From c4c103612216ea66c855799343c5acbdeee670c6 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 26 Aug 2026 15:58:27 -0500
Subject: [PATCH 01/52] feat: add benchmark folder

---
 benchmarks/design-system.ts | 85 +++++++++++++++++++++++++++++++++++++
 benchmarks/package.json     | 13 ++++++
 benchmarks/tsconfig.json    |  5 +++
 pnpm-lock.yaml              | 12 ++++++
 pnpm-workspace.yaml         |  3 +-
 5 files changed, 117 insertions(+), 1 deletion(-)
 create mode 100644 benchmarks/design-system.ts
 create mode 100644 benchmarks/package.json
 create mode 100644 benchmarks/tsconfig.json

diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts
new file mode 100644
index 00000000..90688cba
--- /dev/null
+++ b/benchmarks/design-system.ts
@@ -0,0 +1,85 @@
+import {defineConfig} from '@primer/agent-eval/benchmark'
+
+export const benchmark = defineConfig({
+  name: 'Design System',
+  description: 'Benchmark the performance of different design system approaches.',
+  capabilities: [
+    {
+      name: 'Uses components',
+      scenarios: [
+        // Uses component from the design system unsolicited (e.g. when referring to a button)
+        // Does not re-create a component that already exists
+      ],
+    },
+    {
+      name: 'Infers correct component based on usage',
+      scenarios: [
+        // When talking broadly about an action or case, translates to
+        // components in Primer instead of creating a new component
+      ],
+    },
+    {
+      name: 'Uses tokens',
+      scenarios: [
+        // Color tokens
+        // Size tokens
+        // Typography tokens
+        // Motion tokens
+      ],
+    },
+    {
+      name: 'Infers correct token based on usage',
+      scenarios: [
+        // Given a specific context it can find the correct token
+      ],
+    },
+    {
+      name: 'Uses icons',
+      scenarios: [
+        // Uses icons from Primer before creating its own
+        // Does not use icons from other libraries
+        // Suggests creating an issue or contributing to primer/octicons if
+        // missing
+      ],
+    },
+    {
+      name: 'Infers correct icon based on usage',
+      scenarios: [
+        // Given a specific context it finds the right icon to use
+      ],
+    },
+    {
+      name: 'Uses UI patterns',
+      scenarios: [
+        // Given a specific context it infers what components to use and how to
+        // combine them (forms, loading states, empty states, etc)
+      ],
+    },
+    {
+      name: 'Uses utilities',
+      scenarios: [
+        // Prefers importing and using hooks instead of re-implementing them
+      ],
+    },
+    {
+      name: 'Getting started',
+      scenarios: [
+        // New project
+      ],
+    },
+    {
+      name: 'Theming',
+      scenarios: [
+        // Bringing in primitives
+        // Theme switching
+      ],
+    },
+    {
+      name: 'Works with TailwindCSS',
+      scenarios: [
+        // New project can bring in tailwindcss
+        // Uses correct utility classes
+      ],
+    },
+  ],
+})
diff --git a/benchmarks/package.json b/benchmarks/package.json
new file mode 100644
index 00000000..f93643c7
--- /dev/null
+++ b/benchmarks/package.json
@@ -0,0 +1,13 @@
+{
+  "name": "@primer/benchmarks",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "type-check": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/node": "^26.1.2",
+    "typescript": "^6.0.3"
+  }
+}
diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json
new file mode 100644
index 00000000..df8a02ab
--- /dev/null
+++ b/benchmarks/tsconfig.json
@@ -0,0 +1,5 @@
+{
+  "extends": "../tsconfig.base.json",
+  "include": ["*.ts"]
+}
+
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 88412498..529cbe43 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -54,6 +54,18 @@ importers:
         specifier: ^4.1.11
         version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
+  benchmarks:
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../packages/agent-eval
+      '@types/node':
+        specifier: ^26.1.2
+        version: 26.1.2
+      typescript:
+        specifier: ^6.0.3
+        version: 6.0.3
+
   experiments:
     devDependencies:
       '@primer/agent-eval':
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 261cda7d..9f0e4be8 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -7,7 +7,8 @@ allowBuilds:
 minimumReleaseAge: 10080
 
 packages:
-  - 'scenarios/*'
+  - 'benchmarks'
   - 'experiments'
   - 'packages/*'
+  - 'scenarios/*'
   - 'website'

From 5de02fbf85e98c75618cd3fc3afeff3e980aef50 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 26 Aug 2026 17:05:21 -0500
Subject: [PATCH 02/52] feat: add benchmark scenarios

---
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 18 +++++++++
 .../src/App.tsx                               | 19 +++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 22 +++++++++++
 .../src/App.tsx                               | 11 ++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../008-agent-infers-action-menu/index.html   | 12 ++++++
 .../008-agent-infers-action-menu/package.json | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 22 +++++++++++
 .../008-agent-infers-action-menu/src/App.tsx  | 13 +++++++
 .../008-agent-infers-action-menu/src/main.tsx | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 21 ++++++++++
 .../src/App.tsx                               | 11 ++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 22 +++++++++++
 .../src/App.tsx                               | 13 +++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../011-agent-uses-motion-tokens/index.html   | 12 ++++++
 .../011-agent-uses-motion-tokens/package.json | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 17 ++++++++
 .../011-agent-uses-motion-tokens/src/App.tsx  | 25 ++++++++++++
 .../011-agent-uses-motion-tokens/src/main.tsx | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../012-agent-infers-status-tokens/index.html | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 23 +++++++++++
 .../src/App.tsx                               | 22 +++++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 17 ++++++++
 .../src/App.tsx                               | 12 ++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 21 ++++++++++
 .../src/App.tsx                               | 25 ++++++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../015-agent-infers-copy-icon/index.html     | 12 ++++++
 .../015-agent-infers-copy-icon/package.json   | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 23 +++++++++++
 .../015-agent-infers-copy-icon/src/App.tsx    | 10 +++++
 .../015-agent-infers-copy-icon/src/main.tsx   | 16 ++++++++
 .../015-agent-infers-copy-icon/src/styles.css |  7 ++++
 .../015-agent-infers-copy-icon/tsconfig.json  | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 26 +++++++++++++
 .../src/App.tsx                               | 16 ++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 23 +++++++++++
 .../src/App.tsx                               |  9 +++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../018-agent-uses-filter-pattern/index.html  | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 23 +++++++++++
 .../018-agent-uses-filter-pattern/src/App.tsx | 17 ++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 19 +++++++++
 .../src/App.tsx                               | 31 +++++++++++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 18 +++++++++
 .../src/App.tsx                               | 12 ++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 29 ++++++++++++++
 .../src/App.tsx                               |  3 ++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 22 +++++++++++
 .../src/App.tsx                               |  3 ++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../023-agent-adds-theme-switcher/index.html  | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 24 ++++++++++++
 .../023-agent-adds-theme-switcher/src/App.tsx |  3 ++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../024-agent-sets-up-tailwindcss/index.html  | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 39 +++++++++++++++++++
 .../024-agent-sets-up-tailwindcss/src/App.tsx |  3 ++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 .../index.html                                | 12 ++++++
 .../package.json                              | 24 ++++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 23 +++++++++++
 .../src/App.tsx                               | 11 ++++++
 .../src/main.tsx                              | 16 ++++++++
 .../src/styles.css                            |  9 +++++
 .../tsconfig.json                             | 18 +++++++++
 .../vite.config.ts                            |  6 +++
 .../index.html                                | 12 ++++++
 .../package.json                              | 22 +++++++++++
 .../scenario.config.ts                        |  7 ++++
 .../scenario.test.ts                          | 22 +++++++++++
 .../src/App.tsx                               | 11 ++++++
 .../src/main.tsx                              | 10 +++++
 .../src/styles.css                            |  7 ++++
 .../tsconfig.json                             | 18 +++++++++
 169 files changed, 2480 insertions(+)
 create mode 100644 scenarios/006-agent-uses-pagination-component/index.html
 create mode 100644 scenarios/006-agent-uses-pagination-component/package.json
 create mode 100644 scenarios/006-agent-uses-pagination-component/scenario.config.ts
 create mode 100644 scenarios/006-agent-uses-pagination-component/scenario.test.ts
 create mode 100644 scenarios/006-agent-uses-pagination-component/src/App.tsx
 create mode 100644 scenarios/006-agent-uses-pagination-component/src/main.tsx
 create mode 100644 scenarios/006-agent-uses-pagination-component/src/styles.css
 create mode 100644 scenarios/006-agent-uses-pagination-component/tsconfig.json
 create mode 100644 scenarios/007-agent-infers-billing-banner/index.html
 create mode 100644 scenarios/007-agent-infers-billing-banner/package.json
 create mode 100644 scenarios/007-agent-infers-billing-banner/scenario.config.ts
 create mode 100644 scenarios/007-agent-infers-billing-banner/scenario.test.ts
 create mode 100644 scenarios/007-agent-infers-billing-banner/src/App.tsx
 create mode 100644 scenarios/007-agent-infers-billing-banner/src/main.tsx
 create mode 100644 scenarios/007-agent-infers-billing-banner/src/styles.css
 create mode 100644 scenarios/007-agent-infers-billing-banner/tsconfig.json
 create mode 100644 scenarios/008-agent-infers-action-menu/index.html
 create mode 100644 scenarios/008-agent-infers-action-menu/package.json
 create mode 100644 scenarios/008-agent-infers-action-menu/scenario.config.ts
 create mode 100644 scenarios/008-agent-infers-action-menu/scenario.test.ts
 create mode 100644 scenarios/008-agent-infers-action-menu/src/App.tsx
 create mode 100644 scenarios/008-agent-infers-action-menu/src/main.tsx
 create mode 100644 scenarios/008-agent-infers-action-menu/src/styles.css
 create mode 100644 scenarios/008-agent-infers-action-menu/tsconfig.json
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/index.html
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/package.json
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
 create mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
 create mode 100644 scenarios/010-agent-uses-typography-tokens/index.html
 create mode 100644 scenarios/010-agent-uses-typography-tokens/package.json
 create mode 100644 scenarios/010-agent-uses-typography-tokens/scenario.config.ts
 create mode 100644 scenarios/010-agent-uses-typography-tokens/scenario.test.ts
 create mode 100644 scenarios/010-agent-uses-typography-tokens/src/App.tsx
 create mode 100644 scenarios/010-agent-uses-typography-tokens/src/main.tsx
 create mode 100644 scenarios/010-agent-uses-typography-tokens/src/styles.css
 create mode 100644 scenarios/010-agent-uses-typography-tokens/tsconfig.json
 create mode 100644 scenarios/011-agent-uses-motion-tokens/index.html
 create mode 100644 scenarios/011-agent-uses-motion-tokens/package.json
 create mode 100644 scenarios/011-agent-uses-motion-tokens/scenario.config.ts
 create mode 100644 scenarios/011-agent-uses-motion-tokens/scenario.test.ts
 create mode 100644 scenarios/011-agent-uses-motion-tokens/src/App.tsx
 create mode 100644 scenarios/011-agent-uses-motion-tokens/src/main.tsx
 create mode 100644 scenarios/011-agent-uses-motion-tokens/src/styles.css
 create mode 100644 scenarios/011-agent-uses-motion-tokens/tsconfig.json
 create mode 100644 scenarios/012-agent-infers-status-tokens/index.html
 create mode 100644 scenarios/012-agent-infers-status-tokens/package.json
 create mode 100644 scenarios/012-agent-infers-status-tokens/scenario.config.ts
 create mode 100644 scenarios/012-agent-infers-status-tokens/scenario.test.ts
 create mode 100644 scenarios/012-agent-infers-status-tokens/src/App.tsx
 create mode 100644 scenarios/012-agent-infers-status-tokens/src/main.tsx
 create mode 100644 scenarios/012-agent-infers-status-tokens/src/styles.css
 create mode 100644 scenarios/012-agent-infers-status-tokens/tsconfig.json
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/index.html
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/package.json
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/styles.css
 create mode 100644 scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
 create mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
 create mode 100644 scenarios/015-agent-infers-copy-icon/index.html
 create mode 100644 scenarios/015-agent-infers-copy-icon/package.json
 create mode 100644 scenarios/015-agent-infers-copy-icon/scenario.config.ts
 create mode 100644 scenarios/015-agent-infers-copy-icon/scenario.test.ts
 create mode 100644 scenarios/015-agent-infers-copy-icon/src/App.tsx
 create mode 100644 scenarios/015-agent-infers-copy-icon/src/main.tsx
 create mode 100644 scenarios/015-agent-infers-copy-icon/src/styles.css
 create mode 100644 scenarios/015-agent-infers-copy-icon/tsconfig.json
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
 create mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/index.html
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/package.json
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/styles.css
 create mode 100644 scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
 create mode 100644 scenarios/018-agent-uses-filter-pattern/index.html
 create mode 100644 scenarios/018-agent-uses-filter-pattern/package.json
 create mode 100644 scenarios/018-agent-uses-filter-pattern/scenario.config.ts
 create mode 100644 scenarios/018-agent-uses-filter-pattern/scenario.test.ts
 create mode 100644 scenarios/018-agent-uses-filter-pattern/src/App.tsx
 create mode 100644 scenarios/018-agent-uses-filter-pattern/src/main.tsx
 create mode 100644 scenarios/018-agent-uses-filter-pattern/src/styles.css
 create mode 100644 scenarios/018-agent-uses-filter-pattern/tsconfig.json
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/index.html
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/package.json
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/styles.css
 create mode 100644 scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/index.html
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/package.json
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/styles.css
 create mode 100644 scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/index.html
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/package.json
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
 create mode 100644 scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
 create mode 100644 scenarios/022-agent-enables-automatic-theming/index.html
 create mode 100644 scenarios/022-agent-enables-automatic-theming/package.json
 create mode 100644 scenarios/022-agent-enables-automatic-theming/scenario.config.ts
 create mode 100644 scenarios/022-agent-enables-automatic-theming/scenario.test.ts
 create mode 100644 scenarios/022-agent-enables-automatic-theming/src/App.tsx
 create mode 100644 scenarios/022-agent-enables-automatic-theming/src/main.tsx
 create mode 100644 scenarios/022-agent-enables-automatic-theming/src/styles.css
 create mode 100644 scenarios/022-agent-enables-automatic-theming/tsconfig.json
 create mode 100644 scenarios/023-agent-adds-theme-switcher/index.html
 create mode 100644 scenarios/023-agent-adds-theme-switcher/package.json
 create mode 100644 scenarios/023-agent-adds-theme-switcher/scenario.config.ts
 create mode 100644 scenarios/023-agent-adds-theme-switcher/scenario.test.ts
 create mode 100644 scenarios/023-agent-adds-theme-switcher/src/App.tsx
 create mode 100644 scenarios/023-agent-adds-theme-switcher/src/main.tsx
 create mode 100644 scenarios/023-agent-adds-theme-switcher/src/styles.css
 create mode 100644 scenarios/023-agent-adds-theme-switcher/tsconfig.json
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/index.html
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/package.json
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/styles.css
 create mode 100644 scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
 create mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/index.html
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/package.json
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/styles.css
 create mode 100644 scenarios/026-agent-avoids-deprecated-notification/tsconfig.json

diff --git a/scenarios/006-agent-uses-pagination-component/index.html b/scenarios/006-agent-uses-pagination-component/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/006-agent-uses-pagination-component/package.json b/scenarios/006-agent-uses-pagination-component/package.json
new file mode 100644
index 00000000..6c0786af
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "006-agent-uses-pagination-component",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/006-agent-uses-pagination-component/scenario.config.ts b/scenarios/006-agent-uses-pagination-component/scenario.config.ts
new file mode 100644
index 00000000..fae4973c
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent uses an existing component when adding pagination.',
+  prompt: `Add pagination controls below the issue list. Show 25 issues per page and include previous and next navigation.`,
+  tags: ['component', 'pagination', 'vite'],
+})
diff --git a/scenarios/006-agent-uses-pagination-component/scenario.test.ts b/scenarios/006-agent-uses-pagination-component/scenario.test.ts
new file mode 100644
index 00000000..553184ff
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/scenario.test.ts
@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports Pagination from the design system', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bPagination\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+})
+
+test('renders Pagination', () => {
+  expect(app).toMatch(/<Pagination(?:\s[^>]*)?>/)
+})
+
+test('configures the current page and page count', () => {
+  expect(app).toMatch(/<Pagination[^>]*\bcurrentPage=\{?[^}\s]+}?/)
+  expect(app).toMatch(/<Pagination[^>]*\bpageCount=\{?[^}\s]+}?/)
+})
diff --git a/scenarios/006-agent-uses-pagination-component/src/App.tsx b/scenarios/006-agent-uses-pagination-component/src/App.tsx
new file mode 100644
index 00000000..dccd9e6d
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/src/App.tsx
@@ -0,0 +1,19 @@
+export function App() {
+  const issues = Array.from({length: 25}, (_, index) => {
+    return {
+      id: index + 1,
+      title: `Issue ${index + 1}`,
+    }
+  })
+
+  return (
+    <main>
+      <h1>Issues</h1>
+      <ul>
+        {issues.map(issue => {
+          return <li key={issue.id}>{issue.title}</li>
+        })}
+      </ul>
+    </main>
+  )
+}
diff --git a/scenarios/006-agent-uses-pagination-component/src/main.tsx b/scenarios/006-agent-uses-pagination-component/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/006-agent-uses-pagination-component/src/styles.css b/scenarios/006-agent-uses-pagination-component/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/006-agent-uses-pagination-component/tsconfig.json b/scenarios/006-agent-uses-pagination-component/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/006-agent-uses-pagination-component/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/007-agent-infers-billing-banner/index.html b/scenarios/007-agent-infers-billing-banner/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/007-agent-infers-billing-banner/package.json b/scenarios/007-agent-infers-billing-banner/package.json
new file mode 100644
index 00000000..3114537a
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "007-agent-infers-billing-banner",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/007-agent-infers-billing-banner/scenario.config.ts b/scenarios/007-agent-infers-billing-banner/scenario.config.ts
new file mode 100644
index 00000000..276718d7
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent selects an appropriate component for a persistent warning.',
+  prompt: `Show a persistent warning at the top of the page when an account has a past-due balance. Include a link to billing settings.`,
+  tags: ['banner', 'component', 'vite'],
+})
diff --git a/scenarios/007-agent-infers-billing-banner/scenario.test.ts b/scenarios/007-agent-infers-billing-banner/scenario.test.ts
new file mode 100644
index 00000000..fc7ab5fc
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/scenario.test.ts
@@ -0,0 +1,22 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports the current Banner component', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+})
+
+test('renders an attention Banner for the past-due state', () => {
+  expect(app).toMatch(/<Banner[^>]*variant=["'](?:warning|critical)["'][^>]*>/)
+  expect(app).toMatch(/past[- ]due/i)
+})
+
+test('does not use the deprecated Flash component', () => {
+  expect(app).not.toMatch(/\bFlash\b/)
+})
+
+test('links to billing settings from the warning', () => {
+  expect(app).toMatch(/<Banner[\s\S]*billing[\s\S]*<\/Banner>/i)
+})
diff --git a/scenarios/007-agent-infers-billing-banner/src/App.tsx b/scenarios/007-agent-infers-billing-banner/src/App.tsx
new file mode 100644
index 00000000..2f6f28f5
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/src/App.tsx
@@ -0,0 +1,11 @@
+export function App() {
+  const hasPastDueBalance = true
+
+  return (
+    <main>
+      <h1>Account</h1>
+      <p>{hasPastDueBalance ? 'Payment required' : 'Your account is in good standing'}</p>
+      <a href="/settings/billing">Billing settings</a>
+    </main>
+  )
+}
diff --git a/scenarios/007-agent-infers-billing-banner/src/main.tsx b/scenarios/007-agent-infers-billing-banner/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/007-agent-infers-billing-banner/src/styles.css b/scenarios/007-agent-infers-billing-banner/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/007-agent-infers-billing-banner/tsconfig.json b/scenarios/007-agent-infers-billing-banner/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/007-agent-infers-billing-banner/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/008-agent-infers-action-menu/index.html b/scenarios/008-agent-infers-action-menu/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/008-agent-infers-action-menu/package.json b/scenarios/008-agent-infers-action-menu/package.json
new file mode 100644
index 00000000..539e8506
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "008-agent-infers-action-menu",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/008-agent-infers-action-menu/scenario.config.ts b/scenarios/008-agent-infers-action-menu/scenario.config.ts
new file mode 100644
index 00000000..6d8ff68d
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent selects an appropriate component for secondary actions.',
+  prompt: `Add archive, transfer, and delete actions to the repository header without crowding the existing primary actions.`,
+  tags: ['component', 'menu', 'vite'],
+})
diff --git a/scenarios/008-agent-infers-action-menu/scenario.test.ts b/scenarios/008-agent-infers-action-menu/scenario.test.ts
new file mode 100644
index 00000000..961b51a5
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/scenario.test.ts
@@ -0,0 +1,22 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports ActionMenu and ActionList from the design system', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+})
+
+test('renders an ActionMenu', () => {
+  expect(app).toMatch(/<ActionMenu(?:\s[^>]*)?>[\s\S]*<\/ActionMenu>/)
+})
+
+test.each(['Archive', 'Transfer', 'Delete'])('includes the %s action', action => {
+  expect(app).toContain(action)
+})
+
+test('marks the delete action as destructive', () => {
+  expect(app).toMatch(/<ActionList\.Item[^>]*variant=["']danger["'][^>]*>[\s\S]*Delete/)
+})
diff --git a/scenarios/008-agent-infers-action-menu/src/App.tsx b/scenarios/008-agent-infers-action-menu/src/App.tsx
new file mode 100644
index 00000000..edeaf481
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/src/App.tsx
@@ -0,0 +1,13 @@
+export function App() {
+  return (
+    <main>
+      <header>
+        <h1>octo-repo</h1>
+        <div>
+          <button type="button">Edit</button>
+        </div>
+      </header>
+      <p>Repository settings and activity.</p>
+    </main>
+  )
+}
diff --git a/scenarios/008-agent-infers-action-menu/src/main.tsx b/scenarios/008-agent-infers-action-menu/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/008-agent-infers-action-menu/src/styles.css b/scenarios/008-agent-infers-action-menu/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/008-agent-infers-action-menu/tsconfig.json b/scenarios/008-agent-infers-action-menu/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/008-agent-infers-action-menu/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/index.html b/scenarios/009-agent-uses-layout-and-color-tokens/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/package.json b/scenarios/009-agent-uses-layout-and-color-tokens/package.json
new file mode 100644
index 00000000..756b697a
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "009-agent-uses-layout-and-color-tokens",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
new file mode 100644
index 00000000..6d8092fb
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent uses design tokens for layout and color styling.',
+  prompt: `Style the status summary card so its content is clearly grouped and visually distinct from the page background.`,
+  tags: ['color', 'layout', 'tokens', 'vite'],
+})
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
new file mode 100644
index 00000000..728f1a32
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
@@ -0,0 +1,21 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+
+test('uses semantic background or foreground color tokens', () => {
+  expect(styles).toMatch(/var\(--(?:bgColor|fgColor)-[A-Za-z0-9-]+\)/)
+})
+
+test('uses a semantic border token', () => {
+  expect(styles).toMatch(/var\(--border(?:Color|Width)?-[A-Za-z0-9-]+\)/)
+})
+
+test('uses stack tokens for layout spacing', () => {
+  expect(styles).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/)
+})
+
+test('does not introduce raw hexadecimal colors', () => {
+  expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i)
+})
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
new file mode 100644
index 00000000..d270d223
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
@@ -0,0 +1,11 @@
+export function App() {
+  return (
+    <main>
+      <h1>System status</h1>
+      <section className="status-summary">
+        <h2>All systems operational</h2>
+        <p>Last checked one minute ago.</p>
+      </section>
+    </main>
+  )
+}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css b/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json b/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/010-agent-uses-typography-tokens/index.html b/scenarios/010-agent-uses-typography-tokens/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/010-agent-uses-typography-tokens/package.json b/scenarios/010-agent-uses-typography-tokens/package.json
new file mode 100644
index 00000000..f3d8033e
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "010-agent-uses-typography-tokens",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
new file mode 100644
index 00000000..192e8c31
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent uses role-appropriate typography tokens.',
+  prompt: `Improve the typography of the documentation page. It contains a page title, introductory text, inline code, and a code example.`,
+  tags: ['tokens', 'typography', 'vite'],
+})
diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.test.ts b/scenarios/010-agent-uses-typography-tokens/scenario.test.ts
new file mode 100644
index 00000000..9d8ca2fd
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/scenario.test.ts
@@ -0,0 +1,22 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+
+test('uses a title typography shorthand token', () => {
+  expect(styles).toMatch(/font:\s*var\(--text-title-shorthand-(?:small|medium|large)\)/)
+})
+
+test('uses a body typography shorthand token', () => {
+  expect(styles).toMatch(/font:\s*var\(--text-body-shorthand-(?:small|medium|large)\)/)
+})
+
+test('uses code typography shorthand tokens', () => {
+  expect(styles).toMatch(/font:\s*var\(--text-codeInline-shorthand\)/)
+  expect(styles).toMatch(/font:\s*var\(--text-codeBlock-shorthand\)/)
+})
+
+test('does not set raw font sizes or line heights', () => {
+  expect(styles).not.toMatch(/(?:font-size|line-height):\s*(?:\d|calc\()/)
+})
diff --git a/scenarios/010-agent-uses-typography-tokens/src/App.tsx b/scenarios/010-agent-uses-typography-tokens/src/App.tsx
new file mode 100644
index 00000000..beeddf48
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/src/App.tsx
@@ -0,0 +1,13 @@
+export function App() {
+  return (
+    <main className="documentation">
+      <h1>Configure the CLI</h1>
+      <p>
+        Create a <code>config.json</code> file in your project directory.
+      </p>
+      <pre>
+        <code>{`{"theme": "system"}`}</code>
+      </pre>
+    </main>
+  )
+}
diff --git a/scenarios/010-agent-uses-typography-tokens/src/main.tsx b/scenarios/010-agent-uses-typography-tokens/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/010-agent-uses-typography-tokens/src/styles.css b/scenarios/010-agent-uses-typography-tokens/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/010-agent-uses-typography-tokens/tsconfig.json b/scenarios/010-agent-uses-typography-tokens/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/010-agent-uses-typography-tokens/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/011-agent-uses-motion-tokens/index.html b/scenarios/011-agent-uses-motion-tokens/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/011-agent-uses-motion-tokens/package.json b/scenarios/011-agent-uses-motion-tokens/package.json
new file mode 100644
index 00000000..d906c89c
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "011-agent-uses-motion-tokens",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
new file mode 100644
index 00000000..b494e085
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent uses motion tokens and respects reduced-motion preferences.',
+  prompt: `Add a short transition when the details panel expands or collapses. Keep the interaction comfortable for people who prefer reduced motion.`,
+  tags: ['accessibility', 'motion', 'tokens', 'vite'],
+})
diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.test.ts b/scenarios/011-agent-uses-motion-tokens/scenario.test.ts
new file mode 100644
index 00000000..861b5f15
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/scenario.test.ts
@@ -0,0 +1,17 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+
+test('uses a motion token for the transition', () => {
+  expect(styles).toMatch(/var\(--motion-(?:transition|duration|easing)-[A-Za-z0-9-]+\)/)
+})
+
+test('defines a reduced-motion alternative', () => {
+  expect(styles).toMatch(/@media\s*\(prefers-reduced-motion:\s*reduce\)/)
+})
+
+test('does not use raw transition timing values', () => {
+  expect(styles).not.toMatch(/(?:transition|animation)[^;]*(?:\d+m?s|ease(?:-in|-out|-in-out)?)/)
+})
diff --git a/scenarios/011-agent-uses-motion-tokens/src/App.tsx b/scenarios/011-agent-uses-motion-tokens/src/App.tsx
new file mode 100644
index 00000000..cd0a07d8
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/src/App.tsx
@@ -0,0 +1,25 @@
+import {useState} from 'react'
+
+export function App() {
+  const [isOpen, setIsOpen] = useState(false)
+
+  return (
+    <main>
+      <button
+        aria-expanded={isOpen}
+        aria-controls="details-panel"
+        type="button"
+        onClick={() => {
+          setIsOpen(current => {
+            return !current
+          })
+        }}
+      >
+        View details
+      </button>
+      <section className="details-panel" hidden={!isOpen} id="details-panel">
+        <p>The latest deployment completed successfully.</p>
+      </section>
+    </main>
+  )
+}
diff --git a/scenarios/011-agent-uses-motion-tokens/src/main.tsx b/scenarios/011-agent-uses-motion-tokens/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/011-agent-uses-motion-tokens/src/styles.css b/scenarios/011-agent-uses-motion-tokens/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/011-agent-uses-motion-tokens/tsconfig.json b/scenarios/011-agent-uses-motion-tokens/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/011-agent-uses-motion-tokens/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/012-agent-infers-status-tokens/index.html b/scenarios/012-agent-infers-status-tokens/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/012-agent-infers-status-tokens/package.json b/scenarios/012-agent-infers-status-tokens/package.json
new file mode 100644
index 00000000..7dec141f
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "012-agent-infers-status-tokens",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/012-agent-infers-status-tokens/scenario.config.ts b/scenarios/012-agent-infers-status-tokens/scenario.config.ts
new file mode 100644
index 00000000..75cd19c8
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent chooses semantically correct status tokens.',
+  prompt: `Update the deployment list so successful and failed deployments are easy to distinguish without relying on text alone.`,
+  tags: ['color', 'status', 'tokens', 'vite'],
+})
diff --git a/scenarios/012-agent-infers-status-tokens/scenario.test.ts b/scenarios/012-agent-infers-status-tokens/scenario.test.ts
new file mode 100644
index 00000000..4b181bbe
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/scenario.test.ts
@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+
+test('uses success tokens for successful deployments', () => {
+  expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/)
+})
+
+test('uses danger tokens for failed deployments', () => {
+  expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-danger(?:-[A-Za-z0-9-]+)?\)/)
+})
+
+test('uses a non-text status indicator', () => {
+  expect(app).toMatch(/@primer\/octicons-react/)
+  expect(app).toMatch(/(?:Check|Pass|X|Stop|Alert)[A-Za-z]*Icon/)
+})
+
+test('does not introduce raw hexadecimal colors', () => {
+  expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i)
+})
diff --git a/scenarios/012-agent-infers-status-tokens/src/App.tsx b/scenarios/012-agent-infers-status-tokens/src/App.tsx
new file mode 100644
index 00000000..012f7e30
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/src/App.tsx
@@ -0,0 +1,22 @@
+export function App() {
+  const deployments = [
+    {id: 1, name: 'Production', status: 'success'},
+    {id: 2, name: 'Staging', status: 'failure'},
+  ]
+
+  return (
+    <main>
+      <h1>Deployments</h1>
+      <ul>
+        {deployments.map(deployment => {
+          return (
+            <li className={`deployment deployment-${deployment.status}`} key={deployment.id}>
+              <span>{deployment.name}</span>
+              <span>{deployment.status}</span>
+            </li>
+          )
+        })}
+      </ul>
+    </main>
+  )
+}
diff --git a/scenarios/012-agent-infers-status-tokens/src/main.tsx b/scenarios/012-agent-infers-status-tokens/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/012-agent-infers-status-tokens/src/styles.css b/scenarios/012-agent-infers-status-tokens/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/012-agent-infers-status-tokens/tsconfig.json b/scenarios/012-agent-infers-status-tokens/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/012-agent-infers-status-tokens/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/index.html b/scenarios/013-agent-infers-compact-control-tokens/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/013-agent-infers-compact-control-tokens/package.json b/scenarios/013-agent-infers-compact-control-tokens/package.json
new file mode 100644
index 00000000..959cb638
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "013-agent-infers-compact-control-tokens",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
new file mode 100644
index 00000000..8e007fd8
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent chooses appropriate tokens for a compact toolbar.',
+  prompt: `Make the repository file toolbar more compact while keeping its controls usable and consistently spaced.`,
+  tags: ['controls', 'layout', 'tokens', 'vite'],
+})
diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
new file mode 100644
index 00000000..57bab2ed
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
@@ -0,0 +1,17 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+
+test('uses compact control size or padding tokens', () => {
+  expect(styles).toMatch(/var\(--control-(?:xsmall|small)-(?:size|padding(?:Block|Inline-[A-Za-z]+))\)/)
+})
+
+test('uses a token for spacing between toolbar controls', () => {
+  expect(styles).toMatch(/var\(--(?:controlStack|stack)-[A-Za-z0-9-]*gap[A-Za-z0-9-]*\)/)
+})
+
+test('does not use raw pixel values for control sizing', () => {
+  expect(styles).not.toMatch(/(?:gap|height|padding(?:-block|-inline)?):\s*\d+px/)
+})
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
new file mode 100644
index 00000000..a78da00d
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
@@ -0,0 +1,12 @@
+export function App() {
+  return (
+    <main>
+      <h1>src/components</h1>
+      <div className="file-toolbar" role="toolbar" aria-label="File actions">
+        <button type="button">Add file</button>
+        <button type="button">Go to file</button>
+        <button type="button">Code</button>
+      </div>
+    </main>
+  )
+}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/styles.css b/scenarios/013-agent-infers-compact-control-tokens/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json b/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html b/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
new file mode 100644
index 00000000..ce58381c
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "014-agent-replaces-custom-icons-with-octicons",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
new file mode 100644
index 00000000..83b6ada9
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent replaces custom icons with maintained design-system icons.',
+  prompt: `Replace the hand-drawn search, download, and trash icons in the toolbar with icons from the project's design system.`,
+  tags: ['icon', 'octicons', 'vite'],
+})
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
new file mode 100644
index 00000000..a4a00106
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
@@ -0,0 +1,21 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports icons directly from Primer Octicons', () => {
+  expect(app).toMatch(/from\s+['"]@primer\/octicons-react['"]/)
+})
+
+test.each(['SearchIcon', 'DownloadIcon', 'TrashIcon'])('uses %s', icon => {
+  expect(app).toMatch(new RegExp(`(?:<${icon}(?:\\s[^>]*)?\\/?>|icon=\\{${icon}\\})`))
+})
+
+test('removes the hand-drawn SVG elements', () => {
+  expect(app).not.toMatch(/<svg\b/)
+})
+
+test('does not use the deprecated Octicon wrapper', () => {
+  expect(app).not.toMatch(/<Octicon\b/)
+})
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
new file mode 100644
index 00000000..2a8dce2f
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
@@ -0,0 +1,25 @@
+export function App() {
+  return (
+    <main>
+      <h1>Files</h1>
+      <div role="toolbar" aria-label="File actions">
+        <button type="button" aria-label="Search">
+          <svg aria-hidden="true" viewBox="0 0 16 16">
+            <circle cx="7" cy="7" r="5" />
+            <path d="m11 11 4 4" />
+          </svg>
+        </button>
+        <button type="button" aria-label="Download">
+          <svg aria-hidden="true" viewBox="0 0 16 16">
+            <path d="M8 1v10m-4-4 4 4 4-4M2 15h12" />
+          </svg>
+        </button>
+        <button type="button" aria-label="Delete">
+          <svg aria-hidden="true" viewBox="0 0 16 16">
+            <path d="M3 4h10M6 1h4l1 3H5l1-3Zm-1 5v8m3-8v8m3-8v8" />
+          </svg>
+        </button>
+      </div>
+    </main>
+  )
+}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/015-agent-infers-copy-icon/index.html b/scenarios/015-agent-infers-copy-icon/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/015-agent-infers-copy-icon/package.json b/scenarios/015-agent-infers-copy-icon/package.json
new file mode 100644
index 00000000..4bde6a34
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "015-agent-infers-copy-icon",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/015-agent-infers-copy-icon/scenario.config.ts b/scenarios/015-agent-infers-copy-icon/scenario.config.ts
new file mode 100644
index 00000000..38a586b8
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent infers the appropriate icon for a compact action.',
+  prompt: `Add a compact control next to the commit SHA that copies it to the clipboard.`,
+  tags: ['component', 'icon', 'vite'],
+})
diff --git a/scenarios/015-agent-infers-copy-icon/scenario.test.ts b/scenarios/015-agent-infers-copy-icon/scenario.test.ts
new file mode 100644
index 00000000..4288cb4f
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/scenario.test.ts
@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports and renders CopyIcon', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bCopyIcon\b[^}]*}\s+from\s+['"]@primer\/octicons-react['"]/)
+  expect(app).toMatch(/<CopyIcon(?:\s[^>]*)?\/?>|icon=\{CopyIcon\}/)
+})
+
+test('uses the current IconButton component', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bIconButton\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/<IconButton\b/)
+})
+
+test('gives the compact control an accessible label', () => {
+  expect(app).toMatch(/<IconButton[^>]*aria-label=["'][^"']*copy[^"']*["'][^>]*>/i)
+})
+
+test('copies the commit SHA to the clipboard', () => {
+  expect(app).toMatch(/navigator\.clipboard\.writeText\(/)
+})
diff --git a/scenarios/015-agent-infers-copy-icon/src/App.tsx b/scenarios/015-agent-infers-copy-icon/src/App.tsx
new file mode 100644
index 00000000..e0070a47
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/src/App.tsx
@@ -0,0 +1,10 @@
+export function App() {
+  return (
+    <main>
+      <h1>Latest commit</h1>
+      <p>
+        <code>8f3c2a1</code>
+      </p>
+    </main>
+  )
+}
diff --git a/scenarios/015-agent-infers-copy-icon/src/main.tsx b/scenarios/015-agent-infers-copy-icon/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/015-agent-infers-copy-icon/src/styles.css b/scenarios/015-agent-infers-copy-icon/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/015-agent-infers-copy-icon/tsconfig.json b/scenarios/015-agent-infers-copy-icon/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/015-agent-infers-copy-icon/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html b/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
new file mode 100644
index 00000000..7d9a9548
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "016-agent-uses-loading-and-empty-state-patterns",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
new file mode 100644
index 00000000..ad730b92
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent applies established loading and empty-state patterns.',
+  prompt: `Add appropriate loading and empty states to the repository list. The empty state should help the user create their first repository.`,
+  tags: ['empty-state', 'loading', 'pattern', 'vite'],
+})
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
new file mode 100644
index 00000000..fb4742a0
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
@@ -0,0 +1,26 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('uses skeleton components for the loading state', () => {
+  expect(app).toMatch(
+    /import\s+{[^}]*(?:SkeletonBox|SkeletonText)[^}]*}\s+from\s+['"]@primer\/react(?:\/experimental)?['"]/,
+  )
+  expect(app).toMatch(/<(?:SkeletonBox|SkeletonText)\b/)
+})
+
+test('uses Blankslate for the empty state', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bBlankslate\b[^}]*}\s+from\s+['"]@primer\/react\/experimental['"]/)
+  expect(app).toMatch(/<Blankslate(?:\s[^>]*)?>[\s\S]*<\/Blankslate>/)
+})
+
+test('gives the empty state a create-repository action', () => {
+  expect(app).toMatch(/<Blankslate[\s\S]*(?:Create|New) repository[\s\S]*<\/Blankslate>/i)
+})
+
+test('renders the states conditionally', () => {
+  expect(app).toMatch(/\bisLoading\b[\s\S]*(?:SkeletonBox|SkeletonText)/)
+  expect(app).toMatch(/repositories\.length[\s\S]*Blankslate|Blankslate[\s\S]*repositories\.length/)
+})
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
new file mode 100644
index 00000000..d04ef697
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
@@ -0,0 +1,16 @@
+export function App() {
+  const isLoading = false
+  const repositories = [{id: 1, name: 'octo-repo'}]
+
+  return (
+    <main>
+      <h1>Repositories</h1>
+      <p>{isLoading ? 'Loading repositories' : `${repositories.length} repositories`}</p>
+      <ul>
+        {repositories.map(repository => {
+          return <li key={repository.id}>{repository.name}</li>
+        })}
+      </ul>
+    </main>
+  )
+}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/index.html b/scenarios/017-agent-uses-confirmation-pattern/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/017-agent-uses-confirmation-pattern/package.json b/scenarios/017-agent-uses-confirmation-pattern/package.json
new file mode 100644
index 00000000..d14fc47e
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "017-agent-uses-confirmation-pattern",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
new file mode 100644
index 00000000..aadb828c
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent applies the established confirmation pattern to a destructive action.',
+  prompt: `Let an administrator delete the repository after confirming the destructive action. Include clear cancel and confirm paths.`,
+  tags: ['confirmation', 'dialog', 'pattern', 'vite'],
+})
diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
new file mode 100644
index 00000000..34f7e37d
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('uses the current ConfirmationDialog API', () => {
+  expect(app).toMatch(/import\s+{[^}]*(?:\bConfirmationDialog\b|\buseConfirm\b)[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/<ConfirmationDialog\b|\buseConfirm\(/)
+})
+
+test('does not use the deprecated Dialog component', () => {
+  expect(app).not.toMatch(/<Dialog\b/)
+})
+
+test('provides clear cancel and destructive confirmation labels', () => {
+  expect(app).toMatch(/cancel/i)
+  expect(app).toMatch(/delete repository/i)
+})
+
+test('uses a danger-styled confirmation action', () => {
+  expect(app).toMatch(/(?:confirmButtonType|variant)=["']danger["']/)
+})
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
new file mode 100644
index 00000000..975c0eab
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
@@ -0,0 +1,9 @@
+export function App() {
+  return (
+    <main>
+      <h1>Danger zone</h1>
+      <p>Deleting this repository removes its code, issues, and settings.</p>
+      <button type="button">Delete repository</button>
+    </main>
+  )
+}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/styles.css b/scenarios/017-agent-uses-confirmation-pattern/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json b/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/018-agent-uses-filter-pattern/index.html b/scenarios/018-agent-uses-filter-pattern/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/018-agent-uses-filter-pattern/package.json b/scenarios/018-agent-uses-filter-pattern/package.json
new file mode 100644
index 00000000..38fceca3
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "018-agent-uses-filter-pattern",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
new file mode 100644
index 00000000..d76da367
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent composes current components into an issue-filtering pattern.',
+  prompt: `Add controls for filtering the issue list by author, label, and open or closed status.`,
+  tags: ['filter', 'pattern', 'vite'],
+})
diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.test.ts b/scenarios/018-agent-uses-filter-pattern/scenario.test.ts
new file mode 100644
index 00000000..e3c57d5c
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/scenario.test.ts
@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('composes current menu components for the filters', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app.match(/<ActionMenu(?:\s[^>]*)?>/g)?.length).toBeGreaterThanOrEqual(3)
+})
+
+test.each(['author', 'label', 'status'])('provides a visible %s filter', filter => {
+  expect(app).toMatch(new RegExp(`>[^<]*${filter}[^<]*<`, 'i'))
+})
+
+test('announces the updated result count', () => {
+  expect(app).toMatch(/role=["']status["']/)
+})
+
+test('does not use deprecated filtering components', () => {
+  expect(app).not.toMatch(/\bFilteredSearch\b|\bSelectPanel\b/)
+})
diff --git a/scenarios/018-agent-uses-filter-pattern/src/App.tsx b/scenarios/018-agent-uses-filter-pattern/src/App.tsx
new file mode 100644
index 00000000..8b22478b
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/src/App.tsx
@@ -0,0 +1,17 @@
+export function App() {
+  const issues = [
+    {id: 1, title: 'Improve keyboard navigation'},
+    {id: 2, title: 'Document release process'},
+  ]
+
+  return (
+    <main>
+      <h1>Issues</h1>
+      <ul>
+        {issues.map(issue => {
+          return <li key={issue.id}>{issue.title}</li>
+        })}
+      </ul>
+    </main>
+  )
+}
diff --git a/scenarios/018-agent-uses-filter-pattern/src/main.tsx b/scenarios/018-agent-uses-filter-pattern/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/018-agent-uses-filter-pattern/src/styles.css b/scenarios/018-agent-uses-filter-pattern/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/018-agent-uses-filter-pattern/tsconfig.json b/scenarios/018-agent-uses-filter-pattern/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/018-agent-uses-filter-pattern/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/index.html b/scenarios/019-agent-uses-dismissal-utilities/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/019-agent-uses-dismissal-utilities/package.json b/scenarios/019-agent-uses-dismissal-utilities/package.json
new file mode 100644
index 00000000..2b1372dd
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "019-agent-uses-dismissal-utilities",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
new file mode 100644
index 00000000..7f1b122e
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent reuses design-system utilities for common dismissal behavior.',
+  prompt: `Update the existing floating panel so it closes when the user clicks outside it or presses Escape. Preserve the current markup and positioning.`,
+  tags: ['hooks', 'interaction', 'utilities', 'vite'],
+})
diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
new file mode 100644
index 00000000..1e379243
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
@@ -0,0 +1,19 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports the outside-click utility', () => {
+  expect(app).toMatch(/import\s+{[^}]*\buseOnOutsideClick\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/\buseOnOutsideClick\(/)
+})
+
+test('imports the Escape-key utility', () => {
+  expect(app).toMatch(/import\s+{[^}]*\buseOnEscapePress\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/\buseOnEscapePress\(/)
+})
+
+test('does not add global event listeners directly', () => {
+  expect(app).not.toMatch(/(?:window|document)\.addEventListener\(/)
+})
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
new file mode 100644
index 00000000..9fa93e5a
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
@@ -0,0 +1,31 @@
+import {useState} from 'react'
+
+export function App() {
+  const [isOpen, setIsOpen] = useState(false)
+
+  return (
+    <main>
+      <button
+        type="button"
+        onClick={() => {
+          setIsOpen(true)
+        }}
+      >
+        Open panel
+      </button>
+      {isOpen ? (
+        <aside className="floating-panel">
+          <h2>Notifications</h2>
+          <button
+            type="button"
+            onClick={() => {
+              setIsOpen(false)
+            }}
+          >
+            Close
+          </button>
+        </aside>
+      ) : null}
+    </main>
+  )
+}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/styles.css b/scenarios/019-agent-uses-dismissal-utilities/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json b/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/index.html b/scenarios/020-agent-uses-resize-observer-utility/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/020-agent-uses-resize-observer-utility/package.json b/scenarios/020-agent-uses-resize-observer-utility/package.json
new file mode 100644
index 00000000..77ab4a83
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "020-agent-uses-resize-observer-utility",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
new file mode 100644
index 00000000..e8566909
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent reuses the design-system resize observer utility.',
+  prompt: `Make the contribution chart update its dimensions whenever its container is resized.`,
+  tags: ['hooks', 'responsive', 'utilities', 'vite'],
+})
diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
new file mode 100644
index 00000000..65f7e04c
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('imports and calls useResizeObserver', () => {
+  expect(app).toMatch(/import\s+{[^}]*\buseResizeObserver\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/\buseResizeObserver\(/)
+})
+
+test('uses observed dimensions for the chart', () => {
+  expect(app).toMatch(/<(?:svg|rect)[^>]*(?:width|height)=\{[^}]+\}/)
+})
+
+test('does not instantiate ResizeObserver directly', () => {
+  expect(app).not.toMatch(/new\s+ResizeObserver\(/)
+})
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
new file mode 100644
index 00000000..0ff16788
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
@@ -0,0 +1,12 @@
+export function App() {
+  return (
+    <main>
+      <h1>Contributions</h1>
+      <section className="chart-container">
+        <svg aria-label="Contribution activity" role="img" height="120" width="720">
+          <rect height="80" width="680" x="20" y="20" />
+        </svg>
+      </section>
+    </main>
+  )
+}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/styles.css b/scenarios/020-agent-uses-resize-observer-utility/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json b/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/index.html b/scenarios/021-agent-sets-up-primer-in-vite/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/package.json b/scenarios/021-agent-sets-up-primer-in-vite/package.json
new file mode 100644
index 00000000..8081e4bd
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "021-agent-sets-up-primer-in-vite",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
new file mode 100644
index 00000000..6648b57d
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent correctly configures a Vite application to use the design system.',
+  prompt: `Set up this Vite application to use our design system and update the default page to demonstrate that it is working.`,
+  tags: ['setup', 'vite'],
+})
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
new file mode 100644
index 00000000..302e353c
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
@@ -0,0 +1,29 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8'))
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
+const source = `${app}\n${main}`
+const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies}
+
+test('installs the design-system packages', () => {
+  expect(dependencies).toHaveProperty('@primer/react')
+  expect(dependencies).toHaveProperty('@primer/primitives')
+})
+
+test('loads base primitives and light and dark themes', () => {
+  expect(source).toMatch(/@primer\/primitives\/dist\/css\/primitives\.css/)
+  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/)
+  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/)
+})
+
+test('wraps the application in BaseStyles', () => {
+  expect(source).toMatch(/import\s+{[^}]*\bBaseStyles\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(source).toMatch(/<BaseStyles(?:\s[^>]*)?>[\s\S]*<\/BaseStyles>/)
+})
+
+test('demonstrates a current design-system component', () => {
+  expect(app).toMatch(/import\s+{[^}]*(?:\bButton\b|\bBanner\b|\bCard\b)[^}]*}\s+from\s+['"]@primer\/react['"]/)
+})
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
new file mode 100644
index 00000000..15f8c284
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
@@ -0,0 +1,3 @@
+export function App() {
+  return <main>Hello world</main>
+}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css b/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json b/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/022-agent-enables-automatic-theming/index.html b/scenarios/022-agent-enables-automatic-theming/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/022-agent-enables-automatic-theming/package.json b/scenarios/022-agent-enables-automatic-theming/package.json
new file mode 100644
index 00000000..fd18ce95
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "022-agent-enables-automatic-theming",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
new file mode 100644
index 00000000..c79eeefb
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent configures automatic light and dark theme support.',
+  prompt: `Make the application follow the user's system light or dark appearance setting.`,
+  tags: ['theme', 'theming', 'vite'],
+})
diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.test.ts b/scenarios/022-agent-enables-automatic-theming/scenario.test.ts
new file mode 100644
index 00000000..7e79fe1e
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/scenario.test.ts
@@ -0,0 +1,22 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
+const html = await fs.readFile(path.resolve(import.meta.dirname, 'index.html'), 'utf8')
+const source = `${app}\n${main}\n${html}`
+
+test('loads the light and dark functional themes', () => {
+  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/)
+  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/)
+})
+
+test('uses automatic color mode', () => {
+  expect(source).toMatch(/data-color-mode(?:=|["']\s*,\s*)["']auto["']/)
+})
+
+test('configures light and dark themes', () => {
+  expect(source).toMatch(/data-light-theme(?:=|["']\s*,\s*)["']light["']/)
+  expect(source).toMatch(/data-dark-theme(?:=|["']\s*,\s*)["']dark["']/)
+})
diff --git a/scenarios/022-agent-enables-automatic-theming/src/App.tsx b/scenarios/022-agent-enables-automatic-theming/src/App.tsx
new file mode 100644
index 00000000..15f8c284
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/src/App.tsx
@@ -0,0 +1,3 @@
+export function App() {
+  return <main>Hello world</main>
+}
diff --git a/scenarios/022-agent-enables-automatic-theming/src/main.tsx b/scenarios/022-agent-enables-automatic-theming/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/022-agent-enables-automatic-theming/src/styles.css b/scenarios/022-agent-enables-automatic-theming/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/022-agent-enables-automatic-theming/tsconfig.json b/scenarios/022-agent-enables-automatic-theming/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/022-agent-enables-automatic-theming/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/023-agent-adds-theme-switcher/index.html b/scenarios/023-agent-adds-theme-switcher/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/023-agent-adds-theme-switcher/package.json b/scenarios/023-agent-adds-theme-switcher/package.json
new file mode 100644
index 00000000..f582719b
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "023-agent-adds-theme-switcher",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
new file mode 100644
index 00000000..47a852a9
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent implements a persistent user-controlled theme preference.',
+  prompt: `Add an appearance setting with system, light, and dark choices. Apply the choice immediately and remember it across visits.`,
+  tags: ['interaction', 'theme', 'theming', 'vite'],
+})
diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.test.ts b/scenarios/023-agent-adds-theme-switcher/scenario.test.ts
new file mode 100644
index 00000000..bc663247
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/scenario.test.ts
@@ -0,0 +1,24 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
+const source = `${app}\n${main}`
+
+test.each(['system', 'light', 'dark'])('offers the %s appearance choice', choice => {
+  expect(source).toMatch(new RegExp(`(?:value=["']${choice}["']|>${choice}<)`, 'i'))
+})
+
+test('persists the appearance preference', () => {
+  expect(source).toMatch(/localStorage\.setItem\(/)
+  expect(source).toMatch(/localStorage\.getItem\(/)
+})
+
+test('applies the selected theme using data attributes', () => {
+  expect(source).toMatch(/(?:dataset|setAttribute\()[\s\S]*(?:colorMode|data-color-mode)/)
+})
+
+test('supports the system appearance setting', () => {
+  expect(source).toMatch(/matchMedia\(['"]\(prefers-color-scheme:\s*dark\)['"]\)/)
+})
diff --git a/scenarios/023-agent-adds-theme-switcher/src/App.tsx b/scenarios/023-agent-adds-theme-switcher/src/App.tsx
new file mode 100644
index 00000000..15f8c284
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/src/App.tsx
@@ -0,0 +1,3 @@
+export function App() {
+  return <main>Hello world</main>
+}
diff --git a/scenarios/023-agent-adds-theme-switcher/src/main.tsx b/scenarios/023-agent-adds-theme-switcher/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/023-agent-adds-theme-switcher/src/styles.css b/scenarios/023-agent-adds-theme-switcher/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/023-agent-adds-theme-switcher/tsconfig.json b/scenarios/023-agent-adds-theme-switcher/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/023-agent-adds-theme-switcher/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/index.html b/scenarios/024-agent-sets-up-tailwindcss/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/024-agent-sets-up-tailwindcss/package.json b/scenarios/024-agent-sets-up-tailwindcss/package.json
new file mode 100644
index 00000000..f7daaaaf
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "024-agent-sets-up-tailwindcss",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
new file mode 100644
index 00000000..49b0c522
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent configures Tailwind CSS alongside the design system.',
+  prompt: `Add Tailwind CSS to this Vite application and use it to lay out the default page without breaking the existing design-system styles.`,
+  tags: ['setup', 'tailwindcss', 'vite'],
+})
diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
new file mode 100644
index 00000000..48d1fc52
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
@@ -0,0 +1,39 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+async function readOptional(relativePath: string): Promise<string> {
+  try {
+    return await fs.readFile(path.resolve(import.meta.dirname, relativePath), 'utf8')
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return ''
+    }
+
+    throw error
+  }
+}
+
+const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8'))
+const viteConfig = await readOptional('vite.config.ts')
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies}
+
+test('installs Tailwind CSS and its Vite plugin', () => {
+  expect(dependencies).toHaveProperty('tailwindcss')
+  expect(dependencies).toHaveProperty('@tailwindcss/vite')
+})
+
+test('configures the Tailwind Vite plugin', () => {
+  expect(viteConfig).toMatch(/from\s+['"]@tailwindcss\/vite['"]/)
+  expect(viteConfig).toMatch(/\btailwindcss\(\)/)
+})
+
+test('loads Tailwind CSS', () => {
+  expect(styles).toMatch(/@import\s+['"]tailwindcss['"]/)
+})
+
+test('uses utility classes in the default page', () => {
+  expect(app).toMatch(/className=["'][^"']*(?:flex|grid|gap-|p-|m-)[^"']*["']/)
+})
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
new file mode 100644
index 00000000..15f8c284
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
@@ -0,0 +1,3 @@
+export function App() {
+  return <main>Hello world</main>
+}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/styles.css b/scenarios/024-agent-sets-up-tailwindcss/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json b/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html b/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
new file mode 100644
index 00000000..84f90292
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite template</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
new file mode 100644
index 00000000..0d371a78
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
@@ -0,0 +1,24 @@
+{
+  "name": "025-agent-uses-tokens-with-tailwindcss",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@tailwindcss/vite": "^4.3.3",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "tailwindcss": "^4.3.3",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
new file mode 100644
index 00000000..8adb1f7b
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent uses design tokens when styling with Tailwind CSS.',
+  prompt: `Use Tailwind utility classes to style the deployment status panel while keeping its colors and spacing aligned with the design system.`,
+  tags: ['tailwindcss', 'tokens', 'vite'],
+})
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
new file mode 100644
index 00000000..fd9e8dfd
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
+const source = `${app}\n${styles}`
+
+test('uses Tailwind utility classes on the deployment panel', () => {
+  expect(app).toMatch(/className=["'][^"']*(?:bg-|text-|border-|p-|gap-)[^"']*["']/)
+})
+
+test('uses semantic status tokens with Tailwind', () => {
+  expect(source).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/)
+})
+
+test('uses stack tokens for panel spacing', () => {
+  expect(source).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/)
+})
+
+test('does not introduce raw hexadecimal colors', () => {
+  expect(source).not.toMatch(/#[\da-f]{3,8}\b/i)
+})
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
new file mode 100644
index 00000000..93b598b9
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
@@ -0,0 +1,11 @@
+export function App() {
+  return (
+    <main>
+      <h1>Deployments</h1>
+      <section className="deployment-status">
+        <h2>Production</h2>
+        <p>Deployment succeeded</p>
+      </section>
+    </main>
+  )
+}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
new file mode 100644
index 00000000..4bf28b51
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
@@ -0,0 +1,16 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App'
+import './styles.css'
+
+const root = document.getElementById('root')
+
+if (!root) {
+  throw new Error('Root element not found')
+}
+
+createRoot(root).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
new file mode 100644
index 00000000..43cacaf0
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
@@ -0,0 +1,9 @@
+@import 'tailwindcss';
+
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
new file mode 100644
index 00000000..6fb85ea5
--- /dev/null
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
@@ -0,0 +1,6 @@
+import tailwindcss from '@tailwindcss/vite'
+import {defineConfig} from 'vite'
+
+export default defineConfig({
+  plugins: [tailwindcss()],
+})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/index.html b/scenarios/026-agent-avoids-deprecated-notification/index.html
new file mode 100644
index 00000000..1bd99ac0
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/index.html
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React + TS</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/scenarios/026-agent-avoids-deprecated-notification/package.json b/scenarios/026-agent-avoids-deprecated-notification/package.json
new file mode 100644
index 00000000..7d8b7dc0
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "026-agent-avoids-deprecated-notification",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "react": "19.2.8",
+    "react-dom": "19.2.8"
+  },
+  "devDependencies": {
+    "@primer/agent-eval": "workspace:*",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^6",
+    "vite": "^8",
+    "vitest": "^4.1.8"
+  }
+}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
new file mode 100644
index 00000000..98ff5186
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
@@ -0,0 +1,7 @@
+import {defineScenario} from '@primer/agent-eval/scenario'
+
+export default defineScenario({
+  description: 'Evaluate whether the agent avoids deprecated components when adding a page-level warning.',
+  prompt: `Add a dismissible page-level warning above the repository settings when branch protection is disabled. Include a link to enable branch protection.`,
+  tags: ['banner', 'component', 'deprecated', 'vite'],
+})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
new file mode 100644
index 00000000..99ed9b58
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
@@ -0,0 +1,22 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {expect, test} from 'vitest'
+
+const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
+
+test('uses the current Banner component', () => {
+  expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
+  expect(app).toMatch(/<Banner[^>]*variant=["']warning["'][^>]*>/)
+})
+
+test('makes the warning dismissible', () => {
+  expect(app).toMatch(/<Banner[^>]*\bonDismiss=/)
+})
+
+test('links to branch protection settings', () => {
+  expect(app).toMatch(/<Banner[\s\S]*branch protection[\s\S]*(?:href|Link)[\s\S]*<\/Banner>/i)
+})
+
+test('does not use the deprecated Flash component', () => {
+  expect(app).not.toMatch(/\bFlash\b/)
+})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
new file mode 100644
index 00000000..14b146a4
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
@@ -0,0 +1,11 @@
+export function App() {
+  const isBranchProtectionEnabled = false
+
+  return (
+    <main>
+      <h1>Repository settings</h1>
+      <p>Branch protection is {isBranchProtectionEnabled ? 'enabled' : 'disabled'} for the default branch.</p>
+      <a href="/settings/branches">Branch protection settings</a>
+    </main>
+  )
+}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
new file mode 100644
index 00000000..83f3281d
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
@@ -0,0 +1,10 @@
+import {StrictMode} from 'react'
+import {createRoot} from 'react-dom/client'
+import {App} from './App.tsx'
+import './styles.css'
+
+createRoot(document.getElementById('root')!).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/styles.css b/scenarios/026-agent-avoids-deprecated-notification/src/styles.css
new file mode 100644
index 00000000..2142ede2
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/src/styles.css
@@ -0,0 +1,7 @@
+:root {
+  font-family: system-ui, sans-serif;
+}
+
+body {
+  margin: 0;
+}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json b/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json
new file mode 100644
index 00000000..d7265e99
--- /dev/null
+++ b/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json
@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "types": ["vite/client"]
+  },
+  "include": ["src"]
+}

From ebc72b2944364e5f1d44f93b816f138105da756a Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 26 Aug 2026 20:38:42 -0500
Subject: [PATCH 03/52] feat: add run to benchmark

---
 .changeset/calm-benchmarks-compare.md  |   5 +
 benchmarks/design-system.ts            |  92 ++--
 packages/agent-eval/package.json       |   4 +
 packages/agent-eval/rolldown.config.ts |   1 +
 packages/agent-eval/src/benchmark.ts   | 150 ++++++
 packages/agent-eval/src/cli.ts         |  15 +-
 packages/agent-eval/src/index.ts       |   2 +
 packages/agent-eval/src/scenario.ts    |  59 +++
 packages/agent-eval/src/treatment.ts   |   6 +-
 pnpm-lock.yaml                         | 606 +++++++++++++++++++++++++
 10 files changed, 900 insertions(+), 40 deletions(-)
 create mode 100644 .changeset/calm-benchmarks-compare.md
 create mode 100644 packages/agent-eval/src/benchmark.ts
 create mode 100644 packages/agent-eval/src/scenario.ts

diff --git a/.changeset/calm-benchmarks-compare.md b/.changeset/calm-benchmarks-compare.md
new file mode 100644
index 00000000..3607a2d2
--- /dev/null
+++ b/.changeset/calm-benchmarks-compare.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add a benchmark configuration API for grouping evaluation scenarios by capability.
diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts
index 90688cba..2ad2f016 100644
--- a/benchmarks/design-system.ts
+++ b/benchmarks/design-system.ts
@@ -6,80 +6,96 @@ export const benchmark = defineConfig({
   capabilities: [
     {
       name: 'Uses components',
-      scenarios: [
-        // Uses component from the design system unsolicited (e.g. when referring to a button)
-        // Does not re-create a component that already exists
-      ],
+      scenarios: ['001-agent-uses-button-from-primer', '006-agent-uses-pagination-component'],
     },
     {
       name: 'Infers correct component based on usage',
-      scenarios: [
-        // When talking broadly about an action or case, translates to
-        // components in Primer instead of creating a new component
-      ],
+      scenarios: ['007-agent-infers-billing-banner', '008-agent-infers-action-menu'],
+    },
+    {
+      name: 'Composes components correctly',
+      scenarios: [],
+    },
+    {
+      name: 'Uses documented component APIs',
+      scenarios: [],
     },
     {
       name: 'Uses tokens',
       scenarios: [
-        // Color tokens
-        // Size tokens
-        // Typography tokens
-        // Motion tokens
+        '009-agent-uses-layout-and-color-tokens',
+        '010-agent-uses-typography-tokens',
+        '011-agent-uses-motion-tokens',
       ],
     },
     {
       name: 'Infers correct token based on usage',
-      scenarios: [
-        // Given a specific context it can find the correct token
-      ],
+      scenarios: ['012-agent-infers-status-tokens', '013-agent-infers-compact-control-tokens'],
     },
     {
       name: 'Uses icons',
-      scenarios: [
-        // Uses icons from Primer before creating its own
-        // Does not use icons from other libraries
-        // Suggests creating an issue or contributing to primer/octicons if
-        // missing
-      ],
+      scenarios: ['002-agent-uses-octicon-from-primer', '014-agent-replaces-custom-icons-with-octicons'],
     },
     {
       name: 'Infers correct icon based on usage',
-      scenarios: [
-        // Given a specific context it finds the right icon to use
-      ],
+      scenarios: ['015-agent-infers-copy-icon'],
     },
     {
       name: 'Uses UI patterns',
       scenarios: [
-        // Given a specific context it infers what components to use and how to
-        // combine them (forms, loading states, empty states, etc)
+        '003-agent-uses-form-from-primer',
+        '016-agent-uses-loading-and-empty-state-patterns',
+        '017-agent-uses-confirmation-pattern',
+        '018-agent-uses-filter-pattern',
       ],
     },
+    {
+      name: 'Implements navigation patterns',
+      scenarios: [],
+    },
+    {
+      name: 'Applies accessibility guidance',
+      scenarios: [],
+    },
+    {
+      name: 'Builds responsive interfaces',
+      scenarios: [],
+    },
+    {
+      name: 'Handles interaction states',
+      scenarios: [],
+    },
     {
       name: 'Uses utilities',
-      scenarios: [
-        // Prefers importing and using hooks instead of re-implementing them
-      ],
+      scenarios: ['019-agent-uses-dismissal-utilities', '020-agent-uses-resize-observer-utility'],
+    },
+    {
+      name: 'Extends the design system safely',
+      scenarios: [],
     },
     {
       name: 'Getting started',
-      scenarios: [
-        // New project
-      ],
+      scenarios: ['004-agent-setup-nextjs', '021-agent-sets-up-primer-in-vite'],
     },
     {
       name: 'Theming',
       scenarios: [
-        // Bringing in primitives
-        // Theme switching
+        '005-agent-enables-theme-switching',
+        '022-agent-enables-automatic-theming',
+        '023-agent-adds-theme-switcher',
       ],
     },
+    {
+      name: 'Supports accessible color modes',
+      scenarios: [],
+    },
     {
       name: 'Works with TailwindCSS',
-      scenarios: [
-        // New project can bring in tailwindcss
-        // Uses correct utility classes
-      ],
+      scenarios: ['024-agent-sets-up-tailwindcss', '025-agent-uses-tokens-with-tailwindcss'],
+    },
+    {
+      name: 'Respects component maturity',
+      scenarios: ['026-agent-avoids-deprecated-notification'],
     },
   ],
 })
diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index 673cc8d7..a62b7c7f 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -10,6 +10,10 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./benchmark": {
+      "types": "./dist/benchmark.d.ts",
+      "default": "./dist/benchmark.js"
+    },
     "./cli": {
       "types": "./dist/cli.d.ts",
       "default": "./dist/cli.js"
diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts
index c5ddf713..047b8282 100644
--- a/packages/agent-eval/rolldown.config.ts
+++ b/packages/agent-eval/rolldown.config.ts
@@ -9,6 +9,7 @@ const external = dependencies.map(name => {
 
 const config = defineConfig({
   input: {
+    benchmark: 'src/benchmark.ts',
     cli: 'src/cli.ts',
     experiment: 'src/experiment.ts',
     index: 'src/index.ts',
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
new file mode 100644
index 00000000..080cf0b7
--- /dev/null
+++ b/packages/agent-eval/src/benchmark.ts
@@ -0,0 +1,150 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+// import * as z from 'zod/mini'
+import type {Model, ModelConfig, ReasoningEffort} from './model'
+import type {Sandbox} from './sandbox'
+import {ControlTreatment, type TreatmentConfig} from './experiment-config'
+import {getScenario} from './scenario'
+import type {Scenario} from './scenario'
+import {randomUUID} from 'node:crypto'
+
+// const BenchmarkConfigSchema = z.object({
+//   name: z.string(),
+//   description: z.string(),
+//   capabilities: z.array(
+//     z.object({
+//       name: z.string(),
+//       scenarios: z.array(z.string()),
+//     }),
+//   ),
+// })
+
+type BenchmarkConfig = {
+  name: string
+  description: string
+  models: Array<ModelConfig>
+  setup: Setup
+  capabilities: Array<{
+    name: string
+    scenarios: Array<string>
+  }>
+}
+
+type Setup = ({sandbox}: {sandbox: Sandbox}) => Promise<void>
+
+function defineConfig(config: BenchmarkConfig): BenchmarkConfig {
+  return config
+}
+
+type BenchmarkModule = {
+  benchmark: BenchmarkConfig
+}
+
+const BENCHMARK_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
+
+async function listBenchmarks(directory: string) {
+  const stats = await fs.stat(directory)
+  if (!stats.isDirectory()) {
+    throw new Error('Expected benchmarks to be a directory')
+  }
+
+  const filenames = await fs.readdir(directory)
+  const benchmarks = await Promise.all(
+    filenames
+      .filter(filename => {
+        return isBenchmarkFile(filename)
+      })
+      .map(async filename => {
+        const filepath = path.join(directory, filename)
+        const mod: BenchmarkModule = await import(filepath)
+        if (!mod.benchmark) {
+          throw new Error(`Benchmark file must export "benchmark": ${filepath}`)
+        }
+        return [getBenchmarkId(filename), mod.benchmark] as const
+      }),
+  )
+
+  return benchmarks
+}
+
+async function getBenchmark(directory: string, id: string): Promise<BenchmarkConfig> {
+  const benchmarks = await listBenchmarks(directory)
+  const benchmark = benchmarks.find(([benchmarkId]) => benchmarkId === id)
+  if (!benchmark) {
+    throw new Error(`Benchmark "${id}" was not found in: ${directory}`)
+  }
+
+  return benchmark[1]
+}
+
+function getBenchmarkId(filename: string): string {
+  return path.basename(filename)
+}
+
+function isBenchmarkFile(filename: string): boolean {
+  if (path.extname(filename) === '.d.ts') {
+    return false
+  }
+
+  if (!BENCHMARK_FILE_EXTENSIONS.has(path.extname(filename))) {
+    return false
+  }
+
+  if (path.basename(filename) === 'index') {
+    return false
+  }
+
+  return true
+}
+
+type BenchmarkResult = {}
+
+type Treatment = {
+  id: string
+  scenario: Scenario
+  config: {
+    name: string
+    setup?: Setup
+  }
+  model: Model
+  reasoningEffort: ReasoningEffort | null
+}
+
+type RunOptions = {
+  artifactsDirectory: string
+  benchmarksDirectory: string
+  scenariosDirectory: string
+}
+
+async function run(config: BenchmarkConfig, options: RunOptions): Promise<BenchmarkResult> {
+  const treatments = config.capabilities.flatMap(capability => {
+    return config.models.flatMap(model => {
+      return capability.scenarios.flatMap(scenario => {
+        return [
+          {
+            id: randomUUID(),
+            config: ControlTreatment,
+            scenario,
+            setup: config.setup,
+            model: model.name,
+            reasoningEffort: model.reasoningEfforts.length > 0 ? model.reasoningEfforts[0] : null,
+          },
+          {
+            id: randomUUID(),
+            scenario,
+            config: {
+              name: 'Benchmark',
+              setup: config.setup,
+            },
+            model: model.name,
+            reasoningEffort: model.reasoningEfforts.length > 0 ? model.reasoningEfforts[0] : null,
+          },
+        ]
+      })
+    })
+  })
+  //
+}
+
+export {defineConfig, listBenchmarks, getBenchmark, run}
+export type {BenchmarkConfig}
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index e56a7ff9..1b7f4262 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -20,6 +20,15 @@ const {values} = parseArgs({
       short: 'a',
       description: 'The directory to save artifacts to',
     },
+    benchmark: {
+      type: 'string',
+      short: 'b',
+      description: 'The file name of the benchmark to run',
+    },
+    benchmarks: {
+      type: 'string',
+      description: 'The directory containing local benchmark files',
+    },
     concurrency: {
       type: 'string',
       short: 'c',
@@ -66,6 +75,8 @@ Usage: agent-eval [options]
 
 Options:
   -a, --artifacts <dir>      The directory to save artifacts to
+  -b, --benchmark <file>     The file name of the benchmark to run
+      --benchmarks <dir>     The directory containing local benchmark files
   -c, --concurrency <num>    The number of treatments to run in parallel
       --docker-image <image> The Docker container image to use for running treatments (must be a Debian-based Node image with apt-get and a node user, e.g. node:26.5.0-slim)
   -e, --experiment <file>    The file name of the experiment to run
@@ -102,7 +113,9 @@ if (!existsSync(ARTIFACTS_DIR)) {
   await fs.mkdir(ARTIFACTS_DIR, {recursive: true})
 }
 
-if (values.experiment) {
+if (values.benchmark) {
+  //
+} else if (values.experiment) {
   const config = await findExperiment(values.experiment, {
     directory: values.experiments,
   })
diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts
index 8e0a2f11..d99dfebe 100644
--- a/packages/agent-eval/src/index.ts
+++ b/packages/agent-eval/src/index.ts
@@ -1,3 +1,5 @@
+export {getBenchmark, listBenchmarks} from './benchmark'
+export type {BenchmarkConfig} from './benchmark'
 export {findExperiment, listExperiments, loadExperimentConfigs} from './experiments'
 export type {ExperimentSourceOptions, LoadExperimentOptions} from './experiments'
 export {findScenario, listScenarios} from './scenarios'
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
new file mode 100644
index 00000000..aafb890f
--- /dev/null
+++ b/packages/agent-eval/src/scenario.ts
@@ -0,0 +1,59 @@
+import {existsSync} from 'node:fs'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+type Scenario = {
+  id: string
+  directory: string
+}
+
+async function listScenarios(directory: string): Promise<Array<Scenario>> {
+  const stats = await fs.stat(directory)
+  if (!stats.isDirectory()) {
+    throw new Error('Expected benchmarks to be a directory')
+  }
+
+  const entries = await fs.readdir(directory, {
+    withFileTypes: true,
+  })
+  return entries
+    .filter(entry => {
+      if (!entry.isDirectory()) {
+        return false
+      }
+
+      const packageJsonPath = path.join(directory, entry.name, 'package.json')
+      if (!existsSync(packageJsonPath)) {
+        return false
+      }
+
+      if (entry.name.startsWith('.')) {
+        return false
+      }
+
+      if (entry.name.startsWith('000')) {
+        return false
+      }
+
+      return true
+    })
+    .map(entry => {
+      return {
+        id: entry.name,
+        directory: path.join(directory, entry.name),
+      }
+    })
+}
+
+async function getScenario(directory: string, id: string): Promise<Scenario> {
+  const scenarios = await listScenarios(directory)
+  const scenario = scenarios.find(scenario => scenario.id === id)
+  if (scenario) {
+    return scenario
+  }
+
+  throw new Error(`Scenario "${id}" was not found in: ${directory}`)
+}
+
+export {listScenarios, getScenario}
+export type {Scenario}
diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts
index 40dfb674..8e7d9101 100644
--- a/packages/agent-eval/src/treatment.ts
+++ b/packages/agent-eval/src/treatment.ts
@@ -47,10 +47,14 @@ type TreatmentResult = {
   walkthrough: Walkthrough
 }
 
+const ControlTreatment: TreatmentConfig = {
+  name: 'Control',
+}
+
 type Walkthrough =
   | {type: 'Unavailable'}
   | {type: 'Screenshot'; filepath: string}
   | {type: 'Screenshots'; screenshots: Array<string>}
   | {type: 'Video'; filepath: string}
 
-export type {Treatment, TreatmentResult, Walkthrough}
+export type {Treatment, TreatmentResult, ControlTreatment, Walkthrough}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 529cbe43..f7153f7d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -362,6 +362,600 @@ importers:
         specifier: ^4.1.11
         version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
+  scenarios/006-agent-uses-pagination-component:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/007-agent-infers-billing-banner:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/008-agent-infers-action-menu:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/009-agent-uses-layout-and-color-tokens:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/010-agent-uses-typography-tokens:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/011-agent-uses-motion-tokens:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/012-agent-infers-status-tokens:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/013-agent-infers-compact-control-tokens:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/014-agent-replaces-custom-icons-with-octicons:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/015-agent-infers-copy-icon:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/016-agent-uses-loading-and-empty-state-patterns:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/017-agent-uses-confirmation-pattern:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/018-agent-uses-filter-pattern:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/019-agent-uses-dismissal-utilities:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/020-agent-uses-resize-observer-utility:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/021-agent-sets-up-primer-in-vite:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/022-agent-enables-automatic-theming:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/023-agent-adds-theme-switcher:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/024-agent-sets-up-tailwindcss:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/025-agent-uses-tokens-with-tailwindcss:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@tailwindcss/vite':
+        specifier: ^4.3.3
+        version: 4.3.3(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      tailwindcss:
+        specifier: ^4.3.3
+        version: 4.3.3
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
+  scenarios/026-agent-avoids-deprecated-notification:
+    dependencies:
+      react:
+        specifier: 19.2.8
+        version: 19.2.8
+      react-dom:
+        specifier: 19.2.8
+        version: 19.2.8(react@19.2.8)
+    devDependencies:
+      '@primer/agent-eval':
+        specifier: workspace:*
+        version: link:../../packages/agent-eval
+      '@types/react':
+        specifier: ^19
+        version: 19.2.18
+      '@types/react-dom':
+        specifier: ^19
+        version: 19.2.4(@types/react@19.2.18)
+      typescript:
+        specifier: ^6
+        version: 6.0.3
+      vite:
+        specifier: ^8
+        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vitest:
+        specifier: ^4.1.8
+        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+
   website:
     dependencies:
       '@primer/agent-eval':
@@ -1162,6 +1756,11 @@ packages:
   '@tailwindcss/postcss@4.3.3':
     resolution: {integrity: sha512-JTSZZGQi1AyKirbLN3azmjVzef92tcX7h+iSqPdaeStyFpGpDlKvvpxeOE8njhbUanbRwr3z8DyzhICWnMtQeg==}
 
+  '@tailwindcss/vite@4.3.3':
+    resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==}
+    peerDependencies:
+      vite: ^5.2.0 || ^6 || ^7 || ^8
+
   '@tanstack/react-virtual@3.14.6':
     resolution: {integrity: sha512-4+Uq8m0/gzO4kMCHUEpTtGX1RnONK0C+g88b2ltwPMWUBiaVarBuWKoPJaz7gj1cKCVRAdyu+U8GcKhwCc2beA==}
     peerDependencies:
@@ -3948,6 +4547,13 @@ snapshots:
       postcss: 8.5.26
       tailwindcss: 4.3.3
 
+  '@tailwindcss/vite@4.3.3(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))':
+    dependencies:
+      '@tailwindcss/node': 4.3.3
+      '@tailwindcss/oxide': 4.3.3
+      tailwindcss: 4.3.3
+      vite: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+
   '@tanstack/react-virtual@3.14.6(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
     dependencies:
       '@tanstack/virtual-core': 3.17.4

From 742ebe42a7fbcb0f36b79a0ed7a69841bf219a88 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 27 Aug 2026 00:05:29 -0500
Subject: [PATCH 04/52] test: add host for local testing

---
 packages/agent-eval/src/host.ts | 60 +++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 packages/agent-eval/src/host.ts

diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
new file mode 100644
index 00000000..dd5b7db9
--- /dev/null
+++ b/packages/agent-eval/src/host.ts
@@ -0,0 +1,60 @@
+import {existsSync} from 'node:fs'
+import fs from 'node:fs/promises'
+import {memfs, Volume, type NestedDirectoryJSON} from 'memfs'
+
+type FileSystem = typeof import('node:fs/promises')
+
+interface Host {
+  existsSync: typeof existsSync
+  fs: FileSystem
+  loadModule<T = unknown>(filepath: string): Promise<T>
+}
+
+class SystemHost implements Host {
+  static create() {
+    return new SystemHost()
+  }
+
+  existsSync: typeof existsSync
+  fs: FileSystem
+
+  constructor() {
+    this.existsSync = existsSync
+    this.fs = fs
+  }
+
+  loadModule<T = unknown>(filepath: string): Promise<T> {
+    return import(filepath)
+  }
+}
+
+class VirtualHost implements Host {
+  static create(json?: NestedDirectoryJSON) {
+    return new VirtualHost(json)
+  }
+
+  existsSync: typeof existsSync
+  fs: FileSystem
+  vol: Volume
+
+  constructor(json?: NestedDirectoryJSON) {
+    const {fs, vol} = memfs(json)
+    this.vol = vol
+    this.existsSync = fs.existsSync
+    // @ts-expect-error - not every constant is exposed but memfs should match
+    // for our test cases
+    this.fs = fs.promises
+  }
+
+  async loadModule<T = unknown>(filepath: string): Promise<T> {
+    const contents = await this.fs.readFile(filepath, 'utf-8')
+    const encodedContent = Buffer.from(contents).toString('base64')
+    const dataUri = `data:text/javascript;base64,${encodedContent}`
+    return await import(dataUri)
+  }
+}
+
+const DefaultHost = new SystemHost()
+
+export {SystemHost, VirtualHost, DefaultHost}
+export type {Host}

From 73c5a5539078dd0e7e4aff640f52218af1d643cf Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 27 Aug 2026 00:05:55 -0500
Subject: [PATCH 05/52] refactor: update scenario implementation

---
 packages/agent-eval/src/scenario.test.ts | 203 +++++++++++------------
 packages/agent-eval/src/scenario.ts      | 133 ++++++++++-----
 2 files changed, 196 insertions(+), 140 deletions(-)

diff --git a/packages/agent-eval/src/scenario.test.ts b/packages/agent-eval/src/scenario.test.ts
index 390a65a5..886ff993 100644
--- a/packages/agent-eval/src/scenario.test.ts
+++ b/packages/agent-eval/src/scenario.test.ts
@@ -1,120 +1,117 @@
-import fs from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import {afterEach, describe, expect, test} from 'vitest'
-import {resolveExperimentScenario} from './resolve-experiment-scenario'
+import {test, expect} from 'vitest'
+import {VirtualHost} from './host'
+import {listScenarios, getScenario, defineConfig} from './scenario'
 
-const temporaryDirectories: Array<string> = []
-
-async function createTemporaryDirectory() {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-'))
-  temporaryDirectories.push(directory)
-  return directory
-}
-
-afterEach(async () => {
-  await Promise.all(
-    temporaryDirectories.splice(0).map(directory => {
-      return fs.rm(directory, {recursive: true, force: true})
+test('listScenarios', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      prompt: 'test',
     }),
   )
-})
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '.hidden-directory': {},
+      '.hidden-file': '',
+      '001-scenario': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+      '002-missing-package-json': {
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+      '003-missing-config': {
+        'package.json': '{}',
+        'scenario.test.ts': '',
+      },
+      '004-invalid-config': {
+        'package.json': '{}',
+        'scenario.config.ts': '',
+      },
+      '005-missing-test': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+      },
+    },
+  })
+  const scenarios = await listScenarios(host, '/scenarios')
 
-describe(resolveExperimentScenario, () => {
-  test('resolves named scenarios from the provided directory', async () => {
-    const scenariosDirectory = await createTemporaryDirectory()
-    const directory = path.join(scenariosDirectory, 'button-scenario')
-    await fs.mkdir(directory)
-    await fs.writeFile(path.join(directory, 'scenario.config.ts'), `export default {prompt: 'Use a button'}`)
-    await fs.writeFile(path.join(directory, 'scenario.test.ts'), '')
+  expect(scenarios).toHaveLength(1)
+  expect(scenarios).toContainEqual({
+    id: '001-scenario',
+    directory: '/scenarios/001-scenario',
+    prompt: 'test',
+    tags: [],
+    testPath: '/scenarios/001-scenario/scenario.test.ts',
+  })
 
-    await expect(
-      resolveExperimentScenario('button-scenario', {
-        directory: scenariosDirectory,
-      }),
-    ).resolves.toEqual({
-      id: 'button-scenario',
-      directory,
-      config: {
-        prompt: 'Use a button',
-      },
-      testPath: path.join(directory, 'scenario.test.ts'),
-    })
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '.hidden-directory'}))
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '.hidden-file'}))
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '002-missing-package-json'}))
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '003-missing-config'}))
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '004-invalid-config'}))
+  expect(scenarios).not.toContainEqual(expect.objectContaining({id: '005-missing-test'}))
+})
+
+test('throws if input is not a directory', async () => {
+  const host = VirtualHost.create({
+    '/test': '',
   })
 
-  test('resolves inline scenario directories relative to the provided cwd', async () => {
-    const cwd = await createTemporaryDirectory()
-    const directory = path.join(cwd, 'scenarios', 'local-scenario')
-    await fs.mkdir(directory, {recursive: true})
-    await fs.writeFile(path.join(directory, 'scenario.config.mjs'), `export default {prompt: 'Use ignored config'}`)
-    await fs.writeFile(
-      path.join(directory, 'scenario.config.ts'),
-      `export default {prompt: 'Update the local project'}`,
-    )
-    await fs.writeFile(path.join(directory, 'scenario.test.ts'), '')
+  await expect(() => listScenarios(host, '/test')).rejects.toThrowErrorMatchingInlineSnapshot(
+    `[Error: Expected scenarios path to be a directory]`,
+  )
+})
 
-    await expect(
-      resolveExperimentScenario(
-        {
-          name: 'local-scenario',
-          path: 'scenarios/local-scenario',
-        },
-        {
-          cwd,
-        },
-      ),
-    ).resolves.toEqual({
-      id: 'local-scenario',
-      directory,
-      config: {
-        prompt: 'Update the local project',
+test('getScenario', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      prompt: 'test',
+    }),
+  )
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '001-scenario': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+      '002-scenario': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
       },
-      testPath: path.join(directory, 'scenario.test.ts'),
-    })
+    },
   })
 
-  test('defaults inline scenario names to the directory name', async () => {
-    const cwd = await createTemporaryDirectory()
-    const directory = path.join(cwd, 'scenarios', 'local-button-scenario')
-    await fs.mkdir(directory, {recursive: true})
-    await fs.writeFile(
-      path.join(directory, 'scenario.config.ts'),
-      `export default {prompt: 'Update the local project'}`,
-    )
-    await fs.writeFile(path.join(directory, 'scenario.test.ts'), '')
+  await expect(getScenario(host, '/scenarios', '001-scenario')).resolves.toEqual({
+    id: '001-scenario',
+    directory: '/scenarios/001-scenario',
+    prompt: 'test',
+    tags: [],
+    testPath: '/scenarios/001-scenario/scenario.test.ts',
+  })
 
-    await expect(
-      resolveExperimentScenario(
-        {
-          path: './scenarios/local-button-scenario',
-        },
-        {
-          cwd,
-        },
-      ),
-    ).resolves.toMatchObject({
-      id: 'local-button-scenario',
-      directory,
-    })
+  await expect(getScenario(host, '/scenarios', '002-scenario')).resolves.toEqual({
+    id: '002-scenario',
+    directory: '/scenarios/002-scenario',
+    prompt: 'test',
+    tags: [],
+    testPath: '/scenarios/002-scenario/scenario.test.ts',
   })
 
-  test('requires inline scenarios to use the default scenario file structure', async () => {
-    const cwd = await createTemporaryDirectory()
-    const directory = path.join(cwd, 'fixtures', 'local-scenario')
-    await fs.mkdir(directory, {recursive: true})
-    await fs.writeFile(path.join(directory, 'scenario.config.ts'), `export default {prompt: 'Use default config'}`)
-    await fs.writeFile(path.join(directory, 'custom.test.ts'), '')
+  await expect(getScenario(host, '/scenarios', '003-scenario')).rejects.toThrowErrorMatchingInlineSnapshot(
+    `[Error: Scenario "003-scenario" was not found in: /scenarios]`,
+  )
+})
 
-    await expect(
-      resolveExperimentScenario(
-        {
-          name: 'local-scenario',
-          path: 'fixtures/local-scenario',
-        },
-        {
-          cwd,
-        },
-      ),
-    ).rejects.toThrow(`Scenario "local-scenario" test file was not found: ${path.join(directory, 'scenario.test.ts')}`)
+test('getScenario throws if input is not a directory', async () => {
+  const host = VirtualHost.create({
+    '/test': '',
   })
+
+  await expect(() => getScenario(host, '/test', '001-scenario')).rejects.toThrowErrorMatchingInlineSnapshot(
+    `[Error: Expected scenarios path to be a directory]`,
+  )
 })
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index aafb890f..73ca8a1e 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -1,52 +1,111 @@
-import {existsSync} from 'node:fs'
-import fs from 'node:fs/promises'
 import path from 'node:path'
+import * as z from 'zod/mini'
+import type {Host} from './host'
+
+const ScenarioConfigSchema = z.object({
+  description: z.optional(z.string()),
+  prompt: z.string(),
+  tags: z.optional(z.array(z.string())),
+})
+
+type ScenarioConfig = z.infer<typeof ScenarioConfigSchema>
+
+function defineConfig(config: ScenarioConfig): ScenarioConfig {
+  return config
+}
+
+type ScenarioConfigModule = {
+  default?: unknown
+}
 
 type Scenario = {
   id: string
   directory: string
+  prompt: string
+  description?: string
+  tags: Array<string>
+  testPath: string
+  browserTestPath?: string
 }
 
-async function listScenarios(directory: string): Promise<Array<Scenario>> {
-  const stats = await fs.stat(directory)
+async function listScenarios(host: Host, directory: string): Promise<Array<Scenario>> {
+  const stats = await host.fs.stat(directory)
   if (!stats.isDirectory()) {
-    throw new Error('Expected benchmarks to be a directory')
+    throw new Error('Expected scenarios path to be a directory')
   }
 
-  const entries = await fs.readdir(directory, {
+  const entries = await host.fs.readdir(directory, {
     withFileTypes: true,
   })
-  return entries
-    .filter(entry => {
-      if (!entry.isDirectory()) {
-        return false
-      }
-
-      const packageJsonPath = path.join(directory, entry.name, 'package.json')
-      if (!existsSync(packageJsonPath)) {
-        return false
-      }
-
-      if (entry.name.startsWith('.')) {
-        return false
-      }
-
-      if (entry.name.startsWith('000')) {
-        return false
-      }
-
-      return true
-    })
-    .map(entry => {
-      return {
-        id: entry.name,
-        directory: path.join(directory, entry.name),
-      }
-    })
+  const candidates = entries.filter(entry => {
+    if (!entry.isDirectory()) {
+      return false
+    }
+
+    const packageJsonPath = path.join(directory, entry.name, 'package.json')
+    if (!host.existsSync(packageJsonPath)) {
+      return false
+    }
+
+    if (entry.name.startsWith('.')) {
+      return false
+    }
+
+    if (entry.name.startsWith('000')) {
+      return false
+    }
+
+    const scenarioConfigPath = path.join(directory, entry.name, 'scenario.config.ts')
+    if (!host.existsSync(scenarioConfigPath)) {
+      return false
+    }
+
+    const testPath = path.join(directory, entry.name, 'scenario.test.ts')
+    if (!host.existsSync(testPath)) {
+      return false
+    }
+
+    return true
+  })
+  const scenarios: Array<Scenario> = []
+
+  for (const entry of candidates) {
+    const data: ScenarioConfigModule = await host.loadModule(path.join(directory, entry.name, 'scenario.config.ts'))
+    if (!data.default) {
+      continue
+    }
+
+    const parseResult = ScenarioConfigSchema.safeParse(data.default)
+    if (!parseResult.success) {
+      continue
+    }
+
+    const {data: config} = parseResult
+    const scenario: Scenario = {
+      id: entry.name,
+      directory: path.join(directory, entry.name),
+      prompt: config.prompt,
+      tags: config.tags ?? [],
+      testPath: path.join(directory, entry.name, 'scenario.test.ts'),
+    }
+
+    if (config.description) {
+      scenario.description = config.description
+    }
+
+    const browserTestPath = path.join(directory, entry.name, 'browser.test.ts')
+    if (host.existsSync(browserTestPath)) {
+      scenario.browserTestPath = browserTestPath
+    }
+
+    scenarios.push(scenario)
+  }
+
+  return scenarios
 }
 
-async function getScenario(directory: string, id: string): Promise<Scenario> {
-  const scenarios = await listScenarios(directory)
+async function getScenario(host: Host, directory: string, id: string): Promise<Scenario> {
+  const scenarios = await listScenarios(host, directory)
   const scenario = scenarios.find(scenario => scenario.id === id)
   if (scenario) {
     return scenario
@@ -55,5 +114,5 @@ async function getScenario(directory: string, id: string): Promise<Scenario> {
   throw new Error(`Scenario "${id}" was not found in: ${directory}`)
 }
 
-export {listScenarios, getScenario}
-export type {Scenario}
+export {defineConfig, listScenarios, getScenario, ScenarioConfigSchema}
+export type {ScenarioConfig, Scenario}

From de0d665f3ed26288ce2e1e3dc315dfee41ca6fde Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 27 Aug 2026 00:06:18 -0500
Subject: [PATCH 06/52] refactor: update model implementation

---
 packages/agent-eval/src/model.test.ts |  91 ++++++++++---------
 packages/agent-eval/src/model.ts      | 125 +++++++++++++++++++-------
 2 files changed, 141 insertions(+), 75 deletions(-)

diff --git a/packages/agent-eval/src/model.test.ts b/packages/agent-eval/src/model.test.ts
index 6376d2d5..e7b6646b 100644
--- a/packages/agent-eval/src/model.test.ts
+++ b/packages/agent-eval/src/model.test.ts
@@ -1,48 +1,53 @@
-import {describe, expect, test} from 'vitest'
-import {models} from './index'
-import {resolveModelConfigs} from './model'
+import {test, expect} from 'vitest'
+import {getModelVariants} from './model'
 
-test('provides model information', () => {
-  expect(models).toEqual([
-    {name: 'claude-haiku-4.5', reasoningEfforts: []},
-    {name: 'claude-opus-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max']},
-    {name: 'claude-opus-4.7', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'claude-opus-4.8', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'claude-opus-5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'claude-sonnet-4.5', reasoningEfforts: []},
-    {name: 'claude-sonnet-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max']},
-    {name: 'claude-sonnet-5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'gemini-3.1-pro-preview', reasoningEfforts: ['low', 'medium', 'high']},
-    {name: 'gemini-3.5-flash', reasoningEfforts: ['minimal', 'low', 'medium', 'high']},
-    {name: 'gemini-3.6-flash', reasoningEfforts: []},
-    {name: 'gpt-5.4', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']},
-    {name: 'gpt-5.4-mini', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']},
-    {name: 'gpt-5.5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']},
-    {name: 'gpt-5.6-luna', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'gpt-5.6-sol', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
-    {name: 'gpt-5.6-terra', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']},
+test('getModelVariants', () => {
+  expect(getModelVariants([])).toEqual([])
+
+  expect(getModelVariants(['claude-opus-5'])).toEqual([
+    {
+      name: 'claude-opus-5',
+      reasoningEffort: 'medium',
+    },
+  ])
+
+  expect(getModelVariants([{name: 'claude-opus-5'}])).toEqual([
+    {
+      name: 'claude-opus-5',
+      reasoningEffort: 'medium',
+    },
   ])
-})
 
-describe('resolveModelConfigs', () => {
-  test('resolves each reasoning effort for a model', () => {
-    expect(resolveModelConfigs({name: 'gpt-5.5', reasoningEfforts: ['low', 'high']})).toEqual([
-      {
-        name: 'gpt-5.5',
-        reasoningEffort: 'low',
-      },
-      {
-        name: 'gpt-5.5',
-        reasoningEffort: 'high',
-      },
-    ])
-  })
+  expect(getModelVariants([{name: 'claude-opus-5', reasoningEfforts: []}])).toEqual([
+    {
+      name: 'claude-opus-5',
+      reasoningEffort: 'medium',
+    },
+  ])
+
+  expect(getModelVariants([{name: 'claude-opus-5', reasoningEfforts: ['medium', 'high']}])).toEqual([
+    {
+      name: 'claude-opus-5',
+      reasoningEffort: 'medium',
+    },
+    {
+      name: 'claude-opus-5',
+      reasoningEffort: 'high',
+    },
+  ])
 
-  test('omits reasoning effort for a model without supported efforts', () => {
-    expect(resolveModelConfigs({name: 'claude-haiku-4.5', reasoningEfforts: []})).toEqual([
-      {
-        name: 'claude-haiku-4.5',
-      },
-    ])
-  })
+  expect(getModelVariants(['gpt-5.6-sol', 'gpt-5.6-luna', 'gpt-5.6-terra'])).toEqual([
+    {
+      name: 'gpt-5.6-sol',
+      reasoningEffort: 'medium',
+    },
+    {
+      name: 'gpt-5.6-luna',
+      reasoningEffort: 'medium',
+    },
+    {
+      name: 'gpt-5.6-terra',
+      reasoningEffort: 'medium',
+    },
+  ])
 })
diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts
index c5ff917e..9c4c53c8 100644
--- a/packages/agent-eval/src/model.ts
+++ b/packages/agent-eval/src/model.ts
@@ -1,8 +1,6 @@
+import * as z from 'zod/mini'
+
 const models = [
-  {
-    name: 'claude-haiku-4.5',
-    reasoningEfforts: [],
-  },
   {
     name: 'claude-opus-4.6',
     reasoningEfforts: ['low', 'medium', 'high', 'max'],
@@ -19,10 +17,6 @@ const models = [
     name: 'claude-opus-5',
     reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max'],
   },
-  {
-    name: 'claude-sonnet-4.5',
-    reasoningEfforts: [],
-  },
   {
     name: 'claude-sonnet-4.6',
     reasoningEfforts: ['low', 'medium', 'high', 'max'],
@@ -39,10 +33,6 @@ const models = [
     name: 'gemini-3.5-flash',
     reasoningEfforts: ['minimal', 'low', 'medium', 'high'],
   },
-  {
-    name: 'gemini-3.6-flash',
-    reasoningEfforts: [],
-  },
   {
     name: 'gpt-5.4',
     reasoningEfforts: ['low', 'medium', 'high', 'xhigh'],
@@ -69,27 +59,98 @@ const models = [
   },
 ] as const
 
-type ModelInfo = (typeof models)[number]
-type Model = ModelInfo['name']
-type ReasoningEffort = ModelInfo['reasoningEfforts'][number]
-type ModelConfig = {
-  [Info in ModelInfo as Info['name']]: {
-    name: Info['name']
-    reasoningEfforts: Array<Info['reasoningEfforts'][number]>
-  }
-}[Model]
-type ExperimentModelConfig = ModelConfig
+type ModelConfig = (typeof models)[number]
+type Model = ModelConfig['name']
+type ReasoningEfforts<M extends Model> = Extract<ModelConfig, {name: M}>['reasoningEfforts']
+type ReasoningEffort<M extends Model> = ReasoningEfforts<M>[number]
+type ModelVariant<M extends Model> = {
+  name: M
+  reasoningEffort: ReasoningEffort<M>
+}
+
+const ModelVariantConfigSchema = z.array(
+  z.union([
+    z.enum(models.map(model => model.name)),
+    z.object({
+      name: z.literal(models[0].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[1].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[2].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[3].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[4].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[5].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[6].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[7].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[8].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[9].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[10].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[11].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[12].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[13].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))),
+    }),
+  ]),
+)
+
+type ModelVariantConfig = z.infer<typeof ModelVariantConfigSchema>
+
+function getModelVariants(input: ModelVariantConfig): Array<ModelVariant<Model>> {
+  return input.flatMap(config => {
+    if (typeof config === 'string') {
+      return [{name: config, reasoningEffort: 'medium'}]
+    }
+
+    if (!config.reasoningEfforts) {
+      return [{name: config.name, reasoningEffort: 'medium'}]
+    }
 
-function resolveModelConfigs(config: ExperimentModelConfig): Array<{name: Model; reasoningEffort?: ReasoningEffort}> {
-  if (config.reasoningEfforts.length === 0) {
-    return [{name: config.name}]
-  }
+    if (config.reasoningEfforts.length === 0) {
+      return [{name: config.name, reasoningEffort: 'medium'}]
+    }
 
-  return config.reasoningEfforts.map(reasoningEffort => ({
-    name: config.name,
-    reasoningEffort,
-  }))
+    return config.reasoningEfforts.map(effort => {
+      return {name: config.name, reasoningEffort: effort}
+    })
+  })
 }
 
-export {models, resolveModelConfigs}
-export type {ExperimentModelConfig, Model, ModelConfig, ModelInfo, ReasoningEffort}
+export {models, getModelVariants}
+export type {ModelConfig, Model, ReasoningEffort, ModelVariantConfig}

From 797a9c48907766ba9559b710c5067161e38a0b6f Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 27 Aug 2026 00:06:27 -0500
Subject: [PATCH 07/52] test: add test file for benchmarks

---
 packages/agent-eval/package.json          |   1 +
 packages/agent-eval/src/benchmark.test.ts |   6 +
 pnpm-lock.yaml                            | 399 +++++++++++++++++++---
 3 files changed, 360 insertions(+), 46 deletions(-)
 create mode 100644 packages/agent-eval/src/benchmark.test.ts

diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index a62b7c7f..d175accf 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -72,6 +72,7 @@
     "@types/node": "^26.4.0",
     "@types/tar-fs": "^2.0.4",
     "@types/tar-stream": "^3.1.4",
+    "memfs": "^4.68.1",
     "publint": "^0.3.24",
     "rimraf": "^6.1.3",
     "rolldown": "^1.2.6",
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
new file mode 100644
index 00000000..9c10bd0a
--- /dev/null
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -0,0 +1,6 @@
+import {test, expect} from 'vitest'
+import {defineConfig, listBenchmarks, getBenchmark} from './benchmark'
+
+test('hello', () => {
+  //
+})
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f7153f7d..fc839efb 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -61,7 +61,7 @@ importers:
         version: link:../packages/agent-eval
       '@types/node':
         specifier: ^26.1.2
-        version: 26.1.2
+        version: 26.2.0
       typescript:
         specifier: ^6.0.3
         version: 6.0.3
@@ -105,6 +105,9 @@ importers:
       '@types/tar-stream':
         specifier: ^3.1.4
         version: 3.1.4
+      memfs:
+        specifier: ^4.68.1
+        version: 4.68.1
       publint:
         specifier: ^0.3.24
         version: 0.3.24
@@ -385,10 +388,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/007-agent-infers-billing-banner:
     dependencies:
@@ -413,10 +416,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/008-agent-infers-action-menu:
     dependencies:
@@ -441,10 +444,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/009-agent-uses-layout-and-color-tokens:
     dependencies:
@@ -469,10 +472,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/010-agent-uses-typography-tokens:
     dependencies:
@@ -497,10 +500,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/011-agent-uses-motion-tokens:
     dependencies:
@@ -525,10 +528,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/012-agent-infers-status-tokens:
     dependencies:
@@ -553,10 +556,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/013-agent-infers-compact-control-tokens:
     dependencies:
@@ -581,10 +584,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/014-agent-replaces-custom-icons-with-octicons:
     dependencies:
@@ -609,10 +612,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/015-agent-infers-copy-icon:
     dependencies:
@@ -637,10 +640,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/016-agent-uses-loading-and-empty-state-patterns:
     dependencies:
@@ -665,10 +668,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/017-agent-uses-confirmation-pattern:
     dependencies:
@@ -693,10 +696,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/018-agent-uses-filter-pattern:
     dependencies:
@@ -721,10 +724,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/019-agent-uses-dismissal-utilities:
     dependencies:
@@ -749,10 +752,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/020-agent-uses-resize-observer-utility:
     dependencies:
@@ -777,10 +780,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/021-agent-sets-up-primer-in-vite:
     dependencies:
@@ -805,10 +808,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/022-agent-enables-automatic-theming:
     dependencies:
@@ -833,10 +836,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/023-agent-adds-theme-switcher:
     dependencies:
@@ -861,10 +864,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/024-agent-sets-up-tailwindcss:
     dependencies:
@@ -889,10 +892,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/025-agent-uses-tokens-with-tailwindcss:
     dependencies:
@@ -908,7 +911,7 @@ importers:
         version: link:../../packages/agent-eval
       '@tailwindcss/vite':
         specifier: ^4.3.3
-        version: 4.3.3(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.3.3(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
       '@types/react':
         specifier: ^19
         version: 19.2.18
@@ -923,10 +926,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/026-agent-avoids-deprecated-notification:
     dependencies:
@@ -951,10 +954,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.10(@types/node@26.1.2)(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))
+        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
 
   website:
     dependencies:
@@ -1397,6 +1400,126 @@ packages:
   '@js-sdsl/ordered-map@4.4.2':
     resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==}
 
+  '@jsonjoy.com/base64@1.1.2':
+    resolution: {integrity: sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/base64@17.67.0':
+    resolution: {integrity: sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/buffers@1.2.1':
+    resolution: {integrity: sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/buffers@17.67.0':
+    resolution: {integrity: sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/codegen@1.0.0':
+    resolution: {integrity: sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/codegen@17.67.0':
+    resolution: {integrity: sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-core@4.68.1':
+    resolution: {integrity: sha512-V5oZ4Gt9WJKyQef0n9cAd0N9qjSkIBm3E4MYsgNIWBk5aINCDPKxMPo1i29rBxqiT4Ixf1epklqV9VJMKIxwlw==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-fsa@4.68.1':
+    resolution: {integrity: sha512-HCG72UioncuO7Gw09XNVG+S85e3cq2hrUC/mexBrsWsa3mI7eePkkqWie3uVYbtsb64OR9YGQs5SqaufDRYBcg==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-node-builtins@4.68.1':
+    resolution: {integrity: sha512-HK1BTksysokNZxNspqDH0yPaqN9YgR/AYIlYiIaU2Ys4BOk5CdybI7r6BgiZuiiPiV8n4sK/kZdice7Znpy2Kw==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-node-to-fsa@4.68.1':
+    resolution: {integrity: sha512-lpKmU4X9e/oh8GIuAI7EXaS5QiLNM3KD15CkdhfS6PYmrGvoJqKQcyEfnLgnnaGslh/PFUMYSIZBCf2ejJGw8g==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-node-utils@4.68.1':
+    resolution: {integrity: sha512-/GxfW1DWm9SCdkfbvqevLO/P5duobQfmKkHXxdMIDbcZMQeAgooAstIfZhkXpATzq9QbCQsnoWFM/dGHdZfndw==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-node@4.68.1':
+    resolution: {integrity: sha512-R5D9mWtqdURzcOWj1vdXr3APCwX0xchtFT+kmW7fXLNDifWdDrnh26jSID8pdnUfFBxTyfHtFtTL/NWKzIH7kQ==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-print@4.68.1':
+    resolution: {integrity: sha512-oGeZOGPYKK9v1CgeVeEDsLomH1lCnslSpqUN5GmPzrmAVGQlsmsdcXNA2O4lV8Y4xkuSuynx2ITBkUHJVaTbow==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/fs-snapshot@4.68.1':
+    resolution: {integrity: sha512-XZfP0FDZN32bbc4t2bZN2qRrYHg5AktJnzk22HRoKGK4BprrbNRH2k5ceSNS/kupKYcofCs+O841+xaAbjnxwQ==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/json-pack@1.21.0':
+    resolution: {integrity: sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/json-pack@17.67.0':
+    resolution: {integrity: sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/json-pointer@1.0.2':
+    resolution: {integrity: sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/json-pointer@17.67.0':
+    resolution: {integrity: sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/util@1.9.0':
+    resolution: {integrity: sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
+  '@jsonjoy.com/util@17.67.0':
+    resolution: {integrity: sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
   '@lit-labs/react@1.2.1':
     resolution: {integrity: sha512-DiZdJYFU0tBbdQkfwwRSwYyI/mcWkg3sWesKRsHUd4G+NekTmmeq9fzsurvcKTNVa0comNljwtg4Hvi1ds3V+A==}
 
@@ -2771,6 +2894,12 @@ packages:
     resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==}
     engines: {node: '>=10.13.0'}
 
+  glob-to-regex.js@1.2.0:
+    resolution: {integrity: sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
   glob@13.0.6:
     resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==}
     engines: {node: 18 || 20 || >=22}
@@ -2827,6 +2956,10 @@ packages:
     resolution: {integrity: sha512-K3GbkIWqyvvlpfhBPlbEvD97TtqBpAYA4kt+cn2lD2x2HuohzZCibcA2nOlnJT6exqvJLggoB5nv2dNf192nEA==}
     hasBin: true
 
+  hyperdyperid@1.2.0:
+    resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==}
+    engines: {node: '>=10.18'}
+
   ieee754@1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
 
@@ -3198,6 +3331,9 @@ packages:
     resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
     engines: {node: '>= 0.4'}
 
+  memfs@4.68.1:
+    resolution: {integrity: sha512-OD+IDRUvIxu3QHL+nFm9gdyugInD27FDJ+sl4B5QgomPHXMlbw+GP918P8VNKu2FkNlVeqBkpzkwROpamVifRw==}
+
   minimatch@10.2.6:
     resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
     engines: {node: 18 || 20 || >=22}
@@ -3690,6 +3826,12 @@ packages:
   text-decoder@1.2.7:
     resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==}
 
+  thingies@2.6.1:
+    resolution: {integrity: sha512-cV/CMGTK3M4MlnJ/0At6ismOw/A0EEniDNScajjz/Br3c1sqE72YD01rGpPTKwd27wAxI5Pr+6+0w8yofzFRYw==}
+    engines: {node: '>=10.18'}
+    peerDependencies:
+      tslib: ^2
+
   tinybench@2.9.0:
     resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
 
@@ -3705,6 +3847,12 @@ packages:
     resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==}
     engines: {node: '>=14.0.0'}
 
+  tree-dump@1.1.0:
+    resolution: {integrity: sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==}
+    engines: {node: '>=10.0'}
+    peerDependencies:
+      tslib: '2'
+
   ts-api-utils@2.5.0:
     resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==}
     engines: {node: '>=18.12'}
@@ -4299,6 +4447,134 @@ snapshots:
 
   '@js-sdsl/ordered-map@4.4.2': {}
 
+  '@jsonjoy.com/base64@1.1.2(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/base64@17.67.0(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/buffers@1.2.1(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/buffers@17.67.0(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/codegen@1.0.0(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/codegen@17.67.0(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-core@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      thingies: 2.6.1(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-fsa@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      thingies: 2.6.1(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-node-builtins@4.68.1(tslib@2.8.1)':
+    dependencies:
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-node-to-fsa@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-fsa': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-node-utils@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      glob-to-regex.js: 1.2.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-node@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-print': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-snapshot': 4.68.1(tslib@2.8.1)
+      glob-to-regex.js: 1.2.0(tslib@2.8.1)
+      thingies: 2.6.1(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-print@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      tree-dump: 1.1.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/fs-snapshot@4.68.1(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/json-pack': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/util': 17.67.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/json-pack@1.21.0(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/base64': 1.1.2(tslib@2.8.1)
+      '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1)
+      '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1)
+      '@jsonjoy.com/json-pointer': 1.0.2(tslib@2.8.1)
+      '@jsonjoy.com/util': 1.9.0(tslib@2.8.1)
+      hyperdyperid: 1.2.0
+      thingies: 2.6.1(tslib@2.8.1)
+      tree-dump: 1.1.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/json-pack@17.67.0(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/base64': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/json-pointer': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/util': 17.67.0(tslib@2.8.1)
+      hyperdyperid: 1.2.0
+      thingies: 2.6.1(tslib@2.8.1)
+      tree-dump: 1.1.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/json-pointer@1.0.2(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1)
+      '@jsonjoy.com/util': 1.9.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/json-pointer@17.67.0(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/util': 17.67.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/util@1.9.0(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1)
+      '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1)
+      tslib: 2.8.1
+
+  '@jsonjoy.com/util@17.67.0(tslib@2.8.1)':
+    dependencies:
+      '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1)
+      '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1)
+      tslib: 2.8.1
+
   '@lit-labs/react@1.2.1': {}
 
   '@lit-labs/ssr-dom-shim@1.6.0': {}
@@ -4547,12 +4823,12 @@ snapshots:
       postcss: 8.5.26
       tailwindcss: 4.3.3
 
-  '@tailwindcss/vite@4.3.3(vite@8.2.1(@types/node@26.1.2)(jiti@2.7.0))':
+  '@tailwindcss/vite@4.3.3(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))':
     dependencies:
       '@tailwindcss/node': 4.3.3
       '@tailwindcss/oxide': 4.3.3
       tailwindcss: 4.3.3
-      vite: 8.2.1(@types/node@26.1.2)(jiti@2.7.0)
+      vite: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
 
   '@tanstack/react-virtual@3.14.6(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
     dependencies:
@@ -5691,6 +5967,10 @@ snapshots:
     dependencies:
       is-glob: 4.0.3
 
+  glob-to-regex.js@1.2.0(tslib@2.8.1):
+    dependencies:
+      tslib: 2.8.1
+
   glob@13.0.6:
     dependencies:
       minimatch: 10.2.6
@@ -5738,6 +6018,8 @@ snapshots:
 
   human-id@4.2.0: {}
 
+  hyperdyperid@1.2.0: {}
+
   ieee754@1.2.1: {}
 
   ignore@5.3.2: {}
@@ -6048,6 +6330,23 @@ snapshots:
 
   math-intrinsics@1.1.0: {}
 
+  memfs@4.68.1:
+    dependencies:
+      '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-fsa': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-to-fsa': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-print': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/fs-snapshot': 4.68.1(tslib@2.8.1)
+      '@jsonjoy.com/json-pack': 1.21.0(tslib@2.8.1)
+      '@jsonjoy.com/util': 1.9.0(tslib@2.8.1)
+      glob-to-regex.js: 1.2.0(tslib@2.8.1)
+      thingies: 2.6.1(tslib@2.8.1)
+      tree-dump: 1.1.0(tslib@2.8.1)
+      tslib: 2.8.1
+
   minimatch@10.2.6:
     dependencies:
       brace-expansion: 5.0.9
@@ -6640,6 +6939,10 @@ snapshots:
     transitivePeerDependencies:
       - react-native-b4a
 
+  thingies@2.6.1(tslib@2.8.1):
+    dependencies:
+      tslib: 2.8.1
+
   tinybench@2.9.0: {}
 
   tinyexec@1.3.0: {}
@@ -6651,6 +6954,10 @@ snapshots:
 
   tinyrainbow@3.1.0: {}
 
+  tree-dump@1.1.0(tslib@2.8.1):
+    dependencies:
+      tslib: 2.8.1
+
   ts-api-utils@2.5.0(typescript@6.0.3):
     dependencies:
       typescript: 6.0.3

From ab2f28e58c426ba85d81d6fda6452d092de86883 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 27 Aug 2026 16:09:09 -0500
Subject: [PATCH 08/52] feat: add benchmark functionality

---
 packages/agent-eval/src/benchmark.test.ts | 131 +++++++++++++++++++++-
 packages/agent-eval/src/benchmark.ts      | 114 ++++++++++---------
 packages/agent-eval/src/model.ts          |   2 +-
 3 files changed, 191 insertions(+), 56 deletions(-)

diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 9c10bd0a..3d511eae 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -1,6 +1,133 @@
 import {test, expect} from 'vitest'
 import {defineConfig, listBenchmarks, getBenchmark} from './benchmark'
+import {VirtualHost} from './host'
 
-test('hello', () => {
-  //
+test('listBenchmarks', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      name: 'test',
+      description: 'test',
+      models: [],
+      capabilities: [],
+    }),
+  )
+  const host = VirtualHost.create({
+    '/benchmarks': {
+      '01-benchmark.ts': `export const benchmark = ${config}`,
+      '02-default-export.ts': `export default ${config}`,
+      // '03-invalid-config.ts': `export const benchmark = {}`,
+      '03-no-config.ts': ``,
+    },
+  })
+  const benchmarks = await listBenchmarks(host, '/benchmarks')
+
+  expect(benchmarks).toHaveLength(2)
+  expect(benchmarks).toContainEqual({
+    id: '01-benchmark',
+    name: 'test',
+    description: 'test',
+    models: [],
+    capabilities: [],
+  })
+  expect(benchmarks).toContainEqual({
+    id: '02-default-export',
+    name: 'test',
+    description: 'test',
+    models: [],
+    capabilities: [],
+  })
+
+  expect(benchmarks).not.toContainEqual(
+    expect.objectContaining({
+      id: '03-no-config',
+    }),
+  )
+})
+
+test('listBenchmarks throw error on invalid config', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      name: 'test',
+      description: 'test',
+      models: [],
+      capabilities: [],
+    }),
+  )
+  const host = VirtualHost.create({
+    '/benchmarks': {
+      '01-benchmark.ts': `export const benchmark = ${config}`,
+      '03-invalid-config.ts': `export const benchmark = {}`,
+    },
+  })
+
+  await expect(listBenchmarks(host, '/benchmarks')).rejects.toThrowErrorMatchingInlineSnapshot(`
+    [Error: Benchmark file must export a valid benchmark config: /benchmarks/03-invalid-config.ts
+    ✖ Invalid input
+      → at name
+    ✖ Invalid input
+      → at description
+    ✖ Invalid input
+      → at models
+    ✖ Invalid input
+      → at capabilities]
+  `)
+})
+
+test('getBenchmark', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      name: 'test',
+      description: 'test',
+      models: [],
+      capabilities: [],
+    }),
+  )
+  const host = VirtualHost.create({
+    '/benchmarks': {
+      '01-benchmark.ts': `export const benchmark = ${config}`,
+      '02-benchmark.ts': `export const benchmark = ${config}`,
+    },
+  })
+  const benchmark = await getBenchmark(host, '/benchmarks', '01-benchmark')
+
+  expect(benchmark).toEqual({
+    id: '01-benchmark',
+    name: 'test',
+    description: 'test',
+    models: [],
+    capabilities: [],
+  })
+
+  await expect(getBenchmark(host, '/benchmarks', 'non-existent')).rejects.toThrowErrorMatchingInlineSnapshot(
+    `[Error: Benchmark "non-existent" was not found in: /benchmarks]`,
+  )
+})
+
+test('getBenchmark throw error on invalid config', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      name: 'test',
+      description: 'test',
+      models: [],
+      capabilities: [],
+    }),
+  )
+  const host = VirtualHost.create({
+    '/benchmarks': {
+      '01-benchmark.ts': `export const benchmark = ${config}`,
+      '02-invalid-config.ts': `export const benchmark = {}`,
+    },
+  })
+
+  await expect(getBenchmark(host, '/benchmarks', '02-invalid-config')).rejects.toThrowErrorMatchingInlineSnapshot(`
+    [Error: Benchmark file must export a valid benchmark config: /benchmarks/02-invalid-config.ts
+    ✖ Invalid input
+      → at name
+    ✖ Invalid input
+      → at description
+    ✖ Invalid input
+      → at models
+    ✖ Invalid input
+      → at capabilities]
+  `)
 })
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 080cf0b7..d91545a2 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -1,34 +1,25 @@
-import fs from 'node:fs/promises'
+import {randomUUID} from 'node:crypto'
 import path from 'node:path'
-// import * as z from 'zod/mini'
-import type {Model, ModelConfig, ReasoningEffort} from './model'
+import * as z from 'zod/mini'
+import {ModelVariantConfigSchema, type Model, type ReasoningEffort} from './model'
 import type {Sandbox} from './sandbox'
-import {ControlTreatment, type TreatmentConfig} from './experiment-config'
-import {getScenario} from './scenario'
+import {ControlTreatment} from './experiment-config'
 import type {Scenario} from './scenario'
-import {randomUUID} from 'node:crypto'
-
-// const BenchmarkConfigSchema = z.object({
-//   name: z.string(),
-//   description: z.string(),
-//   capabilities: z.array(
-//     z.object({
-//       name: z.string(),
-//       scenarios: z.array(z.string()),
-//     }),
-//   ),
-// })
-
-type BenchmarkConfig = {
-  name: string
-  description: string
-  models: Array<ModelConfig>
-  setup: Setup
-  capabilities: Array<{
-    name: string
-    scenarios: Array<string>
-  }>
-}
+import type {Host} from './host'
+
+const BenchmarkConfigSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  models: ModelVariantConfigSchema,
+  capabilities: z.array(
+    z.object({
+      name: z.string(),
+      scenarios: z.array(z.string()),
+    }),
+  ),
+})
+
+type BenchmarkConfig = z.infer<typeof BenchmarkConfigSchema>
 
 type Setup = ({sandbox}: {sandbox: Sandbox}) => Promise<void>
 
@@ -37,48 +28,65 @@ function defineConfig(config: BenchmarkConfig): BenchmarkConfig {
 }
 
 type BenchmarkModule = {
-  benchmark: BenchmarkConfig
+  benchmark?: BenchmarkConfig
+  default?: BenchmarkConfig
+}
+
+type Benchmark = BenchmarkConfig & {
+  id: string
 }
 
 const BENCHMARK_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
 
-async function listBenchmarks(directory: string) {
-  const stats = await fs.stat(directory)
+async function listBenchmarks(host: Host, directory: string): Promise<Array<Benchmark>> {
+  const stats = await host.fs.stat(directory)
   if (!stats.isDirectory()) {
     throw new Error('Expected benchmarks to be a directory')
   }
 
-  const filenames = await fs.readdir(directory)
-  const benchmarks = await Promise.all(
-    filenames
-      .filter(filename => {
-        return isBenchmarkFile(filename)
-      })
-      .map(async filename => {
-        const filepath = path.join(directory, filename)
-        const mod: BenchmarkModule = await import(filepath)
-        if (!mod.benchmark) {
-          throw new Error(`Benchmark file must export "benchmark": ${filepath}`)
-        }
-        return [getBenchmarkId(filename), mod.benchmark] as const
-      }),
-  )
+  const filenames = await host.fs.readdir(directory)
+  const benchmarks: Array<Benchmark> = []
+
+  for (const filename of filenames) {
+    if (!isBenchmarkFile(filename)) {
+      continue
+    }
+
+    const filepath = path.join(directory, filename)
+    const mod: BenchmarkModule = await host.loadModule(filepath)
+    const data = mod.benchmark ?? mod.default
+    if (!data) {
+      continue
+    }
+
+    const parseResult = BenchmarkConfigSchema.safeParse(data)
+    if (!parseResult.success) {
+      throw new Error(
+        `Benchmark file must export a valid benchmark config: ${filepath}\n${z.prettifyError(parseResult.error)}`,
+      )
+    }
+
+    benchmarks.push({
+      id: getBenchmarkId(filename),
+      ...parseResult.data,
+    })
+  }
 
   return benchmarks
 }
 
-async function getBenchmark(directory: string, id: string): Promise<BenchmarkConfig> {
-  const benchmarks = await listBenchmarks(directory)
-  const benchmark = benchmarks.find(([benchmarkId]) => benchmarkId === id)
+async function getBenchmark(host: Host, directory: string, id: string): Promise<Benchmark> {
+  const benchmarks = await listBenchmarks(host, directory)
+  const benchmark = benchmarks.find(benchmark => benchmark.id === id)
   if (!benchmark) {
     throw new Error(`Benchmark "${id}" was not found in: ${directory}`)
   }
 
-  return benchmark[1]
+  return benchmark
 }
 
 function getBenchmarkId(filename: string): string {
-  return path.basename(filename)
+  return path.basename(filename, path.extname(filename))
 }
 
 function isBenchmarkFile(filename: string): boolean {
@@ -99,7 +107,7 @@ function isBenchmarkFile(filename: string): boolean {
 
 type BenchmarkResult = {}
 
-type Treatment = {
+type Treatment<M extends Model> = {
   id: string
   scenario: Scenario
   config: {
@@ -107,7 +115,7 @@ type Treatment = {
     setup?: Setup
   }
   model: Model
-  reasoningEffort: ReasoningEffort | null
+  reasoningEffort: ReasoningEffort<M> | null
 }
 
 type RunOptions = {
diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts
index 9c4c53c8..5a2bc0c7 100644
--- a/packages/agent-eval/src/model.ts
+++ b/packages/agent-eval/src/model.ts
@@ -152,5 +152,5 @@ function getModelVariants(input: ModelVariantConfig): Array<ModelVariant<Model>>
   })
 }
 
-export {models, getModelVariants}
+export {models, getModelVariants, ModelVariantConfigSchema}
 export type {ModelConfig, Model, ReasoningEffort, ModelVariantConfig}

From 0a0c7fa72c6f73bfb9131f333fa030a0e4dced63 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 28 Aug 2026 12:17:40 -0500
Subject: [PATCH 09/52] feat: add trial concept for individual treatment runs

---
 packages/agent-eval/src/trial.ts | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 packages/agent-eval/src/trial.ts

diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
new file mode 100644
index 00000000..4c991497
--- /dev/null
+++ b/packages/agent-eval/src/trial.ts
@@ -0,0 +1,17 @@
+import * as z from 'zod/mini'
+import {ModelVariantSchema} from './model'
+import {ScenarioConfigSchema, ScenarioSchema} from './scenario'
+import {TreatmentSchema, TreatmentSetupSchema} from './treatment'
+
+const TrialSchema = z.object({
+  id: z.string(),
+  scenario: ScenarioSchema,
+  treatment: TreatmentSchema,
+  model: ModelVariantSchema,
+  setup: z.optional(TreatmentSetupSchema),
+})
+
+type Trial = z.infer<typeof TrialSchema>
+
+export {TrialSchema}
+export type {Trial}

From f1a7edaf3bb8acebf9dbd8389fc74d6948ca3847 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 28 Aug 2026 12:18:14 -0500
Subject: [PATCH 10/52] feat: add plan concept for ordering trials and storing
 order across shards

---
 packages/agent-eval/src/plan.ts | 34 +++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 packages/agent-eval/src/plan.ts

diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
new file mode 100644
index 00000000..d6a4c3a2
--- /dev/null
+++ b/packages/agent-eval/src/plan.ts
@@ -0,0 +1,34 @@
+import * as z from 'zod/mini'
+import {TrialSchema} from './trial'
+import type {Trial} from './trial'
+
+const PlanSchema = z.object({
+  trials: z.array(TrialSchema),
+})
+
+/**
+ * A plan is an ordered list of trials to be ran.
+ */
+type Plan = z.infer<typeof PlanSchema>
+
+// TODO: support plan with sharding
+async function plan(trials: Array<Trial>): Promise<Plan> {
+  return {
+    trials: randomize(trials),
+  }
+}
+
+function randomize<T>(input: Array<T>): Array<T> {
+  const randomized: Array<T> = input.slice()
+
+  // Fisher–Yates shuffle
+  for (let i = randomized.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1))
+    ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]]
+  }
+
+  return randomized
+}
+
+export {plan}
+export type {Plan}

From 249e75513fdd640ed4ced79d279eb417071a7ecc Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 28 Aug 2026 12:18:31 -0500
Subject: [PATCH 11/52] refactor: update benchmarks, scenario, treatment, and
 run

---
 packages/agent-eval/package.json          |   3 +
 packages/agent-eval/src/benchmark.test.ts |  47 +-
 packages/agent-eval/src/benchmark.ts      | 110 +++--
 packages/agent-eval/src/logger.ts         |  25 +
 packages/agent-eval/src/model.ts          |  36 +-
 packages/agent-eval/src/run-old.test.ts   |  55 +++
 packages/agent-eval/src/run-old.ts        | 486 +++++++++++++++++++
 packages/agent-eval/src/run.test.ts       | 121 +++--
 packages/agent-eval/src/run.ts            | 565 ++++++----------------
 packages/agent-eval/src/sandbox.ts        |   5 +-
 packages/agent-eval/src/scenario.ts       |  22 +-
 packages/agent-eval/src/scenarios.test.ts | 153 ------
 packages/agent-eval/src/scenarios.ts      | 128 -----
 packages/agent-eval/src/treatment-old.ts  |  60 +++
 packages/agent-eval/src/treatment.ts      |  75 +--
 pnpm-lock.yaml                            | 320 +++++++++---
 16 files changed, 1276 insertions(+), 935 deletions(-)
 create mode 100644 packages/agent-eval/src/logger.ts
 create mode 100644 packages/agent-eval/src/run-old.test.ts
 create mode 100644 packages/agent-eval/src/run-old.ts
 delete mode 100644 packages/agent-eval/src/scenarios.test.ts
 delete mode 100644 packages/agent-eval/src/scenarios.ts
 create mode 100644 packages/agent-eval/src/treatment-old.ts

diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index d175accf..2a3e2803 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -63,6 +63,9 @@
   },
   "dependencies": {
     "dockerode": "^5.0.1",
+    "p-queue": "^9.3.3",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3",
     "tar-fs": "^3.1.2",
     "tar-stream": "^3.2.0",
     "zod": "^4.4.3"
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 3d511eae..dfdfb50e 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -1,6 +1,7 @@
 import {test, expect} from 'vitest'
-import {defineConfig, listBenchmarks, getBenchmark} from './benchmark'
+import {defineConfig, listBenchmarks, getBenchmark, run} from './benchmark'
 import {VirtualHost} from './host'
+import {defineConfig as defineScenarioConfig} from './scenario'
 
 test('listBenchmarks', async () => {
   const config = JSON.stringify(
@@ -131,3 +132,47 @@ test('getBenchmark throw error on invalid config', async () => {
       → at capabilities]
   `)
 })
+
+test('run', async () => {
+  const scenario = JSON.stringify(
+    defineScenarioConfig({
+      prompt: 'test',
+    }),
+  )
+  const host = VirtualHost.create({
+    '/artifacts': {},
+    '/benchmarks': {},
+    '/scenarios': {
+      '001-scenario': {
+        'scenario.config.ts': `export default ${scenario}`,
+        'scenario.test.ts': '',
+        'package.json': JSON.stringify({}),
+      },
+      '002-scenario': {
+        'scenario.config.ts': `export default ${scenario}`,
+        'scenario.test.ts': '',
+        'package.json': JSON.stringify({}),
+      },
+    },
+  })
+  await run(
+    {
+      artifactsDirectory: '/artifacts',
+      benchmarksDirectory: '/benchmarks',
+      host,
+      scenariosDirectory: '/scenarios',
+    },
+    {
+      id: 'test',
+      name: 'test',
+      description: 'test',
+      models: ['gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.6-sol'],
+      capabilities: [
+        {
+          name: 'test',
+          scenarios: ['001-scenario', '002-scenario'],
+        },
+      ],
+    },
+  )
+})
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index d91545a2..5d82df37 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -1,27 +1,28 @@
 import {randomUUID} from 'node:crypto'
 import path from 'node:path'
 import * as z from 'zod/mini'
-import {ModelVariantConfigSchema, type Model, type ReasoningEffort} from './model'
-import type {Sandbox} from './sandbox'
-import {ControlTreatment} from './experiment-config'
-import type {Scenario} from './scenario'
+import {getModelVariants, ModelVariantConfigSchema, type Model, type ModelVariant, type ReasoningEffort} from './model'
+import {getScenario, type Scenario} from './scenario'
 import type {Host} from './host'
+import {ControlTreatment, TreatmentSetupSchema} from './treatment'
+import type {Treatment} from './treatment'
+
+const CapabilityConfigSchema = z.object({
+  name: z.string(),
+  scenarios: z.array(z.string()),
+})
 
 const BenchmarkConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
   models: ModelVariantConfigSchema,
-  capabilities: z.array(
-    z.object({
-      name: z.string(),
-      scenarios: z.array(z.string()),
-    }),
-  ),
+  setup: z.optional(TreatmentSetupSchema),
+  capabilities: z.array(CapabilityConfigSchema),
 })
 
 type BenchmarkConfig = z.infer<typeof BenchmarkConfigSchema>
 
-type Setup = ({sandbox}: {sandbox: Sandbox}) => Promise<void>
+type CapabilityConfig = z.infer<typeof CapabilityConfigSchema>
 
 function defineConfig(config: BenchmarkConfig): BenchmarkConfig {
   return config
@@ -105,53 +106,58 @@ function isBenchmarkFile(filename: string): boolean {
   return true
 }
 
-type BenchmarkResult = {}
-
-type Treatment<M extends Model> = {
-  id: string
-  scenario: Scenario
-  config: {
-    name: string
-    setup?: Setup
-  }
-  model: Model
-  reasoningEffort: ReasoningEffort<M> | null
-}
-
-type RunOptions = {
+type RunContext = {
   artifactsDirectory: string
   benchmarksDirectory: string
+  host: Host
   scenariosDirectory: string
 }
 
-async function run(config: BenchmarkConfig, options: RunOptions): Promise<BenchmarkResult> {
-  const treatments = config.capabilities.flatMap(capability => {
-    return config.models.flatMap(model => {
-      return capability.scenarios.flatMap(scenario => {
-        return [
-          {
-            id: randomUUID(),
-            config: ControlTreatment,
-            scenario,
-            setup: config.setup,
-            model: model.name,
-            reasoningEffort: model.reasoningEfforts.length > 0 ? model.reasoningEfforts[0] : null,
-          },
-          {
-            id: randomUUID(),
-            scenario,
-            config: {
-              name: 'Benchmark',
-              setup: config.setup,
-            },
-            model: model.name,
-            reasoningEffort: model.reasoningEfforts.length > 0 ? model.reasoningEfforts[0] : null,
+type Trial = {
+  id: string
+  capability: CapabilityConfig
+  scenario: Scenario
+  treatment: Treatment
+  model: ModelVariant
+}
+
+type TrialRun = {}
+
+type TrialResult = {}
+
+async function run(context: RunContext, benchmark: Benchmark): Promise<TrialResult> {
+  const trials: Array<Trial> = []
+
+  for (const variant of getModelVariants(benchmark.models)) {
+    for (const capability of benchmark.capabilities) {
+      for (const scenarioId of capability.scenarios) {
+        const scenario = await getScenario(context.host, context.scenariosDirectory, scenarioId)
+
+        trials.push({
+          id: randomUUID(),
+          capability,
+          scenario,
+          treatment: ControlTreatment,
+          model: variant,
+        })
+
+        trials.push({
+          id: randomUUID(),
+          capability,
+          scenario,
+          treatment: {
+            name: 'Benchmark',
+            setup: benchmark.setup,
           },
-        ]
-      })
-    })
-  })
-  //
+          model: variant,
+        })
+      }
+    }
+  }
+
+  console.log(trials)
+
+  throw new Error('unimplemented')
 }
 
 export {defineConfig, listBenchmarks, getBenchmark, run}
diff --git a/packages/agent-eval/src/logger.ts b/packages/agent-eval/src/logger.ts
new file mode 100644
index 00000000..92c572d1
--- /dev/null
+++ b/packages/agent-eval/src/logger.ts
@@ -0,0 +1,25 @@
+import process from 'node:process'
+import pino from 'pino'
+import pretty from 'pino-pretty'
+
+const CI = process.env.CI === 'true' || process.env.CI === '1' || process.env.GITHUB_ACTIONS === 'true'
+
+const stream = []
+
+if (!CI) {
+  stream.push(
+    pretty({
+      colorize: true,
+    }),
+  )
+}
+
+export const logger = pino(
+  {
+    base: undefined,
+    level: 'info',
+    timestamp: false,
+    // enabled: process.env.NODE_ENV !== 'test',
+  },
+  ...stream,
+)
diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts
index 5a2bc0c7..8fd6200e 100644
--- a/packages/agent-eval/src/model.ts
+++ b/packages/agent-eval/src/model.ts
@@ -59,14 +59,38 @@ const models = [
   },
 ] as const
 
+const reasoningEffortsByModel = new Map<string, ReadonlySet<string>>(
+  models.map(model => {
+    return [model.name, new Set<string>(model.reasoningEfforts)]
+  }),
+)
+
 type ModelConfig = (typeof models)[number]
 type Model = ModelConfig['name']
 type ReasoningEfforts<M extends Model> = Extract<ModelConfig, {name: M}>['reasoningEfforts']
 type ReasoningEffort<M extends Model> = ReasoningEfforts<M>[number]
-type ModelVariant<M extends Model> = {
-  name: M
-  reasoningEffort: ReasoningEffort<M>
-}
+type ModelVariant<M extends Model = Model> = M extends Model
+  ? {
+      name: M
+      reasoningEffort: ReasoningEffort<M>
+    }
+  : never
+
+const ModelVariantSchema = z.custom<ModelVariant>(value => {
+  if (typeof value !== 'object' || value === null) {
+    return false
+  }
+
+  if (!('name' in value) || typeof value.name !== 'string') {
+    return false
+  }
+
+  if (!('reasoningEffort' in value) || typeof value.reasoningEffort !== 'string') {
+    return false
+  }
+
+  return reasoningEffortsByModel.get(value.name)?.has(value.reasoningEffort) ?? false
+})
 
 const ModelVariantConfigSchema = z.array(
   z.union([
@@ -152,5 +176,5 @@ function getModelVariants(input: ModelVariantConfig): Array<ModelVariant<Model>>
   })
 }
 
-export {models, getModelVariants, ModelVariantConfigSchema}
-export type {ModelConfig, Model, ReasoningEffort, ModelVariantConfig}
+export {models, getModelVariants, ModelVariantSchema, ModelVariantConfigSchema}
+export type {ModelConfig, Model, ModelVariant, ReasoningEffort, ModelVariantConfig}
diff --git a/packages/agent-eval/src/run-old.test.ts b/packages/agent-eval/src/run-old.test.ts
new file mode 100644
index 00000000..fb9abf6a
--- /dev/null
+++ b/packages/agent-eval/src/run-old.test.ts
@@ -0,0 +1,55 @@
+import {describe, expect, test} from 'vitest'
+import {getCopilotArgs, getVitestConfig} from './run'
+
+describe('getCopilotArgs', () => {
+  test('omits reasoning effort when not configured', () => {
+    expect(
+      getCopilotArgs({
+        prompt: 'Update the page',
+        model: 'claude-haiku-4.5',
+      }),
+    ).not.toContain('--reasoning-effort')
+  })
+
+  test('forwards the model and reasoning effort', () => {
+    expect(
+      getCopilotArgs({
+        prompt: 'Update the page',
+        model: 'gpt-5.5',
+        reasoningEffort: 'medium',
+      }),
+    ).toEqual([
+      '-p',
+      'Update the page',
+      '--model',
+      'gpt-5.5',
+      '--allow-all',
+      '--reasoning-effort',
+      'medium',
+      '--mode',
+      'autopilot',
+      '--output-format',
+      'json',
+    ])
+  })
+})
+
+describe('getVitestConfig', () => {
+  test('configures node tests by default', () => {
+    const config = getVitestConfig('test-results.json')
+
+    expect(config).toContain(`outputFile: "test-results.json"`)
+    expect(config).not.toContain('@vitest/browser-playwright')
+  })
+
+  test('configures browser tests with Playwright and Chromium', () => {
+    const config = getVitestConfig('browser-test-results.json', true)
+
+    expect(config).toContain(`import {playwright} from '@vitest/browser-playwright'`)
+    expect(config).toContain('enabled: true')
+    expect(config).toContain('headless: true')
+    expect(config).toContain('provider: playwright()')
+    expect(config).toContain(`instances: [{browser: 'chromium'}]`)
+    expect(config).toContain(`outputFile: "browser-test-results.json"`)
+  })
+})
diff --git a/packages/agent-eval/src/run-old.ts b/packages/agent-eval/src/run-old.ts
new file mode 100644
index 00000000..d43050f7
--- /dev/null
+++ b/packages/agent-eval/src/run-old.ts
@@ -0,0 +1,486 @@
+import {randomUUID} from 'node:crypto'
+import path from 'node:path'
+import fs from 'node:fs/promises'
+import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, Sandbox} from './sandbox'
+import type {Treatment, TreatmentResult, Walkthrough} from './treatment'
+import type {Model, ReasoningEffort} from './model'
+import {isMessageType, parseMessage, type Message} from './copilot-cli'
+import {getTestMetadata, parseTestResults} from './vitest'
+import {existsSync} from 'node:fs'
+
+const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright'
+const CHROMIUM_EXECUTABLE_PATH = '/usr/bin/chromium'
+const WALKTHROUGH_DIR = 'walkthrough'
+const WALKTHROUGH_VIEWPORT_WIDTH = 1440
+const WALKTHROUGH_VIEWPORT_HEIGHT = 900
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
+
+type RunOptions = {
+  artifactsDirectory: string
+  copilotToken: string
+  dockerImage?: string
+  maxConcurrency?: number
+}
+
+function getVitestConfig(outputFile: string, browser = false): string {
+  const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright'\n` : ''
+  const browserConfig = browser
+    ? `    browser: {
+      enabled: true,
+      headless: true,
+      provider: playwright(),
+      instances: [{browser: 'chromium'}],
+    },
+`
+    : ''
+
+  return `
+import {defineConfig} from 'vitest/config'
+${browserImport}
+export default defineConfig({
+  test: {
+${browserConfig}    reporters: [['json', {outputFile: ${JSON.stringify(outputFile)}, includeTaskLocation: true}]],
+  },
+})
+`
+}
+
+function run(treatments: Array<Treatment>, options: RunOptions): Promise<Array<TreatmentResult>> {
+  const maxConcurrency = options.maxConcurrency ?? 1
+  const queue = treatments.slice()
+  const results: Array<TreatmentResult> = []
+  const pending = new Set()
+  let cancelled = false
+
+  let resolve: (value: Array<TreatmentResult>) => void
+  let reject: (reason: unknown) => void
+  const deferred = new Promise<Array<TreatmentResult>>((_resolve, _reject) => {
+    resolve = _resolve
+    reject = _reject
+  })
+
+  function execute() {
+    if (cancelled) {
+      return
+    }
+
+    if (queue.length === 0) {
+      if (pending.size === 0) {
+        resolve(results)
+      }
+      return
+    }
+
+    if (pending.size >= maxConcurrency) {
+      return
+    }
+
+    const treatment = queue.shift()
+    if (!treatment) {
+      return
+    }
+
+    const promise = retry(
+      () =>
+        runTreatment(treatment, {
+          artifactsDirectory: options.artifactsDirectory,
+          copilotToken: options.copilotToken,
+          dockerImage: options.dockerImage,
+        }),
+      3,
+    ).then(
+      result => {
+        results.push(result)
+        pending.delete(promise)
+        execute()
+      },
+      error => {
+        cancelled = true
+        pending.delete(promise)
+        reject(error)
+      },
+    )
+
+    pending.add(promise)
+    execute()
+  }
+
+  execute()
+
+  return deferred
+}
+
+async function retry<T>(fn: () => Promise<T>, retries: number): Promise<T> {
+  try {
+    return await fn()
+  } catch (error) {
+    if (retries > 0) {
+      console.log('Retrying after error: %s', error)
+      return retry(fn, retries - 1)
+    }
+    throw error
+  }
+}
+
+type RunTreatmentOptions = {
+  artifactsDirectory: string
+  copilotToken: string
+  dockerImage?: string
+}
+
+function getCopilotArgs({
+  prompt,
+  model,
+  reasoningEffort,
+}: {
+  prompt: string
+  model: Model
+  reasoningEffort?: ReasoningEffort
+}): Array<string> {
+  const args = ['-p', prompt, '--model', model, '--allow-all']
+
+  if (reasoningEffort) {
+    args.push('--reasoning-effort', reasoningEffort)
+  }
+
+  return [...args, '--mode', 'autopilot', '--output-format', 'json']
+}
+
+async function runTreatment(
+  treatment: Treatment,
+  {artifactsDirectory, copilotToken, dockerImage}: RunTreatmentOptions,
+): Promise<TreatmentResult> {
+  console.log('Running treatment: %s (%s)', treatment.config.name, treatment.id)
+  await using sandbox = await Sandbox.create({dockerImage})
+
+  console.log('Copying files from: %s...', treatment.scenario.directory)
+  await sandbox.copy(treatment.scenario.directory, CONTAINER_WORKDIR, {
+    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
+  })
+  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
+    user: 'root',
+  })
+
+  console.log('Obfuscating package name...')
+  await sandbox.runCommand('npm', ['pkg', 'set', `name=${treatment.id}`], {
+    user: NODE_USER,
+  })
+
+  console.log('Removing workspace dependency...')
+  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
+    user: NODE_USER,
+  })
+
+  console.log('Installing dependencies...')
+  await sandbox.runCommand('npm', ['install'], {
+    user: NODE_USER,
+  })
+
+  if (treatment.experiment.setup) {
+    console.log('Running experiment setup...')
+    await treatment.experiment.setup({
+      sandbox,
+    })
+  }
+
+  if (treatment.config.setup) {
+    console.log('Running treatment setup...')
+    await treatment.config.setup({
+      sandbox,
+    })
+  }
+
+  console.log('Run build script...')
+  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
+    user: NODE_USER,
+  })
+
+  if (treatment.scenario.browserTestPath) {
+    console.log('Installing browser test dependencies...')
+    await sandbox.runCommand(
+      'npm',
+      ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'],
+      {
+        user: NODE_USER,
+      },
+    )
+    console.log('Installing Playwright browser...')
+    await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], {
+      user: 'root',
+      env: {
+        PLAYWRIGHT_BROWSERS_PATH,
+      },
+    })
+  }
+
+  console.log('Running copilot...')
+  const {prompt} = treatment.scenario.config
+  const args = getCopilotArgs({
+    prompt,
+    model: treatment.model,
+    reasoningEffort: treatment.reasoningEffort,
+  })
+  const copilotOutput = await sandbox.runCommand('copilot', args, {
+    user: NODE_USER,
+    env: {
+      COPILOT_GITHUB_TOKEN: copilotToken,
+    },
+  })
+  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
+    const trimmed = line.trim()
+    if (trimmed.length === 0) {
+      return []
+    }
+    return parseMessage(JSON.parse(trimmed))
+  })
+
+  const TEST_PATH = 'scenario.test.ts'
+  const BROWSER_TEST_PATH = 'scenario.browser.test.ts'
+  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
+  const TEST_RESULTS_PATH = 'test-results.json'
+  const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json'
+  const scenarioTests = [
+    {
+      sourcePath: treatment.scenario.testPath,
+      testPath: TEST_PATH,
+      resultsPath: TEST_RESULTS_PATH,
+      browser: false,
+    },
+  ]
+
+  if (treatment.scenario.browserTestPath) {
+    scenarioTests.push({
+      sourcePath: treatment.scenario.browserTestPath,
+      testPath: BROWSER_TEST_PATH,
+      resultsPath: BROWSER_TEST_RESULTS_PATH,
+      browser: true,
+    })
+  }
+
+  let numFailedTests = 0
+  let numPassedTests = 0
+  let numPendingTests = 0
+  let numTodoTests = 0
+  let numTotalTests = 0
+  let testRunSuccess = true
+  const tests: TreatmentResult['testResults']['tests'] = []
+  const rawTestResults: Array<Record<string, unknown> & {testResults: Array<unknown>}> = []
+
+  for (const scenarioTest of scenarioTests) {
+    await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath)
+    await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser))
+    // Always pass vitest calls even if test suite fails
+    await sandbox.runCommand(
+      'sh',
+      ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath],
+      {
+        user: NODE_USER,
+        env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined,
+      },
+    )
+
+    const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath)
+    const rawTestResult: unknown = JSON.parse(testResultsContent)
+    const testResults = parseTestResults(rawTestResult)
+    if (!testResults.success) {
+      throw new Error(`Failed to parse test results: ${testResults.error}`)
+    }
+
+    const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8')
+    numFailedTests += testResults.data.numFailedTests
+    numPassedTests += testResults.data.numPassedTests
+    numPendingTests += testResults.data.numPendingTests
+    numTodoTests += testResults.data.numTodoTests
+    numTotalTests += testResults.data.numTotalTests
+    testRunSuccess &&= testResults.data.success
+    tests.push(...getTestMetadata(testResults.data, testSource))
+    rawTestResults.push(rawTestResult as Record<string, unknown> & {testResults: Array<unknown>})
+  }
+
+  if (rawTestResults.length > 1) {
+    await sandbox.writeFile(
+      TEST_RESULTS_PATH,
+      JSON.stringify({
+        ...rawTestResults[0],
+        numFailedTests,
+        numPassedTests,
+        numPendingTests,
+        numTodoTests,
+        numTotalTests,
+        success: testRunSuccess,
+        testResults: rawTestResults.flatMap(testResult => testResult.testResults),
+      }),
+    )
+  }
+
+  console.log('Capturing walkthrough...')
+  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
+    user: 'root',
+  })
+  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
+    user: NODE_USER,
+  })
+  await sandbox.runCommand(
+    'npx',
+    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
+    {
+      user: NODE_USER,
+    },
+  )
+  await sandbox.writeFile(
+    'agent-browser.json',
+    JSON.stringify({
+      executablePath: CHROMIUM_EXECUTABLE_PATH,
+    }),
+  )
+  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
+
+Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
+
+Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
+
+- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
+- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
+- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
+
+Only capture the walkthrough, do not make any further code changes.`
+  const walkthroughResult = await sandbox.runCommand(
+    'copilot',
+    getCopilotArgs({
+      prompt: walkthroughPrompt,
+      model: 'gpt-5.6-terra',
+      reasoningEffort: 'medium',
+    }),
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: copilotToken,
+      },
+      allowNonZeroExitCode: true,
+    },
+  )
+
+  if (walkthroughResult.exitCode !== 0) {
+    console.warn('Unable to capture walkthrough: %s', walkthroughResult.stderr)
+  }
+
+  // Turns
+  const assistantTurns = new Set()
+  // Tools
+  const toolCalls = new Map()
+  let outputTokens = 0
+
+  for (const message of messages) {
+    if (isMessageType(message, 'assistant.turn_start')) {
+      assistantTurns.add(message.data.turnId)
+    }
+
+    if (isMessageType(message, 'assistant.message')) {
+      outputTokens += message.data.outputTokens ?? 0
+    }
+
+    if (isMessageType(message, 'tool.execution_start')) {
+      const toolName = message.data.toolName
+      toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1)
+    }
+  }
+
+  const result = messages.find(message => isMessageType(message, 'result'))
+  if (!result) {
+    throw new Error('No result message found in copilot output')
+  }
+
+  const artifactDirectory = path.join(artifactsDirectory, treatment.id)
+  const workspacePath = path.join(artifactDirectory, 'workspace')
+  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
+  const copilotConfigPath = path.join(artifactDirectory, '.copilot')
+  const skillsConfigPath = path.join(artifactDirectory, '.agents')
+  const testResultsPath = path.join(workspacePath, 'test-results.json')
+  await fs.mkdir(workspacePath, {recursive: true})
+
+  console.log('Downloading agent workspace to: %s...', workspacePath)
+  await sandbox.download(CONTAINER_WORKDIR, workspacePath, {
+    ignore(name) {
+      return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
+    },
+  })
+
+  console.log('Downloading copilot config to: %s...', copilotConfigPath)
+  await sandbox.download(COPILOT_DIR, copilotConfigPath)
+
+  console.log('Downloading skills config to: %s...', skillsConfigPath)
+  await sandbox.download(AGENTS_DIR, skillsConfigPath)
+
+  let walkthrough: Walkthrough = {
+    type: 'Unavailable',
+  }
+
+  if (existsSync(path.join(workspacePath, WALKTHROUGH_DIR))) {
+    console.log(
+      'Moving walkthrough artifacts from: %s to: %s...',
+      path.join(workspacePath, WALKTHROUGH_DIR),
+      walkthroughPath,
+    )
+    await fs.mkdir(walkthroughPath, {recursive: true})
+    await fs.rename(path.join(workspacePath, WALKTHROUGH_DIR), walkthroughPath)
+
+    if (existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
+      walkthrough = {
+        type: 'Screenshot',
+        filepath: path.join(walkthroughPath, 'screenshot.png'),
+      }
+    } else if (existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
+      walkthrough = {
+        type: 'Video',
+        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
+      }
+    } else if (existsSync(path.join(walkthroughPath, 'screenshots'))) {
+      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
+      const entries = await fs.readdir(screenshotsDir).then(filenames => {
+        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
+      })
+      const screenshots = entries.filter(entry => {
+        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
+      })
+      if (screenshots.length > 0) {
+        walkthrough = {
+          type: 'Screenshots',
+          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
+        }
+      }
+    }
+  }
+
+  return {
+    id: randomUUID(),
+    treatment,
+    artifacts: {
+      directory: artifactDirectory,
+      copilotConfigPath,
+      skillsConfigPath,
+      testResultsPath,
+      workspacePath,
+    },
+    assistant: {
+      logs: messages,
+      turns: assistantTurns.size,
+      outputTokens,
+      premiumRequests: result.usage.premiumRequests,
+      // Time to complete (latency)
+      totalApiDurationMs: result.usage.totalApiDurationMs,
+      sessionDurationMs: result.usage.sessionDurationMs,
+      tools: Object.fromEntries(toolCalls),
+    },
+    testResults: {
+      numFailedTests,
+      numPassedTests,
+      numPendingTests,
+      numTodoTests,
+      numTotalTests,
+      tests,
+    },
+    walkthrough,
+  }
+}
+
+export {getCopilotArgs, getVitestConfig, run}
diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts
index fb9abf6a..015057f1 100644
--- a/packages/agent-eval/src/run.test.ts
+++ b/packages/agent-eval/src/run.test.ts
@@ -1,55 +1,80 @@
-import {describe, expect, test} from 'vitest'
-import {getCopilotArgs, getVitestConfig} from './run'
-
-describe('getCopilotArgs', () => {
-  test('omits reasoning effort when not configured', () => {
-    expect(
-      getCopilotArgs({
-        prompt: 'Update the page',
-        model: 'claude-haiku-4.5',
-      }),
-    ).not.toContain('--reasoning-effort')
-  })
+import {test, expect, vi} from 'vitest'
+import {run, runTrial} from './run'
+import {plan} from './plan'
+import {VirtualHost} from './host'
 
-  test('forwards the model and reasoning effort', () => {
-    expect(
-      getCopilotArgs({
-        prompt: 'Update the page',
-        model: 'gpt-5.5',
-        reasoningEffort: 'medium',
-      }),
-    ).toEqual([
-      '-p',
-      'Update the page',
-      '--model',
-      'gpt-5.5',
-      '--allow-all',
-      '--reasoning-effort',
-      'medium',
-      '--mode',
-      'autopilot',
-      '--output-format',
-      'json',
-    ])
-  })
-})
+vi.mock(import('./sandbox'), async importOriginal => {
+  const actual = await importOriginal()
 
-describe('getVitestConfig', () => {
-  test('configures node tests by default', () => {
-    const config = getVitestConfig('test-results.json')
+  class MockSandbox extends actual.Sandbox {
+    constructor() {
+      super(undefined as never, undefined as never)
+    }
 
-    expect(config).toContain(`outputFile: "test-results.json"`)
-    expect(config).not.toContain('@vitest/browser-playwright')
-  })
+    static async create() {
+      return new MockSandbox()
+    }
+
+    async [Symbol.asyncDispose]() {}
 
-  test('configures browser tests with Playwright and Chromium', () => {
-    const config = getVitestConfig('browser-test-results.json', true)
+    copy = vi.fn()
+    download = vi.fn()
+    readFile = vi.fn()
+    writeFile = vi.fn()
+    exists = vi.fn()
+    addAgentInstruction = vi.fn()
+    addAgentSkill = vi.fn()
+    addCustomAgent = vi.fn()
+    addMcpServer = vi.fn()
+    addCopilotPlugin = vi.fn()
+    runCommand = vi.fn().mockImplementation(async () => {
+      return {
+        stdout: '',
+        stderr: '',
+        exitCode: 0,
+      }
+    })
+  }
 
-    expect(config).toContain(`import {playwright} from '@vitest/browser-playwright'`)
-    expect(config).toContain('enabled: true')
-    expect(config).toContain('headless: true')
-    expect(config).toContain('provider: playwright()')
-    expect(config).toContain(`instances: [{browser: 'chromium'}]`)
-    expect(config).toContain(`outputFile: "browser-test-results.json"`)
+  return {
+    ...actual,
+    Sandbox: MockSandbox,
+  }
+})
+
+test('hello', async () => {
+  const host = new VirtualHost()
+  const results = await run(host, await plan([]), {
+    artifactsDirectory: 'test',
+    copilotToken: 'test',
+    maxConcurrency: 1,
   })
 })
+
+test('runTrial', async () => {
+  const host = new VirtualHost()
+  const result = await runTrial(
+    host,
+    {
+      id: 'test',
+      scenario: {
+        id: 'test',
+        directory: 'test',
+        prompt: 'test',
+        tags: [],
+        testPath: 'test',
+      },
+      treatment: {
+        name: 'test',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    },
+    {
+      artifactsDirectory: '',
+      copilotToken: '',
+    },
+  )
+})
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index a5bc9892..1a704966 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -1,19 +1,54 @@
-import {randomUUID} from 'node:crypto'
-import path from 'node:path'
-import fs from 'node:fs/promises'
-import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, Sandbox} from './sandbox'
-import type {Treatment, TreatmentResult, Walkthrough} from './treatment'
-import type {Model, ReasoningEffort} from './model'
-import {isMessageType, parseMessage, type Message} from './copilot-cli'
-import {getTestMetadata, parseTestResults} from './vitest'
-import {existsSync} from 'node:fs'
+import Queue from 'p-queue'
+import * as z from 'zod/mini'
+import {MessageSchema, parseMessage, type Message} from './copilot-cli'
+import type {Plan} from './plan'
+import {TrialSchema, type Trial} from './trial'
+import type {Host} from './host'
+import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
+import {parseTestResults} from './vitest'
+import {logger} from './logger'
+
+const AssistantSchema = z.object({
+  logs: z.array(MessageSchema),
+  turns: z.number(),
+  outputTokens: z.number(),
+  premiumRequests: z.number(),
+  totalApiDurationMs: z.number(),
+  sessionDurationMs: z.number(),
+  tools: z.record(z.string(), z.number()),
+})
 
-const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright'
-const CHROMIUM_EXECUTABLE_PATH = '/usr/bin/chromium'
-const WALKTHROUGH_DIR = 'walkthrough'
-const WALKTHROUGH_VIEWPORT_WIDTH = 1440
-const WALKTHROUGH_VIEWPORT_HEIGHT = 900
-const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
+const TestResultsSchema = z.object({
+  numTotalTests: z.number(),
+  numPassedTests: z.number(),
+  numFailedTests: z.number(),
+  numPendingTests: z.number(),
+  numTodoTests: z.number(),
+  tests: z.array(
+    z.object({
+      title: z.string(),
+      fullName: z.string(),
+      status: z.enum(['passed', 'failed', 'skipped', 'pending', 'todo', 'disabled']),
+      description: z.optional(z.string()),
+    }),
+  ),
+})
+
+const WalkthroughSchema = z.discriminatedUnion('type', [
+  z.object({type: z.literal('Unavailable')}),
+  z.object({type: z.literal('Screenshot'), filepath: z.string()}),
+  z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}),
+  z.object({type: z.literal('Video'), filepath: z.string()}),
+])
+
+const TrialResultSchema = z.object({
+  trial: TrialSchema,
+  // assistant: AssistantSchema,
+  // testResults: TestResultsSchema,
+  // walkthrough: WalkthroughSchema,
+})
+
+type TrialResult = z.infer<typeof TrialResultSchema>
 
 type RunOptions = {
   artifactsDirectory: string
@@ -22,210 +57,98 @@ type RunOptions = {
   maxConcurrency?: number
 }
 
-function getVitestConfig(outputFile: string, browser = false): string {
-  const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright'\n` : ''
-  const browserConfig = browser
-    ? `    browser: {
-      enabled: true,
-      headless: true,
-      provider: playwright(),
-      instances: [{browser: 'chromium'}],
-    },
-`
-    : ''
-
-  return `
-import {defineConfig} from 'vitest/config'
-${browserImport}
-export default defineConfig({
-  test: {
-${browserConfig}    reporters: [['json', {outputFile: ${JSON.stringify(outputFile)}, includeTaskLocation: true}]],
-  },
-})
-`
-}
-
-function run(treatments: Array<Treatment>, options: RunOptions): Promise<Array<TreatmentResult>> {
-  const maxConcurrency = options.maxConcurrency ?? 1
-  const queue = treatments.slice()
-  const results: Array<TreatmentResult> = []
-  const pending = new Set()
-  let cancelled = false
-
-  let resolve: (value: Array<TreatmentResult>) => void
-  let reject: (reason: unknown) => void
-  const deferred = new Promise<Array<TreatmentResult>>((_resolve, _reject) => {
-    resolve = _resolve
-    reject = _reject
+async function run(host: Host, plan: Plan, options: RunOptions): Promise<Array<TrialResult>> {
+  const {artifactsDirectory, copilotToken, dockerImage, maxConcurrency = 1} = options
+  const queue = new Queue({
+    concurrency: maxConcurrency,
   })
 
-  function execute() {
-    if (cancelled) {
-      return
-    }
-
-    if (queue.length === 0) {
-      if (pending.size === 0) {
-        resolve(results)
-      }
-      return
-    }
-
-    if (pending.size >= maxConcurrency) {
-      return
-    }
-
-    const treatment = queue.shift()
-    if (!treatment) {
-      return
-    }
-
-    const promise = retry(
-      () =>
-        runTreatment(treatment, {
-          artifactsDirectory: options.artifactsDirectory,
-          copilotToken: options.copilotToken,
-          dockerImage: options.dockerImage,
-        }),
-      3,
-    ).then(
-      result => {
-        results.push(result)
-        pending.delete(promise)
-        execute()
-      },
-      error => {
-        cancelled = true
-        pending.delete(promise)
-        reject(error)
-      },
-    )
-
-    pending.add(promise)
-    execute()
-  }
-
-  execute()
+  const results = await Promise.all(
+    plan.trials.map(trial => {
+      return queue.add(() => {
+        return retry(() => {
+          return runTrial(host, trial, {
+            artifactsDirectory,
+            copilotToken,
+            dockerImage,
+          })
+        })
+      })
+    }),
+  )
 
-  return deferred
+  return results
 }
 
-async function retry<T>(fn: () => Promise<T>, retries: number): Promise<T> {
-  try {
-    return await fn()
-  } catch (error) {
-    if (retries > 0) {
-      console.log('Retrying after error: %s', error)
-      return retry(fn, retries - 1)
-    }
-    throw error
-  }
-}
-
-type RunTreatmentOptions = {
+type RunTrialOptions = {
   artifactsDirectory: string
   copilotToken: string
   dockerImage?: string
 }
 
-function getCopilotArgs({
-  prompt,
-  model,
-  reasoningEffort,
-}: {
-  prompt: string
-  model: Model
-  reasoningEffort?: ReasoningEffort
-}): Array<string> {
-  const args = ['-p', prompt, '--model', model, '--allow-all']
+async function runTrial(host: Host, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
+  const {artifactsDirectory, copilotToken, dockerImage} = options
 
-  if (reasoningEffort) {
-    args.push('--reasoning-effort', reasoningEffort)
-  }
+  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
 
-  return [...args, '--mode', 'autopilot', '--output-format', 'json']
-}
+  await using sandbox = await Sandbox.create({
+    dockerImage,
+  })
 
-async function runTreatment(
-  treatment: Treatment,
-  {artifactsDirectory, copilotToken, dockerImage}: RunTreatmentOptions,
-): Promise<TreatmentResult> {
-  console.log('Running treatment: %s (%s)', treatment.config.name, treatment.id)
-  await using sandbox = await Sandbox.create({dockerImage})
+  logger.info('Copying files from: %s...', trial.scenario.directory)
 
-  console.log('Copying files from: %s...', treatment.scenario.directory)
-  await sandbox.copy(treatment.scenario.directory, CONTAINER_WORKDIR, {
+  await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
     exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
   })
   await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
     user: 'root',
   })
 
-  console.log('Obfuscating package name...')
-  await sandbox.runCommand('npm', ['pkg', 'set', `name=${treatment.id}`], {
+  logger.info('Obfuscating package name...')
+  await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
     user: NODE_USER,
   })
 
-  console.log('Removing workspace dependency...')
+  logger.info('Removing workspace dependency...')
   await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
     user: NODE_USER,
   })
 
-  console.log('Installing dependencies...')
+  logger.info('Installing dependencies...')
   await sandbox.runCommand('npm', ['install'], {
     user: NODE_USER,
   })
 
-  if (treatment.experiment.setup) {
-    console.log('Running experiment setup...')
-    await treatment.experiment.setup({
+  if (trial.setup) {
+    logger.info('Running generic setup...')
+    await trial.setup({
       sandbox,
     })
   }
 
-  if (treatment.config.setup) {
-    console.log('Running treatment setup...')
-    await treatment.config.setup({
+  if (trial.treatment.setup) {
+    logger.info('Running treatment setup...')
+    await trial.treatment.setup({
       sandbox,
     })
   }
 
-  console.log('Run build script...')
+  logger.info('Run build script...')
   await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
     user: NODE_USER,
   })
 
-  if (treatment.scenario.browserTestPath) {
-    console.log('Installing browser test dependencies...')
-    await sandbox.runCommand(
-      'npm',
-      ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'],
-      {
-        user: NODE_USER,
-      },
-    )
-    console.log('Installing Playwright browser...')
-    await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], {
-      user: 'root',
+  logger.info('Running copilot...')
+  const copilotOutput = await sandbox.runCommand(
+    'copilot',
+    ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
+    {
+      user: NODE_USER,
       env: {
-        PLAYWRIGHT_BROWSERS_PATH,
+        COPILOT_GITHUB_TOKEN: copilotToken,
       },
-    })
-  }
-
-  console.log('Running copilot...')
-  const {prompt} = treatment.scenario.config
-  const args = getCopilotArgs({
-    prompt,
-    model: treatment.model,
-    reasoningEffort: treatment.reasoningEffort,
-  })
-  const copilotOutput = await sandbox.runCommand('copilot', args, {
-    user: NODE_USER,
-    env: {
-      COPILOT_GITHUB_TOKEN: copilotToken,
     },
-  })
+  )
   const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
     const trimmed = line.trim()
     if (trimmed.length === 0) {
@@ -234,254 +157,66 @@ async function runTreatment(
     return parseMessage(JSON.parse(trimmed))
   })
 
-  const TEST_PATH = 'scenario.test.ts'
-  const BROWSER_TEST_PATH = 'scenario.browser.test.ts'
-  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
-  const TEST_RESULTS_PATH = 'test-results.json'
-  const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json'
-  const scenarioTests = [
-    {
-      sourcePath: treatment.scenario.testPath,
-      testPath: TEST_PATH,
-      resultsPath: TEST_RESULTS_PATH,
-      browser: false,
-    },
-  ]
-
-  if (treatment.scenario.browserTestPath) {
-    scenarioTests.push({
-      sourcePath: treatment.scenario.browserTestPath,
-      testPath: BROWSER_TEST_PATH,
-      resultsPath: BROWSER_TEST_RESULTS_PATH,
-      browser: true,
-    })
-  }
-
-  let numFailedTests = 0
-  let numPassedTests = 0
-  let numPendingTests = 0
-  let numTodoTests = 0
-  let numTotalTests = 0
-  let testRunSuccess = true
-  const tests: TreatmentResult['testResults']['tests'] = []
-  const rawTestResults: Array<Record<string, unknown> & {testResults: Array<unknown>}> = []
-
-  for (const scenarioTest of scenarioTests) {
-    await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath)
-    await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser))
-    // Always pass vitest calls even if test suite fails
-    await sandbox.runCommand(
-      'sh',
-      ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath],
-      {
-        user: NODE_USER,
-        env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined,
-      },
-    )
-
-    const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath)
-    const rawTestResult: unknown = JSON.parse(testResultsContent)
-    const testResults = parseTestResults(rawTestResult)
-    if (!testResults.success) {
-      throw new Error(`Failed to parse test results: ${testResults.error}`)
-    }
+  logger.info('Running tests...')
+
+  // const TEST_PATH = 'scenario.test.ts'
+  // const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
+  // const TEST_RESULTS_PATH = 'test-results.json'
+  //
+  // await sandbox.copy(trial.scenario.testPath, TEST_PATH)
+  // await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
+  // await sandbox.runCommand(
+  //   'sh',
+  //   ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
+  //   {
+  //     user: NODE_USER,
+  //     env: {},
+  //   },
+  // )
+  // const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
+  // const rawTestResult: unknown = JSON.parse(testResultsContent)
+  // const testResults = parseTestResults(rawTestResult)
+  // if (!testResults.success) {
+  //   throw new Error(`Failed to parse test results: ${testResults.error}`)
+  // }
 
-    const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8')
-    numFailedTests += testResults.data.numFailedTests
-    numPassedTests += testResults.data.numPassedTests
-    numPendingTests += testResults.data.numPendingTests
-    numTodoTests += testResults.data.numTodoTests
-    numTotalTests += testResults.data.numTotalTests
-    testRunSuccess &&= testResults.data.success
-    tests.push(...getTestMetadata(testResults.data, testSource))
-    rawTestResults.push(rawTestResult as Record<string, unknown> & {testResults: Array<unknown>})
-  }
-
-  if (rawTestResults.length > 1) {
-    await sandbox.writeFile(
-      TEST_RESULTS_PATH,
-      JSON.stringify({
-        ...rawTestResults[0],
-        numFailedTests,
-        numPassedTests,
-        numPendingTests,
-        numTodoTests,
-        numTotalTests,
-        success: testRunSuccess,
-        testResults: rawTestResults.flatMap(testResult => testResult.testResults),
-      }),
-    )
-  }
-
-  console.log('Capturing walkthrough...')
-  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
-    user: 'root',
-  })
-  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
-    user: NODE_USER,
-  })
-  await sandbox.runCommand(
-    'npx',
-    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
-    {
-      user: NODE_USER,
-    },
-  )
-  await sandbox.writeFile(
-    'agent-browser.json',
-    JSON.stringify({
-      executablePath: CHROMIUM_EXECUTABLE_PATH,
-    }),
-  )
-  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
-
-Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
-
-Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
-
-- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
-- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
-- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
-
-Only capture the walkthrough, do not make any further code changes.`
-  const walkthroughResult = await sandbox.runCommand(
-    'copilot',
-    getCopilotArgs({
-      prompt: walkthroughPrompt,
-      model: 'gpt-5.6-terra',
-      reasoningEffort: 'medium',
-    }),
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: copilotToken,
-      },
-      allowNonZeroExitCode: true,
-    },
-  )
-
-  if (walkthroughResult.exitCode !== 0) {
-    console.warn('Unable to capture walkthrough: %s', walkthroughResult.stderr)
-  }
-
-  // Turns
-  const assistantTurns = new Set()
-  // Tools
-  const toolCalls = new Map()
-  let outputTokens = 0
-
-  for (const message of messages) {
-    if (isMessageType(message, 'assistant.turn_start')) {
-      assistantTurns.add(message.data.turnId)
-    }
-
-    if (isMessageType(message, 'assistant.message')) {
-      outputTokens += message.data.outputTokens ?? 0
-    }
-
-    if (isMessageType(message, 'tool.execution_start')) {
-      const toolName = message.data.toolName
-      toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1)
-    }
-  }
-
-  const result = messages.find(message => isMessageType(message, 'result'))
-  if (!result) {
-    throw new Error('No result message found in copilot output')
+  return {
+    trial,
+    // assistant: {},
+    // testResults: {},
+    // walkthrough: {},
   }
+}
 
-  const artifactDirectory = path.join(artifactsDirectory, treatment.id)
-  const workspacePath = path.join(artifactDirectory, 'workspace')
-  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
-  const copilotConfigPath = path.join(artifactDirectory, '.copilot')
-  const skillsConfigPath = path.join(artifactDirectory, '.agents')
-  const testResultsPath = path.join(workspacePath, 'test-results.json')
-  await fs.rm(artifactDirectory, {recursive: true, force: true})
-  await fs.mkdir(workspacePath, {recursive: true})
-
-  console.log('Downloading agent workspace to: %s...', workspacePath)
-  await sandbox.download(CONTAINER_WORKDIR, workspacePath, {
-    ignore(name) {
-      return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
-    },
-  })
-
-  console.log('Downloading copilot config to: %s...', copilotConfigPath)
-  await sandbox.download(COPILOT_DIR, copilotConfigPath)
-
-  console.log('Downloading skills config to: %s...', skillsConfigPath)
-  await sandbox.download(AGENTS_DIR, skillsConfigPath)
-
-  let walkthrough: Walkthrough = {
-    type: 'Unavailable',
-  }
+function getVitestConfig(outputFile: string) {
+  return `import {defineConfig} from 'vites/tconfig';
 
-  if (existsSync(path.join(workspacePath, WALKTHROUGH_DIR))) {
-    console.log(
-      'Moving walkthrough artifacts from: %s to: %s...',
-      path.join(workspacePath, WALKTHROUGH_DIR),
-      walkthroughPath,
-    )
-    await fs.mkdir(walkthroughPath, {recursive: true})
-    await fs.rename(path.join(workspacePath, WALKTHROUGH_DIR), walkthroughPath)
+export default defineConfig({
+  test: {
+    reporters: [
+      [
+        'json',
+        {
+          outputFile: ${JSON.stringify(outputFile)},
+          includeTaskLocation: true,
+        },
+      ],
+    ],
+  },
+})`
+}
 
-    if (existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
-      walkthrough = {
-        type: 'Screenshot',
-        filepath: path.join(walkthroughPath, 'screenshot.png'),
-      }
-    } else if (existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
-      walkthrough = {
-        type: 'Video',
-        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
-      }
-    } else if (existsSync(path.join(walkthroughPath, 'screenshots'))) {
-      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
-      const entries = await fs.readdir(screenshotsDir).then(filenames => {
-        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
-      })
-      const screenshots = entries.filter(entry => {
-        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
-      })
-      if (screenshots.length > 0) {
-        walkthrough = {
-          type: 'Screenshots',
-          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
-        }
-      }
+async function retry<T>(fn: () => Promise<T>, retries: number = 3): Promise<T> {
+  try {
+    return await fn()
+  } catch (error) {
+    if (retries > 0) {
+      console.log('Retrying after error: %s', error)
+      return retry(fn, retries - 1)
     }
-  }
-
-  return {
-    id: randomUUID(),
-    treatment,
-    artifacts: {
-      directory: artifactDirectory,
-      copilotConfigPath,
-      skillsConfigPath,
-      testResultsPath,
-      workspacePath,
-    },
-    assistant: {
-      logs: messages,
-      turns: assistantTurns.size,
-      outputTokens,
-      premiumRequests: result.usage.premiumRequests,
-      // Time to complete (latency)
-      totalApiDurationMs: result.usage.totalApiDurationMs,
-      sessionDurationMs: result.usage.sessionDurationMs,
-      tools: Object.fromEntries(toolCalls),
-    },
-    testResults: {
-      numFailedTests,
-      numPassedTests,
-      numPendingTests,
-      numTodoTests,
-      numTotalTests,
-      tests,
-    },
-    walkthrough,
+    throw error
   }
 }
 
-export {getCopilotArgs, getVitestConfig, run}
+export {run, runTrial, TrialResultSchema as RunResultSchema}
+export type {RunOptions as RunContext, TrialResult as RunResult}
diff --git a/packages/agent-eval/src/sandbox.ts b/packages/agent-eval/src/sandbox.ts
index b733aab0..062092ef 100644
--- a/packages/agent-eval/src/sandbox.ts
+++ b/packages/agent-eval/src/sandbox.ts
@@ -7,6 +7,7 @@ import Docker from 'dockerode'
 import tarFs from 'tar-fs'
 import type {Headers} from 'tar-fs'
 import tarStream from 'tar-stream'
+import * as z from 'zod/mini'
 import {McpConfigFileSchema} from './mcp-config'
 import type {McpConfigFile, McpServerConfig} from './mcp-config'
 
@@ -778,7 +779,9 @@ function captureStream(destination: NodeJS.WritableStream): {stream: Writable; r
   }
 }
 
-export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox}
+const SandboxSchema = z.instanceof(Sandbox)
+
+export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox, SandboxSchema}
 export type {
   AgentSkillCopiedFile,
   AgentSkillFile,
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index 73ca8a1e..1de788bd 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -18,15 +18,17 @@ type ScenarioConfigModule = {
   default?: unknown
 }
 
-type Scenario = {
-  id: string
-  directory: string
-  prompt: string
-  description?: string
-  tags: Array<string>
-  testPath: string
-  browserTestPath?: string
-}
+const ScenarioSchema = z.object({
+  id: z.string(),
+  directory: z.string(),
+  prompt: z.string(),
+  description: z.optional(z.string()),
+  tags: z.array(z.string()),
+  testPath: z.string(),
+  browserTestPath: z.optional(z.string()),
+})
+
+type Scenario = z.infer<typeof ScenarioSchema>
 
 async function listScenarios(host: Host, directory: string): Promise<Array<Scenario>> {
   const stats = await host.fs.stat(directory)
@@ -114,5 +116,5 @@ async function getScenario(host: Host, directory: string, id: string): Promise<S
   throw new Error(`Scenario "${id}" was not found in: ${directory}`)
 }
 
-export {defineConfig, listScenarios, getScenario, ScenarioConfigSchema}
+export {defineConfig, listScenarios, getScenario, ScenarioSchema, ScenarioConfigSchema}
 export type {ScenarioConfig, Scenario}
diff --git a/packages/agent-eval/src/scenarios.test.ts b/packages/agent-eval/src/scenarios.test.ts
deleted file mode 100644
index 0864c0c1..00000000
--- a/packages/agent-eval/src/scenarios.test.ts
+++ /dev/null
@@ -1,153 +0,0 @@
-import fs from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import {afterEach, describe, expect, test} from 'vitest'
-import {findScenario, listScenarios} from './scenarios'
-
-const temporaryDirectories: Array<string> = []
-
-async function createScenariosDirectory() {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-scenarios-'))
-  temporaryDirectories.push(directory)
-  return directory
-}
-
-async function createScenario(
-  scenariosDirectory: string,
-  id: string,
-  prompt: string,
-  tags?: Array<string>,
-  description?: string,
-) {
-  const directory = path.join(scenariosDirectory, id)
-  await fs.mkdir(directory)
-  await fs.writeFile(
-    path.join(directory, 'scenario.config.ts'),
-    `export default ${JSON.stringify({description, prompt, ...(tags ? {tags} : {})})}`,
-  )
-  await fs.writeFile(path.join(directory, 'scenario.test.ts'), '')
-  return directory
-}
-
-afterEach(async () => {
-  await Promise.all(
-    temporaryDirectories.splice(0).map(directory => {
-      return fs.rm(directory, {recursive: true, force: true})
-    }),
-  )
-})
-
-describe('scenario loading', () => {
-  test('lists scenarios from the provided directory', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    await createScenario(scenariosDirectory, 'second', 'Second prompt')
-    await createScenario(scenariosDirectory, 'first', 'First prompt')
-    await fs.writeFile(path.join(scenariosDirectory, 'README.md'), '')
-
-    await expect(listScenarios({directory: scenariosDirectory})).resolves.toEqual([
-      expect.objectContaining({id: 'first', config: {prompt: 'First prompt'}}),
-      expect.objectContaining({id: 'second', config: {prompt: 'Second prompt'}}),
-    ])
-  })
-
-  test('loads scenario descriptions', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    await createScenario(scenariosDirectory, 'example', 'Example prompt', undefined, 'Example description')
-
-    await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toMatchObject({
-      config: {
-        description: 'Example description',
-        prompt: 'Example prompt',
-      },
-    })
-  })
-
-  test('rejects non-string scenario descriptions', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt')
-    await fs.writeFile(
-      path.join(directory, 'scenario.config.ts'),
-      `export default {description: 42, prompt: 'Example prompt'}`,
-    )
-
-    await expect(findScenario('example', {directory: scenariosDirectory})).rejects.toThrow(
-      'Scenario "example" config must export a default config with a string prompt, optional string description, and optional string[] tags',
-    )
-  })
-
-  test('lists scenarios that match all provided tags', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    await createScenario(scenariosDirectory, 'both', 'Both tags', ['baseline', 'primer'])
-    await createScenario(scenariosDirectory, 'baseline', 'Baseline only', ['baseline'])
-    await createScenario(scenariosDirectory, 'untagged', 'No tags')
-
-    await expect(listScenarios({directory: scenariosDirectory, tags: ['baseline', 'primer']})).resolves.toEqual([
-      expect.objectContaining({id: 'both'}),
-    ])
-  })
-
-  test('lists all scenarios when no tags are provided', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    await createScenario(scenariosDirectory, 'tagged', 'Tagged', ['baseline'])
-    await createScenario(scenariosDirectory, 'untagged', 'Untagged')
-
-    await expect(listScenarios({directory: scenariosDirectory, tags: []})).resolves.toEqual([
-      expect.objectContaining({id: 'tagged'}),
-      expect.objectContaining({id: 'untagged'}),
-    ])
-  })
-
-  test('finds a scenario by id in the provided directory', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt')
-
-    await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({
-      id: 'example',
-      directory,
-      config: {prompt: 'Example prompt'},
-      testPath: path.join(directory, 'scenario.test.ts'),
-    })
-  })
-
-  test('includes an optional browser test', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt')
-    const browserTestPath = path.join(directory, 'scenario.browser.test.ts')
-    await fs.writeFile(browserTestPath, '')
-
-    await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({
-      id: 'example',
-      directory,
-      config: {prompt: 'Example prompt'},
-      testPath: path.join(directory, 'scenario.test.ts'),
-      browserTestPath,
-    })
-  })
-
-  test('returns undefined when a scenario is not found', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-
-    await expect(findScenario('missing-scenario', {directory: scenariosDirectory})).resolves.toBeUndefined()
-  })
-
-  test('returns undefined when a scenario id is not a direct child directory', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    const siblingDirectory = await createScenariosDirectory()
-    await createScenario(siblingDirectory, 'example', 'Example prompt')
-
-    await expect(
-      findScenario(path.relative(scenariosDirectory, path.join(siblingDirectory, 'example')), {
-        directory: scenariosDirectory,
-      }),
-    ).resolves.toBeUndefined()
-  })
-
-  test('throws when the scenarios directory does not exist', async () => {
-    const scenariosDirectory = await createScenariosDirectory()
-    const missingDirectory = path.join(scenariosDirectory, 'missing')
-
-    await expect(findScenario('example', {directory: missingDirectory})).rejects.toThrow(
-      `Scenarios directory does not exist: ${missingDirectory}`,
-    )
-  })
-})
diff --git a/packages/agent-eval/src/scenarios.ts b/packages/agent-eval/src/scenarios.ts
deleted file mode 100644
index 18a56865..00000000
--- a/packages/agent-eval/src/scenarios.ts
+++ /dev/null
@@ -1,128 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {pathToFileURL} from 'node:url'
-import type {ScenarioConfig} from './experiment-config'
-
-type ResolvedScenario = {
-  readonly id: string
-  readonly directory: string
-  readonly config: ScenarioConfig
-  readonly testPath: string
-  readonly browserTestPath?: string
-}
-
-type ScenarioSourceOptions = {
-  directory?: string
-  tags?: ReadonlyArray<string>
-}
-
-function resolveScenariosDirectory(options: ScenarioSourceOptions): string {
-  return path.resolve(options.directory ?? 'scenarios')
-}
-
-async function assertScenariosDirectory(directory: string) {
-  const stats = await fs.stat(directory).catch(() => undefined)
-  if (!stats) {
-    throw new Error(`Scenarios directory does not exist: ${directory}`)
-  }
-  if (!stats.isDirectory()) {
-    throw new Error(`Scenarios path is not a directory: ${directory}`)
-  }
-}
-
-async function assertScenarioDirectory(directory: string, name: string) {
-  const stats = await fs.stat(directory).catch(() => undefined)
-  if (!stats?.isDirectory()) {
-    throw new Error(`Scenario "${name}" directory was not found: ${directory}`)
-  }
-}
-
-async function assertScenarioFile(filepath: string, name: string, kind: 'config' | 'test') {
-  const stats = await fs.stat(filepath).catch(() => undefined)
-  if (!stats?.isFile()) {
-    throw new Error(`Scenario "${name}" ${kind} file was not found: ${filepath}`)
-  }
-}
-
-function isScenarioConfig(value: unknown): value is ScenarioConfig {
-  if (value === null || typeof value !== 'object') {
-    return false
-  }
-
-  const config = value as Record<string, unknown>
-  return (
-    typeof config.prompt === 'string' &&
-    (config.description === undefined || typeof config.description === 'string') &&
-    (config.tags === undefined ||
-      (Array.isArray(config.tags) && config.tags.every((tag: unknown) => typeof tag === 'string')))
-  )
-}
-
-async function loadScenarioConfig(configPath: string, name: string): Promise<ScenarioConfig> {
-  const configModule = (await import(pathToFileURL(configPath).href)) as {default?: unknown}
-  if (!isScenarioConfig(configModule.default)) {
-    throw new Error(
-      `Scenario "${name}" config must export a default config with a string prompt, optional string description, and optional string[] tags`,
-    )
-  }
-  return configModule.default
-}
-
-async function loadScenarioDirectory(directory: string, name = path.basename(directory)): Promise<ResolvedScenario> {
-  await assertScenarioDirectory(directory, name)
-
-  const configPath = path.join(directory, 'scenario.config.ts')
-  const testPath = path.join(directory, 'scenario.test.ts')
-  const browserTestPath = path.join(directory, 'scenario.browser.test.ts')
-  await assertScenarioFile(configPath, name, 'config')
-  await assertScenarioFile(testPath, name, 'test')
-  const browserTestStats = await fs.stat(browserTestPath).catch(() => undefined)
-
-  return {
-    id: name,
-    directory,
-    config: await loadScenarioConfig(configPath, name),
-    testPath,
-    ...(browserTestStats?.isFile() ? {browserTestPath} : {}),
-  }
-}
-
-async function getScenarioDirectoryNames(options: ScenarioSourceOptions): Promise<Array<string>> {
-  const scenariosDirectory = resolveScenariosDirectory(options)
-  await assertScenariosDirectory(scenariosDirectory)
-
-  const entries = await fs.readdir(scenariosDirectory, {withFileTypes: true})
-  return entries
-    .filter(entry => entry.isDirectory())
-    .map(entry => entry.name)
-    .toSorted()
-}
-
-async function listScenarios(options: ScenarioSourceOptions = {}): Promise<ReadonlyArray<ResolvedScenario>> {
-  const scenariosDirectory = resolveScenariosDirectory(options)
-  const names = await getScenarioDirectoryNames(options)
-  const scenarios = await Promise.all(
-    names.map(name => loadScenarioDirectory(path.join(scenariosDirectory, name), name)),
-  )
-  return scenarios.filter(scenario => options.tags?.every(tag => scenario.config.tags?.includes(tag)) ?? true)
-}
-
-async function findScenario(id: string, options: ScenarioSourceOptions = {}): Promise<ResolvedScenario | undefined> {
-  const scenariosDirectory = resolveScenariosDirectory(options)
-  await assertScenariosDirectory(scenariosDirectory)
-
-  const directory = path.resolve(scenariosDirectory, id)
-  if (path.dirname(directory) !== scenariosDirectory) {
-    return undefined
-  }
-
-  const stats = await fs.stat(directory).catch(() => undefined)
-  if (!stats?.isDirectory()) {
-    return undefined
-  }
-
-  return loadScenarioDirectory(directory, id)
-}
-
-export {findScenario, listScenarios, loadScenarioDirectory}
-export type {ResolvedScenario, ScenarioSourceOptions}
diff --git a/packages/agent-eval/src/treatment-old.ts b/packages/agent-eval/src/treatment-old.ts
new file mode 100644
index 00000000..8e7d9101
--- /dev/null
+++ b/packages/agent-eval/src/treatment-old.ts
@@ -0,0 +1,60 @@
+import type {ExperimentConfig, TreatmentConfig} from './experiment-config'
+import type {Model, ReasoningEffort} from './model'
+import type {Message} from './copilot-cli'
+import type {ResolvedScenario} from './resolve-experiment-scenario'
+
+type Treatment = {
+  config: TreatmentConfig
+  scenario: ResolvedScenario
+  experiment: ExperimentConfig
+  id: string
+  model: Model
+  reasoningEffort?: ReasoningEffort
+}
+
+type TreatmentResult = {
+  id: string
+  treatment: Treatment
+  artifacts: {
+    copilotConfigPath: string
+    directory: string
+    skillsConfigPath: string
+    testResultsPath: string
+    workspacePath: string
+  }
+  assistant: {
+    logs: Array<Message>
+    turns: number
+    outputTokens: number
+    premiumRequests: number
+    totalApiDurationMs: number
+    sessionDurationMs: number
+    tools: Record<string, number>
+  }
+  testResults: {
+    numTotalTests: number
+    numPassedTests: number
+    numFailedTests: number
+    numPendingTests: number
+    numTodoTests: number
+    tests: Array<{
+      title: string
+      fullName: string
+      status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
+      description?: string
+    }>
+  }
+  walkthrough: Walkthrough
+}
+
+const ControlTreatment: TreatmentConfig = {
+  name: 'Control',
+}
+
+type Walkthrough =
+  | {type: 'Unavailable'}
+  | {type: 'Screenshot'; filepath: string}
+  | {type: 'Screenshots'; screenshots: Array<string>}
+  | {type: 'Video'; filepath: string}
+
+export type {Treatment, TreatmentResult, ControlTreatment, Walkthrough}
diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts
index 8e7d9101..e453eb3b 100644
--- a/packages/agent-eval/src/treatment.ts
+++ b/packages/agent-eval/src/treatment.ts
@@ -1,60 +1,27 @@
-import type {ExperimentConfig, TreatmentConfig} from './experiment-config'
-import type {Model, ReasoningEffort} from './model'
-import type {Message} from './copilot-cli'
-import type {ResolvedScenario} from './resolve-experiment-scenario'
+import * as z from 'zod/mini'
+import {SandboxSchema} from './sandbox'
 
-type Treatment = {
-  config: TreatmentConfig
-  scenario: ResolvedScenario
-  experiment: ExperimentConfig
-  id: string
-  model: Model
-  reasoningEffort?: ReasoningEffort
-}
+const TreatmentSetupSchema = z.function({
+  input: [
+    z.object({
+      sandbox: SandboxSchema,
+    }),
+  ],
+  output: z.promise(z.void()),
+})
 
-type TreatmentResult = {
-  id: string
-  treatment: Treatment
-  artifacts: {
-    copilotConfigPath: string
-    directory: string
-    skillsConfigPath: string
-    testResultsPath: string
-    workspacePath: string
-  }
-  assistant: {
-    logs: Array<Message>
-    turns: number
-    outputTokens: number
-    premiumRequests: number
-    totalApiDurationMs: number
-    sessionDurationMs: number
-    tools: Record<string, number>
-  }
-  testResults: {
-    numTotalTests: number
-    numPassedTests: number
-    numFailedTests: number
-    numPendingTests: number
-    numTodoTests: number
-    tests: Array<{
-      title: string
-      fullName: string
-      status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
-      description?: string
-    }>
-  }
-  walkthrough: Walkthrough
-}
+type TreatmentSetup = z.infer<typeof TreatmentSetupSchema>
+
+const TreatmentSchema = z.object({
+  name: z.string(),
+  setup: z.optional(TreatmentSetupSchema),
+})
 
-const ControlTreatment: TreatmentConfig = {
+type Treatment = z.infer<typeof TreatmentSchema>
+
+const ControlTreatment: Treatment = {
   name: 'Control',
 }
 
-type Walkthrough =
-  | {type: 'Unavailable'}
-  | {type: 'Screenshot'; filepath: string}
-  | {type: 'Screenshots'; screenshots: Array<string>}
-  | {type: 'Video'; filepath: string}
-
-export type {Treatment, TreatmentResult, ControlTreatment, Walkthrough}
+export {ControlTreatment, TreatmentSchema, TreatmentSetupSchema}
+export type {Treatment, TreatmentSetup}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index fc839efb..0105ec21 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -61,7 +61,7 @@ importers:
         version: link:../packages/agent-eval
       '@types/node':
         specifier: ^26.1.2
-        version: 26.2.0
+        version: 26.4.0
       typescript:
         specifier: ^6.0.3
         version: 6.0.3
@@ -83,6 +83,15 @@ importers:
       dockerode:
         specifier: ^5.0.1
         version: 5.0.1
+      p-queue:
+        specifier: ^9.3.3
+        version: 9.3.3
+      pino:
+        specifier: ^10.3.1
+        version: 10.3.1
+      pino-pretty:
+        specifier: ^13.1.3
+        version: 13.1.3
       tar-fs:
         specifier: ^3.1.2
         version: 3.1.3
@@ -382,16 +391,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/007-agent-infers-billing-banner:
     dependencies:
@@ -410,16 +419,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/008-agent-infers-action-menu:
     dependencies:
@@ -438,16 +447,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/009-agent-uses-layout-and-color-tokens:
     dependencies:
@@ -466,16 +475,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/010-agent-uses-typography-tokens:
     dependencies:
@@ -494,16 +503,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/011-agent-uses-motion-tokens:
     dependencies:
@@ -522,16 +531,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/012-agent-infers-status-tokens:
     dependencies:
@@ -550,16 +559,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/013-agent-infers-compact-control-tokens:
     dependencies:
@@ -578,16 +587,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/014-agent-replaces-custom-icons-with-octicons:
     dependencies:
@@ -606,16 +615,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/015-agent-infers-copy-icon:
     dependencies:
@@ -634,16 +643,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/016-agent-uses-loading-and-empty-state-patterns:
     dependencies:
@@ -662,16 +671,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/017-agent-uses-confirmation-pattern:
     dependencies:
@@ -690,16 +699,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/018-agent-uses-filter-pattern:
     dependencies:
@@ -718,16 +727,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/019-agent-uses-dismissal-utilities:
     dependencies:
@@ -746,16 +755,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/020-agent-uses-resize-observer-utility:
     dependencies:
@@ -774,16 +783,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/021-agent-sets-up-primer-in-vite:
     dependencies:
@@ -802,16 +811,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/022-agent-enables-automatic-theming:
     dependencies:
@@ -830,16 +839,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/023-agent-adds-theme-switcher:
     dependencies:
@@ -858,16 +867,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/024-agent-sets-up-tailwindcss:
     dependencies:
@@ -886,16 +895,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/025-agent-uses-tokens-with-tailwindcss:
     dependencies:
@@ -911,13 +920,13 @@ importers:
         version: link:../../packages/agent-eval
       '@tailwindcss/vite':
         specifier: ^4.3.3
-        version: 4.3.3(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
       '@types/react':
         specifier: ^19
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       tailwindcss:
         specifier: ^4.3.3
         version: 4.3.3
@@ -926,10 +935,10 @@ importers:
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   scenarios/026-agent-avoids-deprecated-notification:
     dependencies:
@@ -948,16 +957,16 @@ importers:
         version: 19.2.18
       '@types/react-dom':
         specifier: ^19
-        version: 19.2.4(@types/react@19.2.18)
+        version: 19.2.5(@types/react@19.2.18)
       typescript:
         specifier: ^6
         version: 6.0.3
       vite:
         specifier: ^8
-        version: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
       vitest:
         specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.2.0)(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))
+        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
   website:
     dependencies:
@@ -1599,6 +1608,9 @@ packages:
   '@oxc-project/types@0.147.0':
     resolution: {integrity: sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==}
 
+  '@pinojs/redact@0.4.0':
+    resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
+
   '@pkgr/core@0.3.6':
     resolution: {integrity: sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==}
     engines: {node: ^14.18.0 || >=16.0.0}
@@ -2318,6 +2330,10 @@ packages:
     resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==}
     engines: {node: '>= 0.4'}
 
+  atomic-sleep@1.0.0:
+    resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
+    engines: {node: '>=8.0.0'}
+
   available-typed-arrays@1.0.7:
     resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==}
     engines: {node: '>= 0.4'}
@@ -2477,6 +2493,9 @@ packages:
   color2k@2.0.4:
     resolution: {integrity: sha512-OXAPGFRNeLFnUfqDtloYdxkwsJoIdXe28+bjbpJiPqyei2HPa3VHmMCWa0Qe62+U4Ftf9Hj7hRssOkxz7WiWbg==}
 
+  colorette@2.0.20:
+    resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
+
   concat-map@0.0.1:
     resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
 
@@ -2509,6 +2528,9 @@ packages:
     resolution: {integrity: sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==}
     engines: {node: '>= 0.4'}
 
+  dateformat@4.6.3:
+    resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==}
+
   debug@3.2.7:
     resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==}
     peerDependencies:
@@ -2781,6 +2803,9 @@ packages:
     resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==}
     engines: {node: '>=0.10.0'}
 
+  eventemitter3@5.0.4:
+    resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==}
+
   events-universal@1.0.1:
     resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
 
@@ -2788,6 +2813,9 @@ packages:
     resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
     engines: {node: '>=12.0.0'}
 
+  fast-copy@4.0.4:
+    resolution: {integrity: sha512-eVAiWVNPSEGIzDl5yPuLrx8fNMogScXvD9xp1Kzd41FjRIz2I3sSIcxsFeM5EzFfHAfobdvs8ZySffUopljvIA==}
+
   fast-deep-equal@3.1.3:
     resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
 
@@ -2803,6 +2831,9 @@ packages:
   fast-levenshtein@2.0.6:
     resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==}
 
+  fast-safe-stringify@2.1.1:
+    resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
+
   fast-string-truncated-width@3.0.3:
     resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==}
 
@@ -2946,6 +2977,9 @@ packages:
     resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==}
     engines: {node: '>= 0.4'}
 
+  help-me@5.0.0:
+    resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==}
+
   history@5.3.0:
     resolution: {integrity: sha512-ZqaKwjjrAYUYfLG+htGaIIZ4nioX2L70ZUMIFysS3xvBsSG4x/n1V6TXV3N8ZYNuFGlDirFg32T7B6WOUPDYcQ==}
 
@@ -3106,6 +3140,10 @@ packages:
   jju@1.4.0:
     resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==}
 
+  joycon@3.1.1:
+    resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==}
+    engines: {node: '>=10'}
+
   js-yaml@4.3.0:
     resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==}
     hasBin: true
@@ -3435,6 +3473,10 @@ packages:
     resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==}
     engines: {node: '>=12.20.0'}
 
+  on-exit-leak-free@2.1.2:
+    resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==}
+    engines: {node: '>=14.0.0'}
+
   once@1.4.0:
     resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
 
@@ -3454,6 +3496,14 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
+  p-queue@9.3.3:
+    resolution: {integrity: sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==}
+    engines: {node: '>=20'}
+
+  p-timeout@7.0.1:
+    resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==}
+    engines: {node: '>=20'}
+
   package-json-from-dist@1.0.1:
     resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
 
@@ -3489,6 +3539,20 @@ packages:
     resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==}
     engines: {node: '>=12'}
 
+  pino-abstract-transport@3.0.0:
+    resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==}
+
+  pino-pretty@13.1.3:
+    resolution: {integrity: sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==}
+    hasBin: true
+
+  pino-std-serializers@7.1.0:
+    resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==}
+
+  pino@10.3.1:
+    resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==}
+    hasBin: true
+
   playwright-core@1.62.1:
     resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==}
     engines: {node: '>=20'}
@@ -3529,6 +3593,9 @@ packages:
     engines: {node: '>=14'}
     hasBin: true
 
+  process-warning@5.1.0:
+    resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==}
+
   protobufjs@7.6.5:
     resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==}
     engines: {node: '>=12.0.0'}
@@ -3545,6 +3612,9 @@ packages:
     resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
     engines: {node: '>=6'}
 
+  quick-format-unescaped@4.0.4:
+    resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
+
   react-compiler-runtime@1.0.0:
     resolution: {integrity: sha512-rRfjYv66HlG8896yPUDONgKzG5BxZD1nV9U6rkm+7VCuvQc903C4MjcoZR4zPw53IKSOX9wMQVpA1IAbRtzQ7w==}
     peerDependencies:
@@ -3575,6 +3645,13 @@ packages:
     resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
     engines: {node: '>= 6'}
 
+  real-require@0.2.0:
+    resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==}
+    engines: {node: '>= 12.13.0'}
+
+  real-require@1.0.0:
+    resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==}
+
   reflect.getprototypeof@1.0.10:
     resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==}
     engines: {node: '>= 0.4'}
@@ -3647,12 +3724,19 @@ packages:
     resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==}
     engines: {node: '>= 0.4'}
 
+  safe-stable-stringify@2.5.0:
+    resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==}
+    engines: {node: '>=10'}
+
   safer-buffer@2.1.2:
     resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
 
   scheduler@0.27.0:
     resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==}
 
+  secure-json-parse@4.1.0:
+    resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==}
+
   semver@6.3.1:
     resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
     hasBin: true
@@ -3717,6 +3801,9 @@ packages:
   sisteransi@1.0.5:
     resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
 
+  sonic-boom@4.2.1:
+    resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==}
+
   source-map-js@1.2.1:
     resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
     engines: {node: '>=0.10.0'}
@@ -3724,6 +3811,10 @@ packages:
   split-ca@1.0.1:
     resolution: {integrity: sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==}
 
+  split2@4.2.0:
+    resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
+    engines: {node: '>= 10.x'}
+
   ssh2@1.17.0:
     resolution: {integrity: sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==}
     engines: {node: '>=10.16.0'}
@@ -3776,6 +3867,10 @@ packages:
     resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
     engines: {node: '>=8'}
 
+  strip-json-comments@5.0.3:
+    resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==}
+    engines: {node: '>=14.16'}
+
   styled-jsx@5.1.6:
     resolution: {integrity: sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==}
     engines: {node: '>= 12.0.0'}
@@ -3832,6 +3927,10 @@ packages:
     peerDependencies:
       tslib: ^2
 
+  thread-stream@4.2.0:
+    resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==}
+    engines: {node: '>=20'}
+
   tinybench@2.9.0:
     resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
 
@@ -4624,6 +4723,8 @@ snapshots:
 
   '@oxc-project/types@0.147.0': {}
 
+  '@pinojs/redact@0.4.0': {}
+
   '@pkgr/core@0.3.6': {}
 
   '@pnpm/deps.graph-sequencer@1100.0.1': {}
@@ -4823,12 +4924,12 @@ snapshots:
       postcss: 8.5.26
       tailwindcss: 4.3.3
 
-  '@tailwindcss/vite@4.3.3(vite@8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0))':
+  '@tailwindcss/vite@4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))':
     dependencies:
       '@tailwindcss/node': 4.3.3
       '@tailwindcss/oxide': 4.3.3
       tailwindcss: 4.3.3
-      vite: 8.2.2(@types/node@26.2.0)(jiti@2.7.0)(yaml@2.9.0)
+      vite: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
 
   '@tanstack/react-virtual@3.14.6(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
     dependencies:
@@ -5291,6 +5392,8 @@ snapshots:
 
   async-function@1.0.0: {}
 
+  atomic-sleep@1.0.0: {}
+
   available-typed-arrays@1.0.7:
     dependencies:
       possible-typed-array-names: 1.1.0
@@ -5426,6 +5529,8 @@ snapshots:
 
   color2k@2.0.4: {}
 
+  colorette@2.0.20: {}
+
   concat-map@0.0.1: {}
 
   convert-source-map@2.0.0: {}
@@ -5464,6 +5569,8 @@ snapshots:
       es-errors: 1.3.0
       is-data-view: 1.0.2
 
+  dateformat@4.6.3: {}
+
   debug@3.2.7:
     dependencies:
       ms: 2.1.3
@@ -5853,6 +5960,8 @@ snapshots:
 
   esutils@2.0.3: {}
 
+  eventemitter3@5.0.4: {}
+
   events-universal@1.0.1:
     dependencies:
       bare-events: 2.9.1
@@ -5861,6 +5970,8 @@ snapshots:
 
   expect-type@1.4.0: {}
 
+  fast-copy@4.0.4: {}
+
   fast-deep-equal@3.1.3: {}
 
   fast-diff@1.3.0: {}
@@ -5871,6 +5982,8 @@ snapshots:
 
   fast-levenshtein@2.0.6: {}
 
+  fast-safe-stringify@2.1.1: {}
+
   fast-string-truncated-width@3.0.3: {}
 
   fast-string-width@3.0.2:
@@ -6010,6 +6123,8 @@ snapshots:
     dependencies:
       function-bind: 1.1.2
 
+  help-me@5.0.0: {}
+
   history@5.3.0:
     dependencies:
       '@babel/runtime': 7.29.7
@@ -6163,6 +6278,8 @@ snapshots:
 
   jju@1.4.0: {}
 
+  joycon@3.1.1: {}
+
   js-yaml@4.3.0:
     dependencies:
       argparse: 2.0.1
@@ -6451,6 +6568,8 @@ snapshots:
 
   obug@2.1.4: {}
 
+  on-exit-leak-free@2.1.2: {}
+
   once@1.4.0:
     dependencies:
       wrappy: 1.0.2
@@ -6479,6 +6598,13 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
+  p-queue@9.3.3:
+    dependencies:
+      eventemitter3: 5.0.4
+      p-timeout: 7.0.1
+
+  p-timeout@7.0.1: {}
+
   package-json-from-dist@1.0.1: {}
 
   package-manager-detector@1.8.0: {}
@@ -6504,6 +6630,42 @@ snapshots:
 
   picomatch@4.0.5: {}
 
+  pino-abstract-transport@3.0.0:
+    dependencies:
+      split2: 4.2.0
+
+  pino-pretty@13.1.3:
+    dependencies:
+      colorette: 2.0.20
+      dateformat: 4.6.3
+      fast-copy: 4.0.4
+      fast-safe-stringify: 2.1.1
+      help-me: 5.0.0
+      joycon: 3.1.1
+      minimist: 1.2.8
+      on-exit-leak-free: 2.1.2
+      pino-abstract-transport: 3.0.0
+      pump: 3.0.4
+      secure-json-parse: 4.1.0
+      sonic-boom: 4.2.1
+      strip-json-comments: 5.0.3
+
+  pino-std-serializers@7.1.0: {}
+
+  pino@10.3.1:
+    dependencies:
+      '@pinojs/redact': 0.4.0
+      atomic-sleep: 1.0.0
+      on-exit-leak-free: 2.1.2
+      pino-abstract-transport: 3.0.0
+      pino-std-serializers: 7.1.0
+      process-warning: 5.1.0
+      quick-format-unescaped: 4.0.4
+      real-require: 0.2.0
+      safe-stable-stringify: 2.5.0
+      sonic-boom: 4.2.1
+      thread-stream: 4.2.0
+
   playwright-core@1.62.1: {}
 
   playwright@1.62.1:
@@ -6536,6 +6698,8 @@ snapshots:
 
   prettier@3.9.6: {}
 
+  process-warning@5.1.0: {}
+
   protobufjs@7.6.5:
     dependencies:
       '@protobufjs/aspromise': 1.1.2
@@ -6564,6 +6728,8 @@ snapshots:
 
   punycode@2.3.1: {}
 
+  quick-format-unescaped@4.0.4: {}
+
   react-compiler-runtime@1.0.0(react@19.2.8):
     dependencies:
       react: 19.2.8
@@ -6589,6 +6755,10 @@ snapshots:
       string_decoder: 1.3.0
       util-deprecate: 1.0.2
 
+  real-require@0.2.0: {}
+
+  real-require@1.0.0: {}
+
   reflect.getprototypeof@1.0.10:
     dependencies:
       call-bind: 1.0.9
@@ -6689,10 +6859,14 @@ snapshots:
       es-errors: 1.3.0
       is-regex: 1.2.1
 
+  safe-stable-stringify@2.5.0: {}
+
   safer-buffer@2.1.2: {}
 
   scheduler@0.27.0: {}
 
+  secure-json-parse@4.1.0: {}
+
   semver@6.3.1: {}
 
   semver@7.8.5: {}
@@ -6793,10 +6967,16 @@ snapshots:
 
   sisteransi@1.0.5: {}
 
+  sonic-boom@4.2.1:
+    dependencies:
+      atomic-sleep: 1.0.0
+
   source-map-js@1.2.1: {}
 
   split-ca@1.0.1: {}
 
+  split2@4.2.0: {}
+
   ssh2@1.17.0:
     dependencies:
       asn1: 0.2.6
@@ -6871,6 +7051,8 @@ snapshots:
 
   strip-json-comments@3.1.1: {}
 
+  strip-json-comments@5.0.3: {}
+
   styled-jsx@5.1.6(react@19.2.8):
     dependencies:
       client-only: 0.0.1
@@ -6943,6 +7125,10 @@ snapshots:
     dependencies:
       tslib: 2.8.1
 
+  thread-stream@4.2.0:
+    dependencies:
+      real-require: 1.0.0
+
   tinybench@2.9.0: {}
 
   tinyexec@1.3.0: {}

From ff1dc6bab581fd332e717f5bcaba6a6c0856af9f Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 28 Aug 2026 16:42:27 -0500
Subject: [PATCH 12/52] refactor: add virtual and system sandboxes

---
 packages/agent-eval/src/experiment-config.ts  |   4 +-
 packages/agent-eval/src/experiment.ts         |   4 +-
 packages/agent-eval/src/host.ts               |   4 +-
 packages/agent-eval/src/run.test.ts           |  51 +-
 packages/agent-eval/src/run.ts                | 156 ++--
 .../{sandbox.test.ts => sandbox-old.test.ts}  |   0
 .../src/{sandbox.ts => sandbox-old.ts}        |   0
 packages/agent-eval/src/sandbox/constants.ts  |  68 ++
 packages/agent-eval/src/sandbox/index.ts      |  25 +
 packages/agent-eval/src/sandbox/system.ts     | 687 ++++++++++++++++++
 packages/agent-eval/src/sandbox/types.ts      | 170 +++++
 .../agent-eval/src/sandbox/virtual.test.ts    |  58 ++
 packages/agent-eval/src/sandbox/virtual.ts    | 226 ++++++
 13 files changed, 1328 insertions(+), 125 deletions(-)
 rename packages/agent-eval/src/{sandbox.test.ts => sandbox-old.test.ts} (100%)
 rename packages/agent-eval/src/{sandbox.ts => sandbox-old.ts} (100%)
 create mode 100644 packages/agent-eval/src/sandbox/constants.ts
 create mode 100644 packages/agent-eval/src/sandbox/index.ts
 create mode 100644 packages/agent-eval/src/sandbox/system.ts
 create mode 100644 packages/agent-eval/src/sandbox/types.ts
 create mode 100644 packages/agent-eval/src/sandbox/virtual.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/virtual.ts

diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts
index 6ccbea10..49a9ce6f 100644
--- a/packages/agent-eval/src/experiment-config.ts
+++ b/packages/agent-eval/src/experiment-config.ts
@@ -1,5 +1,5 @@
 import type {ExperimentModelConfig} from './model'
-import type {Sandbox} from './sandbox'
+import type {SandboxInstance} from './sandbox'
 
 type ScenarioConfig = {
   description?: string
@@ -28,7 +28,7 @@ type TreatmentConfig = {
   setup?: Setup
 }
 
-type Setup = ({sandbox}: {sandbox: Sandbox}) => Promise<void>
+type Setup = ({sandbox}: {sandbox: SandboxInstance}) => Promise<void>
 
 const ControlTreatment: TreatmentConfig = {
   name: 'Control',
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 5cbf7f20..4c60be63 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -5,7 +5,7 @@ import type {
   LocalCopilotPluginSource,
   McpServerConfig,
   RemoteCopilotPluginSource,
-  Sandbox,
+  SandboxInstance,
 } from './sandbox'
 import type {ExperimentModelConfig, Model, ModelConfig, ModelInfo, ReasoningEffort} from './model'
 
@@ -25,7 +25,7 @@ export type {
   ModelInfo,
   ReasoningEffort,
   RemoteCopilotPluginSource,
-  Sandbox,
+  SandboxInstance as Sandbox,
   TreatmentConfig,
 }
 export {defineConfig}
diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
index dd5b7db9..9a29d836 100644
--- a/packages/agent-eval/src/host.ts
+++ b/packages/agent-eval/src/host.ts
@@ -1,6 +1,8 @@
 import {existsSync} from 'node:fs'
 import fs from 'node:fs/promises'
 import {memfs, Volume, type NestedDirectoryJSON} from 'memfs'
+import type {Sandbox} from './sandbox'
+import {VirtualSandbox} from './sandbox'
 
 type FileSystem = typeof import('node:fs/promises')
 
@@ -11,7 +13,7 @@ interface Host {
 }
 
 class SystemHost implements Host {
-  static create() {
+  static async create() {
     return new SystemHost()
   }
 
diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts
index 015057f1..ab6f31e4 100644
--- a/packages/agent-eval/src/run.test.ts
+++ b/packages/agent-eval/src/run.test.ts
@@ -2,49 +2,14 @@ import {test, expect, vi} from 'vitest'
 import {run, runTrial} from './run'
 import {plan} from './plan'
 import {VirtualHost} from './host'
-
-vi.mock(import('./sandbox'), async importOriginal => {
-  const actual = await importOriginal()
-
-  class MockSandbox extends actual.Sandbox {
-    constructor() {
-      super(undefined as never, undefined as never)
-    }
-
-    static async create() {
-      return new MockSandbox()
-    }
-
-    async [Symbol.asyncDispose]() {}
-
-    copy = vi.fn()
-    download = vi.fn()
-    readFile = vi.fn()
-    writeFile = vi.fn()
-    exists = vi.fn()
-    addAgentInstruction = vi.fn()
-    addAgentSkill = vi.fn()
-    addCustomAgent = vi.fn()
-    addMcpServer = vi.fn()
-    addCopilotPlugin = vi.fn()
-    runCommand = vi.fn().mockImplementation(async () => {
-      return {
-        stdout: '',
-        stderr: '',
-        exitCode: 0,
-      }
-    })
-  }
-
-  return {
-    ...actual,
-    Sandbox: MockSandbox,
-  }
-})
+import {VirtualSandbox} from './sandbox'
 
 test('hello', async () => {
-  const host = new VirtualHost()
-  const results = await run(host, await plan([]), {
+  const host = VirtualHost.create()
+  const sandbox = await VirtualSandbox.create({
+    host,
+  })
+  const results = await run(host, sandbox, await plan([]), {
     artifactsDirectory: 'test',
     copilotToken: 'test',
     maxConcurrency: 1,
@@ -53,8 +18,12 @@ test('hello', async () => {
 
 test('runTrial', async () => {
   const host = new VirtualHost()
+  const sandbox = await VirtualSandbox.create({
+    host,
+  })
   const result = await runTrial(
     host,
+    sandbox,
     {
       id: 'test',
       scenario: {
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index 1a704966..2c43645a 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -4,7 +4,8 @@ import {MessageSchema, parseMessage, type Message} from './copilot-cli'
 import type {Plan} from './plan'
 import {TrialSchema, type Trial} from './trial'
 import type {Host} from './host'
-import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
+// import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
+import type {Sandbox} from './sandbox'
 import {parseTestResults} from './vitest'
 import {logger} from './logger'
 
@@ -53,11 +54,10 @@ type TrialResult = z.infer<typeof TrialResultSchema>
 type RunOptions = {
   artifactsDirectory: string
   copilotToken: string
-  dockerImage?: string
   maxConcurrency?: number
 }
 
-async function run(host: Host, plan: Plan, options: RunOptions): Promise<Array<TrialResult>> {
+async function run(host: Host, sandbox: Sandbox, plan: Plan, options: RunOptions): Promise<Array<TrialResult>> {
   const {artifactsDirectory, copilotToken, dockerImage, maxConcurrency = 1} = options
   const queue = new Queue({
     concurrency: maxConcurrency,
@@ -67,10 +67,9 @@ async function run(host: Host, plan: Plan, options: RunOptions): Promise<Array<T
     plan.trials.map(trial => {
       return queue.add(() => {
         return retry(() => {
-          return runTrial(host, trial, {
+          return runTrial(host, sandbox, trial, {
             artifactsDirectory,
             copilotToken,
-            dockerImage,
           })
         })
       })
@@ -83,82 +82,81 @@ async function run(host: Host, plan: Plan, options: RunOptions): Promise<Array<T
 type RunTrialOptions = {
   artifactsDirectory: string
   copilotToken: string
-  dockerImage?: string
 }
 
-async function runTrial(host: Host, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
-  const {artifactsDirectory, copilotToken, dockerImage} = options
-
-  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
-
-  await using sandbox = await Sandbox.create({
-    dockerImage,
-  })
-
-  logger.info('Copying files from: %s...', trial.scenario.directory)
-
-  await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
-    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
-  })
-  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
-    user: 'root',
-  })
-
-  logger.info('Obfuscating package name...')
-  await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
-    user: NODE_USER,
-  })
-
-  logger.info('Removing workspace dependency...')
-  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
-    user: NODE_USER,
-  })
-
-  logger.info('Installing dependencies...')
-  await sandbox.runCommand('npm', ['install'], {
-    user: NODE_USER,
-  })
-
-  if (trial.setup) {
-    logger.info('Running generic setup...')
-    await trial.setup({
-      sandbox,
-    })
-  }
-
-  if (trial.treatment.setup) {
-    logger.info('Running treatment setup...')
-    await trial.treatment.setup({
-      sandbox,
-    })
-  }
-
-  logger.info('Run build script...')
-  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
-    user: NODE_USER,
-  })
-
-  logger.info('Running copilot...')
-  const copilotOutput = await sandbox.runCommand(
-    'copilot',
-    ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: copilotToken,
-      },
-    },
-  )
-  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
-    const trimmed = line.trim()
-    if (trimmed.length === 0) {
-      return []
-    }
-    return parseMessage(JSON.parse(trimmed))
-  })
-
-  logger.info('Running tests...')
-
+async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
+  // const {artifactsDirectory, copilotToken, dockerImage} = options
+  //
+  // logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
+  //
+  // await using sandbox = await Sandbox.create({
+  //   dockerImage,
+  // })
+  //
+  // logger.info('Copying files from: %s...', trial.scenario.directory)
+  //
+  // await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
+  //   exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
+  // })
+  // await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
+  //   user: 'root',
+  // })
+  //
+  // logger.info('Obfuscating package name...')
+  // await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
+  //   user: NODE_USER,
+  // })
+  //
+  // logger.info('Removing workspace dependency...')
+  // await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
+  //   user: NODE_USER,
+  // })
+  //
+  // logger.info('Installing dependencies...')
+  // await sandbox.runCommand('npm', ['install'], {
+  //   user: NODE_USER,
+  // })
+  //
+  // if (trial.setup) {
+  //   logger.info('Running generic setup...')
+  //   await trial.setup({
+  //     sandbox,
+  //   })
+  // }
+  //
+  // if (trial.treatment.setup) {
+  //   logger.info('Running treatment setup...')
+  //   await trial.treatment.setup({
+  //     sandbox,
+  //   })
+  // }
+  //
+  // logger.info('Run build script...')
+  // await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
+  //   user: NODE_USER,
+  // })
+  //
+  // logger.info('Running copilot...')
+  // const copilotOutput = await sandbox.runCommand(
+  //   'copilot',
+  //   ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
+  //   {
+  //     user: NODE_USER,
+  //     env: {
+  //       COPILOT_GITHUB_TOKEN: copilotToken,
+  //     },
+  //   },
+  // )
+  // const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
+  //   const trimmed = line.trim()
+  //   if (trimmed.length === 0) {
+  //     return []
+  //   }
+  //   return parseMessage(JSON.parse(trimmed))
+  // })
+  //
+  // logger.info('Running tests...')
+  //
   // const TEST_PATH = 'scenario.test.ts'
   // const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
   // const TEST_RESULTS_PATH = 'test-results.json'
diff --git a/packages/agent-eval/src/sandbox.test.ts b/packages/agent-eval/src/sandbox-old.test.ts
similarity index 100%
rename from packages/agent-eval/src/sandbox.test.ts
rename to packages/agent-eval/src/sandbox-old.test.ts
diff --git a/packages/agent-eval/src/sandbox.ts b/packages/agent-eval/src/sandbox-old.ts
similarity index 100%
rename from packages/agent-eval/src/sandbox.ts
rename to packages/agent-eval/src/sandbox-old.ts
diff --git a/packages/agent-eval/src/sandbox/constants.ts b/packages/agent-eval/src/sandbox/constants.ts
new file mode 100644
index 00000000..e9b4d48a
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/constants.ts
@@ -0,0 +1,68 @@
+import path from 'node:path'
+
+/**
+ * Working directory inside the sandbox.
+ */
+const CONTAINER_WORKDIR = '/home/sandbox/workspace'
+
+/**
+ * Directory for Copilot CLI configuration.
+ */
+const COPILOT_DIR = '/home/node/.copilot'
+
+/**
+ * Directory for custom Copilot agents.
+ */
+const CUSTOM_AGENTS_DIR = '/home/node/.copilot/agents'
+
+/**
+ * Directory for agent configuration and skills.
+ */
+const AGENTS_DIR = '/home/node/.agents'
+
+/**
+ * Directory for agent skills.
+ */
+const SKILLS_DIR = '/home/node/.agents/skills'
+
+/**
+ * Directory for local plugin sources copied into the sandbox.
+ */
+const COPILOT_PLUGIN_SOURCES_DIR = path.posix.join(COPILOT_DIR, 'plugin-sources')
+
+/**
+ * Path for project agent instructions.
+ */
+const AGENT_INSTRUCTIONS_PATH = path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md')
+
+/**
+ * Path for MCP server configuration.
+ */
+const MCP_CONFIG_PATH = path.posix.join(COPILOT_DIR, 'mcp-config.json')
+
+/**
+ * Node.js images provide a non-root node user with this UID and GID.
+ */
+const SANDBOX_UID = 1000
+const SANDBOX_GID = 1000
+const NODE_USER = `${SANDBOX_UID}:${SANDBOX_GID}` as const
+
+/**
+ * Directory for npm packages installed globally by the non-root user.
+ */
+const NPM_GLOBAL_DIR = '/home/node/.npm-global'
+
+export {
+  AGENT_INSTRUCTIONS_PATH,
+  AGENTS_DIR,
+  CONTAINER_WORKDIR,
+  COPILOT_DIR,
+  COPILOT_PLUGIN_SOURCES_DIR,
+  CUSTOM_AGENTS_DIR,
+  MCP_CONFIG_PATH,
+  NODE_USER,
+  NPM_GLOBAL_DIR,
+  SANDBOX_GID,
+  SANDBOX_UID,
+  SKILLS_DIR,
+}
diff --git a/packages/agent-eval/src/sandbox/index.ts b/packages/agent-eval/src/sandbox/index.ts
new file mode 100644
index 00000000..f33fb11c
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/index.ts
@@ -0,0 +1,25 @@
+export * from './constants'
+export {SandboxSchema, SystemSandbox} from './system'
+export {VirtualSandbox} from './virtual'
+export type {
+  AgentSkillCopiedFile,
+  AgentSkillFile,
+  AgentSkillOptions,
+  AgentSkillWrittenFile,
+  CommandResult,
+  CopilotPluginConfig,
+  CopilotPluginSource,
+  CopyOptions,
+  CustomAgentCopiedFile,
+  CustomAgentFile,
+  CustomAgentOptions,
+  CustomAgentWrittenFile,
+  DownloadOptions,
+  LocalCopilotPluginSource,
+  McpServerConfig,
+  RemoteCopilotPluginSource,
+  RunOptions,
+  Sandbox,
+  SandboxConstructor,
+  SandboxCreateOptions,
+} from './types'
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
new file mode 100644
index 00000000..1eaf1715
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -0,0 +1,687 @@
+import {randomUUID} from 'node:crypto'
+import path from 'node:path'
+import {Writable} from 'node:stream'
+import {pipeline} from 'node:stream/promises'
+import Docker from 'dockerode'
+import tarFs from 'tar-fs'
+import type {Headers} from 'tar-fs'
+import tarStream from 'tar-stream'
+import * as z from 'zod/mini'
+import {McpConfigFileSchema} from '../mcp-config'
+import type {McpConfigFile} from '../mcp-config'
+import {
+  AGENT_INSTRUCTIONS_PATH,
+  AGENTS_DIR,
+  CONTAINER_WORKDIR,
+  COPILOT_DIR,
+  COPILOT_PLUGIN_SOURCES_DIR,
+  CUSTOM_AGENTS_DIR,
+  MCP_CONFIG_PATH,
+  NODE_USER,
+  NPM_GLOBAL_DIR,
+  SANDBOX_GID,
+  SANDBOX_UID,
+  SKILLS_DIR,
+} from './constants'
+import type {
+  AgentSkillFile,
+  AgentSkillOptions,
+  AgentSkillWrittenFile,
+  CommandResult,
+  CopilotPluginConfig,
+  CopilotPluginSource,
+  CopyOptions,
+  CustomAgentFile,
+  CustomAgentOptions,
+  CustomAgentWrittenFile,
+  DownloadOptions,
+  McpServerConfig,
+  RunOptions,
+  Sandbox,
+  SandboxCreateOptions,
+} from './types'
+import {DefaultHost, type Host} from '../host'
+
+const COPILOT_CLI_VERSION = '1.0.80'
+const NPM_VERSION = '12.0.2'
+
+const DEFAULT_MCP_CONFIG: McpConfigFile = {
+  mcpServers: {},
+}
+
+class SystemSandbox implements Sandbox {
+  static async create(options: SandboxCreateOptions = {}) {
+    const docker = new Docker()
+    const dockerImage = options.dockerImage?.trim() || DEFAULT_DOCKER_IMAGE
+    const container = await createContainer(docker, dockerImage)
+    return new SystemSandbox(options.host ?? DefaultHost, docker, container)
+  }
+
+  #container: Docker.Container
+  #docker: Docker
+  #host: Host
+
+  constructor(host: Host, docker: Docker, container: InitializedContainer) {
+    this.#host = host
+    this.#docker = docker
+    this.#container = container
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.#container.stop()
+  }
+
+  async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise<void> {
+    const source = path.resolve(sourcePath)
+    const sourceStats = await this.#host.fs.stat(source)
+    if (!sourceStats.isDirectory() && !sourceStats.isFile()) {
+      throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`)
+    }
+
+    const containerPath = resolveContainerPath(destinationPath)
+    const containerDirectory = path.posix.dirname(containerPath)
+    const destinationName = path.posix.basename(containerPath)
+    if (!destinationName) {
+      throw new Error(`Cannot copy "${sourcePath}" to "${destinationPath}" because the destination must include a name`)
+    }
+
+    await execCommand(this.#docker, this.#container, 'mkdir', ['-p', containerDirectory], {
+      user: NODE_USER,
+    })
+
+    const sourceDirectory = path.dirname(source)
+    const sourceName = path.basename(source)
+    const excludedPaths = new Set(options.exclude?.map(filepath => normalizeExcludedPath(filepath, source)))
+    const archive = tarFs.pack(sourceDirectory, {
+      entries: [sourceName],
+      ignore(name) {
+        const absolutePath = path.isAbsolute(name) ? name : path.resolve(sourceDirectory, name)
+        const relativePath = normalizeCopyPath(path.relative(source, absolutePath))
+        return isExcluded(relativePath, excludedPaths)
+      },
+      map(header) {
+        return mapCopiedHeader(header, sourceName, destinationName)
+      },
+    })
+
+    await this.#container.putArchive(archive, {
+      path: containerDirectory,
+    })
+  }
+
+  async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise<void> {
+    await this.#host.fs.mkdir(hostDestinationPath, {
+      recursive: true,
+    })
+
+    const archive = await this.#container.getArchive({
+      path: containerFilePath,
+    })
+    const sourceName = path.posix.basename(containerFilePath)
+
+    await pipeline(
+      archive,
+      tarFs.extract(hostDestinationPath, {
+        readable: true,
+        writable: true,
+        map(header) {
+          const prefix = `${sourceName}/`
+
+          if (header.name === sourceName) {
+            header.name = '.'
+          } else if (header.name.startsWith(prefix)) {
+            header.name = header.name.slice(prefix.length)
+          }
+
+          return header
+        },
+        ignore: options.ignore,
+      }),
+    )
+  }
+
+  async readFile(filepath: string): Promise<string> {
+    const archive = await this.#container.getArchive({
+      path: resolveContainerPath(filepath),
+    })
+    const buffer = await readFileFromArchive(archive)
+    return buffer.toString('utf8')
+  }
+
+  async writeFile(filepath: string, contents: string): Promise<void> {
+    const containerPath = resolveContainerPath(filepath)
+    const directory = path.dirname(containerPath)
+    const name = path.basename(containerPath)
+    const pack = tarStream.pack()
+    const upload = this.#container.putArchive(pack, {
+      path: directory,
+    })
+
+    pack.entry(
+      {
+        name,
+        mode: 0o644,
+        size: Buffer.byteLength(contents),
+        uid: SANDBOX_UID,
+        gid: SANDBOX_GID,
+      },
+      contents,
+    )
+    pack.finalize()
+
+    await upload
+  }
+
+  async exists(filepath: string): Promise<boolean> {
+    const result = await execCommand(this.#docker, this.#container, 'test', ['-e', resolveContainerPath(filepath)], {
+      user: NODE_USER,
+      allowNonZeroExitCode: true,
+    })
+
+    return result.exitCode === 0
+  }
+
+  async runCommand(command: string, args: Array<string> = [], options?: RunOptions): Promise<CommandResult> {
+    return execCommand(this.#docker, this.#container, command, args, {
+      env: {
+        HOME: options?.user === 'root' ? '/root' : '/home/node',
+        ...options?.env,
+        PATH: `${NPM_GLOBAL_DIR}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
+      },
+      user: options?.user ?? NODE_USER,
+      allowNonZeroExitCode: options?.allowNonZeroExitCode,
+    })
+  }
+
+  async addAgentInstruction(text: string): Promise<void> {
+    const contents = await this.#findOrCreateFile(AGENT_INSTRUCTIONS_PATH)
+    await this.writeFile(AGENT_INSTRUCTIONS_PATH, appendText(contents, text))
+  }
+
+  async addAgentSkill(
+    name: string,
+    description: string,
+    contents: string,
+    options: AgentSkillOptions = {},
+  ): Promise<void> {
+    assertValidSkillName(name)
+
+    const skillDirectory = path.posix.join(SKILLS_DIR, name)
+    const skillPath = path.posix.join(skillDirectory, 'SKILL.md')
+    if (await this.exists(skillPath)) {
+      throw new Error(`Agent skill with name "${name}" already exists`)
+    }
+
+    await this.runCommand('mkdir', ['-p', skillDirectory])
+    await this.writeFile(skillPath, createSkillContents(name, description, contents))
+
+    for (const file of options.files ?? []) {
+      const destinationPath = path.posix.join(skillDirectory, getAgentSkillFileDestination(file))
+
+      if (isWrittenFile(file)) {
+        await this.runCommand('mkdir', ['-p', path.posix.dirname(destinationPath)])
+        await this.writeFile(destinationPath, file.content)
+      } else {
+        await this.copy(file.sourcePath, destinationPath)
+      }
+    }
+  }
+
+  async addCustomAgent(
+    name: string,
+    description: string,
+    contents: string,
+    options: CustomAgentOptions = {},
+  ): Promise<void> {
+    assertValidCustomAgentName(name)
+
+    const agentPath = path.posix.join(CUSTOM_AGENTS_DIR, `${name}.agent.md`)
+    if (await this.exists(agentPath)) {
+      throw new Error(`Custom agent with name "${name}" already exists`)
+    }
+
+    await this.runCommand('mkdir', ['-p', CUSTOM_AGENTS_DIR])
+    await this.writeFile(agentPath, createCustomAgentContents(name, description, contents, options))
+
+    for (const file of options.files ?? []) {
+      const destinationPath = path.posix.join(CUSTOM_AGENTS_DIR, getCustomAgentFileDestination(file))
+
+      if (isWrittenFile(file)) {
+        await this.runCommand('mkdir', ['-p', path.posix.dirname(destinationPath)])
+        await this.writeFile(destinationPath, file.content)
+      } else {
+        await this.copy(file.sourcePath, destinationPath)
+      }
+    }
+  }
+
+  async addMcpServer(name: string, config: McpServerConfig): Promise<void> {
+    const contents = await this.readFile(MCP_CONFIG_PATH)
+    const mcpConfig = contents === '' ? DEFAULT_MCP_CONFIG : McpConfigFileSchema.parse(JSON.parse(contents))
+    if (mcpConfig.mcpServers[name]) {
+      throw new Error(`MCP server with name "${name}" already exists`)
+    }
+
+    const updatedConfig: McpConfigFile = {
+      ...mcpConfig,
+      mcpServers: {
+        ...mcpConfig.mcpServers,
+        [name]: config,
+      },
+    }
+
+    await this.writeFile(MCP_CONFIG_PATH, JSON.stringify(updatedConfig, null, 2))
+    await this.runCommand('chown', ['-R', NODE_USER, MCP_CONFIG_PATH], {
+      user: 'root',
+    })
+  }
+
+  async addCopilotPlugin(config: CopilotPluginConfig): Promise<void> {
+    if (config.type === 'marketplace') {
+      const marketplaceSource = await this.#prepareCopilotPluginSource(config.marketplace.source)
+      await this.runCommand('copilot', ['plugin', 'marketplace', 'add', marketplaceSource])
+      await this.runCommand('copilot', ['plugin', 'install', `${config.name}@${config.marketplace.name}`])
+      return
+    }
+
+    const source = await this.#prepareCopilotPluginSource(config)
+    await this.runCommand('copilot', ['plugin', 'install', source])
+  }
+
+  async #prepareCopilotPluginSource(source: CopilotPluginSource): Promise<string> {
+    if (source.type === 'remote') {
+      return source.version ? `${source.url}#${source.version}` : source.url
+    }
+
+    const destinationPath = path.posix.join(COPILOT_PLUGIN_SOURCES_DIR, randomUUID())
+    await this.copy(source.sourcePath, destinationPath)
+    return destinationPath
+  }
+
+  async #findOrCreateFile(filepath: string): Promise<string> {
+    if (await this.exists(filepath)) {
+      return this.readFile(filepath)
+    }
+
+    await this.writeFile(filepath, '')
+    return ''
+  }
+}
+
+const INITIALIZED_CONTAINER: unique symbol = Symbol('InitializedContainer')
+
+const DEFAULT_DOCKER_IMAGE = 'node:26.5.0-slim'
+
+type InitializedContainer = Docker.Container & {
+  readonly [INITIALIZED_CONTAINER]?: true
+}
+
+async function createContainer(docker: Docker, dockerImage: string): Promise<InitializedContainer> {
+  await pullImage(docker, dockerImage)
+
+  const container = await docker.createContainer({
+    Image: dockerImage,
+    Cmd: ['sleep', 'infinity'],
+    WorkingDir: CONTAINER_WORKDIR,
+    Tty: true,
+    HostConfig: {
+      AutoRemove: true,
+    },
+  })
+
+  await container.start()
+
+  console.log('Creating workspace directory...')
+  await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'chown', ['-R', NODE_USER, CONTAINER_WORKDIR], {
+    user: 'root',
+  })
+
+  console.log('Installing CA certificates...')
+  await execCommand(docker, container, 'apt-get', ['update'], {
+    user: 'root',
+  })
+  await execCommand(
+    docker,
+    container,
+    'apt-get',
+    ['install', '-y', '--no-install-recommends', 'ca-certificates', 'curl'],
+    {
+      user: 'root',
+    },
+  )
+  await execCommand(docker, container, 'test', ['-d', '/etc/ssl/certs'], {
+    user: 'root',
+  })
+
+  console.log('Installing npm...')
+  await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
+    user: 'root',
+  })
+  const npmVersion = await execCommand(docker, container, 'npm', ['--version'], {
+    user: 'root',
+  })
+  if (npmVersion.stdout.trim() !== NPM_VERSION) {
+    throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
+  }
+
+  console.log('Setting up npm for non-root global installs')
+  await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'chown', ['-R', NODE_USER, NPM_GLOBAL_DIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'npm', ['config', 'set', 'prefix', NPM_GLOBAL_DIR], {
+    user: NODE_USER,
+  })
+
+  console.log('Setting up copilot...')
+  await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'chown', ['-R', NODE_USER, COPILOT_DIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'npm', ['install', '-g', `@github/copilot@${COPILOT_CLI_VERSION}`], {
+    user: NODE_USER,
+  })
+  await execCommand(docker, container, 'touch', [path.join(COPILOT_DIR, 'mcp-config.json')], {
+    user: NODE_USER,
+  })
+  await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], {
+    user: NODE_USER,
+  })
+
+  console.log('Setting up agents config...')
+  await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
+    user: 'root',
+  })
+  await execCommand(docker, container, 'chown', ['-R', NODE_USER, AGENTS_DIR], {
+    user: 'root',
+  })
+
+  return container as InitializedContainer
+}
+
+function resolveContainerPath(filepath: string): string {
+  if (path.posix.isAbsolute(filepath)) {
+    return filepath
+  }
+
+  return path.posix.join(CONTAINER_WORKDIR, filepath)
+}
+
+function mapCopiedHeader(header: Headers, sourceName: string, destinationName: string): Headers {
+  const name =
+    header.name === sourceName
+      ? destinationName
+      : path.posix.join(destinationName, header.name.slice(sourceName.length))
+
+  return {
+    ...header,
+    name,
+    uid: SANDBOX_UID,
+    gid: SANDBOX_GID,
+  }
+}
+
+function normalizeCopyPath(filepath: string): string {
+  const normalized = path.posix.normalize(filepath.split(path.sep).join(path.posix.sep))
+  if (normalized === '.') {
+    return ''
+  }
+
+  return normalized.replace(/\/$/, '')
+}
+
+function normalizeExcludedPath(filepath: string, source: string): string {
+  if (path.isAbsolute(filepath)) {
+    return normalizeCopyPath(path.relative(source, filepath))
+  }
+
+  return normalizeCopyPath(filepath)
+}
+
+function isExcluded(relativePath: string, excludedPaths: ReadonlySet<string>): boolean {
+  if (!relativePath) {
+    return false
+  }
+
+  for (const excludedPath of excludedPaths) {
+    if (relativePath === excludedPath || relativePath.startsWith(`${excludedPath}/`)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+function appendText(contents: string, text: string): string {
+  const suffix = ensureTrailingNewline(text)
+  if (contents.length === 0) {
+    return suffix
+  }
+
+  if (contents.endsWith('\n')) {
+    return `${contents}${suffix}`
+  }
+
+  return `${contents}\n${suffix}`
+}
+
+function ensureTrailingNewline(text: string): string {
+  return text.endsWith('\n') ? text : `${text}\n`
+}
+
+function assertValidSkillName(name: string): void {
+  if (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) {
+    return
+  }
+
+  throw new Error(`Invalid agent skill name "${name}". Skill names must be lowercase and use hyphens for spaces.`)
+}
+
+function assertValidCustomAgentName(name: string): void {
+  if (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) {
+    return
+  }
+
+  throw new Error(
+    `Invalid custom agent name "${name}". Custom agent names must be lowercase and use hyphens for spaces.`,
+  )
+}
+
+function createSkillContents(name: string, description: string, contents: string): string {
+  return `---
+name: ${JSON.stringify(name)}
+description: ${JSON.stringify(description)}
+---
+
+${ensureTrailingNewline(contents)}`
+}
+
+function createCustomAgentContents(
+  name: string,
+  description: string,
+  contents: string,
+  options: CustomAgentOptions,
+): string {
+  const tools = options.tools ? `tools: ${JSON.stringify(options.tools)}\n` : ''
+
+  return `---
+name: ${JSON.stringify(name)}
+description: ${JSON.stringify(description)}
+${tools}---
+
+${ensureTrailingNewline(contents)}`
+}
+
+function getCustomAgentFileDestination(file: CustomAgentFile): string {
+  return getAdditionalFileDestination(file, 'custom agent')
+}
+
+function getAgentSkillFileDestination(file: AgentSkillFile): string {
+  return getAdditionalFileDestination(file, 'agent skill')
+}
+
+function getAdditionalFileDestination(file: CustomAgentFile | AgentSkillFile, fileKind: string): string {
+  const destinationPath = isWrittenFile(file) ? file.path : (file.destinationPath ?? path.basename(file.sourcePath))
+  const normalized = normalizeCopyPath(destinationPath)
+
+  if (!normalized || path.posix.isAbsolute(normalized) || normalized === '..' || normalized.startsWith('../')) {
+    throw new Error(`Invalid ${fileKind} file destination "${destinationPath}"`)
+  }
+
+  return normalized
+}
+
+function isWrittenFile(file: CustomAgentFile | AgentSkillFile): file is CustomAgentWrittenFile | AgentSkillWrittenFile {
+  return 'content' in file
+}
+
+async function readFileFromArchive(archive: NodeJS.ReadableStream): Promise<Buffer> {
+  const extract = tarStream.extract()
+
+  return new Promise((resolve, reject) => {
+    const chunks: Array<Buffer> = []
+
+    extract.on('entry', (header, stream, next) => {
+      if (header.type !== 'file') {
+        stream.resume()
+        next()
+        return
+      }
+
+      stream.on('data', (chunk: Buffer) => {
+        chunks.push(chunk)
+      })
+
+      stream.on('end', () => {
+        resolve(Buffer.concat(chunks))
+        next()
+      })
+
+      stream.on('error', reject)
+    })
+
+    extract.on('error', reject)
+    archive.on('error', reject)
+
+    archive.pipe(extract)
+  })
+}
+
+function pullImage(docker: Docker, name: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    docker.pull(name, (error: Error | null, stream: NodeJS.ReadableStream) => {
+      if (error) {
+        reject(error)
+        return
+      }
+
+      // Follow the pull progress
+      docker.modem.followProgress(
+        stream,
+        (progressError: Error | null) => {
+          if (progressError) {
+            reject(progressError)
+          } else {
+            resolve()
+          }
+        },
+        () => {},
+      )
+    })
+  })
+}
+
+class CommandError extends Error {
+  command: ReadonlyArray<string>
+  result: CommandResult
+
+  constructor(command: ReadonlyArray<string>, result: CommandResult) {
+    super(`Command failed with exit code ${result.exitCode}: ${command.join(' ')}`)
+    this.name = 'CommandError'
+    this.command = command
+    this.result = result
+  }
+}
+
+async function execCommand(
+  docker: Docker,
+  container: Docker.Container,
+  command: string,
+  args: Array<string>,
+  options: RunOptions,
+): Promise<CommandResult> {
+  const cmd = [command, ...args]
+  const env = options.env ? Object.entries(options.env).map(([key, value]) => `${key}=${value}`) : undefined
+  const exec = await container.exec({
+    Cmd: cmd,
+    AttachStdout: true,
+    AttachStderr: true,
+    WorkingDir: CONTAINER_WORKDIR,
+    Env: env,
+    User: options.user,
+  })
+
+  const stream = await exec.start({
+    hijack: true,
+    stdin: false,
+  })
+
+  return new Promise((resolve, reject) => {
+    const stdout = captureStream(process.stdout)
+    const stderr = captureStream(process.stderr)
+
+    docker.modem.demuxStream(stream, stdout.stream, stderr.stream)
+
+    stream.on('end', async () => {
+      try {
+        const inspectInfo = await exec.inspect()
+        const exitCode = inspectInfo.ExitCode ?? 0
+        const result = {
+          stdout: stdout.read(),
+          stderr: stderr.read(),
+          exitCode,
+        }
+
+        if (exitCode === 0 || options.allowNonZeroExitCode) {
+          resolve(result)
+          return
+        }
+
+        reject(new CommandError(cmd, result))
+      } catch (error) {
+        reject(error)
+      }
+    })
+    stream.on('error', reject)
+  })
+}
+
+function captureStream(destination: NodeJS.WritableStream): {stream: Writable; read(): string} {
+  const chunks: Array<Buffer> = []
+  const stream = new Writable({
+    write(chunk: Buffer | string, encoding, callback) {
+      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)
+      chunks.push(buffer)
+      destination.write(buffer)
+      callback()
+    },
+  })
+
+  return {
+    stream,
+    read() {
+      return Buffer.concat(chunks).toString('utf8')
+    },
+  }
+}
+
+const SandboxSchema = z.instanceof(SystemSandbox)
+
+export {SandboxSchema, SystemSandbox}
diff --git a/packages/agent-eval/src/sandbox/types.ts b/packages/agent-eval/src/sandbox/types.ts
new file mode 100644
index 00000000..6fb23d10
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/types.ts
@@ -0,0 +1,170 @@
+import type {Host} from '../host'
+import type {McpServerConfig} from '../mcp-config'
+
+type RunOptions = {
+  env?: Record<string, string>
+  user?: string
+  allowNonZeroExitCode?: boolean
+}
+
+type CopyOptions = {
+  exclude?: Array<string>
+}
+
+type DownloadOptions = {
+  ignore?: (name: string) => boolean
+}
+
+type CommandResult = {
+  stdout: string
+  stderr: string
+  exitCode: number
+}
+
+type SandboxCreateOptions = {
+  dockerImage?: string
+  host?: Host
+}
+
+type CustomAgentCopiedFile = {
+  sourcePath: string
+  destinationPath?: string
+}
+
+type CustomAgentWrittenFile = {
+  path: string
+  content: string
+}
+
+type CustomAgentFile = CustomAgentCopiedFile | CustomAgentWrittenFile
+
+type AgentSkillCopiedFile = CustomAgentCopiedFile
+
+type AgentSkillWrittenFile = CustomAgentWrittenFile
+
+type AgentSkillFile = AgentSkillCopiedFile | AgentSkillWrittenFile
+
+type AgentSkillOptions = {
+  files?: Array<AgentSkillFile>
+}
+
+type RemoteCopilotPluginSource = {
+  type: 'remote'
+  url: string
+  version?: string
+}
+
+type LocalCopilotPluginSource = {
+  type: 'local'
+  sourcePath: string
+}
+
+type CopilotPluginSource = RemoteCopilotPluginSource | LocalCopilotPluginSource
+
+type CopilotPluginConfig =
+  | CopilotPluginSource
+  | {
+      type: 'marketplace'
+      name: string
+      marketplace: {
+        name: string
+        source: CopilotPluginSource
+      }
+    }
+
+type CustomAgentOptions = {
+  files?: Array<CustomAgentFile>
+  tools?: Array<string>
+}
+
+interface Sandbox {
+  /**
+   * Stops and removes the sandbox container.
+   */
+  [Symbol.asyncDispose](): Promise<void>
+
+  /**
+   * Copies a host file or directory into the sandbox.
+   */
+  copy(sourcePath: string, destinationPath: string, options?: CopyOptions): Promise<void>
+
+  /**
+   * Downloads a file or directory from the sandbox to the host.
+   */
+  download(containerFilePath: string, hostDestinationPath: string, options?: DownloadOptions): Promise<void>
+
+  /**
+   * Reads a UTF-8 file from the sandbox.
+   */
+  readFile(filepath: string): Promise<string>
+
+  /**
+   * Writes a UTF-8 file to the sandbox.
+   */
+  writeFile(filepath: string, contents: string): Promise<void>
+
+  /**
+   * Checks whether a file or directory exists in the sandbox.
+   */
+  exists(filepath: string): Promise<boolean>
+
+  /**
+   * Runs a command in the sandbox and captures its output and exit code.
+   */
+  runCommand(command: string, args?: Array<string>, options?: RunOptions): Promise<CommandResult>
+
+  /**
+   * Appends instructions to the sandbox's project-level AGENTS.md file.
+   */
+  addAgentInstruction(text: string): Promise<void>
+
+  /**
+   * Adds an agent skill and its supporting files to the sandbox.
+   */
+  addAgentSkill(name: string, description: string, contents: string, options?: AgentSkillOptions): Promise<void>
+
+  /**
+   * Adds a custom agent and its supporting files to the sandbox.
+   */
+  addCustomAgent(name: string, description: string, contents: string, options?: CustomAgentOptions): Promise<void>
+
+  /**
+   * Adds an MCP server to the sandbox's Copilot configuration.
+   */
+  addMcpServer(name: string, config: McpServerConfig): Promise<void>
+
+  /**
+   * Installs a remote, local, or marketplace Copilot plugin in the sandbox.
+   */
+  addCopilotPlugin(config: CopilotPluginConfig): Promise<void>
+}
+
+interface SandboxConstructor {
+  /**
+   * Creates a sandbox using the requested runtime options.
+   */
+  create(options?: SandboxCreateOptions): Promise<Sandbox>
+}
+
+export type {
+  AgentSkillCopiedFile,
+  AgentSkillFile,
+  AgentSkillOptions,
+  AgentSkillWrittenFile,
+  CommandResult,
+  CopilotPluginConfig,
+  CopilotPluginSource,
+  CopyOptions,
+  CustomAgentCopiedFile,
+  CustomAgentFile,
+  CustomAgentOptions,
+  CustomAgentWrittenFile,
+  DownloadOptions,
+  LocalCopilotPluginSource,
+  McpServerConfig,
+  RemoteCopilotPluginSource,
+  RunOptions,
+  Sandbox,
+  SandboxConstructor,
+  SandboxCreateOptions,
+}
diff --git a/packages/agent-eval/src/sandbox/virtual.test.ts b/packages/agent-eval/src/sandbox/virtual.test.ts
new file mode 100644
index 00000000..23a847fd
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/virtual.test.ts
@@ -0,0 +1,58 @@
+import path from 'node:path'
+import {describe, expect, test} from 'vitest'
+import {VirtualHost} from '../host'
+import {CONTAINER_WORKDIR} from './constants'
+import {VirtualSandbox} from './virtual'
+
+describe('VirtualSandbox', () => {
+  test('reads, writes, and checks files in the sandbox workspace', async () => {
+    const host = VirtualHost.create()
+    const sandbox = await VirtualSandbox.create({host})
+
+    expect(await sandbox.exists('nested/example.txt')).toBe(false)
+
+    await sandbox.writeFile('nested/example.txt', 'example')
+
+    expect(await sandbox.exists('nested/example.txt')).toBe(true)
+    expect(await sandbox.readFile('nested/example.txt')).toBe('example')
+    expect(await host.fs.readFile(path.join(CONTAINER_WORKDIR, 'nested/example.txt'), 'utf8')).toBe('example')
+  })
+
+  test('copies host directories into the sandbox with exclusions', async () => {
+    const host = VirtualHost.create({
+      '/fixture/included.txt': 'included',
+      '/fixture/nested/included.txt': 'nested',
+      '/fixture/nested/excluded.txt': 'excluded',
+    })
+    const sandbox = await VirtualSandbox.create({host})
+
+    await sandbox.copy('/fixture', 'copied', {
+      exclude: ['nested/excluded.txt'],
+    })
+
+    expect(await sandbox.readFile('copied/included.txt')).toBe('included')
+    expect(await sandbox.readFile('copied/nested/included.txt')).toBe('nested')
+    expect(await sandbox.exists('copied/nested/excluded.txt')).toBe(false)
+  })
+
+  test('downloads sandbox directories to the host with ignored files', async () => {
+    const host = VirtualHost.create()
+    const sandbox = await VirtualSandbox.create({host})
+
+    await sandbox.writeFile('results/included.txt', 'included')
+    await sandbox.writeFile('results/nested/included.txt', 'nested')
+    await sandbox.writeFile('results/nested/ignored.txt', 'ignored')
+
+    await sandbox.download('results', '/download', {
+      ignore(name) {
+        return name.endsWith('ignored.txt')
+      },
+    })
+
+    expect(await host.fs.readFile('/download/included.txt', 'utf8')).toBe('included')
+    expect(await host.fs.readFile('/download/nested/included.txt', 'utf8')).toBe('nested')
+    await expect(host.fs.access('/download/nested/ignored.txt')).rejects.toMatchObject({
+      code: 'ENOENT',
+    })
+  })
+})
diff --git a/packages/agent-eval/src/sandbox/virtual.ts b/packages/agent-eval/src/sandbox/virtual.ts
new file mode 100644
index 00000000..ee2034c5
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/virtual.ts
@@ -0,0 +1,226 @@
+import path from 'node:path'
+import {VirtualHost, type Host} from '../host'
+import type {McpServerConfig} from '../mcp-config'
+import {CONTAINER_WORKDIR} from './constants'
+import type {
+  AgentSkillOptions,
+  CommandResult,
+  CopilotPluginConfig,
+  CopyOptions,
+  CustomAgentOptions,
+  DownloadOptions,
+  RunOptions,
+  Sandbox,
+  SandboxCreateOptions,
+} from './types'
+
+const defaultCreateOptions: SandboxCreateOptions = {}
+
+export class VirtualSandbox implements Sandbox {
+  static async create(options: SandboxCreateOptions = defaultCreateOptions) {
+    return new VirtualSandbox(options.host ?? VirtualHost.create())
+  }
+
+  [Symbol.asyncDispose](): Promise<void> {
+    return Promise.resolve()
+  }
+
+  #host: Host
+
+  constructor(host: Host) {
+    this.#host = host
+  }
+
+  async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise<void> {
+    const source = path.resolve(sourcePath)
+    const sourceStats = await this.#host.fs.stat(source)
+    if (!sourceStats.isDirectory() && !sourceStats.isFile()) {
+      throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`)
+    }
+
+    const destination = resolveSandboxPath(destinationPath)
+    if (!path.posix.basename(destination)) {
+      throw new Error(`Cannot copy "${sourcePath}" to "${destinationPath}" because the destination must include a name`)
+    }
+
+    const excludedPaths = new Set(options.exclude?.map(filepath => normalizeExcludedPath(filepath, source)))
+    await copyPath(this.#host, source, destination, relativePath => {
+      return isExcluded(relativePath, excludedPaths)
+    })
+  }
+
+  async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise<void> {
+    const source = resolveSandboxPath(containerFilePath)
+    const destination = path.resolve(hostDestinationPath)
+    const sourceStats = await this.#host.fs.stat(source)
+
+    await this.#host.fs.mkdir(destination, {
+      recursive: true,
+    })
+
+    if (sourceStats.isDirectory()) {
+      const entries = await this.#host.fs.readdir(source)
+      for (const entry of entries) {
+        const name = entry.toString()
+        if (options.ignore?.(name)) {
+          continue
+        }
+
+        await copyPath(this.#host, path.join(source, name), path.join(destination, name), relativePath => {
+          return options.ignore?.(path.posix.join(name, relativePath)) ?? false
+        })
+      }
+      return
+    }
+
+    const name = path.basename(source)
+    if (!options.ignore?.(name)) {
+      await copyPath(this.#host, source, path.join(destination, name))
+    }
+  }
+
+  readFile(filepath: string): Promise<string> {
+    return this.#host.fs.readFile(resolveSandboxPath(filepath), 'utf8')
+  }
+
+  async writeFile(filepath: string, contents: string): Promise<void> {
+    const destination = resolveSandboxPath(filepath)
+    await this.#host.fs.mkdir(path.dirname(destination), {
+      recursive: true,
+    })
+    await this.#host.fs.writeFile(destination, contents, 'utf8')
+  }
+
+  async exists(filepath: string): Promise<boolean> {
+    try {
+      await this.#host.fs.access(resolveSandboxPath(filepath))
+      return true
+    } catch (error) {
+      if (isErrorWithCode(error, 'ENOENT')) {
+        return false
+      }
+
+      throw error
+    }
+  }
+
+  runCommand(command: string, args?: Array<string>, options?: RunOptions): Promise<CommandResult> {
+    void command
+    void args
+    void options
+    throw new Error('Method not implemented.')
+  }
+
+  addAgentInstruction(text: string): Promise<void> {
+    void text
+    throw new Error('Method not implemented.')
+  }
+
+  addAgentSkill(name: string, description: string, contents: string, options?: AgentSkillOptions): Promise<void> {
+    void name
+    void description
+    void contents
+    void options
+    throw new Error('Method not implemented.')
+  }
+
+  addCustomAgent(name: string, description: string, contents: string, options?: CustomAgentOptions): Promise<void> {
+    void name
+    void description
+    void contents
+    void options
+    throw new Error('Method not implemented.')
+  }
+
+  addMcpServer(name: string, config: McpServerConfig): Promise<void> {
+    void name
+    void config
+    throw new Error('Method not implemented.')
+  }
+
+  addCopilotPlugin(config: CopilotPluginConfig): Promise<void> {
+    void config
+    throw new Error('Method not implemented.')
+  }
+}
+
+function resolveSandboxPath(filepath: string): string {
+  if (path.posix.isAbsolute(filepath)) {
+    return filepath
+  }
+
+  return path.posix.join(CONTAINER_WORKDIR, filepath)
+}
+
+async function copyPath(
+  host: Host,
+  source: string,
+  destination: string,
+  ignore: (relativePath: string) => boolean = () => {
+    return false
+  },
+  root: string = source,
+): Promise<void> {
+  const relativePath = normalizeCopyPath(path.relative(root, source))
+  if (ignore(relativePath)) {
+    return
+  }
+
+  const stats = await host.fs.stat(source)
+  if (stats.isDirectory()) {
+    await host.fs.mkdir(destination, {
+      recursive: true,
+    })
+
+    const entries = await host.fs.readdir(source)
+    for (const entry of entries) {
+      const name = entry.toString()
+      await copyPath(host, path.join(source, name), path.join(destination, name), ignore, root)
+    }
+    return
+  }
+
+  if (!stats.isFile()) {
+    throw new Error(`Cannot copy "${source}" because it is not a file or directory`)
+  }
+
+  await host.fs.mkdir(path.dirname(destination), {
+    recursive: true,
+  })
+  await host.fs.copyFile(source, destination)
+}
+
+function normalizeCopyPath(filepath: string): string {
+  const normalized = path.posix.normalize(filepath.split(path.sep).join(path.posix.sep))
+  if (normalized === '.') {
+    return ''
+  }
+
+  return normalized.replace(/\/$/, '')
+}
+
+function normalizeExcludedPath(filepath: string, source: string): string {
+  if (path.isAbsolute(filepath)) {
+    return normalizeCopyPath(path.relative(source, filepath))
+  }
+
+  return normalizeCopyPath(filepath)
+}
+
+function isExcluded(relativePath: string, excludedPaths: ReadonlySet<string>): boolean {
+  if (!relativePath) {
+    return false
+  }
+
+  for (const excludedPath of excludedPaths) {
+    if (relativePath === excludedPath || relativePath.startsWith(`${excludedPath}/`)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+function isErrorWithCode(error: unknown, code: string): boolean {
+  return error instanceof Error && 'code' in error && error.code === code
+}

From dc9807cfa0e6e300fb67a7d5cff7852bc7e3cc33 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Mon, 31 Aug 2026 17:16:22 -0500
Subject: [PATCH 13/52] feat: add sandbox interface for testing

---
 packages/agent-eval/src/logger.ts             |   2 +-
 packages/agent-eval/src/run.test.ts           |  50 +++--
 packages/agent-eval/src/run.ts                | 182 +++++++++---------
 packages/agent-eval/src/sandbox/path.ts       |  12 ++
 packages/agent-eval/src/sandbox/system.ts     |  20 +-
 .../agent-eval/src/sandbox/virtual.test.ts    |  52 ++++-
 packages/agent-eval/src/sandbox/virtual.ts    |  99 ++++------
 7 files changed, 238 insertions(+), 179 deletions(-)
 create mode 100644 packages/agent-eval/src/sandbox/path.ts

diff --git a/packages/agent-eval/src/logger.ts b/packages/agent-eval/src/logger.ts
index 92c572d1..a97ce54b 100644
--- a/packages/agent-eval/src/logger.ts
+++ b/packages/agent-eval/src/logger.ts
@@ -19,7 +19,7 @@ export const logger = pino(
     base: undefined,
     level: 'info',
     timestamp: false,
-    // enabled: process.env.NODE_ENV !== 'test',
+    enabled: process.env.NODE_ENV !== 'test',
   },
   ...stream,
 )
diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts
index ab6f31e4..f21967af 100644
--- a/packages/agent-eval/src/run.test.ts
+++ b/packages/agent-eval/src/run.test.ts
@@ -4,23 +4,43 @@ import {plan} from './plan'
 import {VirtualHost} from './host'
 import {VirtualSandbox} from './sandbox'
 
-test('hello', async () => {
-  const host = VirtualHost.create()
-  const sandbox = await VirtualSandbox.create({
-    host,
+test('runTrial', async () => {
+  const host = new VirtualHost({
+    '/test': {
+      'scenario.test.ts': '',
+    },
   })
-  const results = await run(host, sandbox, await plan([]), {
-    artifactsDirectory: 'test',
-    copilotToken: 'test',
-    maxConcurrency: 1,
+  await using sandbox = await VirtualSandbox.create({
+    host,
   })
-})
 
-test('runTrial', async () => {
-  const host = new VirtualHost()
-  const sandbox = await VirtualSandbox.create({
-    host,
+  sandbox.addCommandListener(async (cmd, args) => {
+    if (cmd !== 'sh') {
+      return
+    }
+
+    if (!Array.isArray(args)) {
+      return
+    }
+
+    if (args[0] !== '-c' || !args[1].startsWith('npx vitest run')) {
+      return
+    }
+
+    await sandbox.writeFile(
+      'test-results.json',
+      JSON.stringify({
+        numTotalTests: 0,
+        numPassedTests: 0,
+        numFailedTests: 0,
+        numPendingTests: 0,
+        numTodoTests: 0,
+        success: true,
+        testResults: [],
+      }),
+    )
   })
+
   const result = await runTrial(
     host,
     sandbox,
@@ -28,10 +48,10 @@ test('runTrial', async () => {
       id: 'test',
       scenario: {
         id: 'test',
-        directory: 'test',
+        directory: '/test',
         prompt: 'test',
         tags: [],
-        testPath: 'test',
+        testPath: '/test/scenario.test.ts',
       },
       treatment: {
         name: 'test',
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index 2c43645a..c5838a04 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -5,7 +5,7 @@ import type {Plan} from './plan'
 import {TrialSchema, type Trial} from './trial'
 import type {Host} from './host'
 // import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
-import type {Sandbox} from './sandbox'
+import {CONTAINER_WORKDIR, NODE_USER, type Sandbox} from './sandbox'
 import {parseTestResults} from './vitest'
 import {logger} from './logger'
 
@@ -85,98 +85,94 @@ type RunTrialOptions = {
 }
 
 async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
-  // const {artifactsDirectory, copilotToken, dockerImage} = options
-  //
-  // logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
-  //
-  // await using sandbox = await Sandbox.create({
-  //   dockerImage,
-  // })
-  //
-  // logger.info('Copying files from: %s...', trial.scenario.directory)
-  //
-  // await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
-  //   exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
-  // })
-  // await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
-  //   user: 'root',
-  // })
-  //
-  // logger.info('Obfuscating package name...')
-  // await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
-  //   user: NODE_USER,
-  // })
-  //
-  // logger.info('Removing workspace dependency...')
-  // await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
-  //   user: NODE_USER,
-  // })
-  //
-  // logger.info('Installing dependencies...')
-  // await sandbox.runCommand('npm', ['install'], {
-  //   user: NODE_USER,
-  // })
-  //
-  // if (trial.setup) {
-  //   logger.info('Running generic setup...')
-  //   await trial.setup({
-  //     sandbox,
-  //   })
-  // }
-  //
-  // if (trial.treatment.setup) {
-  //   logger.info('Running treatment setup...')
-  //   await trial.treatment.setup({
-  //     sandbox,
-  //   })
-  // }
-  //
-  // logger.info('Run build script...')
-  // await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
-  //   user: NODE_USER,
-  // })
-  //
-  // logger.info('Running copilot...')
-  // const copilotOutput = await sandbox.runCommand(
-  //   'copilot',
-  //   ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
-  //   {
-  //     user: NODE_USER,
-  //     env: {
-  //       COPILOT_GITHUB_TOKEN: copilotToken,
-  //     },
-  //   },
-  // )
-  // const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
-  //   const trimmed = line.trim()
-  //   if (trimmed.length === 0) {
-  //     return []
-  //   }
-  //   return parseMessage(JSON.parse(trimmed))
-  // })
-  //
-  // logger.info('Running tests...')
-  //
-  // const TEST_PATH = 'scenario.test.ts'
-  // const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
-  // const TEST_RESULTS_PATH = 'test-results.json'
-  //
-  // await sandbox.copy(trial.scenario.testPath, TEST_PATH)
-  // await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
-  // await sandbox.runCommand(
-  //   'sh',
-  //   ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
-  //   {
-  //     user: NODE_USER,
-  //     env: {},
-  //   },
-  // )
-  // const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
-  // const rawTestResult: unknown = JSON.parse(testResultsContent)
-  // const testResults = parseTestResults(rawTestResult)
-  // if (!testResults.success) {
-  //   throw new Error(`Failed to parse test results: ${testResults.error}`)
-  // }
+  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
+
+  const {artifactsDirectory, copilotToken} = options
+
+  logger.info('Copying files from: %s...', trial.scenario.directory)
+
+  await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
+    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
+  })
+  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
+    user: 'root',
+  })
+
+  logger.info('Obfuscating package name...')
+  await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
+    user: NODE_USER,
+  })
+
+  logger.info('Removing workspace dependency...')
+  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
+    user: NODE_USER,
+  })
+
+  logger.info('Installing dependencies...')
+  await sandbox.runCommand('npm', ['install'], {
+    user: NODE_USER,
+  })
+
+  if (trial.setup) {
+    logger.info('Running generic setup...')
+    await trial.setup({
+      sandbox,
+    })
+  }
+
+  if (trial.treatment.setup) {
+    logger.info('Running treatment setup...')
+    await trial.treatment.setup({
+      sandbox,
+    })
+  }
+
+  logger.info('Run build script...')
+  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
+    user: NODE_USER,
+  })
+
+  logger.info('Running copilot...')
+  const copilotOutput = await sandbox.runCommand(
+    'copilot',
+    ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: copilotToken,
+      },
+    },
+  )
+  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
+    const trimmed = line.trim()
+    if (trimmed.length === 0) {
+      return []
+    }
+    return parseMessage(JSON.parse(trimmed))
+  })
+
+  logger.info('Running tests...')
+
+  const TEST_PATH = 'scenario.test.ts'
+  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
+  const TEST_RESULTS_PATH = 'test-results.json'
+
+  await sandbox.copy(trial.scenario.testPath, TEST_PATH)
+  await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
+  await sandbox.runCommand(
+    'sh',
+    ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
+    {
+      user: NODE_USER,
+      env: {},
+    },
+  )
+  const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
+  const rawTestResult: unknown = JSON.parse(testResultsContent)
+  const testResults = parseTestResults(rawTestResult)
+  if (!testResults.success) {
+    throw new Error(`Failed to parse test results: ${testResults.error}`)
+  }
 
   return {
     trial,
diff --git a/packages/agent-eval/src/sandbox/path.ts b/packages/agent-eval/src/sandbox/path.ts
new file mode 100644
index 00000000..febc7e28
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/path.ts
@@ -0,0 +1,12 @@
+import path from 'node:path'
+import {CONTAINER_WORKDIR} from './constants'
+
+function resolveContainerPath(filepath: string): string {
+  if (path.posix.isAbsolute(filepath)) {
+    return path.posix.normalize(filepath)
+  }
+
+  return path.posix.resolve(CONTAINER_WORKDIR, filepath)
+}
+
+export {resolveContainerPath}
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index 1eaf1715..bba695a9 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -41,6 +41,8 @@ import type {
   SandboxCreateOptions,
 } from './types'
 import {DefaultHost, type Host} from '../host'
+import {VirtualSandbox} from './virtual'
+import {resolveContainerPath} from './path'
 
 const COPILOT_CLI_VERSION = '1.0.80'
 const NPM_VERSION = '12.0.2'
@@ -110,14 +112,16 @@ class SystemSandbox implements Sandbox {
   }
 
   async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise<void> {
+    const containerPath = resolveContainerPath(containerFilePath)
+
     await this.#host.fs.mkdir(hostDestinationPath, {
       recursive: true,
     })
 
     const archive = await this.#container.getArchive({
-      path: containerFilePath,
+      path: containerPath,
     })
-    const sourceName = path.posix.basename(containerFilePath)
+    const sourceName = path.posix.basename(containerPath)
 
     await pipeline(
       archive,
@@ -406,14 +410,6 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
   return container as InitializedContainer
 }
 
-function resolveContainerPath(filepath: string): string {
-  if (path.posix.isAbsolute(filepath)) {
-    return filepath
-  }
-
-  return path.posix.join(CONTAINER_WORKDIR, filepath)
-}
-
 function mapCopiedHeader(header: Headers, sourceName: string, destinationName: string): Headers {
   const name =
     header.name === sourceName
@@ -682,6 +678,8 @@ function captureStream(destination: NodeJS.WritableStream): {stream: Writable; r
   }
 }
 
-const SandboxSchema = z.instanceof(SystemSandbox)
+const SandboxSchema = z.custom<Sandbox>(value => {
+  return value instanceof SystemSandbox || value instanceof VirtualSandbox
+})
 
 export {SandboxSchema, SystemSandbox}
diff --git a/packages/agent-eval/src/sandbox/virtual.test.ts b/packages/agent-eval/src/sandbox/virtual.test.ts
index 23a847fd..57116f9d 100644
--- a/packages/agent-eval/src/sandbox/virtual.test.ts
+++ b/packages/agent-eval/src/sandbox/virtual.test.ts
@@ -1,5 +1,5 @@
 import path from 'node:path'
-import {describe, expect, test} from 'vitest'
+import {describe, expect, test, vi} from 'vitest'
 import {VirtualHost} from '../host'
 import {CONTAINER_WORKDIR} from './constants'
 import {VirtualSandbox} from './virtual'
@@ -18,6 +18,20 @@ describe('VirtualSandbox', () => {
     expect(await host.fs.readFile(path.join(CONTAINER_WORKDIR, 'nested/example.txt'), 'utf8')).toBe('example')
   })
 
+  test('resolves absolute and relative container paths', async () => {
+    const host = VirtualHost.create({
+      '/absolute/example.txt': 'absolute',
+      [CONTAINER_WORKDIR]: {
+        'relative.txt': 'relative',
+      },
+    })
+    const sandbox = await VirtualSandbox.create({host})
+
+    expect(await sandbox.readFile('/absolute/../absolute/example.txt')).toBe('absolute')
+    expect(await sandbox.readFile('./nested/../relative.txt')).toBe('relative')
+    expect(await sandbox.exists('/absolute/example.txt')).toBe(true)
+  })
+
   test('copies host directories into the sandbox with exclusions', async () => {
     const host = VirtualHost.create({
       '/fixture/included.txt': 'included',
@@ -55,4 +69,40 @@ describe('VirtualSandbox', () => {
       code: 'ENOENT',
     })
   })
+
+  test('downloads relative container files from the sandbox workspace', async () => {
+    const host = VirtualHost.create({
+      [CONTAINER_WORKDIR]: {
+        'result.txt': 'result',
+      },
+    })
+    const sandbox = await VirtualSandbox.create({host})
+
+    await sandbox.download('./nested/../result.txt', '/download')
+
+    expect(await host.fs.readFile('/download/result.txt', 'utf8')).toBe('result')
+  })
+
+  test('uses asynchronous command listeners', async () => {
+    const sandbox = await VirtualSandbox.create()
+    const listener = vi.fn().mockResolvedValue({
+      stdout: 'installed',
+      stderr: '',
+      exitCode: 0,
+    })
+    sandbox.addCommandListener(listener)
+
+    const result = await sandbox.runCommand('npm', ['install'], {
+      user: '1000:1000',
+    })
+
+    expect(listener).toHaveBeenCalledWith('npm', ['install'], {
+      user: '1000:1000',
+    })
+    expect(result).toEqual({
+      stdout: 'installed',
+      stderr: '',
+      exitCode: 0,
+    })
+  })
 })
diff --git a/packages/agent-eval/src/sandbox/virtual.ts b/packages/agent-eval/src/sandbox/virtual.ts
index ee2034c5..e3d9ee9d 100644
--- a/packages/agent-eval/src/sandbox/virtual.ts
+++ b/packages/agent-eval/src/sandbox/virtual.ts
@@ -1,21 +1,16 @@
 import path from 'node:path'
 import {VirtualHost, type Host} from '../host'
-import type {McpServerConfig} from '../mcp-config'
-import {CONTAINER_WORKDIR} from './constants'
-import type {
-  AgentSkillOptions,
-  CommandResult,
-  CopilotPluginConfig,
-  CopyOptions,
-  CustomAgentOptions,
-  DownloadOptions,
-  RunOptions,
-  Sandbox,
-  SandboxCreateOptions,
-} from './types'
+import {resolveContainerPath} from './path'
+import type {CommandResult, CopyOptions, DownloadOptions, RunOptions, Sandbox, SandboxCreateOptions} from './types'
 
 const defaultCreateOptions: SandboxCreateOptions = {}
 
+type CommandListener = (
+  command: string,
+  args?: Array<string>,
+  options?: RunOptions,
+) => CommandResult | Promise<CommandResult | undefined> | undefined
+
 export class VirtualSandbox implements Sandbox {
   static async create(options: SandboxCreateOptions = defaultCreateOptions) {
     return new VirtualSandbox(options.host ?? VirtualHost.create())
@@ -26,6 +21,8 @@ export class VirtualSandbox implements Sandbox {
   }
 
   #host: Host
+  #commandListeners: Set<CommandListener> = new Set()
+  #commands: Array<[command: string, args: Array<string>, options: RunOptions, result: CommandResult]> = []
 
   constructor(host: Host) {
     this.#host = host
@@ -38,7 +35,7 @@ export class VirtualSandbox implements Sandbox {
       throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`)
     }
 
-    const destination = resolveSandboxPath(destinationPath)
+    const destination = resolveContainerPath(destinationPath)
     if (!path.posix.basename(destination)) {
       throw new Error(`Cannot copy "${sourcePath}" to "${destinationPath}" because the destination must include a name`)
     }
@@ -50,7 +47,7 @@ export class VirtualSandbox implements Sandbox {
   }
 
   async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise<void> {
-    const source = resolveSandboxPath(containerFilePath)
+    const source = resolveContainerPath(containerFilePath)
     const destination = path.resolve(hostDestinationPath)
     const sourceStats = await this.#host.fs.stat(source)
 
@@ -66,7 +63,7 @@ export class VirtualSandbox implements Sandbox {
           continue
         }
 
-        await copyPath(this.#host, path.join(source, name), path.join(destination, name), relativePath => {
+        await copyPath(this.#host, path.posix.join(source, name), path.join(destination, name), relativePath => {
           return options.ignore?.(path.posix.join(name, relativePath)) ?? false
         })
       }
@@ -80,12 +77,12 @@ export class VirtualSandbox implements Sandbox {
   }
 
   readFile(filepath: string): Promise<string> {
-    return this.#host.fs.readFile(resolveSandboxPath(filepath), 'utf8')
+    return this.#host.fs.readFile(resolveContainerPath(filepath), 'utf8')
   }
 
   async writeFile(filepath: string, contents: string): Promise<void> {
-    const destination = resolveSandboxPath(filepath)
-    await this.#host.fs.mkdir(path.dirname(destination), {
+    const destination = resolveContainerPath(filepath)
+    await this.#host.fs.mkdir(path.posix.dirname(destination), {
       recursive: true,
     })
     await this.#host.fs.writeFile(destination, contents, 'utf8')
@@ -93,7 +90,7 @@ export class VirtualSandbox implements Sandbox {
 
   async exists(filepath: string): Promise<boolean> {
     try {
-      await this.#host.fs.access(resolveSandboxPath(filepath))
+      await this.#host.fs.access(resolveContainerPath(filepath))
       return true
     } catch (error) {
       if (isErrorWithCode(error, 'ENOENT')) {
@@ -104,52 +101,38 @@ export class VirtualSandbox implements Sandbox {
     }
   }
 
-  runCommand(command: string, args?: Array<string>, options?: RunOptions): Promise<CommandResult> {
-    void command
-    void args
-    void options
-    throw new Error('Method not implemented.')
-  }
+  async runCommand(command: string, args: Array<string> = [], options: RunOptions = {}): Promise<CommandResult> {
+    for (const listener of this.#commandListeners) {
+      const result = await listener(command, args, options)
+      if (result) {
+        this.#commands.push([command, args, options, result])
+        return result
+      }
+    }
 
-  addAgentInstruction(text: string): Promise<void> {
-    void text
-    throw new Error('Method not implemented.')
-  }
+    const result: CommandResult = {
+      stdout: '',
+      stderr: '',
+      exitCode: 0,
+    }
+    this.#commands.push([command, args, options, result])
 
-  addAgentSkill(name: string, description: string, contents: string, options?: AgentSkillOptions): Promise<void> {
-    void name
-    void description
-    void contents
-    void options
-    throw new Error('Method not implemented.')
+    return result
   }
 
-  addCustomAgent(name: string, description: string, contents: string, options?: CustomAgentOptions): Promise<void> {
-    void name
-    void description
-    void contents
-    void options
-    throw new Error('Method not implemented.')
-  }
+  async addAgentInstruction(): Promise<void> {}
 
-  addMcpServer(name: string, config: McpServerConfig): Promise<void> {
-    void name
-    void config
-    throw new Error('Method not implemented.')
-  }
+  async addAgentSkill(): Promise<void> {}
 
-  addCopilotPlugin(config: CopilotPluginConfig): Promise<void> {
-    void config
-    throw new Error('Method not implemented.')
-  }
-}
+  async addCustomAgent(): Promise<void> {}
 
-function resolveSandboxPath(filepath: string): string {
-  if (path.posix.isAbsolute(filepath)) {
-    return filepath
-  }
+  async addMcpServer(): Promise<void> {}
+
+  async addCopilotPlugin(): Promise<void> {}
 
-  return path.posix.join(CONTAINER_WORKDIR, filepath)
+  addCommandListener(listener: CommandListener): void {
+    this.#commandListeners.add(listener)
+  }
 }
 
 async function copyPath(

From 4a6ea53ae5f7fe05fcdebd1cc737e5d85f866743 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 10:14:04 -0500
Subject: [PATCH 14/52] refactor: remove host arg from runTrial

---
 packages/agent-eval/src/run.test.ts |  3 +--
 packages/agent-eval/src/run.ts      | 40 +++++++++++------------------
 2 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts
index f21967af..804c1ffa 100644
--- a/packages/agent-eval/src/run.test.ts
+++ b/packages/agent-eval/src/run.test.ts
@@ -5,7 +5,7 @@ import {VirtualHost} from './host'
 import {VirtualSandbox} from './sandbox'
 
 test('runTrial', async () => {
-  const host = new VirtualHost({
+  const host = VirtualHost.create({
     '/test': {
       'scenario.test.ts': '',
     },
@@ -42,7 +42,6 @@ test('runTrial', async () => {
   })
 
   const result = await runTrial(
-    host,
     sandbox,
     {
       id: 'test',
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index c5838a04..4877d9ef 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -6,7 +6,7 @@ import {TrialSchema, type Trial} from './trial'
 import type {Host} from './host'
 // import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
 import {CONTAINER_WORKDIR, NODE_USER, type Sandbox} from './sandbox'
-import {parseTestResults} from './vitest'
+import {parseTestResults, TestResultsSchema} from './vitest'
 import {logger} from './logger'
 
 const AssistantSchema = z.object({
@@ -19,22 +19,6 @@ const AssistantSchema = z.object({
   tools: z.record(z.string(), z.number()),
 })
 
-const TestResultsSchema = z.object({
-  numTotalTests: z.number(),
-  numPassedTests: z.number(),
-  numFailedTests: z.number(),
-  numPendingTests: z.number(),
-  numTodoTests: z.number(),
-  tests: z.array(
-    z.object({
-      title: z.string(),
-      fullName: z.string(),
-      status: z.enum(['passed', 'failed', 'skipped', 'pending', 'todo', 'disabled']),
-      description: z.optional(z.string()),
-    }),
-  ),
-})
-
 const WalkthroughSchema = z.discriminatedUnion('type', [
   z.object({type: z.literal('Unavailable')}),
   z.object({type: z.literal('Screenshot'), filepath: z.string()}),
@@ -44,9 +28,11 @@ const WalkthroughSchema = z.discriminatedUnion('type', [
 
 const TrialResultSchema = z.object({
   trial: TrialSchema,
-  // assistant: AssistantSchema,
-  // testResults: TestResultsSchema,
-  // walkthrough: WalkthroughSchema,
+  assistant: z.object({
+    sessions: z.array(z.array(MessageSchema)),
+  }),
+  testResults: TestResultsSchema,
+  walkthrough: WalkthroughSchema,
 })
 
 type TrialResult = z.infer<typeof TrialResultSchema>
@@ -58,7 +44,7 @@ type RunOptions = {
 }
 
 async function run(host: Host, sandbox: Sandbox, plan: Plan, options: RunOptions): Promise<Array<TrialResult>> {
-  const {artifactsDirectory, copilotToken, dockerImage, maxConcurrency = 1} = options
+  const {artifactsDirectory, copilotToken, maxConcurrency = 1} = options
   const queue = new Queue({
     concurrency: maxConcurrency,
   })
@@ -84,7 +70,7 @@ type RunTrialOptions = {
   copilotToken: string
 }
 
-async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
+async function runTrial(sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
   logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
 
   const {artifactsDirectory, copilotToken} = options
@@ -176,9 +162,13 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
 
   return {
     trial,
-    // assistant: {},
-    // testResults: {},
-    // walkthrough: {},
+    assistant: {
+      sessions: [messages],
+    },
+    testResults: testResults.data,
+    walkthrough: {
+      type: 'Unavailable',
+    },
   }
 }
 

From cc2cc263216f669e844875ae4df69e73de4a1f00 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 10:23:53 -0500
Subject: [PATCH 15/52] refactor: mock virtual sandbox instead of adding custom
 implementation

---
 packages/agent-eval/src/run.test.ts           | 71 +++++++++----------
 .../agent-eval/src/sandbox/virtual.test.ts    | 25 +------
 packages/agent-eval/src/sandbox/virtual.ts    | 29 +-------
 3 files changed, 39 insertions(+), 86 deletions(-)

diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts
index 804c1ffa..f05e3356 100644
--- a/packages/agent-eval/src/run.test.ts
+++ b/packages/agent-eval/src/run.test.ts
@@ -1,44 +1,31 @@
-import {test, expect, vi} from 'vitest'
-import {run, runTrial} from './run'
-import {plan} from './plan'
-import {VirtualHost} from './host'
-import {VirtualSandbox} from './sandbox'
+import {expect, test, vi} from 'vitest'
+import {runTrial} from './run'
+import {NODE_USER, VirtualSandbox} from './sandbox'
 
 test('runTrial', async () => {
-  const host = VirtualHost.create({
-    '/test': {
-      'scenario.test.ts': '',
-    },
-  })
-  await using sandbox = await VirtualSandbox.create({
-    host,
-  })
-
-  sandbox.addCommandListener(async (cmd, args) => {
-    if (cmd !== 'sh') {
-      return
-    }
-
-    if (!Array.isArray(args)) {
-      return
+  await using sandbox = await VirtualSandbox.create()
+  vi.spyOn(sandbox, 'copy').mockResolvedValue()
+  const runCommand = vi.spyOn(sandbox, 'runCommand').mockImplementation(async command => {
+    if (command === 'sh') {
+      await sandbox.writeFile(
+        'test-results.json',
+        JSON.stringify({
+          numTotalTests: 0,
+          numPassedTests: 0,
+          numFailedTests: 0,
+          numPendingTests: 0,
+          numTodoTests: 0,
+          success: true,
+          testResults: [],
+        }),
+      )
     }
 
-    if (args[0] !== '-c' || !args[1].startsWith('npx vitest run')) {
-      return
+    return {
+      stdout: '',
+      stderr: '',
+      exitCode: 0,
     }
-
-    await sandbox.writeFile(
-      'test-results.json',
-      JSON.stringify({
-        numTotalTests: 0,
-        numPassedTests: 0,
-        numFailedTests: 0,
-        numPendingTests: 0,
-        numTodoTests: 0,
-        success: true,
-        testResults: [],
-      }),
-    )
   })
 
   const result = await runTrial(
@@ -65,4 +52,16 @@ test('runTrial', async () => {
       copilotToken: '',
     },
   )
+
+  expect(runCommand).toHaveBeenCalledWith(
+    'copilot',
+    ['--prompt', 'test', '--model', 'gpt-5.6-sol', '--reasoning-effort', 'medium'],
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: '',
+      },
+    },
+  )
+  expect(result.assistant.sessions).toEqual([[]])
 })
diff --git a/packages/agent-eval/src/sandbox/virtual.test.ts b/packages/agent-eval/src/sandbox/virtual.test.ts
index 57116f9d..0fb8ea85 100644
--- a/packages/agent-eval/src/sandbox/virtual.test.ts
+++ b/packages/agent-eval/src/sandbox/virtual.test.ts
@@ -1,5 +1,5 @@
 import path from 'node:path'
-import {describe, expect, test, vi} from 'vitest'
+import {describe, expect, test} from 'vitest'
 import {VirtualHost} from '../host'
 import {CONTAINER_WORKDIR} from './constants'
 import {VirtualSandbox} from './virtual'
@@ -82,27 +82,4 @@ describe('VirtualSandbox', () => {
 
     expect(await host.fs.readFile('/download/result.txt', 'utf8')).toBe('result')
   })
-
-  test('uses asynchronous command listeners', async () => {
-    const sandbox = await VirtualSandbox.create()
-    const listener = vi.fn().mockResolvedValue({
-      stdout: 'installed',
-      stderr: '',
-      exitCode: 0,
-    })
-    sandbox.addCommandListener(listener)
-
-    const result = await sandbox.runCommand('npm', ['install'], {
-      user: '1000:1000',
-    })
-
-    expect(listener).toHaveBeenCalledWith('npm', ['install'], {
-      user: '1000:1000',
-    })
-    expect(result).toEqual({
-      stdout: 'installed',
-      stderr: '',
-      exitCode: 0,
-    })
-  })
 })
diff --git a/packages/agent-eval/src/sandbox/virtual.ts b/packages/agent-eval/src/sandbox/virtual.ts
index e3d9ee9d..e84c1b40 100644
--- a/packages/agent-eval/src/sandbox/virtual.ts
+++ b/packages/agent-eval/src/sandbox/virtual.ts
@@ -1,16 +1,10 @@
 import path from 'node:path'
 import {VirtualHost, type Host} from '../host'
 import {resolveContainerPath} from './path'
-import type {CommandResult, CopyOptions, DownloadOptions, RunOptions, Sandbox, SandboxCreateOptions} from './types'
+import type {CommandResult, CopyOptions, DownloadOptions, Sandbox, SandboxCreateOptions} from './types'
 
 const defaultCreateOptions: SandboxCreateOptions = {}
 
-type CommandListener = (
-  command: string,
-  args?: Array<string>,
-  options?: RunOptions,
-) => CommandResult | Promise<CommandResult | undefined> | undefined
-
 export class VirtualSandbox implements Sandbox {
   static async create(options: SandboxCreateOptions = defaultCreateOptions) {
     return new VirtualSandbox(options.host ?? VirtualHost.create())
@@ -21,8 +15,6 @@ export class VirtualSandbox implements Sandbox {
   }
 
   #host: Host
-  #commandListeners: Set<CommandListener> = new Set()
-  #commands: Array<[command: string, args: Array<string>, options: RunOptions, result: CommandResult]> = []
 
   constructor(host: Host) {
     this.#host = host
@@ -101,23 +93,12 @@ export class VirtualSandbox implements Sandbox {
     }
   }
 
-  async runCommand(command: string, args: Array<string> = [], options: RunOptions = {}): Promise<CommandResult> {
-    for (const listener of this.#commandListeners) {
-      const result = await listener(command, args, options)
-      if (result) {
-        this.#commands.push([command, args, options, result])
-        return result
-      }
-    }
-
-    const result: CommandResult = {
+  async runCommand(): Promise<CommandResult> {
+    return {
       stdout: '',
       stderr: '',
       exitCode: 0,
     }
-    this.#commands.push([command, args, options, result])
-
-    return result
   }
 
   async addAgentInstruction(): Promise<void> {}
@@ -129,10 +110,6 @@ export class VirtualSandbox implements Sandbox {
   async addMcpServer(): Promise<void> {}
 
   async addCopilotPlugin(): Promise<void> {}
-
-  addCommandListener(listener: CommandListener): void {
-    this.#commandListeners.add(listener)
-  }
 }
 
 async function copyPath(

From e05cb5cb35520c976f10ce9786efc7f64c58770f Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 15:05:27 -0500
Subject: [PATCH 16/52] refactor: change defineScenario to defineConfig

---
 scenarios/000-nextjs-template/scenario.config.ts              | 4 ++--
 scenarios/000-vite-template/scenario.config.ts                | 4 ++--
 .../001-agent-uses-button-from-primer/scenario.config.ts      | 4 ++--
 .../002-agent-uses-octicon-from-primer/scenario.config.ts     | 4 ++--
 scenarios/003-agent-uses-form-from-primer/scenario.config.ts  | 4 ++--
 scenarios/004-agent-setup-nextjs/scenario.config.ts           | 4 ++--
 .../005-agent-enables-theme-switching/scenario.config.ts      | 4 ++--
 .../006-agent-uses-pagination-component/scenario.config.ts    | 4 ++--
 scenarios/007-agent-infers-billing-banner/scenario.config.ts  | 4 ++--
 scenarios/008-agent-infers-action-menu/scenario.config.ts     | 4 ++--
 .../009-agent-uses-layout-and-color-tokens/scenario.config.ts | 4 ++--
 scenarios/010-agent-uses-typography-tokens/scenario.config.ts | 4 ++--
 scenarios/011-agent-uses-motion-tokens/scenario.config.ts     | 4 ++--
 scenarios/012-agent-infers-status-tokens/scenario.config.ts   | 4 ++--
 .../scenario.config.ts                                        | 4 ++--
 .../scenario.config.ts                                        | 4 ++--
 scenarios/015-agent-infers-copy-icon/scenario.config.ts       | 4 ++--
 .../scenario.config.ts                                        | 4 ++--
 .../017-agent-uses-confirmation-pattern/scenario.config.ts    | 4 ++--
 scenarios/018-agent-uses-filter-pattern/scenario.config.ts    | 4 ++--
 .../019-agent-uses-dismissal-utilities/scenario.config.ts     | 4 ++--
 .../020-agent-uses-resize-observer-utility/scenario.config.ts | 4 ++--
 scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts | 4 ++--
 .../022-agent-enables-automatic-theming/scenario.config.ts    | 4 ++--
 scenarios/023-agent-adds-theme-switcher/scenario.config.ts    | 4 ++--
 scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts    | 4 ++--
 .../025-agent-uses-tokens-with-tailwindcss/scenario.config.ts | 4 ++--
 .../scenario.config.ts                                        | 4 ++--
 28 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/scenarios/000-nextjs-template/scenario.config.ts b/scenarios/000-nextjs-template/scenario.config.ts
index 89ab54e3..d153a49d 100644
--- a/scenarios/000-nextjs-template/scenario.config.ts
+++ b/scenarios/000-nextjs-template/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   prompt: `Example prompt`,
   tags: [],
 })
diff --git a/scenarios/000-vite-template/scenario.config.ts b/scenarios/000-vite-template/scenario.config.ts
index 02803bff..9c437b85 100644
--- a/scenarios/000-vite-template/scenario.config.ts
+++ b/scenarios/000-vite-template/scenario.config.ts
@@ -1,5 +1,5 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   prompt: `Example prompt`,
 })
diff --git a/scenarios/001-agent-uses-button-from-primer/scenario.config.ts b/scenarios/001-agent-uses-button-from-primer/scenario.config.ts
index b3e98827..67f0a0d6 100644
--- a/scenarios/001-agent-uses-button-from-primer/scenario.config.ts
+++ b/scenarios/001-agent-uses-button-from-primer/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses a primary Primer button with the requested label.',
   prompt: `Update the index page to use a primary button with the text 'Submit'`,
   tags: ['baseline', 'button', 'component', 'nextjs', 'primer'],
diff --git a/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts b/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts
index f4806c80..98885963 100644
--- a/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts
+++ b/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses the requested icon from Primer Octicons.',
   prompt: `Update the index page to use a Search icon`,
   tags: ['baseline', 'icon', 'nextjs', 'primer'],
diff --git a/scenarios/003-agent-uses-form-from-primer/scenario.config.ts b/scenarios/003-agent-uses-form-from-primer/scenario.config.ts
index 0fe02048..eede7936 100644
--- a/scenarios/003-agent-uses-form-from-primer/scenario.config.ts
+++ b/scenarios/003-agent-uses-form-from-primer/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent builds a sign-up form with Primer form components.',
   prompt: `Update the index page to render a sign-up form. The form does not need to post to an endpoint, I am only working on the UI for now.`,
   tags: ['baseline', 'component', 'form', 'nextjs', 'primer'],
diff --git a/scenarios/004-agent-setup-nextjs/scenario.config.ts b/scenarios/004-agent-setup-nextjs/scenario.config.ts
index c74cebe4..cd855be8 100644
--- a/scenarios/004-agent-setup-nextjs/scenario.config.ts
+++ b/scenarios/004-agent-setup-nextjs/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent configures a Next.js project to use Primer.',
   prompt: `Setup this project to work with Primer. Including a default page layout in app.tsx.`,
   tags: ['baseline', 'nextjs', 'primer', 'setup'],
diff --git a/scenarios/005-agent-enables-theme-switching/scenario.config.ts b/scenarios/005-agent-enables-theme-switching/scenario.config.ts
index 7eab4360..b01f5e6a 100644
--- a/scenarios/005-agent-enables-theme-switching/scenario.config.ts
+++ b/scenarios/005-agent-enables-theme-switching/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent enables every available Primer color scheme.',
   prompt: `Enable support for switching between all available color schemes.`,
   tags: ['accessibility', 'baseline', 'nextjs', 'primer', 'theming'],
diff --git a/scenarios/006-agent-uses-pagination-component/scenario.config.ts b/scenarios/006-agent-uses-pagination-component/scenario.config.ts
index fae4973c..59de7fc8 100644
--- a/scenarios/006-agent-uses-pagination-component/scenario.config.ts
+++ b/scenarios/006-agent-uses-pagination-component/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses an existing component when adding pagination.',
   prompt: `Add pagination controls below the issue list. Show 25 issues per page and include previous and next navigation.`,
   tags: ['component', 'pagination', 'vite'],
diff --git a/scenarios/007-agent-infers-billing-banner/scenario.config.ts b/scenarios/007-agent-infers-billing-banner/scenario.config.ts
index 276718d7..a1fb46aa 100644
--- a/scenarios/007-agent-infers-billing-banner/scenario.config.ts
+++ b/scenarios/007-agent-infers-billing-banner/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent selects an appropriate component for a persistent warning.',
   prompt: `Show a persistent warning at the top of the page when an account has a past-due balance. Include a link to billing settings.`,
   tags: ['banner', 'component', 'vite'],
diff --git a/scenarios/008-agent-infers-action-menu/scenario.config.ts b/scenarios/008-agent-infers-action-menu/scenario.config.ts
index 6d8ff68d..654ef042 100644
--- a/scenarios/008-agent-infers-action-menu/scenario.config.ts
+++ b/scenarios/008-agent-infers-action-menu/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent selects an appropriate component for secondary actions.',
   prompt: `Add archive, transfer, and delete actions to the repository header without crowding the existing primary actions.`,
   tags: ['component', 'menu', 'vite'],
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
index 6d8092fb..5cc514b6 100644
--- a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
+++ b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses design tokens for layout and color styling.',
   prompt: `Style the status summary card so its content is clearly grouped and visually distinct from the page background.`,
   tags: ['color', 'layout', 'tokens', 'vite'],
diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
index 192e8c31..7ef60583 100644
--- a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
+++ b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses role-appropriate typography tokens.',
   prompt: `Improve the typography of the documentation page. It contains a page title, introductory text, inline code, and a code example.`,
   tags: ['tokens', 'typography', 'vite'],
diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
index b494e085..83f268e7 100644
--- a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
+++ b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses motion tokens and respects reduced-motion preferences.',
   prompt: `Add a short transition when the details panel expands or collapses. Keep the interaction comfortable for people who prefer reduced motion.`,
   tags: ['accessibility', 'motion', 'tokens', 'vite'],
diff --git a/scenarios/012-agent-infers-status-tokens/scenario.config.ts b/scenarios/012-agent-infers-status-tokens/scenario.config.ts
index 75cd19c8..17fe6215 100644
--- a/scenarios/012-agent-infers-status-tokens/scenario.config.ts
+++ b/scenarios/012-agent-infers-status-tokens/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent chooses semantically correct status tokens.',
   prompt: `Update the deployment list so successful and failed deployments are easy to distinguish without relying on text alone.`,
   tags: ['color', 'status', 'tokens', 'vite'],
diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
index 8e007fd8..55f240e0 100644
--- a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
+++ b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent chooses appropriate tokens for a compact toolbar.',
   prompt: `Make the repository file toolbar more compact while keeping its controls usable and consistently spaced.`,
   tags: ['controls', 'layout', 'tokens', 'vite'],
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
index 83b6ada9..070ea270 100644
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
+++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent replaces custom icons with maintained design-system icons.',
   prompt: `Replace the hand-drawn search, download, and trash icons in the toolbar with icons from the project's design system.`,
   tags: ['icon', 'octicons', 'vite'],
diff --git a/scenarios/015-agent-infers-copy-icon/scenario.config.ts b/scenarios/015-agent-infers-copy-icon/scenario.config.ts
index 38a586b8..946fd008 100644
--- a/scenarios/015-agent-infers-copy-icon/scenario.config.ts
+++ b/scenarios/015-agent-infers-copy-icon/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent infers the appropriate icon for a compact action.',
   prompt: `Add a compact control next to the commit SHA that copies it to the clipboard.`,
   tags: ['component', 'icon', 'vite'],
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
index ad730b92..aa01276b 100644
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
+++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent applies established loading and empty-state patterns.',
   prompt: `Add appropriate loading and empty states to the repository list. The empty state should help the user create their first repository.`,
   tags: ['empty-state', 'loading', 'pattern', 'vite'],
diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
index aadb828c..6fefaca9 100644
--- a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
+++ b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent applies the established confirmation pattern to a destructive action.',
   prompt: `Let an administrator delete the repository after confirming the destructive action. Include clear cancel and confirm paths.`,
   tags: ['confirmation', 'dialog', 'pattern', 'vite'],
diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
index d76da367..ba9008ea 100644
--- a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
+++ b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent composes current components into an issue-filtering pattern.',
   prompt: `Add controls for filtering the issue list by author, label, and open or closed status.`,
   tags: ['filter', 'pattern', 'vite'],
diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
index 7f1b122e..2954db45 100644
--- a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
+++ b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent reuses design-system utilities for common dismissal behavior.',
   prompt: `Update the existing floating panel so it closes when the user clicks outside it or presses Escape. Preserve the current markup and positioning.`,
   tags: ['hooks', 'interaction', 'utilities', 'vite'],
diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
index e8566909..37450d53 100644
--- a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
+++ b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent reuses the design-system resize observer utility.',
   prompt: `Make the contribution chart update its dimensions whenever its container is resized.`,
   tags: ['hooks', 'responsive', 'utilities', 'vite'],
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
index 6648b57d..e36c9608 100644
--- a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
+++ b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent correctly configures a Vite application to use the design system.',
   prompt: `Set up this Vite application to use our design system and update the default page to demonstrate that it is working.`,
   tags: ['setup', 'vite'],
diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
index c79eeefb..b5f91a29 100644
--- a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
+++ b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent configures automatic light and dark theme support.',
   prompt: `Make the application follow the user's system light or dark appearance setting.`,
   tags: ['theme', 'theming', 'vite'],
diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
index 47a852a9..286ecea8 100644
--- a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
+++ b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent implements a persistent user-controlled theme preference.',
   prompt: `Add an appearance setting with system, light, and dark choices. Apply the choice immediately and remember it across visits.`,
   tags: ['interaction', 'theme', 'theming', 'vite'],
diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
index 49b0c522..a4350ac4 100644
--- a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
+++ b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent configures Tailwind CSS alongside the design system.',
   prompt: `Add Tailwind CSS to this Vite application and use it to lay out the default page without breaking the existing design-system styles.`,
   tags: ['setup', 'tailwindcss', 'vite'],
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
index 8adb1f7b..1f1f6c1c 100644
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
+++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses design tokens when styling with Tailwind CSS.',
   prompt: `Use Tailwind utility classes to style the deployment status panel while keeping its colors and spacing aligned with the design system.`,
   tags: ['tailwindcss', 'tokens', 'vite'],
diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
index 98ff5186..f884cb24 100644
--- a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
+++ b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
@@ -1,6 +1,6 @@
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent avoids deprecated components when adding a page-level warning.',
   prompt: `Add a dismissible page-level warning above the repository settings when branch protection is disabled. Include a link to enable branch protection.`,
   tags: ['banner', 'component', 'deprecated', 'vite'],

From 4abe36710f27c30cf278da0a021cbba1428c69cd Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 15:05:40 -0500
Subject: [PATCH 17/52] feat: build through experiments run flow

---
 experiments/baseline.ts                       |  74 +-
 experiments/noop.ts                           |   9 +-
 packages/agent-eval/rolldown.config.ts        |   2 +-
 packages/agent-eval/src/benchmark.ts          | 117 ++-
 packages/agent-eval/src/cli.ts                | 908 +++++++++---------
 packages/agent-eval/src/environment.ts        |  51 +
 packages/agent-eval/src/experiment-config.ts  |  38 -
 packages/agent-eval/src/experiment.test.ts    | 152 +++
 packages/agent-eval/src/experiment.ts         | 195 +++-
 packages/agent-eval/src/experiments.test.ts   |  76 --
 packages/agent-eval/src/experiments.ts        |  91 --
 packages/agent-eval/src/host.test.ts          |  53 +
 packages/agent-eval/src/host.ts               |  12 +-
 packages/agent-eval/src/index.ts              |  40 +-
 packages/agent-eval/src/model.ts              | 122 ++-
 packages/agent-eval/src/plan.ts               |   4 +-
 .../src/resolve-experiment-scenario.ts        |  32 -
 packages/agent-eval/src/run-old.test.ts       |  55 --
 packages/agent-eval/src/run-old.ts            | 486 ----------
 packages/agent-eval/src/run.ts                | 133 ++-
 packages/agent-eval/src/sandbox-old.test.ts   | 119 ---
 packages/agent-eval/src/sandbox-old.ts        | 799 ---------------
 packages/agent-eval/src/sandbox/index.ts      |   2 +-
 packages/agent-eval/src/sandbox/system.ts     |  15 +-
 packages/agent-eval/src/scenario-config.ts    |   8 -
 packages/agent-eval/src/scenario.test.ts      |  70 ++
 packages/agent-eval/src/scenario.ts           |  16 +-
 packages/agent-eval/src/treatment-old.ts      |  60 --
 28 files changed, 1296 insertions(+), 2443 deletions(-)
 create mode 100644 packages/agent-eval/src/environment.ts
 delete mode 100644 packages/agent-eval/src/experiment-config.ts
 create mode 100644 packages/agent-eval/src/experiment.test.ts
 delete mode 100644 packages/agent-eval/src/experiments.test.ts
 delete mode 100644 packages/agent-eval/src/experiments.ts
 create mode 100644 packages/agent-eval/src/host.test.ts
 delete mode 100644 packages/agent-eval/src/resolve-experiment-scenario.ts
 delete mode 100644 packages/agent-eval/src/run-old.test.ts
 delete mode 100644 packages/agent-eval/src/run-old.ts
 delete mode 100644 packages/agent-eval/src/sandbox-old.test.ts
 delete mode 100644 packages/agent-eval/src/sandbox-old.ts
 delete mode 100644 packages/agent-eval/src/scenario-config.ts
 delete mode 100644 packages/agent-eval/src/treatment-old.ts

diff --git a/experiments/baseline.ts b/experiments/baseline.ts
index 40d440ca..bfd4f20e 100644
--- a/experiments/baseline.ts
+++ b/experiments/baseline.ts
@@ -1,37 +1,37 @@
-import path from 'node:path'
-import {defineConfig} from '@primer/agent-eval/experiment'
-import {listScenarios} from '@primer/agent-eval'
-
-const scenarios = await listScenarios({
-  directory: path.resolve(import.meta.dirname, '..', 'scenarios'),
-  tags: ['baseline'],
-})
-
-export const experiment = defineConfig({
-  name: 'Baseline',
-  description: 'Baseline experiment to evaluate the performance of the agent with our recommended setup.',
-  models: [
-    {name: 'gpt-5.6-sol', reasoningEfforts: []},
-    {name: 'gpt-5.6-terra', reasoningEfforts: []},
-    {name: 'claude-opus-5', reasoningEfforts: []},
-    {name: 'claude-sonnet-5', reasoningEfforts: []},
-    {name: 'gemini-3.1-pro-preview', reasoningEfforts: []},
-    {name: 'gemini-3.6-flash', reasoningEfforts: []},
-  ],
-  scenarios: scenarios.filter(scenario => !scenario.id.startsWith('000')).map(scenario => scenario.id),
-  treatments: [
-    {
-      name: 'Recommended',
-      async setup({sandbox}) {
-        // Setup the Primer MCP server locally
-        await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest'])
-        await sandbox.addMcpServer('primer', {
-          type: 'local',
-          command: 'npx',
-          args: ['--no-install', '@primer/mcp'],
-          tools: ['*'],
-        })
-      },
-    },
-  ],
-})
+// import path from 'node:path'
+// import {defineConfig} from '@primer/agent-eval/experiment'
+// import {listScenarios} from '@primer/agent-eval'
+//
+// const scenarios = await listScenarios({
+//   directory: path.resolve(import.meta.dirname, '..', 'scenarios'),
+//   tags: ['baseline'],
+// })
+//
+// export const experiment = defineConfig({
+//   name: 'Baseline',
+//   description: 'Baseline experiment to evaluate the performance of the agent with our recommended setup.',
+//   models: [
+//     {name: 'gpt-5.6-sol', reasoningEfforts: []},
+//     {name: 'gpt-5.6-terra', reasoningEfforts: []},
+//     {name: 'claude-opus-5', reasoningEfforts: []},
+//     {name: 'claude-sonnet-5', reasoningEfforts: []},
+//     {name: 'gemini-3.1-pro-preview', reasoningEfforts: []},
+//     {name: 'gemini-3.6-flash', reasoningEfforts: []},
+//   ],
+//   scenarios: scenarios.filter(scenario => !scenario.id.startsWith('000')).map(scenario => scenario.id),
+//   treatments: [
+//     {
+//       name: 'Recommended',
+//       async setup({sandbox}) {
+//         // Setup the Primer MCP server locally
+//         await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest'])
+//         await sandbox.addMcpServer('primer', {
+//           type: 'local',
+//           command: 'npx',
+//           args: ['--no-install', '@primer/mcp'],
+//           tools: ['*'],
+//         })
+//       },
+//     },
+//   ],
+// })
diff --git a/experiments/noop.ts b/experiments/noop.ts
index 30cf76e4..9b2ea568 100644
--- a/experiments/noop.ts
+++ b/experiments/noop.ts
@@ -10,5 +10,12 @@ export const experiment = defineConfig({
     },
   ],
   scenarios: ['001-agent-uses-button-from-primer'],
-  treatments: [],
+  treatments: [
+    {
+      name: 'noop',
+      async setup() {
+        //
+      },
+    },
+  ],
 })
diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts
index 047b8282..7e1d288b 100644
--- a/packages/agent-eval/rolldown.config.ts
+++ b/packages/agent-eval/rolldown.config.ts
@@ -13,7 +13,7 @@ const config = defineConfig({
     cli: 'src/cli.ts',
     experiment: 'src/experiment.ts',
     index: 'src/index.ts',
-    scenario: 'src/scenario-config.ts',
+    scenario: 'src/scenario.ts',
   },
   platform: 'node',
   external,
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 5d82df37..47598e96 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -3,9 +3,10 @@ import path from 'node:path'
 import * as z from 'zod/mini'
 import {getModelVariants, ModelVariantConfigSchema, type Model, type ModelVariant, type ReasoningEffort} from './model'
 import {getScenario, type Scenario} from './scenario'
-import type {Host} from './host'
+import {DefaultHost, type Host} from './host'
 import {ControlTreatment, TreatmentSetupSchema} from './treatment'
 import type {Treatment} from './treatment'
+import type {EnvironmentConfig} from './environment'
 
 const CapabilityConfigSchema = z.object({
   name: z.string(),
@@ -15,7 +16,7 @@ const CapabilityConfigSchema = z.object({
 const BenchmarkConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
-  models: ModelVariantConfigSchema,
+  models: z.array(ModelVariantConfigSchema),
   setup: z.optional(TreatmentSetupSchema),
   capabilities: z.array(CapabilityConfigSchema),
 })
@@ -106,59 +107,71 @@ function isBenchmarkFile(filename: string): boolean {
   return true
 }
 
-type RunContext = {
-  artifactsDirectory: string
-  benchmarksDirectory: string
-  host: Host
-  scenariosDirectory: string
-}
-
-type Trial = {
-  id: string
-  capability: CapabilityConfig
-  scenario: Scenario
-  treatment: Treatment
-  model: ModelVariant
-}
-
-type TrialRun = {}
-
+// type RunContext = {
+//   artifactsDirectory: string
+//   benchmarksDirectory: string
+//   host: Host
+//   scenariosDirectory: string
+// }
+//
+// type Trial = {
+//   id: string
+//   capability: CapabilityConfig
+//   scenario: Scenario
+//   treatment: Treatment
+//   model: ModelVariant
+// }
+//
+// type TrialRun = {}
+//
 type TrialResult = {}
 
-async function run(context: RunContext, benchmark: Benchmark): Promise<TrialResult> {
-  const trials: Array<Trial> = []
-
-  for (const variant of getModelVariants(benchmark.models)) {
-    for (const capability of benchmark.capabilities) {
-      for (const scenarioId of capability.scenarios) {
-        const scenario = await getScenario(context.host, context.scenariosDirectory, scenarioId)
-
-        trials.push({
-          id: randomUUID(),
-          capability,
-          scenario,
-          treatment: ControlTreatment,
-          model: variant,
-        })
-
-        trials.push({
-          id: randomUUID(),
-          capability,
-          scenario,
-          treatment: {
-            name: 'Benchmark',
-            setup: benchmark.setup,
-          },
-          model: variant,
-        })
-      }
-    }
-  }
-
-  console.log(trials)
+async function run({
+  env,
+  host = DefaultHost,
+  id,
+}: {
+  env: EnvironmentConfig
+  host?: Host
+  id: string
+}): Promise<TrialResult> {
+  const benchmark = await getBenchmark(host, env.benchmarksDirectory, id)
+  console.log(benchmark)
 
   throw new Error('unimplemented')
+  // const trials: Array<Trial> = []
+  //
+  // for (const variant of getModelVariants(benchmark.models)) {
+  //   for (const capability of benchmark.capabilities) {
+  //     for (const scenarioId of capability.scenarios) {
+  //       const scenario = await getScenario(context.host, context.scenariosDirectory, scenarioId)
+  //
+  //       trials.push({
+  //         id: randomUUID(),
+  //         capability,
+  //         scenario,
+  //         treatment: ControlTreatment,
+  //         model: variant,
+  //       })
+  //
+  //       trials.push({
+  //         id: randomUUID(),
+  //         capability,
+  //         scenario,
+  //         treatment: {
+  //           name: 'Benchmark',
+  //           setup: benchmark.setup,
+  //         },
+  //         model: variant,
+  //       })
+  //     }
+  //   }
+  // }
+  //
+  // console.log(trials)
+  //
+  // throw new Error('unimplemented')
 }
 
-export {defineConfig, listBenchmarks, getBenchmark, run}
-export type {BenchmarkConfig}
+export {defineConfig, listBenchmarks, getBenchmark, run, BenchmarkConfigSchema}
+export type {BenchmarkConfig, Benchmark}
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 1b7f4262..cfdd4186 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -1,17 +1,9 @@
 #!/usr/bin/env node
-import {randomUUID} from 'node:crypto'
-import {existsSync} from 'node:fs'
-import path from 'node:path'
-import fs from 'node:fs/promises'
+
 import {parseArgs} from 'node:util'
-import {ControlTreatment, type ExperimentConfig} from './experiment-config'
-import {resolveModelConfigs, type Model, type ReasoningEffort} from './model'
-import {createAgentEvalOutput} from './output'
-import type {Treatment, TreatmentResult} from './treatment'
-import {findExperiment, listExperiments} from './experiments'
-import {resolveExperimentScenario} from './resolve-experiment-scenario'
-import {run} from './run'
-import {parseShard, selectShard} from './shard'
+import {getEnvironmentConfig} from './environment'
+import {run as runBenchmark} from './benchmark'
+import {run as runExperiment} from './experiment'
 
 const {values} = parseArgs({
   options: {
@@ -62,30 +54,29 @@ const {values} = parseArgs({
       type: 'string',
       description: 'The directory containing scenario directories',
     },
-    shard: {
-      type: 'string',
-      description: 'The shard to run in <order>/<total> format',
-    },
   },
 })
 
-if (values.help) {
+function displayHelp() {
   console.log(`
 Usage: agent-eval [options]
 
 Options:
-  -a, --artifacts <dir>      The directory to save artifacts to
+  -a, --artifacts <dir>      The directory to save artifacts to (default: ./artifacts)
   -b, --benchmark <file>     The file name of the benchmark to run
-      --benchmarks <dir>     The directory containing local benchmark files
+      --benchmarks <dir>     The directory containing local benchmark files (default: ./benchmarks)
   -c, --concurrency <num>    The number of treatments to run in parallel
       --docker-image <image> The Docker container image to use for running treatments (must be a Debian-based Node image with apt-get and a node user, e.g. node:26.5.0-slim)
   -e, --experiment <file>    The file name of the experiment to run
-      --experiments <dir>    The directory containing local experiment files
+      --experiments <dir>    The directory containing local experiment files (default: ./experiments)
   -h, --help                 Learn more about the command and its options
       --output <file>        The target file in which results are written (default: output.json)
-      --scenarios <dir>      The directory containing scenario directories
-      --shard <order/total>  The shard to run
+      --scenarios <dir>      The directory containing scenario directories (default: ./scenarios)
 `)
+}
+
+if (values.help) {
+  displayHelp()
   process.exit(0)
 }
 
@@ -93,440 +84,457 @@ const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN
 const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY
 
 if (!COPILOT_GITHUB_TOKEN) {
-  throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run the experiments')
-}
-
-const ARTIFACTS_DIR = path.resolve(values.artifacts ?? 'artifacts')
-const DOCKER_IMAGE = values['docker-image']?.trim() || undefined
-const parsedConcurrency = values.concurrency ? parseInt(values.concurrency, 10) : 1
-const MAX_CONCURRENCY =
-  Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1
-    ? parsedConcurrency
-    : 1
-const SHARD = values.shard ? parseShard(values.shard) : undefined
-let selectedExperiment: {
-  id: string
-  config: ExperimentConfig
+  throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval')
 }
 
-if (!existsSync(ARTIFACTS_DIR)) {
-  await fs.mkdir(ARTIFACTS_DIR, {recursive: true})
-}
+const env = getEnvironmentConfig({
+  artifactsDirectory: values.artifacts,
+  benchmarksDirectory: values.benchmarks,
+  concurrency: values.concurrency,
+  copilotToken: COPILOT_GITHUB_TOKEN,
+  dockerImage: values['docker-image']?.trim(),
+  experimentsDirectory: values.experiments,
+  outputPath: values.output,
+  scenariosDirectory: values.scenarios,
+})
 
 if (values.benchmark) {
-  //
+  await runBenchmark({
+    env,
+    id: values.benchmark,
+  })
 } else if (values.experiment) {
-  const config = await findExperiment(values.experiment, {
-    directory: values.experiments,
+  await runExperiment({
+    env,
+    id: values.experiment,
   })
-  if (!config) {
-    const experiments = await listExperiments({
-      directory: values.experiments,
-    })
-    throw new Error(
-      `Experiment "${values.experiment}" was not found. Available experiments:\n${experiments
-        .map(([id]) => id)
-        .join('\n')}`,
-    )
-  }
-
-  selectedExperiment = {
-    id: existsSync(values.experiment)
-      ? path.basename(values.experiment, path.extname(values.experiment))
-      : values.experiment,
-    config,
-  }
 } else {
-  const experiments = await listExperiments({
-    directory: values.experiments,
-  })
-  if (experiments.length !== 1) {
-    throw new Error(
-      `Select an experiment with --experiment. Available experiments:\n${experiments.map(([id]) => id).join('\n')}`,
-    )
-  }
-
-  selectedExperiment = {
-    id: experiments[0][0],
-    config: experiments[0][1],
-  }
-}
-
-function randomize<T>(input: Array<T>): Array<T> {
-  const randomized: Array<T> = input.slice()
-
-  // Fisher–Yates shuffle
-  for (let i = randomized.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1))
-    ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]]
-  }
-
-  return randomized
-}
-
-function getSuccessRate(result: TreatmentResult): number {
-  if (result.testResults.numTotalTests === 0) {
-    return 0
-  }
-
-  return result.testResults.numPassedTests / result.testResults.numTotalTests
-}
-
-function compareResults(a: TreatmentResult, b: TreatmentResult): number {
-  return (
-    getSuccessRate(b) - getSuccessRate(a) ||
-    a.assistant.outputTokens - b.assistant.outputTokens ||
-    a.assistant.sessionDurationMs - b.assistant.sessionDurationMs ||
-    a.assistant.premiumRequests - b.assistant.premiumRequests ||
-    a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) ||
-    a.treatment.config.name.localeCompare(b.treatment.config.name) ||
-    a.treatment.model.localeCompare(b.treatment.model) ||
-    (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') ||
-    a.treatment.scenario.id.localeCompare(b.treatment.scenario.id)
-  )
-}
-
-type ResultSummary = {
-  experiment: string
-  treatment?: string
-  scenario?: string
-  model?: Model
-  reasoningEffort?: ReasoningEffort
-  runs: number
-  numPassedTests: number
-  numTotalTests: number
-  outputTokens: number
-  premiumRequests: number
-  sessionDurationMs: number
-  totalApiDurationMs: number
-}
-
-type ResultSummaryValues = {
-  treatment?: string
-  scenario?: string
-  model?: Model
-  reasoningEffort?: ReasoningEffort
-}
-
-function createResultSummary(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): ResultSummary {
-  return {
-    experiment: result.treatment.experiment.name,
-    treatment: summaryValues.treatment,
-    scenario: summaryValues.scenario,
-    model: summaryValues.model,
-    reasoningEffort: summaryValues.reasoningEffort,
-    runs: 0,
-    numPassedTests: 0,
-    numTotalTests: 0,
-    outputTokens: 0,
-    premiumRequests: 0,
-    sessionDurationMs: 0,
-    totalApiDurationMs: 0,
-  }
-}
-
-function addResultToSummary(summary: ResultSummary, result: TreatmentResult) {
-  summary.runs += 1
-  summary.numPassedTests += result.testResults.numPassedTests
-  summary.numTotalTests += result.testResults.numTotalTests
-  summary.outputTokens += result.assistant.outputTokens
-  summary.premiumRequests += result.assistant.premiumRequests
-  summary.sessionDurationMs += result.assistant.sessionDurationMs
-  summary.totalApiDurationMs += result.assistant.totalApiDurationMs
-}
-
-function getSummarySuccessRate(summary: ResultSummary): number {
-  if (summary.numTotalTests === 0) {
-    return 0
-  }
-
-  return summary.numPassedTests / summary.numTotalTests
-}
-
-function compareSummaries(a: ResultSummary, b: ResultSummary): number {
-  return (
-    getSummarySuccessRate(b) - getSummarySuccessRate(a) ||
-    a.outputTokens - b.outputTokens ||
-    a.sessionDurationMs - b.sessionDurationMs ||
-    a.premiumRequests - b.premiumRequests ||
-    a.experiment.localeCompare(b.experiment) ||
-    (a.treatment ?? '').localeCompare(b.treatment ?? '') ||
-    (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
-    (a.model ?? '').localeCompare(b.model ?? '') ||
-    (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
-  )
-}
-
-function formatPercent(value: number): string {
-  return `${(value * 100).toFixed(1)}%`
-}
-
-function formatDuration(ms: number): string {
-  const seconds = ms / 1000
-
-  if (seconds < 60) {
-    return `${seconds.toFixed(1)}s`
-  }
-
-  const minutes = Math.floor(seconds / 60)
-  const remainingSeconds = seconds - minutes * 60
-  return `${minutes}m ${remainingSeconds.toFixed(1)}s`
-}
-
-function formatNumber(value: number): string {
-  return new Intl.NumberFormat('en-US').format(value)
+  displayHelp()
 }
 
-type TableRow = Record<string, string | number>
-
-function formatTable(rows: Array<TableRow>, columns: Array<string>): string {
-  const columnWidths = columns.map(column => {
-    let width = column.length
-
-    for (const row of rows) {
-      width = Math.max(width, String(row[column] ?? '').length)
-    }
-
-    return width
-  })
-
-  const formatRow = (row: TableRow) => {
-    return columns
-      .map((column, index) => {
-        return String(row[column] ?? '').padEnd(columnWidths[index])
-      })
-      .join('  ')
-  }
-
-  return [
-    formatRow(Object.fromEntries(columns.map(column => [column, column]))),
-    columnWidths.map(width => '-'.repeat(width)).join('  '),
-    ...rows.map(formatRow),
-  ].join('\n')
-}
-
-function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): string {
-  return [
-    result.treatment.experiment.name,
-    summaryValues.treatment ?? '',
-    summaryValues.scenario ?? '',
-    summaryValues.model ?? '',
-    summaryValues.reasoningEffort ?? '',
-  ].join('\0')
-}
-
-type ResultHierarchy = Array<{
-  experiment: string
-  treatments: Array<{
-    summary: ResultSummary
-    scenarios: Array<{
-      summary: ResultSummary
-      models: Array<ResultSummary>
-    }>
-  }>
-}>
-
-function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
-  const experiments = new Set<string>()
-  const treatmentSummaries = new Map<string, ResultSummary>()
-  const scenarioSummaries = new Map<string, ResultSummary>()
-  const modelSummaries = new Map<string, ResultSummary>()
-
-  for (const result of results) {
-    experiments.add(result.treatment.experiment.name)
-
-    const treatmentValues = {
-      treatment: result.treatment.config.name,
-    }
-    const treatmentKey = getSummaryKey(result, treatmentValues)
-    const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues)
-    addResultToSummary(treatmentSummary, result)
-    treatmentSummaries.set(treatmentKey, treatmentSummary)
-
-    const scenarioValues = {
-      treatment: result.treatment.config.name,
-      scenario: result.treatment.scenario.id,
-    }
-    const scenarioKey = getSummaryKey(result, scenarioValues)
-    const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(result, scenarioValues)
-    addResultToSummary(scenarioSummary, result)
-    scenarioSummaries.set(scenarioKey, scenarioSummary)
-
-    const modelValues = {
-      treatment: result.treatment.config.name,
-      scenario: result.treatment.scenario.id,
-      model: result.treatment.model,
-      reasoningEffort: result.treatment.reasoningEffort,
-    }
-    const modelKey = getSummaryKey(result, modelValues)
-    const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(result, modelValues)
-    addResultToSummary(modelSummary, result)
-    modelSummaries.set(modelKey, modelSummary)
-  }
-
-  return [...experiments].toSorted().map(experiment => {
-    return {
-      experiment,
-      treatments: [...treatmentSummaries.values()]
-        .filter(treatmentSummary => {
-          return treatmentSummary.experiment === experiment
-        })
-        .toSorted(compareSummaries)
-        .map(summary => {
-          return {
-            summary,
-            scenarios: [...scenarioSummaries.values()]
-              .filter(scenarioSummary => {
-                return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment
-              })
-              .toSorted(compareSummaries)
-              .map(scenarioSummary => {
-                return {
-                  summary: scenarioSummary,
-                  models: [...modelSummaries.values()]
-                    .filter(modelSummary => {
-                      return (
-                        modelSummary.experiment === experiment &&
-                        modelSummary.treatment === summary.treatment &&
-                        modelSummary.scenario === scenarioSummary.scenario
-                      )
-                    })
-                    .toSorted(compareSummaries),
-                }
-              }),
-          }
-        }),
-    }
-  })
-}
-
-function formatResultSummaries(results: Array<TreatmentResult>): string {
-  const columns = [
-    'Experiment',
-    'Treatment',
-    'Scenario',
-    'Model',
-    'Reasoning Effort',
-    'Success Rate',
-    'Tests',
-    'Runs',
-    'Output Tokens',
-    'Premium Requests',
-    'Session Time',
-    'API Time',
-  ]
-  const rows: Array<TableRow> = []
-
-  for (const {treatments} of getResultSummaries(results)) {
-    for (const {summary, scenarios} of treatments) {
-      rows.push(formatSummaryRow(summary, 'treatment'))
-
-      for (const {summary: scenarioSummary, models} of scenarios) {
-        rows.push(formatSummaryRow(scenarioSummary, 'scenario'))
-
-        for (const model of models) {
-          rows.push(formatSummaryRow(model, 'model'))
-        }
-      }
-    }
-  }
-
-  return formatTable(rows, columns)
-}
-
-async function appendResultsToJobSummary(resultSummaries: string) {
-  if (!GITHUB_STEP_SUMMARY) {
-    return
-  }
-
-  await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`)
-}
-
-function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow {
-  return {
-    Experiment: level === 'treatment' ? summary.experiment : '',
-    Treatment: level === 'treatment' ? (summary.treatment ?? '') : '',
-    Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario ?? ''}` : '',
-    Model: level === 'model' ? `    ${summary.model ?? ''}` : 'All models',
-    'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '',
-    'Success Rate': formatPercent(getSummarySuccessRate(summary)),
-    Tests: `${summary.numPassedTests}/${summary.numTotalTests}`,
-    Runs: summary.runs,
-    'Output Tokens': formatNumber(summary.outputTokens),
-    'Premium Requests': formatNumber(summary.premiumRequests),
-    'Session Time': formatDuration(summary.sessionDurationMs),
-    'API Time': formatDuration(summary.totalApiDurationMs),
-  }
-}
-
-const config = selectedExperiment.config
-
-console.log('Running experiment:', config.name)
-
-const scenarios = await Promise.all(
-  config.scenarios.map(scenarioConfig => {
-    return resolveExperimentScenario(scenarioConfig, {
-      directory: values.scenarios,
-    })
-  }),
-)
-
-const treatments: Array<Treatment> = config.models.flatMap(modelConfig => {
-  return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => {
-    return scenarios.flatMap(scenarioConfig => {
-      return [
-        {
-          config: ControlTreatment,
-          scenario: scenarioConfig,
-          experiment: config,
-          id: randomUUID(),
-          model,
-          reasoningEffort,
-        },
-        ...config.treatments.map(treatment => {
-          return {
-            config: treatment,
-            scenario: scenarioConfig,
-            experiment: config,
-            id: randomUUID(),
-            model,
-            reasoningEffort,
-          }
-        }),
-      ]
-    })
-  })
-})
-
-// Randomize treatments to mitigate any ordering effects. We want to make sure
-// that if there are any external factors that could impact the scenarios (e.g.
-// rate limits, resource constraints), they are more likely to impact all
-// scenarios rather than just the ones at the end.
-const selectedTreatments = SHARD ? selectShard(treatments, SHARD) : treatments
-const results: Array<TreatmentResult> = await run(randomize(selectedTreatments), {
-  artifactsDirectory: ARTIFACTS_DIR,
-  copilotToken: COPILOT_GITHUB_TOKEN,
-  dockerImage: DOCKER_IMAGE,
-  maxConcurrency: MAX_CONCURRENCY,
-})
-
-const sortedResults = results.toSorted(compareResults)
-const resultSummaries = formatResultSummaries(sortedResults)
-console.log(resultSummaries)
-await appendResultsToJobSummary(resultSummaries)
-
-const outputFilePath = path.isAbsolute(values.output) ? values.output : path.resolve(process.cwd(), values.output)
-
-if (!existsSync(path.dirname(outputFilePath))) {
-  await fs.mkdir(path.dirname(outputFilePath), {recursive: true})
-}
-
-const output = createAgentEvalOutput({
-  id: randomUUID(),
-  experimentId: selectedExperiment.id,
-  experiment: config,
-  scenarios,
-  results: sortedResults,
-})
-
-await fs.writeFile(outputFilePath, JSON.stringify(output, null, 2))
+// let selectedExperiment: {
+//   id: string
+//   config: ExperimentConfig
+// }
+//
+// if (!existsSync(ARTIFACTS_DIR)) {
+//   await fs.mkdir(ARTIFACTS_DIR, {recursive: true})
+// }
+//
+// if (values.benchmark) {
+//   //
+// } else if (values.experiment) {
+//   const config = await findExperiment(values.experiment, {
+//     directory: values.experiments,
+//   })
+//   if (!config) {
+//     const experiments = await listExperiments({
+//       directory: values.experiments,
+//     })
+//     throw new Error(
+//       `Experiment "${values.experiment}" was not found. Available experiments:\n${experiments
+//         .map(([id]) => id)
+//         .join('\n')}`,
+//     )
+//   }
+//
+//   selectedExperiment = {
+//     id: existsSync(values.experiment)
+//       ? path.basename(values.experiment, path.extname(values.experiment))
+//       : values.experiment,
+//     config,
+//   }
+// } else {
+//   const experiments = await listExperiments({
+//     directory: values.experiments,
+//   })
+//   if (experiments.length !== 1) {
+//     throw new Error(
+//       `Select an experiment with --experiment. Available experiments:\n${experiments.map(([id]) => id).join('\n')}`,
+//     )
+//   }
+//
+//   selectedExperiment = {
+//     id: experiments[0][0],
+//     config: experiments[0][1],
+//   }
+// }
+//
+// function randomize<T>(input: Array<T>): Array<T> {
+//   const randomized: Array<T> = input.slice()
+//
+//   // Fisher–Yates shuffle
+//   for (let i = randomized.length - 1; i > 0; i--) {
+//     const j = Math.floor(Math.random() * (i + 1))
+//     ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]]
+//   }
+//
+//   return randomized
+// }
+//
+// function getSuccessRate(result: TreatmentResult): number {
+//   if (result.testResults.numTotalTests === 0) {
+//     return 0
+//   }
+//
+//   return result.testResults.numPassedTests / result.testResults.numTotalTests
+// }
+//
+// function compareResults(a: TreatmentResult, b: TreatmentResult): number {
+//   return (
+//     getSuccessRate(b) - getSuccessRate(a) ||
+//     a.assistant.outputTokens - b.assistant.outputTokens ||
+//     a.assistant.sessionDurationMs - b.assistant.sessionDurationMs ||
+//     a.assistant.premiumRequests - b.assistant.premiumRequests ||
+//     a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) ||
+//     a.treatment.config.name.localeCompare(b.treatment.config.name) ||
+//     a.treatment.model.localeCompare(b.treatment.model) ||
+//     (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') ||
+//     a.treatment.scenario.id.localeCompare(b.treatment.scenario.id)
+//   )
+// }
+//
+// type ResultSummary = {
+//   experiment: string
+//   treatment?: string
+//   scenario?: string
+//   model?: Model
+//   reasoningEffort?: ReasoningEffort
+//   runs: number
+//   numPassedTests: number
+//   numTotalTests: number
+//   outputTokens: number
+//   premiumRequests: number
+//   sessionDurationMs: number
+//   totalApiDurationMs: number
+// }
+//
+// type ResultSummaryValues = {
+//   treatment?: string
+//   scenario?: string
+//   model?: Model
+//   reasoningEffort?: ReasoningEffort
+// }
+//
+// function createResultSummary(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): ResultSummary {
+//   return {
+//     experiment: result.treatment.experiment.name,
+//     treatment: summaryValues.treatment,
+//     scenario: summaryValues.scenario,
+//     model: summaryValues.model,
+//     reasoningEffort: summaryValues.reasoningEffort,
+//     runs: 0,
+//     numPassedTests: 0,
+//     numTotalTests: 0,
+//     outputTokens: 0,
+//     premiumRequests: 0,
+//     sessionDurationMs: 0,
+//     totalApiDurationMs: 0,
+//   }
+// }
+//
+// function addResultToSummary(summary: ResultSummary, result: TreatmentResult) {
+//   summary.runs += 1
+//   summary.numPassedTests += result.testResults.numPassedTests
+//   summary.numTotalTests += result.testResults.numTotalTests
+//   summary.outputTokens += result.assistant.outputTokens
+//   summary.premiumRequests += result.assistant.premiumRequests
+//   summary.sessionDurationMs += result.assistant.sessionDurationMs
+//   summary.totalApiDurationMs += result.assistant.totalApiDurationMs
+// }
+//
+// function getSummarySuccessRate(summary: ResultSummary): number {
+//   if (summary.numTotalTests === 0) {
+//     return 0
+//   }
+//
+//   return summary.numPassedTests / summary.numTotalTests
+// }
+//
+// function compareSummaries(a: ResultSummary, b: ResultSummary): number {
+//   return (
+//     getSummarySuccessRate(b) - getSummarySuccessRate(a) ||
+//     a.outputTokens - b.outputTokens ||
+//     a.sessionDurationMs - b.sessionDurationMs ||
+//     a.premiumRequests - b.premiumRequests ||
+//     a.experiment.localeCompare(b.experiment) ||
+//     (a.treatment ?? '').localeCompare(b.treatment ?? '') ||
+//     (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
+//     (a.model ?? '').localeCompare(b.model ?? '') ||
+//     (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
+//   )
+// }
+//
+// function formatPercent(value: number): string {
+//   return `${(value * 100).toFixed(1)}%`
+// }
+//
+// function formatDuration(ms: number): string {
+//   const seconds = ms / 1000
+//
+//   if (seconds < 60) {
+//     return `${seconds.toFixed(1)}s`
+//   }
+//
+//   const minutes = Math.floor(seconds / 60)
+//   const remainingSeconds = seconds - minutes * 60
+//   return `${minutes}m ${remainingSeconds.toFixed(1)}s`
+// }
+//
+// function formatNumber(value: number): string {
+//   return new Intl.NumberFormat('en-US').format(value)
+// }
+//
+// type TableRow = Record<string, string | number>
+//
+// function formatTable(rows: Array<TableRow>, columns: Array<string>): string {
+//   const columnWidths = columns.map(column => {
+//     let width = column.length
+//
+//     for (const row of rows) {
+//       width = Math.max(width, String(row[column] ?? '').length)
+//     }
+//
+//     return width
+//   })
+//
+//   const formatRow = (row: TableRow) => {
+//     return columns
+//       .map((column, index) => {
+//         return String(row[column] ?? '').padEnd(columnWidths[index])
+//       })
+//       .join('  ')
+//   }
+//
+//   return [
+//     formatRow(Object.fromEntries(columns.map(column => [column, column]))),
+//     columnWidths.map(width => '-'.repeat(width)).join('  '),
+//     ...rows.map(formatRow),
+//   ].join('\n')
+// }
+//
+// function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): string {
+//   return [
+//     result.treatment.experiment.name,
+//     summaryValues.treatment ?? '',
+//     summaryValues.scenario ?? '',
+//     summaryValues.model ?? '',
+//     summaryValues.reasoningEffort ?? '',
+//   ].join('\0')
+// }
+//
+// type ResultHierarchy = Array<{
+//   experiment: string
+//   treatments: Array<{
+//     summary: ResultSummary
+//     scenarios: Array<{
+//       summary: ResultSummary
+//       models: Array<ResultSummary>
+//     }>
+//   }>
+// }>
+//
+// function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
+//   const experiments = new Set<string>()
+//   const treatmentSummaries = new Map<string, ResultSummary>()
+//   const scenarioSummaries = new Map<string, ResultSummary>()
+//   const modelSummaries = new Map<string, ResultSummary>()
+//
+//   for (const result of results) {
+//     experiments.add(result.treatment.experiment.name)
+//
+//     const treatmentValues = {
+//       treatment: result.treatment.config.name,
+//     }
+//     const treatmentKey = getSummaryKey(result, treatmentValues)
+//     const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues)
+//     addResultToSummary(treatmentSummary, result)
+//     treatmentSummaries.set(treatmentKey, treatmentSummary)
+//
+//     const scenarioValues = {
+//       treatment: result.treatment.config.name,
+//       scenario: result.treatment.scenario.id,
+//     }
+//     const scenarioKey = getSummaryKey(result, scenarioValues)
+//     const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(result, scenarioValues)
+//     addResultToSummary(scenarioSummary, result)
+//     scenarioSummaries.set(scenarioKey, scenarioSummary)
+//
+//     const modelValues = {
+//       treatment: result.treatment.config.name,
+//       scenario: result.treatment.scenario.id,
+//       model: result.treatment.model,
+//       reasoningEffort: result.treatment.reasoningEffort,
+//     }
+//     const modelKey = getSummaryKey(result, modelValues)
+//     const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(result, modelValues)
+//     addResultToSummary(modelSummary, result)
+//     modelSummaries.set(modelKey, modelSummary)
+//   }
+//
+//   return [...experiments].toSorted().map(experiment => {
+//     return {
+//       experiment,
+//       treatments: [...treatmentSummaries.values()]
+//         .filter(treatmentSummary => {
+//           return treatmentSummary.experiment === experiment
+//         })
+//         .toSorted(compareSummaries)
+//         .map(summary => {
+//           return {
+//             summary,
+//             scenarios: [...scenarioSummaries.values()]
+//               .filter(scenarioSummary => {
+//                 return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment
+//               })
+//               .toSorted(compareSummaries)
+//               .map(scenarioSummary => {
+//                 return {
+//                   summary: scenarioSummary,
+//                   models: [...modelSummaries.values()]
+//                     .filter(modelSummary => {
+//                       return (
+//                         modelSummary.experiment === experiment &&
+//                         modelSummary.treatment === summary.treatment &&
+//                         modelSummary.scenario === scenarioSummary.scenario
+//                       )
+//                     })
+//                     .toSorted(compareSummaries),
+//                 }
+//               }),
+//           }
+//         }),
+//     }
+//   })
+// }
+//
+// function formatResultSummaries(results: Array<TreatmentResult>): string {
+//   const columns = [
+//     'Experiment',
+//     'Treatment',
+//     'Scenario',
+//     'Model',
+//     'Reasoning Effort',
+//     'Success Rate',
+//     'Tests',
+//     'Runs',
+//     'Output Tokens',
+//     'Premium Requests',
+//     'Session Time',
+//     'API Time',
+//   ]
+//   const rows: Array<TableRow> = []
+//
+//   for (const {treatments} of getResultSummaries(results)) {
+//     for (const {summary, scenarios} of treatments) {
+//       rows.push(formatSummaryRow(summary, 'treatment'))
+//
+//       for (const {summary: scenarioSummary, models} of scenarios) {
+//         rows.push(formatSummaryRow(scenarioSummary, 'scenario'))
+//
+//         for (const model of models) {
+//           rows.push(formatSummaryRow(model, 'model'))
+//         }
+//       }
+//     }
+//   }
+//
+//   return formatTable(rows, columns)
+// }
+//
+// async function appendResultsToJobSummary(resultSummaries: string) {
+//   if (!GITHUB_STEP_SUMMARY) {
+//     return
+//   }
+//
+//   await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`)
+// }
+//
+// function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow {
+//   return {
+//     Experiment: level === 'treatment' ? summary.experiment : '',
+//     Treatment: level === 'treatment' ? (summary.treatment ?? '') : '',
+//     Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario ?? ''}` : '',
+//     Model: level === 'model' ? `    ${summary.model ?? ''}` : 'All models',
+//     'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '',
+//     'Success Rate': formatPercent(getSummarySuccessRate(summary)),
+//     Tests: `${summary.numPassedTests}/${summary.numTotalTests}`,
+//     Runs: summary.runs,
+//     'Output Tokens': formatNumber(summary.outputTokens),
+//     'Premium Requests': formatNumber(summary.premiumRequests),
+//     'Session Time': formatDuration(summary.sessionDurationMs),
+//     'API Time': formatDuration(summary.totalApiDurationMs),
+//   }
+// }
+//
+// const config = selectedExperiment.config
+//
+// console.log('Running experiment:', config.name)
+//
+// const scenarios = await Promise.all(
+//   config.scenarios.map(scenarioConfig => {
+//     return resolveExperimentScenario(scenarioConfig, {
+//       directory: values.scenarios,
+//     })
+//   }),
+// )
+//
+// const treatments: Array<Treatment> = config.models.flatMap(modelConfig => {
+//   return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => {
+//     return scenarios.flatMap(scenarioConfig => {
+//       return [
+//         {
+//           config: ControlTreatment,
+//           scenario: scenarioConfig,
+//           experiment: config,
+//           id: randomUUID(),
+//           model,
+//           reasoningEffort,
+//         },
+//         ...config.treatments.map(treatment => {
+//           return {
+//             config: treatment,
+//             scenario: scenarioConfig,
+//             experiment: config,
+//             id: randomUUID(),
+//             model,
+//             reasoningEffort,
+//           }
+//         }),
+//       ]
+//     })
+//   })
+// })
+//
+// // Randomize treatments to mitigate any ordering effects. We want to make sure
+// // that if there are any external factors that could impact the scenarios (e.g.
+// // rate limits, resource constraints), they are more likely to impact all
+// // scenarios rather than just the ones at the end.
+// const selectedTreatments = SHARD ? selectShard(treatments, SHARD) : treatments
+// const results: Array<TreatmentResult> = await run(randomize(selectedTreatments), {
+//   artifactsDirectory: ARTIFACTS_DIR,
+//   copilotToken: COPILOT_GITHUB_TOKEN,
+//   dockerImage: DOCKER_IMAGE,
+//   maxConcurrency: MAX_CONCURRENCY,
+// })
+//
+// const sortedResults = results.toSorted(compareResults)
+// const resultSummaries = formatResultSummaries(sortedResults)
+// console.log(resultSummaries)
+// await appendResultsToJobSummary(resultSummaries)
+//
+// const outputFilePath = path.isAbsolute(values.output) ? values.output : path.resolve(process.cwd(), values.output)
+//
+// if (!existsSync(path.dirname(outputFilePath))) {
+//   await fs.mkdir(path.dirname(outputFilePath), {recursive: true})
+// }
+//
+// const output = createAgentEvalOutput({
+//   id: randomUUID(),
+//   experimentId: selectedExperiment.id,
+//   experiment: config,
+//   scenarios,
+//   results: sortedResults,
+// })
+//
+// await fs.writeFile(outputFilePath, JSON.stringify(output, null, 2))
diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts
new file mode 100644
index 00000000..6bf65893
--- /dev/null
+++ b/packages/agent-eval/src/environment.ts
@@ -0,0 +1,51 @@
+import path from 'node:path'
+import {DEFAULT_DOCKER_IMAGE} from './sandbox'
+
+type EnvironmentConfig = {
+  artifactsDirectory: string
+  benchmarksDirectory: string
+  concurrency: number
+  copilotToken: string
+  dockerImage: string
+  experimentsDirectory: string
+  outputPath: string
+  scenariosDirectory: string
+}
+
+type EnvironmentOptions = {
+  artifactsDirectory?: string
+  benchmarksDirectory?: string
+  concurrency?: string
+  copilotToken: string
+  dockerImage?: string
+  experimentsDirectory?: string
+  outputPath?: string
+  scenariosDirectory?: string
+}
+
+function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig {
+  const artifactsDirectory = path.resolve(options.artifactsDirectory ?? 'artifacts')
+  const benchmarksDirectory = path.resolve(options.benchmarksDirectory ?? 'benchmarks')
+  const parsedConcurrency = options.concurrency ? parseInt(options.concurrency, 10) : 1
+  const concurrency =
+    Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1
+      ? parsedConcurrency
+      : 1
+  const experimentsDirectory = path.resolve(options.experimentsDirectory ?? 'experiments')
+  const outputPath = path.resolve(options.outputPath ?? 'output.json')
+  const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios')
+
+  return {
+    artifactsDirectory,
+    benchmarksDirectory,
+    concurrency,
+    copilotToken: options.copilotToken,
+    dockerImage: options.dockerImage ?? DEFAULT_DOCKER_IMAGE,
+    experimentsDirectory,
+    outputPath,
+    scenariosDirectory,
+  }
+}
+
+export {getEnvironmentConfig}
+export type {EnvironmentConfig}
diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts
deleted file mode 100644
index 49a9ce6f..00000000
--- a/packages/agent-eval/src/experiment-config.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-import type {ExperimentModelConfig} from './model'
-import type {SandboxInstance} from './sandbox'
-
-type ScenarioConfig = {
-  description?: string
-  prompt: string
-  tags?: Array<string>
-}
-
-type InlineScenarioConfig = {
-  name?: string
-  path: string
-}
-
-type ExperimentScenarioConfig = string | InlineScenarioConfig
-
-type ExperimentConfig = {
-  name: string
-  description: string
-  models: Array<ExperimentModelConfig>
-  scenarios: Array<ExperimentScenarioConfig>
-  setup?: Setup
-  treatments: Array<TreatmentConfig>
-}
-
-type TreatmentConfig = {
-  name: string
-  setup?: Setup
-}
-
-type Setup = ({sandbox}: {sandbox: SandboxInstance}) => Promise<void>
-
-const ControlTreatment: TreatmentConfig = {
-  name: 'Control',
-}
-
-export {ControlTreatment}
-export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig}
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
new file mode 100644
index 00000000..1a9c40da
--- /dev/null
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -0,0 +1,152 @@
+import {expect, test} from 'vitest'
+import {defineConfig, getExperiment, listExperiments} from './experiment'
+import {VirtualHost} from './host'
+
+const config = defineConfig({
+  name: 'Test experiment',
+  description: 'Tests an experiment',
+  models: ['gpt-5.6-sol'],
+  scenarios: [
+    {
+      prompt: 'Complete the task',
+    },
+  ],
+  treatments: [
+    {
+      name: 'Test treatment',
+    },
+  ],
+})
+
+test('listExperiments loads named and default exports from supported files', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = VirtualHost.create({
+    '/experiments': {
+      'named.ts': `export const experiment = ${serializedConfig}`,
+      'default.js': `export default ${serializedConfig}`,
+      'commonjs.cjs': `export default ${serializedConfig}`,
+      'module.mjs': `export default ${serializedConfig}`,
+    },
+  })
+
+  const experiments = await listExperiments(host, '/experiments')
+
+  expect(experiments).toHaveLength(4)
+  expect(experiments).toEqual(
+    expect.arrayContaining([
+      {
+        ...config,
+        id: 'named',
+        filepath: '/experiments/named.ts',
+      },
+      {
+        ...config,
+        id: 'default',
+        filepath: '/experiments/default.js',
+      },
+      {
+        ...config,
+        id: 'commonjs',
+        filepath: '/experiments/commonjs.cjs',
+      },
+      {
+        ...config,
+        id: 'module',
+        filepath: '/experiments/module.mjs',
+      },
+    ]),
+  )
+})
+
+test('listExperiments prefers the named experiment export', async () => {
+  const namedConfig = {
+    ...config,
+    name: 'Named experiment',
+  }
+  const defaultConfig = {
+    ...config,
+    name: 'Default experiment',
+  }
+  const host = VirtualHost.create({
+    '/experiments': {
+      'experiment.ts': [
+        `export const experiment = ${JSON.stringify(namedConfig)}`,
+        `export default ${JSON.stringify(defaultConfig)}`,
+      ].join('\n'),
+    },
+  })
+
+  await expect(listExperiments(host, '/experiments')).resolves.toEqual([
+    {
+      ...namedConfig,
+      id: 'experiment',
+      filepath: '/experiments/experiment.ts',
+    },
+  ])
+})
+
+test('listExperiments ignores unsupported, reserved, missing, and invalid configs', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = VirtualHost.create({
+    '/experiments': {
+      'valid.ts': `export const experiment = ${serializedConfig}`,
+      'types.d.ts': `export const experiment = ${serializedConfig}`,
+      'index.ts': `export const experiment = ${serializedConfig}`,
+      'unsupported.json': serializedConfig,
+      'missing.ts': 'export const value = true',
+      'invalid.ts': 'export const experiment = {}',
+    },
+  })
+
+  await expect(listExperiments(host, '/experiments')).resolves.toEqual([
+    {
+      ...config,
+      id: 'valid',
+      filepath: '/experiments/valid.ts',
+    },
+  ])
+})
+
+test('listExperiments throws when the directory does not exist', async () => {
+  const host = VirtualHost.create()
+
+  await expect(listExperiments(host, '/experiments')).rejects.toThrowError(
+    'Experiments directory does not exist: /experiments',
+  )
+})
+
+test('listExperiments throws when the path is not a directory', async () => {
+  const host = VirtualHost.create({
+    '/experiments': '',
+  })
+
+  await expect(listExperiments(host, '/experiments')).rejects.toThrowError(
+    'Experiments path is not a directory: /experiments',
+  )
+})
+
+test('getExperiment returns the experiment matching the id', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = VirtualHost.create({
+    '/experiments': {
+      'first.ts': `export const experiment = ${serializedConfig}`,
+      'second.ts': `export const experiment = ${serializedConfig}`,
+    },
+  })
+
+  await expect(getExperiment(host, '/experiments', 'second')).resolves.toEqual({
+    ...config,
+    id: 'second',
+    filepath: '/experiments/second.ts',
+  })
+})
+
+test('getExperiment throws when the experiment is not found', async () => {
+  const host = VirtualHost.create({
+    '/experiments': {},
+  })
+
+  await expect(getExperiment(host, '/experiments', 'missing')).rejects.toThrowError(
+    'Experiment "missing" was not found in: /experiments',
+  )
+})
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 4c60be63..bf9a8c39 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -1,31 +1,176 @@
-import type {ExperimentConfig, TreatmentConfig} from './experiment-config'
-import type {
-  CopilotPluginConfig,
-  CopilotPluginSource,
-  LocalCopilotPluginSource,
-  McpServerConfig,
-  RemoteCopilotPluginSource,
-  SandboxInstance,
-} from './sandbox'
-import type {ExperimentModelConfig, Model, ModelConfig, ModelInfo, ReasoningEffort} from './model'
+import path from 'node:path'
+import * as z from 'zod/mini'
+import {ModelVariantConfigSchema, type ModelVariant, type ModelVariantConfig} from './model'
+import {getScenario, type Scenario} from './scenario'
+import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
+import {DefaultHost, type Host} from './host'
+import type {EnvironmentConfig} from './environment'
+import {logger} from './logger'
+import {createPlan} from './plan'
+import {run as runTrials} from './run'
+import type {Trial} from './trial'
+import {randomUUID} from 'node:crypto'
+
+const ExperimentConfigSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  models: z.array(ModelVariantConfigSchema),
+  scenarios: z.array(z.string()),
+  setup: z.optional(TreatmentSetupSchema),
+  treatments: z.array(TreatmentSchema),
+})
+
+type ExperimentConfig = z.infer<typeof ExperimentConfigSchema>
 
 function defineConfig(config: ExperimentConfig): ExperimentConfig {
   return config
 }
 
-export type {
-  ExperimentConfig,
-  ExperimentModelConfig,
-  CopilotPluginConfig,
-  CopilotPluginSource,
-  LocalCopilotPluginSource,
-  McpServerConfig,
-  Model,
-  ModelConfig,
-  ModelInfo,
-  ReasoningEffort,
-  RemoteCopilotPluginSource,
-  SandboxInstance as Sandbox,
-  TreatmentConfig,
+type Experiment = {
+  id: string
+  filepath: string
+  name: ExperimentConfig['name']
+  description: ExperimentConfig['description']
+  models: Array<ModelVariant>
+  scenarios: Array<Scenario>
+  setup?: TreatmentSetup
+  treatments: Array<Treatment>
+}
+
+type ExperimentModule = {
+  default?: unknown
+  experiment?: unknown
 }
-export {defineConfig}
+
+const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
+
+async function listExperiments({
+  host = DefaultHost,
+  experimentsDirectory,
+  scenariosDirectory,
+}: {
+  host?: Host
+  experimentsDirectory: string
+  scenariosDirectory: string
+}): Promise<Array<Experiment>> {
+  if (!host.existsSync(experimentsDirectory)) {
+    throw new Error(`Experiments directory does not exist: ${experimentsDirectory}`)
+  }
+
+  const stats = await host.fs.stat(experimentsDirectory)
+  if (!stats.isDirectory()) {
+    throw new Error(`Experiments path is not a directory: ${experimentsDirectory}`)
+  }
+
+  const filenames = await host.fs.readdir(experimentsDirectory)
+  const experiments: Array<Experiment> = []
+
+  for (const filename of filenames) {
+    if (
+      filename.endsWith('.d.ts') ||
+      filename === 'index.ts' ||
+      !EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename))
+    ) {
+      continue
+    }
+
+    const filepath = path.join(experimentsDirectory, filename)
+    const mod: ExperimentModule = await host.loadModule(filepath)
+    const data = mod.experiment ?? mod.default
+    if (!data) {
+      continue
+    }
+
+    const parseResult = ExperimentConfigSchema.safeParse(data)
+    if (!parseResult.success) {
+      logger.warn(
+        `Failed to parse experiment config for file: ${filepath}. Error: ${z.prettifyError(parseResult.error)}`,
+      )
+      continue
+    }
+
+    const {data: config} = parseResult
+    const scenarios = await Promise.all(
+      config.scenarios.map(scenario => {
+        return getScenario(host, scenariosDirectory, scenario)
+      }),
+    )
+    const id = path.basename(filename, path.extname(filename))
+    const experiment: Experiment = {
+      id,
+      filepath,
+      ...config,
+      scenarios,
+    }
+    experiments.push(experiment)
+  }
+
+  return experiments
+}
+
+async function getExperiment({
+  host = DefaultHost,
+  experimentsDirectory,
+  scenariosDirectory,
+  id,
+}: {
+  host?: Host
+  experimentsDirectory: string
+  scenariosDirectory: string
+  id: string
+}): Promise<Experiment> {
+  const experiments = await listExperiments({
+    host,
+    experimentsDirectory,
+    scenariosDirectory,
+  })
+  const experiment = experiments.find(experiment => experiment.id === id)
+  if (experiment) {
+    return experiment
+  }
+
+  throw new Error(`Experiment "${id}" was not found in: ${experimentsDirectory}`)
+}
+
+async function run({env, host = DefaultHost, id}: {env: EnvironmentConfig; host?: Host; id: string}) {
+  const experiment = await getExperiment({
+    host,
+    experimentsDirectory: env.experimentsDirectory,
+    scenariosDirectory: env.scenariosDirectory,
+    id,
+  })
+  const trials: Array<Trial> = experiment.models.flatMap(model => {
+    return experiment.scenarios.flatMap(scenario => {
+      return [
+        {
+          id: randomUUID(),
+          scenario,
+          treatment: ControlTreatment,
+          model,
+          setup: experiment.setup,
+        },
+        ...experiment.treatments.map(treatment => {
+          return {
+            id: randomUUID(),
+            scenario,
+            treatment,
+            model,
+            setup: experiment.setup,
+          }
+        }),
+      ]
+    })
+  })
+  const plan = await createPlan(trials)
+  const results = await runTrials({
+    env,
+    host,
+    plan,
+  })
+  console.log(results)
+
+  // throw new Error('unimplemented')
+}
+
+export {defineConfig, listExperiments, getExperiment, ExperimentConfigSchema, run}
+export type {ExperimentConfig, Experiment}
diff --git a/packages/agent-eval/src/experiments.test.ts b/packages/agent-eval/src/experiments.test.ts
deleted file mode 100644
index 005ed2fd..00000000
--- a/packages/agent-eval/src/experiments.test.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-import fs from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import {describe, expect, test} from 'vitest'
-import {findExperiment, listExperiments, loadExperimentConfigs} from './experiments'
-
-async function createExperimentsDirectory() {
-  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-experiments-'))
-  await fs.writeFile(
-    path.join(directory, 'example.mjs'),
-    `export const experiment = {
-      name: 'Example',
-      description: 'Example experiment',
-      models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
-      scenarios: ['001-agent-uses-button-from-primer'],
-      treatments: []
-    }`,
-  )
-  await fs.writeFile(
-    path.join(directory, 'default-export.mjs'),
-    `export default {
-      name: 'Default export',
-      description: 'Default export experiment',
-      models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
-      scenarios: ['001-agent-uses-button-from-primer'],
-      treatments: []
-    }`,
-  )
-  await fs.writeFile(path.join(directory, 'index.ts'), 'throw new Error("index should be ignored")')
-  return directory
-}
-
-describe('local experiment loading', () => {
-  test('lists experiments from a local directory', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(listExperiments({directory})).resolves.toEqual([
-      ['default-export', expect.objectContaining({name: 'Default export'})],
-      ['example', expect.objectContaining({name: 'Example'})],
-    ])
-  })
-
-  test('finds a named experiment from a local directory', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(findExperiment('example', {directory})).resolves.toEqual(expect.objectContaining({name: 'Example'}))
-  })
-
-  test('finds an experiment from a local file path', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(findExperiment(path.join(directory, 'example.mjs'))).resolves.toEqual(
-      expect.objectContaining({name: 'Example'}),
-    )
-  })
-
-  test('returns undefined when an experiment is not found', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(findExperiment('missing', {directory})).resolves.toBeUndefined()
-  })
-
-  test('loads an experiment from a local file path', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(loadExperimentConfigs({experiment: path.join(directory, 'example.mjs')})).resolves.toEqual([
-      expect.objectContaining({name: 'Example'}),
-    ])
-  })
-
-  test('loads all experiments when no experiment is specified', async () => {
-    const directory = await createExperimentsDirectory()
-
-    await expect(loadExperimentConfigs({directory})).resolves.toHaveLength(2)
-  })
-})
diff --git a/packages/agent-eval/src/experiments.ts b/packages/agent-eval/src/experiments.ts
deleted file mode 100644
index 581f7f1f..00000000
--- a/packages/agent-eval/src/experiments.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-import {existsSync} from 'node:fs'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {pathToFileURL} from 'node:url'
-import type {ExperimentConfig} from './experiment-config'
-
-type ExperimentModule = {
-  default?: ExperimentConfig
-  experiment?: ExperimentConfig
-}
-
-type ExperimentSourceOptions = {
-  directory?: string
-}
-
-type LoadExperimentOptions = ExperimentSourceOptions & {
-  experiment?: string
-}
-
-const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
-
-function getExperimentId(filename: string): string {
-  return filename.replace(/\.(?:cjs|js|mjs|ts)$/, '')
-}
-
-function isExperimentFile(filename: string): boolean {
-  if (filename.endsWith('.d.ts')) {
-    return false
-  }
-
-  return filename !== 'index.ts' && EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename))
-}
-
-async function loadExperimentFile(filepath: string): Promise<ExperimentConfig> {
-  const resolvedPath = path.resolve(filepath)
-  const mod = (await import(pathToFileURL(resolvedPath).href)) as ExperimentModule
-  const experiment = mod.experiment ?? mod.default
-  if (!experiment) {
-    throw new Error(`Experiment file must export "experiment" or a default export: ${resolvedPath}`)
-  }
-
-  return experiment
-}
-
-async function getExperimentEntries(sourceDirectory: string): Promise<Array<[string, ExperimentConfig]>> {
-  const directory = path.resolve(sourceDirectory)
-  if (!existsSync(directory)) {
-    throw new Error(`Experiments directory does not exist: ${directory}`)
-  }
-
-  const stats = await fs.stat(directory)
-  if (!stats.isDirectory()) {
-    throw new Error(`Experiments path is not a directory: ${directory}`)
-  }
-
-  const filenames = (await fs.readdir(directory)).toSorted()
-  return Promise.all(
-    filenames.filter(isExperimentFile).map(async filename => {
-      const filepath = path.join(directory, filename)
-      return [getExperimentId(filename), await loadExperimentFile(filepath)]
-    }),
-  )
-}
-
-async function listExperiments(options: ExperimentSourceOptions = {}): Promise<Array<[string, ExperimentConfig]>> {
-  return getExperimentEntries(options.directory ?? 'experiments')
-}
-
-async function findExperiment(
-  id: string,
-  options: ExperimentSourceOptions = {},
-): Promise<ExperimentConfig | undefined> {
-  if (existsSync(id)) {
-    return loadExperimentFile(id)
-  }
-
-  const experiments = await getExperimentEntries(options.directory ?? 'experiments')
-  return experiments.find(([name]) => name === id)?.[1]
-}
-
-async function loadExperimentConfigs(options: LoadExperimentOptions = {}): Promise<Array<ExperimentConfig>> {
-  if (options.experiment) {
-    const experiment = await findExperiment(options.experiment, options)
-    return experiment ? [experiment] : []
-  }
-
-  return (await listExperiments(options)).map(([, experiment]) => experiment)
-}
-
-export {findExperiment, listExperiments, loadExperimentConfigs}
-export type {ExperimentSourceOptions, LoadExperimentOptions}
diff --git a/packages/agent-eval/src/host.test.ts b/packages/agent-eval/src/host.test.ts
new file mode 100644
index 00000000..ffb5e99b
--- /dev/null
+++ b/packages/agent-eval/src/host.test.ts
@@ -0,0 +1,53 @@
+import os from 'node:os'
+import path from 'node:path'
+import fs from 'node:fs/promises'
+import {pathToFileURL} from 'node:url'
+import {expect, test} from 'vitest'
+import {DefaultHost, SystemHost, VirtualHost} from './host'
+
+test('SystemHost provides access to the system filesystem and module loader', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-host-'))
+  const textFilepath = path.join(directory, 'example.txt')
+  const moduleFilepath = path.join(directory, 'example.mjs')
+
+  try {
+    await fs.writeFile(textFilepath, 'example')
+    await fs.writeFile(moduleFilepath, 'export const value = "loaded"')
+
+    const host = await SystemHost.create()
+
+    expect(host.existsSync(textFilepath)).toBe(true)
+    await expect(host.fs.readFile(textFilepath, 'utf8')).resolves.toBe('example')
+    await expect(host.loadModule<{value: string}>(pathToFileURL(moduleFilepath).href)).resolves.toMatchObject({
+      value: 'loaded',
+    })
+  } finally {
+    await fs.rm(directory, {recursive: true, force: true})
+  }
+})
+
+test('DefaultHost is a system host', () => {
+  expect(DefaultHost).toBeInstanceOf(SystemHost)
+})
+
+test('VirtualHost provides access to an in-memory filesystem and module loader', async () => {
+  const host = VirtualHost.create({
+    '/example.txt': 'example',
+    '/example.mjs': 'export const value = "loaded"',
+  })
+
+  expect(host.existsSync('/example.txt')).toBe(true)
+  expect(host.existsSync('/missing.txt')).toBe(false)
+  await expect(host.fs.readFile('/example.txt', 'utf8')).resolves.toBe('example')
+  await expect(host.loadModule<{value: string}>('/example.mjs')).resolves.toMatchObject({
+    value: 'loaded',
+  })
+})
+
+test('VirtualHost creates an empty writable filesystem by default', async () => {
+  const host = VirtualHost.create()
+
+  await host.fs.writeFile('/example.txt', 'example')
+
+  expect(host.vol.readFileSync('/example.txt', 'utf8')).toBe('example')
+})
diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
index 9a29d836..fe98d5d8 100644
--- a/packages/agent-eval/src/host.ts
+++ b/packages/agent-eval/src/host.ts
@@ -1,8 +1,7 @@
 import {existsSync} from 'node:fs'
 import fs from 'node:fs/promises'
 import {memfs, Volume, type NestedDirectoryJSON} from 'memfs'
-import type {Sandbox} from './sandbox'
-import {VirtualSandbox} from './sandbox'
+import {SystemSandbox, VirtualSandbox, type Sandbox, type SandboxCreateOptions} from './sandbox'
 
 type FileSystem = typeof import('node:fs/promises')
 
@@ -10,6 +9,7 @@ interface Host {
   existsSync: typeof existsSync
   fs: FileSystem
   loadModule<T = unknown>(filepath: string): Promise<T>
+  createSandbox: (options?: SandboxCreateOptions) => Promise<Sandbox>
 }
 
 class SystemHost implements Host {
@@ -28,6 +28,10 @@ class SystemHost implements Host {
   loadModule<T = unknown>(filepath: string): Promise<T> {
     return import(filepath)
   }
+
+  createSandbox(options?: SandboxCreateOptions): Promise<Sandbox> {
+    return SystemSandbox.create(options)
+  }
 }
 
 class VirtualHost implements Host {
@@ -54,6 +58,10 @@ class VirtualHost implements Host {
     const dataUri = `data:text/javascript;base64,${encodedContent}`
     return await import(dataUri)
   }
+
+  crateSandbox(options?: SandboxCreateOptions): Promise<Sandbox> {
+    return VirtualSandbox.create(options)
+  }
 }
 
 const DefaultHost = new SystemHost()
diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts
index d99dfebe..3b7ff1e4 100644
--- a/packages/agent-eval/src/index.ts
+++ b/packages/agent-eval/src/index.ts
@@ -1,21 +1,21 @@
-export {getBenchmark, listBenchmarks} from './benchmark'
-export type {BenchmarkConfig} from './benchmark'
-export {findExperiment, listExperiments, loadExperimentConfigs} from './experiments'
-export type {ExperimentSourceOptions, LoadExperimentOptions} from './experiments'
-export {findScenario, listScenarios} from './scenarios'
-export type {ResolvedScenario, ScenarioSourceOptions} from './scenarios'
-export {run} from './run'
-export type {Treatment, TreatmentResult} from './treatment'
-export {defineConfig} from './experiment'
-export {models} from './model'
-export type {
-  ExperimentConfig,
-  ExperimentModelConfig,
-  Model,
-  ModelConfig,
-  ModelInfo,
-  ReasoningEffort,
-  TreatmentConfig,
+export {defineConfig as defineBenchmarkConfig, getBenchmark, listBenchmarks, BenchmarkConfigSchema} from './benchmark'
+export type {BenchmarkConfig, Benchmark} from './benchmark'
+
+export {
+  defineConfig as defineExperimentConfig,
+  getExperiment,
+  listExperiments,
+  ExperimentConfigSchema,
 } from './experiment'
-export {createAgentEvalOutput, parseAgentEvalOutput} from './output'
-export type {AgentEvalOutput, AgentEvalOutputResult} from './output'
+export type {ExperimentConfig, Experiment} from './experiment'
+
+export {defineConfig as defineScenarioConfig, getScenario, listScenarios, ScenarioConfigSchema} from './scenario'
+export type {ScenarioConfig, Scenario} from './scenario'
+
+export {TreatmentSchema, ControlTreatment} from './treatment'
+export type {Treatment} from './treatment'
+
+// export {run} from './run'
+// export {models} from './model'
+// export {createAgentEvalOutput, parseAgentEvalOutput} from './output'
+// export type {AgentEvalOutput, AgentEvalOutputResult} from './output'
diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts
index 8fd6200e..5b6e5224 100644
--- a/packages/agent-eval/src/model.ts
+++ b/packages/agent-eval/src/model.ts
@@ -92,71 +92,69 @@ const ModelVariantSchema = z.custom<ModelVariant>(value => {
   return reasoningEffortsByModel.get(value.name)?.has(value.reasoningEffort) ?? false
 })
 
-const ModelVariantConfigSchema = z.array(
-  z.union([
-    z.enum(models.map(model => model.name)),
-    z.object({
-      name: z.literal(models[0].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[1].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[2].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[3].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[4].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[5].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[6].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[7].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[8].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[9].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[10].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[11].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[12].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))),
-    }),
-    z.object({
-      name: z.literal(models[13].name),
-      reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))),
-    }),
-  ]),
-)
+const ModelVariantConfigSchema = z.union([
+  z.enum(models.map(model => model.name)),
+  z.object({
+    name: z.literal(models[0].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[1].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[2].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[3].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[4].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[5].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[6].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[7].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[8].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[9].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[10].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[11].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[12].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))),
+  }),
+  z.object({
+    name: z.literal(models[13].name),
+    reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))),
+  }),
+])
 
 type ModelVariantConfig = z.infer<typeof ModelVariantConfigSchema>
 
-function getModelVariants(input: ModelVariantConfig): Array<ModelVariant<Model>> {
+function getModelVariants(input: Array<ModelVariantConfig>): Array<ModelVariant<Model>> {
   return input.flatMap(config => {
     if (typeof config === 'string') {
       return [{name: config, reasoningEffort: 'medium'}]
diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
index d6a4c3a2..f2e0ba79 100644
--- a/packages/agent-eval/src/plan.ts
+++ b/packages/agent-eval/src/plan.ts
@@ -12,7 +12,7 @@ const PlanSchema = z.object({
 type Plan = z.infer<typeof PlanSchema>
 
 // TODO: support plan with sharding
-async function plan(trials: Array<Trial>): Promise<Plan> {
+async function createPlan(trials: Array<Trial>): Promise<Plan> {
   return {
     trials: randomize(trials),
   }
@@ -30,5 +30,5 @@ function randomize<T>(input: Array<T>): Array<T> {
   return randomized
 }
 
-export {plan}
+export {createPlan}
 export type {Plan}
diff --git a/packages/agent-eval/src/resolve-experiment-scenario.ts b/packages/agent-eval/src/resolve-experiment-scenario.ts
deleted file mode 100644
index 61b22ccd..00000000
--- a/packages/agent-eval/src/resolve-experiment-scenario.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import path from 'node:path'
-import type {ExperimentScenarioConfig} from './experiment-config'
-import {findScenario, loadScenarioDirectory, type ResolvedScenario, type ScenarioSourceOptions} from './scenarios'
-
-type ResolveScenarioOptions = ScenarioSourceOptions & {
-  cwd?: string
-}
-
-async function resolveExperimentScenario(
-  scenarioConfig: ExperimentScenarioConfig,
-  options: ResolveScenarioOptions,
-): Promise<ResolvedScenario> {
-  if (typeof scenarioConfig === 'string') {
-    const scenario = await findScenario(scenarioConfig, {
-      directory: options.directory,
-    })
-    if (!scenario) {
-      throw new Error(
-        `Scenario "${scenarioConfig}" was not found in: ${path.resolve(options.directory ?? 'scenarios')}`,
-      )
-    }
-    return scenario
-  }
-
-  const cwd = options.cwd ?? process.cwd()
-  const directory = path.resolve(cwd, scenarioConfig.path)
-  const name = scenarioConfig.name ?? path.basename(directory)
-  return loadScenarioDirectory(directory, name)
-}
-
-export {resolveExperimentScenario}
-export type {ResolvedScenario} from './scenarios'
diff --git a/packages/agent-eval/src/run-old.test.ts b/packages/agent-eval/src/run-old.test.ts
deleted file mode 100644
index fb9abf6a..00000000
--- a/packages/agent-eval/src/run-old.test.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-import {describe, expect, test} from 'vitest'
-import {getCopilotArgs, getVitestConfig} from './run'
-
-describe('getCopilotArgs', () => {
-  test('omits reasoning effort when not configured', () => {
-    expect(
-      getCopilotArgs({
-        prompt: 'Update the page',
-        model: 'claude-haiku-4.5',
-      }),
-    ).not.toContain('--reasoning-effort')
-  })
-
-  test('forwards the model and reasoning effort', () => {
-    expect(
-      getCopilotArgs({
-        prompt: 'Update the page',
-        model: 'gpt-5.5',
-        reasoningEffort: 'medium',
-      }),
-    ).toEqual([
-      '-p',
-      'Update the page',
-      '--model',
-      'gpt-5.5',
-      '--allow-all',
-      '--reasoning-effort',
-      'medium',
-      '--mode',
-      'autopilot',
-      '--output-format',
-      'json',
-    ])
-  })
-})
-
-describe('getVitestConfig', () => {
-  test('configures node tests by default', () => {
-    const config = getVitestConfig('test-results.json')
-
-    expect(config).toContain(`outputFile: "test-results.json"`)
-    expect(config).not.toContain('@vitest/browser-playwright')
-  })
-
-  test('configures browser tests with Playwright and Chromium', () => {
-    const config = getVitestConfig('browser-test-results.json', true)
-
-    expect(config).toContain(`import {playwright} from '@vitest/browser-playwright'`)
-    expect(config).toContain('enabled: true')
-    expect(config).toContain('headless: true')
-    expect(config).toContain('provider: playwright()')
-    expect(config).toContain(`instances: [{browser: 'chromium'}]`)
-    expect(config).toContain(`outputFile: "browser-test-results.json"`)
-  })
-})
diff --git a/packages/agent-eval/src/run-old.ts b/packages/agent-eval/src/run-old.ts
deleted file mode 100644
index d43050f7..00000000
--- a/packages/agent-eval/src/run-old.ts
+++ /dev/null
@@ -1,486 +0,0 @@
-import {randomUUID} from 'node:crypto'
-import path from 'node:path'
-import fs from 'node:fs/promises'
-import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, Sandbox} from './sandbox'
-import type {Treatment, TreatmentResult, Walkthrough} from './treatment'
-import type {Model, ReasoningEffort} from './model'
-import {isMessageType, parseMessage, type Message} from './copilot-cli'
-import {getTestMetadata, parseTestResults} from './vitest'
-import {existsSync} from 'node:fs'
-
-const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright'
-const CHROMIUM_EXECUTABLE_PATH = '/usr/bin/chromium'
-const WALKTHROUGH_DIR = 'walkthrough'
-const WALKTHROUGH_VIEWPORT_WIDTH = 1440
-const WALKTHROUGH_VIEWPORT_HEIGHT = 900
-const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
-
-type RunOptions = {
-  artifactsDirectory: string
-  copilotToken: string
-  dockerImage?: string
-  maxConcurrency?: number
-}
-
-function getVitestConfig(outputFile: string, browser = false): string {
-  const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright'\n` : ''
-  const browserConfig = browser
-    ? `    browser: {
-      enabled: true,
-      headless: true,
-      provider: playwright(),
-      instances: [{browser: 'chromium'}],
-    },
-`
-    : ''
-
-  return `
-import {defineConfig} from 'vitest/config'
-${browserImport}
-export default defineConfig({
-  test: {
-${browserConfig}    reporters: [['json', {outputFile: ${JSON.stringify(outputFile)}, includeTaskLocation: true}]],
-  },
-})
-`
-}
-
-function run(treatments: Array<Treatment>, options: RunOptions): Promise<Array<TreatmentResult>> {
-  const maxConcurrency = options.maxConcurrency ?? 1
-  const queue = treatments.slice()
-  const results: Array<TreatmentResult> = []
-  const pending = new Set()
-  let cancelled = false
-
-  let resolve: (value: Array<TreatmentResult>) => void
-  let reject: (reason: unknown) => void
-  const deferred = new Promise<Array<TreatmentResult>>((_resolve, _reject) => {
-    resolve = _resolve
-    reject = _reject
-  })
-
-  function execute() {
-    if (cancelled) {
-      return
-    }
-
-    if (queue.length === 0) {
-      if (pending.size === 0) {
-        resolve(results)
-      }
-      return
-    }
-
-    if (pending.size >= maxConcurrency) {
-      return
-    }
-
-    const treatment = queue.shift()
-    if (!treatment) {
-      return
-    }
-
-    const promise = retry(
-      () =>
-        runTreatment(treatment, {
-          artifactsDirectory: options.artifactsDirectory,
-          copilotToken: options.copilotToken,
-          dockerImage: options.dockerImage,
-        }),
-      3,
-    ).then(
-      result => {
-        results.push(result)
-        pending.delete(promise)
-        execute()
-      },
-      error => {
-        cancelled = true
-        pending.delete(promise)
-        reject(error)
-      },
-    )
-
-    pending.add(promise)
-    execute()
-  }
-
-  execute()
-
-  return deferred
-}
-
-async function retry<T>(fn: () => Promise<T>, retries: number): Promise<T> {
-  try {
-    return await fn()
-  } catch (error) {
-    if (retries > 0) {
-      console.log('Retrying after error: %s', error)
-      return retry(fn, retries - 1)
-    }
-    throw error
-  }
-}
-
-type RunTreatmentOptions = {
-  artifactsDirectory: string
-  copilotToken: string
-  dockerImage?: string
-}
-
-function getCopilotArgs({
-  prompt,
-  model,
-  reasoningEffort,
-}: {
-  prompt: string
-  model: Model
-  reasoningEffort?: ReasoningEffort
-}): Array<string> {
-  const args = ['-p', prompt, '--model', model, '--allow-all']
-
-  if (reasoningEffort) {
-    args.push('--reasoning-effort', reasoningEffort)
-  }
-
-  return [...args, '--mode', 'autopilot', '--output-format', 'json']
-}
-
-async function runTreatment(
-  treatment: Treatment,
-  {artifactsDirectory, copilotToken, dockerImage}: RunTreatmentOptions,
-): Promise<TreatmentResult> {
-  console.log('Running treatment: %s (%s)', treatment.config.name, treatment.id)
-  await using sandbox = await Sandbox.create({dockerImage})
-
-  console.log('Copying files from: %s...', treatment.scenario.directory)
-  await sandbox.copy(treatment.scenario.directory, CONTAINER_WORKDIR, {
-    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
-  })
-  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
-    user: 'root',
-  })
-
-  console.log('Obfuscating package name...')
-  await sandbox.runCommand('npm', ['pkg', 'set', `name=${treatment.id}`], {
-    user: NODE_USER,
-  })
-
-  console.log('Removing workspace dependency...')
-  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
-    user: NODE_USER,
-  })
-
-  console.log('Installing dependencies...')
-  await sandbox.runCommand('npm', ['install'], {
-    user: NODE_USER,
-  })
-
-  if (treatment.experiment.setup) {
-    console.log('Running experiment setup...')
-    await treatment.experiment.setup({
-      sandbox,
-    })
-  }
-
-  if (treatment.config.setup) {
-    console.log('Running treatment setup...')
-    await treatment.config.setup({
-      sandbox,
-    })
-  }
-
-  console.log('Run build script...')
-  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
-    user: NODE_USER,
-  })
-
-  if (treatment.scenario.browserTestPath) {
-    console.log('Installing browser test dependencies...')
-    await sandbox.runCommand(
-      'npm',
-      ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'],
-      {
-        user: NODE_USER,
-      },
-    )
-    console.log('Installing Playwright browser...')
-    await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], {
-      user: 'root',
-      env: {
-        PLAYWRIGHT_BROWSERS_PATH,
-      },
-    })
-  }
-
-  console.log('Running copilot...')
-  const {prompt} = treatment.scenario.config
-  const args = getCopilotArgs({
-    prompt,
-    model: treatment.model,
-    reasoningEffort: treatment.reasoningEffort,
-  })
-  const copilotOutput = await sandbox.runCommand('copilot', args, {
-    user: NODE_USER,
-    env: {
-      COPILOT_GITHUB_TOKEN: copilotToken,
-    },
-  })
-  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
-    const trimmed = line.trim()
-    if (trimmed.length === 0) {
-      return []
-    }
-    return parseMessage(JSON.parse(trimmed))
-  })
-
-  const TEST_PATH = 'scenario.test.ts'
-  const BROWSER_TEST_PATH = 'scenario.browser.test.ts'
-  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
-  const TEST_RESULTS_PATH = 'test-results.json'
-  const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json'
-  const scenarioTests = [
-    {
-      sourcePath: treatment.scenario.testPath,
-      testPath: TEST_PATH,
-      resultsPath: TEST_RESULTS_PATH,
-      browser: false,
-    },
-  ]
-
-  if (treatment.scenario.browserTestPath) {
-    scenarioTests.push({
-      sourcePath: treatment.scenario.browserTestPath,
-      testPath: BROWSER_TEST_PATH,
-      resultsPath: BROWSER_TEST_RESULTS_PATH,
-      browser: true,
-    })
-  }
-
-  let numFailedTests = 0
-  let numPassedTests = 0
-  let numPendingTests = 0
-  let numTodoTests = 0
-  let numTotalTests = 0
-  let testRunSuccess = true
-  const tests: TreatmentResult['testResults']['tests'] = []
-  const rawTestResults: Array<Record<string, unknown> & {testResults: Array<unknown>}> = []
-
-  for (const scenarioTest of scenarioTests) {
-    await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath)
-    await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser))
-    // Always pass vitest calls even if test suite fails
-    await sandbox.runCommand(
-      'sh',
-      ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath],
-      {
-        user: NODE_USER,
-        env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined,
-      },
-    )
-
-    const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath)
-    const rawTestResult: unknown = JSON.parse(testResultsContent)
-    const testResults = parseTestResults(rawTestResult)
-    if (!testResults.success) {
-      throw new Error(`Failed to parse test results: ${testResults.error}`)
-    }
-
-    const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8')
-    numFailedTests += testResults.data.numFailedTests
-    numPassedTests += testResults.data.numPassedTests
-    numPendingTests += testResults.data.numPendingTests
-    numTodoTests += testResults.data.numTodoTests
-    numTotalTests += testResults.data.numTotalTests
-    testRunSuccess &&= testResults.data.success
-    tests.push(...getTestMetadata(testResults.data, testSource))
-    rawTestResults.push(rawTestResult as Record<string, unknown> & {testResults: Array<unknown>})
-  }
-
-  if (rawTestResults.length > 1) {
-    await sandbox.writeFile(
-      TEST_RESULTS_PATH,
-      JSON.stringify({
-        ...rawTestResults[0],
-        numFailedTests,
-        numPassedTests,
-        numPendingTests,
-        numTodoTests,
-        numTotalTests,
-        success: testRunSuccess,
-        testResults: rawTestResults.flatMap(testResult => testResult.testResults),
-      }),
-    )
-  }
-
-  console.log('Capturing walkthrough...')
-  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
-    user: 'root',
-  })
-  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
-    user: NODE_USER,
-  })
-  await sandbox.runCommand(
-    'npx',
-    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
-    {
-      user: NODE_USER,
-    },
-  )
-  await sandbox.writeFile(
-    'agent-browser.json',
-    JSON.stringify({
-      executablePath: CHROMIUM_EXECUTABLE_PATH,
-    }),
-  )
-  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
-
-Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
-
-Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
-
-- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
-- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
-- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
-
-Only capture the walkthrough, do not make any further code changes.`
-  const walkthroughResult = await sandbox.runCommand(
-    'copilot',
-    getCopilotArgs({
-      prompt: walkthroughPrompt,
-      model: 'gpt-5.6-terra',
-      reasoningEffort: 'medium',
-    }),
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: copilotToken,
-      },
-      allowNonZeroExitCode: true,
-    },
-  )
-
-  if (walkthroughResult.exitCode !== 0) {
-    console.warn('Unable to capture walkthrough: %s', walkthroughResult.stderr)
-  }
-
-  // Turns
-  const assistantTurns = new Set()
-  // Tools
-  const toolCalls = new Map()
-  let outputTokens = 0
-
-  for (const message of messages) {
-    if (isMessageType(message, 'assistant.turn_start')) {
-      assistantTurns.add(message.data.turnId)
-    }
-
-    if (isMessageType(message, 'assistant.message')) {
-      outputTokens += message.data.outputTokens ?? 0
-    }
-
-    if (isMessageType(message, 'tool.execution_start')) {
-      const toolName = message.data.toolName
-      toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1)
-    }
-  }
-
-  const result = messages.find(message => isMessageType(message, 'result'))
-  if (!result) {
-    throw new Error('No result message found in copilot output')
-  }
-
-  const artifactDirectory = path.join(artifactsDirectory, treatment.id)
-  const workspacePath = path.join(artifactDirectory, 'workspace')
-  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
-  const copilotConfigPath = path.join(artifactDirectory, '.copilot')
-  const skillsConfigPath = path.join(artifactDirectory, '.agents')
-  const testResultsPath = path.join(workspacePath, 'test-results.json')
-  await fs.mkdir(workspacePath, {recursive: true})
-
-  console.log('Downloading agent workspace to: %s...', workspacePath)
-  await sandbox.download(CONTAINER_WORKDIR, workspacePath, {
-    ignore(name) {
-      return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
-    },
-  })
-
-  console.log('Downloading copilot config to: %s...', copilotConfigPath)
-  await sandbox.download(COPILOT_DIR, copilotConfigPath)
-
-  console.log('Downloading skills config to: %s...', skillsConfigPath)
-  await sandbox.download(AGENTS_DIR, skillsConfigPath)
-
-  let walkthrough: Walkthrough = {
-    type: 'Unavailable',
-  }
-
-  if (existsSync(path.join(workspacePath, WALKTHROUGH_DIR))) {
-    console.log(
-      'Moving walkthrough artifacts from: %s to: %s...',
-      path.join(workspacePath, WALKTHROUGH_DIR),
-      walkthroughPath,
-    )
-    await fs.mkdir(walkthroughPath, {recursive: true})
-    await fs.rename(path.join(workspacePath, WALKTHROUGH_DIR), walkthroughPath)
-
-    if (existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
-      walkthrough = {
-        type: 'Screenshot',
-        filepath: path.join(walkthroughPath, 'screenshot.png'),
-      }
-    } else if (existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
-      walkthrough = {
-        type: 'Video',
-        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
-      }
-    } else if (existsSync(path.join(walkthroughPath, 'screenshots'))) {
-      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
-      const entries = await fs.readdir(screenshotsDir).then(filenames => {
-        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
-      })
-      const screenshots = entries.filter(entry => {
-        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
-      })
-      if (screenshots.length > 0) {
-        walkthrough = {
-          type: 'Screenshots',
-          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
-        }
-      }
-    }
-  }
-
-  return {
-    id: randomUUID(),
-    treatment,
-    artifacts: {
-      directory: artifactDirectory,
-      copilotConfigPath,
-      skillsConfigPath,
-      testResultsPath,
-      workspacePath,
-    },
-    assistant: {
-      logs: messages,
-      turns: assistantTurns.size,
-      outputTokens,
-      premiumRequests: result.usage.premiumRequests,
-      // Time to complete (latency)
-      totalApiDurationMs: result.usage.totalApiDurationMs,
-      sessionDurationMs: result.usage.sessionDurationMs,
-      tools: Object.fromEntries(toolCalls),
-    },
-    testResults: {
-      numFailedTests,
-      numPassedTests,
-      numPendingTests,
-      numTodoTests,
-      numTotalTests,
-      tests,
-    },
-    walkthrough,
-  }
-}
-
-export {getCopilotArgs, getVitestConfig, run}
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index 4877d9ef..310a1653 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -1,13 +1,17 @@
+import path from 'node:path'
 import Queue from 'p-queue'
 import * as z from 'zod/mini'
 import {MessageSchema, parseMessage, type Message} from './copilot-cli'
 import type {Plan} from './plan'
 import {TrialSchema, type Trial} from './trial'
-import type {Host} from './host'
+import {DefaultHost, type Host} from './host'
 // import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
-import {CONTAINER_WORKDIR, NODE_USER, type Sandbox} from './sandbox'
+import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type Sandbox} from './sandbox'
 import {parseTestResults, TestResultsSchema} from './vitest'
 import {logger} from './logger'
+import type {EnvironmentConfig} from './environment'
+
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
 
 const AssistantSchema = z.object({
   logs: z.array(MessageSchema),
@@ -26,10 +30,23 @@ const WalkthroughSchema = z.discriminatedUnion('type', [
   z.object({type: z.literal('Video'), filepath: z.string()}),
 ])
 
+type Walkthrough = z.infer<typeof WalkthroughSchema>
+
 const TrialResultSchema = z.object({
+  artifacts: z.object({
+    directory: z.string(),
+    copilotConfigDirectory: z.string(),
+    skillsConfigDirectory: z.string(),
+    testResultsPath: z.string(),
+    workspaceDirectory: z.string(),
+  }),
   trial: TrialSchema,
   assistant: z.object({
-    sessions: z.array(z.array(MessageSchema)),
+    sessions: z.array(
+      z.object({
+        messages: z.array(MessageSchema),
+      }),
+    ),
   }),
   testResults: TestResultsSchema,
   walkthrough: WalkthroughSchema,
@@ -43,19 +60,29 @@ type RunOptions = {
   maxConcurrency?: number
 }
 
-async function run(host: Host, sandbox: Sandbox, plan: Plan, options: RunOptions): Promise<Array<TrialResult>> {
-  const {artifactsDirectory, copilotToken, maxConcurrency = 1} = options
+async function run({
+  env,
+  host = DefaultHost,
+  plan,
+}: {
+  env: EnvironmentConfig
+  host?: Host
+  plan: Plan
+}): Promise<Array<TrialResult>> {
   const queue = new Queue({
-    concurrency: maxConcurrency,
+    concurrency: env.concurrency,
   })
 
   const results = await Promise.all(
     plan.trials.map(trial => {
       return queue.add(() => {
-        return retry(() => {
+        return retry(async () => {
+          await using sandbox = await host.createSandbox({
+            dockerImage: env.dockerImage,
+          })
           return runTrial(host, sandbox, trial, {
-            artifactsDirectory,
-            copilotToken,
+            artifactsDirectory: env.artifactsDirectory,
+            copilotToken: env.copilotToken,
           })
         })
       })
@@ -70,7 +97,7 @@ type RunTrialOptions = {
   copilotToken: string
 }
 
-async function runTrial(sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
+async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
   logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
 
   const {artifactsDirectory, copilotToken} = options
@@ -160,15 +187,93 @@ async function runTrial(sandbox: Sandbox, trial: Trial, options: RunTrialOptions
     throw new Error(`Failed to parse test results: ${testResults.error}`)
   }
 
+  const WALKTHROUGH_DIR = 'walkthrough'
+  const WALKTHROUGH_VIEWPORT_WIDTH = 1440
+  const WALKTHROUGH_VIEWPORT_HEIGHT = 900
+
+  const artifactDirectory = path.join(artifactsDirectory, trial.id)
+  const workspaceDirectory = path.join(artifactDirectory, 'workspace')
+  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
+  const copilotConfigDirectory = path.join(artifactDirectory, '.copilot')
+  const skillsConfigDirectory = path.join(artifactDirectory, '.agents')
+  const testResultsPath = path.join(workspaceDirectory, 'test-results.json')
+
+  if (host.existsSync(artifactDirectory)) {
+    await host.fs.rm(artifactDirectory, {recursive: true, force: true})
+  }
+  await host.fs.mkdir(workspaceDirectory, {recursive: true})
+
+  logger.info('Downloading agent workspace to: %s...', workspaceDirectory)
+  await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
+    ignore(name) {
+      return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
+    },
+  })
+
+  logger.info('Downloading copilot config to: %s...', copilotConfigDirectory)
+  await sandbox.download(COPILOT_DIR, copilotConfigDirectory)
+
+  logger.info('Downloading skills config to: %s...', skillsConfigDirectory)
+  await sandbox.download(AGENTS_DIR, skillsConfigDirectory)
+
+  let walkthrough: Walkthrough = {
+    type: 'Unavailable',
+  }
+
+  if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) {
+    logger.info(
+      'Moving walkthrough artifacts from: %s to: %s...',
+      path.join(workspaceDirectory, WALKTHROUGH_DIR),
+      walkthroughPath,
+    )
+    await host.fs.mkdir(walkthroughPath, {recursive: true})
+    await host.fs.rename(path.join(workspaceDirectory, WALKTHROUGH_DIR), walkthroughPath)
+
+    if (host.existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
+      walkthrough = {
+        type: 'Screenshot',
+        filepath: path.join(walkthroughPath, 'screenshot.png'),
+      }
+    } else if (host.existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
+      walkthrough = {
+        type: 'Video',
+        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
+      }
+    } else if (host.existsSync(path.join(walkthroughPath, 'screenshots'))) {
+      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
+      const entries = await host.fs.readdir(screenshotsDir).then(filenames => {
+        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
+      })
+      const screenshots = entries.filter(entry => {
+        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
+      })
+      if (screenshots.length > 0) {
+        walkthrough = {
+          type: 'Screenshots',
+          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
+        }
+      }
+    }
+  }
+
   return {
+    artifacts: {
+      directory: artifactDirectory,
+      copilotConfigDirectory,
+      skillsConfigDirectory,
+      testResultsPath,
+      workspaceDirectory,
+    },
     trial,
     assistant: {
-      sessions: [messages],
+      sessions: [
+        {
+          messages,
+        },
+      ],
     },
     testResults: testResults.data,
-    walkthrough: {
-      type: 'Unavailable',
-    },
+    walkthrough,
   }
 }
 
diff --git a/packages/agent-eval/src/sandbox-old.test.ts b/packages/agent-eval/src/sandbox-old.test.ts
deleted file mode 100644
index e468228c..00000000
--- a/packages/agent-eval/src/sandbox-old.test.ts
+++ /dev/null
@@ -1,119 +0,0 @@
-import {describe, expect, test, vi} from 'vitest'
-import {Sandbox} from './sandbox'
-
-function createSandbox() {
-  const sandbox = new Sandbox({} as never, {} as never)
-  const copy = vi.spyOn(sandbox, 'copy').mockResolvedValue()
-  const runCommand = vi.spyOn(sandbox, 'runCommand').mockResolvedValue({
-    stdout: '',
-    stderr: '',
-    exitCode: 0,
-  })
-
-  return {sandbox, copy, runCommand}
-}
-
-describe('addCopilotPlugin', () => {
-  test('installs a remote plugin', async () => {
-    const {sandbox, copy, runCommand} = createSandbox()
-
-    await sandbox.addCopilotPlugin({
-      type: 'remote',
-      url: 'https://github.com/example/plugin.git',
-    })
-
-    expect(copy).not.toHaveBeenCalled()
-    expect(runCommand).toHaveBeenCalledOnce()
-    expect(runCommand).toHaveBeenCalledWith('copilot', ['plugin', 'install', 'https://github.com/example/plugin.git'])
-  })
-
-  test('installs a versioned remote plugin', async () => {
-    const {sandbox, copy, runCommand} = createSandbox()
-
-    await sandbox.addCopilotPlugin({
-      type: 'remote',
-      url: 'https://github.com/example/plugin.git',
-      version: 'v1.2.3',
-    })
-
-    expect(copy).not.toHaveBeenCalled()
-    expect(runCommand).toHaveBeenCalledOnce()
-    expect(runCommand).toHaveBeenCalledWith('copilot', [
-      'plugin',
-      'install',
-      'https://github.com/example/plugin.git#v1.2.3',
-    ])
-  })
-
-  test('copies and installs a local plugin', async () => {
-    const {sandbox, copy, runCommand} = createSandbox()
-
-    await sandbox.addCopilotPlugin({
-      type: 'local',
-      sourcePath: './plugins/local-plugin',
-    })
-
-    expect(copy).toHaveBeenCalledOnce()
-    const pluginPath = copy.mock.calls[0][1]
-    expect(pluginPath).toMatch(/^\/home\/node\/\.copilot\/plugin-sources\//)
-    expect(copy).toHaveBeenCalledWith('./plugins/local-plugin', pluginPath)
-    expect(runCommand).toHaveBeenCalledOnce()
-    expect(runCommand).toHaveBeenCalledWith('copilot', ['plugin', 'install', pluginPath])
-  })
-
-  test('adds a remote marketplace and installs its plugin', async () => {
-    const {sandbox, copy, runCommand} = createSandbox()
-
-    await sandbox.addCopilotPlugin({
-      type: 'marketplace',
-      name: 'example-plugin',
-      marketplace: {
-        name: 'example-marketplace',
-        source: {
-          type: 'remote',
-          url: 'https://github.com/example/marketplace.git',
-          version: 'v1.2.3',
-        },
-      },
-    })
-
-    expect(copy).not.toHaveBeenCalled()
-    expect(runCommand).toHaveBeenNthCalledWith(1, 'copilot', [
-      'plugin',
-      'marketplace',
-      'add',
-      'https://github.com/example/marketplace.git#v1.2.3',
-    ])
-    expect(runCommand).toHaveBeenNthCalledWith(2, 'copilot', [
-      'plugin',
-      'install',
-      'example-plugin@example-marketplace',
-    ])
-  })
-
-  test('copies a local marketplace and installs its plugin', async () => {
-    const {sandbox, copy, runCommand} = createSandbox()
-
-    await sandbox.addCopilotPlugin({
-      type: 'marketplace',
-      name: 'example-plugin',
-      marketplace: {
-        name: 'example-marketplace',
-        source: {
-          type: 'local',
-          sourcePath: './plugins/marketplace',
-        },
-      },
-    })
-
-    const marketplacePath = copy.mock.calls[0][1]
-    expect(marketplacePath).toMatch(/^\/home\/node\/\.copilot\/plugin-sources\//)
-    expect(copy).toHaveBeenCalledWith('./plugins/marketplace', marketplacePath)
-    expect(runCommand).toHaveBeenNthCalledWith(1, 'copilot', ['plugin', 'marketplace', 'add', marketplacePath])
-    expect(runCommand).toHaveBeenNthCalledWith(2, 'copilot', [
-      'plugin',
-      'install',
-      'example-plugin@example-marketplace',
-    ])
-  })
-})
diff --git a/packages/agent-eval/src/sandbox-old.ts b/packages/agent-eval/src/sandbox-old.ts
deleted file mode 100644
index 062092ef..00000000
--- a/packages/agent-eval/src/sandbox-old.ts
+++ /dev/null
@@ -1,799 +0,0 @@
-import {randomUUID} from 'node:crypto'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {Writable} from 'node:stream'
-import {pipeline} from 'node:stream/promises'
-import Docker from 'dockerode'
-import tarFs from 'tar-fs'
-import type {Headers} from 'tar-fs'
-import tarStream from 'tar-stream'
-import * as z from 'zod/mini'
-import {McpConfigFileSchema} from './mcp-config'
-import type {McpConfigFile, McpServerConfig} from './mcp-config'
-
-const COPILOT_CLI_VERSION = '1.0.80'
-const NPM_VERSION = '12.0.2'
-
-/**
- * Working directory inside the container.
- */
-const CONTAINER_WORKDIR = '/home/sandbox/workspace'
-
-/**
- * Directory for copilot cli configuration.
- */
-const COPILOT_DIR = '/home/node/.copilot'
-
-/**
- * Directory for custom Copilot sub-agents.
- */
-const CUSTOM_AGENTS_DIR = '/home/node/.copilot/agents'
-
-/**
- * Directory for agents configuration and skills.
- */
-const AGENTS_DIR = '/home/node/.agents'
-
-/**
- * Directory for skills.
- */
-const SKILLS_DIR = '/home/node/.agents/skills'
-
-/**
- * Directory for local plugin sources copied into the container.
- */
-const COPILOT_PLUGIN_SOURCES_DIR = path.posix.join(COPILOT_DIR, 'plugin-sources')
-
-/**
- * Path for project agent instructions.
- */
-const AGENT_INSTRUCTIONS_PATH = path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md')
-
-/**
- * Path for MCP server configuration file.
- */
-const MCP_CONFIG_PATH = path.join(COPILOT_DIR, 'mcp-config.json')
-
-/**
- * Non-root user configuration.
- * Running as non-root is important for security and compatibility
- * (e.g., Claude Code refuses --dangerously-skip-permissions as root).
- * Node.js images already have a 'node' user with UID/GID 1000.
- */
-const SANDBOX_UID = 1000
-const SANDBOX_GID = 1000
-const NODE_USER = `${SANDBOX_UID}:${SANDBOX_GID}` as const
-
-/**
- * Directory for npm global packages (non-root install location).
- */
-const NPM_GLOBAL_DIR = '/home/node/.npm-global'
-
-type RunOptions = {
-  env?: Record<string, string>
-  user?: string
-  allowNonZeroExitCode?: boolean
-}
-
-type CopyOptions = {
-  exclude?: string[]
-}
-
-type CustomAgentCopiedFile = {
-  sourcePath: string
-  destinationPath?: string
-}
-
-type CustomAgentWrittenFile = {
-  path: string
-  content: string
-}
-
-type CustomAgentFile = CustomAgentCopiedFile | CustomAgentWrittenFile
-
-type AgentSkillCopiedFile = CustomAgentCopiedFile
-
-type AgentSkillWrittenFile = CustomAgentWrittenFile
-
-type AgentSkillFile = AgentSkillCopiedFile | AgentSkillWrittenFile
-
-type AgentSkillOptions = {
-  files?: Array<AgentSkillFile>
-}
-
-type RemoteCopilotPluginSource = {
-  type: 'remote'
-  url: string
-  version?: string
-}
-
-type LocalCopilotPluginSource = {
-  type: 'local'
-  sourcePath: string
-}
-
-type CopilotPluginSource = RemoteCopilotPluginSource | LocalCopilotPluginSource
-
-type CopilotPluginConfig =
-  | CopilotPluginSource
-  | {
-      type: 'marketplace'
-      name: string
-      marketplace: {
-        name: string
-        source: CopilotPluginSource
-      }
-    }
-
-type CustomAgentOptions = {
-  files?: Array<CustomAgentFile>
-  tools?: Array<string>
-}
-
-type DownloadOptions = {
-  ignore?: (name: string) => boolean
-}
-
-type SandboxCreateOptions = {
-  dockerImage?: string
-}
-
-const DEFAULT_MCP_CONFIG: McpConfigFile = {
-  mcpServers: {},
-}
-
-class Sandbox {
-  static async create(options: SandboxCreateOptions = {}) {
-    const docker = new Docker()
-    const dockerImage = options.dockerImage?.trim() || DEFAULT_DOCKER_IMAGE
-    const container = await createContainer(docker, dockerImage)
-    return new Sandbox(docker, container)
-  }
-
-  #docker: Docker
-  #container: Docker.Container
-
-  constructor(docker: Docker, container: InitializedContainer) {
-    this.#docker = docker
-    this.#container = container
-  }
-
-  async [Symbol.asyncDispose]() {
-    await this.#container.stop()
-  }
-
-  async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise<void> {
-    const source = path.resolve(sourcePath)
-    const sourceStats = await fs.stat(source)
-    if (!sourceStats.isDirectory() && !sourceStats.isFile()) {
-      throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`)
-    }
-
-    const containerPath = resolveContainerPath(destinationPath)
-    const containerDirectory = path.posix.dirname(containerPath)
-    const destinationName = path.posix.basename(containerPath)
-    if (!destinationName) {
-      throw new Error(`Cannot copy "${sourcePath}" to "${destinationPath}" because the destination must include a name`)
-    }
-
-    await execCommand(this.#docker, this.#container, 'mkdir', ['-p', containerDirectory], {
-      user: NODE_USER,
-    })
-
-    const sourceDirectory = path.dirname(source)
-    const sourceName = path.basename(source)
-    const excludedPaths = new Set(options.exclude?.map(filepath => normalizeExcludedPath(filepath, source)))
-    const archive = tarFs.pack(sourceDirectory, {
-      entries: [sourceName],
-      ignore(name) {
-        const absolutePath = path.isAbsolute(name) ? name : path.resolve(sourceDirectory, name)
-        const relativePath = normalizeCopyPath(path.relative(source, absolutePath))
-        return isExcluded(relativePath, excludedPaths)
-      },
-      map(header) {
-        return mapCopiedHeader(header, sourceName, destinationName)
-      },
-    })
-
-    await this.#container.putArchive(archive, {
-      path: containerDirectory,
-    })
-  }
-
-  async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise<void> {
-    await fs.mkdir(hostDestinationPath, {
-      recursive: true,
-    })
-
-    const archive = await this.#container.getArchive({
-      path: containerFilePath,
-    })
-    const sourceName = path.posix.basename(containerFilePath)
-
-    await pipeline(
-      archive,
-      tarFs.extract(hostDestinationPath, {
-        readable: true,
-        writable: true,
-        map(header) {
-          const prefix = `${sourceName}/`
-
-          if (header.name === sourceName) {
-            header.name = '.'
-          } else if (header.name.startsWith(prefix)) {
-            header.name = header.name.slice(prefix.length)
-          }
-
-          return header
-        },
-        ignore: options.ignore,
-      }),
-    )
-  }
-
-  async readFile(filepath: string): Promise<string> {
-    const archive = await this.#container.getArchive({
-      path: resolveContainerPath(filepath),
-    })
-    const buffer = await readFileFromArchive(archive)
-    return buffer.toString('utf8')
-  }
-
-  async writeFile(filepath: string, contents: string): Promise<void> {
-    const containerPath = resolveContainerPath(filepath)
-    const directory = path.dirname(containerPath)
-    const name = path.basename(containerPath)
-    const pack = tarStream.pack()
-    const upload = this.#container.putArchive(pack, {
-      path: directory,
-    })
-
-    pack.entry(
-      {
-        name,
-        mode: 0o644,
-        size: Buffer.byteLength(contents),
-        uid: SANDBOX_UID,
-        gid: SANDBOX_GID,
-      },
-      contents,
-    )
-    pack.finalize()
-
-    await upload
-  }
-
-  async exists(filepath: string): Promise<boolean> {
-    const result = await execCommand(this.#docker, this.#container, 'test', ['-e', resolveContainerPath(filepath)], {
-      user: NODE_USER,
-      allowNonZeroExitCode: true,
-    })
-
-    return result.exitCode === 0
-  }
-
-  async runCommand(command: string, args: Array<string> = [], options?: RunOptions): Promise<CommandResult> {
-    return execCommand(this.#docker, this.#container, command, args, {
-      env: {
-        HOME: options?.user === 'root' ? '/root' : '/home/node',
-        ...options?.env,
-        PATH: `${NPM_GLOBAL_DIR}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
-      },
-      user: options?.user ?? NODE_USER,
-      allowNonZeroExitCode: options?.allowNonZeroExitCode,
-    })
-  }
-
-  async addAgentInstruction(text: string): Promise<void> {
-    const contents = await this.#findOrCreateFile(AGENT_INSTRUCTIONS_PATH)
-    await this.writeFile(AGENT_INSTRUCTIONS_PATH, appendText(contents, text))
-  }
-
-  async addAgentSkill(
-    name: string,
-    description: string,
-    contents: string,
-    options: AgentSkillOptions = {},
-  ): Promise<void> {
-    assertValidSkillName(name)
-
-    const skillDirectory = path.posix.join(SKILLS_DIR, name)
-    const skillPath = path.posix.join(skillDirectory, 'SKILL.md')
-    if (await this.exists(skillPath)) {
-      throw new Error(`Agent skill with name "${name}" already exists`)
-    }
-
-    await this.runCommand('mkdir', ['-p', skillDirectory])
-    await this.writeFile(skillPath, createSkillContents(name, description, contents))
-
-    for (const file of options.files ?? []) {
-      const destinationPath = path.posix.join(skillDirectory, getAgentSkillFileDestination(file))
-
-      if (isWrittenFile(file)) {
-        await this.runCommand('mkdir', ['-p', path.posix.dirname(destinationPath)])
-        await this.writeFile(destinationPath, file.content)
-      } else {
-        await this.copy(file.sourcePath, destinationPath)
-      }
-    }
-  }
-
-  async addCustomAgent(
-    name: string,
-    description: string,
-    contents: string,
-    options: CustomAgentOptions = {},
-  ): Promise<void> {
-    assertValidCustomAgentName(name)
-
-    const agentPath = path.posix.join(CUSTOM_AGENTS_DIR, `${name}.agent.md`)
-    if (await this.exists(agentPath)) {
-      throw new Error(`Custom agent with name "${name}" already exists`)
-    }
-
-    await this.runCommand('mkdir', ['-p', CUSTOM_AGENTS_DIR])
-    await this.writeFile(agentPath, createCustomAgentContents(name, description, contents, options))
-
-    for (const file of options.files ?? []) {
-      const destinationPath = path.posix.join(CUSTOM_AGENTS_DIR, getCustomAgentFileDestination(file))
-
-      if (isWrittenFile(file)) {
-        await this.runCommand('mkdir', ['-p', path.posix.dirname(destinationPath)])
-        await this.writeFile(destinationPath, file.content)
-      } else {
-        await this.copy(file.sourcePath, destinationPath)
-      }
-    }
-  }
-
-  async addMcpServer(name: string, config: McpServerConfig): Promise<void> {
-    const contents = await this.readFile(MCP_CONFIG_PATH)
-    const mcpConfig = contents === '' ? DEFAULT_MCP_CONFIG : McpConfigFileSchema.parse(JSON.parse(contents))
-    if (mcpConfig.mcpServers[name]) {
-      throw new Error(`MCP server with name "${name}" already exists`)
-    }
-
-    const updatedConfig: McpConfigFile = {
-      ...mcpConfig,
-      mcpServers: {
-        ...mcpConfig.mcpServers,
-        [name]: config,
-      },
-    }
-
-    await this.writeFile(MCP_CONFIG_PATH, JSON.stringify(updatedConfig, null, 2))
-    await this.runCommand('chown', ['-R', NODE_USER, MCP_CONFIG_PATH], {
-      user: 'root',
-    })
-  }
-
-  async addCopilotPlugin(config: CopilotPluginConfig): Promise<void> {
-    if (config.type === 'marketplace') {
-      const marketplaceSource = await this.#prepareCopilotPluginSource(config.marketplace.source)
-      await this.runCommand('copilot', ['plugin', 'marketplace', 'add', marketplaceSource])
-      await this.runCommand('copilot', ['plugin', 'install', `${config.name}@${config.marketplace.name}`])
-      return
-    }
-
-    const source = await this.#prepareCopilotPluginSource(config)
-    await this.runCommand('copilot', ['plugin', 'install', source])
-  }
-
-  async #prepareCopilotPluginSource(source: CopilotPluginSource): Promise<string> {
-    if (source.type === 'remote') {
-      return source.version ? `${source.url}#${source.version}` : source.url
-    }
-
-    const destinationPath = path.posix.join(COPILOT_PLUGIN_SOURCES_DIR, randomUUID())
-    await this.copy(source.sourcePath, destinationPath)
-    return destinationPath
-  }
-
-  async #findOrCreateFile(filepath: string): Promise<string> {
-    if (await this.exists(filepath)) {
-      return this.readFile(filepath)
-    }
-
-    await this.writeFile(filepath, '')
-    return ''
-  }
-}
-
-const INITIALIZED_CONTAINER: unique symbol = Symbol('InitializedContainer')
-
-const DEFAULT_DOCKER_IMAGE = 'node:26.5.0-slim'
-
-type InitializedContainer = Docker.Container & {
-  readonly [INITIALIZED_CONTAINER]?: true
-}
-
-async function createContainer(docker: Docker, dockerImage: string): Promise<InitializedContainer> {
-  await pullImage(docker, dockerImage)
-
-  const container = await docker.createContainer({
-    Image: dockerImage,
-    Cmd: ['sleep', 'infinity'],
-    WorkingDir: CONTAINER_WORKDIR,
-    Tty: true,
-    HostConfig: {
-      AutoRemove: true,
-    },
-  })
-
-  await container.start()
-
-  console.log('Creating workspace directory...')
-  await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, CONTAINER_WORKDIR], {
-    user: 'root',
-  })
-
-  console.log('Installing CA certificates...')
-  await execCommand(docker, container, 'apt-get', ['update'], {
-    user: 'root',
-  })
-  await execCommand(
-    docker,
-    container,
-    'apt-get',
-    ['install', '-y', '--no-install-recommends', 'ca-certificates', 'curl'],
-    {
-      user: 'root',
-    },
-  )
-  await execCommand(docker, container, 'test', ['-d', '/etc/ssl/certs'], {
-    user: 'root',
-  })
-
-  console.log('Installing npm...')
-  await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
-    user: 'root',
-  })
-  const npmVersion = await execCommand(docker, container, 'npm', ['--version'], {
-    user: 'root',
-  })
-  if (npmVersion.stdout.trim() !== NPM_VERSION) {
-    throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
-  }
-
-  console.log('Setting up npm for non-root global installs')
-  await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, NPM_GLOBAL_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'npm', ['config', 'set', 'prefix', NPM_GLOBAL_DIR], {
-    user: NODE_USER,
-  })
-
-  console.log('Setting up copilot...')
-  await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, COPILOT_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'npm', ['install', '-g', `@github/copilot@${COPILOT_CLI_VERSION}`], {
-    user: NODE_USER,
-  })
-  await execCommand(docker, container, 'touch', [path.join(COPILOT_DIR, 'mcp-config.json')], {
-    user: NODE_USER,
-  })
-  await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], {
-    user: NODE_USER,
-  })
-
-  console.log('Setting up agents config...')
-  await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, AGENTS_DIR], {
-    user: 'root',
-  })
-
-  return container as InitializedContainer
-}
-
-function resolveContainerPath(filepath: string): string {
-  if (path.posix.isAbsolute(filepath)) {
-    return filepath
-  }
-
-  return path.posix.join(CONTAINER_WORKDIR, filepath)
-}
-
-function mapCopiedHeader(header: Headers, sourceName: string, destinationName: string): Headers {
-  const name =
-    header.name === sourceName
-      ? destinationName
-      : path.posix.join(destinationName, header.name.slice(sourceName.length))
-
-  return {
-    ...header,
-    name,
-    uid: SANDBOX_UID,
-    gid: SANDBOX_GID,
-  }
-}
-
-function normalizeCopyPath(filepath: string): string {
-  const normalized = path.posix.normalize(filepath.split(path.sep).join(path.posix.sep))
-  if (normalized === '.') {
-    return ''
-  }
-
-  return normalized.replace(/\/$/, '')
-}
-
-function normalizeExcludedPath(filepath: string, source: string): string {
-  if (path.isAbsolute(filepath)) {
-    return normalizeCopyPath(path.relative(source, filepath))
-  }
-
-  return normalizeCopyPath(filepath)
-}
-
-function isExcluded(relativePath: string, excludedPaths: ReadonlySet<string>): boolean {
-  if (!relativePath) {
-    return false
-  }
-
-  for (const excludedPath of excludedPaths) {
-    if (relativePath === excludedPath || relativePath.startsWith(`${excludedPath}/`)) {
-      return true
-    }
-  }
-
-  return false
-}
-
-function appendText(contents: string, text: string): string {
-  const suffix = ensureTrailingNewline(text)
-  if (contents.length === 0) {
-    return suffix
-  }
-
-  if (contents.endsWith('\n')) {
-    return `${contents}${suffix}`
-  }
-
-  return `${contents}\n${suffix}`
-}
-
-function ensureTrailingNewline(text: string): string {
-  return text.endsWith('\n') ? text : `${text}\n`
-}
-
-function assertValidSkillName(name: string): void {
-  if (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) {
-    return
-  }
-
-  throw new Error(`Invalid agent skill name "${name}". Skill names must be lowercase and use hyphens for spaces.`)
-}
-
-function assertValidCustomAgentName(name: string): void {
-  if (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name)) {
-    return
-  }
-
-  throw new Error(
-    `Invalid custom agent name "${name}". Custom agent names must be lowercase and use hyphens for spaces.`,
-  )
-}
-
-function createSkillContents(name: string, description: string, contents: string): string {
-  return `---
-name: ${JSON.stringify(name)}
-description: ${JSON.stringify(description)}
----
-
-${ensureTrailingNewline(contents)}`
-}
-
-function createCustomAgentContents(
-  name: string,
-  description: string,
-  contents: string,
-  options: CustomAgentOptions,
-): string {
-  const tools = options.tools ? `tools: ${JSON.stringify(options.tools)}\n` : ''
-
-  return `---
-name: ${JSON.stringify(name)}
-description: ${JSON.stringify(description)}
-${tools}---
-
-${ensureTrailingNewline(contents)}`
-}
-
-function getCustomAgentFileDestination(file: CustomAgentFile): string {
-  return getAdditionalFileDestination(file, 'custom agent')
-}
-
-function getAgentSkillFileDestination(file: AgentSkillFile): string {
-  return getAdditionalFileDestination(file, 'agent skill')
-}
-
-function getAdditionalFileDestination(file: CustomAgentFile | AgentSkillFile, fileKind: string): string {
-  const destinationPath = isWrittenFile(file) ? file.path : (file.destinationPath ?? path.basename(file.sourcePath))
-  const normalized = normalizeCopyPath(destinationPath)
-
-  if (!normalized || path.posix.isAbsolute(normalized) || normalized === '..' || normalized.startsWith('../')) {
-    throw new Error(`Invalid ${fileKind} file destination "${destinationPath}"`)
-  }
-
-  return normalized
-}
-
-function isWrittenFile(file: CustomAgentFile | AgentSkillFile): file is CustomAgentWrittenFile | AgentSkillWrittenFile {
-  return 'content' in file
-}
-
-async function readFileFromArchive(archive: NodeJS.ReadableStream): Promise<Buffer> {
-  const extract = tarStream.extract()
-
-  return new Promise((resolve, reject) => {
-    const chunks: Array<Buffer> = []
-
-    extract.on('entry', (header, stream, next) => {
-      if (header.type !== 'file') {
-        stream.resume()
-        next()
-        return
-      }
-
-      stream.on('data', (chunk: Buffer) => {
-        chunks.push(chunk)
-      })
-
-      stream.on('end', () => {
-        resolve(Buffer.concat(chunks))
-        next()
-      })
-
-      stream.on('error', reject)
-    })
-
-    extract.on('error', reject)
-    archive.on('error', reject)
-
-    archive.pipe(extract)
-  })
-}
-
-function pullImage(docker: Docker, name: string): Promise<void> {
-  return new Promise((resolve, reject) => {
-    docker.pull(name, (error: Error | null, stream: NodeJS.ReadableStream) => {
-      if (error) {
-        reject(error)
-        return
-      }
-
-      // Follow the pull progress
-      docker.modem.followProgress(
-        stream,
-        (progressError: Error | null) => {
-          if (progressError) {
-            reject(progressError)
-          } else {
-            resolve()
-          }
-        },
-        () => {},
-      )
-    })
-  })
-}
-
-type CommandResult = {
-  stdout: string
-  stderr: string
-  exitCode: number
-}
-
-class CommandError extends Error {
-  command: ReadonlyArray<string>
-  result: CommandResult
-
-  constructor(command: ReadonlyArray<string>, result: CommandResult) {
-    super(`Command failed with exit code ${result.exitCode}: ${command.join(' ')}`)
-    this.name = 'CommandError'
-    this.command = command
-    this.result = result
-  }
-}
-
-async function execCommand(
-  docker: Docker,
-  container: Docker.Container,
-  command: string,
-  args: Array<string>,
-  options: RunOptions,
-): Promise<CommandResult> {
-  const cmd = [command, ...args]
-  const env = options.env ? Object.entries(options.env).map(([key, value]) => `${key}=${value}`) : undefined
-  const exec = await container.exec({
-    Cmd: cmd,
-    AttachStdout: true,
-    AttachStderr: true,
-    WorkingDir: CONTAINER_WORKDIR,
-    Env: env,
-    User: options.user,
-  })
-
-  const stream = await exec.start({
-    hijack: true,
-    stdin: false,
-  })
-
-  return new Promise((resolve, reject) => {
-    const stdout = captureStream(process.stdout)
-    const stderr = captureStream(process.stderr)
-
-    docker.modem.demuxStream(stream, stdout.stream, stderr.stream)
-
-    stream.on('end', async () => {
-      try {
-        const inspectInfo = await exec.inspect()
-        const exitCode = inspectInfo.ExitCode ?? 0
-        const result = {
-          stdout: stdout.read(),
-          stderr: stderr.read(),
-          exitCode,
-        }
-
-        if (exitCode === 0 || options.allowNonZeroExitCode) {
-          resolve(result)
-          return
-        }
-
-        reject(new CommandError(cmd, result))
-      } catch (error) {
-        reject(error)
-      }
-    })
-    stream.on('error', reject)
-  })
-}
-
-function captureStream(destination: NodeJS.WritableStream): {stream: Writable; read(): string} {
-  const chunks: Array<Buffer> = []
-  const stream = new Writable({
-    write(chunk: Buffer | string, encoding, callback) {
-      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)
-      chunks.push(buffer)
-      destination.write(buffer)
-      callback()
-    },
-  })
-
-  return {
-    stream,
-    read() {
-      return Buffer.concat(chunks).toString('utf8')
-    },
-  }
-}
-
-const SandboxSchema = z.instanceof(Sandbox)
-
-export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox, SandboxSchema}
-export type {
-  AgentSkillCopiedFile,
-  AgentSkillFile,
-  AgentSkillOptions,
-  AgentSkillWrittenFile,
-  CopilotPluginConfig,
-  CopilotPluginSource,
-  CustomAgentCopiedFile,
-  CustomAgentFile,
-  CustomAgentOptions,
-  CustomAgentWrittenFile,
-  LocalCopilotPluginSource,
-  RemoteCopilotPluginSource,
-}
-export type {McpServerConfig} from './mcp-config'
diff --git a/packages/agent-eval/src/sandbox/index.ts b/packages/agent-eval/src/sandbox/index.ts
index f33fb11c..c49da6f4 100644
--- a/packages/agent-eval/src/sandbox/index.ts
+++ b/packages/agent-eval/src/sandbox/index.ts
@@ -1,5 +1,5 @@
 export * from './constants'
-export {SandboxSchema, SystemSandbox} from './system'
+export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE} from './system'
 export {VirtualSandbox} from './virtual'
 export type {
   AgentSkillCopiedFile,
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index bba695a9..cc410781 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -43,6 +43,7 @@ import type {
 import {DefaultHost, type Host} from '../host'
 import {VirtualSandbox} from './virtual'
 import {resolveContainerPath} from './path'
+import {logger} from '../logger'
 
 const COPILOT_CLI_VERSION = '1.0.80'
 const NPM_VERSION = '12.0.2'
@@ -335,7 +336,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
 
   await container.start()
 
-  console.log('Creating workspace directory...')
+  logger.info('Creating workspace directory...')
   await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
     user: 'root',
   })
@@ -343,7 +344,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: 'root',
   })
 
-  console.log('Installing CA certificates...')
+  logger.info('Installing CA certificates...')
   await execCommand(docker, container, 'apt-get', ['update'], {
     user: 'root',
   })
@@ -360,7 +361,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: 'root',
   })
 
-  console.log('Installing npm...')
+  logger.info('Installing npm...')
   await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
     user: 'root',
   })
@@ -371,7 +372,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
   }
 
-  console.log('Setting up npm for non-root global installs')
+  logger.info('Setting up npm for non-root global installs')
   await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
     user: 'root',
   })
@@ -382,7 +383,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: NODE_USER,
   })
 
-  console.log('Setting up copilot...')
+  logger.info('Setting up copilot...')
   await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
     user: 'root',
   })
@@ -399,7 +400,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: NODE_USER,
   })
 
-  console.log('Setting up agents config...')
+  logger.info('Setting up agents config...')
   await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
     user: 'root',
   })
@@ -682,4 +683,4 @@ const SandboxSchema = z.custom<Sandbox>(value => {
   return value instanceof SystemSandbox || value instanceof VirtualSandbox
 })
 
-export {SandboxSchema, SystemSandbox}
+export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE}
diff --git a/packages/agent-eval/src/scenario-config.ts b/packages/agent-eval/src/scenario-config.ts
deleted file mode 100644
index 0a056418..00000000
--- a/packages/agent-eval/src/scenario-config.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import type {ScenarioConfig} from './experiment-config'
-
-function defineScenario(config: ScenarioConfig) {
-  return config
-}
-
-export {defineScenario}
-export type {ScenarioConfig}
diff --git a/packages/agent-eval/src/scenario.test.ts b/packages/agent-eval/src/scenario.test.ts
index 886ff993..2bdc8062 100644
--- a/packages/agent-eval/src/scenario.test.ts
+++ b/packages/agent-eval/src/scenario.test.ts
@@ -54,6 +54,76 @@ test('listScenarios', async () => {
   expect(scenarios).not.toContainEqual(expect.objectContaining({id: '005-missing-test'}))
 })
 
+test('listScenarios includes optional metadata and browser tests', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      description: 'Test scenario',
+      prompt: 'Complete the task',
+      tags: ['test', 'browser'],
+    }),
+  )
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '001-scenario': {
+        'browser.test.ts': '',
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+    },
+  })
+
+  await expect(listScenarios(host, '/scenarios')).resolves.toEqual([
+    {
+      id: '001-scenario',
+      directory: '/scenarios/001-scenario',
+      prompt: 'Complete the task',
+      description: 'Test scenario',
+      tags: ['test', 'browser'],
+      testPath: '/scenarios/001-scenario/scenario.test.ts',
+      browserTestPath: '/scenarios/001-scenario/browser.test.ts',
+    },
+  ])
+})
+
+test('listScenarios ignores configs without a default export', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      prompt: 'test',
+    }),
+  )
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '001-scenario': {
+        'package.json': '{}',
+        'scenario.config.ts': `export const scenario = ${config}`,
+        'scenario.test.ts': '',
+      },
+    },
+  })
+
+  await expect(listScenarios(host, '/scenarios')).resolves.toEqual([])
+})
+
+test('listScenarios excludes template directories', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      prompt: 'test',
+    }),
+  )
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '000-template': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+    },
+  })
+
+  await expect(listScenarios(host, '/scenarios')).resolves.toEqual([])
+})
+
 test('throws if input is not a directory', async () => {
   const host = VirtualHost.create({
     '/test': '',
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index 1de788bd..4dd9e00d 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -2,11 +2,17 @@ import path from 'node:path'
 import * as z from 'zod/mini'
 import type {Host} from './host'
 
-const ScenarioConfigSchema = z.object({
-  description: z.optional(z.string()),
-  prompt: z.string(),
-  tags: z.optional(z.array(z.string())),
-})
+const ScenarioConfigSchema = z.union([
+  z.object({
+    description: z.optional(z.string()),
+    prompt: z.string(),
+    tags: z.optional(z.array(z.string())),
+  }),
+  z.object({
+    name: z.optional(z.string()),
+    path: z.string(),
+  }),
+])
 
 type ScenarioConfig = z.infer<typeof ScenarioConfigSchema>
 
diff --git a/packages/agent-eval/src/treatment-old.ts b/packages/agent-eval/src/treatment-old.ts
deleted file mode 100644
index 8e7d9101..00000000
--- a/packages/agent-eval/src/treatment-old.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-import type {ExperimentConfig, TreatmentConfig} from './experiment-config'
-import type {Model, ReasoningEffort} from './model'
-import type {Message} from './copilot-cli'
-import type {ResolvedScenario} from './resolve-experiment-scenario'
-
-type Treatment = {
-  config: TreatmentConfig
-  scenario: ResolvedScenario
-  experiment: ExperimentConfig
-  id: string
-  model: Model
-  reasoningEffort?: ReasoningEffort
-}
-
-type TreatmentResult = {
-  id: string
-  treatment: Treatment
-  artifacts: {
-    copilotConfigPath: string
-    directory: string
-    skillsConfigPath: string
-    testResultsPath: string
-    workspacePath: string
-  }
-  assistant: {
-    logs: Array<Message>
-    turns: number
-    outputTokens: number
-    premiumRequests: number
-    totalApiDurationMs: number
-    sessionDurationMs: number
-    tools: Record<string, number>
-  }
-  testResults: {
-    numTotalTests: number
-    numPassedTests: number
-    numFailedTests: number
-    numPendingTests: number
-    numTodoTests: number
-    tests: Array<{
-      title: string
-      fullName: string
-      status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
-      description?: string
-    }>
-  }
-  walkthrough: Walkthrough
-}
-
-const ControlTreatment: TreatmentConfig = {
-  name: 'Control',
-}
-
-type Walkthrough =
-  | {type: 'Unavailable'}
-  | {type: 'Screenshot'; filepath: string}
-  | {type: 'Screenshots'; screenshots: Array<string>}
-  | {type: 'Video'; filepath: string}
-
-export type {Treatment, TreatmentResult, ControlTreatment, Walkthrough}

From 18cfd65283c62b1ba452242fb6d916ade59289ad Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 16:03:55 -0500
Subject: [PATCH 18/52] feat: support running trials

---
 packages/agent-eval/src/cli.ts                | 439 +-----------------
 packages/agent-eval/src/experiment.ts         |  13 +-
 packages/agent-eval/src/model.ts              | 124 ++---
 packages/agent-eval/src/run.ts                | 121 ++++-
 .../src/sandbox/captured-stream.test.ts       |  50 ++
 .../agent-eval/src/sandbox/captured-stream.ts |  59 +++
 packages/agent-eval/src/sandbox/system.ts     |  50 +-
 7 files changed, 313 insertions(+), 543 deletions(-)
 create mode 100644 packages/agent-eval/src/sandbox/captured-stream.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/captured-stream.ts

diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index cfdd4186..d8c4fbe1 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -4,6 +4,7 @@ import {parseArgs} from 'node:util'
 import {getEnvironmentConfig} from './environment'
 import {run as runBenchmark} from './benchmark'
 import {run as runExperiment} from './experiment'
+import {logger} from './logger'
 
 const {values} = parseArgs({
   options: {
@@ -45,6 +46,10 @@ const {values} = parseArgs({
       short: 'h',
       description: 'Learn more about the command and its options',
     },
+    'log-level': {
+      type: 'string',
+      description: 'The log level to use',
+    },
     output: {
       type: 'string',
       description: 'The target file in which results are written',
@@ -70,6 +75,7 @@ Options:
   -e, --experiment <file>    The file name of the experiment to run
       --experiments <dir>    The directory containing local experiment files (default: ./experiments)
   -h, --help                 Learn more about the command and its options
+      --log-level <level>    The log level to use (default: info)
       --output <file>        The target file in which results are written (default: output.json)
       --scenarios <dir>      The directory containing scenario directories (default: ./scenarios)
 `)
@@ -80,6 +86,10 @@ if (values.help) {
   process.exit(0)
 }
 
+if (values['log-level']) {
+  logger.level = values['log-level']
+}
+
 const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN
 const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY
 
@@ -99,11 +109,13 @@ const env = getEnvironmentConfig({
 })
 
 if (values.benchmark) {
+  logger.info('Running benchmark: %s', values.benchmark)
   await runBenchmark({
     env,
     id: values.benchmark,
   })
 } else if (values.experiment) {
+  logger.info('Running experiment: %s', values.benchmark)
   await runExperiment({
     env,
     id: values.experiment,
@@ -111,430 +123,3 @@ if (values.benchmark) {
 } else {
   displayHelp()
 }
-
-// let selectedExperiment: {
-//   id: string
-//   config: ExperimentConfig
-// }
-//
-// if (!existsSync(ARTIFACTS_DIR)) {
-//   await fs.mkdir(ARTIFACTS_DIR, {recursive: true})
-// }
-//
-// if (values.benchmark) {
-//   //
-// } else if (values.experiment) {
-//   const config = await findExperiment(values.experiment, {
-//     directory: values.experiments,
-//   })
-//   if (!config) {
-//     const experiments = await listExperiments({
-//       directory: values.experiments,
-//     })
-//     throw new Error(
-//       `Experiment "${values.experiment}" was not found. Available experiments:\n${experiments
-//         .map(([id]) => id)
-//         .join('\n')}`,
-//     )
-//   }
-//
-//   selectedExperiment = {
-//     id: existsSync(values.experiment)
-//       ? path.basename(values.experiment, path.extname(values.experiment))
-//       : values.experiment,
-//     config,
-//   }
-// } else {
-//   const experiments = await listExperiments({
-//     directory: values.experiments,
-//   })
-//   if (experiments.length !== 1) {
-//     throw new Error(
-//       `Select an experiment with --experiment. Available experiments:\n${experiments.map(([id]) => id).join('\n')}`,
-//     )
-//   }
-//
-//   selectedExperiment = {
-//     id: experiments[0][0],
-//     config: experiments[0][1],
-//   }
-// }
-//
-// function randomize<T>(input: Array<T>): Array<T> {
-//   const randomized: Array<T> = input.slice()
-//
-//   // Fisher–Yates shuffle
-//   for (let i = randomized.length - 1; i > 0; i--) {
-//     const j = Math.floor(Math.random() * (i + 1))
-//     ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]]
-//   }
-//
-//   return randomized
-// }
-//
-// function getSuccessRate(result: TreatmentResult): number {
-//   if (result.testResults.numTotalTests === 0) {
-//     return 0
-//   }
-//
-//   return result.testResults.numPassedTests / result.testResults.numTotalTests
-// }
-//
-// function compareResults(a: TreatmentResult, b: TreatmentResult): number {
-//   return (
-//     getSuccessRate(b) - getSuccessRate(a) ||
-//     a.assistant.outputTokens - b.assistant.outputTokens ||
-//     a.assistant.sessionDurationMs - b.assistant.sessionDurationMs ||
-//     a.assistant.premiumRequests - b.assistant.premiumRequests ||
-//     a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) ||
-//     a.treatment.config.name.localeCompare(b.treatment.config.name) ||
-//     a.treatment.model.localeCompare(b.treatment.model) ||
-//     (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') ||
-//     a.treatment.scenario.id.localeCompare(b.treatment.scenario.id)
-//   )
-// }
-//
-// type ResultSummary = {
-//   experiment: string
-//   treatment?: string
-//   scenario?: string
-//   model?: Model
-//   reasoningEffort?: ReasoningEffort
-//   runs: number
-//   numPassedTests: number
-//   numTotalTests: number
-//   outputTokens: number
-//   premiumRequests: number
-//   sessionDurationMs: number
-//   totalApiDurationMs: number
-// }
-//
-// type ResultSummaryValues = {
-//   treatment?: string
-//   scenario?: string
-//   model?: Model
-//   reasoningEffort?: ReasoningEffort
-// }
-//
-// function createResultSummary(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): ResultSummary {
-//   return {
-//     experiment: result.treatment.experiment.name,
-//     treatment: summaryValues.treatment,
-//     scenario: summaryValues.scenario,
-//     model: summaryValues.model,
-//     reasoningEffort: summaryValues.reasoningEffort,
-//     runs: 0,
-//     numPassedTests: 0,
-//     numTotalTests: 0,
-//     outputTokens: 0,
-//     premiumRequests: 0,
-//     sessionDurationMs: 0,
-//     totalApiDurationMs: 0,
-//   }
-// }
-//
-// function addResultToSummary(summary: ResultSummary, result: TreatmentResult) {
-//   summary.runs += 1
-//   summary.numPassedTests += result.testResults.numPassedTests
-//   summary.numTotalTests += result.testResults.numTotalTests
-//   summary.outputTokens += result.assistant.outputTokens
-//   summary.premiumRequests += result.assistant.premiumRequests
-//   summary.sessionDurationMs += result.assistant.sessionDurationMs
-//   summary.totalApiDurationMs += result.assistant.totalApiDurationMs
-// }
-//
-// function getSummarySuccessRate(summary: ResultSummary): number {
-//   if (summary.numTotalTests === 0) {
-//     return 0
-//   }
-//
-//   return summary.numPassedTests / summary.numTotalTests
-// }
-//
-// function compareSummaries(a: ResultSummary, b: ResultSummary): number {
-//   return (
-//     getSummarySuccessRate(b) - getSummarySuccessRate(a) ||
-//     a.outputTokens - b.outputTokens ||
-//     a.sessionDurationMs - b.sessionDurationMs ||
-//     a.premiumRequests - b.premiumRequests ||
-//     a.experiment.localeCompare(b.experiment) ||
-//     (a.treatment ?? '').localeCompare(b.treatment ?? '') ||
-//     (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
-//     (a.model ?? '').localeCompare(b.model ?? '') ||
-//     (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
-//   )
-// }
-//
-// function formatPercent(value: number): string {
-//   return `${(value * 100).toFixed(1)}%`
-// }
-//
-// function formatDuration(ms: number): string {
-//   const seconds = ms / 1000
-//
-//   if (seconds < 60) {
-//     return `${seconds.toFixed(1)}s`
-//   }
-//
-//   const minutes = Math.floor(seconds / 60)
-//   const remainingSeconds = seconds - minutes * 60
-//   return `${minutes}m ${remainingSeconds.toFixed(1)}s`
-// }
-//
-// function formatNumber(value: number): string {
-//   return new Intl.NumberFormat('en-US').format(value)
-// }
-//
-// type TableRow = Record<string, string | number>
-//
-// function formatTable(rows: Array<TableRow>, columns: Array<string>): string {
-//   const columnWidths = columns.map(column => {
-//     let width = column.length
-//
-//     for (const row of rows) {
-//       width = Math.max(width, String(row[column] ?? '').length)
-//     }
-//
-//     return width
-//   })
-//
-//   const formatRow = (row: TableRow) => {
-//     return columns
-//       .map((column, index) => {
-//         return String(row[column] ?? '').padEnd(columnWidths[index])
-//       })
-//       .join('  ')
-//   }
-//
-//   return [
-//     formatRow(Object.fromEntries(columns.map(column => [column, column]))),
-//     columnWidths.map(width => '-'.repeat(width)).join('  '),
-//     ...rows.map(formatRow),
-//   ].join('\n')
-// }
-//
-// function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): string {
-//   return [
-//     result.treatment.experiment.name,
-//     summaryValues.treatment ?? '',
-//     summaryValues.scenario ?? '',
-//     summaryValues.model ?? '',
-//     summaryValues.reasoningEffort ?? '',
-//   ].join('\0')
-// }
-//
-// type ResultHierarchy = Array<{
-//   experiment: string
-//   treatments: Array<{
-//     summary: ResultSummary
-//     scenarios: Array<{
-//       summary: ResultSummary
-//       models: Array<ResultSummary>
-//     }>
-//   }>
-// }>
-//
-// function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
-//   const experiments = new Set<string>()
-//   const treatmentSummaries = new Map<string, ResultSummary>()
-//   const scenarioSummaries = new Map<string, ResultSummary>()
-//   const modelSummaries = new Map<string, ResultSummary>()
-//
-//   for (const result of results) {
-//     experiments.add(result.treatment.experiment.name)
-//
-//     const treatmentValues = {
-//       treatment: result.treatment.config.name,
-//     }
-//     const treatmentKey = getSummaryKey(result, treatmentValues)
-//     const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues)
-//     addResultToSummary(treatmentSummary, result)
-//     treatmentSummaries.set(treatmentKey, treatmentSummary)
-//
-//     const scenarioValues = {
-//       treatment: result.treatment.config.name,
-//       scenario: result.treatment.scenario.id,
-//     }
-//     const scenarioKey = getSummaryKey(result, scenarioValues)
-//     const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(result, scenarioValues)
-//     addResultToSummary(scenarioSummary, result)
-//     scenarioSummaries.set(scenarioKey, scenarioSummary)
-//
-//     const modelValues = {
-//       treatment: result.treatment.config.name,
-//       scenario: result.treatment.scenario.id,
-//       model: result.treatment.model,
-//       reasoningEffort: result.treatment.reasoningEffort,
-//     }
-//     const modelKey = getSummaryKey(result, modelValues)
-//     const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(result, modelValues)
-//     addResultToSummary(modelSummary, result)
-//     modelSummaries.set(modelKey, modelSummary)
-//   }
-//
-//   return [...experiments].toSorted().map(experiment => {
-//     return {
-//       experiment,
-//       treatments: [...treatmentSummaries.values()]
-//         .filter(treatmentSummary => {
-//           return treatmentSummary.experiment === experiment
-//         })
-//         .toSorted(compareSummaries)
-//         .map(summary => {
-//           return {
-//             summary,
-//             scenarios: [...scenarioSummaries.values()]
-//               .filter(scenarioSummary => {
-//                 return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment
-//               })
-//               .toSorted(compareSummaries)
-//               .map(scenarioSummary => {
-//                 return {
-//                   summary: scenarioSummary,
-//                   models: [...modelSummaries.values()]
-//                     .filter(modelSummary => {
-//                       return (
-//                         modelSummary.experiment === experiment &&
-//                         modelSummary.treatment === summary.treatment &&
-//                         modelSummary.scenario === scenarioSummary.scenario
-//                       )
-//                     })
-//                     .toSorted(compareSummaries),
-//                 }
-//               }),
-//           }
-//         }),
-//     }
-//   })
-// }
-//
-// function formatResultSummaries(results: Array<TreatmentResult>): string {
-//   const columns = [
-//     'Experiment',
-//     'Treatment',
-//     'Scenario',
-//     'Model',
-//     'Reasoning Effort',
-//     'Success Rate',
-//     'Tests',
-//     'Runs',
-//     'Output Tokens',
-//     'Premium Requests',
-//     'Session Time',
-//     'API Time',
-//   ]
-//   const rows: Array<TableRow> = []
-//
-//   for (const {treatments} of getResultSummaries(results)) {
-//     for (const {summary, scenarios} of treatments) {
-//       rows.push(formatSummaryRow(summary, 'treatment'))
-//
-//       for (const {summary: scenarioSummary, models} of scenarios) {
-//         rows.push(formatSummaryRow(scenarioSummary, 'scenario'))
-//
-//         for (const model of models) {
-//           rows.push(formatSummaryRow(model, 'model'))
-//         }
-//       }
-//     }
-//   }
-//
-//   return formatTable(rows, columns)
-// }
-//
-// async function appendResultsToJobSummary(resultSummaries: string) {
-//   if (!GITHUB_STEP_SUMMARY) {
-//     return
-//   }
-//
-//   await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`)
-// }
-//
-// function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow {
-//   return {
-//     Experiment: level === 'treatment' ? summary.experiment : '',
-//     Treatment: level === 'treatment' ? (summary.treatment ?? '') : '',
-//     Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario ?? ''}` : '',
-//     Model: level === 'model' ? `    ${summary.model ?? ''}` : 'All models',
-//     'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '',
-//     'Success Rate': formatPercent(getSummarySuccessRate(summary)),
-//     Tests: `${summary.numPassedTests}/${summary.numTotalTests}`,
-//     Runs: summary.runs,
-//     'Output Tokens': formatNumber(summary.outputTokens),
-//     'Premium Requests': formatNumber(summary.premiumRequests),
-//     'Session Time': formatDuration(summary.sessionDurationMs),
-//     'API Time': formatDuration(summary.totalApiDurationMs),
-//   }
-// }
-//
-// const config = selectedExperiment.config
-//
-// console.log('Running experiment:', config.name)
-//
-// const scenarios = await Promise.all(
-//   config.scenarios.map(scenarioConfig => {
-//     return resolveExperimentScenario(scenarioConfig, {
-//       directory: values.scenarios,
-//     })
-//   }),
-// )
-//
-// const treatments: Array<Treatment> = config.models.flatMap(modelConfig => {
-//   return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => {
-//     return scenarios.flatMap(scenarioConfig => {
-//       return [
-//         {
-//           config: ControlTreatment,
-//           scenario: scenarioConfig,
-//           experiment: config,
-//           id: randomUUID(),
-//           model,
-//           reasoningEffort,
-//         },
-//         ...config.treatments.map(treatment => {
-//           return {
-//             config: treatment,
-//             scenario: scenarioConfig,
-//             experiment: config,
-//             id: randomUUID(),
-//             model,
-//             reasoningEffort,
-//           }
-//         }),
-//       ]
-//     })
-//   })
-// })
-//
-// // Randomize treatments to mitigate any ordering effects. We want to make sure
-// // that if there are any external factors that could impact the scenarios (e.g.
-// // rate limits, resource constraints), they are more likely to impact all
-// // scenarios rather than just the ones at the end.
-// const selectedTreatments = SHARD ? selectShard(treatments, SHARD) : treatments
-// const results: Array<TreatmentResult> = await run(randomize(selectedTreatments), {
-//   artifactsDirectory: ARTIFACTS_DIR,
-//   copilotToken: COPILOT_GITHUB_TOKEN,
-//   dockerImage: DOCKER_IMAGE,
-//   maxConcurrency: MAX_CONCURRENCY,
-// })
-//
-// const sortedResults = results.toSorted(compareResults)
-// const resultSummaries = formatResultSummaries(sortedResults)
-// console.log(resultSummaries)
-// await appendResultsToJobSummary(resultSummaries)
-//
-// const outputFilePath = path.isAbsolute(values.output) ? values.output : path.resolve(process.cwd(), values.output)
-//
-// if (!existsSync(path.dirname(outputFilePath))) {
-//   await fs.mkdir(path.dirname(outputFilePath), {recursive: true})
-// }
-//
-// const output = createAgentEvalOutput({
-//   id: randomUUID(),
-//   experimentId: selectedExperiment.id,
-//   experiment: config,
-//   scenarios,
-//   results: sortedResults,
-// })
-//
-// await fs.writeFile(outputFilePath, JSON.stringify(output, null, 2))
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index bf9a8c39..f29d2975 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -1,6 +1,6 @@
 import path from 'node:path'
 import * as z from 'zod/mini'
-import {ModelVariantConfigSchema, type ModelVariant, type ModelVariantConfig} from './model'
+import {getModelVariants, ModelVariantConfigSchema, type ModelVariant} from './model'
 import {getScenario, type Scenario} from './scenario'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
 import {DefaultHost, type Host} from './host'
@@ -14,7 +14,7 @@ import {randomUUID} from 'node:crypto'
 const ExperimentConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
-  models: z.array(ModelVariantConfigSchema),
+  models: ModelVariantConfigSchema,
   scenarios: z.array(z.string()),
   setup: z.optional(TreatmentSetupSchema),
   treatments: z.array(TreatmentSchema),
@@ -99,8 +99,12 @@ async function listExperiments({
     const experiment: Experiment = {
       id,
       filepath,
-      ...config,
+      name: config.name,
+      description: config.description,
+      models: getModelVariants(config.models),
       scenarios,
+      setup: config.setup,
+      treatments: config.treatments,
     }
     experiments.push(experiment)
   }
@@ -167,9 +171,8 @@ async function run({env, host = DefaultHost, id}: {env: EnvironmentConfig; host?
     host,
     plan,
   })
-  console.log(results)
 
-  // throw new Error('unimplemented')
+  return results
 }
 
 export {defineConfig, listExperiments, getExperiment, ExperimentConfigSchema, run}
diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts
index 5b6e5224..40381059 100644
--- a/packages/agent-eval/src/model.ts
+++ b/packages/agent-eval/src/model.ts
@@ -92,69 +92,71 @@ const ModelVariantSchema = z.custom<ModelVariant>(value => {
   return reasoningEffortsByModel.get(value.name)?.has(value.reasoningEffort) ?? false
 })
 
-const ModelVariantConfigSchema = z.union([
-  z.enum(models.map(model => model.name)),
-  z.object({
-    name: z.literal(models[0].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[1].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[2].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[3].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[4].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[5].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[6].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[7].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[8].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[9].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[10].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[11].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[12].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))),
-  }),
-  z.object({
-    name: z.literal(models[13].name),
-    reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))),
-  }),
-])
+const ModelVariantConfigSchema = z.array(
+  z.union([
+    z.enum(models.map(model => model.name)),
+    z.object({
+      name: z.literal(models[0].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[1].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[2].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[3].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[4].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[5].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[6].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[7].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[8].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[9].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[10].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[11].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[12].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))),
+    }),
+    z.object({
+      name: z.literal(models[13].name),
+      reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))),
+    }),
+  ]),
+)
 
 type ModelVariantConfig = z.infer<typeof ModelVariantConfigSchema>
 
-function getModelVariants(input: Array<ModelVariantConfig>): Array<ModelVariant<Model>> {
+function getModelVariants(input: ModelVariantConfig): Array<ModelVariant<Model>> {
   return input.flatMap(config => {
     if (typeof config === 'string') {
       return [{name: config, reasoningEffort: 'medium'}]
@@ -171,7 +173,7 @@ function getModelVariants(input: Array<ModelVariantConfig>): Array<ModelVariant<
     return config.reasoningEfforts.map(effort => {
       return {name: config.name, reasoningEffort: effort}
     })
-  })
+  }) as Array<ModelVariant<Model>>
 }
 
 export {models, getModelVariants, ModelVariantSchema, ModelVariantConfigSchema}
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index 310a1653..40f7be82 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -80,9 +80,12 @@ async function run({
           await using sandbox = await host.createSandbox({
             dockerImage: env.dockerImage,
           })
-          return runTrial(host, sandbox, trial, {
+          return await runTrial({
             artifactsDirectory: env.artifactsDirectory,
             copilotToken: env.copilotToken,
+            host,
+            sandbox,
+            trial,
           })
         })
       })
@@ -92,16 +95,21 @@ async function run({
   return results
 }
 
-type RunTrialOptions = {
+async function runTrial({
+  artifactsDirectory,
+  copilotToken,
+  host = DefaultHost,
+  sandbox,
+  trial,
+}: {
   artifactsDirectory: string
   copilotToken: string
-}
-
-async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: RunTrialOptions): Promise<TrialResult> {
+  host?: Host
+  sandbox: Sandbox
+  trial: Trial
+}): Promise<TrialResult> {
   logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
 
-  const {artifactsDirectory, copilotToken} = options
-
   logger.info('Copying files from: %s...', trial.scenario.directory)
 
   await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
@@ -111,44 +119,55 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
     user: 'root',
   })
 
-  logger.info('Obfuscating package name...')
+  logger.info('[%s] Obfuscating package name...', trial.treatment.name)
   await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
     user: NODE_USER,
   })
 
-  logger.info('Removing workspace dependency...')
+  logger.info('[%s] Removing workspace dependency...', trial.treatment.name)
   await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
     user: NODE_USER,
   })
 
-  logger.info('Installing dependencies...')
+  logger.info('[%s] Installing dependencies...', trial.treatment.name)
   await sandbox.runCommand('npm', ['install'], {
     user: NODE_USER,
   })
 
   if (trial.setup) {
-    logger.info('Running generic setup...')
+    logger.info('[%s] Running generic setup...', trial.treatment.name)
     await trial.setup({
       sandbox,
     })
   }
 
   if (trial.treatment.setup) {
-    logger.info('Running treatment setup...')
+    logger.info('[%s] Running treatment setup...', trial.treatment.name)
     await trial.treatment.setup({
       sandbox,
     })
   }
 
-  logger.info('Run build script...')
+  logger.info('[%s] Run build script...', trial.treatment.name)
   await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
     user: NODE_USER,
   })
 
-  logger.info('Running copilot...')
+  logger.info('[%s] Running copilot...', trial.treatment.name)
   const copilotOutput = await sandbox.runCommand(
     'copilot',
-    ['--prompt', trial.scenario.prompt, '--model', trial.model.name, '--reasoning-effort', trial.model.reasoningEffort],
+    [
+      '--prompt',
+      trial.scenario.prompt,
+      '--model',
+      trial.model.name,
+      '--reasoning-effort',
+      trial.model.reasoningEffort,
+      '--mode',
+      'autopilot',
+      '--output-format',
+      'json',
+    ],
     {
       user: NODE_USER,
       env: {
@@ -164,7 +183,7 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
     return parseMessage(JSON.parse(trimmed))
   })
 
-  logger.info('Running tests...')
+  logger.info('[%s] Running tests...', trial.treatment.name)
 
   const TEST_PATH = 'scenario.test.ts'
   const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
@@ -190,6 +209,63 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
   const WALKTHROUGH_DIR = 'walkthrough'
   const WALKTHROUGH_VIEWPORT_WIDTH = 1440
   const WALKTHROUGH_VIEWPORT_HEIGHT = 900
+  console.log('Capturing walkthrough...')
+  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
+    user: 'root',
+  })
+  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
+    user: NODE_USER,
+  })
+  await sandbox.runCommand(
+    'npx',
+    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
+    {
+      user: NODE_USER,
+    },
+  )
+  await sandbox.writeFile(
+    'agent-browser.json',
+    JSON.stringify({
+      executablePath: CHROMIUM_EXECUTABLE_PATH,
+    }),
+  )
+  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
+
+Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
+
+Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
+
+- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
+- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
+- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
+
+Only capture the walkthrough, do not make any further code changes.`
+  const walkthroughResult = await sandbox.runCommand(
+    'copilot',
+    [
+      '--prompt',
+      walkthroughPrompt,
+      '--model',
+      'gpt-5.6-terra',
+      '--reasoning-effort',
+      'medium',
+      '--mode',
+      'autopilot',
+      '--output-format',
+      'json',
+    ],
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: copilotToken,
+      },
+      allowNonZeroExitCode: true,
+    },
+  )
+
+  if (walkthroughResult.exitCode !== 0) {
+    logger.warn('[%s] Unable to capture walkthrough: %s', trial.treatment.name, walkthroughResult.stderr)
+  }
 
   const artifactDirectory = path.join(artifactsDirectory, trial.id)
   const workspaceDirectory = path.join(artifactDirectory, 'workspace')
@@ -203,17 +279,19 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
   }
   await host.fs.mkdir(workspaceDirectory, {recursive: true})
 
-  logger.info('Downloading agent workspace to: %s...', workspaceDirectory)
+  logger.info('[%s] Downloading artifacts to: %s...', trial.treatment.name, artifactDirectory)
+
+  logger.debug('[%s] Downloading agent workspace to: %s...', trial.treatment.name, workspaceDirectory)
   await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
     ignore(name) {
       return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
     },
   })
 
-  logger.info('Downloading copilot config to: %s...', copilotConfigDirectory)
+  logger.debug('[%s] Downloading copilot config to: %s...', trial.treatment.name, copilotConfigDirectory)
   await sandbox.download(COPILOT_DIR, copilotConfigDirectory)
 
-  logger.info('Downloading skills config to: %s...', skillsConfigDirectory)
+  logger.debug('[%s] Downloading skills config to: %s...', trial.treatment.name, skillsConfigDirectory)
   await sandbox.download(AGENTS_DIR, skillsConfigDirectory)
 
   let walkthrough: Walkthrough = {
@@ -221,8 +299,9 @@ async function runTrial(host: Host, sandbox: Sandbox, trial: Trial, options: Run
   }
 
   if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) {
-    logger.info(
-      'Moving walkthrough artifacts from: %s to: %s...',
+    logger.debug(
+      '[%s] Moving walkthrough artifacts from: %s to: %s...',
+      trial.treatment.name,
       path.join(workspaceDirectory, WALKTHROUGH_DIR),
       walkthroughPath,
     )
diff --git a/packages/agent-eval/src/sandbox/captured-stream.test.ts b/packages/agent-eval/src/sandbox/captured-stream.test.ts
new file mode 100644
index 00000000..20f64d3f
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/captured-stream.test.ts
@@ -0,0 +1,50 @@
+import {describe, expect, test, vi} from 'vitest'
+import {createCapturedStream} from './captured-stream'
+
+describe('createCapturedStream', () => {
+  test('logs complete lines while preserving captured output', () => {
+    const onLine = vi.fn()
+    const captured = createCapturedStream(onLine)
+
+    captured.stream.write('first line\nsecond ')
+    captured.stream.write('line\r\nthird line\r')
+    captured.stream.write('\nfourth line\rfifth line')
+    captured.flush()
+
+    expect(onLine.mock.calls).toEqual([
+      ['first line'],
+      ['second line'],
+      ['third line'],
+      ['fourth line'],
+      ['fifth line'],
+    ])
+    expect(captured.read()).toBe('first line\nsecond line\r\nthird line\r\nfourth line\rfifth line')
+  })
+
+  test('flushes a final line without a delimiter', () => {
+    const onLine = vi.fn()
+    const captured = createCapturedStream(onLine)
+
+    captured.stream.write('final line')
+
+    expect(onLine).not.toHaveBeenCalled()
+
+    captured.flush()
+
+    expect(onLine).toHaveBeenCalledOnce()
+    expect(onLine).toHaveBeenCalledWith('final line')
+  })
+
+  test('preserves multibyte characters split across chunks', () => {
+    const onLine = vi.fn()
+    const captured = createCapturedStream(onLine)
+    const output = Buffer.from('hello 👋\n')
+
+    captured.stream.write(output.subarray(0, output.length - 3))
+    captured.stream.write(output.subarray(output.length - 3))
+    captured.flush()
+
+    expect(onLine).toHaveBeenCalledWith('hello 👋')
+    expect(captured.read()).toBe('hello 👋\n')
+  })
+})
diff --git a/packages/agent-eval/src/sandbox/captured-stream.ts b/packages/agent-eval/src/sandbox/captured-stream.ts
new file mode 100644
index 00000000..7c14aa1f
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/captured-stream.ts
@@ -0,0 +1,59 @@
+import {StringDecoder} from 'node:string_decoder'
+import {Writable} from 'node:stream'
+
+type CapturedStream = {
+  stream: Writable
+  read(): string
+  flush(): void
+}
+
+function createCapturedStream(onLine: (line: string) => void): CapturedStream {
+  const chunks: Array<Buffer> = []
+  const decoder = new StringDecoder('utf8')
+  let pending = ''
+
+  function emitLines(final: boolean): void {
+    while (pending.length > 0) {
+      const delimiterIndex = pending.search(/[\r\n]/)
+      if (delimiterIndex === -1) {
+        break
+      }
+
+      if (pending[delimiterIndex] === '\r' && delimiterIndex === pending.length - 1 && !final) {
+        break
+      }
+
+      const delimiterLength = pending.slice(delimiterIndex, delimiterIndex + 2) === '\r\n' ? 2 : 1
+      onLine(pending.slice(0, delimiterIndex))
+      pending = pending.slice(delimiterIndex + delimiterLength)
+    }
+
+    if (final && pending.length > 0) {
+      onLine(pending)
+      pending = ''
+    }
+  }
+
+  const stream = new Writable({
+    write(chunk: Buffer | string, encoding, callback) {
+      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)
+      chunks.push(buffer)
+      pending += decoder.write(buffer)
+      emitLines(false)
+      callback()
+    },
+  })
+
+  return {
+    stream,
+    read() {
+      return Buffer.concat(chunks).toString('utf8')
+    },
+    flush() {
+      pending += decoder.end()
+      emitLines(true)
+    },
+  }
+}
+
+export {createCapturedStream}
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index cc410781..6bcd8b0a 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -1,6 +1,5 @@
 import {randomUUID} from 'node:crypto'
 import path from 'node:path'
-import {Writable} from 'node:stream'
 import {pipeline} from 'node:stream/promises'
 import Docker from 'dockerode'
 import tarFs from 'tar-fs'
@@ -44,6 +43,7 @@ import {DefaultHost, type Host} from '../host'
 import {VirtualSandbox} from './virtual'
 import {resolveContainerPath} from './path'
 import {logger} from '../logger'
+import {createCapturedStream} from './captured-stream'
 
 const COPILOT_CLI_VERSION = '1.0.80'
 const NPM_VERSION = '12.0.2'
@@ -336,7 +336,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
 
   await container.start()
 
-  logger.info('Creating workspace directory...')
+  logger.debug('Creating workspace directory...')
   await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
     user: 'root',
   })
@@ -344,7 +344,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: 'root',
   })
 
-  logger.info('Installing CA certificates...')
+  logger.debug('Installing CA certificates...')
   await execCommand(docker, container, 'apt-get', ['update'], {
     user: 'root',
   })
@@ -361,7 +361,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: 'root',
   })
 
-  logger.info('Installing npm...')
+  logger.debug('Installing npm...')
   await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
     user: 'root',
   })
@@ -372,7 +372,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
   }
 
-  logger.info('Setting up npm for non-root global installs')
+  logger.debug('Setting up npm for non-root global installs')
   await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
     user: 'root',
   })
@@ -383,7 +383,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: NODE_USER,
   })
 
-  logger.info('Setting up copilot...')
+  logger.debug('Setting up copilot...')
   await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
     user: 'root',
   })
@@ -400,7 +400,7 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     user: NODE_USER,
   })
 
-  logger.info('Setting up agents config...')
+  logger.debug('Setting up agents config...')
   await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
     user: 'root',
   })
@@ -631,13 +631,20 @@ async function execCommand(
   })
 
   return new Promise((resolve, reject) => {
-    const stdout = captureStream(process.stdout)
-    const stderr = captureStream(process.stderr)
+    const stdout = createCapturedStream(line => {
+      logger.debug('[sandbox]: %s', line)
+    })
+    const stderr = createCapturedStream(line => {
+      logger.debug('[sandbox]: %s', line)
+    })
 
     docker.modem.demuxStream(stream, stdout.stream, stderr.stream)
 
     stream.on('end', async () => {
       try {
+        stdout.flush()
+        stderr.flush()
+
         const inspectInfo = await exec.inspect()
         const exitCode = inspectInfo.ExitCode ?? 0
         const result = {
@@ -656,27 +663,12 @@ async function execCommand(
         reject(error)
       }
     })
-    stream.on('error', reject)
-  })
-}
-
-function captureStream(destination: NodeJS.WritableStream): {stream: Writable; read(): string} {
-  const chunks: Array<Buffer> = []
-  const stream = new Writable({
-    write(chunk: Buffer | string, encoding, callback) {
-      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)
-      chunks.push(buffer)
-      destination.write(buffer)
-      callback()
-    },
+    stream.on('error', error => {
+      stdout.flush()
+      stderr.flush()
+      reject(error)
+    })
   })
-
-  return {
-    stream,
-    read() {
-      return Buffer.concat(chunks).toString('utf8')
-    },
-  }
 }
 
 const SandboxSchema = z.custom<Sandbox>(value => {

From 46a22c44068cbe4f42484c44b51d231e9b35ba66 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Tue, 1 Sep 2026 16:56:02 -0500
Subject: [PATCH 19/52] feat: improve copilot cli message parsing and run trial
 behavior

---
 packages/agent-eval/src/cli.ts              |  4 ++-
 packages/agent-eval/src/copilot-cli.test.ts | 40 +++++++++++++++++++++
 packages/agent-eval/src/copilot-cli.ts      |  4 +--
 packages/agent-eval/src/run.ts              |  8 +++--
 packages/agent-eval/src/sandbox/system.ts   | 11 +++++-
 5 files changed, 60 insertions(+), 7 deletions(-)

diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index d8c4fbe1..0daf01b3 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -108,6 +108,8 @@ const env = getEnvironmentConfig({
   scenariosDirectory: values.scenarios,
 })
 
+logger.debug('Environment configuration: %o', env)
+
 if (values.benchmark) {
   logger.info('Running benchmark: %s', values.benchmark)
   await runBenchmark({
@@ -115,7 +117,7 @@ if (values.benchmark) {
     id: values.benchmark,
   })
 } else if (values.experiment) {
-  logger.info('Running experiment: %s', values.benchmark)
+  logger.info('Running experiment: %s', values.experiment)
   await runExperiment({
     env,
     id: values.experiment,
diff --git a/packages/agent-eval/src/copilot-cli.test.ts b/packages/agent-eval/src/copilot-cli.test.ts
index 4794b432..878d33a3 100644
--- a/packages/agent-eval/src/copilot-cli.test.ts
+++ b/packages/agent-eval/src/copilot-cli.test.ts
@@ -59,6 +59,20 @@ describe(parseMessage, () => {
       timestamp: '2026-07-24T01:57:27.191Z',
       parentId: '080d2918-29d2-4da6-b115-3fe37825b5d2',
     },
+    {
+      type: 'user.message',
+      data: {
+        content: "Update the index page to use a primary button with the text 'Submit'",
+        transformedContent: "Update the index page to use a primary button with the text 'Submit'",
+        supportedNativeDocumentMimeTypes: [],
+        agentMode: 'autopilot',
+        interactionId: '40a03f57-636f-41be-bef9-cfe71591862a',
+        parentAgentTaskId: 'b5c96c9d-a61b-4689-9720-455a5a1e2644',
+      },
+      id: '30d361b9-068e-4ebe-bef0-857a8d92e5ab',
+      timestamp: '2026-09-01T21:29:34.818Z',
+      parentId: '0344de20-b203-4642-a644-4cc19a9283ac',
+    },
     {
       type: 'assistant.turn_start',
       data: {
@@ -168,6 +182,32 @@ describe(parseMessage, () => {
       timestamp: '2026-08-17T00:23:26.072Z',
       parentId: '2b1722fe-509e-4368-8c8b-2ae8985099b6',
     },
+    {
+      type: 'assistant.message',
+      data: {
+        messageId: '99e762aa-0196-42be-8756-39ba3c294a07',
+        content: '',
+        toolRequests: [
+          {
+            toolCallId: 'call_oLjXYouZFO1PA5D6u8d54cFX',
+            name: 'glob',
+            arguments: {
+              pattern: '**/*',
+              paths: '/home/sandbox/workspace',
+            },
+            type: 'function',
+            intentionSummary: '**/*',
+          },
+        ],
+        interactionId: 'feb477c9-0fb1-47d5-bd08-b397389ec2da',
+        turnId: '0',
+        reasoningOpaque: 'opaque',
+        encryptedContent: 'encrypted',
+      },
+      id: '0c89050a-a6f6-4cd5-8ded-7e25d70b01bc',
+      timestamp: '2026-09-01T21:35:25.627Z',
+      parentId: '2ac75cd3-5bf3-4986-a5b2-00cfa4b6e9f5',
+    },
     {
       type: 'assistant.idle',
       data: {},
diff --git a/packages/agent-eval/src/copilot-cli.ts b/packages/agent-eval/src/copilot-cli.ts
index 76982ff9..73c94beb 100644
--- a/packages/agent-eval/src/copilot-cli.ts
+++ b/packages/agent-eval/src/copilot-cli.ts
@@ -101,7 +101,7 @@ const UserMessageSchema = z.object({
   data: z.object({
     content: z.string(),
     transformedContent: z.string(),
-    attachments: z.array(z.unknown()),
+    attachments: z.optional(z.array(z.unknown())),
     supportedNativeDocumentMimeTypes: z.array(z.string()),
     agentMode: z.string(),
     interactionId: z.string(),
@@ -149,7 +149,7 @@ const AssistantMessageSchema = z.object({
     encryptedContent: z.optional(z.string()),
     phase: z.optional(z.string()),
     outputTokens: z.optional(z.number()),
-    requestId: z.string(),
+    requestId: z.optional(z.string()),
   }),
 })
 
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
index 40f7be82..92f1b21b 100644
--- a/packages/agent-eval/src/run.ts
+++ b/packages/agent-eval/src/run.ts
@@ -165,6 +165,7 @@ async function runTrial({
       trial.model.reasoningEffort,
       '--mode',
       'autopilot',
+      '--allow-all',
       '--output-format',
       'json',
     ],
@@ -226,7 +227,7 @@ async function runTrial({
   await sandbox.writeFile(
     'agent-browser.json',
     JSON.stringify({
-      executablePath: CHROMIUM_EXECUTABLE_PATH,
+      executablePath: '/usr/bin/chromium',
     }),
   )
   const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
@@ -251,6 +252,7 @@ Only capture the walkthrough, do not make any further code changes.`
       'medium',
       '--mode',
       'autopilot',
+      '--allow-all',
       '--output-format',
       'json',
     ],
@@ -284,7 +286,7 @@ Only capture the walkthrough, do not make any further code changes.`
   logger.debug('[%s] Downloading agent workspace to: %s...', trial.treatment.name, workspaceDirectory)
   await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
     ignore(name) {
-      return name.includes('node_modules') || name.includes('.next') || name.includes('dist')
+      return name.includes('node_modules') || name.includes('.next') || name.includes('.turbo') || name.includes('dist')
     },
   })
 
@@ -357,7 +359,7 @@ Only capture the walkthrough, do not make any further code changes.`
 }
 
 function getVitestConfig(outputFile: string) {
-  return `import {defineConfig} from 'vites/tconfig';
+  return `import {defineConfig} from 'vitest/config';
 
 export default defineConfig({
   test: {
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index 6bcd8b0a..a60030be 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -45,7 +45,7 @@ import {resolveContainerPath} from './path'
 import {logger} from '../logger'
 import {createCapturedStream} from './captured-stream'
 
-const COPILOT_CLI_VERSION = '1.0.80'
+const COPILOT_CLI_VERSION = '1.0.82'
 const NPM_VERSION = '12.0.2'
 
 const DEFAULT_MCP_CONFIG: McpConfigFile = {
@@ -396,6 +396,15 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
   await execCommand(docker, container, 'touch', [path.join(COPILOT_DIR, 'mcp-config.json')], {
     user: NODE_USER,
   })
+  await execCommand(
+    docker,
+    container,
+    'bash',
+    ['-c', `echo '${JSON.stringify(DEFAULT_MCP_CONFIG)}' > ${MCP_CONFIG_PATH}`],
+    {
+      user: NODE_USER,
+    },
+  )
   await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], {
     user: NODE_USER,
   })

From 68ad16864fe2becceafc5f6566f92dd0634f6432 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 2 Sep 2026 12:21:37 -0500
Subject: [PATCH 20/52] refactor: move run into trial and plan

---
 packages/agent-eval/src/plan.ts               |  54 ++-
 packages/agent-eval/src/run.ts                | 392 ------------------
 .../src/{run.test.ts => trial.test.ts}        |   0
 packages/agent-eval/src/trial.ts              | 366 +++++++++++++++-
 4 files changed, 414 insertions(+), 398 deletions(-)
 delete mode 100644 packages/agent-eval/src/run.ts
 rename packages/agent-eval/src/{run.test.ts => trial.test.ts} (100%)

diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
index f2e0ba79..d98c8c6f 100644
--- a/packages/agent-eval/src/plan.ts
+++ b/packages/agent-eval/src/plan.ts
@@ -1,6 +1,9 @@
+import Queue from 'p-queue'
 import * as z from 'zod/mini'
-import {TrialSchema} from './trial'
-import type {Trial} from './trial'
+import {TrialSchema, run as runTrial} from './trial'
+import type {Trial, TrialResult} from './trial'
+import type {EnvironmentConfig} from './environment'
+import {DefaultHost, type Host} from './host'
 
 const PlanSchema = z.object({
   trials: z.array(TrialSchema),
@@ -30,5 +33,50 @@ function randomize<T>(input: Array<T>): Array<T> {
   return randomized
 }
 
-export {createPlan}
+type RunPlanOptions = {
+  env: EnvironmentConfig
+  host?: Host
+  plan: Plan
+}
+
+async function run({env, host = DefaultHost, plan}: RunPlanOptions): Promise<Array<TrialResult>> {
+  const queue = new Queue({
+    concurrency: env.concurrency,
+  })
+
+  const results = await Promise.all(
+    plan.trials.map(trial => {
+      return queue.add(() => {
+        return retry(async () => {
+          await using sandbox = await host.createSandbox({
+            dockerImage: env.dockerImage,
+          })
+          return await runTrial({
+            artifactsDirectory: env.artifactsDirectory,
+            copilotToken: env.copilotToken,
+            host,
+            sandbox,
+            trial,
+          })
+        })
+      })
+    }),
+  )
+
+  return results
+}
+
+async function retry<T>(fn: () => Promise<T>, retries: number = 3): Promise<T> {
+  try {
+    return await fn()
+  } catch (error) {
+    if (retries > 0) {
+      console.log('Retrying after error: %s', error)
+      return retry(fn, retries - 1)
+    }
+    throw error
+  }
+}
+
+export {createPlan, run}
 export type {Plan}
diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts
deleted file mode 100644
index 92f1b21b..00000000
--- a/packages/agent-eval/src/run.ts
+++ /dev/null
@@ -1,392 +0,0 @@
-import path from 'node:path'
-import Queue from 'p-queue'
-import * as z from 'zod/mini'
-import {MessageSchema, parseMessage, type Message} from './copilot-cli'
-import type {Plan} from './plan'
-import {TrialSchema, type Trial} from './trial'
-import {DefaultHost, type Host} from './host'
-// import {CONTAINER_WORKDIR, NODE_USER, Sandbox} from './sandbox'
-import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type Sandbox} from './sandbox'
-import {parseTestResults, TestResultsSchema} from './vitest'
-import {logger} from './logger'
-import type {EnvironmentConfig} from './environment'
-
-const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
-
-const AssistantSchema = z.object({
-  logs: z.array(MessageSchema),
-  turns: z.number(),
-  outputTokens: z.number(),
-  premiumRequests: z.number(),
-  totalApiDurationMs: z.number(),
-  sessionDurationMs: z.number(),
-  tools: z.record(z.string(), z.number()),
-})
-
-const WalkthroughSchema = z.discriminatedUnion('type', [
-  z.object({type: z.literal('Unavailable')}),
-  z.object({type: z.literal('Screenshot'), filepath: z.string()}),
-  z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}),
-  z.object({type: z.literal('Video'), filepath: z.string()}),
-])
-
-type Walkthrough = z.infer<typeof WalkthroughSchema>
-
-const TrialResultSchema = z.object({
-  artifacts: z.object({
-    directory: z.string(),
-    copilotConfigDirectory: z.string(),
-    skillsConfigDirectory: z.string(),
-    testResultsPath: z.string(),
-    workspaceDirectory: z.string(),
-  }),
-  trial: TrialSchema,
-  assistant: z.object({
-    sessions: z.array(
-      z.object({
-        messages: z.array(MessageSchema),
-      }),
-    ),
-  }),
-  testResults: TestResultsSchema,
-  walkthrough: WalkthroughSchema,
-})
-
-type TrialResult = z.infer<typeof TrialResultSchema>
-
-type RunOptions = {
-  artifactsDirectory: string
-  copilotToken: string
-  maxConcurrency?: number
-}
-
-async function run({
-  env,
-  host = DefaultHost,
-  plan,
-}: {
-  env: EnvironmentConfig
-  host?: Host
-  plan: Plan
-}): Promise<Array<TrialResult>> {
-  const queue = new Queue({
-    concurrency: env.concurrency,
-  })
-
-  const results = await Promise.all(
-    plan.trials.map(trial => {
-      return queue.add(() => {
-        return retry(async () => {
-          await using sandbox = await host.createSandbox({
-            dockerImage: env.dockerImage,
-          })
-          return await runTrial({
-            artifactsDirectory: env.artifactsDirectory,
-            copilotToken: env.copilotToken,
-            host,
-            sandbox,
-            trial,
-          })
-        })
-      })
-    }),
-  )
-
-  return results
-}
-
-async function runTrial({
-  artifactsDirectory,
-  copilotToken,
-  host = DefaultHost,
-  sandbox,
-  trial,
-}: {
-  artifactsDirectory: string
-  copilotToken: string
-  host?: Host
-  sandbox: Sandbox
-  trial: Trial
-}): Promise<TrialResult> {
-  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
-
-  logger.info('Copying files from: %s...', trial.scenario.directory)
-
-  await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
-    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
-  })
-  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
-    user: 'root',
-  })
-
-  logger.info('[%s] Obfuscating package name...', trial.treatment.name)
-  await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
-    user: NODE_USER,
-  })
-
-  logger.info('[%s] Removing workspace dependency...', trial.treatment.name)
-  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
-    user: NODE_USER,
-  })
-
-  logger.info('[%s] Installing dependencies...', trial.treatment.name)
-  await sandbox.runCommand('npm', ['install'], {
-    user: NODE_USER,
-  })
-
-  if (trial.setup) {
-    logger.info('[%s] Running generic setup...', trial.treatment.name)
-    await trial.setup({
-      sandbox,
-    })
-  }
-
-  if (trial.treatment.setup) {
-    logger.info('[%s] Running treatment setup...', trial.treatment.name)
-    await trial.treatment.setup({
-      sandbox,
-    })
-  }
-
-  logger.info('[%s] Run build script...', trial.treatment.name)
-  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
-    user: NODE_USER,
-  })
-
-  logger.info('[%s] Running copilot...', trial.treatment.name)
-  const copilotOutput = await sandbox.runCommand(
-    'copilot',
-    [
-      '--prompt',
-      trial.scenario.prompt,
-      '--model',
-      trial.model.name,
-      '--reasoning-effort',
-      trial.model.reasoningEffort,
-      '--mode',
-      'autopilot',
-      '--allow-all',
-      '--output-format',
-      'json',
-    ],
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: copilotToken,
-      },
-    },
-  )
-  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
-    const trimmed = line.trim()
-    if (trimmed.length === 0) {
-      return []
-    }
-    return parseMessage(JSON.parse(trimmed))
-  })
-
-  logger.info('[%s] Running tests...', trial.treatment.name)
-
-  const TEST_PATH = 'scenario.test.ts'
-  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
-  const TEST_RESULTS_PATH = 'test-results.json'
-
-  await sandbox.copy(trial.scenario.testPath, TEST_PATH)
-  await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
-  await sandbox.runCommand(
-    'sh',
-    ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
-    {
-      user: NODE_USER,
-      env: {},
-    },
-  )
-  const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
-  const rawTestResult: unknown = JSON.parse(testResultsContent)
-  const testResults = parseTestResults(rawTestResult)
-  if (!testResults.success) {
-    throw new Error(`Failed to parse test results: ${testResults.error}`)
-  }
-
-  const WALKTHROUGH_DIR = 'walkthrough'
-  const WALKTHROUGH_VIEWPORT_WIDTH = 1440
-  const WALKTHROUGH_VIEWPORT_HEIGHT = 900
-  console.log('Capturing walkthrough...')
-  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
-    user: 'root',
-  })
-  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
-    user: NODE_USER,
-  })
-  await sandbox.runCommand(
-    'npx',
-    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
-    {
-      user: NODE_USER,
-    },
-  )
-  await sandbox.writeFile(
-    'agent-browser.json',
-    JSON.stringify({
-      executablePath: '/usr/bin/chromium',
-    }),
-  )
-  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
-
-Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
-
-Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
-
-- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
-- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
-- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
-
-Only capture the walkthrough, do not make any further code changes.`
-  const walkthroughResult = await sandbox.runCommand(
-    'copilot',
-    [
-      '--prompt',
-      walkthroughPrompt,
-      '--model',
-      'gpt-5.6-terra',
-      '--reasoning-effort',
-      'medium',
-      '--mode',
-      'autopilot',
-      '--allow-all',
-      '--output-format',
-      'json',
-    ],
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: copilotToken,
-      },
-      allowNonZeroExitCode: true,
-    },
-  )
-
-  if (walkthroughResult.exitCode !== 0) {
-    logger.warn('[%s] Unable to capture walkthrough: %s', trial.treatment.name, walkthroughResult.stderr)
-  }
-
-  const artifactDirectory = path.join(artifactsDirectory, trial.id)
-  const workspaceDirectory = path.join(artifactDirectory, 'workspace')
-  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
-  const copilotConfigDirectory = path.join(artifactDirectory, '.copilot')
-  const skillsConfigDirectory = path.join(artifactDirectory, '.agents')
-  const testResultsPath = path.join(workspaceDirectory, 'test-results.json')
-
-  if (host.existsSync(artifactDirectory)) {
-    await host.fs.rm(artifactDirectory, {recursive: true, force: true})
-  }
-  await host.fs.mkdir(workspaceDirectory, {recursive: true})
-
-  logger.info('[%s] Downloading artifacts to: %s...', trial.treatment.name, artifactDirectory)
-
-  logger.debug('[%s] Downloading agent workspace to: %s...', trial.treatment.name, workspaceDirectory)
-  await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
-    ignore(name) {
-      return name.includes('node_modules') || name.includes('.next') || name.includes('.turbo') || name.includes('dist')
-    },
-  })
-
-  logger.debug('[%s] Downloading copilot config to: %s...', trial.treatment.name, copilotConfigDirectory)
-  await sandbox.download(COPILOT_DIR, copilotConfigDirectory)
-
-  logger.debug('[%s] Downloading skills config to: %s...', trial.treatment.name, skillsConfigDirectory)
-  await sandbox.download(AGENTS_DIR, skillsConfigDirectory)
-
-  let walkthrough: Walkthrough = {
-    type: 'Unavailable',
-  }
-
-  if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) {
-    logger.debug(
-      '[%s] Moving walkthrough artifacts from: %s to: %s...',
-      trial.treatment.name,
-      path.join(workspaceDirectory, WALKTHROUGH_DIR),
-      walkthroughPath,
-    )
-    await host.fs.mkdir(walkthroughPath, {recursive: true})
-    await host.fs.rename(path.join(workspaceDirectory, WALKTHROUGH_DIR), walkthroughPath)
-
-    if (host.existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
-      walkthrough = {
-        type: 'Screenshot',
-        filepath: path.join(walkthroughPath, 'screenshot.png'),
-      }
-    } else if (host.existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
-      walkthrough = {
-        type: 'Video',
-        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
-      }
-    } else if (host.existsSync(path.join(walkthroughPath, 'screenshots'))) {
-      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
-      const entries = await host.fs.readdir(screenshotsDir).then(filenames => {
-        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
-      })
-      const screenshots = entries.filter(entry => {
-        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
-      })
-      if (screenshots.length > 0) {
-        walkthrough = {
-          type: 'Screenshots',
-          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
-        }
-      }
-    }
-  }
-
-  return {
-    artifacts: {
-      directory: artifactDirectory,
-      copilotConfigDirectory,
-      skillsConfigDirectory,
-      testResultsPath,
-      workspaceDirectory,
-    },
-    trial,
-    assistant: {
-      sessions: [
-        {
-          messages,
-        },
-      ],
-    },
-    testResults: testResults.data,
-    walkthrough,
-  }
-}
-
-function getVitestConfig(outputFile: string) {
-  return `import {defineConfig} from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    reporters: [
-      [
-        'json',
-        {
-          outputFile: ${JSON.stringify(outputFile)},
-          includeTaskLocation: true,
-        },
-      ],
-    ],
-  },
-})`
-}
-
-async function retry<T>(fn: () => Promise<T>, retries: number = 3): Promise<T> {
-  try {
-    return await fn()
-  } catch (error) {
-    if (retries > 0) {
-      console.log('Retrying after error: %s', error)
-      return retry(fn, retries - 1)
-    }
-    throw error
-  }
-}
-
-export {run, runTrial, TrialResultSchema as RunResultSchema}
-export type {RunOptions as RunContext, TrialResult as RunResult}
diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/trial.test.ts
similarity index 100%
rename from packages/agent-eval/src/run.test.ts
rename to packages/agent-eval/src/trial.test.ts
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index 4c991497..742e8321 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -1,6 +1,12 @@
+import path from 'node:path'
+import {isMessageType, MessageSchema, parseMessage, type Message} from './copilot-cli'
+import {DefaultHost, type Host} from './host'
+import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type Sandbox} from './sandbox'
+import {parseTestResults, TestResultsSchema} from './vitest'
+import {logger} from './logger'
 import * as z from 'zod/mini'
 import {ModelVariantSchema} from './model'
-import {ScenarioConfigSchema, ScenarioSchema} from './scenario'
+import {ScenarioSchema} from './scenario'
 import {TreatmentSchema, TreatmentSetupSchema} from './treatment'
 
 const TrialSchema = z.object({
@@ -13,5 +19,359 @@ const TrialSchema = z.object({
 
 type Trial = z.infer<typeof TrialSchema>
 
-export {TrialSchema}
-export type {Trial}
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
+
+const WalkthroughSchema = z.discriminatedUnion('type', [
+  z.object({type: z.literal('Unavailable')}),
+  z.object({type: z.literal('Screenshot'), filepath: z.string()}),
+  z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}),
+  z.object({type: z.literal('Video'), filepath: z.string()}),
+])
+
+type Walkthrough = z.infer<typeof WalkthroughSchema>
+
+const AgentSessionSchema = z.object({
+  turns: z.number(),
+  outputTokens: z.number(),
+  premiumRequests: z.number(),
+  totalApiDurationMs: z.number(),
+  sessionDurationMs: z.number(),
+  tools: z.record(z.string(), z.number()),
+  messages: z.array(MessageSchema),
+})
+
+type AgentSession = z.infer<typeof AgentSessionSchema>
+
+const TrialResultSchema = z.object({
+  artifacts: z.object({
+    directory: z.string(),
+    copilotConfigDirectory: z.string(),
+    skillsConfigDirectory: z.string(),
+    testResultsPath: z.string(),
+    workspaceDirectory: z.string(),
+  }),
+  trial: TrialSchema,
+  agent: z.object({
+    sessions: z.array(AgentSessionSchema),
+  }),
+  testResults: TestResultsSchema,
+  walkthrough: WalkthroughSchema,
+})
+
+type TrialResult = z.infer<typeof TrialResultSchema>
+
+async function run({
+  artifactsDirectory,
+  copilotToken,
+  host = DefaultHost,
+  sandbox,
+  trial,
+}: {
+  artifactsDirectory: string
+  copilotToken: string
+  host?: Host
+  sandbox: Sandbox
+  trial: Trial
+}): Promise<TrialResult> {
+  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
+
+  logger.info('Copying files from: %s...', trial.scenario.directory)
+
+  await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
+    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
+  })
+  await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
+    user: 'root',
+  })
+
+  logger.info('[%s] Obfuscating package name...', trial.treatment.name)
+  await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
+    user: NODE_USER,
+  })
+
+  logger.info('[%s] Removing workspace dependency...', trial.treatment.name)
+  await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
+    user: NODE_USER,
+  })
+
+  logger.info('[%s] Installing dependencies...', trial.treatment.name)
+  await sandbox.runCommand('npm', ['install'], {
+    user: NODE_USER,
+  })
+
+  if (trial.setup) {
+    logger.info('[%s] Running generic setup...', trial.treatment.name)
+    await trial.setup({
+      sandbox,
+    })
+  }
+
+  if (trial.treatment.setup) {
+    logger.info('[%s] Running treatment setup...', trial.treatment.name)
+    await trial.treatment.setup({
+      sandbox,
+    })
+  }
+
+  logger.info('[%s] Run build script...', trial.treatment.name)
+  await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
+    user: NODE_USER,
+  })
+
+  logger.info('[%s] Running copilot...', trial.treatment.name)
+  const copilotOutput = await sandbox.runCommand(
+    'copilot',
+    [
+      '--prompt',
+      trial.scenario.prompt,
+      '--model',
+      trial.model.name,
+      '--reasoning-effort',
+      trial.model.reasoningEffort,
+      '--mode',
+      'autopilot',
+      '--allow-all',
+      '--output-format',
+      'json',
+    ],
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: copilotToken,
+      },
+    },
+  )
+  const messages: Array<Message> = copilotOutput.stdout.split('\n').flatMap(line => {
+    const trimmed = line.trim()
+    if (trimmed.length === 0) {
+      return []
+    }
+    return parseMessage(JSON.parse(trimmed))
+  })
+
+  logger.info('[%s] Running tests...', trial.treatment.name)
+
+  const TEST_PATH = 'scenario.test.ts'
+  const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
+  const TEST_RESULTS_PATH = 'test-results.json'
+
+  await sandbox.copy(trial.scenario.testPath, TEST_PATH)
+  await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
+  await sandbox.runCommand(
+    'sh',
+    ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
+    {
+      user: NODE_USER,
+      env: {},
+    },
+  )
+  const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
+  const rawTestResult: unknown = JSON.parse(testResultsContent)
+  const testResults = parseTestResults(rawTestResult)
+  if (!testResults.success) {
+    throw new Error(`Failed to parse test results: ${testResults.error}`)
+  }
+
+  const WALKTHROUGH_DIR = 'walkthrough'
+  const WALKTHROUGH_VIEWPORT_WIDTH = 1440
+  const WALKTHROUGH_VIEWPORT_HEIGHT = 900
+  console.log('Capturing walkthrough...')
+  await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
+    user: 'root',
+  })
+  await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], {
+    user: NODE_USER,
+  })
+  await sandbox.runCommand(
+    'npx',
+    ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'],
+    {
+      user: NODE_USER,
+    },
+  )
+  await sandbox.writeFile(
+    'agent-browser.json',
+    JSON.stringify({
+      executablePath: '/usr/bin/chromium',
+    }),
+  )
+  const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves.
+
+Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything.
+
+Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project:
+
+- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png.
+- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc.
+- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm.
+
+Only capture the walkthrough, do not make any further code changes.`
+  const walkthroughResult = await sandbox.runCommand(
+    'copilot',
+    [
+      '--prompt',
+      walkthroughPrompt,
+      '--model',
+      'gpt-5.6-terra',
+      '--reasoning-effort',
+      'medium',
+      '--mode',
+      'autopilot',
+      '--allow-all',
+      '--output-format',
+      'json',
+    ],
+    {
+      user: NODE_USER,
+      env: {
+        COPILOT_GITHUB_TOKEN: copilotToken,
+      },
+      allowNonZeroExitCode: true,
+    },
+  )
+
+  if (walkthroughResult.exitCode !== 0) {
+    logger.warn('[%s] Unable to capture walkthrough: %s', trial.treatment.name, walkthroughResult.stderr)
+  }
+
+  const artifactDirectory = path.join(artifactsDirectory, trial.id)
+  const workspaceDirectory = path.join(artifactDirectory, 'workspace')
+  const walkthroughPath = path.join(artifactDirectory, 'walkthrough')
+  const copilotConfigDirectory = path.join(artifactDirectory, '.copilot')
+  const skillsConfigDirectory = path.join(artifactDirectory, '.agents')
+  const testResultsPath = path.join(workspaceDirectory, 'test-results.json')
+
+  if (host.existsSync(artifactDirectory)) {
+    await host.fs.rm(artifactDirectory, {recursive: true, force: true})
+  }
+  await host.fs.mkdir(workspaceDirectory, {recursive: true})
+
+  logger.info('[%s] Downloading artifacts to: %s...', trial.treatment.name, artifactDirectory)
+
+  logger.debug('[%s] Downloading agent workspace to: %s...', trial.treatment.name, workspaceDirectory)
+  await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
+    ignore(name) {
+      return name.includes('node_modules') || name.includes('.next') || name.includes('.turbo') || name.includes('dist')
+    },
+  })
+
+  logger.debug('[%s] Downloading copilot config to: %s...', trial.treatment.name, copilotConfigDirectory)
+  await sandbox.download(COPILOT_DIR, copilotConfigDirectory)
+
+  logger.debug('[%s] Downloading skills config to: %s...', trial.treatment.name, skillsConfigDirectory)
+  await sandbox.download(AGENTS_DIR, skillsConfigDirectory)
+
+  let walkthrough: Walkthrough = {
+    type: 'Unavailable',
+  }
+
+  if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) {
+    logger.debug(
+      '[%s] Moving walkthrough artifacts from: %s to: %s...',
+      trial.treatment.name,
+      path.join(workspaceDirectory, WALKTHROUGH_DIR),
+      walkthroughPath,
+    )
+    await host.fs.mkdir(walkthroughPath, {recursive: true})
+    await host.fs.rename(path.join(workspaceDirectory, WALKTHROUGH_DIR), walkthroughPath)
+
+    if (host.existsSync(path.join(walkthroughPath, 'screenshot.png'))) {
+      walkthrough = {
+        type: 'Screenshot',
+        filepath: path.join(walkthroughPath, 'screenshot.png'),
+      }
+    } else if (host.existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) {
+      walkthrough = {
+        type: 'Video',
+        filepath: path.join(walkthroughPath, 'walkthrough.webm'),
+      }
+    } else if (host.existsSync(path.join(walkthroughPath, 'screenshots'))) {
+      const screenshotsDir = path.join(walkthroughPath, 'screenshots')
+      const entries = await host.fs.readdir(screenshotsDir).then(filenames => {
+        return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true}))
+      })
+      const screenshots = entries.filter(entry => {
+        return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())
+      })
+      if (screenshots.length > 0) {
+        walkthrough = {
+          type: 'Screenshots',
+          screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)),
+        }
+      }
+    }
+  }
+
+  return {
+    artifacts: {
+      directory: artifactDirectory,
+      copilotConfigDirectory,
+      skillsConfigDirectory,
+      testResultsPath,
+      workspaceDirectory,
+    },
+    trial,
+    agent: {
+      sessions: [getAgentSession(messages)],
+    },
+    testResults: testResults.data,
+    walkthrough,
+  }
+}
+
+function getAgentSession(messages: Array<Message>): AgentSession {
+  const turns = new Set()
+  const toolCalls = new Map()
+  let outputTokens = 0
+
+  for (const message of messages) {
+    if (isMessageType(message, 'assistant.turn_start')) {
+      turns.add(message.data.turnId)
+    }
+
+    if (isMessageType(message, 'assistant.message')) {
+      outputTokens += message.data.outputTokens ?? 0
+    }
+
+    if (isMessageType(message, 'tool.execution_start')) {
+      const toolName = message.data.toolName
+      toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1)
+    }
+  }
+
+  const result = messages.find(message => isMessageType(message, 'result'))
+  if (!result) {
+    throw new Error('No result message found in copilot output')
+  }
+
+  return {
+    messages,
+    outputTokens,
+    premiumRequests: result.usage.premiumRequests,
+    sessionDurationMs: result.usage.sessionDurationMs,
+    tools: Object.fromEntries(toolCalls),
+    totalApiDurationMs: result.usage.totalApiDurationMs,
+    turns: turns.size,
+  }
+}
+
+function getVitestConfig(outputFile: string) {
+  return `import {defineConfig} from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    reporters: [
+      [
+        'json',
+        {
+          outputFile: ${JSON.stringify(outputFile)},
+          includeTaskLocation: true,
+        },
+      ],
+    ],
+  },
+})`
+}
+
+export {TrialSchema, TrialResultSchema, run}
+export type {Trial, TrialResult}

From 85068982c66bfbe5f59c25e19fb50f06a9efe5e5 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 2 Sep 2026 12:43:24 -0500
Subject: [PATCH 21/52] refactor: add support for testing trial runs

---
 packages/agent-eval/src/copilot-cli.ts |   4 +-
 packages/agent-eval/src/experiment.ts  |   5 +-
 packages/agent-eval/src/host.ts        |   7 +-
 packages/agent-eval/src/plan.ts        |   4 +-
 packages/agent-eval/src/trial.test.ts  | 171 ++++++++++++++++++-------
 packages/agent-eval/src/trial.ts       |   2 +-
 6 files changed, 140 insertions(+), 53 deletions(-)

diff --git a/packages/agent-eval/src/copilot-cli.ts b/packages/agent-eval/src/copilot-cli.ts
index 73c94beb..b1bb53fd 100644
--- a/packages/agent-eval/src/copilot-cli.ts
+++ b/packages/agent-eval/src/copilot-cli.ts
@@ -298,6 +298,8 @@ const ResultMessageSchema = z.object({
   }),
 })
 
+type ResultMessage = z.infer<typeof ResultMessageSchema>
+
 const KnownMessageSchema = z.discriminatedUnion('type', [
   SessionMcpServerStatusChangedMessageSchema,
   SessionMcpServersLoadedMessageSchema,
@@ -390,4 +392,4 @@ export {
   isMessageType,
   parseMessage,
 }
-export type {KnownMessage, Message, UnknownMessage, UnknownMessageType}
+export type {KnownMessage, Message, ResultMessage, UnknownMessage, UnknownMessageType}
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index f29d2975..e5f1b14c 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -6,8 +6,7 @@ import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment,
 import {DefaultHost, type Host} from './host'
 import type {EnvironmentConfig} from './environment'
 import {logger} from './logger'
-import {createPlan} from './plan'
-import {run as runTrials} from './run'
+import {create as createPlan, run as runPlan} from './plan'
 import type {Trial} from './trial'
 import {randomUUID} from 'node:crypto'
 
@@ -166,7 +165,7 @@ async function run({env, host = DefaultHost, id}: {env: EnvironmentConfig; host?
     })
   })
   const plan = await createPlan(trials)
-  const results = await runTrials({
+  const results = await runPlan({
     env,
     host,
     plan,
diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
index fe98d5d8..8e386719 100644
--- a/packages/agent-eval/src/host.ts
+++ b/packages/agent-eval/src/host.ts
@@ -59,8 +59,11 @@ class VirtualHost implements Host {
     return await import(dataUri)
   }
 
-  crateSandbox(options?: SandboxCreateOptions): Promise<Sandbox> {
-    return VirtualSandbox.create(options)
+  createSandbox(options?: SandboxCreateOptions): Promise<Sandbox> {
+    return VirtualSandbox.create({
+      host: this,
+      ...options,
+    })
   }
 }
 
diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
index d98c8c6f..7425112c 100644
--- a/packages/agent-eval/src/plan.ts
+++ b/packages/agent-eval/src/plan.ts
@@ -15,7 +15,7 @@ const PlanSchema = z.object({
 type Plan = z.infer<typeof PlanSchema>
 
 // TODO: support plan with sharding
-async function createPlan(trials: Array<Trial>): Promise<Plan> {
+async function create(trials: Array<Trial>): Promise<Plan> {
   return {
     trials: randomize(trials),
   }
@@ -78,5 +78,5 @@ async function retry<T>(fn: () => Promise<T>, retries: number = 3): Promise<T> {
   }
 }
 
-export {createPlan, run}
+export {create, run}
 export type {Plan}
diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts
index f05e3356..6addb427 100644
--- a/packages/agent-eval/src/trial.test.ts
+++ b/packages/agent-eval/src/trial.test.ts
@@ -1,11 +1,24 @@
-import {expect, test, vi} from 'vitest'
-import {runTrial} from './run'
-import {NODE_USER, VirtualSandbox} from './sandbox'
+import {test, vi} from 'vitest'
+import type {ResultMessage} from './copilot-cli'
+import {run, type Trial} from './trial'
+import {COPILOT_DIR, AGENTS_DIR} from './sandbox'
+import {VirtualHost} from './host'
+import {randomUUID} from 'node:crypto'
 
-test('runTrial', async () => {
-  await using sandbox = await VirtualSandbox.create()
-  vi.spyOn(sandbox, 'copy').mockResolvedValue()
-  const runCommand = vi.spyOn(sandbox, 'runCommand').mockImplementation(async command => {
+test('run', async () => {
+  const host = VirtualHost.create({
+    [AGENTS_DIR]: {},
+    [COPILOT_DIR]: {},
+    '/scenarios/test': {
+      'scenario.config.ts': '',
+      'scenario.test.ts': '',
+    },
+  })
+  await using sandbox = await host.createSandbox()
+
+  const runCommand = sandbox.runCommand
+
+  vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, options) => {
     if (command === 'sh') {
       await sandbox.writeFile(
         'test-results.json',
@@ -21,47 +34,117 @@ test('runTrial', async () => {
       )
     }
 
-    return {
-      stdout: '',
-      stderr: '',
-      exitCode: 0,
+    if (command === 'copilot') {
+      const result: ResultMessage = {
+        type: 'result',
+        timestamp: '',
+        sessionId: '',
+        exitCode: 0,
+        usage: {
+          premiumRequests: 0,
+          totalApiDurationMs: 0,
+          sessionDurationMs: 0,
+          codeChanges: {
+            linesAdded: 0,
+            linesRemoved: 0,
+            filesModified: [],
+          },
+        },
+      }
+      return {
+        stdout: [JSON.stringify(result)].join('\n'),
+        stderr: '',
+        exitCode: 0,
+      }
     }
+
+    return runCommand(command, args, options)
   })
 
-  const result = await runTrial(
-    sandbox,
-    {
-      id: 'test',
-      scenario: {
-        id: 'test',
-        directory: '/test',
-        prompt: 'test',
-        tags: [],
-        testPath: '/test/scenario.test.ts',
-      },
-      treatment: {
-        name: 'test',
-      },
-      model: {
-        name: 'gpt-5.6-sol',
-        reasoningEffort: 'medium',
-      },
+  const trial: Trial = {
+    id: randomUUID(),
+    scenario: {
+      id: 'test-id',
+      directory: '/scenarios/test',
+      prompt: 'test-prompt',
+      tags: [],
+      testPath: '/scenarios/test/scenario.test.ts',
     },
-    {
-      artifactsDirectory: '',
-      copilotToken: '',
+    treatment: {
+      name: 'test-treatment-name',
     },
-  )
-
-  expect(runCommand).toHaveBeenCalledWith(
-    'copilot',
-    ['--prompt', 'test', '--model', 'gpt-5.6-sol', '--reasoning-effort', 'medium'],
-    {
-      user: NODE_USER,
-      env: {
-        COPILOT_GITHUB_TOKEN: '',
-      },
+    model: {
+      name: 'gpt-5.6-sol',
+      reasoningEffort: 'medium',
     },
-  )
-  expect(result.assistant.sessions).toEqual([[]])
+  }
+
+  const result = await run({
+    artifactsDirectory: '/artifacts',
+    copilotToken: 'test',
+    host,
+    sandbox,
+    trial,
+  })
+
+  // vi.spyOn(sandbox, 'copy').mockResolvedValue()
+  // const runCommand = vi.spyOn(sandbox, 'runCommand').mockImplementation(async command => {
+  //   if (command === 'sh') {
+  //     await sandbox.writeFile(
+  //       'test-results.json',
+  //       JSON.stringify({
+  //         numTotalTests: 0,
+  //         numPassedTests: 0,
+  //         numFailedTests: 0,
+  //         numPendingTests: 0,
+  //         numTodoTests: 0,
+  //         success: true,
+  //         testResults: [],
+  //       }),
+  //     )
+  //   }
+  //
+  //   return {
+  //     stdout: '',
+  //     stderr: '',
+  //     exitCode: 0,
+  //   }
+  // })
+  //
+  // const result = await runTrial(
+  //   sandbox,
+  //   {
+  //     id: 'test',
+  //     scenario: {
+  //       id: 'test',
+  //       directory: '/test',
+  //       prompt: 'test',
+  //       tags: [],
+  //       testPath: '/test/scenario.test.ts',
+  //     },
+  //     treatment: {
+  //       name: 'test',
+  //     },
+  //     model: {
+  //       name: 'gpt-5.6-sol',
+  //       reasoningEffort: 'medium',
+  //     },
+  //   },
+  //   {
+  //     artifactsDirectory: '',
+  //     copilotToken: '',
+  //   },
+  // )
+  //
+  // expect(runCommand).toHaveBeenCalledWith(
+  //   'copilot',
+  //   ['--prompt', 'test', '--model', 'gpt-5.6-sol', '--reasoning-effort', 'medium'],
+  //   {
+  //     user: NODE_USER,
+  //     env: {
+  //       COPILOT_GITHUB_TOKEN: '',
+  //     },
+  //   },
+  // )
+  // expect(result.assistant.sessions).toEqual([[]])
 })
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index 742e8321..b4f4a731 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -175,7 +175,7 @@ async function run({
   const WALKTHROUGH_DIR = 'walkthrough'
   const WALKTHROUGH_VIEWPORT_WIDTH = 1440
   const WALKTHROUGH_VIEWPORT_HEIGHT = 900
-  console.log('Capturing walkthrough...')
+  logger.debug('Capturing walkthrough...')
   await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
     user: 'root',
   })

From c8eb0545d49a67cde6e7865b3f64281292035513 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Wed, 2 Sep 2026 16:45:30 -0500
Subject: [PATCH 22/52] test: fill out trial tests

---
 packages/agent-eval/src/trial.test.ts | 891 ++++++++++++++++++++++----
 packages/agent-eval/src/trial.ts      |   2 +-
 2 files changed, 776 insertions(+), 117 deletions(-)

diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts
index 6addb427..ce2e0c5d 100644
--- a/packages/agent-eval/src/trial.test.ts
+++ b/packages/agent-eval/src/trial.test.ts
@@ -1,68 +1,106 @@
-import {test, vi} from 'vitest'
-import type {ResultMessage} from './copilot-cli'
-import {run, type Trial} from './trial'
-import {COPILOT_DIR, AGENTS_DIR} from './sandbox'
+import path from 'node:path'
+import {describe, expect, test, vi} from 'vitest'
 import {VirtualHost} from './host'
-import {randomUUID} from 'node:crypto'
+import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type CommandResult, type Sandbox} from './sandbox'
+import {run} from './trial'
+import type {Trial} from './trial'
+import type {ResultMessage} from './copilot-cli'
 
-test('run', async () => {
+async function setup(trial: Trial) {
+  const artifactsDirectory = '/artifacts'
   const host = VirtualHost.create({
     [AGENTS_DIR]: {},
     [COPILOT_DIR]: {},
-    '/scenarios/test': {
+    [artifactsDirectory]: {},
+    [trial.scenario.directory]: {
+      '.next': {
+        'build.txt': '',
+      },
+      node_modules: {
+        'dependency.txt': '',
+      },
       'scenario.config.ts': '',
       'scenario.test.ts': '',
     },
   })
-  await using sandbox = await host.createSandbox()
+  const sandbox = await host.createSandbox()
 
-  const runCommand = sandbox.runCommand
+  vi.spyOn(sandbox, 'copy')
 
-  vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, options) => {
-    if (command === 'sh') {
-      await sandbox.writeFile(
-        'test-results.json',
-        JSON.stringify({
-          numTotalTests: 0,
-          numPassedTests: 0,
-          numFailedTests: 0,
-          numPendingTests: 0,
-          numTodoTests: 0,
-          success: true,
-          testResults: [],
-        }),
-      )
+  return {
+    artifactsDirectory,
+    copilotToken: '',
+    host,
+    sandbox,
+  }
+}
+
+type RunCommandMockOptions = {
+  params: Parameters<Sandbox['runCommand']>
+  sandbox: Sandbox
+}
+type RunCommandMock = (options: RunCommandMockOptions) => Promise<CommandResult | null | undefined>
+
+async function applyCommandMocks(
+  mocks: Array<RunCommandMock>,
+  options: RunCommandMockOptions,
+): Promise<CommandResult | null | undefined> {
+  for (const mock of mocks) {
+    const result = await mock(options)
+    if (result) {
+      return result
     }
+  }
+}
 
-    if (command === 'copilot') {
-      const result: ResultMessage = {
-        type: 'result',
-        timestamp: '',
-        sessionId: '',
-        exitCode: 0,
-        usage: {
-          premiumRequests: 0,
-          totalApiDurationMs: 0,
-          sessionDurationMs: 0,
-          codeChanges: {
-            linesAdded: 0,
-            linesRemoved: 0,
-            filesModified: [],
-          },
+const writeCopilotResult: RunCommandMock = async ({params}) => {
+  const [command, args] = params
+  if (command === 'copilot' && Array.isArray(args) && args[0] === '--prompt') {
+    const result: ResultMessage = {
+      type: 'result',
+      timestamp: '',
+      sessionId: '',
+      exitCode: 0,
+      usage: {
+        premiumRequests: 0,
+        totalApiDurationMs: 0,
+        sessionDurationMs: 0,
+        codeChanges: {
+          linesAdded: 0,
+          linesRemoved: 0,
+          filesModified: [],
         },
-      }
-      return {
-        stdout: [JSON.stringify(result)].join('\n'),
-        stderr: '',
-        exitCode: 0,
-      }
+      },
+    }
+    return {
+      stdout: [JSON.stringify(result)].join('\n'),
+      stderr: '',
+      exitCode: 0,
     }
+  }
+}
 
-    return runCommand(command, args, options)
-  })
+const writeTestFile: RunCommandMock = async ({params, sandbox}) => {
+  const [command, args] = params
+  if (command === 'sh' && Array.isArray(args) && args[0] === '-c' && args[1].startsWith('npx vitest run')) {
+    await sandbox.writeFile(
+      'test-results.json',
+      JSON.stringify({
+        numTotalTests: 0,
+        numPassedTests: 0,
+        numFailedTests: 0,
+        numPendingTests: 0,
+        numTodoTests: 0,
+        success: true,
+        testResults: [],
+      }),
+    )
+  }
+}
 
-  const trial: Trial = {
-    id: randomUUID(),
+function createTrial(): Trial {
+  return {
+    id: 'test-id',
     scenario: {
       id: 'test-id',
       directory: '/scenarios/test',
@@ -78,73 +116,694 @@ test('run', async () => {
       reasoningEffort: 'medium',
     },
   }
+}
 
-  const result = await run({
-    artifactsDirectory: '/artifacts',
-    copilotToken: 'test',
-    host,
-    sandbox,
-    trial,
-  })
-
-  // vi.spyOn(sandbox, 'copy').mockResolvedValue()
-  // const runCommand = vi.spyOn(sandbox, 'runCommand').mockImplementation(async command => {
-  //   if (command === 'sh') {
-  //     await sandbox.writeFile(
-  //       'test-results.json',
-  //       JSON.stringify({
-  //         numTotalTests: 0,
-  //         numPassedTests: 0,
-  //         numFailedTests: 0,
-  //         numPendingTests: 0,
-  //         numTodoTests: 0,
-  //         success: true,
-  //         testResults: [],
-  //       }),
-  //     )
-  //   }
-  //
-  //   return {
-  //     stdout: '',
-  //     stderr: '',
-  //     exitCode: 0,
-  //   }
-  // })
-  //
-  // const result = await runTrial(
-  //   sandbox,
-  //   {
-  //     id: 'test',
-  //     scenario: {
-  //       id: 'test',
-  //       directory: '/test',
-  //       prompt: 'test',
-  //       tags: [],
-  //       testPath: '/test/scenario.test.ts',
-  //     },
-  //     treatment: {
-  //       name: 'test',
-  //     },
-  //     model: {
-  //       name: 'gpt-5.6-sol',
-  //       reasoningEffort: 'medium',
-  //     },
-  //   },
-  //   {
-  //     artifactsDirectory: '',
-  //     copilotToken: '',
-  //   },
-  // )
-  //
-  // expect(runCommand).toHaveBeenCalledWith(
-  //   'copilot',
-  //   ['--prompt', 'test', '--model', 'gpt-5.6-sol', '--reasoning-effort', 'medium'],
-  //   {
-  //     user: NODE_USER,
-  //     env: {
-  //       COPILOT_GITHUB_TOKEN: '',
-  //     },
-  //   },
-  // )
-  // expect(result.assistant.sessions).toEqual([[]])
+function mockRunCommand(sandbox: Sandbox, mocks: Array<RunCommandMock> = []) {
+  const runCommand = sandbox.runCommand
+
+  vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+    const result = await applyCommandMocks([...mocks, writeTestFile, writeCopilotResult], {
+      params: [command, args, commandOptions],
+      sandbox,
+    })
+    if (result) {
+      return result
+    }
+    return runCommand(command, args, commandOptions)
+  })
+}
+
+function writeWalkthroughArtifact(filepath: string, contents = ''): RunCommandMock {
+  return async ({params, sandbox}) => {
+    const [command, args] = params
+    if (
+      command === 'copilot' &&
+      Array.isArray(args) &&
+      args[0] === '--prompt' &&
+      args[1].startsWith('Record a visual walkthrough')
+    ) {
+      await sandbox.writeFile(filepath, contents)
+    }
+  }
+}
+
+describe('run', () => {
+  test('copies scenario files into the container workdir', async () => {
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.directory, CONTAINER_WORKDIR, {
+      exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next', 'dist'],
+    })
+  })
+
+  test('sets permissions for the scenario files', async () => {
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('chown', ['-R', NODE_USER, '.'], {
+      user: 'root',
+    })
+  })
+
+  test('obfuscates the package name', async () => {
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['pkg', 'set', `name=${trial.id}`], {
+      user: NODE_USER,
+    })
+  })
+
+  test('removes @primer/agent-eval from the workspace dependencies', async () => {
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
+      user: NODE_USER,
+    })
+  })
+
+  test('installs workspace dependencies', async () => {
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['install'], {
+      user: NODE_USER,
+    })
+  })
+
+  test('runs the generic setup', async () => {
+    const genericSetup = vi.fn(async () => {
+      //
+    })
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+      setup: genericSetup,
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(genericSetup).toHaveBeenCalledWith({
+      sandbox,
+    })
+  })
+
+  test('runs the treatment setup', async () => {
+    const treatmentSetup = vi.fn(async () => {
+      //
+    })
+    const trial: Trial = {
+      id: 'test-id',
+      scenario: {
+        id: 'test-id',
+        directory: '/scenarios/test',
+        prompt: 'test-prompt',
+        tags: [],
+        testPath: '/scenarios/test/scenario.test.ts',
+      },
+      treatment: {
+        name: 'test-treatment-name',
+        setup: treatmentSetup,
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    }
+    const {sandbox, ...runOptions} = await setup(trial)
+    const runCommand = sandbox.runCommand
+
+    vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => {
+      const result = await applyCommandMocks([writeTestFile, writeCopilotResult], {
+        params: [command, args, commandOptions],
+        sandbox,
+      })
+      if (result) {
+        return result
+      }
+      return runCommand(command, args, commandOptions)
+    })
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(treatmentSetup).toHaveBeenCalledWith({
+      sandbox,
+    })
+  })
+
+  test('runs the build script when one exists', async () => {
+    const trial = createTrial()
+    const {sandbox, host, ...runOptions} = await setup(trial)
+    await host.fs.writeFile(
+      path.join(trial.scenario.directory, 'package.json'),
+      JSON.stringify({
+        scripts: {
+          build: 'build',
+        },
+      }),
+    )
+    mockRunCommand(sandbox)
+
+    await run({
+      ...runOptions,
+      host,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['run', 'build', '--if-present'], {
+      user: NODE_USER,
+    })
+  })
+
+  test('continues when no build script exists', async () => {
+    const trial = createTrial()
+    const {sandbox, ...runOptions} = await setup(trial)
+    mockRunCommand(sandbox)
+
+    await expect(
+      run({
+        ...runOptions,
+        sandbox,
+        trial,
+      }),
+    ).resolves.toMatchObject({
+      trial,
+    })
+  })
+
+  test('runs Copilot with the trial arguments', async () => {
+    const trial = createTrial()
+    const {sandbox, ...runOptions} = await setup(trial)
+    const copilotToken = 'test-token'
+    mockRunCommand(sandbox)
+
+    await run({
+      ...runOptions,
+      copilotToken,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      'copilot',
+      [
+        '--prompt',
+        trial.scenario.prompt,
+        '--model',
+        trial.model.name,
+        '--reasoning-effort',
+        trial.model.reasoningEffort,
+        '--mode',
+        'autopilot',
+        '--allow-all',
+        '--output-format',
+        'json',
+      ],
+      {
+        user: NODE_USER,
+        env: {
+          COPILOT_GITHUB_TOKEN: copilotToken,
+        },
+      },
+    )
+  })
+
+  test('runs the scenario tests', async () => {
+    const trial = createTrial()
+    const {sandbox, ...runOptions} = await setup(trial)
+    vi.spyOn(sandbox, 'writeFile')
+    mockRunCommand(sandbox)
+
+    await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.testPath, 'scenario.test.ts')
+    expect(sandbox.writeFile).toHaveBeenCalledWith(
+      'vitest.agent-eval.config.ts',
+      expect.stringContaining('outputFile: "test-results.json"'),
+    )
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      'sh',
+      [
+        '-c',
+        'npx vitest run --config "$1" "$2" || true',
+        'vitest-run',
+        'vitest.agent-eval.config.ts',
+        'scenario.test.ts',
+      ],
+      {
+        user: NODE_USER,
+        env: {},
+      },
+    )
+  })
+
+  test('runs Copilot with the walkthrough arguments', async () => {
+    const trial = createTrial()
+    const {sandbox, ...runOptions} = await setup(trial)
+    const copilotToken = 'test-token'
+    mockRunCommand(sandbox)
+
+    await run({
+      ...runOptions,
+      copilotToken,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      'copilot',
+      [
+        '--prompt',
+        expect.stringContaining('Record a visual walkthrough'),
+        '--model',
+        'gpt-5.6-terra',
+        '--reasoning-effort',
+        'medium',
+        '--mode',
+        'autopilot',
+        '--allow-all',
+        '--output-format',
+        'json',
+      ],
+      {
+        user: NODE_USER,
+        env: {
+          COPILOT_GITHUB_TOKEN: copilotToken,
+        },
+        allowNonZeroExitCode: true,
+      },
+    )
+  })
+
+  describe('walkthrough artifacts', () => {
+    test('returns unavailable when no walkthrough artifact exists', async () => {
+      const trial = createTrial()
+      const {sandbox, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox)
+
+      const result = await run({
+        ...runOptions,
+        sandbox,
+        trial,
+      })
+
+      expect(result.walkthrough).toEqual({
+        type: 'Unavailable',
+      })
+    })
+
+    test('returns a video walkthrough', async () => {
+      const trial = createTrial()
+      const {sandbox, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/walkthrough.webm')])
+
+      const result = await run({
+        ...runOptions,
+        sandbox,
+        trial,
+      })
+
+      expect(result.walkthrough).toEqual({
+        type: 'Video',
+        filepath: '/artifacts/test-id/walkthrough/walkthrough.webm',
+      })
+    })
+
+    test('returns multiple walkthrough screenshots', async () => {
+      const trial = createTrial()
+      const {sandbox, ...runOptions} = await setup(trial)
+      const writeScreenshots: RunCommandMock = async options => {
+        await writeWalkthroughArtifact('walkthrough/screenshots/10.png')(options)
+        await writeWalkthroughArtifact('walkthrough/screenshots/2.jpg')(options)
+        await writeWalkthroughArtifact('walkthrough/screenshots/01.jpeg')(options)
+        await writeWalkthroughArtifact('walkthrough/screenshots/notes.txt')(options)
+      }
+      mockRunCommand(sandbox, [writeScreenshots])
+
+      const result = await run({
+        ...runOptions,
+        sandbox,
+        trial,
+      })
+
+      expect(result.walkthrough).toEqual({
+        type: 'Screenshots',
+        screenshots: [
+          '/artifacts/test-id/walkthrough/screenshots/01.jpeg',
+          '/artifacts/test-id/walkthrough/screenshots/2.jpg',
+          '/artifacts/test-id/walkthrough/screenshots/10.png',
+        ],
+      })
+    })
+
+    test('returns a single walkthrough screenshot', async () => {
+      const trial = createTrial()
+      const {sandbox, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/screenshot.png')])
+
+      const result = await run({
+        ...runOptions,
+        sandbox,
+        trial,
+      })
+
+      expect(result.walkthrough).toEqual({
+        type: 'Screenshot',
+        filepath: '/artifacts/test-id/walkthrough/screenshot.png',
+      })
+    })
+  })
+
+  describe('artifacts', () => {
+    test('creates the artifact directory when it does not exist', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox)
+
+      expect(host.existsSync('/artifacts/test-id')).toBe(false)
+
+      await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      expect(host.existsSync('/artifacts/test-id/workspace')).toBe(true)
+    })
+
+    test('empties the artifact directory when it already exists', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      await host.fs.mkdir('/artifacts/test-id', {
+        recursive: true,
+      })
+      await host.fs.writeFile('/artifacts/test-id/stale.txt', 'stale')
+      mockRunCommand(sandbox)
+
+      await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      expect(host.existsSync('/artifacts/test-id/stale.txt')).toBe(false)
+      expect(host.existsSync('/artifacts/test-id/workspace')).toBe(true)
+    })
+
+    test('downloads the workspace results', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox)
+
+      const result = await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      await expect(host.fs.readFile(result.artifacts.testResultsPath, 'utf8')).resolves.toBe(
+        JSON.stringify({
+          numTotalTests: 0,
+          numPassedTests: 0,
+          numFailedTests: 0,
+          numPendingTests: 0,
+          numTodoTests: 0,
+          success: true,
+          testResults: [],
+        }),
+      )
+    })
+
+    test('downloads the Copilot configuration', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      await host.fs.writeFile(path.join(COPILOT_DIR, 'config.json'), 'copilot config')
+      mockRunCommand(sandbox)
+
+      const result = await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      await expect(
+        host.fs.readFile(path.join(result.artifacts.copilotConfigDirectory, 'config.json'), 'utf8'),
+      ).resolves.toBe('copilot config')
+    })
+
+    test('downloads the agent configuration', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      await host.fs.writeFile(path.join(AGENTS_DIR, 'AGENTS.md'), 'agent config')
+      mockRunCommand(sandbox)
+
+      const result = await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      await expect(
+        host.fs.readFile(path.join(result.artifacts.skillsConfigDirectory, 'AGENTS.md'), 'utf8'),
+      ).resolves.toBe('agent config')
+    })
+
+    test('downloads the walkthrough', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/screenshot.png', 'screenshot')])
+
+      const result = await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      expect(result.walkthrough).toEqual({
+        type: 'Screenshot',
+        filepath: '/artifacts/test-id/walkthrough/screenshot.png',
+      })
+      await expect(host.fs.readFile('/artifacts/test-id/walkthrough/screenshot.png', 'utf8')).resolves.toBe(
+        'screenshot',
+      )
+      expect(host.existsSync('/artifacts/test-id/workspace/walkthrough')).toBe(false)
+    })
+  })
 })
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index b4f4a731..d48173b6 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -78,7 +78,7 @@ async function run({
   logger.info('Copying files from: %s...', trial.scenario.directory)
 
   await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
-    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'],
+    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next', 'dist'],
   })
   await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
     user: 'root',

From daa9b02259d51c6cd61ed27cc1eb7b2b88bd838b Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 12:02:36 -0500
Subject: [PATCH 23/52] feat: update output format and serialize, deserialize

---
 packages/agent-eval/src/cli.ts         |  38 +++-
 packages/agent-eval/src/experiment.ts  | 115 ++++++++++-
 packages/agent-eval/src/index.ts       |  12 +-
 packages/agent-eval/src/output.test.ts | 239 ----------------------
 packages/agent-eval/src/output.ts      | 268 -------------------------
 packages/agent-eval/src/trial.ts       |  63 +++++-
 6 files changed, 202 insertions(+), 533 deletions(-)
 delete mode 100644 packages/agent-eval/src/output.test.ts
 delete mode 100644 packages/agent-eval/src/output.ts

diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 0daf01b3..4319b2d0 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -1,11 +1,15 @@
 #!/usr/bin/env node
 
+import {existsSync} from 'node:fs'
+import fs from 'node:fs/promises'
+import path from 'node:path'
 import {parseArgs} from 'node:util'
 import {getEnvironmentConfig} from './environment'
 import {run as runBenchmark} from './benchmark'
-import {run as runExperiment} from './experiment'
+import {run as runExperiment, output as getExperimentOutput, serialize as serializeExperimentOutput} from './experiment'
 import {logger} from './logger'
 
+const start = Date.now()
 const {values} = parseArgs({
   options: {
     artifacts: {
@@ -116,12 +120,42 @@ if (values.benchmark) {
     env,
     id: values.benchmark,
   })
+
+  if (!existsSync(path.dirname(env.outputPath))) {
+    await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
+  }
+
+  logger.info('Writing benchmark output to: %s', env.outputPath)
+  // await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8')
+
+  logger.info(
+    'Done in %s',
+    new Intl.DurationFormat('en', {style: 'short'}).format({
+      milliseconds: Date.now() - start,
+    }),
+  )
 } else if (values.experiment) {
   logger.info('Running experiment: %s', values.experiment)
-  await runExperiment({
+
+  const result = await runExperiment({
     env,
     id: values.experiment,
   })
+  const output = getExperimentOutput(result)
+
+  if (!existsSync(path.dirname(env.outputPath))) {
+    await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
+  }
+
+  logger.info('Writing experiment output to: %s', env.outputPath)
+  await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8')
+
+  logger.info(
+    'Done in %s',
+    new Intl.DurationFormat('en', {style: 'short'}).format({
+      milliseconds: Date.now() - start,
+    }),
+  )
 } else {
   displayHelp()
 }
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index e5f1b14c..43c7c69b 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -1,14 +1,15 @@
+import {randomUUID} from 'node:crypto'
 import path from 'node:path'
 import * as z from 'zod/mini'
-import {getModelVariants, ModelVariantConfigSchema, type ModelVariant} from './model'
-import {getScenario, type Scenario} from './scenario'
-import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
-import {DefaultHost, type Host} from './host'
 import type {EnvironmentConfig} from './environment'
+import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
+import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
 import {create as createPlan, run as runPlan} from './plan'
-import type {Trial} from './trial'
-import {randomUUID} from 'node:crypto'
+import {getScenario, ScenarioSchema, type Scenario} from './scenario'
+import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
+import {TrialAgentSchema, TrialArtifactsSchema, WalkthroughSchema, type Trial, type TrialResult} from './trial'
+import {TestResultsSchema} from './vitest'
 
 const ExperimentConfigSchema = z.object({
   name: z.string(),
@@ -135,7 +136,17 @@ async function getExperiment({
   throw new Error(`Experiment "${id}" was not found in: ${experimentsDirectory}`)
 }
 
-async function run({env, host = DefaultHost, id}: {env: EnvironmentConfig; host?: Host; id: string}) {
+type ExperimentRunResult = Array<TrialResult>
+
+async function run({
+  env,
+  host = DefaultHost,
+  id,
+}: {
+  env: EnvironmentConfig
+  host?: Host
+  id: string
+}): Promise<ExperimentRunResult> {
   const experiment = await getExperiment({
     host,
     experimentsDirectory: env.experimentsDirectory,
@@ -174,5 +185,93 @@ async function run({env, host = DefaultHost, id}: {env: EnvironmentConfig; host?
   return results
 }
 
-export {defineConfig, listExperiments, getExperiment, ExperimentConfigSchema, run}
+const ExperimentOutputSchema = z.object({
+  scenarios: z.map(
+    z.string(),
+    z.pick(ScenarioSchema, {
+      id: true,
+      directory: true,
+      prompt: true,
+      description: true,
+      tags: true,
+      testPath: true,
+      browserTestPath: true,
+    }),
+  ),
+  treatments: z.map(
+    z.string(),
+    z.pick(TreatmentSchema, {
+      name: true,
+    }),
+  ),
+  trials: z.map(
+    z.string(),
+    z.object({
+      agent: TrialAgentSchema,
+      artifacts: TrialArtifactsSchema,
+      id: z.string(),
+      model: ModelVariantSchema,
+      scenarioId: z.string(),
+      testResults: TestResultsSchema,
+      treatmentId: z.string(),
+      walkthrough: WalkthroughSchema,
+    }),
+  ),
+})
+
+type ExperimentOutput = z.infer<typeof ExperimentOutputSchema>
+
+function output(trialResults: ExperimentRunResult): ExperimentOutput {
+  const result: ExperimentOutput = {
+    scenarios: new Map(),
+    treatments: new Map(),
+    trials: new Map(),
+  }
+
+  for (const trialResult of trialResults) {
+    const {artifacts, trial} = trialResult
+
+    if (!result.scenarios.has(trial.scenario.id)) {
+      result.scenarios.set(trial.scenario.id, trial.scenario)
+    }
+
+    if (!result.treatments.has(trial.treatment.name)) {
+      result.treatments.set(trial.treatment.name, trial.treatment)
+    }
+
+    result.trials.set(trial.id, {
+      agent: trialResult.agent,
+      artifacts,
+      id: trial.id,
+      model: trial.model,
+      scenarioId: trial.scenario.id,
+      testResults: trialResult.testResults,
+      treatmentId: trial.treatment.name,
+      walkthrough: trialResult.walkthrough,
+    })
+  }
+
+  return result
+}
+
+function serialize(output: ExperimentOutput): string {
+  return JSON.stringify({
+    scenarios: Object.fromEntries(output.scenarios),
+    treatments: Object.fromEntries(output.treatments),
+    trials: Object.fromEntries(output.trials),
+  })
+}
+
+function deserialize(input: unknown): ExperimentOutput {
+  const parsed = typeof input === 'string' ? JSON.parse(input) : input
+  const result = ExperimentOutputSchema.parse(parsed, {reportInput: true})
+
+  return {
+    scenarios: new Map(Object.entries(result.scenarios)),
+    treatments: new Map(Object.entries(result.treatments)),
+    trials: new Map(Object.entries(result.trials)),
+  }
+}
+
+export {ExperimentConfigSchema, defineConfig, deserialize, getExperiment, listExperiments, output, run, serialize}
 export type {ExperimentConfig, Experiment}
diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts
index 3b7ff1e4..6e67bebd 100644
--- a/packages/agent-eval/src/index.ts
+++ b/packages/agent-eval/src/index.ts
@@ -2,10 +2,14 @@ export {defineConfig as defineBenchmarkConfig, getBenchmark, listBenchmarks, Ben
 export type {BenchmarkConfig, Benchmark} from './benchmark'
 
 export {
+  ExperimentConfigSchema,
   defineConfig as defineExperimentConfig,
+  deserialize as deserializeExperimentOutput,
   getExperiment,
   listExperiments,
-  ExperimentConfigSchema,
+  output as getExperimentOutput,
+  run as runExperiment,
+  serialize as serializeExperimentOutput,
 } from './experiment'
 export type {ExperimentConfig, Experiment} from './experiment'
 
@@ -15,7 +19,5 @@ export type {ScenarioConfig, Scenario} from './scenario'
 export {TreatmentSchema, ControlTreatment} from './treatment'
 export type {Treatment} from './treatment'
 
-// export {run} from './run'
-// export {models} from './model'
-// export {createAgentEvalOutput, parseAgentEvalOutput} from './output'
-// export type {AgentEvalOutput, AgentEvalOutputResult} from './output'
+export {TrialSchema, TrialResultSchema, run as runTrial, compare as compareTrial} from './trial'
+export type {Trial, TrialResult} from './trial'
diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts
deleted file mode 100644
index da378252..00000000
--- a/packages/agent-eval/src/output.test.ts
+++ /dev/null
@@ -1,239 +0,0 @@
-import {describe, expect, test} from 'vitest'
-import {
-  createAgentEvalOutput,
-  parseAgentEvalOutput,
-  type AgentEvalOutput,
-  type ExperimentConfig,
-  type ResolvedScenario,
-  type TreatmentResult,
-} from './index'
-
-const output: AgentEvalOutput = {
-  id: 'run-id',
-  experiment: {
-    id: 'example',
-    name: 'Example',
-    description: 'An example experiment',
-    models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
-    scenarios: ['example'],
-  },
-  scenarios: [
-    {
-      id: 'example',
-      directory: '/scenarios/example',
-      config: {
-        description: 'Evaluate whether the agent builds an example',
-        prompt: 'Build an example',
-      },
-      testPath: '/scenarios/example/scenario.test.ts',
-      browserTestPath: '/scenarios/example/scenario.browser.test.ts',
-    },
-  ],
-  treatments: [
-    {
-      id: 'treatment-id',
-      config: {
-        name: 'Control',
-      },
-    },
-  ],
-  results: [
-    {
-      id: 'result-id',
-      treatmentId: 'treatment-id',
-      model: 'gpt-5.5',
-      reasoningEffort: 'high',
-      scenarioId: 'example',
-      artifacts: {
-        copilotConfigPath: '/artifacts/.copilot',
-        directory: '/artifacts',
-        skillsConfigPath: '/artifacts/.agents',
-        testResultsPath: '/artifacts/workspace/test-results.json',
-        workspacePath: '/artifacts/workspace',
-      },
-      assistant: {
-        logs: [],
-        turns: 1,
-        outputTokens: 100,
-        premiumRequests: 1,
-        totalApiDurationMs: 1000,
-        sessionDurationMs: 2000,
-        tools: {
-          view: 1,
-        },
-      },
-      testResults: {
-        numTotalTests: 1,
-        numPassedTests: 1,
-        numFailedTests: 0,
-        numPendingTests: 0,
-        numTodoTests: 0,
-        tests: [
-          {
-            title: 'renders an example',
-            fullName: 'example > renders an example',
-            status: 'passed',
-          },
-        ],
-      },
-      walkthrough: {
-        type: 'Unavailable',
-      },
-    },
-  ],
-}
-
-describe(createAgentEvalOutput, () => {
-  test('deduplicates experiment, scenario, and treatment metadata', () => {
-    const experiment: ExperimentConfig = {
-      name: 'Example',
-      description: 'An example experiment',
-      models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
-      scenarios: ['example'],
-      treatments: [],
-    }
-    const scenario: ResolvedScenario = output.scenarios[0]
-    const result: TreatmentResult = {
-      ...output.results[0],
-      treatment: {
-        config: {
-          name: 'Control',
-        },
-        scenario,
-        experiment,
-        id: 'treatment-id',
-        model: 'gpt-5.5',
-        reasoningEffort: 'high',
-      },
-    }
-    const duplicateTreatmentResult: TreatmentResult = {
-      ...result,
-      id: 'second-result-id',
-      treatment: {
-        ...result.treatment,
-        id: 'second-treatment-id',
-      },
-    }
-
-    expect(
-      createAgentEvalOutput({
-        id: 'run-id',
-        experimentId: 'example',
-        experiment,
-        scenarios: [scenario],
-        results: [result, duplicateTreatmentResult],
-      }),
-    ).toEqual({
-      ...output,
-      results: [
-        output.results[0],
-        {
-          ...output.results[0],
-          id: 'second-result-id',
-        },
-      ],
-    })
-  })
-})
-
-describe(parseAgentEvalOutput, () => {
-  test('parses agent eval output', () => {
-    expect(parseAgentEvalOutput(output)).toEqual(output)
-  })
-
-  test('parses serialized agent eval output', () => {
-    expect(parseAgentEvalOutput(JSON.stringify(output))).toEqual(output)
-  })
-
-  test('preserves unknown Copilot messages', () => {
-    const unknownMessage = {
-      type: 'unknown.event',
-      data: {
-        nested: {
-          value: 42,
-        },
-      },
-      metadata: ['one', 'two'],
-    }
-    const outputWithUnknownMessage = {
-      ...output,
-      results: [
-        {
-          ...output.results[0],
-          assistant: {
-            ...output.results[0].assistant,
-            logs: [unknownMessage],
-          },
-        },
-      ],
-    }
-
-    expect(parseAgentEvalOutput(outputWithUnknownMessage)).toEqual(outputWithUnknownMessage)
-  })
-
-  test('defaults the walkthrough for results without one', () => {
-    const {walkthrough, ...resultWithoutWalkthrough} = output.results[0]
-
-    expect(walkthrough).toEqual({type: 'Unavailable'})
-    expect(
-      parseAgentEvalOutput({
-        ...output,
-        results: [resultWithoutWalkthrough],
-      }),
-    ).toEqual(output)
-  })
-
-  test('throws for invalid agent eval output', () => {
-    expect(() =>
-      parseAgentEvalOutput({
-        ...output,
-        results: [
-          {
-            ...output.results[0],
-            testResults: {
-              ...output.results[0].testResults,
-              tests: [{title: 'invalid', fullName: 'invalid', status: 'unknown'}],
-            },
-          },
-        ],
-      }),
-    ).toThrow()
-  })
-
-  test('throws for an invalid reasoning effort', () => {
-    expect(() =>
-      parseAgentEvalOutput({
-        ...output,
-        results: [
-          {
-            ...output.results[0],
-            reasoningEffort: 'invalid',
-          },
-        ],
-      }),
-    ).toThrow()
-  })
-
-  test('throws for an unknown treatment reference', () => {
-    expect(() =>
-      parseAgentEvalOutput({
-        ...output,
-        results: [
-          {
-            ...output.results[0],
-            treatmentId: 'unknown',
-          },
-        ],
-      }),
-    ).toThrow('references unknown treatment')
-  })
-
-  test('throws for duplicate treatment IDs', () => {
-    expect(() =>
-      parseAgentEvalOutput({
-        ...output,
-        treatments: [...output.treatments, output.treatments[0]],
-      }),
-    ).toThrow('Treatment IDs must be unique')
-  })
-})
diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts
deleted file mode 100644
index 7844909e..00000000
--- a/packages/agent-eval/src/output.ts
+++ /dev/null
@@ -1,268 +0,0 @@
-import * as z from 'zod/mini'
-import {MessageSchema, type Message} from './copilot-cli'
-import type {ExperimentConfig, ExperimentScenarioConfig} from './experiment-config'
-import {models} from './model'
-import type {Model, ReasoningEffort} from './model'
-import type {ResolvedScenario} from './resolve-experiment-scenario'
-import type {TreatmentResult, Walkthrough} from './treatment'
-
-type AgentEvalOutputResult = {
-  id: string
-  treatmentId: string
-  model: Model
-  reasoningEffort?: ReasoningEffort
-  scenarioId: string
-  artifacts: {
-    copilotConfigPath: string
-    directory: string
-    skillsConfigPath: string
-    testResultsPath: string
-    workspacePath: string
-  }
-  assistant: {
-    logs: Array<Message>
-    turns: number
-    outputTokens: number
-    premiumRequests: number
-    totalApiDurationMs: number
-    sessionDurationMs: number
-    tools: Record<string, number>
-  }
-  testResults: {
-    numTotalTests: number
-    numPassedTests: number
-    numFailedTests: number
-    numPendingTests: number
-    numTodoTests: number
-    tests: Array<{
-      title: string
-      fullName: string
-      status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
-      description?: string
-    }>
-  }
-  walkthrough: Walkthrough
-}
-
-type AgentEvalOutput = {
-  id: string
-  experiment: {
-    id: string
-    name: string
-    description: string
-    models: Array<{
-      name: Model
-      reasoningEfforts: Array<ReasoningEffort>
-    }>
-    scenarios: Array<ExperimentScenarioConfig>
-  }
-  scenarios: Array<ResolvedScenario>
-  treatments: Array<{
-    id: string
-    config: {
-      name: string
-    }
-  }>
-  results: Array<AgentEvalOutputResult>
-}
-
-const modelNames = new Set<string>(models.map(model => model.name))
-const reasoningEfforts = new Set<string>(models.flatMap(model => model.reasoningEfforts))
-const ModelSchema = z.custom<Model, string>(
-  value => typeof value === 'string' && modelNames.has(value),
-  'Expected a supported model',
-)
-const ReasoningEffortSchema = z.custom<ReasoningEffort, string>(
-  value => typeof value === 'string' && reasoningEfforts.has(value),
-  'Expected a supported reasoning effort',
-)
-
-const ExperimentScenarioSchema = z.union([
-  z.string(),
-  z.object({
-    name: z.optional(z.string()),
-    path: z.string(),
-  }),
-])
-
-const TreatmentConfigSchema = z.object({
-  name: z.string(),
-})
-
-const ExperimentModelConfigSchema = z.object({
-  name: ModelSchema,
-  reasoningEfforts: z.array(ReasoningEffortSchema),
-})
-
-const AgentEvalOutputExperimentSchema = z.object({
-  id: z.string(),
-  name: z.string(),
-  description: z.string(),
-  models: z.array(ExperimentModelConfigSchema),
-  scenarios: z.array(ExperimentScenarioSchema),
-})
-
-const ResolvedScenarioSchema = z.object({
-  id: z.string(),
-  directory: z.string(),
-  config: z.object({
-    description: z.optional(z.string()),
-    prompt: z.string(),
-  }),
-  testPath: z.string(),
-  browserTestPath: z.optional(z.string()),
-})
-
-const unavailableWalkthrough = {type: 'Unavailable'} as const
-
-const AgentEvalOutputResultSchema = z.object({
-  id: z.string(),
-  treatmentId: z.string(),
-  model: ModelSchema,
-  reasoningEffort: z.optional(ReasoningEffortSchema),
-  scenarioId: z.string(),
-  artifacts: z.object({
-    copilotConfigPath: z.string(),
-    directory: z.string(),
-    skillsConfigPath: z.string(),
-    testResultsPath: z.string(),
-    workspacePath: z.string(),
-  }),
-  assistant: z.object({
-    logs: z.array(MessageSchema),
-    turns: z.number(),
-    outputTokens: z.number(),
-    premiumRequests: z.number(),
-    totalApiDurationMs: z.number(),
-    sessionDurationMs: z.number(),
-    tools: z.record(z.string(), z.number()),
-  }),
-  testResults: z.object({
-    numTotalTests: z.number(),
-    numPassedTests: z.number(),
-    numFailedTests: z.number(),
-    numPendingTests: z.number(),
-    numTodoTests: z.number(),
-    tests: z.array(
-      z.object({
-        title: z.string(),
-        fullName: z.string(),
-        status: z.enum(['passed', 'failed', 'skipped', 'pending', 'todo', 'disabled']),
-        description: z.optional(z.string()),
-      }),
-    ),
-  }),
-  // Runs created before walkthroughs were supported do not include this field
-  walkthrough: z.pipe(
-    z.optional(
-      z.discriminatedUnion('type', [
-        z.object({type: z.literal('Unavailable')}),
-        z.object({type: z.literal('Screenshot'), filepath: z.string()}),
-        z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}),
-        z.object({type: z.literal('Video'), filepath: z.string()}),
-      ]),
-    ),
-    z.transform(value => value ?? unavailableWalkthrough),
-  ),
-})
-
-const AgentEvalOutputSchema = z.object({
-  id: z.string(),
-  experiment: AgentEvalOutputExperimentSchema,
-  scenarios: z.array(ResolvedScenarioSchema),
-  treatments: z.array(
-    z.object({
-      id: z.string(),
-      config: TreatmentConfigSchema,
-    }),
-  ),
-  results: z.array(AgentEvalOutputResultSchema),
-})
-
-type CreateAgentEvalOutputOptions = {
-  id: string
-  experimentId: string
-  experiment: ExperimentConfig
-  scenarios: Array<ResolvedScenario>
-  results: Array<TreatmentResult>
-}
-
-function createAgentEvalOutput({
-  id,
-  experimentId,
-  experiment,
-  scenarios,
-  results,
-}: CreateAgentEvalOutputOptions): AgentEvalOutput {
-  const treatmentsByName = new Map<
-    string,
-    {
-      id: string
-      config: {
-        name: string
-      }
-    }
-  >()
-
-  for (const result of results) {
-    if (!treatmentsByName.has(result.treatment.config.name)) {
-      treatmentsByName.set(result.treatment.config.name, {
-        id: result.treatment.id,
-        config: {
-          name: result.treatment.config.name,
-        },
-      })
-    }
-  }
-
-  return {
-    id,
-    experiment: {
-      id: experimentId,
-      name: experiment.name,
-      description: experiment.description,
-      models: experiment.models,
-      scenarios: experiment.scenarios,
-    },
-    scenarios,
-    treatments: [...treatmentsByName.values()],
-    results: results.map(result => {
-      const treatment = treatmentsByName.get(result.treatment.config.name)
-      if (!treatment) {
-        throw new Error(`Treatment "${result.treatment.config.name}" was not normalized`)
-      }
-
-      return {
-        id: result.id,
-        treatmentId: treatment.id,
-        model: result.treatment.model,
-        reasoningEffort: result.treatment.reasoningEffort,
-        scenarioId: result.treatment.scenario.id,
-        artifacts: result.artifacts,
-        assistant: result.assistant,
-        testResults: result.testResults,
-        walkthrough: result.walkthrough,
-      }
-    }),
-  }
-}
-
-function parseAgentEvalOutput(value: unknown): AgentEvalOutput {
-  const input = typeof value === 'string' ? JSON.parse(value) : value
-  const output = AgentEvalOutputSchema.parse(input, {reportInput: true})
-  const treatmentIds = new Set(output.treatments.map(treatment => treatment.id))
-  if (treatmentIds.size !== output.treatments.length) {
-    throw new Error('Treatment IDs must be unique')
-  }
-
-  for (const result of output.results) {
-    if (!treatmentIds.has(result.treatmentId)) {
-      throw new Error(`Result "${result.id}" references unknown treatment "${result.treatmentId}"`)
-    }
-  }
-
-  return output
-}
-
-export {createAgentEvalOutput, parseAgentEvalOutput}
-export type {AgentEvalOutput, AgentEvalOutputResult}
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index d48173b6..1ac8627a 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -42,18 +42,22 @@ const AgentSessionSchema = z.object({
 
 type AgentSession = z.infer<typeof AgentSessionSchema>
 
+const TrialArtifactsSchema = z.object({
+  directory: z.string(),
+  copilotConfigDirectory: z.string(),
+  skillsConfigDirectory: z.string(),
+  testResultsPath: z.string(),
+  workspaceDirectory: z.string(),
+})
+
+const TrialAgentSchema = z.object({
+  sessions: z.array(AgentSessionSchema),
+})
+
 const TrialResultSchema = z.object({
-  artifacts: z.object({
-    directory: z.string(),
-    copilotConfigDirectory: z.string(),
-    skillsConfigDirectory: z.string(),
-    testResultsPath: z.string(),
-    workspaceDirectory: z.string(),
-  }),
+  artifacts: TrialArtifactsSchema,
   trial: TrialSchema,
-  agent: z.object({
-    sessions: z.array(AgentSessionSchema),
-  }),
+  agent: TrialAgentSchema,
   testResults: TestResultsSchema,
   walkthrough: WalkthroughSchema,
 })
@@ -373,5 +377,42 @@ export default defineConfig({
 })`
 }
 
-export {TrialSchema, TrialResultSchema, run}
+type CompareTrialResult = Pick<TrialResult, 'testResults' | 'agent'>
+
+/**
+ * Compare to trial results to determine which treatment performed better. We
+ * compare trials based on:
+ *
+ * - Test success rate (higher is better)
+ * - Output tokens (lower is better)
+ * - Total API duration (lower is better)
+ * - Number of turns (lower is better)
+ * - Number of premium requests (lower is better)
+ */
+function compare(a: CompareTrialResult, b: CompareTrialResult): number {
+  const successRateA = a.testResults.numTotalTests > 0 ? a.testResults.numPassedTests / a.testResults.numTotalTests : 0
+  const successRateB = b.testResults.numTotalTests > 0 ? b.testResults.numPassedTests / b.testResults.numTotalTests : 0
+
+  const outputTokensA = a.agent.sessions.reduce((sum, session) => sum + session.outputTokens, 0)
+  const outputTokensB = b.agent.sessions.reduce((sum, session) => sum + session.outputTokens, 0)
+
+  const totalApiDurationA = a.agent.sessions.reduce((sum, session) => sum + session.totalApiDurationMs, 0)
+  const totalApiDurationB = b.agent.sessions.reduce((sum, session) => sum + session.totalApiDurationMs, 0)
+
+  const turnsA = a.agent.sessions.reduce((sum, session) => sum + session.turns, 0)
+  const turnsB = b.agent.sessions.reduce((sum, session) => sum + session.turns, 0)
+
+  const premiumRequestsA = a.agent.sessions.reduce((sum, session) => sum + session.premiumRequests, 0)
+  const premiumRequestsB = b.agent.sessions.reduce((sum, session) => sum + session.premiumRequests, 0)
+
+  return (
+    successRateB - successRateA ||
+    outputTokensA - outputTokensB ||
+    totalApiDurationA - totalApiDurationB ||
+    turnsA - turnsB ||
+    premiumRequestsA - premiumRequestsB
+  )
+}
+
+export {TrialSchema, TrialResultSchema, TrialArtifactsSchema, TrialAgentSchema, WalkthroughSchema, run, compare}
 export type {Trial, TrialResult}

From dc21648912c0cf70affea832763a1f339ce18b3c Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 15:31:00 -0500
Subject: [PATCH 24/52] feat: restore table emit behavior

---
 packages/agent-eval/src/cli.ts         |  40 ++--
 packages/agent-eval/src/report.test.ts | 125 ++++++++++++
 packages/agent-eval/src/report.ts      | 262 +++++++++++++++++++++++++
 3 files changed, 411 insertions(+), 16 deletions(-)
 create mode 100644 packages/agent-eval/src/report.test.ts
 create mode 100644 packages/agent-eval/src/report.ts

diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 4319b2d0..601feb10 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -6,8 +6,15 @@ import path from 'node:path'
 import {parseArgs} from 'node:util'
 import {getEnvironmentConfig} from './environment'
 import {run as runBenchmark} from './benchmark'
-import {run as runExperiment, output as getExperimentOutput, serialize as serializeExperimentOutput} from './experiment'
+import {
+  getExperiment,
+  run as runExperiment,
+  output as getExperimentOutput,
+  serialize as serializeExperimentOutput,
+} from './experiment'
 import {logger} from './logger'
+import {formatExperimentResults} from './report'
+import {compare as compareTrial} from './trial'
 
 const start = Date.now()
 const {values} = parseArgs({
@@ -127,35 +134,36 @@ if (values.benchmark) {
 
   logger.info('Writing benchmark output to: %s', env.outputPath)
   // await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8')
-
-  logger.info(
-    'Done in %s',
-    new Intl.DurationFormat('en', {style: 'short'}).format({
-      milliseconds: Date.now() - start,
-    }),
-  )
 } else if (values.experiment) {
   logger.info('Running experiment: %s', values.experiment)
 
+  const experiment = await getExperiment({
+    experimentsDirectory: env.experimentsDirectory,
+    scenariosDirectory: env.scenariosDirectory,
+    id: values.experiment,
+  })
   const result = await runExperiment({
     env,
     id: values.experiment,
   })
-  const output = getExperimentOutput(result)
+  const sorted = result.toSorted(compareTrial)
+
+  logger.info('Writing experiment output to: %s', env.outputPath)
+
+  const output = getExperimentOutput(sorted)
 
   if (!existsSync(path.dirname(env.outputPath))) {
     await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
   }
 
-  logger.info('Writing experiment output to: %s', env.outputPath)
   await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8')
 
-  logger.info(
-    'Done in %s',
-    new Intl.DurationFormat('en', {style: 'short'}).format({
-      milliseconds: Date.now() - start,
-    }),
-  )
+  const resultSummaries = formatExperimentResults(experiment.name, sorted)
+  console.log(resultSummaries)
+
+  if (GITHUB_STEP_SUMMARY) {
+    await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`)
+  }
 } else {
   displayHelp()
 }
diff --git a/packages/agent-eval/src/report.test.ts b/packages/agent-eval/src/report.test.ts
new file mode 100644
index 00000000..50adfb04
--- /dev/null
+++ b/packages/agent-eval/src/report.test.ts
@@ -0,0 +1,125 @@
+import {expect, test} from 'vitest'
+import type {TrialResult} from './trial'
+import {formatExperimentResults} from './report'
+
+function createResult({
+  treatment,
+  scenario,
+  model,
+  numPassedTests,
+  numTotalTests,
+  sessions,
+}: {
+  treatment: string
+  scenario: string
+  model: TrialResult['trial']['model']
+  numPassedTests: number
+  numTotalTests: number
+  sessions: TrialResult['agent']['sessions']
+}): TrialResult {
+  return {
+    artifacts: {
+      directory: '/artifacts/trial',
+      copilotConfigDirectory: '/artifacts/trial/.copilot',
+      skillsConfigDirectory: '/artifacts/trial/.agents',
+      testResultsPath: '/artifacts/trial/workspace/test-results.json',
+      workspaceDirectory: '/artifacts/trial/workspace',
+    },
+    trial: {
+      id: `${treatment}-${scenario}-${model.name}-${model.reasoningEffort}`,
+      scenario: {
+        id: scenario,
+        directory: `/scenarios/${scenario}`,
+        prompt: 'Complete the task',
+        tags: [],
+        testPath: `/scenarios/${scenario}/scenario.test.ts`,
+      },
+      treatment: {
+        name: treatment,
+      },
+      model,
+    },
+    agent: {
+      sessions,
+    },
+    testResults: {
+      numTotalTests,
+      numPassedTests,
+      numFailedTests: numTotalTests - numPassedTests,
+      numPendingTests: 0,
+      numTodoTests: 0,
+      success: numPassedTests === numTotalTests,
+      testResults: [],
+    },
+    walkthrough: {
+      type: 'Unavailable',
+    },
+  }
+}
+
+test('formats trial results as an aggregated hierarchy', () => {
+  const results = [
+    createResult({
+      treatment: 'Control',
+      scenario: 'button',
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+      numPassedTests: 1,
+      numTotalTests: 2,
+      sessions: [
+        {
+          turns: 2,
+          outputTokens: 1_000,
+          premiumRequests: 1,
+          totalApiDurationMs: 30_000,
+          sessionDurationMs: 45_000,
+          tools: {},
+          messages: [],
+        },
+        {
+          turns: 1,
+          outputTokens: 250,
+          premiumRequests: 2,
+          totalApiDurationMs: 5_000,
+          sessionDurationMs: 10_000,
+          tools: {},
+          messages: [],
+        },
+      ],
+    }),
+    createResult({
+      treatment: 'Recommended',
+      scenario: 'button',
+      model: {
+        name: 'claude-sonnet-5',
+        reasoningEffort: 'high',
+      },
+      numPassedTests: 2,
+      numTotalTests: 2,
+      sessions: [
+        {
+          turns: 1,
+          outputTokens: 900,
+          premiumRequests: 1,
+          totalApiDurationMs: 65_000,
+          sessionDurationMs: 90_000,
+          tools: {},
+          messages: [],
+        },
+      ],
+    }),
+  ]
+
+  expect(formatExperimentResults('Design system', results)).toMatchInlineSnapshot(`
+    "Experiment     Treatment    Scenario       Model                Reasoning Effort  Success Rate  Tests  Runs  Output Tokens  Premium Requests  Session Time  API Time
+    -------------  -----------  -------------  -------------------  ----------------  ------------  -----  ----  -------------  ----------------  ------------  --------
+    Design system  Recommended  All scenarios  All models                             100.0%        2/2    1     900            1                 1m 30.0s      1m 5.0s 
+                                  button       All models                             100.0%        2/2    1     900            1                 1m 30.0s      1m 5.0s 
+                                                   claude-sonnet-5  high              100.0%        2/2    1     900            1                 1m 30.0s      1m 5.0s 
+    Design system  Control      All scenarios  All models                             50.0%         1/2    1     1,250          3                 55.0s         35.0s   
+                                  button       All models                             50.0%         1/2    1     1,250          3                 55.0s         35.0s   
+                                                   gpt-5.6-sol      medium            50.0%         1/2    1     1,250          3                 55.0s         35.0s   "
+  `)
+})
diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts
new file mode 100644
index 00000000..560d7bd0
--- /dev/null
+++ b/packages/agent-eval/src/report.ts
@@ -0,0 +1,262 @@
+import type {Model, ReasoningEffort} from './model'
+import type {TrialResult} from './trial'
+
+type ResultSummary = {
+  experiment: string
+  treatment?: string
+  scenario?: string
+  model?: Model
+  reasoningEffort?: ReasoningEffort<Model>
+  runs: number
+  numPassedTests: number
+  numTotalTests: number
+  outputTokens: number
+  premiumRequests: number
+  sessionDurationMs: number
+  totalApiDurationMs: number
+}
+
+type ResultSummaryValues = {
+  treatment?: string
+  scenario?: string
+  model?: Model
+  reasoningEffort?: ReasoningEffort<Model>
+}
+
+type ResultHierarchy = Array<{
+  experiment: string
+  treatments: Array<{
+    summary: ResultSummary
+    scenarios: Array<{
+      summary: ResultSummary
+      models: Array<ResultSummary>
+    }>
+  }>
+}>
+
+type TableRow = Record<string, string | number>
+
+function createResultSummary(experiment: string, values: ResultSummaryValues = {}): ResultSummary {
+  return {
+    experiment,
+    treatment: values.treatment,
+    scenario: values.scenario,
+    model: values.model,
+    reasoningEffort: values.reasoningEffort,
+    runs: 0,
+    numPassedTests: 0,
+    numTotalTests: 0,
+    outputTokens: 0,
+    premiumRequests: 0,
+    sessionDurationMs: 0,
+    totalApiDurationMs: 0,
+  }
+}
+
+function addResultToSummary(summary: ResultSummary, result: TrialResult): void {
+  summary.runs += 1
+  summary.numPassedTests += result.testResults.numPassedTests
+  summary.numTotalTests += result.testResults.numTotalTests
+
+  for (const session of result.agent.sessions) {
+    summary.outputTokens += session.outputTokens
+    summary.premiumRequests += session.premiumRequests
+    summary.sessionDurationMs += session.sessionDurationMs
+    summary.totalApiDurationMs += session.totalApiDurationMs
+  }
+}
+
+function getSuccessRate(summary: ResultSummary): number {
+  if (summary.numTotalTests === 0) {
+    return 0
+  }
+
+  return summary.numPassedTests / summary.numTotalTests
+}
+
+function compareSummaries(a: ResultSummary, b: ResultSummary): number {
+  return (
+    getSuccessRate(b) - getSuccessRate(a) ||
+    a.outputTokens - b.outputTokens ||
+    a.sessionDurationMs - b.sessionDurationMs ||
+    a.premiumRequests - b.premiumRequests ||
+    a.experiment.localeCompare(b.experiment) ||
+    (a.treatment ?? '').localeCompare(b.treatment ?? '') ||
+    (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
+    (a.model ?? '').localeCompare(b.model ?? '') ||
+    (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
+  )
+}
+
+function getSummaryKey(experiment: string, values: ResultSummaryValues = {}): string {
+  return [
+    experiment,
+    values.treatment ?? '',
+    values.scenario ?? '',
+    values.model ?? '',
+    values.reasoningEffort ?? '',
+  ].join('\0')
+}
+
+function getResultSummaries(experiment: string, results: Array<TrialResult>): ResultHierarchy {
+  const treatmentSummaries = new Map<string, ResultSummary>()
+  const scenarioSummaries = new Map<string, ResultSummary>()
+  const modelSummaries = new Map<string, ResultSummary>()
+
+  for (const result of results) {
+    const treatmentValues = {
+      treatment: result.trial.treatment.name,
+    }
+    const treatmentKey = getSummaryKey(experiment, treatmentValues)
+    const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(experiment, treatmentValues)
+    addResultToSummary(treatmentSummary, result)
+    treatmentSummaries.set(treatmentKey, treatmentSummary)
+
+    const scenarioValues = {
+      treatment: result.trial.treatment.name,
+      scenario: result.trial.scenario.id,
+    }
+    const scenarioKey = getSummaryKey(experiment, scenarioValues)
+    const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(experiment, scenarioValues)
+    addResultToSummary(scenarioSummary, result)
+    scenarioSummaries.set(scenarioKey, scenarioSummary)
+
+    const modelValues = {
+      treatment: result.trial.treatment.name,
+      scenario: result.trial.scenario.id,
+      model: result.trial.model.name,
+      reasoningEffort: result.trial.model.reasoningEffort,
+    }
+    const modelKey = getSummaryKey(experiment, modelValues)
+    const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(experiment, modelValues)
+    addResultToSummary(modelSummary, result)
+    modelSummaries.set(modelKey, modelSummary)
+  }
+
+  return [
+    {
+      experiment,
+      treatments: [...treatmentSummaries.values()].toSorted(compareSummaries).map(summary => {
+        return {
+          summary,
+          scenarios: [...scenarioSummaries.values()]
+            .filter(scenarioSummary => {
+              return scenarioSummary.treatment === summary.treatment
+            })
+            .toSorted(compareSummaries)
+            .map(scenarioSummary => {
+              return {
+                summary: scenarioSummary,
+                models: [...modelSummaries.values()]
+                  .filter(modelSummary => {
+                    return (
+                      modelSummary.treatment === summary.treatment && modelSummary.scenario === scenarioSummary.scenario
+                    )
+                  })
+                  .toSorted(compareSummaries),
+              }
+            }),
+        }
+      }),
+    },
+  ]
+}
+
+function formatPercent(value: number): string {
+  return `${(value * 100).toFixed(1)}%`
+}
+
+function formatDuration(ms: number): string {
+  const seconds = ms / 1000
+
+  if (seconds < 60) {
+    return `${seconds.toFixed(1)}s`
+  }
+
+  const minutes = Math.floor(seconds / 60)
+  const remainingSeconds = seconds - minutes * 60
+  return `${minutes}m ${remainingSeconds.toFixed(1)}s`
+}
+
+function formatNumber(value: number): string {
+  return new Intl.NumberFormat('en-US').format(value)
+}
+
+function formatTable(rows: Array<TableRow>, columns: Array<string>): string {
+  const columnWidths = columns.map(column => {
+    let width = column.length
+
+    for (const row of rows) {
+      width = Math.max(width, String(row[column] ?? '').length)
+    }
+
+    return width
+  })
+
+  const formatRow = (row: TableRow): string => {
+    return columns
+      .map((column, index) => {
+        return String(row[column] ?? '').padEnd(columnWidths[index])
+      })
+      .join('  ')
+  }
+
+  return [
+    formatRow(Object.fromEntries(columns.map(column => [column, column]))),
+    columnWidths.map(width => '-'.repeat(width)).join('  '),
+    ...rows.map(formatRow),
+  ].join('\n')
+}
+
+function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow {
+  return {
+    Experiment: level === 'treatment' ? summary.experiment : '',
+    Treatment: level === 'treatment' ? (summary.treatment ?? '') : '',
+    Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario ?? ''}` : '',
+    Model: level === 'model' ? `    ${summary.model ?? ''}` : 'All models',
+    'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '',
+    'Success Rate': formatPercent(getSuccessRate(summary)),
+    Tests: `${summary.numPassedTests}/${summary.numTotalTests}`,
+    Runs: summary.runs,
+    'Output Tokens': formatNumber(summary.outputTokens),
+    'Premium Requests': formatNumber(summary.premiumRequests),
+    'Session Time': formatDuration(summary.sessionDurationMs),
+    'API Time': formatDuration(summary.totalApiDurationMs),
+  }
+}
+
+function formatExperimentResults(experiment: string, results: Array<TrialResult>): string {
+  const columns = [
+    'Experiment',
+    'Treatment',
+    'Scenario',
+    'Model',
+    'Reasoning Effort',
+    'Success Rate',
+    'Tests',
+    'Runs',
+    'Output Tokens',
+    'Premium Requests',
+    'Session Time',
+    'API Time',
+  ]
+  const rows: Array<TableRow> = []
+
+  for (const {treatments} of getResultSummaries(experiment, results)) {
+    for (const {summary, scenarios} of treatments) {
+      rows.push(formatSummaryRow(summary, 'treatment'))
+
+      for (const {summary: scenarioSummary, models} of scenarios) {
+        rows.push(formatSummaryRow(scenarioSummary, 'scenario'))
+
+        for (const model of models) {
+          rows.push(formatSummaryRow(model, 'model'))
+        }
+      }
+    }
+  }
+
+  return formatTable(rows, columns)
+}
+
+export {formatExperimentResults}

From dc35b3bc2bb3dea33923dc68f19e8233ed996eba Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 15:39:54 -0500
Subject: [PATCH 25/52] test: add tests and changeset

---
 .changeset/calm-benchmarks-compare.md         |   2 +-
 packages/agent-eval/src/benchmark.test.ts     | 385 +++++++++++-------
 packages/agent-eval/src/benchmark.ts          | 349 +++++++++++-----
 packages/agent-eval/src/cli.test.ts           |  47 +++
 packages/agent-eval/src/cli.ts                |  28 +-
 packages/agent-eval/src/environment.test.ts   |  56 +++
 .../agent-eval/src/experiment-output.test.ts  |  20 +
 packages/agent-eval/src/experiment.test.ts    |  95 +++--
 packages/agent-eval/src/experiment.ts         |  88 ++--
 packages/agent-eval/src/index.test.ts         |  25 ++
 packages/agent-eval/src/index.ts              |  24 +-
 packages/agent-eval/src/logger.test.ts        |   7 +
 packages/agent-eval/src/mcp-config.test.ts    |  55 +++
 packages/agent-eval/src/plan.test.ts          | 168 ++++++++
 .../agent-eval/src/sandbox/constants.test.ts  |  26 ++
 packages/agent-eval/src/sandbox/index.test.ts |  10 +
 packages/agent-eval/src/sandbox/path.test.ts  |  13 +
 .../agent-eval/src/sandbox/system.test.ts     | 227 +++++++++++
 packages/agent-eval/src/scenario.ts           |  71 +++-
 packages/agent-eval/src/treatment.test.ts     |  33 ++
 20 files changed, 1397 insertions(+), 332 deletions(-)
 create mode 100644 packages/agent-eval/src/cli.test.ts
 create mode 100644 packages/agent-eval/src/environment.test.ts
 create mode 100644 packages/agent-eval/src/experiment-output.test.ts
 create mode 100644 packages/agent-eval/src/index.test.ts
 create mode 100644 packages/agent-eval/src/logger.test.ts
 create mode 100644 packages/agent-eval/src/mcp-config.test.ts
 create mode 100644 packages/agent-eval/src/plan.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/constants.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/index.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/path.test.ts
 create mode 100644 packages/agent-eval/src/sandbox/system.test.ts
 create mode 100644 packages/agent-eval/src/treatment.test.ts

diff --git a/.changeset/calm-benchmarks-compare.md b/.changeset/calm-benchmarks-compare.md
index 3607a2d2..4c2c3de5 100644
--- a/.changeset/calm-benchmarks-compare.md
+++ b/.changeset/calm-benchmarks-compare.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add a benchmark configuration API for grouping evaluation scenarios by capability.
+Add support for benchmarks
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index dfdfb50e..5abcd58f 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -1,178 +1,279 @@
-import {test, expect} from 'vitest'
-import {defineConfig, listBenchmarks, getBenchmark, run} from './benchmark'
+import {expect, test} from 'vitest'
+import {
+  defineConfig,
+  deserialize,
+  getBenchmark,
+  listBenchmarks,
+  output,
+  run,
+  serialize,
+  type BenchmarkTrialResult,
+} from './benchmark'
 import {VirtualHost} from './host'
 import {defineConfig as defineScenarioConfig} from './scenario'
 
-test('listBenchmarks', async () => {
-  const config = JSON.stringify(
-    defineConfig({
-      name: 'test',
-      description: 'test',
-      models: [],
-      capabilities: [],
-    }),
-  )
-  const host = VirtualHost.create({
-    '/benchmarks': {
-      '01-benchmark.ts': `export const benchmark = ${config}`,
-      '02-default-export.ts': `export default ${config}`,
-      // '03-invalid-config.ts': `export const benchmark = {}`,
-      '03-no-config.ts': ``,
+const config = defineConfig({
+  name: 'Test benchmark',
+  description: 'Tests a benchmark',
+  models: ['gpt-5.6-sol'],
+  capabilities: [
+    {
+      name: 'Test capability',
+      scenarios: ['001-scenario'],
+    },
+  ],
+})
+
+const scenario = defineScenarioConfig({
+  prompt: 'Complete the task',
+})
+
+function createHost(files: Record<string, string>) {
+  return VirtualHost.create({
+    '/benchmarks': files,
+    '/scenarios': {
+      '001-scenario': {
+        'scenario.config.ts': `export default ${JSON.stringify(scenario)}`,
+        'scenario.test.ts': '',
+        'package.json': JSON.stringify({}),
+      },
     },
   })
-  const benchmarks = await listBenchmarks(host, '/benchmarks')
+}
 
-  expect(benchmarks).toHaveLength(2)
-  expect(benchmarks).toContainEqual({
-    id: '01-benchmark',
-    name: 'test',
-    description: 'test',
-    models: [],
-    capabilities: [],
+test('listBenchmarks loads configs and resolves models and capability scenarios', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = createHost({
+    'named.ts': `export const benchmark = ${serializedConfig}`,
+    'default.js': `export default ${serializedConfig}`,
+    'types.d.ts': `export const benchmark = ${serializedConfig}`,
+    'index.ts': `export const benchmark = ${serializedConfig}`,
+    'unsupported.json': serializedConfig,
+    'missing.ts': 'export const value = true',
+    'invalid.ts': 'export const benchmark = {}',
   })
-  expect(benchmarks).toContainEqual({
-    id: '02-default-export',
-    name: 'test',
-    description: 'test',
-    models: [],
-    capabilities: [],
+
+  const benchmarks = await listBenchmarks({
+    host,
+    benchmarksDirectory: '/benchmarks',
+    scenariosDirectory: '/scenarios',
   })
 
-  expect(benchmarks).not.toContainEqual(
-    expect.objectContaining({
-      id: '03-no-config',
-    }),
+  expect(benchmarks).toHaveLength(2)
+  expect(benchmarks).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining({
+        id: 'named',
+        filepath: '/benchmarks/named.ts',
+        name: config.name,
+        models: [
+          {
+            name: 'gpt-5.6-sol',
+            reasoningEffort: 'medium',
+          },
+        ],
+        capabilities: [
+          {
+            name: 'Test capability',
+            scenarios: [
+              expect.objectContaining({
+                id: '001-scenario',
+                prompt: 'Complete the task',
+              }),
+            ],
+          },
+        ],
+      }),
+      expect.objectContaining({
+        id: 'default',
+        filepath: '/benchmarks/default.js',
+      }),
+    ]),
   )
 })
 
-test('listBenchmarks throw error on invalid config', async () => {
-  const config = JSON.stringify(
-    defineConfig({
-      name: 'test',
-      description: 'test',
-      models: [],
-      capabilities: [],
-    }),
-  )
-  const host = VirtualHost.create({
-    '/benchmarks': {
-      '01-benchmark.ts': `export const benchmark = ${config}`,
-      '03-invalid-config.ts': `export const benchmark = {}`,
-    },
+test('listBenchmarks prefers the named benchmark export', async () => {
+  const namedConfig = {
+    ...config,
+    name: 'Named benchmark',
+  }
+  const defaultConfig = {
+    ...config,
+    name: 'Default benchmark',
+  }
+  const host = createHost({
+    'benchmark.ts': [
+      `export const benchmark = ${JSON.stringify(namedConfig)}`,
+      `export default ${JSON.stringify(defaultConfig)}`,
+    ].join('\n'),
   })
 
-  await expect(listBenchmarks(host, '/benchmarks')).rejects.toThrowErrorMatchingInlineSnapshot(`
-    [Error: Benchmark file must export a valid benchmark config: /benchmarks/03-invalid-config.ts
-    ✖ Invalid input
-      → at name
-    ✖ Invalid input
-      → at description
-    ✖ Invalid input
-      → at models
-    ✖ Invalid input
-      → at capabilities]
-  `)
+  await expect(
+    listBenchmarks({
+      host,
+      benchmarksDirectory: '/benchmarks',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).resolves.toEqual([
+    expect.objectContaining({
+      id: 'benchmark',
+      name: 'Named benchmark',
+    }),
+  ])
 })
 
-test('getBenchmark', async () => {
-  const config = JSON.stringify(
-    defineConfig({
-      name: 'test',
-      description: 'test',
-      models: [],
-      capabilities: [],
+test('listBenchmarks validates the benchmarks directory', async () => {
+  const host = VirtualHost.create()
+
+  await expect(
+    listBenchmarks({
+      host,
+      benchmarksDirectory: '/benchmarks',
+      scenariosDirectory: '/scenarios',
     }),
-  )
-  const host = VirtualHost.create({
-    '/benchmarks': {
-      '01-benchmark.ts': `export const benchmark = ${config}`,
-      '02-benchmark.ts': `export const benchmark = ${config}`,
-    },
+  ).rejects.toThrowError('Benchmarks directory does not exist: /benchmarks')
+
+  const fileHost = VirtualHost.create({
+    '/benchmarks': '',
   })
-  const benchmark = await getBenchmark(host, '/benchmarks', '01-benchmark')
 
-  expect(benchmark).toEqual({
-    id: '01-benchmark',
-    name: 'test',
-    description: 'test',
-    models: [],
-    capabilities: [],
+  await expect(
+    listBenchmarks({
+      host: fileHost,
+      benchmarksDirectory: '/benchmarks',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).rejects.toThrowError('Benchmarks path is not a directory: /benchmarks')
+})
+
+test('getBenchmark returns the benchmark matching the id', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = createHost({
+    'first.ts': `export const benchmark = ${serializedConfig}`,
+    'second.ts': `export const benchmark = ${serializedConfig}`,
   })
 
-  await expect(getBenchmark(host, '/benchmarks', 'non-existent')).rejects.toThrowErrorMatchingInlineSnapshot(
-    `[Error: Benchmark "non-existent" was not found in: /benchmarks]`,
+  await expect(
+    getBenchmark({
+      host,
+      benchmarksDirectory: '/benchmarks',
+      scenariosDirectory: '/scenarios',
+      id: 'second',
+    }),
+  ).resolves.toEqual(
+    expect.objectContaining({
+      id: 'second',
+      filepath: '/benchmarks/second.ts',
+      name: config.name,
+    }),
   )
 })
 
-test('getBenchmark throw error on invalid config', async () => {
-  const config = JSON.stringify(
-    defineConfig({
-      name: 'test',
-      description: 'test',
-      models: [],
-      capabilities: [],
+test('getBenchmark throws when the benchmark is not found', async () => {
+  const host = createHost({})
+
+  await expect(
+    getBenchmark({
+      host,
+      benchmarksDirectory: '/benchmarks',
+      scenariosDirectory: '/scenarios',
+      id: 'missing',
     }),
-  )
-  const host = VirtualHost.create({
-    '/benchmarks': {
-      '01-benchmark.ts': `export const benchmark = ${config}`,
-      '02-invalid-config.ts': `export const benchmark = {}`,
-    },
+  ).rejects.toThrowError('Benchmark "missing" was not found in: /benchmarks')
+})
+
+test('run returns an empty result when the benchmark has no trials', async () => {
+  const emptyConfig = defineConfig({
+    name: 'Empty benchmark',
+    description: 'Has no trials',
+    models: [],
+    capabilities: [],
+  })
+  const host = createHost({
+    'empty.ts': `export const benchmark = ${JSON.stringify(emptyConfig)}`,
   })
 
-  await expect(getBenchmark(host, '/benchmarks', '02-invalid-config')).rejects.toThrowErrorMatchingInlineSnapshot(`
-    [Error: Benchmark file must export a valid benchmark config: /benchmarks/02-invalid-config.ts
-    ✖ Invalid input
-      → at name
-    ✖ Invalid input
-      → at description
-    ✖ Invalid input
-      → at models
-    ✖ Invalid input
-      → at capabilities]
-  `)
+  await expect(
+    run({
+      env: {
+        artifactsDirectory: '/artifacts',
+        benchmarksDirectory: '/benchmarks',
+        concurrency: 1,
+        copilotToken: 'token',
+        dockerImage: 'node:26-slim',
+        experimentsDirectory: '/experiments',
+        outputPath: '/output.json',
+        scenariosDirectory: '/scenarios',
+      },
+      host,
+      id: 'empty',
+    }),
+  ).resolves.toEqual([])
 })
 
-test('run', async () => {
-  const scenario = JSON.stringify(
-    defineScenarioConfig({
-      prompt: 'test',
-    }),
-  )
-  const host = VirtualHost.create({
-    '/artifacts': {},
-    '/benchmarks': {},
-    '/scenarios': {
-      '001-scenario': {
-        'scenario.config.ts': `export default ${scenario}`,
-        'scenario.test.ts': '',
-        'package.json': JSON.stringify({}),
+test('output serializes and deserializes benchmark capability metadata', () => {
+  const capability = {
+    name: 'Test capability',
+    scenarios: [
+      {
+        id: '001-scenario',
+        directory: '/scenarios/001-scenario',
+        prompt: 'Complete the task',
+        tags: [],
+        testPath: '/scenarios/001-scenario/scenario.test.ts',
       },
-      '002-scenario': {
-        'scenario.config.ts': `export default ${scenario}`,
-        'scenario.test.ts': '',
-        'package.json': JSON.stringify({}),
+    ],
+  }
+  const trialResult: BenchmarkTrialResult = {
+    capability,
+    artifacts: {
+      directory: '/artifacts/trial',
+      copilotConfigDirectory: '/artifacts/trial/.copilot',
+      skillsConfigDirectory: '/artifacts/trial/.agents',
+      testResultsPath: '/artifacts/trial/workspace/test-results.json',
+      workspaceDirectory: '/artifacts/trial/workspace',
+    },
+    trial: {
+      id: 'trial',
+      scenario: capability.scenarios[0],
+      treatment: {
+        name: 'Benchmark',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
       },
     },
-  })
-  await run(
-    {
-      artifactsDirectory: '/artifacts',
-      benchmarksDirectory: '/benchmarks',
-      host,
-      scenariosDirectory: '/scenarios',
+    agent: {
+      sessions: [],
     },
-    {
-      id: 'test',
-      name: 'test',
-      description: 'test',
-      models: ['gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.6-sol'],
-      capabilities: [
-        {
-          name: 'test',
-          scenarios: ['001-scenario', '002-scenario'],
-        },
-      ],
+    testResults: {
+      numTotalTests: 1,
+      numPassedTests: 1,
+      numFailedTests: 0,
+      numPendingTests: 0,
+      numTodoTests: 0,
+      success: true,
+      testResults: [],
+    },
+    walkthrough: {
+      type: 'Unavailable',
     },
+  }
+
+  const benchmarkOutput = output('test-benchmark', [trialResult])
+
+  expect(benchmarkOutput.benchmarkId).toBe('test-benchmark')
+  expect(benchmarkOutput.capabilities.get('Test capability')).toEqual({
+    name: 'Test capability',
+    scenarioIds: ['001-scenario'],
+  })
+  expect(benchmarkOutput.trials.get('trial')).toEqual(
+    expect.objectContaining({
+      capabilityId: 'Test capability',
+      scenarioId: '001-scenario',
+      treatmentId: 'Benchmark',
+    }),
   )
+  expect(deserialize(serialize(benchmarkOutput))).toEqual(benchmarkOutput)
 })
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 47598e96..73914fe3 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -1,12 +1,15 @@
 import {randomUUID} from 'node:crypto'
 import path from 'node:path'
 import * as z from 'zod/mini'
-import {getModelVariants, ModelVariantConfigSchema, type Model, type ModelVariant, type ReasoningEffort} from './model'
-import {getScenario, type Scenario} from './scenario'
-import {DefaultHost, type Host} from './host'
-import {ControlTreatment, TreatmentSetupSchema} from './treatment'
-import type {Treatment} from './treatment'
 import type {EnvironmentConfig} from './environment'
+import {DefaultHost, type Host} from './host'
+import {logger} from './logger'
+import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
+import {create as createPlan, run as runPlan} from './plan'
+import {getScenario, ScenarioSchema, type Scenario} from './scenario'
+import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type TreatmentSetup} from './treatment'
+import {TrialAgentSchema, TrialArtifactsSchema, WalkthroughSchema, type Trial, type TrialResult} from './trial'
+import {TestResultsSchema} from './vitest'
 
 const CapabilityConfigSchema = z.object({
   name: z.string(),
@@ -16,37 +19,58 @@ const CapabilityConfigSchema = z.object({
 const BenchmarkConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
-  models: z.array(ModelVariantConfigSchema),
+  models: ModelVariantConfigSchema,
   setup: z.optional(TreatmentSetupSchema),
   capabilities: z.array(CapabilityConfigSchema),
 })
 
 type BenchmarkConfig = z.infer<typeof BenchmarkConfigSchema>
 
-type CapabilityConfig = z.infer<typeof CapabilityConfigSchema>
-
 function defineConfig(config: BenchmarkConfig): BenchmarkConfig {
   return config
 }
 
-type BenchmarkModule = {
-  benchmark?: BenchmarkConfig
-  default?: BenchmarkConfig
+type Capability = {
+  name: string
+  scenarios: Array<Scenario>
 }
 
-type Benchmark = BenchmarkConfig & {
+type Benchmark = {
   id: string
+  filepath: string
+  name: BenchmarkConfig['name']
+  description: BenchmarkConfig['description']
+  models: Array<ModelVariant>
+  setup?: TreatmentSetup
+  capabilities: Array<Capability>
+}
+
+type BenchmarkModule = {
+  benchmark?: unknown
+  default?: unknown
 }
 
 const BENCHMARK_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
 
-async function listBenchmarks(host: Host, directory: string): Promise<Array<Benchmark>> {
-  const stats = await host.fs.stat(directory)
+async function listBenchmarks({
+  host = DefaultHost,
+  benchmarksDirectory,
+  scenariosDirectory,
+}: {
+  host?: Host
+  benchmarksDirectory: string
+  scenariosDirectory: string
+}): Promise<Array<Benchmark>> {
+  if (!host.existsSync(benchmarksDirectory)) {
+    throw new Error(`Benchmarks directory does not exist: ${benchmarksDirectory}`)
+  }
+
+  const stats = await host.fs.stat(benchmarksDirectory)
   if (!stats.isDirectory()) {
-    throw new Error('Expected benchmarks to be a directory')
+    throw new Error(`Benchmarks path is not a directory: ${benchmarksDirectory}`)
   }
 
-  const filenames = await host.fs.readdir(directory)
+  const filenames = await host.fs.readdir(benchmarksDirectory)
   const benchmarks: Array<Benchmark> = []
 
   for (const filename of filenames) {
@@ -54,7 +78,7 @@ async function listBenchmarks(host: Host, directory: string): Promise<Array<Benc
       continue
     }
 
-    const filepath = path.join(directory, filename)
+    const filepath = path.join(benchmarksDirectory, filename)
     const mod: BenchmarkModule = await host.loadModule(filepath)
     const data = mod.benchmark ?? mod.default
     if (!data) {
@@ -63,28 +87,64 @@ async function listBenchmarks(host: Host, directory: string): Promise<Array<Benc
 
     const parseResult = BenchmarkConfigSchema.safeParse(data)
     if (!parseResult.success) {
-      throw new Error(
-        `Benchmark file must export a valid benchmark config: ${filepath}\n${z.prettifyError(parseResult.error)}`,
+      logger.warn(
+        `Failed to parse benchmark config for file: ${filepath}. Error: ${z.prettifyError(parseResult.error)}`,
       )
+      continue
     }
 
+    const {data: config} = parseResult
+    const capabilities = await Promise.all(
+      config.capabilities.map(async capability => {
+        const scenarios = await Promise.all(
+          capability.scenarios.map(scenario => {
+            return getScenario(host, scenariosDirectory, scenario)
+          }),
+        )
+
+        return {
+          name: capability.name,
+          scenarios,
+        }
+      }),
+    )
+
     benchmarks.push({
       id: getBenchmarkId(filename),
-      ...parseResult.data,
+      filepath,
+      name: config.name,
+      description: config.description,
+      models: getModelVariants(config.models),
+      setup: config.setup,
+      capabilities,
     })
   }
 
   return benchmarks
 }
 
-async function getBenchmark(host: Host, directory: string, id: string): Promise<Benchmark> {
-  const benchmarks = await listBenchmarks(host, directory)
-  const benchmark = benchmarks.find(benchmark => benchmark.id === id)
-  if (!benchmark) {
-    throw new Error(`Benchmark "${id}" was not found in: ${directory}`)
+async function getBenchmark({
+  host = DefaultHost,
+  benchmarksDirectory,
+  scenariosDirectory,
+  id,
+}: {
+  host?: Host
+  benchmarksDirectory: string
+  scenariosDirectory: string
+  id: string
+}): Promise<Benchmark> {
+  const benchmarks = await listBenchmarks({
+    host,
+    benchmarksDirectory,
+    scenariosDirectory,
+  })
+  const benchmark = benchmarks.find(candidate => candidate.id === id)
+  if (benchmark) {
+    return benchmark
   }
 
-  return benchmark
+  throw new Error(`Benchmark "${id}" was not found in: ${benchmarksDirectory}`)
 }
 
 function getBenchmarkId(filename: string): string {
@@ -92,39 +152,14 @@ function getBenchmarkId(filename: string): string {
 }
 
 function isBenchmarkFile(filename: string): boolean {
-  if (path.extname(filename) === '.d.ts') {
-    return false
-  }
-
-  if (!BENCHMARK_FILE_EXTENSIONS.has(path.extname(filename))) {
-    return false
-  }
-
-  if (path.basename(filename) === 'index') {
-    return false
-  }
+  return !filename.endsWith('.d.ts') && filename !== 'index.ts' && BENCHMARK_FILE_EXTENSIONS.has(path.extname(filename))
+}
 
-  return true
+type BenchmarkTrialResult = TrialResult & {
+  capability: Capability
 }
 
-// type RunContext = {
-//   artifactsDirectory: string
-//   benchmarksDirectory: string
-//   host: Host
-//   scenariosDirectory: string
-// }
-//
-// type Trial = {
-//   id: string
-//   capability: CapabilityConfig
-//   scenario: Scenario
-//   treatment: Treatment
-//   model: ModelVariant
-// }
-//
-// type TrialRun = {}
-//
-type TrialResult = {}
+type BenchmarkRunResult = Array<BenchmarkTrialResult>
 
 async function run({
   env,
@@ -134,44 +169,168 @@ async function run({
   env: EnvironmentConfig
   host?: Host
   id: string
-}): Promise<TrialResult> {
-  const benchmark = await getBenchmark(host, env.benchmarksDirectory, id)
-  console.log(benchmark)
-
-  throw new Error('unimplemented')
-  // const trials: Array<Trial> = []
-  //
-  // for (const variant of getModelVariants(benchmark.models)) {
-  //   for (const capability of benchmark.capabilities) {
-  //     for (const scenarioId of capability.scenarios) {
-  //       const scenario = await getScenario(context.host, context.scenariosDirectory, scenarioId)
-  //
-  //       trials.push({
-  //         id: randomUUID(),
-  //         capability,
-  //         scenario,
-  //         treatment: ControlTreatment,
-  //         model: variant,
-  //       })
-  //
-  //       trials.push({
-  //         id: randomUUID(),
-  //         capability,
-  //         scenario,
-  //         treatment: {
-  //           name: 'Benchmark',
-  //           setup: benchmark.setup,
-  //         },
-  //         model: variant,
-  //       })
-  //     }
-  //   }
-  // }
-  //
-  // console.log(trials)
-  //
-  // throw new Error('unimplemented')
+}): Promise<BenchmarkRunResult> {
+  const benchmark = await getBenchmark({
+    host,
+    benchmarksDirectory: env.benchmarksDirectory,
+    scenariosDirectory: env.scenariosDirectory,
+    id,
+  })
+  const benchmarkTreatment = {
+    name: 'Benchmark',
+    setup: benchmark.setup,
+  }
+  const trialCapabilities = new Map<string, Capability>()
+  const trials: Array<Trial> = benchmark.models.flatMap(model => {
+    return benchmark.capabilities.flatMap(capability => {
+      return capability.scenarios.flatMap(scenario => {
+        return [ControlTreatment, benchmarkTreatment].map(treatment => {
+          const trial = {
+            id: randomUUID(),
+            scenario,
+            treatment,
+            model,
+          }
+          trialCapabilities.set(trial.id, capability)
+          return trial
+        })
+      })
+    })
+  })
+  const plan = await createPlan(trials)
+  const results = await runPlan({
+    env,
+    host,
+    plan,
+  })
+
+  return results.map(result => {
+    const capability = trialCapabilities.get(result.trial.id)
+    if (!capability) {
+      throw new Error(`Capability was not found for trial: ${result.trial.id}`)
+    }
+
+    return {
+      ...result,
+      capability,
+    }
+  })
+}
+
+const CapabilityOutputSchema = z.object({
+  name: z.string(),
+  scenarioIds: z.array(z.string()),
+})
+
+const BenchmarkTrialOutputSchema = z.object({
+  agent: TrialAgentSchema,
+  artifacts: TrialArtifactsSchema,
+  capabilityId: z.string(),
+  id: z.string(),
+  model: ModelVariantSchema,
+  scenarioId: z.string(),
+  testResults: TestResultsSchema,
+  treatmentId: z.string(),
+  walkthrough: WalkthroughSchema,
+})
+
+const SerializedBenchmarkOutputSchema = z.object({
+  benchmarkId: z.string(),
+  capabilities: z.record(z.string(), CapabilityOutputSchema),
+  scenarios: z.record(
+    z.string(),
+    z.pick(ScenarioSchema, {
+      id: true,
+      directory: true,
+      prompt: true,
+      description: true,
+      tags: true,
+      testPath: true,
+      browserTestPath: true,
+    }),
+  ),
+  treatments: z.record(
+    z.string(),
+    z.pick(TreatmentSchema, {
+      name: true,
+    }),
+  ),
+  trials: z.record(z.string(), BenchmarkTrialOutputSchema),
+})
+
+type BenchmarkOutput = {
+  benchmarkId: string
+  capabilities: Map<string, z.infer<typeof CapabilityOutputSchema>>
+  scenarios: Map<string, Scenario>
+  treatments: Map<string, {name: string}>
+  trials: Map<string, z.infer<typeof BenchmarkTrialOutputSchema>>
+}
+
+function output(benchmarkId: string, trialResults: BenchmarkRunResult): BenchmarkOutput {
+  const result: BenchmarkOutput = {
+    benchmarkId,
+    capabilities: new Map(),
+    scenarios: new Map(),
+    treatments: new Map(),
+    trials: new Map(),
+  }
+
+  for (const trialResult of trialResults) {
+    const {artifacts, capability, trial} = trialResult
+
+    if (!result.capabilities.has(capability.name)) {
+      result.capabilities.set(capability.name, {
+        name: capability.name,
+        scenarioIds: capability.scenarios.map(scenario => scenario.id),
+      })
+    }
+
+    if (!result.scenarios.has(trial.scenario.id)) {
+      result.scenarios.set(trial.scenario.id, trial.scenario)
+    }
+
+    if (!result.treatments.has(trial.treatment.name)) {
+      result.treatments.set(trial.treatment.name, trial.treatment)
+    }
+
+    result.trials.set(trial.id, {
+      agent: trialResult.agent,
+      artifacts,
+      capabilityId: capability.name,
+      id: trial.id,
+      model: trial.model,
+      scenarioId: trial.scenario.id,
+      testResults: trialResult.testResults,
+      treatmentId: trial.treatment.name,
+      walkthrough: trialResult.walkthrough,
+    })
+  }
+
+  return result
+}
+
+function serialize(benchmarkOutput: BenchmarkOutput): string {
+  return JSON.stringify({
+    benchmarkId: benchmarkOutput.benchmarkId,
+    capabilities: Object.fromEntries(benchmarkOutput.capabilities),
+    scenarios: Object.fromEntries(benchmarkOutput.scenarios),
+    treatments: Object.fromEntries(benchmarkOutput.treatments),
+    trials: Object.fromEntries(benchmarkOutput.trials),
+  })
+}
+
+function deserialize(input: unknown): BenchmarkOutput {
+  const parsed = typeof input === 'string' ? JSON.parse(input) : input
+  const result = SerializedBenchmarkOutputSchema.parse(parsed, {reportInput: true})
+
+  return {
+    benchmarkId: result.benchmarkId,
+    capabilities: new Map(Object.entries(result.capabilities)),
+    scenarios: new Map(Object.entries(result.scenarios)),
+    treatments: new Map(Object.entries(result.treatments)),
+    trials: new Map(Object.entries(result.trials)),
+  }
 }
 
-export {defineConfig, listBenchmarks, getBenchmark, run, BenchmarkConfigSchema}
-export type {BenchmarkConfig, Benchmark}
+export {BenchmarkConfigSchema, defineConfig, deserialize, getBenchmark, listBenchmarks, output, run, serialize}
+export type {BenchmarkConfig, Benchmark, BenchmarkOutput, BenchmarkRunResult, BenchmarkTrialResult, Capability}
diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts
new file mode 100644
index 00000000..a55e8dcb
--- /dev/null
+++ b/packages/agent-eval/src/cli.test.ts
@@ -0,0 +1,47 @@
+import {afterEach, describe, expect, test, vi} from 'vitest'
+
+const originalArgv = process.argv
+const originalToken = process.env.COPILOT_GITHUB_TOKEN
+
+afterEach(() => {
+  process.argv = originalArgv
+  if (originalToken === undefined) {
+    delete process.env.COPILOT_GITHUB_TOKEN
+  } else {
+    process.env.COPILOT_GITHUB_TOKEN = originalToken
+  }
+  vi.restoreAllMocks()
+  vi.resetModules()
+})
+
+describe('cli', () => {
+  test('displays help when requested', async () => {
+    process.argv = ['node', 'agent-eval', '--help']
+    const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+    vi.spyOn(process, 'exit').mockImplementation(() => {
+      throw new Error('process.exit')
+    })
+
+    await expect(import('./cli')).rejects.toThrow('process.exit')
+    expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]'))
+  })
+
+  test('requires a Copilot token before running', async () => {
+    process.argv = ['node', 'agent-eval']
+    delete process.env.COPILOT_GITHUB_TOKEN
+
+    await expect(import('./cli')).rejects.toThrow(
+      'COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval',
+    )
+  })
+
+  test('displays help when no benchmark or experiment is selected', async () => {
+    process.argv = ['node', 'agent-eval']
+    process.env.COPILOT_GITHUB_TOKEN = 'token'
+    const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+
+    await import('./cli')
+
+    expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]'))
+  })
+})
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 601feb10..036347ea 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -5,7 +5,12 @@ import fs from 'node:fs/promises'
 import path from 'node:path'
 import {parseArgs} from 'node:util'
 import {getEnvironmentConfig} from './environment'
-import {run as runBenchmark} from './benchmark'
+import {
+  getBenchmark,
+  run as runBenchmark,
+  output as getBenchmarkOutput,
+  serialize as serializeBenchmarkOutput,
+} from './benchmark'
 import {
   getExperiment,
   run as runExperiment,
@@ -16,7 +21,6 @@ import {logger} from './logger'
 import {formatExperimentResults} from './report'
 import {compare as compareTrial} from './trial'
 
-const start = Date.now()
 const {values} = parseArgs({
   options: {
     artifacts: {
@@ -123,17 +127,31 @@ logger.debug('Environment configuration: %o', env)
 
 if (values.benchmark) {
   logger.info('Running benchmark: %s', values.benchmark)
-  await runBenchmark({
+
+  const benchmark = await getBenchmark({
+    benchmarksDirectory: env.benchmarksDirectory,
+    scenariosDirectory: env.scenariosDirectory,
+    id: values.benchmark,
+  })
+  const result = await runBenchmark({
     env,
     id: values.benchmark,
   })
+  const sorted = result.toSorted(compareTrial)
 
   if (!existsSync(path.dirname(env.outputPath))) {
     await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
   }
 
   logger.info('Writing benchmark output to: %s', env.outputPath)
-  // await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8')
+  await fs.writeFile(env.outputPath, serializeBenchmarkOutput(getBenchmarkOutput(benchmark.id, sorted)), 'utf-8')
+
+  const resultSummaries = formatExperimentResults(benchmark.name, sorted)
+  console.log(resultSummaries)
+
+  if (GITHUB_STEP_SUMMARY) {
+    await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`)
+  }
 } else if (values.experiment) {
   logger.info('Running experiment: %s', values.experiment)
 
@@ -150,7 +168,7 @@ if (values.benchmark) {
 
   logger.info('Writing experiment output to: %s', env.outputPath)
 
-  const output = getExperimentOutput(sorted)
+  const output = getExperimentOutput(experiment.id, sorted)
 
   if (!existsSync(path.dirname(env.outputPath))) {
     await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts
new file mode 100644
index 00000000..32f4c189
--- /dev/null
+++ b/packages/agent-eval/src/environment.test.ts
@@ -0,0 +1,56 @@
+import path from 'node:path'
+import {describe, expect, test} from 'vitest'
+import {DEFAULT_DOCKER_IMAGE} from './sandbox'
+import {getEnvironmentConfig} from './environment'
+
+describe('getEnvironmentConfig', () => {
+  test('resolves default paths and values', () => {
+    expect(
+      getEnvironmentConfig({
+        copilotToken: 'token',
+      }),
+    ).toEqual({
+      artifactsDirectory: path.resolve('artifacts'),
+      benchmarksDirectory: path.resolve('benchmarks'),
+      concurrency: 1,
+      copilotToken: 'token',
+      dockerImage: DEFAULT_DOCKER_IMAGE,
+      experimentsDirectory: path.resolve('experiments'),
+      outputPath: path.resolve('output.json'),
+      scenariosDirectory: path.resolve('scenarios'),
+    })
+  })
+
+  test('uses valid custom values', () => {
+    expect(
+      getEnvironmentConfig({
+        artifactsDirectory: './custom-artifacts',
+        benchmarksDirectory: './custom-benchmarks',
+        concurrency: '4',
+        copilotToken: 'token',
+        dockerImage: 'node:custom',
+        experimentsDirectory: './custom-experiments',
+        outputPath: './results/output.json',
+        scenariosDirectory: './custom-scenarios',
+      }),
+    ).toEqual({
+      artifactsDirectory: path.resolve('custom-artifacts'),
+      benchmarksDirectory: path.resolve('custom-benchmarks'),
+      concurrency: 4,
+      copilotToken: 'token',
+      dockerImage: 'node:custom',
+      experimentsDirectory: path.resolve('custom-experiments'),
+      outputPath: path.resolve('results/output.json'),
+      scenariosDirectory: path.resolve('custom-scenarios'),
+    })
+  })
+
+  test.each(['0', '-1', 'invalid', '1.5'])('falls back to one for invalid concurrency %s', concurrency => {
+    expect(
+      getEnvironmentConfig({
+        concurrency,
+        copilotToken: 'token',
+      }).concurrency,
+    ).toBe(1)
+  })
+})
diff --git a/packages/agent-eval/src/experiment-output.test.ts b/packages/agent-eval/src/experiment-output.test.ts
new file mode 100644
index 00000000..8ea03b3d
--- /dev/null
+++ b/packages/agent-eval/src/experiment-output.test.ts
@@ -0,0 +1,20 @@
+import {expect, test} from 'vitest'
+import {deserialize, output, serialize} from './experiment'
+
+test('serializes and deserializes experiment identity and result maps', () => {
+  const experimentOutput = output('baseline', [])
+  const serialized = serialize(experimentOutput)
+
+  expect(JSON.parse(serialized)).toEqual({
+    experimentId: 'baseline',
+    scenarios: {},
+    treatments: {},
+    trials: {},
+  })
+  expect(deserialize(serialized)).toEqual({
+    experimentId: 'baseline',
+    scenarios: new Map(),
+    treatments: new Map(),
+    trials: new Map(),
+  })
+})
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
index 1a9c40da..533d873d 100644
--- a/packages/agent-eval/src/experiment.test.ts
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -6,11 +6,7 @@ const config = defineConfig({
   name: 'Test experiment',
   description: 'Tests an experiment',
   models: ['gpt-5.6-sol'],
-  scenarios: [
-    {
-      prompt: 'Complete the task',
-    },
-  ],
+  scenarios: [],
   treatments: [
     {
       name: 'Test treatment',
@@ -18,6 +14,19 @@ const config = defineConfig({
   ],
 })
 
+function resolveConfig(input: typeof config) {
+  return {
+    ...input,
+    models: [
+      {
+        name: 'gpt-5.6-sol' as const,
+        reasoningEffort: 'medium' as const,
+      },
+    ],
+    setup: undefined,
+  }
+}
+
 test('listExperiments loads named and default exports from supported files', async () => {
   const serializedConfig = JSON.stringify(config)
   const host = VirtualHost.create({
@@ -29,28 +38,32 @@ test('listExperiments loads named and default exports from supported files', asy
     },
   })
 
-  const experiments = await listExperiments(host, '/experiments')
+  const experiments = await listExperiments({
+    host,
+    experimentsDirectory: '/experiments',
+    scenariosDirectory: '/scenarios',
+  })
 
   expect(experiments).toHaveLength(4)
   expect(experiments).toEqual(
     expect.arrayContaining([
       {
-        ...config,
+        ...resolveConfig(config),
         id: 'named',
         filepath: '/experiments/named.ts',
       },
       {
-        ...config,
+        ...resolveConfig(config),
         id: 'default',
         filepath: '/experiments/default.js',
       },
       {
-        ...config,
+        ...resolveConfig(config),
         id: 'commonjs',
         filepath: '/experiments/commonjs.cjs',
       },
       {
-        ...config,
+        ...resolveConfig(config),
         id: 'module',
         filepath: '/experiments/module.mjs',
       },
@@ -76,9 +89,15 @@ test('listExperiments prefers the named experiment export', async () => {
     },
   })
 
-  await expect(listExperiments(host, '/experiments')).resolves.toEqual([
+  await expect(
+    listExperiments({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).resolves.toEqual([
     {
-      ...namedConfig,
+      ...resolveConfig(namedConfig),
       id: 'experiment',
       filepath: '/experiments/experiment.ts',
     },
@@ -98,9 +117,15 @@ test('listExperiments ignores unsupported, reserved, missing, and invalid config
     },
   })
 
-  await expect(listExperiments(host, '/experiments')).resolves.toEqual([
+  await expect(
+    listExperiments({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).resolves.toEqual([
     {
-      ...config,
+      ...resolveConfig(config),
       id: 'valid',
       filepath: '/experiments/valid.ts',
     },
@@ -110,9 +135,13 @@ test('listExperiments ignores unsupported, reserved, missing, and invalid config
 test('listExperiments throws when the directory does not exist', async () => {
   const host = VirtualHost.create()
 
-  await expect(listExperiments(host, '/experiments')).rejects.toThrowError(
-    'Experiments directory does not exist: /experiments',
-  )
+  await expect(
+    listExperiments({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).rejects.toThrowError('Experiments directory does not exist: /experiments')
 })
 
 test('listExperiments throws when the path is not a directory', async () => {
@@ -120,9 +149,13 @@ test('listExperiments throws when the path is not a directory', async () => {
     '/experiments': '',
   })
 
-  await expect(listExperiments(host, '/experiments')).rejects.toThrowError(
-    'Experiments path is not a directory: /experiments',
-  )
+  await expect(
+    listExperiments({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).rejects.toThrowError('Experiments path is not a directory: /experiments')
 })
 
 test('getExperiment returns the experiment matching the id', async () => {
@@ -134,8 +167,15 @@ test('getExperiment returns the experiment matching the id', async () => {
     },
   })
 
-  await expect(getExperiment(host, '/experiments', 'second')).resolves.toEqual({
-    ...config,
+  await expect(
+    getExperiment({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+      id: 'second',
+    }),
+  ).resolves.toEqual({
+    ...resolveConfig(config),
     id: 'second',
     filepath: '/experiments/second.ts',
   })
@@ -146,7 +186,12 @@ test('getExperiment throws when the experiment is not found', async () => {
     '/experiments': {},
   })
 
-  await expect(getExperiment(host, '/experiments', 'missing')).rejects.toThrowError(
-    'Experiment "missing" was not found in: /experiments',
-  )
+  await expect(
+    getExperiment({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+      id: 'missing',
+    }),
+  ).rejects.toThrowError('Experiment "missing" was not found in: /experiments')
 })
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 43c7c69b..3f7bd870 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -128,7 +128,7 @@ async function getExperiment({
     experimentsDirectory,
     scenariosDirectory,
   })
-  const experiment = experiments.find(experiment => experiment.id === id)
+  const experiment = experiments.find(candidate => candidate.id === id)
   if (experiment) {
     return experiment
   }
@@ -185,44 +185,48 @@ async function run({
   return results
 }
 
-const ExperimentOutputSchema = z.object({
-  scenarios: z.map(
-    z.string(),
-    z.pick(ScenarioSchema, {
-      id: true,
-      directory: true,
-      prompt: true,
-      description: true,
-      tags: true,
-      testPath: true,
-      browserTestPath: true,
-    }),
-  ),
-  treatments: z.map(
-    z.string(),
-    z.pick(TreatmentSchema, {
-      name: true,
-    }),
-  ),
-  trials: z.map(
-    z.string(),
-    z.object({
-      agent: TrialAgentSchema,
-      artifacts: TrialArtifactsSchema,
-      id: z.string(),
-      model: ModelVariantSchema,
-      scenarioId: z.string(),
-      testResults: TestResultsSchema,
-      treatmentId: z.string(),
-      walkthrough: WalkthroughSchema,
-    }),
-  ),
+const ExperimentOutputScenarioSchema = z.pick(ScenarioSchema, {
+  id: true,
+  directory: true,
+  prompt: true,
+  description: true,
+  tags: true,
+  testPath: true,
+  browserTestPath: true,
 })
 
-type ExperimentOutput = z.infer<typeof ExperimentOutputSchema>
+const ExperimentOutputTreatmentSchema = z.pick(TreatmentSchema, {
+  name: true,
+})
+
+const ExperimentOutputTrialSchema = z.object({
+  agent: TrialAgentSchema,
+  artifacts: TrialArtifactsSchema,
+  id: z.string(),
+  model: ModelVariantSchema,
+  scenarioId: z.string(),
+  testResults: TestResultsSchema,
+  treatmentId: z.string(),
+  walkthrough: WalkthroughSchema,
+})
+
+type ExperimentOutput = {
+  experimentId: string
+  scenarios: Map<string, z.infer<typeof ExperimentOutputScenarioSchema>>
+  treatments: Map<string, z.infer<typeof ExperimentOutputTreatmentSchema>>
+  trials: Map<string, z.infer<typeof ExperimentOutputTrialSchema>>
+}
+
+const SerializedExperimentOutputSchema = z.object({
+  experimentId: z.string(),
+  scenarios: z.record(z.string(), ExperimentOutputScenarioSchema),
+  treatments: z.record(z.string(), ExperimentOutputTreatmentSchema),
+  trials: z.record(z.string(), ExperimentOutputTrialSchema),
+})
 
-function output(trialResults: ExperimentRunResult): ExperimentOutput {
+function output(experimentId: string, trialResults: ExperimentRunResult): ExperimentOutput {
   const result: ExperimentOutput = {
+    experimentId,
     scenarios: new Map(),
     treatments: new Map(),
     trials: new Map(),
@@ -254,19 +258,21 @@ function output(trialResults: ExperimentRunResult): ExperimentOutput {
   return result
 }
 
-function serialize(output: ExperimentOutput): string {
+function serialize(experimentOutput: ExperimentOutput): string {
   return JSON.stringify({
-    scenarios: Object.fromEntries(output.scenarios),
-    treatments: Object.fromEntries(output.treatments),
-    trials: Object.fromEntries(output.trials),
+    experimentId: experimentOutput.experimentId,
+    scenarios: Object.fromEntries(experimentOutput.scenarios),
+    treatments: Object.fromEntries(experimentOutput.treatments),
+    trials: Object.fromEntries(experimentOutput.trials),
   })
 }
 
 function deserialize(input: unknown): ExperimentOutput {
   const parsed = typeof input === 'string' ? JSON.parse(input) : input
-  const result = ExperimentOutputSchema.parse(parsed, {reportInput: true})
+  const result = SerializedExperimentOutputSchema.parse(parsed, {reportInput: true})
 
   return {
+    experimentId: result.experimentId,
     scenarios: new Map(Object.entries(result.scenarios)),
     treatments: new Map(Object.entries(result.treatments)),
     trials: new Map(Object.entries(result.trials)),
@@ -274,4 +280,4 @@ function deserialize(input: unknown): ExperimentOutput {
 }
 
 export {ExperimentConfigSchema, defineConfig, deserialize, getExperiment, listExperiments, output, run, serialize}
-export type {ExperimentConfig, Experiment}
+export type {ExperimentConfig, Experiment, ExperimentOutput}
diff --git a/packages/agent-eval/src/index.test.ts b/packages/agent-eval/src/index.test.ts
new file mode 100644
index 00000000..c74c9077
--- /dev/null
+++ b/packages/agent-eval/src/index.test.ts
@@ -0,0 +1,25 @@
+import {expect, test} from 'vitest'
+import {
+  BenchmarkConfigSchema,
+  ControlTreatment,
+  ExperimentConfigSchema,
+  ScenarioConfigSchema,
+  TreatmentSchema,
+  TrialResultSchema,
+  TrialSchema,
+  defineBenchmarkConfig,
+  defineExperimentConfig,
+  defineScenarioConfig,
+} from './index'
+
+test('exports the public configuration helpers and schemas', () => {
+  expect(defineBenchmarkConfig).toBeTypeOf('function')
+  expect(defineExperimentConfig).toBeTypeOf('function')
+  expect(defineScenarioConfig).toBeTypeOf('function')
+  expect(BenchmarkConfigSchema).toBeDefined()
+  expect(ExperimentConfigSchema).toBeDefined()
+  expect(ScenarioConfigSchema).toBeDefined()
+  expect(TreatmentSchema.parse(ControlTreatment)).toEqual(ControlTreatment)
+  expect(TrialSchema).toBeDefined()
+  expect(TrialResultSchema).toBeDefined()
+})
diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts
index 6e67bebd..39dd3dc2 100644
--- a/packages/agent-eval/src/index.ts
+++ b/packages/agent-eval/src/index.ts
@@ -1,5 +1,21 @@
-export {defineConfig as defineBenchmarkConfig, getBenchmark, listBenchmarks, BenchmarkConfigSchema} from './benchmark'
-export type {BenchmarkConfig, Benchmark} from './benchmark'
+export {
+  BenchmarkConfigSchema,
+  defineConfig as defineBenchmarkConfig,
+  deserialize as deserializeBenchmarkOutput,
+  getBenchmark,
+  listBenchmarks,
+  output as getBenchmarkOutput,
+  run as runBenchmark,
+  serialize as serializeBenchmarkOutput,
+} from './benchmark'
+export type {
+  BenchmarkConfig,
+  Benchmark,
+  BenchmarkOutput,
+  BenchmarkRunResult,
+  BenchmarkTrialResult,
+  Capability,
+} from './benchmark'
 
 export {
   ExperimentConfigSchema,
@@ -11,10 +27,10 @@ export {
   run as runExperiment,
   serialize as serializeExperimentOutput,
 } from './experiment'
-export type {ExperimentConfig, Experiment} from './experiment'
+export type {ExperimentConfig, Experiment, ExperimentOutput} from './experiment'
 
 export {defineConfig as defineScenarioConfig, getScenario, listScenarios, ScenarioConfigSchema} from './scenario'
-export type {ScenarioConfig, Scenario} from './scenario'
+export type {ScenarioConfig, Scenario, ScenarioSourceOptions} from './scenario'
 
 export {TreatmentSchema, ControlTreatment} from './treatment'
 export type {Treatment} from './treatment'
diff --git a/packages/agent-eval/src/logger.test.ts b/packages/agent-eval/src/logger.test.ts
new file mode 100644
index 00000000..15debe9c
--- /dev/null
+++ b/packages/agent-eval/src/logger.test.ts
@@ -0,0 +1,7 @@
+import {expect, test} from 'vitest'
+import {logger} from './logger'
+
+test('logger uses the default level and is disabled during tests', () => {
+  expect(logger.level).toBe('silent')
+  expect(logger.isLevelEnabled('info')).toBe(false)
+})
diff --git a/packages/agent-eval/src/mcp-config.test.ts b/packages/agent-eval/src/mcp-config.test.ts
new file mode 100644
index 00000000..9692792c
--- /dev/null
+++ b/packages/agent-eval/src/mcp-config.test.ts
@@ -0,0 +1,55 @@
+import {describe, expect, test} from 'vitest'
+import {McpConfigFileSchema, McpServerConfigSchema} from './mcp-config'
+
+describe('McpServerConfigSchema', () => {
+  test('parses a local MCP server configuration', () => {
+    expect(
+      McpServerConfigSchema.parse({
+        command: 'npx',
+        type: 'local',
+        args: ['example-server'],
+        env: {
+          TOKEN: 'token',
+        },
+        tools: ['example'],
+      }),
+    ).toEqual({
+      command: 'npx',
+      type: 'local',
+      args: ['example-server'],
+      env: {
+        TOKEN: 'token',
+      },
+      tools: ['example'],
+    })
+  })
+
+  test('rejects unsupported server types', () => {
+    expect(() => {
+      McpServerConfigSchema.parse({
+        command: 'https://example.com',
+        type: 'remote',
+      })
+    }).toThrow()
+  })
+})
+
+test('McpConfigFileSchema parses named servers', () => {
+  expect(
+    McpConfigFileSchema.parse({
+      mcpServers: {
+        example: {
+          command: 'example-server',
+          type: 'local',
+        },
+      },
+    }),
+  ).toEqual({
+    mcpServers: {
+      example: {
+        command: 'example-server',
+        type: 'local',
+      },
+    },
+  })
+})
diff --git a/packages/agent-eval/src/plan.test.ts b/packages/agent-eval/src/plan.test.ts
new file mode 100644
index 00000000..f85de6f8
--- /dev/null
+++ b/packages/agent-eval/src/plan.test.ts
@@ -0,0 +1,168 @@
+import {afterEach, describe, expect, test, vi} from 'vitest'
+import {VirtualHost} from './host'
+import {create, run} from './plan'
+import {run as runTrial} from './trial'
+import type {Trial, TrialResult} from './trial'
+
+vi.mock('./trial', async importOriginal => {
+  const original = await importOriginal<typeof import('./trial')>()
+  return {
+    ...original,
+    run: vi.fn(),
+  }
+})
+
+afterEach(() => {
+  vi.clearAllMocks()
+  vi.restoreAllMocks()
+})
+
+function createTrial(id: string): Trial {
+  return {
+    id,
+    scenario: {
+      id: 'scenario',
+      directory: '/scenario',
+      prompt: 'prompt',
+      tags: [],
+      testPath: '/scenario/scenario.test.ts',
+    },
+    treatment: {
+      name: 'Control',
+    },
+    model: {
+      name: 'gpt-5.6-sol',
+      reasoningEffort: 'medium',
+    },
+  }
+}
+
+function createResult(trial: Trial): TrialResult {
+  return {
+    artifacts: {
+      directory: '/artifacts',
+      copilotConfigDirectory: '/artifacts/.copilot',
+      skillsConfigDirectory: '/artifacts/.agents',
+      testResultsPath: '/artifacts/test-results.json',
+      workspaceDirectory: '/artifacts/workspace',
+    },
+    trial,
+    agent: {
+      sessions: [],
+    },
+    testResults: {
+      numTotalTests: 0,
+      numPassedTests: 0,
+      numFailedTests: 0,
+      numPendingTests: 0,
+      numTodoTests: 0,
+      success: true,
+      testResults: [],
+    },
+    walkthrough: {
+      type: 'Unavailable',
+    },
+  }
+}
+
+describe('create', () => {
+  test('randomizes trials without mutating the input', async () => {
+    const trials = [createTrial('one'), createTrial('two'), createTrial('three')]
+    vi.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(0)
+
+    const plan = await create(trials)
+
+    expect(plan.trials.map(trial => trial.id)).toEqual(['two', 'three', 'one'])
+    expect(trials.map(trial => trial.id)).toEqual(['one', 'two', 'three'])
+  })
+})
+
+describe('run', () => {
+  test('runs each trial and returns results in plan order', async () => {
+    const trials = [createTrial('one'), createTrial('two')]
+    const results = trials.map(createResult)
+    const host = VirtualHost.create()
+    vi.mocked(runTrial).mockImplementation(async ({trial}) => {
+      return createResult(trial)
+    })
+
+    await expect(
+      run({
+        env: {
+          artifactsDirectory: '/artifacts',
+          benchmarksDirectory: '/benchmarks',
+          concurrency: 2,
+          copilotToken: 'token',
+          dockerImage: 'node:test',
+          experimentsDirectory: '/experiments',
+          outputPath: '/output.json',
+          scenariosDirectory: '/scenarios',
+        },
+        host,
+        plan: {
+          trials,
+        },
+      }),
+    ).resolves.toEqual(results)
+    expect(runTrial).toHaveBeenCalledTimes(2)
+  })
+
+  test('retries a failed trial three times', async () => {
+    const trial = createTrial('one')
+    const result = createResult(trial)
+    const host = VirtualHost.create()
+    vi.spyOn(console, 'log').mockImplementation(() => {})
+    vi.mocked(runTrial)
+      .mockRejectedValueOnce(new Error('first'))
+      .mockRejectedValueOnce(new Error('second'))
+      .mockRejectedValueOnce(new Error('third'))
+      .mockResolvedValueOnce(result)
+
+    await expect(
+      run({
+        env: {
+          artifactsDirectory: '/artifacts',
+          benchmarksDirectory: '/benchmarks',
+          concurrency: 1,
+          copilotToken: 'token',
+          dockerImage: 'node:test',
+          experimentsDirectory: '/experiments',
+          outputPath: '/output.json',
+          scenariosDirectory: '/scenarios',
+        },
+        host,
+        plan: {
+          trials: [trial],
+        },
+      }),
+    ).resolves.toEqual([result])
+    expect(runTrial).toHaveBeenCalledTimes(4)
+  })
+
+  test('throws after all retry attempts fail', async () => {
+    const host = VirtualHost.create()
+    const error = new Error('failure')
+    vi.spyOn(console, 'log').mockImplementation(() => {})
+    vi.mocked(runTrial).mockRejectedValue(error)
+
+    await expect(
+      run({
+        env: {
+          artifactsDirectory: '/artifacts',
+          benchmarksDirectory: '/benchmarks',
+          concurrency: 1,
+          copilotToken: 'token',
+          dockerImage: 'node:test',
+          experimentsDirectory: '/experiments',
+          outputPath: '/output.json',
+          scenariosDirectory: '/scenarios',
+        },
+        host,
+        plan: {
+          trials: [createTrial('one')],
+        },
+      }),
+    ).rejects.toBe(error)
+    expect(runTrial).toHaveBeenCalledTimes(4)
+  })
+})
diff --git a/packages/agent-eval/src/sandbox/constants.test.ts b/packages/agent-eval/src/sandbox/constants.test.ts
new file mode 100644
index 00000000..17c9943d
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/constants.test.ts
@@ -0,0 +1,26 @@
+import path from 'node:path'
+import {expect, test} from 'vitest'
+import {
+  AGENT_INSTRUCTIONS_PATH,
+  AGENTS_DIR,
+  CONTAINER_WORKDIR,
+  COPILOT_DIR,
+  COPILOT_PLUGIN_SOURCES_DIR,
+  CUSTOM_AGENTS_DIR,
+  MCP_CONFIG_PATH,
+  NODE_USER,
+  NPM_GLOBAL_DIR,
+  SANDBOX_GID,
+  SANDBOX_UID,
+  SKILLS_DIR,
+} from './constants'
+
+test('sandbox paths and user values remain internally consistent', () => {
+  expect(AGENT_INSTRUCTIONS_PATH).toBe(path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md'))
+  expect(CUSTOM_AGENTS_DIR).toBe(path.posix.join(COPILOT_DIR, 'agents'))
+  expect(COPILOT_PLUGIN_SOURCES_DIR).toBe(path.posix.join(COPILOT_DIR, 'plugin-sources'))
+  expect(MCP_CONFIG_PATH).toBe(path.posix.join(COPILOT_DIR, 'mcp-config.json'))
+  expect(SKILLS_DIR).toBe(path.posix.join(AGENTS_DIR, 'skills'))
+  expect(NODE_USER).toBe(`${SANDBOX_UID}:${SANDBOX_GID}`)
+  expect(NPM_GLOBAL_DIR).toBe('/home/node/.npm-global')
+})
diff --git a/packages/agent-eval/src/sandbox/index.test.ts b/packages/agent-eval/src/sandbox/index.test.ts
new file mode 100644
index 00000000..8f1cff0e
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/index.test.ts
@@ -0,0 +1,10 @@
+import {expect, test} from 'vitest'
+import {CONTAINER_WORKDIR, DEFAULT_DOCKER_IMAGE, SandboxSchema, SystemSandbox, VirtualSandbox} from './index'
+
+test('exports sandbox implementations, schema, and constants', () => {
+  expect(CONTAINER_WORKDIR).toBe('/home/sandbox/workspace')
+  expect(DEFAULT_DOCKER_IMAGE).toBe('node:26.5.0-slim')
+  expect(SandboxSchema).toBeDefined()
+  expect(SystemSandbox).toBeTypeOf('function')
+  expect(VirtualSandbox).toBeTypeOf('function')
+})
diff --git a/packages/agent-eval/src/sandbox/path.test.ts b/packages/agent-eval/src/sandbox/path.test.ts
new file mode 100644
index 00000000..a78a74de
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/path.test.ts
@@ -0,0 +1,13 @@
+import {describe, expect, test} from 'vitest'
+import {CONTAINER_WORKDIR} from './constants'
+import {resolveContainerPath} from './path'
+
+describe('resolveContainerPath', () => {
+  test('resolves relative paths from the container workdir', () => {
+    expect(resolveContainerPath('nested/../example.txt')).toBe(`${CONTAINER_WORKDIR}/example.txt`)
+  })
+
+  test('normalizes absolute container paths', () => {
+    expect(resolveContainerPath('/tmp/nested/../example.txt')).toBe('/tmp/example.txt')
+  })
+})
diff --git a/packages/agent-eval/src/sandbox/system.test.ts b/packages/agent-eval/src/sandbox/system.test.ts
new file mode 100644
index 00000000..f7e3a8d6
--- /dev/null
+++ b/packages/agent-eval/src/sandbox/system.test.ts
@@ -0,0 +1,227 @@
+import Docker from 'dockerode'
+import {beforeEach, describe, expect, test, vi} from 'vitest'
+import {VirtualHost} from '../host'
+import {MCP_CONFIG_PATH, NODE_USER, SKILLS_DIR} from './constants'
+import {SandboxSchema, SystemSandbox} from './system'
+import {VirtualSandbox} from './virtual'
+
+function createSandbox() {
+  const container = {
+    stop: vi.fn(),
+  }
+  // @ts-expect-error This test only exercises methods whose container operations are mocked.
+  return new SystemSandbox(VirtualHost.create(), new Docker(), container)
+}
+
+describe('SandboxSchema', () => {
+  test('accepts system and virtual sandboxes', async () => {
+    const systemSandbox = createSandbox()
+    const virtualSandbox = await VirtualSandbox.create()
+
+    expect(SandboxSchema.parse(systemSandbox)).toBe(systemSandbox)
+    expect(SandboxSchema.parse(virtualSandbox)).toBe(virtualSandbox)
+    expect(() => {
+      SandboxSchema.parse({})
+    }).toThrow()
+  })
+})
+
+describe('SystemSandbox configuration helpers', () => {
+  let sandbox: SystemSandbox
+
+  beforeEach(() => {
+    sandbox = createSandbox()
+    vi.spyOn(sandbox, 'runCommand').mockResolvedValue({
+      stdout: '',
+      stderr: '',
+      exitCode: 0,
+    })
+    vi.spyOn(sandbox, 'writeFile').mockResolvedValue()
+    vi.spyOn(sandbox, 'copy').mockResolvedValue()
+  })
+
+  test('appends agent instructions with normalized newlines', async () => {
+    vi.spyOn(sandbox, 'exists').mockResolvedValue(true)
+    vi.spyOn(sandbox, 'readFile').mockResolvedValue('Existing instructions')
+
+    await sandbox.addAgentInstruction('New instructions')
+
+    expect(sandbox.writeFile).toHaveBeenCalledWith(
+      '/home/sandbox/workspace/AGENTS.md',
+      'Existing instructions\nNew instructions\n',
+    )
+  })
+
+  test('creates an agent skill and supporting files', async () => {
+    vi.spyOn(sandbox, 'exists').mockResolvedValue(false)
+
+    await sandbox.addAgentSkill('example-skill', 'Example description', 'Skill instructions', {
+      files: [
+        {
+          path: 'references/example.md',
+          content: 'reference',
+        },
+        {
+          sourcePath: '/fixtures/script.js',
+          destinationPath: 'scripts/script.js',
+        },
+      ],
+    })
+
+    expect(sandbox.writeFile).toHaveBeenCalledWith(
+      `${SKILLS_DIR}/example-skill/SKILL.md`,
+      `---
+name: "example-skill"
+description: "Example description"
+---
+
+Skill instructions
+`,
+    )
+    expect(sandbox.writeFile).toHaveBeenCalledWith(`${SKILLS_DIR}/example-skill/references/example.md`, 'reference')
+    expect(sandbox.copy).toHaveBeenCalledWith('/fixtures/script.js', `${SKILLS_DIR}/example-skill/scripts/script.js`)
+  })
+
+  test('rejects invalid skill names and file destinations', async () => {
+    vi.spyOn(sandbox, 'exists').mockResolvedValue(false)
+
+    await expect(sandbox.addAgentSkill('Invalid Skill', 'description', 'contents')).rejects.toThrow(
+      'Skill names must be lowercase and use hyphens for spaces',
+    )
+    await expect(
+      sandbox.addAgentSkill('valid-skill', 'description', 'contents', {
+        files: [
+          {
+            path: '../outside.md',
+            content: 'outside',
+          },
+        ],
+      }),
+    ).rejects.toThrow('Invalid agent skill file destination "../outside.md"')
+  })
+
+  test('creates a custom agent with tools', async () => {
+    vi.spyOn(sandbox, 'exists').mockResolvedValue(false)
+
+    await sandbox.addCustomAgent('example-agent', 'Example description', 'Agent instructions', {
+      tools: ['view', 'grep'],
+    })
+
+    expect(sandbox.writeFile).toHaveBeenCalledWith(
+      '/home/node/.copilot/agents/example-agent.agent.md',
+      `---
+name: "example-agent"
+description: "Example description"
+tools: ["view","grep"]
+---
+
+Agent instructions
+`,
+    )
+  })
+
+  test('adds an MCP server to the existing configuration', async () => {
+    vi.spyOn(sandbox, 'readFile').mockResolvedValue(
+      JSON.stringify({
+        mcpServers: {
+          existing: {
+            command: 'existing-server',
+            type: 'local',
+          },
+        },
+      }),
+    )
+
+    await sandbox.addMcpServer('example', {
+      command: 'example-server',
+      type: 'local',
+    })
+
+    expect(sandbox.writeFile).toHaveBeenCalledWith(
+      MCP_CONFIG_PATH,
+      JSON.stringify(
+        {
+          mcpServers: {
+            existing: {
+              command: 'existing-server',
+              type: 'local',
+            },
+            example: {
+              command: 'example-server',
+              type: 'local',
+            },
+          },
+        },
+        null,
+        2,
+      ),
+    )
+    expect(sandbox.runCommand).toHaveBeenCalledWith('chown', ['-R', NODE_USER, MCP_CONFIG_PATH], {
+      user: 'root',
+    })
+  })
+
+  test('rejects duplicate MCP server names', async () => {
+    vi.spyOn(sandbox, 'readFile').mockResolvedValue(
+      JSON.stringify({
+        mcpServers: {
+          example: {
+            command: 'example-server',
+            type: 'local',
+          },
+        },
+      }),
+    )
+
+    await expect(
+      sandbox.addMcpServer('example', {
+        command: 'other-server',
+        type: 'local',
+      }),
+    ).rejects.toThrow('MCP server with name "example" already exists')
+  })
+
+  test('installs remote, local, and marketplace plugins', async () => {
+    await sandbox.addCopilotPlugin({
+      type: 'remote',
+      url: 'https://example.com/plugin.git',
+      version: 'v1',
+    })
+    await sandbox.addCopilotPlugin({
+      type: 'local',
+      sourcePath: '/fixtures/plugin',
+    })
+    await sandbox.addCopilotPlugin({
+      type: 'marketplace',
+      name: 'example-plugin',
+      marketplace: {
+        name: 'example-marketplace',
+        source: {
+          type: 'remote',
+          url: 'https://example.com/marketplace.git',
+        },
+      },
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [
+      'plugin',
+      'install',
+      'https://example.com/plugin.git#v1',
+    ])
+    expect(sandbox.copy).toHaveBeenCalledWith(
+      '/fixtures/plugin',
+      expect.stringMatching(/^\/home\/node\/\.copilot\/plugin-sources\/.+/),
+    )
+    expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [
+      'plugin',
+      'marketplace',
+      'add',
+      'https://example.com/marketplace.git',
+    ])
+    expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [
+      'plugin',
+      'install',
+      'example-plugin@example-marketplace',
+    ])
+  })
+})
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index 4dd9e00d..c54b2eb3 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -1,18 +1,12 @@
 import path from 'node:path'
 import * as z from 'zod/mini'
-import type {Host} from './host'
-
-const ScenarioConfigSchema = z.union([
-  z.object({
-    description: z.optional(z.string()),
-    prompt: z.string(),
-    tags: z.optional(z.array(z.string())),
-  }),
-  z.object({
-    name: z.optional(z.string()),
-    path: z.string(),
-  }),
-])
+import {DefaultHost, type Host} from './host'
+
+const ScenarioConfigSchema = z.object({
+  description: z.optional(z.string()),
+  prompt: z.string(),
+  tags: z.optional(z.array(z.string())),
+})
 
 type ScenarioConfig = z.infer<typeof ScenarioConfigSchema>
 
@@ -36,7 +30,38 @@ const ScenarioSchema = z.object({
 
 type Scenario = z.infer<typeof ScenarioSchema>
 
-async function listScenarios(host: Host, directory: string): Promise<Array<Scenario>> {
+type ScenarioSourceOptions = {
+  host?: Host
+  directory: string
+}
+
+function getScenarioSource(
+  hostOrOptions: Host | ScenarioSourceOptions,
+  directory?: string,
+): {host: Host; directory: string} {
+  if (directory !== undefined) {
+    return {
+      host: hostOrOptions as Host,
+      directory,
+    }
+  }
+
+  const options = hostOrOptions as ScenarioSourceOptions
+  return {
+    host: options.host ?? DefaultHost,
+    directory: options.directory,
+  }
+}
+
+async function listScenarios(options: ScenarioSourceOptions): Promise<Array<Scenario>>
+async function listScenarios(host: Host, directory: string): Promise<Array<Scenario>>
+async function listScenarios(
+  hostOrOptions: Host | ScenarioSourceOptions,
+  directory?: string,
+): Promise<Array<Scenario>> {
+  const source = getScenarioSource(hostOrOptions, directory)
+  const {host} = source
+  directory = source.directory
   const stats = await host.fs.stat(directory)
   if (!stats.isDirectory()) {
     throw new Error('Expected scenarios path to be a directory')
@@ -112,15 +137,23 @@ async function listScenarios(host: Host, directory: string): Promise<Array<Scena
   return scenarios
 }
 
-async function getScenario(host: Host, directory: string, id: string): Promise<Scenario> {
-  const scenarios = await listScenarios(host, directory)
-  const scenario = scenarios.find(scenario => scenario.id === id)
+async function getScenario(options: ScenarioSourceOptions & {id: string}): Promise<Scenario>
+async function getScenario(host: Host, directory: string, id: string): Promise<Scenario>
+async function getScenario(
+  hostOrOptions: Host | (ScenarioSourceOptions & {id: string}),
+  directory?: string,
+  id?: string,
+): Promise<Scenario> {
+  const source = getScenarioSource(hostOrOptions, directory)
+  id = id ?? (hostOrOptions as ScenarioSourceOptions & {id: string}).id
+  const scenarios = await listScenarios(source)
+  const scenario = scenarios.find(candidate => candidate.id === id)
   if (scenario) {
     return scenario
   }
 
-  throw new Error(`Scenario "${id}" was not found in: ${directory}`)
+  throw new Error(`Scenario "${id}" was not found in: ${source.directory}`)
 }
 
 export {defineConfig, listScenarios, getScenario, ScenarioSchema, ScenarioConfigSchema}
-export type {ScenarioConfig, Scenario}
+export type {ScenarioConfig, Scenario, ScenarioSourceOptions}
diff --git a/packages/agent-eval/src/treatment.test.ts b/packages/agent-eval/src/treatment.test.ts
new file mode 100644
index 00000000..f57c69fb
--- /dev/null
+++ b/packages/agent-eval/src/treatment.test.ts
@@ -0,0 +1,33 @@
+import {describe, expect, test, vi} from 'vitest'
+import {VirtualSandbox} from './sandbox'
+import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema} from './treatment'
+
+describe('TreatmentSchema', () => {
+  test('parses the control treatment', () => {
+    expect(TreatmentSchema.parse(ControlTreatment)).toEqual({
+      name: 'Control',
+    })
+  })
+
+  test('parses and runs a setup function', async () => {
+    const setup = vi.fn(async () => {})
+    const treatment = TreatmentSchema.parse({
+      name: 'Example',
+      setup,
+    })
+    const sandbox = await VirtualSandbox.create()
+
+    await treatment.setup?.({sandbox})
+
+    expect(setup).toHaveBeenCalledWith({sandbox})
+  })
+
+  test('rejects invalid setup return values', async () => {
+    const setup = TreatmentSetupSchema.parse(() => {
+      return 'invalid'
+    })
+    const sandbox = await VirtualSandbox.create()
+
+    await expect(setup({sandbox})).rejects.toThrow()
+  })
+})

From e3784de87ad665d31b5f4fed824d8fb81389c0be Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 15:40:03 -0500
Subject: [PATCH 26/52] refactor: update to new config

---
 .../app/experiments/[id]/runs/[date]/page.tsx |   8 +-
 website/src/baseline-results.ts               |  23 ++-
 website/src/experiments.ts                    |  44 ++---
 website/src/runs.ts                           | 174 +++++++++++++++++-
 website/src/scenarios.ts                      |  33 ++--
 5 files changed, 213 insertions(+), 69 deletions(-)

diff --git a/website/src/app/experiments/[id]/runs/[date]/page.tsx b/website/src/app/experiments/[id]/runs/[date]/page.tsx
index 0b3244d0..abd4b965 100644
--- a/website/src/app/experiments/[id]/runs/[date]/page.tsx
+++ b/website/src/app/experiments/[id]/runs/[date]/page.tsx
@@ -1,12 +1,11 @@
-import type {AgentEvalOutput} from '@primer/agent-eval/output'
 import {get as getExperiment} from '../../../../../experiments'
 import {get as getRun, list as listRuns} from '../../../../../runs'
+import type {RunOutput} from '../../../../../runs'
 import {notFound} from 'next/navigation'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import {Page} from './components/Page'
 import type {RunDetails, TranscriptEntry} from './components/Page'
-import type {Walkthrough} from '../../../../../../../packages/agent-eval/src/treatment'
 
 const EMPTY_RUN_PARAM = '__no-runs__'
 const REPOSITORY_ROOT = path.resolve(process.cwd(), '..')
@@ -19,7 +18,8 @@ type RunPageProps = {
   }>
 }
 
-type LogMessage = AgentEvalOutput['results'][number]['assistant']['logs'][number]
+type LogMessage = RunOutput['results'][number]['assistant']['logs'][number]
+type Walkthrough = RunOutput['results'][number]['walkthrough']
 
 function asRecord(value: unknown): Record<string, unknown> | null {
   return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null
@@ -216,7 +216,7 @@ async function getWalkthroughDataUrls(walkthrough: Walkthrough): Promise<Walkthr
   }
 }
 
-async function createRunDetails(date: string, output: AgentEvalOutput): Promise<RunDetails> {
+async function createRunDetails(date: string, output: RunOutput): Promise<RunDetails> {
   const treatments = new Map(output.treatments.map(treatment => [treatment.id, treatment.config.name]))
   return {
     date,
diff --git a/website/src/baseline-results.ts b/website/src/baseline-results.ts
index fcb5e3a1..d2699067 100644
--- a/website/src/baseline-results.ts
+++ b/website/src/baseline-results.ts
@@ -1,11 +1,10 @@
-import type {AgentEvalOutput, AgentEvalOutputResult} from '@primer/agent-eval/output'
 import type {BaselineComparison, BaselineResult} from './app/components/Index'
 import type {BaselineTrendPoint} from './app/components/BaselineTrends'
-import {list as listRuns} from './runs'
+import {list as listRuns, type RunOutput, type RunOutputResult} from './runs'
 
 type TreatmentResults = {
-  control?: AgentEvalOutputResult
-  baseline?: AgentEvalOutputResult
+  control?: RunOutputResult
+  baseline?: RunOutputResult
 }
 
 type ModelTreatmentResults = TreatmentResults & {
@@ -54,7 +53,7 @@ function getPercentChangeValue(control: number | undefined, baseline: number | u
   return ((baseline - control) / Math.abs(control)) * 100
 }
 
-function getTestPassRate(result: AgentEvalOutputResult | undefined) {
+function getTestPassRate(result: RunOutputResult | undefined) {
   if (!result) {
     return undefined
   }
@@ -66,7 +65,7 @@ function getTestPassRate(result: AgentEvalOutputResult | undefined) {
   return result.testResults.numPassedTests / result.testResults.numTotalTests
 }
 
-function countToolCalls(result: AgentEvalOutputResult | undefined) {
+function countToolCalls(result: RunOutputResult | undefined) {
   if (!result) {
     return undefined
   }
@@ -102,7 +101,7 @@ function formatDuration(milliseconds: number): string {
   return `${percentFormatter.format(milliseconds / 1000)}s`
 }
 
-function getBaselineComparisons(output: AgentEvalOutput): Array<BaselineComparison> {
+function getBaselineComparisons(output: RunOutput): Array<BaselineComparison> {
   const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
   const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
 
@@ -212,7 +211,7 @@ function getBaselineComparisons(output: AgentEvalOutput): Array<BaselineComparis
   })
 }
 
-function getAggregateBaselineResults(output: AgentEvalOutput): Array<BaselineResult> {
+function getAggregateBaselineResults(output: RunOutput): Array<BaselineResult> {
   const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
   const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
 
@@ -223,7 +222,7 @@ function getAggregateBaselineResults(output: AgentEvalOutput): Array<BaselineRes
   const results = output.experiment.models.flatMap(model => {
     const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined]
     return reasoningEfforts.map(reasoningEffort => {
-      const matches = (result: AgentEvalOutputResult) =>
+      const matches = (result: RunOutputResult) =>
         result.model === model.name && result.reasoningEffort === reasoningEffort
       const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result))
       const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result))
@@ -303,7 +302,7 @@ function getAggregateBaselineResults(output: AgentEvalOutput): Array<BaselineRes
     }))
 }
 
-function getBaselineTrendPoints(date: string, output: AgentEvalOutput): Array<BaselineTrendPoint> {
+function getBaselineTrendPoints(date: string, output: RunOutput): Array<BaselineTrendPoint> {
   const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
   const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
   if (!controlTreatment || !baselineTreatment) {
@@ -370,7 +369,7 @@ function getBaselineTrendPoints(date: string, output: AgentEvalOutput): Array<Ba
     })
 }
 
-function getAggregateTrendPoints(date: string, output: AgentEvalOutput): Array<BaselineTrendPoint> {
+function getAggregateTrendPoints(date: string, output: RunOutput): Array<BaselineTrendPoint> {
   const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
   const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
   if (!controlTreatment || !baselineTreatment) {
@@ -380,7 +379,7 @@ function getAggregateTrendPoints(date: string, output: AgentEvalOutput): Array<B
   return output.experiment.models.flatMap(model => {
     const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined]
     return reasoningEfforts.map(reasoningEffort => {
-      const matches = (result: AgentEvalOutputResult) =>
+      const matches = (result: RunOutputResult) =>
         result.model === model.name && result.reasoningEffort === reasoningEffort
       const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result))
       const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result))
diff --git a/website/src/experiments.ts b/website/src/experiments.ts
index fe7a37cc..1c386a06 100644
--- a/website/src/experiments.ts
+++ b/website/src/experiments.ts
@@ -1,50 +1,50 @@
 import path from 'node:path'
-import type {ExperimentConfig} from '@primer/agent-eval/experiment'
+import type {Experiment as AgentEvalExperiment} from '@primer/agent-eval/experiment'
 
-const {listExperiments, findExperiment} = await import(
+const {listExperiments, getExperiment} = await import(
   /* turbopackIgnore: true */
-  '@primer/agent-eval/experiments'
+  '@primer/agent-eval/experiment'
 )
 
 const EXPERIMENTS_DIR = path.resolve(process.cwd(), '..', 'experiments')
+const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios')
 
-export type Experiment = Pick<ExperimentConfig, 'name' | 'description' | 'models' | 'scenarios'> & {
-  id: string
+export type Experiment = Pick<AgentEvalExperiment, 'id' | 'name' | 'description' | 'models'> & {
+  scenarios: Array<{id: string}>
   treatments: Array<{name: string}>
 }
 
 export async function list(): Promise<Array<Experiment>> {
   const experiments = await listExperiments({
-    directory: EXPERIMENTS_DIR,
+    experimentsDirectory: EXPERIMENTS_DIR,
+    scenariosDirectory: SCENARIOS_DIR,
   })
 
-  return experiments.map(([id, experiment]) => {
+  return experiments.map(experiment => {
     return {
-      id,
+      id: experiment.id,
       name: experiment.name,
       description: experiment.description,
       models: experiment.models,
-      scenarios: experiment.scenarios,
+      scenarios: experiment.scenarios.map(scenario => ({id: scenario.id})),
       treatments: experiment.treatments.map(t => ({name: t.name})),
     }
   })
 }
 
 export async function get(id: string): Promise<Experiment> {
-  const experiment = await findExperiment(id, {
-    directory: EXPERIMENTS_DIR,
+  const experiment = await getExperiment({
+    experimentsDirectory: EXPERIMENTS_DIR,
+    scenariosDirectory: SCENARIOS_DIR,
+    id,
   })
 
-  if (!experiment) {
-    throw new Error(`Experiment "${id}" was not found in: ${EXPERIMENTS_DIR}`)
-  }
-
   return {
-    id,
+    id: experiment.id,
     name: experiment.name,
     description: experiment.description,
     models: experiment.models,
-    scenarios: experiment.scenarios,
+    scenarios: experiment.scenarios.map(scenario => ({id: scenario.id})),
     treatments: experiment.treatments.map(t => ({name: t.name})),
   }
 }
@@ -52,13 +52,5 @@ export async function get(id: string): Promise<Experiment> {
 export async function listForScenario(id: string): Promise<Array<Experiment>> {
   const experiments = await list()
 
-  return experiments.filter(experiment =>
-    experiment.scenarios.some(scenario => {
-      if (typeof scenario === 'string') {
-        return scenario === id
-      }
-
-      return (scenario.name ?? path.basename(path.resolve(scenario.path))) === id
-    }),
-  )
+  return experiments.filter(experiment => experiment.scenarios.some(scenario => scenario.id === id))
 }
diff --git a/website/src/runs.ts b/website/src/runs.ts
index 2da7a8d7..e0c857b7 100644
--- a/website/src/runs.ts
+++ b/website/src/runs.ts
@@ -2,16 +2,103 @@ import fs from 'node:fs/promises'
 import {existsSync, type Dirent} from 'node:fs'
 import path from 'node:path'
 
-import {parseAgentEvalOutput, type AgentEvalOutput} from '@primer/agent-eval/output'
+import type {ExperimentOutput} from '@primer/agent-eval/experiment'
+
+const {deserialize} = await import(
+  /* turbopackIgnore: true */
+  '@primer/agent-eval/experiment'
+)
 
 const RESULTS_DIR = path.resolve(process.cwd(), '..', 'results')
 
+type ExperimentOutputTrial = ExperimentOutput['trials'] extends Map<string, infer Trial> ? Trial : never
+
+type RunOutputResult = {
+  id: string
+  treatmentId: string
+  model: ExperimentOutputTrial['model']['name']
+  reasoningEffort: ExperimentOutputTrial['model']['reasoningEffort']
+  scenarioId: string
+  assistant: {
+    logs: ExperimentOutputTrial['agent']['sessions'][number]['messages']
+    turns: number
+    outputTokens: number
+    premiumRequests: number
+    totalApiDurationMs: number
+    sessionDurationMs: number
+    tools: Record<string, number>
+  }
+  testResults: ExperimentOutputTrial['testResults'] & {
+    tests: Array<{
+      title: string
+      fullName: string
+      status: string
+      description?: string
+    }>
+  }
+  walkthrough: ExperimentOutputTrial['walkthrough']
+}
+
+type RunOutput = {
+  experiment: {
+    id: string
+    models: Array<{
+      name: ExperimentOutputTrial['model']['name']
+      reasoningEfforts: Array<ExperimentOutputTrial['model']['reasoningEffort']>
+    }>
+  }
+  scenarios: Array<ExperimentOutput['scenarios'] extends Map<string, infer Scenario> ? Scenario : never>
+  treatments: Array<{
+    id: string
+    config: {
+      name: string
+    }
+  }>
+  results: Array<RunOutputResult>
+}
+
 type Run = {
   id: string
   name: string
   directory: string
   date: Date
-  output: AgentEvalOutput
+  output: RunOutput
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  if (typeof value !== 'object' || value === null) {
+    return null
+  }
+
+  return value as Record<string, unknown>
+}
+
+function isLegacyRunOutput(value: unknown): value is RunOutput {
+  const output = asRecord(value)
+  const experiment = asRecord(output?.experiment)
+
+  return (
+    typeof experiment?.id === 'string' &&
+    Array.isArray(experiment.models) &&
+    Array.isArray(output?.scenarios) &&
+    Array.isArray(output.treatments) &&
+    Array.isArray(output.results)
+  )
+}
+
+function parseOutput(contents: string): RunOutput {
+  const parsed: unknown = JSON.parse(contents)
+  const output = asRecord(parsed)
+
+  if (typeof output?.experimentId === 'string') {
+    return normalizeOutput(deserialize(parsed))
+  }
+
+  if (isLegacyRunOutput(parsed)) {
+    return parsed
+  }
+
+  throw new Error('Result output does not match a supported experiment output format')
 }
 
 function isRunName(name: string): boolean {
@@ -43,7 +130,7 @@ async function list(): Promise<Array<Run>> {
         const directory = path.join(RESULTS_DIR, entry.name)
         const outputFile = path.join(directory, 'output.json')
         const contents = await fs.readFile(outputFile, 'utf-8')
-        const output = parseAgentEvalOutput(contents)
+        const output = parseOutput(contents)
         return [directory, entry.name, output] as const
       }),
     )
@@ -52,7 +139,7 @@ async function list(): Promise<Array<Run>> {
   return results
     .map(([directory, name, output]) => {
       const date = new Date(`${name}T00:00:00.000Z`)
-      return {id: output.id, name, directory, date, output}
+      return {id: name, name, directory, date, output}
     })
     .toSorted((a, b) => b.date.getTime() - a.date.getTime())
 }
@@ -96,9 +183,9 @@ async function find(name: string): Promise<Run | null> {
 
   const outputFile = path.join(directory, 'output.json')
   const contents = await fs.readFile(outputFile, 'utf-8')
-  const output = parseAgentEvalOutput(contents)
+  const output = parseOutput(contents)
   return {
-    id: output.id,
+    id: name,
     name,
     directory,
     date: new Date(`${name}T00:00:00.000Z`),
@@ -115,5 +202,78 @@ async function get(name: string): Promise<Run> {
   return run
 }
 
+function normalizeOutput(output: ExperimentOutput): RunOutput {
+  const modelReasoningEfforts = new Map<
+    ExperimentOutputTrial['model']['name'],
+    Set<ExperimentOutputTrial['model']['reasoningEffort']>
+  >()
+
+  const results = [...output.trials.values()].map(trial => {
+    const reasoningEfforts = modelReasoningEfforts.get(trial.model.name) ?? new Set()
+    reasoningEfforts.add(trial.model.reasoningEffort)
+    modelReasoningEfforts.set(trial.model.name, reasoningEfforts)
+
+    const tools: Record<string, number> = {}
+    for (const session of trial.agent.sessions) {
+      for (const [name, count] of Object.entries(session.tools)) {
+        tools[name] = (tools[name] ?? 0) + count
+      }
+    }
+
+    return {
+      id: trial.id,
+      treatmentId: trial.treatmentId,
+      model: trial.model.name,
+      reasoningEffort: trial.model.reasoningEffort,
+      scenarioId: trial.scenarioId,
+      assistant: {
+        logs: trial.agent.sessions.flatMap(session => session.messages),
+        turns: trial.agent.sessions.reduce((total, session) => total + session.turns, 0),
+        outputTokens: trial.agent.sessions.reduce((total, session) => total + session.outputTokens, 0),
+        premiumRequests: trial.agent.sessions.reduce((total, session) => total + session.premiumRequests, 0),
+        totalApiDurationMs: trial.agent.sessions.reduce((total, session) => total + session.totalApiDurationMs, 0),
+        sessionDurationMs: trial.agent.sessions.reduce((total, session) => total + session.sessionDurationMs, 0),
+        tools,
+      },
+      testResults: {
+        ...trial.testResults,
+        tests: trial.testResults.testResults.flatMap(testResult => {
+          return testResult.assertionResults.map(assertion => {
+            return {
+              title: assertion.title,
+              fullName: assertion.fullName,
+              status: assertion.status,
+              description: assertion.meta.description,
+            }
+          })
+        }),
+      },
+      walkthrough: trial.walkthrough,
+    }
+  })
+
+  return {
+    experiment: {
+      id: output.experimentId,
+      models: [...modelReasoningEfforts].map(([name, reasoningEfforts]) => {
+        return {
+          name,
+          reasoningEfforts: [...reasoningEfforts],
+        }
+      }),
+    },
+    scenarios: [...output.scenarios.values()],
+    treatments: [...output.treatments].map(([id, treatment]) => {
+      return {
+        id,
+        config: {
+          name: treatment.name,
+        },
+      }
+    }),
+    results,
+  }
+}
+
 export {list, listForExperiment, latest, get}
-export type {Run}
+export type {Run, RunOutput, RunOutputResult}
diff --git a/website/src/scenarios.ts b/website/src/scenarios.ts
index d6ac3d2c..61de85e5 100644
--- a/website/src/scenarios.ts
+++ b/website/src/scenarios.ts
@@ -1,17 +1,15 @@
 import fs from 'node:fs/promises'
 import path from 'node:path'
-import type {ResolvedScenario} from '@primer/agent-eval/scenarios'
+import type {Scenario as AgentEvalScenario} from '@primer/agent-eval/scenario'
 
-const {listScenarios, findScenario} = await import(
+const {listScenarios, getScenario} = await import(
   /* turbopackIgnore: true */
-  '@primer/agent-eval/scenarios'
+  '@primer/agent-eval/scenario'
 )
 
 const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios')
 
-export type ScenarioSummary = Pick<ResolvedScenario['config'], 'prompt'> & {
-  id: string
-}
+export type ScenarioSummary = Pick<AgentEvalScenario, 'id' | 'prompt'>
 
 export type Scenario = ScenarioSummary & {
   test: string
@@ -22,28 +20,23 @@ export async function list(): Promise<Array<ScenarioSummary>> {
     directory: SCENARIOS_DIR,
   })
 
-  return scenarios
-    .filter(scenario => !scenario.id.startsWith('000-'))
-    .map(scenario => {
-      return {
-        id: scenario.id,
-        prompt: scenario.config.prompt,
-      }
-    })
+  return scenarios.map(scenario => {
+    return {
+      id: scenario.id,
+      prompt: scenario.prompt,
+    }
+  })
 }
 
 export async function get(id: string): Promise<Scenario> {
-  const scenario = await findScenario(id, {
+  const scenario = await getScenario({
     directory: SCENARIOS_DIR,
+    id,
   })
 
-  if (!scenario) {
-    throw new Error(`Scenario "${id}" was not found in: ${SCENARIOS_DIR}`)
-  }
-
   return {
     id: scenario.id,
-    prompt: scenario.config.prompt,
+    prompt: scenario.prompt,
     test: await fs.readFile(scenario.testPath, 'utf8'),
   }
 }

From 07c41a4d78b490634b2a7cfd9de37ac444a303ad Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 15:40:20 -0500
Subject: [PATCH 27/52] chore: update design system benchmark

---
 benchmarks/design-system.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts
index 2ad2f016..d25f0d7f 100644
--- a/benchmarks/design-system.ts
+++ b/benchmarks/design-system.ts
@@ -3,6 +3,7 @@ import {defineConfig} from '@primer/agent-eval/benchmark'
 export const benchmark = defineConfig({
   name: 'Design System',
   description: 'Benchmark the performance of different design system approaches.',
+  models: ['gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.6-sol'],
   capabilities: [
     {
       name: 'Uses components',

From e6c44b64e00bdd79dc5dd8647114a197844d52e1 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 15:43:21 -0500
Subject: [PATCH 28/52] fix: make config const generic so the type does not
 have to be explicit

---
 packages/agent-eval/src/benchmark.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 73914fe3..b8131de2 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -26,7 +26,7 @@ const BenchmarkConfigSchema = z.object({
 
 type BenchmarkConfig = z.infer<typeof BenchmarkConfigSchema>
 
-function defineConfig(config: BenchmarkConfig): BenchmarkConfig {
+function defineConfig<const Config extends BenchmarkConfig>(config: Config): Config {
   return config
 }
 

From d09dff14aa4b1a50548f37dd084ea5eefe1023ec Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 16:02:10 -0500
Subject: [PATCH 29/52] refactor: update tests and benchmark summaries

---
 benchmarks/noop.ts                         | 24 ++++++
 experiments/noop.ts                        |  5 +-
 packages/agent-eval/src/benchmark.test.ts  |  6 +-
 packages/agent-eval/src/benchmark.ts       |  7 +-
 packages/agent-eval/src/cli.ts             |  4 +-
 packages/agent-eval/src/experiment.test.ts |  6 +-
 packages/agent-eval/src/experiment.ts      | 21 ++++-
 packages/agent-eval/src/report.test.ts     | 90 +++++++++++++++++++++-
 packages/agent-eval/src/report.ts          | 86 ++++++++++++++++++++-
 9 files changed, 233 insertions(+), 16 deletions(-)
 create mode 100644 benchmarks/noop.ts

diff --git a/benchmarks/noop.ts b/benchmarks/noop.ts
new file mode 100644
index 00000000..1e7fbde5
--- /dev/null
+++ b/benchmarks/noop.ts
@@ -0,0 +1,24 @@
+import {defineConfig} from '@primer/agent-eval/benchmark'
+
+export default defineConfig({
+  name: 'noop',
+  description: 'noop benchmark for end-to-end testing',
+  models: [
+    {
+      name: 'gpt-5.6-luna',
+      reasoningEfforts: ['low'],
+    },
+  ],
+  async setup() {
+    console.log('global setup')
+  },
+  capabilities: [
+    {
+      name: 'noop',
+      scenarios: ['001-agent-uses-button-from-primer'],
+      async setup() {
+        console.log('local setup')
+      },
+    },
+  ],
+})
diff --git a/experiments/noop.ts b/experiments/noop.ts
index 9b2ea568..e7aa5198 100644
--- a/experiments/noop.ts
+++ b/experiments/noop.ts
@@ -10,11 +10,14 @@ export const experiment = defineConfig({
     },
   ],
   scenarios: ['001-agent-uses-button-from-primer'],
+  async setup() {
+    console.log('global setup')
+  },
   treatments: [
     {
       name: 'noop',
       async setup() {
-        //
+        console.log('local setup')
       },
     },
   ],
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 5abcd58f..93dcf3b7 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -131,7 +131,7 @@ test('listBenchmarks validates the benchmarks directory', async () => {
       benchmarksDirectory: '/benchmarks',
       scenariosDirectory: '/scenarios',
     }),
-  ).rejects.toThrowError('Benchmarks directory does not exist: /benchmarks')
+  ).rejects.toThrow('Benchmarks directory does not exist: /benchmarks')
 
   const fileHost = VirtualHost.create({
     '/benchmarks': '',
@@ -143,7 +143,7 @@ test('listBenchmarks validates the benchmarks directory', async () => {
       benchmarksDirectory: '/benchmarks',
       scenariosDirectory: '/scenarios',
     }),
-  ).rejects.toThrowError('Benchmarks path is not a directory: /benchmarks')
+  ).rejects.toThrow('Benchmarks path is not a directory: /benchmarks')
 })
 
 test('getBenchmark returns the benchmark matching the id', async () => {
@@ -179,7 +179,7 @@ test('getBenchmark throws when the benchmark is not found', async () => {
       scenariosDirectory: '/scenarios',
       id: 'missing',
     }),
-  ).rejects.toThrowError('Benchmark "missing" was not found in: /benchmarks')
+  ).rejects.toThrow('Benchmark "missing" was not found in: /benchmarks')
 })
 
 test('run returns an empty result when the benchmark has no trials', async () => {
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index b8131de2..9ba61bd7 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -4,7 +4,12 @@ import * as z from 'zod/mini'
 import type {EnvironmentConfig} from './environment'
 import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
-import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
+import {
+  getModelVariants,
+  ModelVariantConfigSchema,
+  ModelVariantSchema,
+  type ModelVariant,
+} from './model'
 import {create as createPlan, run as runPlan} from './plan'
 import {getScenario, ScenarioSchema, type Scenario} from './scenario'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type TreatmentSetup} from './treatment'
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 036347ea..f31d1e81 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -18,7 +18,7 @@ import {
   serialize as serializeExperimentOutput,
 } from './experiment'
 import {logger} from './logger'
-import {formatExperimentResults} from './report'
+import {formatBenchmarkResults, formatExperimentResults} from './report'
 import {compare as compareTrial} from './trial'
 
 const {values} = parseArgs({
@@ -146,7 +146,7 @@ if (values.benchmark) {
   logger.info('Writing benchmark output to: %s', env.outputPath)
   await fs.writeFile(env.outputPath, serializeBenchmarkOutput(getBenchmarkOutput(benchmark.id, sorted)), 'utf-8')
 
-  const resultSummaries = formatExperimentResults(benchmark.name, sorted)
+  const resultSummaries = formatBenchmarkResults(benchmark.name, sorted)
   console.log(resultSummaries)
 
   if (GITHUB_STEP_SUMMARY) {
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
index 533d873d..3d83ade6 100644
--- a/packages/agent-eval/src/experiment.test.ts
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -141,7 +141,7 @@ test('listExperiments throws when the directory does not exist', async () => {
       experimentsDirectory: '/experiments',
       scenariosDirectory: '/scenarios',
     }),
-  ).rejects.toThrowError('Experiments directory does not exist: /experiments')
+  ).rejects.toThrow('Experiments directory does not exist: /experiments')
 })
 
 test('listExperiments throws when the path is not a directory', async () => {
@@ -155,7 +155,7 @@ test('listExperiments throws when the path is not a directory', async () => {
       experimentsDirectory: '/experiments',
       scenariosDirectory: '/scenarios',
     }),
-  ).rejects.toThrowError('Experiments path is not a directory: /experiments')
+  ).rejects.toThrow('Experiments path is not a directory: /experiments')
 })
 
 test('getExperiment returns the experiment matching the id', async () => {
@@ -193,5 +193,5 @@ test('getExperiment throws when the experiment is not found', async () => {
       scenariosDirectory: '/scenarios',
       id: 'missing',
     }),
-  ).rejects.toThrowError('Experiment "missing" was not found in: /experiments')
+  ).rejects.toThrow('Experiment "missing" was not found in: /experiments')
 })
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 3f7bd870..005e38bd 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -2,7 +2,13 @@ import {randomUUID} from 'node:crypto'
 import path from 'node:path'
 import * as z from 'zod/mini'
 import type {EnvironmentConfig} from './environment'
-import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
+import {
+  getModelVariants,
+  ModelVariantConfigSchema,
+  ModelVariantSchema,
+  type ModelVariant,
+  type ModelVariantConfig,
+} from './model'
 import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
 import {create as createPlan, run as runPlan} from './plan'
@@ -11,6 +17,15 @@ import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment,
 import {TrialAgentSchema, TrialArtifactsSchema, WalkthroughSchema, type Trial, type TrialResult} from './trial'
 import {TestResultsSchema} from './vitest'
 
+type ExperimentConfig = {
+  name: string
+  description: string
+  models: ModelVariantConfig
+  scenarios: Array<string>
+  setup?: TreatmentSetup
+  treatments: Array<Treatment>
+}
+
 const ExperimentConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
@@ -18,9 +33,7 @@ const ExperimentConfigSchema = z.object({
   scenarios: z.array(z.string()),
   setup: z.optional(TreatmentSetupSchema),
   treatments: z.array(TreatmentSchema),
-})
-
-type ExperimentConfig = z.infer<typeof ExperimentConfigSchema>
+}) satisfies z.ZodMiniType<ExperimentConfig>
 
 function defineConfig(config: ExperimentConfig): ExperimentConfig {
   return config
diff --git a/packages/agent-eval/src/report.test.ts b/packages/agent-eval/src/report.test.ts
index 50adfb04..c64edbb0 100644
--- a/packages/agent-eval/src/report.test.ts
+++ b/packages/agent-eval/src/report.test.ts
@@ -1,6 +1,7 @@
 import {expect, test} from 'vitest'
+import type {BenchmarkTrialResult, Capability} from './benchmark'
 import type {TrialResult} from './trial'
-import {formatExperimentResults} from './report'
+import {formatBenchmarkResults, formatExperimentResults} from './report'
 
 function createResult({
   treatment,
@@ -123,3 +124,90 @@ test('formats trial results as an aggregated hierarchy', () => {
                                                    gpt-5.6-sol      medium            50.0%         1/2    1     1,250          3                 55.0s         35.0s   "
   `)
 })
+
+test('formats benchmark results as capability comparisons by scenario', () => {
+  const capabilities: Array<Capability> = [
+    {
+      name: 'Authoring',
+      scenarios: [],
+    },
+    {
+      name: 'Migration',
+      scenarios: [],
+    },
+  ]
+  const results: Array<BenchmarkTrialResult> = [
+    {
+      ...createResult({
+        treatment: 'Control',
+        scenario: 'create-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 1,
+        numTotalTests: 2,
+        sessions: [],
+      }),
+      capability: capabilities[0],
+    },
+    {
+      ...createResult({
+        treatment: 'Benchmark',
+        scenario: 'create-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 2,
+        numTotalTests: 2,
+        sessions: [],
+      }),
+      capability: capabilities[0],
+    },
+    {
+      ...createResult({
+        treatment: 'Control',
+        scenario: 'migrate-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 2,
+        numTotalTests: 2,
+        sessions: [],
+      }),
+      capability: capabilities[1],
+    },
+    {
+      ...createResult({
+        treatment: 'Benchmark',
+        scenario: 'migrate-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 1,
+        numTotalTests: 2,
+        sessions: [],
+      }),
+      capability: capabilities[1],
+    },
+  ]
+
+  const formatted = formatBenchmarkResults('Design system', results)
+    .split('\n')
+    .map(line => {
+      return line.trimEnd()
+    })
+    .join('\n')
+
+  expect(formatted).toMatchInlineSnapshot(`
+    "Benchmark      Capability  Scenario             Control  With benchmark  Delta     Control tests  Benchmark tests
+    -------------  ----------  -------------------  -------  --------------  --------  -------------  ---------------
+    Design system  Authoring   All scenarios        50.0%    100.0%          +50.0 pp  1/2            2/2
+                                 create-component   50.0%    100.0%          +50.0 pp  1/2            2/2
+    Design system  Migration   All scenarios        100.0%   50.0%           -50.0 pp  2/2            1/2
+                                 migrate-component  100.0%   50.0%           -50.0 pp  2/2            1/2"
+  `)
+})
diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts
index 560d7bd0..0b278bb7 100644
--- a/packages/agent-eval/src/report.ts
+++ b/packages/agent-eval/src/report.ts
@@ -1,4 +1,5 @@
 import type {Model, ReasoningEffort} from './model'
+import type {BenchmarkTrialResult} from './benchmark'
 import type {TrialResult} from './trial'
 
 type ResultSummary = {
@@ -36,6 +37,14 @@ type ResultHierarchy = Array<{
 
 type TableRow = Record<string, string | number>
 
+type BenchmarkComparison = {
+  benchmark: string
+  capability: string
+  scenario?: string
+  control: ResultSummary
+  benchmarkTreatment: ResultSummary
+}
+
 function createResultSummary(experiment: string, values: ResultSummaryValues = {}): ResultSummary {
   return {
     experiment,
@@ -225,6 +234,81 @@ function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario
   }
 }
 
+function formatPercentagePointDifference(value: number): string {
+  const sign = value > 0 ? '+' : ''
+  return `${sign}${(value * 100).toFixed(1)} pp`
+}
+
+function getBenchmarkComparisons(benchmark: string, results: Array<BenchmarkTrialResult>): Array<BenchmarkComparison> {
+  const comparisons = new Map<string, BenchmarkComparison>()
+
+  for (const result of results) {
+    const values = [
+      {
+        key: result.capability.name,
+        scenario: undefined,
+      },
+      {
+        key: `${result.capability.name}\0${result.trial.scenario.id}`,
+        scenario: result.trial.scenario.id,
+      },
+    ]
+
+    for (const value of values) {
+      const comparison = comparisons.get(value.key) ?? {
+        benchmark,
+        capability: result.capability.name,
+        scenario: value.scenario,
+        control: createResultSummary(benchmark),
+        benchmarkTreatment: createResultSummary(benchmark),
+      }
+      const summary = result.trial.treatment.name === 'Control' ? comparison.control : comparison.benchmarkTreatment
+      addResultToSummary(summary, result)
+      comparisons.set(value.key, comparison)
+    }
+  }
+
+  return [...comparisons.values()].toSorted((a, b) => {
+    return (
+      a.capability.localeCompare(b.capability) ||
+      Number(Boolean(a.scenario)) - Number(Boolean(b.scenario)) ||
+      (a.scenario ?? '').localeCompare(b.scenario ?? '')
+    )
+  })
+}
+
+function formatBenchmarkComparison(comparison: BenchmarkComparison): TableRow {
+  const controlSuccessRate = getSuccessRate(comparison.control)
+  const benchmarkSuccessRate = getSuccessRate(comparison.benchmarkTreatment)
+
+  return {
+    Benchmark: comparison.scenario ? '' : comparison.benchmark,
+    Capability: comparison.scenario ? '' : comparison.capability,
+    Scenario: comparison.scenario ? `  ${comparison.scenario}` : 'All scenarios',
+    Control: formatPercent(controlSuccessRate),
+    'With benchmark': formatPercent(benchmarkSuccessRate),
+    Delta: formatPercentagePointDifference(benchmarkSuccessRate - controlSuccessRate),
+    'Control tests': `${comparison.control.numPassedTests}/${comparison.control.numTotalTests}`,
+    'Benchmark tests': `${comparison.benchmarkTreatment.numPassedTests}/${comparison.benchmarkTreatment.numTotalTests}`,
+  }
+}
+
+function formatBenchmarkResults(benchmark: string, results: Array<BenchmarkTrialResult>): string {
+  const columns = [
+    'Benchmark',
+    'Capability',
+    'Scenario',
+    'Control',
+    'With benchmark',
+    'Delta',
+    'Control tests',
+    'Benchmark tests',
+  ]
+  const rows = getBenchmarkComparisons(benchmark, results).map(formatBenchmarkComparison)
+
+  return formatTable(rows, columns)
+}
+
 function formatExperimentResults(experiment: string, results: Array<TrialResult>): string {
   const columns = [
     'Experiment',
@@ -259,4 +343,4 @@ function formatExperimentResults(experiment: string, results: Array<TrialResult>
   return formatTable(rows, columns)
 }
 
-export {formatExperimentResults}
+export {formatBenchmarkResults, formatExperimentResults}

From 02f360f1c75beb813fc3bd8eb0e6ec8df9d28677 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 16:06:14 -0500
Subject: [PATCH 30/52] ci: add workflows for benchmark

---
 .github/workflows/benchmark.yml | 63 +++++++++++++++++++++++++++++++++
 .github/workflows/deploy.yml    |  5 +--
 script/run-benchmark.sh         | 26 ++++++++++++++
 3 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/benchmark.yml
 create mode 100755 script/run-benchmark.sh

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 00000000..4c77a905
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,63 @@
+name: benchmark
+on:
+  workflow_dispatch:
+    inputs:
+      benchmark:
+        description: Benchmark name to run
+        required: true
+        type: string
+      concurrency:
+        description: Number of trials to run in parallel
+        required: false
+        default: '1'
+        type: string
+      docker-image:
+        description: >-
+          Docker container image to use for running trials.
+          Must be a Debian-based Node image with apt-get and a node user (e.g. node:26.5.0-slim).
+        required: false
+        default: 'node:26.5.0-slim'
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.benchmark }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: build project
+        run: pnpm run build
+      - name: run benchmark
+        env:
+          BENCHMARK_NAME: ${{ inputs.benchmark }}
+          CONCURRENCY: ${{ inputs.concurrency }}
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+          DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
+        run: script/run-benchmark.sh
+      - name: upload benchmark results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: benchmark-${{ github.run_id }}
+          path: |
+            results
+            artifacts
+          if-no-files-found: error
+          retention-days: 90
+          compression-level: 9
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 4fb02fc3..2c230c6c 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -6,6 +6,7 @@ on:
   workflow_run:
     workflows:
       - baseline
+      - benchmark
     types:
       - completed
     branches:
@@ -31,13 +32,13 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.repository.default_branch }}
-      - name: download baseline results
+      - name: download evaluation results
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh api --paginate \
             "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[] | select(.expired == false and (.name | test("^baseline-[0-9]+$"))) | [.created_at, .id] | @tsv' |
+            --jq '.artifacts[] | select(.expired == false and (.name | test("^(baseline|benchmark)-[0-9]+$"))) | [.created_at, .id] | @tsv' |
             sort |
             cut -f2 |
             while read -r artifact_id; do
diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh
new file mode 100755
index 00000000..0a309523
--- /dev/null
+++ b/script/run-benchmark.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+benchmark_name="${BENCHMARK_NAME:?BENCHMARK_NAME is required}"
+run_date="${RUN_DATE:-$(date -u +%F)}"
+
+if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+  echo "BENCHMARK_NAME must be a benchmark file name without its extension" >&2
+  exit 1
+fi
+
+if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
+  echo "RUN_DATE must use the YYYY-MM-DD format" >&2
+  exit 1
+fi
+
+node "$repository_root/packages/agent-eval/dist/cli.js" \
+  --artifacts "$repository_root/artifacts" \
+  --benchmark "$benchmark_name" \
+  --benchmarks "$repository_root/benchmarks" \
+  --concurrency "${CONCURRENCY:-1}" \
+  --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \
+  --output "$repository_root/results/benchmarks/$benchmark_name/$run_date/output.json" \
+  --scenarios "$repository_root/scenarios"

From d3b45e1b27b2dff795a2a31be25fb69590533812 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 16:47:37 -0500
Subject: [PATCH 31/52] feat: switch up output format, change website

---
 .changeset/sixty-ways-taste.md                |   5 +
 .github/workflows/baseline.yml                |  19 +-
 .github/workflows/benchmark.yml               |  18 +-
 .github/workflows/experiment.yml              |  24 +-
 .gitignore                                    |   1 +
 packages/agent-eval/README.md                 |  23 ++
 packages/agent-eval/src/benchmark.test.ts     |  34 +-
 packages/agent-eval/src/benchmark.ts          |  43 ++-
 packages/agent-eval/src/cli.test.ts           |   8 +
 packages/agent-eval/src/cli.ts                |  25 +-
 packages/agent-eval/src/environment.test.ts   |  36 ++
 packages/agent-eval/src/environment.ts        |  14 +-
 .../agent-eval/src/experiment-output.test.ts  |  66 ++++
 packages/agent-eval/src/experiment.ts         |  28 +-
 packages/agent-eval/src/report.test.ts        | 159 +++++++--
 packages/agent-eval/src/report.ts             | 154 ++++++--
 packages/agent-eval/src/trial.ts              |  49 ++-
 script/run-baseline.sh                        |   5 +-
 script/run-benchmark.sh                       |   9 +-
 script/setup                                  |   5 +-
 .../app/benchmarks/[id]/components/Page.tsx   | 226 ++++++++++++
 website/src/app/benchmarks/[id]/page.tsx      |  47 +++
 .../app/benchmarks/[id]/runs/[date]/page.tsx  | 121 +++++++
 website/src/app/benchmarks/page.tsx           |  11 +
 website/src/app/components/PageHeader.tsx     |   7 +
 website/src/app/components/ResourceTables.tsx |  59 +++
 .../RunDetailsPage.tsx}                       |  59 +--
 .../app/experiments/[id]/runs/[date]/page.tsx | 265 ++------------
 website/src/benchmark-results.ts              | 257 ++++++++++++++
 website/src/benchmarks.ts                     |  53 +++
 website/src/run-details.ts                    | 335 ++++++++++++++++++
 31 files changed, 1767 insertions(+), 398 deletions(-)
 create mode 100644 .changeset/sixty-ways-taste.md
 create mode 100644 website/src/app/benchmarks/[id]/components/Page.tsx
 create mode 100644 website/src/app/benchmarks/[id]/page.tsx
 create mode 100644 website/src/app/benchmarks/[id]/runs/[date]/page.tsx
 create mode 100644 website/src/app/benchmarks/page.tsx
 rename website/src/app/{experiments/[id]/runs/[date]/components/Page.tsx => components/RunDetailsPage.tsx} (92%)
 create mode 100644 website/src/benchmark-results.ts
 create mode 100644 website/src/benchmarks.ts
 create mode 100644 website/src/run-details.ts

diff --git a/.changeset/sixty-ways-taste.md b/.changeset/sixty-ways-taste.md
new file mode 100644
index 00000000..f27b0da1
--- /dev/null
+++ b/.changeset/sixty-ways-taste.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add `--output-dir` for creating portable experiment and benchmark bundles containing `output.json` and its artifacts.
diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml
index df7b7cc7..12b5f0ed 100644
--- a/.github/workflows/baseline.yml
+++ b/.github/workflows/baseline.yml
@@ -50,14 +50,19 @@ jobs:
           RUN_DATE: ${{ needs.setup.outputs.run-date }}
           SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
         run: script/run-baseline.sh
+      - name: prepare baseline artifact
+        if: ${{ always() }}
+        run: |
+          mkdir -p workflow-artifact
+          if [[ -d results ]]; then
+            mv results workflow-artifact/results
+          fi
       - name: upload baseline results
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: baseline-shard-${{ github.run_id }}-${{ matrix.order }}
-          path: |
-            results
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 1
           compression-level: 9
@@ -88,13 +93,15 @@ jobs:
               ]
           ' "$output_directory"/output-*.json > "$output_directory/output.json"
           rm "$output_directory"/output-*.json
+      - name: prepare baseline artifact
+        run: |
+          mkdir -p workflow-artifact
+          mv results workflow-artifact/results
       - name: upload baseline results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: baseline-${{ github.run_id }}
-          path: |
-            results
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 90
           compression-level: 9
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 4c77a905..465ab09c 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -2,10 +2,6 @@ name: benchmark
 on:
   workflow_dispatch:
     inputs:
-      benchmark:
-        description: Benchmark name to run
-        required: true
-        type: string
       concurrency:
         description: Number of trials to run in parallel
         required: false
@@ -20,7 +16,7 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ inputs.benchmark }}
+  group: ${{ github.workflow }}
   cancel-in-progress: false
 
 permissions:
@@ -45,19 +41,23 @@ jobs:
         run: pnpm run build
       - name: run benchmark
         env:
-          BENCHMARK_NAME: ${{ inputs.benchmark }}
           CONCURRENCY: ${{ inputs.concurrency }}
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
         run: script/run-benchmark.sh
+      - name: prepare benchmark artifact
+        if: ${{ always() }}
+        run: |
+          mkdir -p workflow-artifact
+          if [[ -d results ]]; then
+            mv results workflow-artifact/results
+          fi
       - name: upload benchmark results
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: benchmark-${{ github.run_id }}
-          path: |
-            results
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 90
           compression-level: 9
diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml
index 75ef944f..66bb5342 100644
--- a/.github/workflows/experiment.yml
+++ b/.github/workflows/experiment.yml
@@ -53,7 +53,7 @@ jobs:
           CONCURRENCY: ${{ inputs.concurrency }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
           EXPERIMENT_NAME: ${{ inputs.experiment }}
-          OUTPUT_FILE: output-${{ matrix.order }}.json
+          OUTPUT_FILE: results/experiment/output-${{ matrix.order }}.json
           SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
         run: |
           set -o pipefail
@@ -63,17 +63,22 @@ jobs:
             --experiment "$EXPERIMENT_NAME" \
             --experiments experiments \
             --scenarios scenarios \
-            --artifacts artifacts \
+            --artifacts results/experiment/artifacts \
             --output "$OUTPUT_FILE" \
             --shard "$SHARD"
+      - name: Prepare experiment artifact
+        if: ${{ always() }}
+        run: |
+          mkdir -p workflow-artifact
+          if [[ -d results ]]; then
+            mv results workflow-artifact/results
+          fi
       - name: Upload experiment results
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: experiment-shard-${{ github.run_id }}-${{ matrix.order }}
-          path: |
-            output-${{ matrix.order }}.json
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 1
           compression-level: 9
@@ -89,6 +94,7 @@ jobs:
           merge-multiple: true
       - name: merge experiment results
         run: |
+          cd results/experiment
           jq --slurp '
             . as $outputs
             | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments
@@ -103,13 +109,15 @@ jobs:
               ]
           ' output-*.json > output.json
           rm output-*.json
+      - name: Prepare experiment artifact
+        run: |
+          mkdir -p workflow-artifact
+          mv results workflow-artifact/results
       - name: upload experiment results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: experiment
-          path: |
-            output.json
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 90
           compression-level: 9
diff --git a/.gitignore b/.gitignore
index 83298f2e..b0ba3844 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ out
 # Generated files
 artifacts/
 results/
+output.json
diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md
index 3556a3f5..da242a60 100644
--- a/packages/agent-eval/README.md
+++ b/packages/agent-eval/README.md
@@ -128,6 +128,29 @@ experiment. The experiments directory defaults to `./experiments`. Use
 `--scenarios` to set the directory containing scenario directories; it defaults
 to `./scenarios`.
 
+### Result bundles
+
+Keep the output file and artifacts in one directory so results can be moved
+between machines without rewriting paths:
+
+```text
+run/
+├── output.json
+└── artifacts/
+```
+
+```sh
+agent-eval \
+  --experiment example \
+  --output-dir run
+```
+
+Artifact and walkthrough references written by the CLI are relative to the
+directory containing `output.json`. Upload or download the complete `run`
+directory to preserve those references. `--output-dir` creates `output.json`
+and `artifacts/` within the selected directory. Use the existing `--output` and
+`--artifacts` options separately when a custom layout is required.
+
 ## Scenario config authoring
 
 Use `defineScenario` from `@primer/agent-eval/scenario` in each
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 93dcf3b7..4c764730 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -227,11 +227,11 @@ test('output serializes and deserializes benchmark capability metadata', () => {
   const trialResult: BenchmarkTrialResult = {
     capability,
     artifacts: {
-      directory: '/artifacts/trial',
-      copilotConfigDirectory: '/artifacts/trial/.copilot',
-      skillsConfigDirectory: '/artifacts/trial/.agents',
-      testResultsPath: '/artifacts/trial/workspace/test-results.json',
-      workspaceDirectory: '/artifacts/trial/workspace',
+      directory: '/bundle/artifacts/trial',
+      copilotConfigDirectory: '/bundle/artifacts/trial/.copilot',
+      skillsConfigDirectory: '/bundle/artifacts/trial/.agents',
+      testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json',
+      workspaceDirectory: '/bundle/artifacts/trial/workspace',
     },
     trial: {
       id: 'trial',
@@ -257,11 +257,17 @@ test('output serializes and deserializes benchmark capability metadata', () => {
       testResults: [],
     },
     walkthrough: {
-      type: 'Unavailable',
+      type: 'Screenshots',
+      screenshots: [
+        '/bundle/artifacts/trial/walkthrough/screenshots/01.png',
+        '/bundle/artifacts/trial/walkthrough/screenshots/02.png',
+      ],
     },
   }
 
-  const benchmarkOutput = output('test-benchmark', [trialResult])
+  const benchmarkOutput = output('test-benchmark', [trialResult], {
+    baseDirectory: '/bundle',
+  })
 
   expect(benchmarkOutput.benchmarkId).toBe('test-benchmark')
   expect(benchmarkOutput.capabilities.get('Test capability')).toEqual({
@@ -271,8 +277,22 @@ test('output serializes and deserializes benchmark capability metadata', () => {
   expect(benchmarkOutput.trials.get('trial')).toEqual(
     expect.objectContaining({
       capabilityId: 'Test capability',
+      artifacts: {
+        directory: 'artifacts/trial',
+        copilotConfigDirectory: 'artifacts/trial/.copilot',
+        skillsConfigDirectory: 'artifacts/trial/.agents',
+        testResultsPath: 'artifacts/trial/workspace/test-results.json',
+        workspaceDirectory: 'artifacts/trial/workspace',
+      },
       scenarioId: '001-scenario',
       treatmentId: 'Benchmark',
+      walkthrough: {
+        type: 'Screenshots',
+        screenshots: [
+          'artifacts/trial/walkthrough/screenshots/01.png',
+          'artifacts/trial/walkthrough/screenshots/02.png',
+        ],
+      },
     }),
   )
   expect(deserialize(serialize(benchmarkOutput))).toEqual(benchmarkOutput)
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 9ba61bd7..4dfc5d9c 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -4,16 +4,18 @@ import * as z from 'zod/mini'
 import type {EnvironmentConfig} from './environment'
 import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
-import {
-  getModelVariants,
-  ModelVariantConfigSchema,
-  ModelVariantSchema,
-  type ModelVariant,
-} from './model'
+import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
 import {create as createPlan, run as runPlan} from './plan'
 import {getScenario, ScenarioSchema, type Scenario} from './scenario'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type TreatmentSetup} from './treatment'
-import {TrialAgentSchema, TrialArtifactsSchema, WalkthroughSchema, type Trial, type TrialResult} from './trial'
+import {
+  getPortableTrialPaths,
+  TrialAgentSchema,
+  TrialArtifactsSchema,
+  WalkthroughSchema,
+  type Trial,
+  type TrialResult,
+} from './trial'
 import {TestResultsSchema} from './vitest'
 
 const CapabilityConfigSchema = z.object({
@@ -271,7 +273,15 @@ type BenchmarkOutput = {
   trials: Map<string, z.infer<typeof BenchmarkTrialOutputSchema>>
 }
 
-function output(benchmarkId: string, trialResults: BenchmarkRunResult): BenchmarkOutput {
+type BenchmarkOutputOptions = {
+  baseDirectory?: string
+}
+
+function output(
+  benchmarkId: string,
+  trialResults: BenchmarkRunResult,
+  options: BenchmarkOutputOptions = {},
+): BenchmarkOutput {
   const result: BenchmarkOutput = {
     benchmarkId,
     capabilities: new Map(),
@@ -281,7 +291,10 @@ function output(benchmarkId: string, trialResults: BenchmarkRunResult): Benchmar
   }
 
   for (const trialResult of trialResults) {
-    const {artifacts, capability, trial} = trialResult
+    const {capability, trial} = trialResult
+    const {artifacts, walkthrough} = options.baseDirectory
+      ? getPortableTrialPaths(trialResult, options.baseDirectory)
+      : trialResult
 
     if (!result.capabilities.has(capability.name)) {
       result.capabilities.set(capability.name, {
@@ -307,7 +320,7 @@ function output(benchmarkId: string, trialResults: BenchmarkRunResult): Benchmar
       scenarioId: trial.scenario.id,
       testResults: trialResult.testResults,
       treatmentId: trial.treatment.name,
-      walkthrough: trialResult.walkthrough,
+      walkthrough,
     })
   }
 
@@ -338,4 +351,12 @@ function deserialize(input: unknown): BenchmarkOutput {
 }
 
 export {BenchmarkConfigSchema, defineConfig, deserialize, getBenchmark, listBenchmarks, output, run, serialize}
-export type {BenchmarkConfig, Benchmark, BenchmarkOutput, BenchmarkRunResult, BenchmarkTrialResult, Capability}
+export type {
+  BenchmarkConfig,
+  Benchmark,
+  BenchmarkOutput,
+  BenchmarkOutputOptions,
+  BenchmarkRunResult,
+  BenchmarkTrialResult,
+  Capability,
+}
diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts
index a55e8dcb..6c2427a9 100644
--- a/packages/agent-eval/src/cli.test.ts
+++ b/packages/agent-eval/src/cli.test.ts
@@ -24,6 +24,7 @@ describe('cli', () => {
 
     await expect(import('./cli')).rejects.toThrow('process.exit')
     expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]'))
+    expect(log).toHaveBeenCalledWith(expect.stringContaining('--output-dir <dir>'))
   })
 
   test('requires a Copilot token before running', async () => {
@@ -44,4 +45,11 @@ describe('cli', () => {
 
     expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]'))
   })
+
+  test('rejects output directory combinations with explicit artifact paths', async () => {
+    process.argv = ['node', 'agent-eval', '--output-dir', 'results/run', '--artifacts', 'artifacts']
+    process.env.COPILOT_GITHUB_TOKEN = 'token'
+
+    await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --artifacts or --output')
+  })
 })
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index f31d1e81..42e6ac4f 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -68,7 +68,10 @@ const {values} = parseArgs({
     output: {
       type: 'string',
       description: 'The target file in which results are written',
-      default: 'output.json',
+    },
+    'output-dir': {
+      type: 'string',
+      description: 'The directory containing output.json and its artifacts',
     },
     scenarios: {
       type: 'string',
@@ -92,6 +95,7 @@ Options:
   -h, --help                 Learn more about the command and its options
       --log-level <level>    The log level to use (default: info)
       --output <file>        The target file in which results are written (default: output.json)
+      --output-dir <dir>     The directory containing output.json and its artifacts
       --scenarios <dir>      The directory containing scenario directories (default: ./scenarios)
 `)
 }
@@ -119,6 +123,7 @@ const env = getEnvironmentConfig({
   copilotToken: COPILOT_GITHUB_TOKEN,
   dockerImage: values['docker-image']?.trim(),
   experimentsDirectory: values.experiments,
+  outputDirectory: values['output-dir'],
   outputPath: values.output,
   scenariosDirectory: values.scenarios,
 })
@@ -144,9 +149,17 @@ if (values.benchmark) {
   }
 
   logger.info('Writing benchmark output to: %s', env.outputPath)
-  await fs.writeFile(env.outputPath, serializeBenchmarkOutput(getBenchmarkOutput(benchmark.id, sorted)), 'utf-8')
-
-  const resultSummaries = formatBenchmarkResults(benchmark.name, sorted)
+  await fs.writeFile(
+    env.outputPath,
+    serializeBenchmarkOutput(
+      getBenchmarkOutput(benchmark.id, sorted, {
+        baseDirectory: path.dirname(env.outputPath),
+      }),
+    ),
+    'utf-8',
+  )
+
+  const resultSummaries = formatBenchmarkResults(benchmark, sorted)
   console.log(resultSummaries)
 
   if (GITHUB_STEP_SUMMARY) {
@@ -168,7 +181,9 @@ if (values.benchmark) {
 
   logger.info('Writing experiment output to: %s', env.outputPath)
 
-  const output = getExperimentOutput(experiment.id, sorted)
+  const output = getExperimentOutput(experiment.id, sorted, {
+    baseDirectory: path.dirname(env.outputPath),
+  })
 
   if (!existsSync(path.dirname(env.outputPath))) {
     await fs.mkdir(path.dirname(env.outputPath), {recursive: true})
diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts
index 32f4c189..33a1c0f0 100644
--- a/packages/agent-eval/src/environment.test.ts
+++ b/packages/agent-eval/src/environment.test.ts
@@ -45,6 +45,42 @@ describe('getEnvironmentConfig', () => {
     })
   })
 
+  test('derives the output and artifacts paths from an output directory', () => {
+    expect(
+      getEnvironmentConfig({
+        copilotToken: 'token',
+        outputDirectory: './results/run',
+      }),
+    ).toEqual({
+      artifactsDirectory: path.resolve('results/run/artifacts'),
+      benchmarksDirectory: path.resolve('benchmarks'),
+      concurrency: 1,
+      copilotToken: 'token',
+      dockerImage: DEFAULT_DOCKER_IMAGE,
+      experimentsDirectory: path.resolve('experiments'),
+      outputPath: path.resolve('results/run/output.json'),
+      scenariosDirectory: path.resolve('scenarios'),
+    })
+  })
+
+  test('rejects output directory combinations with explicit output paths', () => {
+    expect(() => {
+      getEnvironmentConfig({
+        artifactsDirectory: './artifacts',
+        copilotToken: 'token',
+        outputDirectory: './results/run',
+      })
+    }).toThrow('--output-dir cannot be combined with --artifacts or --output')
+
+    expect(() => {
+      getEnvironmentConfig({
+        copilotToken: 'token',
+        outputDirectory: './results/run',
+        outputPath: './output.json',
+      })
+    }).toThrow('--output-dir cannot be combined with --artifacts or --output')
+  })
+
   test.each(['0', '-1', 'invalid', '1.5'])('falls back to one for invalid concurrency %s', concurrency => {
     expect(
       getEnvironmentConfig({
diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts
index 6bf65893..66b77ec9 100644
--- a/packages/agent-eval/src/environment.ts
+++ b/packages/agent-eval/src/environment.ts
@@ -19,12 +19,20 @@ type EnvironmentOptions = {
   copilotToken: string
   dockerImage?: string
   experimentsDirectory?: string
+  outputDirectory?: string
   outputPath?: string
   scenariosDirectory?: string
 }
 
 function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig {
-  const artifactsDirectory = path.resolve(options.artifactsDirectory ?? 'artifacts')
+  if (options.outputDirectory && (options.artifactsDirectory || options.outputPath)) {
+    throw new Error('--output-dir cannot be combined with --artifacts or --output')
+  }
+
+  const outputDirectory = options.outputDirectory ? path.resolve(options.outputDirectory) : undefined
+  const artifactsDirectory = outputDirectory
+    ? path.join(outputDirectory, 'artifacts')
+    : path.resolve(options.artifactsDirectory ?? 'artifacts')
   const benchmarksDirectory = path.resolve(options.benchmarksDirectory ?? 'benchmarks')
   const parsedConcurrency = options.concurrency ? parseInt(options.concurrency, 10) : 1
   const concurrency =
@@ -32,7 +40,9 @@ function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig {
       ? parsedConcurrency
       : 1
   const experimentsDirectory = path.resolve(options.experimentsDirectory ?? 'experiments')
-  const outputPath = path.resolve(options.outputPath ?? 'output.json')
+  const outputPath = outputDirectory
+    ? path.join(outputDirectory, 'output.json')
+    : path.resolve(options.outputPath ?? 'output.json')
   const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios')
 
   return {
diff --git a/packages/agent-eval/src/experiment-output.test.ts b/packages/agent-eval/src/experiment-output.test.ts
index 8ea03b3d..e93dc809 100644
--- a/packages/agent-eval/src/experiment-output.test.ts
+++ b/packages/agent-eval/src/experiment-output.test.ts
@@ -1,5 +1,6 @@
 import {expect, test} from 'vitest'
 import {deserialize, output, serialize} from './experiment'
+import type {TrialResult} from './trial'
 
 test('serializes and deserializes experiment identity and result maps', () => {
   const experimentOutput = output('baseline', [])
@@ -18,3 +19,68 @@ test('serializes and deserializes experiment identity and result maps', () => {
     trials: new Map(),
   })
 })
+
+test('creates portable artifact paths relative to the output directory', () => {
+  const trialResult: TrialResult = {
+    artifacts: {
+      directory: '/bundle/artifacts/trial',
+      copilotConfigDirectory: '/bundle/artifacts/trial/.copilot',
+      skillsConfigDirectory: '/bundle/artifacts/trial/.agents',
+      testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json',
+      workspaceDirectory: '/bundle/artifacts/trial/workspace',
+    },
+    trial: {
+      id: 'trial',
+      scenario: {
+        id: 'scenario',
+        directory: '/scenarios/scenario',
+        prompt: 'Complete the task',
+        tags: [],
+        testPath: '/scenarios/scenario/scenario.test.ts',
+      },
+      treatment: {
+        name: 'Control',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    },
+    agent: {
+      sessions: [],
+    },
+    testResults: {
+      numTotalTests: 1,
+      numPassedTests: 1,
+      numFailedTests: 0,
+      numPendingTests: 0,
+      numTodoTests: 0,
+      success: true,
+      testResults: [],
+    },
+    walkthrough: {
+      type: 'Screenshot',
+      filepath: '/bundle/artifacts/trial/walkthrough/screenshot.png',
+    },
+  }
+
+  const portableOutput = output('baseline', [trialResult], {
+    baseDirectory: '/bundle',
+  })
+
+  expect(portableOutput.trials.get('trial')).toEqual(
+    expect.objectContaining({
+      artifacts: {
+        directory: 'artifacts/trial',
+        copilotConfigDirectory: 'artifacts/trial/.copilot',
+        skillsConfigDirectory: 'artifacts/trial/.agents',
+        testResultsPath: 'artifacts/trial/workspace/test-results.json',
+        workspaceDirectory: 'artifacts/trial/workspace',
+      },
+      walkthrough: {
+        type: 'Screenshot',
+        filepath: 'artifacts/trial/walkthrough/screenshot.png',
+      },
+    }),
+  )
+})
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 005e38bd..7667971d 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -14,7 +14,14 @@ import {logger} from './logger'
 import {create as createPlan, run as runPlan} from './plan'
 import {getScenario, ScenarioSchema, type Scenario} from './scenario'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
-import {TrialAgentSchema, TrialArtifactsSchema, WalkthroughSchema, type Trial, type TrialResult} from './trial'
+import {
+  getPortableTrialPaths,
+  TrialAgentSchema,
+  TrialArtifactsSchema,
+  WalkthroughSchema,
+  type Trial,
+  type TrialResult,
+} from './trial'
 import {TestResultsSchema} from './vitest'
 
 type ExperimentConfig = {
@@ -230,6 +237,10 @@ type ExperimentOutput = {
   trials: Map<string, z.infer<typeof ExperimentOutputTrialSchema>>
 }
 
+type ExperimentOutputOptions = {
+  baseDirectory?: string
+}
+
 const SerializedExperimentOutputSchema = z.object({
   experimentId: z.string(),
   scenarios: z.record(z.string(), ExperimentOutputScenarioSchema),
@@ -237,7 +248,11 @@ const SerializedExperimentOutputSchema = z.object({
   trials: z.record(z.string(), ExperimentOutputTrialSchema),
 })
 
-function output(experimentId: string, trialResults: ExperimentRunResult): ExperimentOutput {
+function output(
+  experimentId: string,
+  trialResults: ExperimentRunResult,
+  options: ExperimentOutputOptions = {},
+): ExperimentOutput {
   const result: ExperimentOutput = {
     experimentId,
     scenarios: new Map(),
@@ -246,7 +261,10 @@ function output(experimentId: string, trialResults: ExperimentRunResult): Experi
   }
 
   for (const trialResult of trialResults) {
-    const {artifacts, trial} = trialResult
+    const {trial} = trialResult
+    const {artifacts, walkthrough} = options.baseDirectory
+      ? getPortableTrialPaths(trialResult, options.baseDirectory)
+      : trialResult
 
     if (!result.scenarios.has(trial.scenario.id)) {
       result.scenarios.set(trial.scenario.id, trial.scenario)
@@ -264,7 +282,7 @@ function output(experimentId: string, trialResults: ExperimentRunResult): Experi
       scenarioId: trial.scenario.id,
       testResults: trialResult.testResults,
       treatmentId: trial.treatment.name,
-      walkthrough: trialResult.walkthrough,
+      walkthrough,
     })
   }
 
@@ -293,4 +311,4 @@ function deserialize(input: unknown): ExperimentOutput {
 }
 
 export {ExperimentConfigSchema, defineConfig, deserialize, getExperiment, listExperiments, output, run, serialize}
-export type {ExperimentConfig, Experiment, ExperimentOutput}
+export type {ExperimentConfig, Experiment, ExperimentOutput, ExperimentOutputOptions}
diff --git a/packages/agent-eval/src/report.test.ts b/packages/agent-eval/src/report.test.ts
index c64edbb0..19d85876 100644
--- a/packages/agent-eval/src/report.test.ts
+++ b/packages/agent-eval/src/report.test.ts
@@ -128,12 +128,28 @@ test('formats trial results as an aggregated hierarchy', () => {
 test('formats benchmark results as capability comparisons by scenario', () => {
   const capabilities: Array<Capability> = [
     {
-      name: 'Authoring',
-      scenarios: [],
+      name: 'Migration',
+      scenarios: [
+        {
+          id: 'migrate-component',
+          directory: '/scenarios/migrate-component',
+          prompt: 'Complete the task',
+          tags: [],
+          testPath: '/scenarios/migrate-component/scenario.test.ts',
+        },
+      ],
     },
     {
-      name: 'Migration',
-      scenarios: [],
+      name: 'Authoring',
+      scenarios: [
+        {
+          id: 'create-component',
+          directory: '/scenarios/create-component',
+          prompt: 'Complete the task',
+          tags: [],
+          testPath: '/scenarios/create-component/scenario.test.ts',
+        },
+      ],
     },
   ]
   const results: Array<BenchmarkTrialResult> = [
@@ -147,9 +163,19 @@ test('formats benchmark results as capability comparisons by scenario', () => {
         },
         numPassedTests: 1,
         numTotalTests: 2,
-        sessions: [],
+        sessions: [
+          {
+            turns: 2,
+            outputTokens: 1_000,
+            premiumRequests: 1,
+            totalApiDurationMs: 30_000,
+            sessionDurationMs: 45_000,
+            tools: {},
+            messages: [],
+          },
+        ],
       }),
-      capability: capabilities[0],
+      capability: capabilities[1],
     },
     {
       ...createResult({
@@ -161,41 +187,125 @@ test('formats benchmark results as capability comparisons by scenario', () => {
         },
         numPassedTests: 2,
         numTotalTests: 2,
-        sessions: [],
+        sessions: [
+          {
+            turns: 1,
+            outputTokens: 800,
+            premiumRequests: 2,
+            totalApiDurationMs: 20_000,
+            sessionDurationMs: 35_000,
+            tools: {},
+            messages: [],
+          },
+        ],
       }),
-      capability: capabilities[0],
+      capability: capabilities[1],
+    },
+    {
+      ...createResult({
+        treatment: 'Control',
+        scenario: 'create-component',
+        model: {
+          name: 'claude-sonnet-5',
+          reasoningEffort: 'high',
+        },
+        numPassedTests: 0,
+        numTotalTests: 2,
+        sessions: [
+          {
+            turns: 1,
+            outputTokens: 700,
+            premiumRequests: 1,
+            totalApiDurationMs: 20_000,
+            sessionDurationMs: 30_000,
+            tools: {},
+            messages: [],
+          },
+        ],
+      }),
+      capability: capabilities[1],
+    },
+    {
+      ...createResult({
+        treatment: 'Benchmark',
+        scenario: 'create-component',
+        model: {
+          name: 'claude-sonnet-5',
+          reasoningEffort: 'high',
+        },
+        numPassedTests: 0,
+        numTotalTests: 2,
+        sessions: [
+          {
+            turns: 1,
+            outputTokens: 700,
+            premiumRequests: 1,
+            totalApiDurationMs: 15_000,
+            sessionDurationMs: 25_000,
+            tools: {},
+            messages: [],
+          },
+        ],
+      }),
+      capability: capabilities[1],
     },
     {
       ...createResult({
         treatment: 'Control',
         scenario: 'migrate-component',
         model: {
-          name: 'gpt-5.6-sol',
-          reasoningEffort: 'medium',
+          name: 'claude-sonnet-5',
+          reasoningEffort: 'high',
         },
         numPassedTests: 2,
         numTotalTests: 2,
-        sessions: [],
+        sessions: [
+          {
+            turns: 2,
+            outputTokens: 900,
+            premiumRequests: 2,
+            totalApiDurationMs: 40_000,
+            sessionDurationMs: 60_000,
+            tools: {},
+            messages: [],
+          },
+        ],
       }),
-      capability: capabilities[1],
+      capability: capabilities[0],
     },
     {
       ...createResult({
         treatment: 'Benchmark',
         scenario: 'migrate-component',
         model: {
-          name: 'gpt-5.6-sol',
-          reasoningEffort: 'medium',
+          name: 'claude-sonnet-5',
+          reasoningEffort: 'high',
         },
         numPassedTests: 1,
         numTotalTests: 2,
-        sessions: [],
+        sessions: [
+          {
+            turns: 3,
+            outputTokens: 1_100,
+            premiumRequests: 3,
+            totalApiDurationMs: 50_000,
+            sessionDurationMs: 75_000,
+            tools: {},
+            messages: [],
+          },
+        ],
       }),
-      capability: capabilities[1],
+      capability: capabilities[0],
     },
   ]
 
-  const formatted = formatBenchmarkResults('Design system', results)
+  const formatted = formatBenchmarkResults(
+    {
+      name: 'Design system',
+      capabilities,
+    },
+    results,
+  )
     .split('\n')
     .map(line => {
       return line.trimEnd()
@@ -203,11 +313,14 @@ test('formats benchmark results as capability comparisons by scenario', () => {
     .join('\n')
 
   expect(formatted).toMatchInlineSnapshot(`
-    "Benchmark      Capability  Scenario             Control  With benchmark  Delta     Control tests  Benchmark tests
-    -------------  ----------  -------------------  -------  --------------  --------  -------------  ---------------
-    Design system  Authoring   All scenarios        50.0%    100.0%          +50.0 pp  1/2            2/2
-                                 create-component   50.0%    100.0%          +50.0 pp  1/2            2/2
-    Design system  Migration   All scenarios        100.0%   50.0%           -50.0 pp  2/2            1/2
-                                 migrate-component  100.0%   50.0%           -50.0 pp  2/2            1/2"
+    "Benchmark      Capability  Scenario             Model                Reasoning Effort  Tests          Output Tokens   Premium Requests  Session Time       API Time
+    -------------  ----------  -------------------  -------------------  ----------------  -------------  --------------  ----------------  -----------------  --------------
+    Design system  Migration   All scenarios        All models                             1/2 (-50.0%)   1,100 (+22.2%)  3 (+50.0%)        1m 15.0s (+25.0%)  50.0s (+25.0%)
+                                 migrate-component  All models                             1/2 (-50.0%)   1,100 (+22.2%)  3 (+50.0%)        1m 15.0s (+25.0%)  50.0s (+25.0%)
+                                                        claude-sonnet-5  high              1/2 (-50.0%)   1,100 (+22.2%)  3 (+50.0%)        1m 15.0s (+25.0%)  50.0s (+25.0%)
+    Design system  Authoring   All scenarios        All models                             2/4 (+100.0%)  1,500 (-11.8%)  3 (+50.0%)        1m 0.0s (-20.0%)   35.0s (-30.0%)
+                                 create-component   All models                             2/4 (+100.0%)  1,500 (-11.8%)  3 (+50.0%)        1m 0.0s (-20.0%)   35.0s (-30.0%)
+                                                        gpt-5.6-sol      medium            2/2 (+100.0%)  800 (-20.0%)    2 (+100.0%)       35.0s (-22.2%)     20.0s (-33.3%)
+                                                        claude-sonnet-5  high              0/2 (0.0%)     700 (0.0%)      1 (0.0%)          25.0s (-16.7%)     15.0s (-25.0%)"
   `)
 })
diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts
index 0b278bb7..cc7be24e 100644
--- a/packages/agent-eval/src/report.ts
+++ b/packages/agent-eval/src/report.ts
@@ -1,5 +1,5 @@
 import type {Model, ReasoningEffort} from './model'
-import type {BenchmarkTrialResult} from './benchmark'
+import type {Benchmark, BenchmarkTrialResult} from './benchmark'
 import type {TrialResult} from './trial'
 
 type ResultSummary = {
@@ -41,6 +41,8 @@ type BenchmarkComparison = {
   benchmark: string
   capability: string
   scenario?: string
+  model?: Model
+  reasoningEffort?: ReasoningEffort<Model>
   control: ResultSummary
   benchmarkTreatment: ResultSummary
 }
@@ -234,12 +236,34 @@ function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario
   }
 }
 
-function formatPercentagePointDifference(value: number): string {
-  const sign = value > 0 ? '+' : ''
-  return `${sign}${(value * 100).toFixed(1)} pp`
+function formatPercentDelta(control: number, benchmark: number): string {
+  if (control === 0) {
+    return benchmark === 0 ? '0.0%' : 'N/A'
+  }
+
+  const delta = (benchmark - control) / control
+  const sign = delta > 0 ? '+' : ''
+  return `${sign}${(delta * 100).toFixed(1)}%`
+}
+
+function formatBenchmarkValue(value: string, control: number, benchmark: number): string {
+  return `${value} (${formatPercentDelta(control, benchmark)})`
+}
+
+function compareBenchmarkModelPerformance(a: ResultSummary, b: ResultSummary): number {
+  return (
+    getSuccessRate(b) - getSuccessRate(a) ||
+    a.outputTokens - b.outputTokens ||
+    a.premiumRequests - b.premiumRequests ||
+    a.sessionDurationMs - b.sessionDurationMs ||
+    a.totalApiDurationMs - b.totalApiDurationMs
+  )
 }
 
-function getBenchmarkComparisons(benchmark: string, results: Array<BenchmarkTrialResult>): Array<BenchmarkComparison> {
+function getBenchmarkComparisons(
+  benchmark: Pick<Benchmark, 'name' | 'capabilities'>,
+  results: Array<BenchmarkTrialResult>,
+): Array<BenchmarkComparison> {
   const comparisons = new Map<string, BenchmarkComparison>()
 
   for (const result of results) {
@@ -252,13 +276,26 @@ function getBenchmarkComparisons(benchmark: string, results: Array<BenchmarkTria
         key: `${result.capability.name}\0${result.trial.scenario.id}`,
         scenario: result.trial.scenario.id,
       },
+      {
+        key: [
+          result.capability.name,
+          result.trial.scenario.id,
+          result.trial.model.name,
+          result.trial.model.reasoningEffort,
+        ].join('\0'),
+        scenario: result.trial.scenario.id,
+        model: result.trial.model.name,
+        reasoningEffort: result.trial.model.reasoningEffort,
+      },
     ]
 
     for (const value of values) {
       const comparison = comparisons.get(value.key) ?? {
-        benchmark,
+        benchmark: benchmark.name,
         capability: result.capability.name,
         scenario: value.scenario,
+        model: value.model,
+        reasoningEffort: value.reasoningEffort,
         control: createResultSummary(benchmark),
         benchmarkTreatment: createResultSummary(benchmark),
       }
@@ -268,41 +305,112 @@ function getBenchmarkComparisons(benchmark: string, results: Array<BenchmarkTria
     }
   }
 
+  const capabilityOrder = new Map(
+    benchmark.capabilities.map((capability, index) => {
+      return [capability.name, index]
+    }),
+  )
+  const scenarioOrder = new Map(
+    benchmark.capabilities.map(capability => {
+      return [
+        capability.name,
+        new Map(
+          capability.scenarios.map((scenario, index) => {
+            return [scenario.id, index]
+          }),
+        ),
+      ]
+    }),
+  )
+
   return [...comparisons.values()].toSorted((a, b) => {
-    return (
-      a.capability.localeCompare(b.capability) ||
+    const capabilityDifference =
+      (capabilityOrder.get(a.capability) ?? Number.MAX_SAFE_INTEGER) -
+      (capabilityOrder.get(b.capability) ?? Number.MAX_SAFE_INTEGER)
+    if (capabilityDifference !== 0) {
+      return capabilityDifference
+    }
+
+    const scenarioDifference =
       Number(Boolean(a.scenario)) - Number(Boolean(b.scenario)) ||
+      (scenarioOrder.get(a.capability)?.get(a.scenario ?? '') ?? Number.MAX_SAFE_INTEGER) -
+        (scenarioOrder.get(b.capability)?.get(b.scenario ?? '') ?? Number.MAX_SAFE_INTEGER) ||
       (a.scenario ?? '').localeCompare(b.scenario ?? '')
+    if (scenarioDifference !== 0) {
+      return scenarioDifference
+    }
+
+    const modelDifference = Number(Boolean(a.model)) - Number(Boolean(b.model))
+    if (modelDifference !== 0) {
+      return modelDifference
+    }
+
+    if (a.model && b.model) {
+      return (
+        compareBenchmarkModelPerformance(a.benchmarkTreatment, b.benchmarkTreatment) ||
+        a.model.localeCompare(b.model) ||
+        (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
+      )
+    }
+
+    return (
+      a.capability.localeCompare(b.capability) ||
+      (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
+      (a.model ?? '').localeCompare(b.model ?? '')
     )
   })
 }
 
 function formatBenchmarkComparison(comparison: BenchmarkComparison): TableRow {
-  const controlSuccessRate = getSuccessRate(comparison.control)
-  const benchmarkSuccessRate = getSuccessRate(comparison.benchmarkTreatment)
-
   return {
     Benchmark: comparison.scenario ? '' : comparison.benchmark,
     Capability: comparison.scenario ? '' : comparison.capability,
-    Scenario: comparison.scenario ? `  ${comparison.scenario}` : 'All scenarios',
-    Control: formatPercent(controlSuccessRate),
-    'With benchmark': formatPercent(benchmarkSuccessRate),
-    Delta: formatPercentagePointDifference(benchmarkSuccessRate - controlSuccessRate),
-    'Control tests': `${comparison.control.numPassedTests}/${comparison.control.numTotalTests}`,
-    'Benchmark tests': `${comparison.benchmarkTreatment.numPassedTests}/${comparison.benchmarkTreatment.numTotalTests}`,
+    Scenario: comparison.model ? '' : comparison.scenario ? `  ${comparison.scenario}` : 'All scenarios',
+    Model: comparison.model ? `    ${comparison.model}` : 'All models',
+    'Reasoning Effort': comparison.reasoningEffort ?? '',
+    Tests: formatBenchmarkValue(
+      `${comparison.benchmarkTreatment.numPassedTests}/${comparison.benchmarkTreatment.numTotalTests}`,
+      comparison.control.numPassedTests,
+      comparison.benchmarkTreatment.numPassedTests,
+    ),
+    'Output Tokens': formatBenchmarkValue(
+      formatNumber(comparison.benchmarkTreatment.outputTokens),
+      comparison.control.outputTokens,
+      comparison.benchmarkTreatment.outputTokens,
+    ),
+    'Premium Requests': formatBenchmarkValue(
+      formatNumber(comparison.benchmarkTreatment.premiumRequests),
+      comparison.control.premiumRequests,
+      comparison.benchmarkTreatment.premiumRequests,
+    ),
+    'Session Time': formatBenchmarkValue(
+      formatDuration(comparison.benchmarkTreatment.sessionDurationMs),
+      comparison.control.sessionDurationMs,
+      comparison.benchmarkTreatment.sessionDurationMs,
+    ),
+    'API Time': formatBenchmarkValue(
+      formatDuration(comparison.benchmarkTreatment.totalApiDurationMs),
+      comparison.control.totalApiDurationMs,
+      comparison.benchmarkTreatment.totalApiDurationMs,
+    ),
   }
 }
 
-function formatBenchmarkResults(benchmark: string, results: Array<BenchmarkTrialResult>): string {
+function formatBenchmarkResults(
+  benchmark: Pick<Benchmark, 'name' | 'capabilities'>,
+  results: Array<BenchmarkTrialResult>,
+): string {
   const columns = [
     'Benchmark',
     'Capability',
     'Scenario',
-    'Control',
-    'With benchmark',
-    'Delta',
-    'Control tests',
-    'Benchmark tests',
+    'Model',
+    'Reasoning Effort',
+    'Tests',
+    'Output Tokens',
+    'Premium Requests',
+    'Session Time',
+    'API Time',
   ]
   const rows = getBenchmarkComparisons(benchmark, results).map(formatBenchmarkComparison)
 
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index 1ac8627a..e7faafbd 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -64,6 +64,44 @@ const TrialResultSchema = z.object({
 
 type TrialResult = z.infer<typeof TrialResultSchema>
 
+type PortableTrialPaths = Pick<TrialResult, 'artifacts' | 'walkthrough'>
+
+function getPortableTrialPaths(result: TrialResult, baseDirectory: string): PortableTrialPaths {
+  const toPortablePath = (filepath: string): string => {
+    if (!path.isAbsolute(filepath)) {
+      return filepath.split(path.sep).join(path.posix.sep)
+    }
+
+    return path.relative(baseDirectory, filepath).split(path.sep).join(path.posix.sep)
+  }
+
+  let walkthrough: Walkthrough
+  if (result.walkthrough.type === 'Screenshots') {
+    walkthrough = {
+      type: 'Screenshots',
+      screenshots: result.walkthrough.screenshots.map(toPortablePath),
+    }
+  } else if (result.walkthrough.type === 'Screenshot' || result.walkthrough.type === 'Video') {
+    walkthrough = {
+      ...result.walkthrough,
+      filepath: toPortablePath(result.walkthrough.filepath),
+    }
+  } else {
+    walkthrough = result.walkthrough
+  }
+
+  return {
+    artifacts: {
+      directory: toPortablePath(result.artifacts.directory),
+      copilotConfigDirectory: toPortablePath(result.artifacts.copilotConfigDirectory),
+      skillsConfigDirectory: toPortablePath(result.artifacts.skillsConfigDirectory),
+      testResultsPath: toPortablePath(result.artifacts.testResultsPath),
+      workspaceDirectory: toPortablePath(result.artifacts.workspaceDirectory),
+    },
+    walkthrough,
+  }
+}
+
 async function run({
   artifactsDirectory,
   copilotToken,
@@ -414,5 +452,14 @@ function compare(a: CompareTrialResult, b: CompareTrialResult): number {
   )
 }
 
-export {TrialSchema, TrialResultSchema, TrialArtifactsSchema, TrialAgentSchema, WalkthroughSchema, run, compare}
+export {
+  TrialSchema,
+  TrialResultSchema,
+  TrialArtifactsSchema,
+  TrialAgentSchema,
+  WalkthroughSchema,
+  run,
+  compare,
+  getPortableTrialPaths,
+}
 export type {Trial, TrialResult}
diff --git a/script/run-baseline.sh b/script/run-baseline.sh
index acc4f35e..68cefe88 100755
--- a/script/run-baseline.sh
+++ b/script/run-baseline.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 run_date="${RUN_DATE:-$(date -u +%F)}"
 output_file="${OUTPUT_FILE:-output.json}"
+run_directory="$repository_root/results/$run_date"
 
 if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
   echo "RUN_DATE must use the YYYY-MM-DD format" >&2
@@ -12,12 +13,12 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
 fi
 
 arguments=(
-  --artifacts "$repository_root/artifacts"
+  --artifacts "$run_directory/artifacts"
   --concurrency "${CONCURRENCY:-1}"
   --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}"
   --experiment baseline
   --experiments "$repository_root/experiments"
-  --output "$repository_root/results/$run_date/$output_file"
+  --output "$run_directory/$output_file"
   --scenarios "$repository_root/scenarios"
 )
 
diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh
index 0a309523..dbf10527 100755
--- a/script/run-benchmark.sh
+++ b/script/run-benchmark.sh
@@ -3,8 +3,9 @@
 set -euo pipefail
 
 repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-benchmark_name="${BENCHMARK_NAME:?BENCHMARK_NAME is required}"
+benchmark_name='design-system'
 run_date="${RUN_DATE:-$(date -u +%F)}"
+run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date"
 
 if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
   echo "BENCHMARK_NAME must be a benchmark file name without its extension" >&2
@@ -16,11 +17,9 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
   exit 1
 fi
 
-node "$repository_root/packages/agent-eval/dist/cli.js" \
-  --artifacts "$repository_root/artifacts" \
+node "$repository_root/packages/agent-eval/bin/agent-eval" \
   --benchmark "$benchmark_name" \
   --benchmarks "$repository_root/benchmarks" \
   --concurrency "${CONCURRENCY:-1}" \
-  --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \
-  --output "$repository_root/results/benchmarks/$benchmark_name/$run_date/output.json" \
+  --output-dir "$run_directory" \
   --scenarios "$repository_root/scenarios"
diff --git a/script/setup b/script/setup
index 48f14108..7ec54741 100755
--- a/script/setup
+++ b/script/setup
@@ -9,10 +9,7 @@ corepack install
 # Install dependencies
 pnpm install --frozen-lockfile
 
-# Baseline workspaces are not needed after their merged results are produced.
-rm -rf "$repository_root/artifacts"
-
-# Download results from the 10 most recent retained baseline artifacts
+# Download the 10 most recent retained baseline result bundles.
 artifact_ids="$(
   gh api \
     --paginate \
diff --git a/website/src/app/benchmarks/[id]/components/Page.tsx b/website/src/app/benchmarks/[id]/components/Page.tsx
new file mode 100644
index 00000000..94c78206
--- /dev/null
+++ b/website/src/app/benchmarks/[id]/components/Page.tsx
@@ -0,0 +1,226 @@
+'use client'
+
+import {Breadcrumbs, Stack} from '@primer/react'
+import {Blankslate, DataTable, Table} from '@primer/react/experimental'
+import type {Route} from 'next'
+import NextLink from 'next/link'
+import type {BenchmarkPageResults} from '../../../../benchmark-results'
+import type {Benchmark} from '../../../../benchmarks'
+import {Link} from '../../../../components/Link'
+
+type Comparison = BenchmarkPageResults['capabilities'][number]['comparison']
+
+type TableResult = {
+  id: string
+  capability: string
+  scenario: string
+  scenarioId?: string
+  comparison: Comparison
+}
+
+type BenchmarkRun = {
+  id: string
+  name: string
+  resultCount: number
+  passedTests: number
+  totalTests: number
+}
+
+function Delta({comparison}: {comparison: Comparison}) {
+  const tone =
+    comparison.deltaValue === null
+      ? 'text-muted'
+      : comparison.deltaValue > 0
+        ? 'text-success'
+        : comparison.deltaValue < 0
+          ? 'text-danger'
+          : 'text-muted'
+
+  return <span className={tone}>{comparison.delta}</span>
+}
+
+function createTableResults(results: BenchmarkPageResults): Array<TableResult> {
+  return results.capabilities.flatMap(capability => {
+    return [
+      {
+        id: capability.id,
+        capability: capability.name,
+        scenario: capability.scenarios.length > 0 ? 'All scenarios' : 'Not measured',
+        comparison: capability.comparison,
+      },
+      ...capability.scenarios.map(scenario => {
+        return {
+          id: `${capability.id}-${scenario.id}`,
+          capability: '',
+          scenario: scenario.id,
+          scenarioId: scenario.id,
+          comparison: scenario.comparison,
+        }
+      }),
+    ]
+  })
+}
+
+export function Page({
+  benchmark,
+  results,
+  runs,
+}: {
+  benchmark: Benchmark
+  results: BenchmarkPageResults | null
+  runs: Array<BenchmarkRun>
+}) {
+  const tableResults = results ? createTableResults(results) : []
+
+  return (
+    <Stack padding="normal" gap="spacious">
+      <Breadcrumbs>
+        <Breadcrumbs.Item as={NextLink} href="/benchmarks">
+          Benchmarks
+        </Breadcrumbs.Item>
+        <Breadcrumbs.Item selected>{benchmark.id}</Breadcrumbs.Item>
+      </Breadcrumbs>
+      <header>
+        <h1>{benchmark.name}</h1>
+        <p>{benchmark.description}</p>
+      </header>
+      <section>
+        <h2 className="text-title-medium pb-4">Runs</h2>
+        {runs.length > 0 ? (
+          <Table.Container>
+            <DataTable
+              aria-label={`Runs for ${benchmark.name}`}
+              columns={[
+                {
+                  id: 'date',
+                  header: 'Date',
+                  field: 'name',
+                  rowHeader: true,
+                  renderCell: row => {
+                    return (
+                      <Link href={`/benchmarks/${benchmark.id}/runs/${row.name}` as Route}>
+                        <time dateTime={row.name}>{row.name}</time>
+                      </Link>
+                    )
+                  },
+                },
+                {id: 'results', header: 'Results', field: 'resultCount', align: 'end'},
+                {
+                  id: 'tests',
+                  header: 'Tests passed',
+                  field: 'passedTests',
+                  align: 'end',
+                  renderCell: row => {
+                    return `${row.passedTests}/${row.totalTests}`
+                  },
+                },
+              ]}
+              data={runs}
+            />
+          </Table.Container>
+        ) : (
+          <Blankslate border>
+            <Blankslate.Heading as="h3">No runs</Blankslate.Heading>
+            <Blankslate.Description>No results have been recorded for this benchmark yet.</Blankslate.Description>
+          </Blankslate>
+        )}
+      </section>
+      {results ? (
+        <Table.Container>
+          <Table.Title as="h2" id="capability-results-heading">
+            Capability performance
+          </Table.Title>
+          <Table.Subtitle as="p" id="capability-results-description">
+            Benchmark pass rates are compared with Control across every configured scenario. Capability rows aggregate
+            all of their scenario and model results. Latest results:{' '}
+            <Link href={`/benchmarks/${benchmark.id}/runs/${results.date}` as Route}>
+              <time dateTime={results.date}>{results.date}</time>
+            </Link>
+            .
+          </Table.Subtitle>
+          <DataTable
+            aria-describedby="capability-results-description"
+            aria-labelledby="capability-results-heading"
+            cellPadding="condensed"
+            columns={[
+              {
+                id: 'capability',
+                header: 'Capability',
+                field: 'capability',
+                rowHeader: true,
+                maxWidth: '32ch',
+              },
+              {
+                id: 'scenario',
+                header: 'Scenario',
+                field: 'scenario',
+                maxWidth: '1fr',
+                renderCell: row => {
+                  return row.scenarioId ? (
+                    <Link href={`/scenarios/${row.scenarioId}` as Route}>{row.scenario}</Link>
+                  ) : (
+                    row.scenario
+                  )
+                },
+              },
+              {
+                id: 'control',
+                header: 'Control',
+                field: 'comparison',
+                align: 'end',
+                renderCell: row => {
+                  return row.comparison.control
+                },
+              },
+              {
+                id: 'benchmark',
+                header: 'Benchmark',
+                field: 'comparison',
+                align: 'end',
+                renderCell: row => {
+                  return row.comparison.benchmark
+                },
+              },
+              {
+                id: 'delta',
+                header: 'Difference',
+                field: 'comparison',
+                align: 'end',
+                renderCell: row => {
+                  return <Delta comparison={row.comparison} />
+                },
+              },
+              {
+                id: 'control-tests',
+                header: 'Control tests',
+                field: 'comparison',
+                align: 'end',
+                renderCell: row => {
+                  return row.comparison.controlTests
+                },
+              },
+              {
+                id: 'benchmark-tests',
+                header: 'Benchmark tests',
+                field: 'comparison',
+                align: 'end',
+                renderCell: row => {
+                  return row.comparison.benchmarkTests
+                },
+              },
+            ]}
+            data={tableResults}
+          />
+        </Table.Container>
+      ) : (
+        <Blankslate border>
+          <Blankslate.Heading as="h2">No results</Blankslate.Heading>
+          <Blankslate.Description>
+            No results have been recorded for this benchmark yet. Run the benchmark to compare its capabilities with
+            Control.
+          </Blankslate.Description>
+        </Blankslate>
+      )}
+    </Stack>
+  )
+}
diff --git a/website/src/app/benchmarks/[id]/page.tsx b/website/src/app/benchmarks/[id]/page.tsx
new file mode 100644
index 00000000..13d9361a
--- /dev/null
+++ b/website/src/app/benchmarks/[id]/page.tsx
@@ -0,0 +1,47 @@
+import {get, list} from '../../../benchmarks'
+import {getBenchmarkPageResults, listBenchmarkRuns} from '../../../benchmark-results'
+import {Page} from './components/Page'
+
+type BenchmarkPageProps = {
+  params: Promise<{
+    id: string
+  }>
+}
+
+export const dynamicParams = false
+
+export default async function BenchmarkPage(props: BenchmarkPageProps) {
+  const {id} = await props.params
+  const [benchmark, runs] = await Promise.all([get(id), listBenchmarkRuns(id)])
+  const results = getBenchmarkPageResults(benchmark, runs[0])
+
+  return (
+    <Page
+      benchmark={benchmark}
+      results={results}
+      runs={runs.map(run => {
+        const trials = [...run.output.trials.values()]
+        return {
+          id: run.id,
+          name: run.name,
+          resultCount: trials.length,
+          passedTests: trials.reduce((total, trial) => {
+            return total + trial.testResults.numPassedTests
+          }, 0),
+          totalTests: trials.reduce((total, trial) => {
+            return total + trial.testResults.numTotalTests
+          }, 0),
+        }
+      })}
+    />
+  )
+}
+
+export async function generateStaticParams() {
+  const benchmarks = await list()
+  return benchmarks.map(benchmark => {
+    return {
+      id: benchmark.id,
+    }
+  })
+}
diff --git a/website/src/app/benchmarks/[id]/runs/[date]/page.tsx b/website/src/app/benchmarks/[id]/runs/[date]/page.tsx
new file mode 100644
index 00000000..4ed36847
--- /dev/null
+++ b/website/src/app/benchmarks/[id]/runs/[date]/page.tsx
@@ -0,0 +1,121 @@
+import type {BenchmarkOutput} from '@primer/agent-eval/benchmark'
+import type {Route} from 'next'
+import {notFound} from 'next/navigation'
+import {get as getBenchmark, list as listBenchmarks} from '../../../../../benchmarks'
+import {getBenchmarkRun, listBenchmarkRuns, type BenchmarkRun} from '../../../../../benchmark-results'
+import {createTranscript, getWalkthroughDataUrls, type RunDetails} from '../../../../../run-details'
+import {RunDetailsPage} from '../../../../components/RunDetailsPage'
+
+const EMPTY_RUN_PARAM = '__no-runs__'
+
+type BenchmarkOutputTrial = BenchmarkOutput['trials'] extends Map<string, infer Trial> ? Trial : never
+
+type RunPageProps = {
+  params: Promise<{
+    id: string
+    date: string
+  }>
+}
+
+async function createBenchmarkRunDetails(run: BenchmarkRun): Promise<RunDetails> {
+  const treatments = new Map(
+    [...run.output.treatments].map(([id, treatment]) => {
+      return [id, treatment.name]
+    }),
+  )
+
+  return {
+    date: run.name,
+    results: await Promise.all(
+      [...run.output.trials.values()].map(async (trial: BenchmarkOutputTrial) => {
+        const sessions = trial.agent.sessions
+        return {
+          id: trial.id,
+          scenarioId: trial.scenarioId,
+          context: trial.capabilityId,
+          treatment: treatments.get(trial.treatmentId) ?? 'Unknown treatment',
+          model: trial.model.name,
+          reasoningEffort: trial.model.reasoningEffort,
+          testsPassed: trial.testResults.numPassedTests,
+          totalTests: trial.testResults.numTotalTests,
+          turns: sessions.reduce((total, session) => {
+            return total + session.turns
+          }, 0),
+          outputTokens: sessions.reduce((total, session) => {
+            return total + session.outputTokens
+          }, 0),
+          premiumRequests: sessions.reduce((total, session) => {
+            return total + session.premiumRequests
+          }, 0),
+          totalApiDurationMs: sessions.reduce((total, session) => {
+            return total + session.totalApiDurationMs
+          }, 0),
+          sessionDurationMs: sessions.reduce((total, session) => {
+            return total + session.sessionDurationMs
+          }, 0),
+          tests: trial.testResults.testResults.flatMap(testResult => {
+            return testResult.assertionResults.map(assertion => {
+              return {
+                fullName: assertion.fullName,
+                status: assertion.status,
+                description: assertion.meta.description,
+              }
+            })
+          }),
+          walkthrough: await getWalkthroughDataUrls(trial.walkthrough, run.directory),
+          transcript: createTranscript(
+            sessions.flatMap(session => {
+              return session.messages
+            }),
+          ),
+        }
+      }),
+    ),
+  }
+}
+
+export const dynamicParams = false
+
+export default async function BenchmarkRunPage(props: RunPageProps) {
+  const {id, date} = await props.params
+  if (id === EMPTY_RUN_PARAM && date === EMPTY_RUN_PARAM) {
+    notFound()
+  }
+
+  const [benchmark, run] = await Promise.all([getBenchmark(id), getBenchmarkRun(id, date)])
+  if (!run) {
+    notFound()
+  }
+
+  return (
+    <RunDetailsPage
+      resource={{
+        id: benchmark.id,
+        name: benchmark.name,
+        collectionLabel: 'Benchmarks',
+        collectionHref: '/benchmarks',
+        href: `/benchmarks/${benchmark.id}` as Route,
+      }}
+      run={await createBenchmarkRunDetails(run)}
+    />
+  )
+}
+
+export async function generateStaticParams() {
+  const benchmarks = await listBenchmarks()
+  const params = (
+    await Promise.all(
+      benchmarks.map(async benchmark => {
+        const runs = await listBenchmarkRuns(benchmark.id)
+        return runs.map(run => {
+          return {
+            id: benchmark.id,
+            date: run.name,
+          }
+        })
+      }),
+    )
+  ).flat()
+
+  return params.length > 0 ? params : [{id: EMPTY_RUN_PARAM, date: EMPTY_RUN_PARAM}]
+}
diff --git a/website/src/app/benchmarks/page.tsx b/website/src/app/benchmarks/page.tsx
new file mode 100644
index 00000000..a5c0d55f
--- /dev/null
+++ b/website/src/app/benchmarks/page.tsx
@@ -0,0 +1,11 @@
+import {BenchmarksTable} from '../components/ResourceTables'
+import {list as listBenchmarks} from '../../benchmarks'
+
+export const metadata = {
+  title: 'Benchmarks',
+}
+
+export default async function BenchmarksPage() {
+  const benchmarks = await listBenchmarks()
+  return <BenchmarksTable benchmarks={benchmarks} headingLevel="h1" standalone />
+}
diff --git a/website/src/app/components/PageHeader.tsx b/website/src/app/components/PageHeader.tsx
index 2de09f9a..b4b88ef1 100644
--- a/website/src/app/components/PageHeader.tsx
+++ b/website/src/app/components/PageHeader.tsx
@@ -37,6 +37,13 @@ export function PageHeader() {
           >
             Experiments
           </UnderlineNav.Item>
+          <UnderlineNav.Item
+            as={Link}
+            href="/benchmarks"
+            aria-current={pathname.startsWith('/benchmarks') ? 'page' : undefined}
+          >
+            Benchmarks
+          </UnderlineNav.Item>
           <UnderlineNav.Item
             as={Link}
             href="/scenarios"
diff --git a/website/src/app/components/ResourceTables.tsx b/website/src/app/components/ResourceTables.tsx
index b279ce06..f1ed59cd 100644
--- a/website/src/app/components/ResourceTables.tsx
+++ b/website/src/app/components/ResourceTables.tsx
@@ -2,6 +2,7 @@
 
 import {Stack} from '@primer/react'
 import {DataTable, Table} from '@primer/react/experimental'
+import type {Benchmark} from '../../benchmarks'
 import type {Experiment} from '../../experiments'
 import {Link} from '../../components/Link'
 import type {ScenarioSummary} from '../../scenarios'
@@ -103,3 +104,61 @@ export function ExperimentsTable({
     standalone,
   )
 }
+
+export function BenchmarksTable({
+  benchmarks,
+  headingLevel = 'h2',
+  showViewAll = false,
+  standalone = false,
+}: ResourceTableProps & {
+  benchmarks: Array<Benchmark>
+}) {
+  return withStandaloneLayout(
+    <section>
+      <Table.Container>
+        <Table.Title as={headingLevel} id="benchmarks-heading">
+          Benchmarks
+        </Table.Title>
+        {showViewAll ? (
+          <Table.Actions>
+            <Link href="/benchmarks">View all</Link>
+          </Table.Actions>
+        ) : null}
+        <DataTable
+          aria-labelledby="benchmarks-heading"
+          columns={[
+            {
+              id: 'name',
+              header: 'Name',
+              field: 'name',
+              rowHeader: true,
+              maxWidth: '40ch',
+              renderCell: row => {
+                return <Link href={`/benchmarks/${row.id}`}>{row.name}</Link>
+              },
+            },
+            {id: 'description', header: 'Description', field: 'description', maxWidth: '60ch'},
+            {id: 'models', header: 'Models', field: 'models', align: 'end'},
+            {id: 'capabilities', header: 'Capabilities', field: 'capabilities', align: 'end'},
+            {id: 'scenarios', header: 'Scenarios', field: 'scenarios', align: 'end'},
+          ]}
+          data={benchmarks.map(benchmark => ({
+            id: benchmark.id,
+            name: benchmark.name,
+            description: benchmark.description,
+            models: benchmark.models.length,
+            capabilities: benchmark.capabilities.length,
+            scenarios: new Set(
+              benchmark.capabilities.flatMap(capability => {
+                return capability.scenarios.map(scenario => {
+                  return scenario.id
+                })
+              }),
+            ).size,
+          }))}
+        />
+      </Table.Container>
+    </section>,
+    standalone,
+  )
+}
diff --git a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx b/website/src/app/components/RunDetailsPage.tsx
similarity index 92%
rename from website/src/app/experiments/[id]/runs/[date]/components/Page.tsx
rename to website/src/app/components/RunDetailsPage.tsx
index b20d3218..28ee1d22 100644
--- a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx
+++ b/website/src/app/components/RunDetailsPage.tsx
@@ -2,46 +2,13 @@
 
 import {CheckCircleFillIcon, CopilotIcon, PersonIcon, XCircleFillIcon} from '@primer/octicons-react'
 import {Breadcrumbs, FormControl, Select, Stack, UnderlineNav} from '@primer/react'
-import type {Experiment} from '../../../../../../experiments'
+import type {RunDetails, TranscriptEntry, WalkthroughDataUrl} from '../../run-details'
 import type {Route} from 'next'
 import Link from 'next/link'
 import Image from 'next/image'
 import {useState} from 'react'
-import type {WalkthroughDataUrl} from '../page'
 
-type TranscriptEntry = {
-  id: string
-  label: string
-  timestamp?: string
-  content: string
-}
-
-type RunResult = {
-  id: string
-  scenarioId: string
-  treatment: string
-  model: string
-  reasoningEffort?: string
-  testsPassed: number
-  totalTests: number
-  turns: number
-  outputTokens: number
-  premiumRequests: number
-  totalApiDurationMs: number
-  sessionDurationMs: number
-  tests: Array<{
-    fullName: string
-    status: string
-    description?: string
-  }>
-  transcript: Array<TranscriptEntry>
-  walkthrough: WalkthroughDataUrl
-}
-
-type RunDetails = {
-  date: string
-  results: Array<RunResult>
-}
+type RunResult = RunDetails['results'][number]
 
 type ScenarioResultGroup = {
   scenarioId: string
@@ -394,26 +361,32 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num
 }
 
 type Props = {
-  experiment: Experiment
+  resource: {
+    id: string
+    name: string
+    collectionLabel: string
+    collectionHref: Route
+    href: Route
+  }
   run: RunDetails
 }
 
-export function Page({experiment, run}: Props) {
+export function RunDetailsPage({resource, run}: Props) {
   const resultGroups = groupResultsByScenario(run.results)
 
   return (
     <Stack padding="normal">
       <div className="w-full max-w-screen-xl mx-auto flex flex-col gap-6">
         <Breadcrumbs>
-          <Breadcrumbs.Item as={Link} href="/experiments">
-            Experiments
+          <Breadcrumbs.Item as={Link} href={resource.collectionHref}>
+            {resource.collectionLabel}
           </Breadcrumbs.Item>
-          <Breadcrumbs.Item as={Link} href={`/experiments/${experiment.id}` as Route}>
-            {experiment.id}
+          <Breadcrumbs.Item as={Link} href={resource.href}>
+            {resource.id}
           </Breadcrumbs.Item>
           <Breadcrumbs.Item selected>{run.date}</Breadcrumbs.Item>
         </Breadcrumbs>
-        <h1 className="sr-only">Run results for {experiment.name}</h1>
+        <h1 className="sr-only">Run results for {resource.name}</h1>
         <div className="flex flex-col gap-8">
           {resultGroups.map((group, index) => {
             return <ScenarioResults group={group} index={index} key={`${run.date}:${group.scenarioId}`} />
@@ -423,5 +396,3 @@ export function Page({experiment, run}: Props) {
     </Stack>
   )
 }
-
-export type {RunDetails, TranscriptEntry}
diff --git a/website/src/app/experiments/[id]/runs/[date]/page.tsx b/website/src/app/experiments/[id]/runs/[date]/page.tsx
index abd4b965..6593984a 100644
--- a/website/src/app/experiments/[id]/runs/[date]/page.tsx
+++ b/website/src/app/experiments/[id]/runs/[date]/page.tsx
@@ -1,15 +1,11 @@
+import type {Route} from 'next'
+import {notFound} from 'next/navigation'
 import {get as getExperiment} from '../../../../../experiments'
+import {createExperimentRunDetails} from '../../../../../run-details'
 import {get as getRun, list as listRuns} from '../../../../../runs'
-import type {RunOutput} from '../../../../../runs'
-import {notFound} from 'next/navigation'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {Page} from './components/Page'
-import type {RunDetails, TranscriptEntry} from './components/Page'
+import {RunDetailsPage} from '../../../../components/RunDetailsPage'
 
 const EMPTY_RUN_PARAM = '__no-runs__'
-const REPOSITORY_ROOT = path.resolve(process.cwd(), '..')
-const ARTIFACTS_DIRECTORY = path.join(REPOSITORY_ROOT, 'artifacts')
 
 type RunPageProps = {
   params: Promise<{
@@ -18,236 +14,6 @@ type RunPageProps = {
   }>
 }
 
-type LogMessage = RunOutput['results'][number]['assistant']['logs'][number]
-type Walkthrough = RunOutput['results'][number]['walkthrough']
-
-function asRecord(value: unknown): Record<string, unknown> | null {
-  return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null
-}
-
-function getString(record: Record<string, unknown> | null, key: string): string | undefined {
-  const value = record?.[key]
-  return typeof value === 'string' ? value : undefined
-}
-
-function createTranscript(logs: Array<LogMessage>): Array<TranscriptEntry> {
-  const entries: Array<TranscriptEntry> = []
-  const messageEntries = new Map<string, TranscriptEntry>()
-  const reasoningEntries = new Map<string, TranscriptEntry>()
-  const toolNames = new Map<string, string>()
-
-  for (const [index, message] of logs.entries()) {
-    const record = asRecord(message)
-    const data = asRecord(record?.data)
-    const timestamp = getString(record, 'timestamp')
-    const id = getString(record, 'id') ?? `${message.type}-${index}`
-
-    switch (message.type) {
-      case 'user.message': {
-        const content = getString(data, 'content')
-        if (content) {
-          entries.push({id, label: 'User', timestamp, content})
-        }
-        break
-      }
-      case 'assistant.message_delta': {
-        const messageId = getString(data, 'messageId')
-        const delta = getString(data, 'deltaContent')
-        if (!messageId || !delta) {
-          break
-        }
-
-        let entry = messageEntries.get(messageId)
-        if (!entry) {
-          entry = {id, label: 'Assistant', timestamp, content: ''}
-          messageEntries.set(messageId, entry)
-          entries.push(entry)
-        }
-        entry.content += delta
-        break
-      }
-      case 'assistant.message': {
-        const messageId = getString(data, 'messageId')
-        const content = getString(data, 'content')
-        const entry = messageId ? messageEntries.get(messageId) : undefined
-        if (entry) {
-          if (content) {
-            entry.content = content
-          }
-        } else if (content) {
-          entries.push({id, label: 'Assistant', timestamp, content})
-        }
-        break
-      }
-      case 'assistant.reasoning_delta': {
-        const reasoningId = getString(data, 'reasoningId')
-        const delta = getString(data, 'deltaContent')
-        if (!reasoningId || !delta) {
-          break
-        }
-
-        let entry = reasoningEntries.get(reasoningId)
-        if (!entry) {
-          entry = {id, label: 'Reasoning', timestamp, content: ''}
-          reasoningEntries.set(reasoningId, entry)
-          entries.push(entry)
-        }
-        entry.content += delta
-        break
-      }
-      case 'assistant.reasoning': {
-        const reasoningId = getString(data, 'reasoningId')
-        const content = getString(data, 'content')
-        const entry = reasoningId ? reasoningEntries.get(reasoningId) : undefined
-        if (entry) {
-          if (content) {
-            entry.content = content
-          }
-        } else if (content) {
-          entries.push({id, label: 'Reasoning', timestamp, content})
-        }
-        break
-      }
-      case 'tool.execution_start': {
-        const toolName = getString(data, 'toolName') ?? 'Unknown tool'
-        const toolCallId = getString(data, 'toolCallId')
-        if (toolCallId) {
-          toolNames.set(toolCallId, toolName)
-        }
-        entries.push({id, label: `Tool call: ${toolName}`, timestamp, content: 'Started'})
-        break
-      }
-      case 'tool.execution_complete': {
-        const toolCallId = getString(data, 'toolCallId')
-        const toolName = toolCallId ? toolNames.get(toolCallId) : undefined
-        entries.push({
-          id,
-          label: `Tool result: ${toolName ?? 'Unknown tool'}`,
-          timestamp,
-          content: data?.success === true ? 'Completed successfully' : 'Failed',
-        })
-        break
-      }
-      case 'session.info': {
-        const content = getString(data, 'message')
-        if (content) {
-          entries.push({id, label: 'Session', timestamp, content})
-        }
-        break
-      }
-      case 'session.task_complete': {
-        const content = getString(data, 'summary')
-        if (content) {
-          entries.push({id, label: 'Summary', timestamp, content})
-        }
-        break
-      }
-    }
-  }
-
-  return entries.filter(entry => entry.content.length > 0)
-}
-
-async function getArtifactDataUrl(artifactPath: string | undefined, mimeType: string): Promise<string | undefined> {
-  if (!artifactPath) {
-    return undefined
-  }
-
-  const absolutePath = path.isAbsolute(artifactPath) ? artifactPath : path.resolve(REPOSITORY_ROOT, artifactPath)
-  const relativePath = path.relative(ARTIFACTS_DIRECTORY, absolutePath)
-  if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) {
-    return undefined
-  }
-
-  try {
-    const contents = await fs.readFile(absolutePath)
-    return `data:${mimeType};base64,${contents.toString('base64')}`
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return undefined
-    }
-    throw error
-  }
-}
-
-function getImageMimeType(artifactPath: string): string {
-  const extension = path.extname(artifactPath).toLowerCase()
-  if (extension === '.jpg' || extension === '.jpeg') {
-    return 'image/jpeg'
-  }
-  return 'image/png'
-}
-
-export type WalkthroughDataUrl =
-  | {
-      type: 'Unavailable'
-    }
-  | {type: 'Screenshot'; screenshot: string}
-  | {type: 'Screenshots'; screenshots: Array<string>}
-  | {type: 'Video'; video: string}
-
-async function getWalkthroughDataUrls(walkthrough: Walkthrough): Promise<WalkthroughDataUrl> {
-  if (walkthrough.type === 'Screenshot') {
-    return {
-      type: 'Screenshot',
-      screenshot: await getArtifactDataUrl(walkthrough.filepath, getImageMimeType(walkthrough.filepath)),
-    }
-  }
-
-  if (walkthrough.type === 'Screenshots') {
-    const sources = await Promise.all(
-      walkthrough.screenshots.map(artifactPath => getArtifactDataUrl(artifactPath, getImageMimeType(artifactPath))),
-    )
-    return {
-      type: 'Screenshots',
-      screenshots: sources.filter((source): source is string => source !== undefined),
-    }
-  }
-
-  if (walkthrough.type === 'Video') {
-    return {
-      type: 'Video',
-      video: await getArtifactDataUrl(walkthrough.filepath, 'video/webm'),
-    }
-  }
-
-  return {
-    type: 'Unavailable',
-  }
-}
-
-async function createRunDetails(date: string, output: RunOutput): Promise<RunDetails> {
-  const treatments = new Map(output.treatments.map(treatment => [treatment.id, treatment.config.name]))
-  return {
-    date,
-    results: await Promise.all(
-      output.results.map(async result => ({
-        id: result.id,
-        scenarioId: result.scenarioId,
-        treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment',
-        model: result.model,
-        reasoningEffort: result.reasoningEffort,
-        testsPassed: result.testResults.numPassedTests,
-        totalTests: result.testResults.numTotalTests,
-        turns: result.assistant.turns,
-        outputTokens: result.assistant.outputTokens,
-        premiumRequests: result.assistant.premiumRequests,
-        totalApiDurationMs: result.assistant.totalApiDurationMs,
-        sessionDurationMs: result.assistant.sessionDurationMs,
-        tests: result.testResults.tests.map(test => {
-          return {
-            fullName: test.fullName,
-            status: test.status,
-            description: test.description,
-          }
-        }),
-        walkthrough: await getWalkthroughDataUrls(result.walkthrough),
-        transcript: createTranscript(result.assistant.logs),
-      })),
-    ),
-  }
-}
-
 export const dynamicParams = false
 
 export default async function RunPage(props: RunPageProps) {
@@ -261,7 +27,18 @@ export default async function RunPage(props: RunPageProps) {
     notFound()
   }
 
-  return <Page experiment={experiment} run={await createRunDetails(date, run.output)} />
+  return (
+    <RunDetailsPage
+      resource={{
+        id: experiment.id,
+        name: experiment.name,
+        collectionLabel: 'Experiments',
+        collectionHref: '/experiments',
+        href: `/experiments/${experiment.id}` as Route,
+      }}
+      run={await createExperimentRunDetails(date, run.output, run.directory)}
+    />
+  )
 }
 
 export async function generateStaticParams() {
@@ -270,8 +47,10 @@ export async function generateStaticParams() {
     return [{id: EMPTY_RUN_PARAM, date: EMPTY_RUN_PARAM}]
   }
 
-  return runs.map(run => ({
-    id: run.output.experiment.id,
-    date: run.name,
-  }))
+  return runs.map(run => {
+    return {
+      id: run.output.experiment.id,
+      date: run.name,
+    }
+  })
 }
diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts
new file mode 100644
index 00000000..7aaa67fb
--- /dev/null
+++ b/website/src/benchmark-results.ts
@@ -0,0 +1,257 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import type {BenchmarkOutput} from '@primer/agent-eval/benchmark'
+import type {Benchmark} from './benchmarks'
+
+const {deserialize} = await import(
+  /* turbopackIgnore: true */
+  '@primer/agent-eval/benchmark'
+)
+
+const REPOSITORY_ROOT = path.resolve(process.cwd(), '..')
+const BENCHMARK_RESULTS_DIR = path.join(REPOSITORY_ROOT, 'results', 'benchmarks')
+
+type BenchmarkOutputTrial = BenchmarkOutput['trials'] extends Map<string, infer Trial> ? Trial : never
+
+type ResultTotals = {
+  passed: number
+  total: number
+}
+
+export type BenchmarkComparison = {
+  control: string
+  benchmark: string
+  delta: string
+  controlTests: string
+  benchmarkTests: string
+  deltaValue: number | null
+}
+
+export type BenchmarkCapabilityResult = {
+  id: string
+  name: string
+  comparison: BenchmarkComparison
+  scenarios: Array<{
+    id: string
+    comparison: BenchmarkComparison
+  }>
+}
+
+export type BenchmarkPageResults = {
+  date: string
+  capabilities: Array<BenchmarkCapabilityResult>
+}
+
+type OutputCandidate = {
+  filepath: string
+  date: string
+}
+
+export type BenchmarkRun = {
+  id: string
+  name: string
+  directory: string
+  date: Date
+  output: BenchmarkOutput
+}
+
+function isRunDate(value: string): boolean {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+    return false
+  }
+
+  const date = new Date(`${value}T00:00:00.000Z`)
+  return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(value)
+}
+
+async function getDatedCandidates(benchmarkId: string): Promise<Array<OutputCandidate>> {
+  const directory = path.join(BENCHMARK_RESULTS_DIR, benchmarkId)
+
+  try {
+    const entries = await fs.readdir(directory, {withFileTypes: true})
+    const candidates = await Promise.all(
+      entries
+        .filter(entry => {
+          return entry.isDirectory() && isRunDate(entry.name)
+        })
+        .map(async entry => {
+          const filepath = path.join(directory, entry.name, 'output.json')
+          try {
+            await fs.access(filepath)
+            return {
+              filepath,
+              date: entry.name,
+            }
+          } catch (error) {
+            if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+              return null
+            }
+
+            throw error
+          }
+        }),
+    )
+
+    return candidates.filter((candidate): candidate is OutputCandidate => {
+      return candidate !== null
+    })
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return []
+    }
+
+    throw error
+  }
+}
+
+async function readBenchmarkOutput(candidate: OutputCandidate, benchmarkId: string): Promise<BenchmarkOutput | null> {
+  const contents = await fs.readFile(candidate.filepath, 'utf-8')
+  const parsed: unknown = JSON.parse(contents)
+  const output = deserialize(parsed)
+  return output.benchmarkId === benchmarkId ? output : null
+}
+
+function getTotals(trials: Array<BenchmarkOutputTrial>): ResultTotals {
+  return trials.reduce(
+    (totals, trial) => {
+      totals.passed += trial.testResults.numPassedTests
+      totals.total += trial.testResults.numTotalTests
+      return totals
+    },
+    {passed: 0, total: 0},
+  )
+}
+
+function getPassRate(totals: ResultTotals): number | null {
+  if (totals.total === 0) {
+    return null
+  }
+
+  return totals.passed / totals.total
+}
+
+function formatRate(rate: number | null): string {
+  if (rate === null) {
+    return 'N/A'
+  }
+
+  return `${(rate * 100).toFixed(1)}%`
+}
+
+function formatTests(totals: ResultTotals): string {
+  if (totals.total === 0) {
+    return 'N/A'
+  }
+
+  return `${totals.passed}/${totals.total}`
+}
+
+function createComparison(
+  trials: Array<BenchmarkOutputTrial>,
+  controlTreatmentId: string,
+  benchmarkTreatmentId: string,
+): BenchmarkComparison {
+  const controlTotals = getTotals(
+    trials.filter(trial => {
+      return trial.treatmentId === controlTreatmentId
+    }),
+  )
+  const benchmarkTotals = getTotals(
+    trials.filter(trial => {
+      return trial.treatmentId === benchmarkTreatmentId
+    }),
+  )
+  const controlRate = getPassRate(controlTotals)
+  const benchmarkRate = getPassRate(benchmarkTotals)
+  const deltaValue = controlRate === null || benchmarkRate === null ? null : benchmarkRate - controlRate
+
+  return {
+    control: formatRate(controlRate),
+    benchmark: formatRate(benchmarkRate),
+    delta:
+      deltaValue === null ? 'N/A' : `${deltaValue > 0 ? '+' : ''}${(deltaValue * 100).toFixed(1)} percentage points`,
+    controlTests: formatTests(controlTotals),
+    benchmarkTests: formatTests(benchmarkTotals),
+    deltaValue,
+  }
+}
+
+function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date: string): BenchmarkPageResults {
+  const controlTreatment = [...output.treatments].find(([, treatment]) => {
+    return treatment.name === 'Control'
+  })
+  const benchmarkTreatment = [...output.treatments].find(([, treatment]) => {
+    return treatment.name === 'Benchmark'
+  })
+
+  if (!controlTreatment || !benchmarkTreatment) {
+    throw new Error(`Benchmark "${benchmark.id}" results must include Control and Benchmark treatments`)
+  }
+
+  const trials = [...output.trials.values()]
+  const capabilities = benchmark.capabilities.map((capability, capabilityIndex) => {
+    const capabilityTrials = trials.filter(trial => {
+      return trial.capabilityId === capability.name
+    })
+
+    return {
+      id: `${capabilityIndex}-${capability.name}`,
+      name: capability.name,
+      comparison: createComparison(capabilityTrials, controlTreatment[0], benchmarkTreatment[0]),
+      scenarios: capability.scenarios.map(scenario => {
+        const scenarioTrials = capabilityTrials.filter(trial => {
+          return trial.scenarioId === scenario.id
+        })
+
+        return {
+          id: scenario.id,
+          comparison: createComparison(scenarioTrials, controlTreatment[0], benchmarkTreatment[0]),
+        }
+      }),
+    }
+  })
+
+  return {date, capabilities}
+}
+
+export async function listBenchmarkRuns(benchmarkId: string): Promise<Array<BenchmarkRun>> {
+  const candidates = (await getDatedCandidates(benchmarkId)).toSorted((a, b) => {
+    return b.date.localeCompare(a.date)
+  })
+
+  const runs: Array<BenchmarkRun> = []
+  for (const candidate of candidates) {
+    const output = await readBenchmarkOutput(candidate, benchmarkId)
+    if (output) {
+      runs.push({
+        id: candidate.date,
+        name: candidate.date,
+        directory: path.dirname(candidate.filepath),
+        date: new Date(`${candidate.date}T00:00:00.000Z`),
+        output,
+      })
+    }
+  }
+
+  return runs
+}
+
+export async function getBenchmarkRun(benchmarkId: string, date: string): Promise<BenchmarkRun | null> {
+  if (!isRunDate(date)) {
+    return null
+  }
+
+  const runs = await listBenchmarkRuns(benchmarkId)
+  return (
+    runs.find(run => {
+      return run.name === date
+    }) ?? null
+  )
+}
+
+export function getBenchmarkPageResults(
+  benchmark: Benchmark,
+  run: BenchmarkRun | undefined,
+): BenchmarkPageResults | null {
+  return run ? createPageResults(benchmark, run.output, run.name) : null
+}
diff --git a/website/src/benchmarks.ts b/website/src/benchmarks.ts
new file mode 100644
index 00000000..bf6b45e0
--- /dev/null
+++ b/website/src/benchmarks.ts
@@ -0,0 +1,53 @@
+import path from 'node:path'
+import type {Benchmark as AgentEvalBenchmark} from '@primer/agent-eval/benchmark'
+
+const {getBenchmark, listBenchmarks} = await import(
+  /* turbopackIgnore: true */
+  '@primer/agent-eval/benchmark'
+)
+
+const BENCHMARKS_DIR = path.resolve(process.cwd(), '..', 'benchmarks')
+const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios')
+
+export type Benchmark = Pick<AgentEvalBenchmark, 'id' | 'name' | 'description' | 'models'> & {
+  capabilities: Array<{
+    name: string
+    scenarios: Array<{id: string}>
+  }>
+}
+
+function normalizeBenchmark(benchmark: AgentEvalBenchmark): Benchmark {
+  return {
+    id: benchmark.id,
+    name: benchmark.name,
+    description: benchmark.description,
+    models: benchmark.models,
+    capabilities: benchmark.capabilities.map(capability => {
+      return {
+        name: capability.name,
+        scenarios: capability.scenarios.map(scenario => {
+          return {id: scenario.id}
+        }),
+      }
+    }),
+  }
+}
+
+export async function list(): Promise<Array<Benchmark>> {
+  const benchmarks = await listBenchmarks({
+    benchmarksDirectory: BENCHMARKS_DIR,
+    scenariosDirectory: SCENARIOS_DIR,
+  })
+
+  return benchmarks.map(normalizeBenchmark)
+}
+
+export async function get(id: string): Promise<Benchmark> {
+  const benchmark = await getBenchmark({
+    benchmarksDirectory: BENCHMARKS_DIR,
+    scenariosDirectory: SCENARIOS_DIR,
+    id,
+  })
+
+  return normalizeBenchmark(benchmark)
+}
diff --git a/website/src/run-details.ts b/website/src/run-details.ts
new file mode 100644
index 00000000..330067c2
--- /dev/null
+++ b/website/src/run-details.ts
@@ -0,0 +1,335 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import type {RunOutput, RunOutputResult} from './runs'
+
+const REPOSITORY_ROOT = path.resolve(process.cwd(), '..')
+const LEGACY_ARTIFACTS_DIRECTORY = path.join(REPOSITORY_ROOT, 'artifacts')
+
+type LogMessage = RunOutputResult['assistant']['logs'][number]
+type Walkthrough = RunOutputResult['walkthrough']
+
+type TranscriptEntry = {
+  id: string
+  label: string
+  timestamp?: string
+  content: string
+}
+
+type WalkthroughDataUrl =
+  | {
+      type: 'Unavailable'
+    }
+  | {type: 'Screenshot'; screenshot: string}
+  | {type: 'Screenshots'; screenshots: Array<string>}
+  | {type: 'Video'; video: string}
+
+type RunResult = {
+  id: string
+  scenarioId: string
+  context?: string
+  treatment: string
+  model: string
+  reasoningEffort?: string
+  testsPassed: number
+  totalTests: number
+  turns: number
+  outputTokens: number
+  premiumRequests: number
+  totalApiDurationMs: number
+  sessionDurationMs: number
+  tests: Array<{
+    fullName: string
+    status: string
+    description?: string
+  }>
+  transcript: Array<TranscriptEntry>
+  walkthrough: WalkthroughDataUrl
+}
+
+type RunDetails = {
+  date: string
+  results: Array<RunResult>
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null
+}
+
+function getString(record: Record<string, unknown> | null, key: string): string | undefined {
+  const value = record?.[key]
+  return typeof value === 'string' ? value : undefined
+}
+
+function createTranscript(logs: Array<LogMessage>): Array<TranscriptEntry> {
+  const entries: Array<TranscriptEntry> = []
+  const messageEntries = new Map<string, TranscriptEntry>()
+  const reasoningEntries = new Map<string, TranscriptEntry>()
+  const toolNames = new Map<string, string>()
+
+  for (const [index, message] of logs.entries()) {
+    const record = asRecord(message)
+    const data = asRecord(record?.data)
+    const timestamp = getString(record, 'timestamp')
+    const id = getString(record, 'id') ?? `${message.type}-${index}`
+
+    switch (message.type) {
+      case 'user.message': {
+        const content = getString(data, 'content')
+        if (content) {
+          entries.push({id, label: 'User', timestamp, content})
+        }
+        break
+      }
+      case 'assistant.message_delta': {
+        const messageId = getString(data, 'messageId')
+        const delta = getString(data, 'deltaContent')
+        if (!messageId || !delta) {
+          break
+        }
+
+        let entry = messageEntries.get(messageId)
+        if (!entry) {
+          entry = {id, label: 'Assistant', timestamp, content: ''}
+          messageEntries.set(messageId, entry)
+          entries.push(entry)
+        }
+        entry.content += delta
+        break
+      }
+      case 'assistant.message': {
+        const messageId = getString(data, 'messageId')
+        const content = getString(data, 'content')
+        const entry = messageId ? messageEntries.get(messageId) : undefined
+        if (entry) {
+          if (content) {
+            entry.content = content
+          }
+        } else if (content) {
+          entries.push({id, label: 'Assistant', timestamp, content})
+        }
+        break
+      }
+      case 'assistant.reasoning_delta': {
+        const reasoningId = getString(data, 'reasoningId')
+        const delta = getString(data, 'deltaContent')
+        if (!reasoningId || !delta) {
+          break
+        }
+
+        let entry = reasoningEntries.get(reasoningId)
+        if (!entry) {
+          entry = {id, label: 'Reasoning', timestamp, content: ''}
+          reasoningEntries.set(reasoningId, entry)
+          entries.push(entry)
+        }
+        entry.content += delta
+        break
+      }
+      case 'assistant.reasoning': {
+        const reasoningId = getString(data, 'reasoningId')
+        const content = getString(data, 'content')
+        const entry = reasoningId ? reasoningEntries.get(reasoningId) : undefined
+        if (entry) {
+          if (content) {
+            entry.content = content
+          }
+        } else if (content) {
+          entries.push({id, label: 'Reasoning', timestamp, content})
+        }
+        break
+      }
+      case 'tool.execution_start': {
+        const toolName = getString(data, 'toolName') ?? 'Unknown tool'
+        const toolCallId = getString(data, 'toolCallId')
+        if (toolCallId) {
+          toolNames.set(toolCallId, toolName)
+        }
+        entries.push({id, label: `Tool call: ${toolName}`, timestamp, content: 'Started'})
+        break
+      }
+      case 'tool.execution_complete': {
+        const toolCallId = getString(data, 'toolCallId')
+        const toolName = toolCallId ? toolNames.get(toolCallId) : undefined
+        entries.push({
+          id,
+          label: `Tool result: ${toolName ?? 'Unknown tool'}`,
+          timestamp,
+          content: data?.success === true ? 'Completed successfully' : 'Failed',
+        })
+        break
+      }
+      case 'session.info': {
+        const content = getString(data, 'message')
+        if (content) {
+          entries.push({id, label: 'Session', timestamp, content})
+        }
+        break
+      }
+      case 'session.task_complete': {
+        const content = getString(data, 'summary')
+        if (content) {
+          entries.push({id, label: 'Summary', timestamp, content})
+        }
+        break
+      }
+    }
+  }
+
+  return entries.filter(entry => {
+    return entry.content.length > 0
+  })
+}
+
+function isWithinDirectory(directory: string, filepath: string): boolean {
+  const relativePath = path.relative(directory, filepath)
+  return relativePath !== '..' && !relativePath.startsWith(`..${path.sep}`) && !path.isAbsolute(relativePath)
+}
+
+function getArtifactCandidates(artifactPath: string, runDirectory: string): Array<string> {
+  const runArtifactsDirectory = path.join(runDirectory, 'artifacts')
+
+  if (!path.isAbsolute(artifactPath)) {
+    const candidate = path.resolve(runDirectory, artifactPath)
+    return isWithinDirectory(runArtifactsDirectory, candidate) ? [candidate] : []
+  }
+
+  if (isWithinDirectory(LEGACY_ARTIFACTS_DIRECTORY, artifactPath)) {
+    return [artifactPath]
+  }
+
+  const segments = artifactPath.split(/[\\/]+/)
+  const artifactsIndex = segments.lastIndexOf('artifacts')
+  if (artifactsIndex === -1) {
+    return []
+  }
+
+  const artifactSegments = segments.slice(artifactsIndex + 1)
+  return [
+    path.join(runArtifactsDirectory, ...artifactSegments),
+    path.join(LEGACY_ARTIFACTS_DIRECTORY, ...artifactSegments),
+  ].filter(candidate => {
+    return (
+      isWithinDirectory(runArtifactsDirectory, candidate) || isWithinDirectory(LEGACY_ARTIFACTS_DIRECTORY, candidate)
+    )
+  })
+}
+
+async function getArtifactDataUrl(
+  artifactPath: string | undefined,
+  mimeType: string,
+  runDirectory: string,
+): Promise<string | undefined> {
+  if (!artifactPath) {
+    return undefined
+  }
+
+  for (const candidate of getArtifactCandidates(artifactPath, runDirectory)) {
+    try {
+      const contents = await fs.readFile(candidate)
+      return `data:${mimeType};base64,${contents.toString('base64')}`
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error
+      }
+    }
+  }
+
+  return undefined
+}
+
+function getImageMimeType(artifactPath: string): string {
+  const extension = path.extname(artifactPath).toLowerCase()
+  if (extension === '.jpg' || extension === '.jpeg') {
+    return 'image/jpeg'
+  }
+  return 'image/png'
+}
+
+async function getWalkthroughDataUrls(walkthrough: Walkthrough, runDirectory: string): Promise<WalkthroughDataUrl> {
+  if (walkthrough.type === 'Screenshot') {
+    const screenshot = await getArtifactDataUrl(
+      walkthrough.filepath,
+      getImageMimeType(walkthrough.filepath),
+      runDirectory,
+    )
+    return screenshot
+      ? {
+          type: 'Screenshot',
+          screenshot,
+        }
+      : {type: 'Unavailable'}
+  }
+
+  if (walkthrough.type === 'Screenshots') {
+    const sources = await Promise.all(
+      walkthrough.screenshots.map(artifactPath => {
+        return getArtifactDataUrl(artifactPath, getImageMimeType(artifactPath), runDirectory)
+      }),
+    )
+    const screenshots = sources.filter((source): source is string => {
+      return source !== undefined
+    })
+    return screenshots.length > 0
+      ? {
+          type: 'Screenshots',
+          screenshots,
+        }
+      : {type: 'Unavailable'}
+  }
+
+  if (walkthrough.type === 'Video') {
+    const video = await getArtifactDataUrl(walkthrough.filepath, 'video/webm', runDirectory)
+    return video
+      ? {
+          type: 'Video',
+          video,
+        }
+      : {type: 'Unavailable'}
+  }
+
+  return {
+    type: 'Unavailable',
+  }
+}
+
+async function createExperimentRunDetails(date: string, output: RunOutput, runDirectory: string): Promise<RunDetails> {
+  const treatments = new Map(
+    output.treatments.map(treatment => {
+      return [treatment.id, treatment.config.name]
+    }),
+  )
+
+  return {
+    date,
+    results: await Promise.all(
+      output.results.map(async result => {
+        return {
+          id: result.id,
+          scenarioId: result.scenarioId,
+          treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment',
+          model: result.model,
+          reasoningEffort: result.reasoningEffort,
+          testsPassed: result.testResults.numPassedTests,
+          totalTests: result.testResults.numTotalTests,
+          turns: result.assistant.turns,
+          outputTokens: result.assistant.outputTokens,
+          premiumRequests: result.assistant.premiumRequests,
+          totalApiDurationMs: result.assistant.totalApiDurationMs,
+          sessionDurationMs: result.assistant.sessionDurationMs,
+          tests: result.testResults.tests.map(test => {
+            return {
+              fullName: test.fullName,
+              status: test.status,
+              description: test.description,
+            }
+          }),
+          walkthrough: await getWalkthroughDataUrls(result.walkthrough, runDirectory),
+          transcript: createTranscript(result.assistant.logs),
+        }
+      }),
+    ),
+  }
+}
+
+export {createExperimentRunDetails, createTranscript, getWalkthroughDataUrls}
+export type {RunDetails, TranscriptEntry, WalkthroughDataUrl}

From 2f0e34a37f35a4bd814ac4b36dd27773ce3ad40c Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 16:49:41 -0500
Subject: [PATCH 32/52] refactor: remove baseline results

---
 .github/workflows/baseline.yml  | 107 --------------------------------
 .github/workflows/benchmark.yml |   4 +-
 script/run-baseline.sh          |  29 ---------
 script/setup                    |   4 +-
 4 files changed, 5 insertions(+), 139 deletions(-)
 delete mode 100644 .github/workflows/baseline.yml
 delete mode 100755 script/run-baseline.sh

diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml
deleted file mode 100644
index 12b5f0ed..00000000
--- a/.github/workflows/baseline.yml
+++ /dev/null
@@ -1,107 +0,0 @@
-name: baseline
-on:
-  schedule:
-    - cron: '0 0 * * 1'
-  workflow_dispatch:
-
-concurrency:
-  group: weekly-baseline
-  cancel-in-progress: false
-
-permissions:
-  contents: read
-
-jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      run-date: ${{ steps.run.outputs.date }}
-    steps:
-      - name: set run date
-        id: run
-        run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT"
-
-  run:
-    needs: setup
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: true
-      matrix:
-        order: [1, 2, 3, 4]
-    steps:
-      - name: checkout repository
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: set up pnpm
-        uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
-        with:
-          cache: true
-          require-lockfile: true
-      - name: set up Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
-        with:
-          node-version-file: '.nvmrc'
-      - name: build project
-        run: pnpm run build
-      - name: run baseline
-        env:
-          CONCURRENCY: 2
-          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
-          OUTPUT_FILE: output-${{ matrix.order }}.json
-          RUN_DATE: ${{ needs.setup.outputs.run-date }}
-          SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
-        run: script/run-baseline.sh
-      - name: prepare baseline artifact
-        if: ${{ always() }}
-        run: |
-          mkdir -p workflow-artifact
-          if [[ -d results ]]; then
-            mv results workflow-artifact/results
-          fi
-      - name: upload baseline results
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: baseline-shard-${{ github.run_id }}-${{ matrix.order }}
-          path: workflow-artifact
-          if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-
-  merge:
-    needs: run
-    runs-on: ubuntu-latest
-    steps:
-      - name: download baseline results
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          pattern: baseline-shard-${{ github.run_id }}-*
-          merge-multiple: true
-      - name: merge baseline results
-        run: |
-          output_directory="$(dirname "$(find results -type f -name 'output-*.json' -print -quit)")"
-          jq --slurp '
-            . as $outputs
-            | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments
-            | $outputs[0]
-            | .treatments = $treatments
-            | .results = [
-                $outputs[] as $output
-                | $output.results[]
-                | . as $result
-                | ($output.treatments[] | select(.id == $result.treatmentId).config.name) as $name
-                | .treatmentId = ($treatments[] | select(.config.name == $name).id)
-              ]
-          ' "$output_directory"/output-*.json > "$output_directory/output.json"
-          rm "$output_directory"/output-*.json
-      - name: prepare baseline artifact
-        run: |
-          mkdir -p workflow-artifact
-          mv results workflow-artifact/results
-      - name: upload baseline results
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: baseline-${{ github.run_id }}
-          path: workflow-artifact
-          if-no-files-found: error
-          retention-days: 90
-          compression-level: 9
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 465ab09c..f762a1ab 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,5 +1,7 @@
 name: benchmark
 on:
+  schedule:
+    - cron: '0 0 * * 1'
   workflow_dispatch:
     inputs:
       concurrency:
@@ -16,7 +18,7 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.workflow }}
+  group: weekly-design-system-benchmark
   cancel-in-progress: false
 
 permissions:
diff --git a/script/run-baseline.sh b/script/run-baseline.sh
deleted file mode 100755
index 68cefe88..00000000
--- a/script/run-baseline.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-run_date="${RUN_DATE:-$(date -u +%F)}"
-output_file="${OUTPUT_FILE:-output.json}"
-run_directory="$repository_root/results/$run_date"
-
-if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
-  echo "RUN_DATE must use the YYYY-MM-DD format" >&2
-  exit 1
-fi
-
-arguments=(
-  --artifacts "$run_directory/artifacts"
-  --concurrency "${CONCURRENCY:-1}"
-  --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}"
-  --experiment baseline
-  --experiments "$repository_root/experiments"
-  --output "$run_directory/$output_file"
-  --scenarios "$repository_root/scenarios"
-)
-
-if [[ -n "${SHARD:-}" ]]; then
-  arguments+=(--shard "$SHARD")
-fi
-
-node "$repository_root/packages/agent-eval/dist/cli.js" "${arguments[@]}"
diff --git a/script/setup b/script/setup
index 7ec54741..2305a4fd 100755
--- a/script/setup
+++ b/script/setup
@@ -9,7 +9,7 @@ corepack install
 # Install dependencies
 pnpm install --frozen-lockfile
 
-# Download the 10 most recent retained baseline result bundles.
+# Download the 10 most recent retained benchmark result bundles.
 artifact_ids="$(
   gh api \
     --paginate \
@@ -17,7 +17,7 @@ artifact_ids="$(
     "/repos/primer/agent-eval/actions/artifacts?per_page=100" |
     jq --raw-output '
       [.[].artifacts[]
-        | select(.expired == false and (.name | test("^baseline-[0-9]+$")))
+        | select(.expired == false and (.name | test("^benchmark-[0-9]+$")))
       ]
       | sort_by(.created_at)
       | .[-10:][].id

From db929c3707d9087f0381d873e0b441cedee117e3 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 17:21:34 -0500
Subject: [PATCH 33/52] feat: move towards benchmark trends

---
 .github/workflows/deploy.yml                  |   3 +-
 .github/workflows/experiment.yml              |  38 +-
 .../workflows/report-workflow-failures.yml    |   2 +-
 benchmarks/design-system.ts                   | 213 ++++---
 packages/agent-eval/package.json              |   4 +
 packages/agent-eval/src/cli.ts                |   7 +
 packages/agent-eval/src/experiment.ts         |   5 +-
 packages/agent-eval/src/trial.ts              |  38 +-
 website/README.md                             |  32 +-
 .../src/app/baseline/[scenarioId]/page.tsx    |  28 -
 website/src/app/baseline/page.tsx             |   7 -
 .../app/benchmarks/[id]/components/Page.tsx   | 101 +--
 website/src/app/benchmarks/[id]/page.tsx      |  28 +-
 .../src/app/components/BaselineOverview.tsx   |  33 -
 website/src/app/components/BaselineTrends.tsx | 581 ------------------
 .../components/BenchmarkOverview.module.css   |   8 +
 .../src/app/components/BenchmarkOverview.tsx  | 142 +++++
 ....module.css => BenchmarkTrends.module.css} |  74 +--
 .../src/app/components/BenchmarkTrends.tsx    | 368 +++++++++++
 website/src/app/components/Index.module.css   |  15 -
 website/src/app/components/Index.tsx          | 220 -------
 website/src/app/components/PageHeader.tsx     |  13 +-
 .../app/experiments/[id]/runs/[date]/page.tsx |   5 +-
 website/src/app/layout.tsx                    |   2 +-
 website/src/app/page.tsx                      |   8 +-
 website/src/baseline-results.ts               | 476 --------------
 website/src/benchmark-page-data.ts            |  28 +
 website/src/benchmark-results.ts              | 333 +++++++++-
 website/src/runs.ts                           |  90 +--
 29 files changed, 1172 insertions(+), 1730 deletions(-)
 delete mode 100644 website/src/app/baseline/[scenarioId]/page.tsx
 delete mode 100644 website/src/app/baseline/page.tsx
 delete mode 100644 website/src/app/components/BaselineOverview.tsx
 delete mode 100644 website/src/app/components/BaselineTrends.tsx
 create mode 100644 website/src/app/components/BenchmarkOverview.module.css
 create mode 100644 website/src/app/components/BenchmarkOverview.tsx
 rename website/src/app/components/{BaselineTrends.module.css => BenchmarkTrends.module.css} (50%)
 create mode 100644 website/src/app/components/BenchmarkTrends.tsx
 delete mode 100644 website/src/app/components/Index.module.css
 delete mode 100644 website/src/app/components/Index.tsx
 delete mode 100644 website/src/baseline-results.ts
 create mode 100644 website/src/benchmark-page-data.ts

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 2c230c6c..bd76352d 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -5,7 +5,6 @@ on:
       - main
   workflow_run:
     workflows:
-      - baseline
       - benchmark
     types:
       - completed
@@ -38,7 +37,7 @@ jobs:
         run: |
           gh api --paginate \
             "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[] | select(.expired == false and (.name | test("^(baseline|benchmark)-[0-9]+$"))) | [.created_at, .id] | @tsv' |
+            --jq '.artifacts[] | select(.expired == false and (.name | test("^benchmark-[0-9]+$"))) | [.created_at, .id] | @tsv' |
             sort |
             cut -f2 |
             while read -r artifact_id; do
diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml
index 66bb5342..bbfed459 100644
--- a/.github/workflows/experiment.yml
+++ b/.github/workflows/experiment.yml
@@ -27,7 +27,17 @@ permissions:
   contents: read
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      run-date: ${{ steps.run.outputs.date }}
+    steps:
+      - name: Set run date
+        id: run
+        run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT"
+
   run:
+    needs: setup
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -53,18 +63,19 @@ jobs:
           CONCURRENCY: ${{ inputs.concurrency }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
           EXPERIMENT_NAME: ${{ inputs.experiment }}
-          OUTPUT_FILE: results/experiment/output-${{ matrix.order }}.json
+          RUN_DATE: ${{ needs.setup.outputs.run-date }}
           SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
         run: |
           set -o pipefail
+          run_directory="results/experiments/$EXPERIMENT_NAME/$RUN_DATE"
           node packages/agent-eval/dist/cli.js \
             --concurrency "$CONCURRENCY" \
             --docker-image "$DOCKER_IMAGE" \
             --experiment "$EXPERIMENT_NAME" \
             --experiments experiments \
             --scenarios scenarios \
-            --artifacts results/experiment/artifacts \
-            --output "$OUTPUT_FILE" \
+            --artifacts "$run_directory/artifacts" \
+            --output "$run_directory/output-${{ matrix.order }}.json" \
             --shard "$SHARD"
       - name: Prepare experiment artifact
         if: ${{ always() }}
@@ -84,7 +95,7 @@ jobs:
           compression-level: 9
 
   merge:
-    needs: run
+    needs: [setup, run]
     runs-on: ubuntu-latest
     steps:
       - name: download experiment results
@@ -94,19 +105,14 @@ jobs:
           merge-multiple: true
       - name: merge experiment results
         run: |
-          cd results/experiment
+          cd "results/experiments/${{ inputs.experiment }}/${{ needs.setup.outputs.run-date }}"
           jq --slurp '
-            . as $outputs
-            | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments
-            | $outputs[0]
-            | .treatments = $treatments
-            | .results = [
-                $outputs[] as $output
-                | $output.results[]
-                | . as $result
-                | ($output.treatments[] | select(.id == $result.treatmentId).config.name) as $name
-                | .treatmentId = ($treatments[] | select(.config.name == $name).id)
-              ]
+            {
+              experimentId: .[0].experimentId,
+              scenarios: (reduce .[].scenarios as $scenarios ({}; . * $scenarios)),
+              treatments: (reduce .[].treatments as $treatments ({}; . * $treatments)),
+              trials: (reduce .[].trials as $trials ({}; . * $trials))
+            }
           ' output-*.json > output.json
           rm output-*.json
       - name: Prepare experiment artifact
diff --git a/.github/workflows/report-workflow-failures.yml b/.github/workflows/report-workflow-failures.yml
index 4ed8f179..c24d1861 100644
--- a/.github/workflows/report-workflow-failures.yml
+++ b/.github/workflows/report-workflow-failures.yml
@@ -3,7 +3,7 @@ name: Report workflow failures
 on:
   workflow_run:
     workflows:
-      - baseline
+      - benchmark
       - deploy
       - update-pnpm
       - update-sandbox-tools
diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts
index d25f0d7f..00955953 100644
--- a/benchmarks/design-system.ts
+++ b/benchmarks/design-system.ts
@@ -2,101 +2,126 @@ import {defineConfig} from '@primer/agent-eval/benchmark'
 
 export const benchmark = defineConfig({
   name: 'Design System',
-  description: 'Benchmark the performance of different design system approaches.',
-  models: ['gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.6-sol'],
+  description: 'Benchmark the performance of agents with different design system tasks.',
+  models: [
+    'gpt-5.6-sol',
+    'gpt-5.6-terra',
+    'claude-opus-5',
+    'claude-sonnet-5',
+    'gemini-3.1-pro-preview',
+    'gemini-3.5-flash',
+  ],
+  async setup({sandbox}) {
+    // Setup the Primer MCP server locally
+    await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest'])
+    await sandbox.addMcpServer('primer', {
+      type: 'local',
+      command: 'npx',
+      args: ['--no-install', '@primer/mcp'],
+      tools: ['*'],
+    })
+  },
   capabilities: [
     {
-      name: 'Uses components',
-      scenarios: ['001-agent-uses-button-from-primer', '006-agent-uses-pagination-component'],
-    },
-    {
-      name: 'Infers correct component based on usage',
-      scenarios: ['007-agent-infers-billing-banner', '008-agent-infers-action-menu'],
-    },
-    {
-      name: 'Composes components correctly',
-      scenarios: [],
-    },
-    {
-      name: 'Uses documented component APIs',
-      scenarios: [],
-    },
-    {
-      name: 'Uses tokens',
-      scenarios: [
-        '009-agent-uses-layout-and-color-tokens',
-        '010-agent-uses-typography-tokens',
-        '011-agent-uses-motion-tokens',
-      ],
-    },
-    {
-      name: 'Infers correct token based on usage',
-      scenarios: ['012-agent-infers-status-tokens', '013-agent-infers-compact-control-tokens'],
-    },
-    {
-      name: 'Uses icons',
-      scenarios: ['002-agent-uses-octicon-from-primer', '014-agent-replaces-custom-icons-with-octicons'],
-    },
-    {
-      name: 'Infers correct icon based on usage',
-      scenarios: ['015-agent-infers-copy-icon'],
-    },
-    {
-      name: 'Uses UI patterns',
-      scenarios: [
-        '003-agent-uses-form-from-primer',
-        '016-agent-uses-loading-and-empty-state-patterns',
-        '017-agent-uses-confirmation-pattern',
-        '018-agent-uses-filter-pattern',
-      ],
-    },
-    {
-      name: 'Implements navigation patterns',
-      scenarios: [],
-    },
-    {
-      name: 'Applies accessibility guidance',
-      scenarios: [],
-    },
-    {
-      name: 'Builds responsive interfaces',
-      scenarios: [],
-    },
-    {
-      name: 'Handles interaction states',
-      scenarios: [],
-    },
-    {
-      name: 'Uses utilities',
-      scenarios: ['019-agent-uses-dismissal-utilities', '020-agent-uses-resize-observer-utility'],
-    },
-    {
-      name: 'Extends the design system safely',
-      scenarios: [],
-    },
-    {
-      name: 'Getting started',
-      scenarios: ['004-agent-setup-nextjs', '021-agent-sets-up-primer-in-vite'],
-    },
-    {
-      name: 'Theming',
-      scenarios: [
-        '005-agent-enables-theme-switching',
-        '022-agent-enables-automatic-theming',
-        '023-agent-adds-theme-switcher',
-      ],
-    },
-    {
-      name: 'Supports accessible color modes',
-      scenarios: [],
-    },
-    {
-      name: 'Works with TailwindCSS',
-      scenarios: ['024-agent-sets-up-tailwindcss', '025-agent-uses-tokens-with-tailwindcss'],
-    },
-    {
-      name: 'Respects component maturity',
-      scenarios: ['026-agent-avoids-deprecated-notification'],
-    },
+      name: 'Uses appropriate components by default',
+      scenarios: ['001-agent-uses-button-from-primer', '003-agent-uses-form-from-primer'],
+    },
+    {
+      name: 'Uses Octicons by default for iconography',
+      scenarios: ['002-agent-uses-octicon-from-primer'],
+    },
+    {
+      name: 'Sets up new projects with Primer',
+      scenarios: ['004-agent-setup-nextjs', '005-agent-enables-theme-switching'],
+    },
+    // {
+    //   name: 'Infers correct component based on usage',
+    //   scenarios: ['007-agent-infers-billing-banner', '008-agent-infers-action-menu'],
+    // },
+    // {
+    //   name: 'Composes components correctly',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Uses documented component APIs',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Uses tokens',
+    //   scenarios: [
+    //     '009-agent-uses-layout-and-color-tokens',
+    //     '010-agent-uses-typography-tokens',
+    //     '011-agent-uses-motion-tokens',
+    //   ],
+    // },
+    // {
+    //   name: 'Infers correct token based on usage',
+    //   scenarios: ['012-agent-infers-status-tokens', '013-agent-infers-compact-control-tokens'],
+    // },
+    // {
+    //   name: 'Uses icons',
+    //   scenarios: ['002-agent-uses-octicon-from-primer', '014-agent-replaces-custom-icons-with-octicons'],
+    // },
+    // {
+    //   name: 'Infers correct icon based on usage',
+    //   scenarios: ['015-agent-infers-copy-icon'],
+    // },
+    // {
+    //   name: 'Uses UI patterns',
+    //   scenarios: [
+    //     '003-agent-uses-form-from-primer',
+    //     '016-agent-uses-loading-and-empty-state-patterns',
+    //     '017-agent-uses-confirmation-pattern',
+    //     '018-agent-uses-filter-pattern',
+    //   ],
+    // },
+    // {
+    //   name: 'Implements navigation patterns',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Applies accessibility guidance',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Builds responsive interfaces',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Handles interaction states',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Uses utilities',
+    //   scenarios: ['019-agent-uses-dismissal-utilities', '020-agent-uses-resize-observer-utility'],
+    // },
+    // {
+    //   name: 'Extends the design system safely',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Getting started',
+    //   scenarios: ['004-agent-setup-nextjs', '021-agent-sets-up-primer-in-vite'],
+    // },
+    // {
+    //   name: 'Theming',
+    //   scenarios: [
+    //     '005-agent-enables-theme-switching',
+    //     '022-agent-enables-automatic-theming',
+    //     '023-agent-adds-theme-switcher',
+    //   ],
+    // },
+    // {
+    //   name: 'Supports accessible color modes',
+    //   scenarios: [],
+    // },
+    // {
+    //   name: 'Works with TailwindCSS',
+    //   scenarios: ['024-agent-sets-up-tailwindcss', '025-agent-uses-tokens-with-tailwindcss'],
+    // },
+    // {
+    //   name: 'Respects component maturity',
+    //   scenarios: ['026-agent-avoids-deprecated-notification'],
+    // },
   ],
 })
diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index 2a3e2803..2ea4d9d3 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -34,6 +34,10 @@
       "types": "./dist/scenario.d.ts",
       "default": "./dist/scenario.js"
     },
+    "./sandbox": {
+      "types": "./dist/sandbox/index.d.ts",
+      "default": "./dist/sandbox/index.js"
+    },
     "./scenarios": {
       "types": "./dist/scenarios.d.ts",
       "default": "./dist/scenarios.js"
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index 42e6ac4f..b77460fd 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -19,6 +19,7 @@ import {
 } from './experiment'
 import {logger} from './logger'
 import {formatBenchmarkResults, formatExperimentResults} from './report'
+import {parseShard} from './shard'
 import {compare as compareTrial} from './trial'
 
 const {values} = parseArgs({
@@ -77,6 +78,10 @@ const {values} = parseArgs({
       type: 'string',
       description: 'The directory containing scenario directories',
     },
+    shard: {
+      type: 'string',
+      description: 'The experiment shard to run, formatted as order/total',
+    },
   },
 })
 
@@ -97,6 +102,7 @@ Options:
       --output <file>        The target file in which results are written (default: output.json)
       --output-dir <dir>     The directory containing output.json and its artifacts
       --scenarios <dir>      The directory containing scenario directories (default: ./scenarios)
+      --shard <order/total>  The experiment shard to run
 `)
 }
 
@@ -176,6 +182,7 @@ if (values.benchmark) {
   const result = await runExperiment({
     env,
     id: values.experiment,
+    shard: values.shard ? parseShard(values.shard) : undefined,
   })
   const sorted = result.toSorted(compareTrial)
 
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 7667971d..aa6dce49 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -13,6 +13,7 @@ import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
 import {create as createPlan, run as runPlan} from './plan'
 import {getScenario, ScenarioSchema, type Scenario} from './scenario'
+import {selectShard, type Shard} from './shard'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
 import {
   getPortableTrialPaths,
@@ -162,10 +163,12 @@ async function run({
   env,
   host = DefaultHost,
   id,
+  shard,
 }: {
   env: EnvironmentConfig
   host?: Host
   id: string
+  shard?: Shard
 }): Promise<ExperimentRunResult> {
   const experiment = await getExperiment({
     host,
@@ -195,7 +198,7 @@ async function run({
       ]
     })
   })
-  const plan = await createPlan(trials)
+  const plan = await createPlan(shard ? selectShard(trials, shard) : trials)
   const results = await runPlan({
     env,
     host,
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index e7faafbd..1dd4ed3f 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -115,9 +115,11 @@ async function run({
   sandbox: Sandbox
   trial: Trial
 }): Promise<TrialResult> {
-  logger.info('Running trial treatment: %s (%s)', trial.treatment.name, trial.id)
+  const logPrefix = `[${trial.scenario.id}] [${trial.treatment.name}] [${trial.model.name} (${trial.model.reasoningEffort})]`
 
-  logger.info('Copying files from: %s...', trial.scenario.directory)
+  logger.info('%s Running trial: %s', logPrefix, trial.id)
+
+  logger.info('%s Copying files from: %s...', logPrefix, trial.scenario.directory)
 
   await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
     exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next', 'dist'],
@@ -126,41 +128,41 @@ async function run({
     user: 'root',
   })
 
-  logger.info('[%s] Obfuscating package name...', trial.treatment.name)
+  logger.info('%s Obfuscating package name...', logPrefix)
   await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], {
     user: NODE_USER,
   })
 
-  logger.info('[%s] Removing workspace dependency...', trial.treatment.name)
+  logger.info('%s Removing workspace dependency...', logPrefix)
   await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], {
     user: NODE_USER,
   })
 
-  logger.info('[%s] Installing dependencies...', trial.treatment.name)
+  logger.info('%s Installing dependencies...', logPrefix)
   await sandbox.runCommand('npm', ['install'], {
     user: NODE_USER,
   })
 
   if (trial.setup) {
-    logger.info('[%s] Running generic setup...', trial.treatment.name)
+    logger.info('%s Running generic setup...', logPrefix)
     await trial.setup({
       sandbox,
     })
   }
 
   if (trial.treatment.setup) {
-    logger.info('[%s] Running treatment setup...', trial.treatment.name)
+    logger.info('%s Running treatment setup...', logPrefix)
     await trial.treatment.setup({
       sandbox,
     })
   }
 
-  logger.info('[%s] Run build script...', trial.treatment.name)
+  logger.info('%s Run build script...', logPrefix)
   await sandbox.runCommand('npm', ['run', 'build', '--if-present'], {
     user: NODE_USER,
   })
 
-  logger.info('[%s] Running copilot...', trial.treatment.name)
+  logger.info('%s Running copilot...', logPrefix)
   const copilotOutput = await sandbox.runCommand(
     'copilot',
     [
@@ -191,7 +193,7 @@ async function run({
     return parseMessage(JSON.parse(trimmed))
   })
 
-  logger.info('[%s] Running tests...', trial.treatment.name)
+  logger.info('%s Running tests...', logPrefix)
 
   const TEST_PATH = 'scenario.test.ts'
   const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
@@ -217,7 +219,7 @@ async function run({
   const WALKTHROUGH_DIR = 'walkthrough'
   const WALKTHROUGH_VIEWPORT_WIDTH = 1440
   const WALKTHROUGH_VIEWPORT_HEIGHT = 900
-  logger.debug('Capturing walkthrough...')
+  logger.debug('%s Capturing walkthrough...', logPrefix)
   await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], {
     user: 'root',
   })
@@ -273,7 +275,7 @@ Only capture the walkthrough, do not make any further code changes.`
   )
 
   if (walkthroughResult.exitCode !== 0) {
-    logger.warn('[%s] Unable to capture walkthrough: %s', trial.treatment.name, walkthroughResult.stderr)
+    logger.warn('%s Unable to capture walkthrough: %s', logPrefix, walkthroughResult.stderr)
   }
 
   const artifactDirectory = path.join(artifactsDirectory, trial.id)
@@ -288,19 +290,19 @@ Only capture the walkthrough, do not make any further code changes.`
   }
   await host.fs.mkdir(workspaceDirectory, {recursive: true})
 
-  logger.info('[%s] Downloading artifacts to: %s...', trial.treatment.name, artifactDirectory)
+  logger.info('%s Downloading artifacts to: %s...', logPrefix, artifactDirectory)
 
-  logger.debug('[%s] Downloading agent workspace to: %s...', trial.treatment.name, workspaceDirectory)
+  logger.debug('%s Downloading agent workspace to: %s...', logPrefix, workspaceDirectory)
   await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, {
     ignore(name) {
       return name.includes('node_modules') || name.includes('.next') || name.includes('.turbo') || name.includes('dist')
     },
   })
 
-  logger.debug('[%s] Downloading copilot config to: %s...', trial.treatment.name, copilotConfigDirectory)
+  logger.debug('%s Downloading copilot config to: %s...', logPrefix, copilotConfigDirectory)
   await sandbox.download(COPILOT_DIR, copilotConfigDirectory)
 
-  logger.debug('[%s] Downloading skills config to: %s...', trial.treatment.name, skillsConfigDirectory)
+  logger.debug('%s Downloading skills config to: %s...', logPrefix, skillsConfigDirectory)
   await sandbox.download(AGENTS_DIR, skillsConfigDirectory)
 
   let walkthrough: Walkthrough = {
@@ -309,8 +311,8 @@ Only capture the walkthrough, do not make any further code changes.`
 
   if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) {
     logger.debug(
-      '[%s] Moving walkthrough artifacts from: %s to: %s...',
-      trial.treatment.name,
+      '%s Moving walkthrough artifacts from: %s to: %s...',
+      logPrefix,
       path.join(workspaceDirectory, WALKTHROUGH_DIR),
       walkthroughPath,
     )
diff --git a/website/README.md b/website/README.md
index 0d1824b8..cafef617 100644
--- a/website/README.md
+++ b/website/README.md
@@ -2,10 +2,28 @@
 
 ## Routes
 
-| URL                | Description                                                                  |
-| :----------------- | :--------------------------------------------------------------------------- |
-| `/`                | View baseline above the fold, section for experiments, section for scenarios |
-| `/experiments`     | List experiments                                                             |
-| `/experiments/:id` | View experiment details                                                      |
-| `/scenarios`       | List scenarios                                                               |
-| `/scenarios/:id`   | View scenario details and baseline results                                   |
+| URL                           | Description                                     |
+| :---------------------------- | :---------------------------------------------- |
+| `/`                           | View the latest design system benchmark results |
+| `/benchmarks`                 | List benchmarks                                 |
+| `/benchmarks/:id`             | View benchmark results and dated runs           |
+| `/benchmarks/:id/runs/:date`  | View benchmark run details and walkthroughs     |
+| `/experiments`                | List experiments                                |
+| `/experiments/:id`            | View experiment details                         |
+| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs    |
+| `/scenarios`                  | List scenarios                                  |
+| `/scenarios/:id`              | View scenario details                           |
+
+## Results
+
+The website reads portable result bundles from:
+
+```text
+results/
+├── benchmarks/<benchmark-id>/<YYYY-MM-DD>/
+│   ├── output.json
+│   └── artifacts/
+└── experiments/<experiment-id>/<YYYY-MM-DD>/
+    ├── output.json
+    └── artifacts/
+```
diff --git a/website/src/app/baseline/[scenarioId]/page.tsx b/website/src/app/baseline/[scenarioId]/page.tsx
deleted file mode 100644
index fbbe0d33..00000000
--- a/website/src/app/baseline/[scenarioId]/page.tsx
+++ /dev/null
@@ -1,28 +0,0 @@
-import {notFound} from 'next/navigation'
-import {getBaselinePageData} from '../../../baseline-results'
-import {list as listScenarios} from '../../../scenarios'
-import {Index} from '../../components/Index'
-
-export const dynamicParams = false
-
-type BaselineScenarioPageProps = {
-  params: Promise<{
-    scenarioId: string
-  }>
-}
-
-export default async function BaselineScenarioPage({params}: BaselineScenarioPageProps) {
-  const {scenarioId} = await params
-  const [{baseline, baselineTrends}, scenarios] = await Promise.all([getBaselinePageData(), listScenarios()])
-
-  if (!scenarios.some(scenario => scenario.id === scenarioId)) {
-    notFound()
-  }
-
-  return <Index baseline={baseline} baselineTrends={baselineTrends} selectedScenarioId={scenarioId} />
-}
-
-export async function generateStaticParams() {
-  const scenarios = await listScenarios()
-  return scenarios.map(scenario => ({scenarioId: scenario.id}))
-}
diff --git a/website/src/app/baseline/page.tsx b/website/src/app/baseline/page.tsx
deleted file mode 100644
index 79d4b3ea..00000000
--- a/website/src/app/baseline/page.tsx
+++ /dev/null
@@ -1,7 +0,0 @@
-import {getBaselinePageData} from '../../baseline-results'
-import {Index} from '../components/Index'
-
-export default async function BaselinePage() {
-  const {baseline, baselineTrends} = await getBaselinePageData()
-  return <Index baseline={baseline} baselineTrends={baselineTrends} />
-}
diff --git a/website/src/app/benchmarks/[id]/components/Page.tsx b/website/src/app/benchmarks/[id]/components/Page.tsx
index 94c78206..5fefaa57 100644
--- a/website/src/app/benchmarks/[id]/components/Page.tsx
+++ b/website/src/app/benchmarks/[id]/components/Page.tsx
@@ -15,6 +15,8 @@ type TableResult = {
   capability: string
   scenario: string
   scenarioId?: string
+  model: string
+  reasoningEffort: string
   comparison: Comparison
 }
 
@@ -26,19 +28,6 @@ type BenchmarkRun = {
   totalTests: number
 }
 
-function Delta({comparison}: {comparison: Comparison}) {
-  const tone =
-    comparison.deltaValue === null
-      ? 'text-muted'
-      : comparison.deltaValue > 0
-        ? 'text-success'
-        : comparison.deltaValue < 0
-          ? 'text-danger'
-          : 'text-muted'
-
-  return <span className={tone}>{comparison.delta}</span>
-}
-
 function createTableResults(results: BenchmarkPageResults): Array<TableResult> {
   return results.capabilities.flatMap(capability => {
     return [
@@ -46,16 +35,32 @@ function createTableResults(results: BenchmarkPageResults): Array<TableResult> {
         id: capability.id,
         capability: capability.name,
         scenario: capability.scenarios.length > 0 ? 'All scenarios' : 'Not measured',
+        model: 'All models',
+        reasoningEffort: '',
         comparison: capability.comparison,
       },
-      ...capability.scenarios.map(scenario => {
-        return {
-          id: `${capability.id}-${scenario.id}`,
-          capability: '',
-          scenario: scenario.id,
-          scenarioId: scenario.id,
-          comparison: scenario.comparison,
-        }
+      ...capability.scenarios.flatMap(scenario => {
+        return [
+          {
+            id: `${capability.id}-${scenario.id}`,
+            capability: '',
+            scenario: scenario.id,
+            scenarioId: scenario.id,
+            model: 'All models',
+            reasoningEffort: '',
+            comparison: scenario.comparison,
+          },
+          ...scenario.models.map(model => {
+            return {
+              id: `${capability.id}-${scenario.id}-${model.id}`,
+              capability: '',
+              scenario: '',
+              model: model.name,
+              reasoningEffort: model.reasoningEffort,
+              comparison: model.comparison,
+            }
+          }),
+        ]
       }),
     ]
   })
@@ -131,8 +136,8 @@ export function Page({
             Capability performance
           </Table.Title>
           <Table.Subtitle as="p" id="capability-results-description">
-            Benchmark pass rates are compared with Control across every configured scenario. Capability rows aggregate
-            all of their scenario and model results. Latest results:{' '}
+            Benchmark results are shown first, followed by the percent change from Control in parentheses. Capability
+            rows aggregate all of their scenario and model results. Latest results:{' '}
             <Link href={`/benchmarks/${benchmark.id}/runs/${results.date}` as Route}>
               <time dateTime={results.date}>{results.date}</time>
             </Link>
@@ -164,63 +169,65 @@ export function Page({
                 },
               },
               {
-                id: 'control',
-                header: 'Control',
+                id: 'model',
+                header: 'Model',
+                field: 'model',
+              },
+              {
+                id: 'reasoning-effort',
+                header: 'Reasoning effort',
+                field: 'reasoningEffort',
+              },
+              {
+                id: 'tests',
+                header: 'Tests',
                 field: 'comparison',
                 align: 'end',
                 renderCell: row => {
-                  return row.comparison.control
+                  return row.comparison.tests
                 },
               },
               {
-                id: 'benchmark',
-                header: 'Benchmark',
+                id: 'output-tokens',
+                header: 'Output tokens',
                 field: 'comparison',
                 align: 'end',
                 renderCell: row => {
-                  return row.comparison.benchmark
+                  return row.comparison.outputTokens
                 },
               },
               {
-                id: 'delta',
-                header: 'Difference',
+                id: 'premium-requests',
+                header: 'Premium requests',
                 field: 'comparison',
                 align: 'end',
                 renderCell: row => {
-                  return <Delta comparison={row.comparison} />
+                  return row.comparison.premiumRequests
                 },
               },
               {
-                id: 'control-tests',
-                header: 'Control tests',
+                id: 'session-time',
+                header: 'Session time',
                 field: 'comparison',
                 align: 'end',
                 renderCell: row => {
-                  return row.comparison.controlTests
+                  return row.comparison.sessionTime
                 },
               },
               {
-                id: 'benchmark-tests',
-                header: 'Benchmark tests',
+                id: 'api-time',
+                header: 'API time',
                 field: 'comparison',
                 align: 'end',
                 renderCell: row => {
-                  return row.comparison.benchmarkTests
+                  return row.comparison.apiTime
                 },
               },
             ]}
             data={tableResults}
           />
         </Table.Container>
-      ) : (
-        <Blankslate border>
-          <Blankslate.Heading as="h2">No results</Blankslate.Heading>
-          <Blankslate.Description>
-            No results have been recorded for this benchmark yet. Run the benchmark to compare its capabilities with
-            Control.
-          </Blankslate.Description>
-        </Blankslate>
-      )}
+      ) : null}
     </Stack>
   )
 }
diff --git a/website/src/app/benchmarks/[id]/page.tsx b/website/src/app/benchmarks/[id]/page.tsx
index 13d9361a..a0a395c9 100644
--- a/website/src/app/benchmarks/[id]/page.tsx
+++ b/website/src/app/benchmarks/[id]/page.tsx
@@ -1,5 +1,5 @@
-import {get, list} from '../../../benchmarks'
-import {getBenchmarkPageResults, listBenchmarkRuns} from '../../../benchmark-results'
+import {getBenchmarkPageData} from '../../../benchmark-page-data'
+import {list} from '../../../benchmarks'
 import {Page} from './components/Page'
 
 type BenchmarkPageProps = {
@@ -12,29 +12,9 @@ export const dynamicParams = false
 
 export default async function BenchmarkPage(props: BenchmarkPageProps) {
   const {id} = await props.params
-  const [benchmark, runs] = await Promise.all([get(id), listBenchmarkRuns(id)])
-  const results = getBenchmarkPageResults(benchmark, runs[0])
+  const {benchmark, results, runs} = await getBenchmarkPageData(id)
 
-  return (
-    <Page
-      benchmark={benchmark}
-      results={results}
-      runs={runs.map(run => {
-        const trials = [...run.output.trials.values()]
-        return {
-          id: run.id,
-          name: run.name,
-          resultCount: trials.length,
-          passedTests: trials.reduce((total, trial) => {
-            return total + trial.testResults.numPassedTests
-          }, 0),
-          totalTests: trials.reduce((total, trial) => {
-            return total + trial.testResults.numTotalTests
-          }, 0),
-        }
-      })}
-    />
-  )
+  return <Page benchmark={benchmark} results={results} runs={runs} />
 }
 
 export async function generateStaticParams() {
diff --git a/website/src/app/components/BaselineOverview.tsx b/website/src/app/components/BaselineOverview.tsx
deleted file mode 100644
index a019098e..00000000
--- a/website/src/app/components/BaselineOverview.tsx
+++ /dev/null
@@ -1,33 +0,0 @@
-'use client'
-
-import {Stack} from '@primer/react'
-import type {BaselineTrendPoint} from './BaselineTrends'
-import {BaselineTrends} from './BaselineTrends'
-import type {BaselineResult} from './Index'
-import {BaselineResultsTable} from './Index'
-
-export function BaselineOverview({
-  results,
-  trends,
-}: {
-  results: Array<BaselineResult>
-  trends: Array<BaselineTrendPoint>
-}) {
-  return (
-    <Stack padding="normal" gap="spacious">
-      <BaselineResultsTable
-        description={
-          <>
-            Each metric is the average Recommended result across all baseline scenarios, followed by the percent change
-            from the average Control result in parentheses.
-          </>
-        }
-        heading="Baseline overview"
-        headingId="aggregate-baseline-heading"
-        headingLevel="h1"
-        results={results}
-      />
-      <BaselineTrends points={trends} scenarioId="aggregate" />
-    </Stack>
-  )
-}
diff --git a/website/src/app/components/BaselineTrends.tsx b/website/src/app/components/BaselineTrends.tsx
deleted file mode 100644
index 5fe39887..00000000
--- a/website/src/app/components/BaselineTrends.tsx
+++ /dev/null
@@ -1,581 +0,0 @@
-'use client'
-
-import {Button, FormControl, Heading, Select, Stack, Text} from '@primer/react'
-import {DataTable, Table} from '@primer/react/experimental'
-import {type MouseEvent, useState} from 'react'
-import styles from './BaselineTrends.module.css'
-
-type BaselineTrendMetric = {
-  value: number | null
-  raw: string
-  change: number | null
-  controlValue: number | null
-  controlRaw: string | null
-}
-
-type BaselineTrendMetricId =
-  'tests' | 'turns' | 'outputTokens' | 'premiumRequests' | 'apiDuration' | 'sessionDuration' | 'toolCalls'
-
-export type BaselineTrendPoint = {
-  id: string
-  date: string
-  scenarioId: string
-  model: string
-  reasoningEffort: string
-  metrics: Record<BaselineTrendMetricId, BaselineTrendMetric>
-}
-
-const metrics: Array<{
-  id: BaselineTrendMetricId
-  label: string
-}> = [
-  {id: 'tests', label: 'Tests passed'},
-  {id: 'turns', label: 'Turns'},
-  {id: 'outputTokens', label: 'Output tokens'},
-  {id: 'premiumRequests', label: 'Premium requests'},
-  {id: 'apiDuration', label: 'API time'},
-  {id: 'sessionDuration', label: 'Session time'},
-  {id: 'toolCalls', label: 'Tool calls'},
-]
-
-const lineStyles = [
-  {color: 'var(--data-blue-color-emphasis, var(--data-blue-color))', dash: undefined},
-  {color: 'var(--data-green-color-emphasis, var(--data-green-color))', dash: '7 3'},
-  {color: 'var(--data-orange-color-emphasis, var(--data-orange-color))', dash: '2 3'},
-  {color: 'var(--data-pink-color-emphasis, var(--data-pink-color))', dash: '10 3 2 3'},
-  {color: 'var(--data-yellow-color-emphasis, var(--data-yellow-color))', dash: '4 4'},
-  {color: 'var(--data-red-color-emphasis, var(--data-red-color))', dash: '12 4'},
-]
-
-const percentFormatter = new Intl.NumberFormat('en-US', {
-  maximumFractionDigits: 1,
-})
-
-const numberFormatter = new Intl.NumberFormat('en-US', {
-  maximumFractionDigits: 1,
-})
-
-function getSeriesName(point: BaselineTrendPoint) {
-  return point.reasoningEffort === '—' ? point.model : `${point.model} (${point.reasoningEffort})`
-}
-
-function formatChange(change: number | null) {
-  if (change === null) {
-    return '—'
-  }
-
-  return `${change > 0 ? '+' : ''}${percentFormatter.format(change)}%`
-}
-
-function TrendChart({
-  dates,
-  metric,
-  points,
-  series,
-  highlightedSeries,
-  highlightedTreatment,
-}: {
-  dates: Array<string>
-  metric: (typeof metrics)[number]
-  points: Array<BaselineTrendPoint>
-  series: Array<string>
-  highlightedSeries: string | null
-  highlightedTreatment: 'recommended' | 'control' | null
-}) {
-  const [activeTrend, setActiveTrend] = useState<{
-    pointId: string
-    treatment: 'recommended' | 'control'
-  } | null>(null)
-  const width = 520
-  const height = 240
-  const padding = {top: 16, right: 16, bottom: 52, left: 48}
-  const plotWidth = width - padding.left - padding.right
-  const plotHeight = height - padding.top - padding.bottom
-  const values = points.flatMap(point => {
-    const value = point.metrics[metric.id]
-    return [value.value, value.controlValue].filter(item => item !== null)
-  })
-  const rawMin = values.length > 0 ? Math.min(...values) : 0
-  const rawMax = values.length > 0 ? Math.max(...values) : 0
-  const range = rawMax - rawMin
-  const min = Math.max(0, rawMin - (range || Math.max(rawMax, 1)) * 0.1)
-  const max = rawMax + (range || Math.max(rawMax, 1)) * 0.1
-  const getX = (date: string) =>
-    padding.left + (dates.length === 1 ? plotWidth / 2 : (dates.indexOf(date) / (dates.length - 1)) * plotWidth)
-  const getY = (value: number) => padding.top + (1 - (value - min) / (max - min)) * plotHeight
-  const activePoint = points.find(point => point.id === activeTrend?.pointId)
-  const activeMetric = activePoint?.metrics[metric.id]
-  const activeValue =
-    activeTrend?.treatment === 'control'
-      ? (activeMetric?.controlValue ?? undefined)
-      : (activeMetric?.value ?? undefined)
-  const activeRaw = activeTrend?.treatment === 'control' ? activeMetric?.controlRaw : activeMetric?.raw
-  const tooltipWidth = 248
-  const tooltipHeight = 52
-  const tooltipX = activePoint
-    ? Math.min(Math.max(getX(activePoint.date) - tooltipWidth / 2, padding.left), width - padding.right - tooltipWidth)
-    : 0
-  const tooltipY =
-    activePoint && activeValue !== undefined
-      ? Math.max(padding.top, Math.min(getY(activeValue) - tooltipHeight - 10, height - padding.bottom - tooltipHeight))
-      : 0
-  const tooltipId = `${metric.id}-chart-tooltip`
-  const controlOpacity = highlightedTreatment === 'recommended' ? 0.15 : highlightedTreatment === 'control' ? 1 : 0.4
-  const recommendedOpacity = highlightedTreatment === 'control' ? 0.15 : 1
-  const activateClosestPoint = (
-    event: MouseEvent<SVGPathElement>,
-    seriesPoints: Array<BaselineTrendPoint>,
-    treatment: 'recommended' | 'control',
-  ) => {
-    const svg = event.currentTarget.ownerSVGElement
-    if (!svg || seriesPoints.length === 0) {
-      return
-    }
-
-    const bounds = svg.getBoundingClientRect()
-    const pointerX = ((event.clientX - bounds.left) / bounds.width) * width
-    const closestPoint = seriesPoints.reduce((closest, point) => {
-      return Math.abs(getX(point.date) - pointerX) < Math.abs(getX(closest.date) - pointerX) ? point : closest
-    })
-    setActiveTrend({pointId: closestPoint.id, treatment})
-  }
-
-  return (
-    <article className={styles.chart}>
-      <Heading as="h3" variant="small">
-        {metric.label}
-      </Heading>
-      <svg
-        aria-label={`${metric.label} by model over time`}
-        className={styles.chartSvg}
-        role="img"
-        viewBox={`0 0 ${width} ${height}`}
-      >
-        {[max, (max + min) / 2, min].map(value => {
-          const y = getY(value)
-          return (
-            <g key={value}>
-              <line className={styles.gridLine} x1={padding.left} x2={width - padding.right} y1={y} y2={y} />
-              <text className={styles.axisLabel} textAnchor="end" x={padding.left - 8} y={y + 4}>
-                {numberFormatter.format(value)}
-              </text>
-            </g>
-          )
-        })}
-        {dates.map(date => (
-          <text className={styles.axisLabel} key={date} textAnchor="middle" x={getX(date)} y={height - 24}>
-            {date.slice(5)}
-          </text>
-        ))}
-        <text className={styles.axisLabel} textAnchor="middle" x={padding.left + plotWidth / 2} y={height - 4}>
-          Run date
-        </text>
-        {series.map((seriesName, index) => {
-          const seriesPoints = points
-            .filter(point => getSeriesName(point) === seriesName)
-            .toSorted((a, b) => a.date.localeCompare(b.date))
-          const controlPoints = seriesPoints.filter(point => point.metrics[metric.id].controlValue !== null)
-          const recommendedPoints = seriesPoints.filter(point => point.metrics[metric.id].value !== null)
-          const style = lineStyles[index % lineStyles.length]
-          const recommendedPath = recommendedPoints
-            .map((point, pointIndex) => {
-              return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(point.metrics[metric.id].value ?? 0)}`
-            })
-            .join(' ')
-          const controlPath = controlPoints
-            .map((point, pointIndex) => {
-              return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(
-                point.metrics[metric.id].controlValue ?? 0,
-              )}`
-            })
-            .join(' ')
-
-          return (
-            <g
-              className={styles.seriesGroup}
-              key={seriesName}
-              opacity={highlightedSeries && highlightedSeries !== seriesName ? 0.15 : 1}
-            >
-              {controlPoints.length > 0 ? (
-                <>
-                  <path
-                    className={`${styles.controlLine} ${styles.treatmentGroup}`}
-                    d={controlPath}
-                    fill="none"
-                    opacity={controlOpacity}
-                    stroke={style.color}
-                    strokeDasharray={style.dash}
-                    strokeLinecap="round"
-                    strokeLinejoin="round"
-                    strokeWidth="1.5"
-                  />
-                  <path
-                    aria-describedby={
-                      activePoint && getSeriesName(activePoint) === seriesName && activeTrend?.treatment === 'control'
-                        ? tooltipId
-                        : undefined
-                    }
-                    aria-label={`${seriesName}, Control ${metric.label} trend`}
-                    className={styles.lineHitTarget}
-                    d={controlPath}
-                    fill="none"
-                    onBlur={() => setActiveTrend(null)}
-                    onFocus={() => {
-                      const pointId = controlPoints.at(-1)?.id
-                      setActiveTrend(pointId ? {pointId, treatment: 'control'} : null)
-                    }}
-                    onMouseLeave={() => setActiveTrend(null)}
-                    onMouseMove={event => activateClosestPoint(event, controlPoints, 'control')}
-                    stroke="transparent"
-                    strokeWidth="16"
-                    tabIndex={0}
-                  />
-                </>
-              ) : null}
-              {recommendedPoints.length > 0 ? (
-                <>
-                  <path
-                    className={`${styles.seriesLine} ${styles.treatmentGroup}`}
-                    d={recommendedPath}
-                    fill="none"
-                    opacity={recommendedOpacity}
-                    stroke={style.color}
-                    strokeDasharray={style.dash}
-                    strokeLinecap="round"
-                    strokeLinejoin="round"
-                    strokeWidth="2"
-                  />
-                  <path
-                    aria-describedby={
-                      activePoint &&
-                      getSeriesName(activePoint) === seriesName &&
-                      activeTrend?.treatment === 'recommended'
-                        ? tooltipId
-                        : undefined
-                    }
-                    aria-label={`${seriesName}, Recommended ${metric.label} trend`}
-                    className={styles.lineHitTarget}
-                    d={recommendedPath}
-                    fill="none"
-                    onBlur={() => setActiveTrend(null)}
-                    onFocus={() => {
-                      const pointId = recommendedPoints.at(-1)?.id
-                      setActiveTrend(pointId ? {pointId, treatment: 'recommended'} : null)
-                    }}
-                    onMouseLeave={() => setActiveTrend(null)}
-                    onMouseMove={event => activateClosestPoint(event, recommendedPoints, 'recommended')}
-                    stroke="transparent"
-                    strokeWidth="16"
-                    tabIndex={0}
-                  />
-                </>
-              ) : null}
-              {seriesPoints.map(point => {
-                const value = point.metrics[metric.id]
-                return (
-                  <g aria-hidden="true" key={point.id}>
-                    {value.controlValue !== null ? (
-                      <circle
-                        className={`${styles.controlPoint} ${styles.treatmentGroup}`}
-                        cx={getX(point.date)}
-                        cy={getY(value.controlValue)}
-                        fill="var(--bgColor-default)"
-                        opacity={controlOpacity}
-                        r={activeTrend?.pointId === point.id && activeTrend.treatment === 'control' ? '4.5' : '3'}
-                        stroke={style.color}
-                        strokeWidth="1.5"
-                      />
-                    ) : null}
-                    {value.value !== null ? (
-                      <circle
-                        className={styles.treatmentGroup}
-                        cx={getX(point.date)}
-                        cy={getY(value.value)}
-                        fill="var(--bgColor-default)"
-                        opacity={recommendedOpacity}
-                        r={activeTrend?.pointId === point.id && activeTrend.treatment === 'recommended' ? '5' : '3.5'}
-                        stroke={style.color}
-                        strokeWidth="2"
-                      />
-                    ) : null}
-                  </g>
-                )
-              })}
-            </g>
-          )
-        })}
-        {activePoint && activeMetric && activeRaw ? (
-          <g className={styles.tooltip} id={tooltipId} role="tooltip" transform={`translate(${tooltipX} ${tooltipY})`}>
-            <rect className={styles.tooltipBackground} height={tooltipHeight} width={tooltipWidth} />
-            <text className={styles.tooltipTitle} x="10" y="20">
-              {getSeriesName(activePoint)} · {activeTrend?.treatment === 'control' ? 'Control' : 'Recommended'}
-            </text>
-            <text className={styles.tooltipValue} x="10" y="40">
-              {activePoint.date}: {activeRaw}
-              {activeTrend?.treatment === 'recommended' ? ` (${formatChange(activeMetric.change)})` : ''}
-            </text>
-          </g>
-        ) : null}
-      </svg>
-    </article>
-  )
-}
-
-function ControlTable({
-  dates,
-  metric,
-  points,
-  series,
-}: {
-  dates: Array<string>
-  metric: (typeof metrics)[number]
-  points: Array<BaselineTrendPoint>
-  series: Array<string>
-}) {
-  const rows = series.map(seriesName => ({
-    id: seriesName,
-    model: seriesName,
-    values: Object.fromEntries(
-      dates.map(date => {
-        const point = points.find(item => getSeriesName(item) === seriesName && item.date === date)
-        return [date, point?.metrics[metric.id]]
-      }),
-    ),
-  }))
-
-  return (
-    <DataTable
-      aria-labelledby="baseline-heatmap-heading"
-      aria-describedby="baseline-heatmap-description"
-      cellPadding="condensed"
-      columns={[
-        {
-          id: 'model',
-          header: 'Model',
-          field: 'model',
-          rowHeader: true,
-        },
-        ...dates.map(date => ({
-          id: date,
-          header: () => <time dateTime={date}>{date}</time>,
-          field: 'values' as const,
-          align: 'end' as const,
-          renderCell: (row: (typeof rows)[number]) => {
-            const value = row.values[date]
-
-            return value ? (
-              <span className={styles.metricValue}>
-                {value.raw} <span className={styles.change}>({formatChange(value.change)})</span>
-              </span>
-            ) : (
-              '—'
-            )
-          },
-        })),
-      ]}
-      data={rows}
-    />
-  )
-}
-
-export function BaselineTrends({points, scenarioId}: {points: Array<BaselineTrendPoint>; scenarioId: string}) {
-  const [heatmapMetricId, setHeatmapMetricId] = useState<BaselineTrendMetricId>('tests')
-  const [selectedSeries, setSelectedSeries] = useState<string | null>(null)
-  const [hoveredSeries, setHoveredSeries] = useState<string | null>(null)
-  const [selectedTreatment, setSelectedTreatment] = useState<'recommended' | 'control' | null>(null)
-  const [hoveredTreatment, setHoveredTreatment] = useState<'recommended' | 'control' | null>(null)
-  const scenarioPoints = points.filter(point => point.scenarioId === scenarioId)
-  const dates = [...new Set(scenarioPoints.map(point => point.date))].toSorted()
-  const series = [...new Set(scenarioPoints.map(getSeriesName))].toSorted()
-  const heatmapMetric = metrics.find(metric => metric.id === heatmapMetricId) ?? metrics[0]
-  const highlightedSeries = hoveredSeries ?? selectedSeries
-  const highlightedTreatment = hoveredTreatment ?? selectedTreatment
-  const rawTrendRows = scenarioPoints
-    .toSorted((a, b) => a.date.localeCompare(b.date) || getSeriesName(a).localeCompare(getSeriesName(b)))
-    .map(point => ({
-      ...point,
-      seriesName: getSeriesName(point),
-    }))
-
-  if (scenarioPoints.length === 0) {
-    return null
-  }
-
-  return (
-    <section className={styles.section} aria-labelledby="baseline-trends-heading">
-      <Stack gap="spacious">
-        <Stack gap="condensed">
-          <Heading as="h2" id="baseline-trends-heading">
-            Trends
-          </Heading>
-          <Text as="p" className={styles.description}>
-            Strong lines show Recommended results and muted lines show Control over time. The comparison table shows
-            each Recommended result relative to Control.
-          </Text>
-        </Stack>
-        <Stack gap="normal">
-          <div className={styles.legendGroups}>
-            <ul className={styles.legend} aria-label="Model chart legend">
-              {series.map((seriesName, index) => {
-                const style = lineStyles[index % lineStyles.length]
-                return (
-                  <li key={seriesName}>
-                    <Button
-                      aria-pressed={selectedSeries === seriesName}
-                      className={styles.legendButton}
-                      onBlur={() => setHoveredSeries(null)}
-                      onClick={() => setSelectedSeries(current => (current === seriesName ? null : seriesName))}
-                      onFocus={() => setHoveredSeries(seriesName)}
-                      onMouseEnter={() => setHoveredSeries(seriesName)}
-                      onMouseLeave={() => setHoveredSeries(null)}
-                      size="small"
-                      variant="invisible"
-                    >
-                      <svg aria-hidden="true" height="8" width="28">
-                        <line
-                          stroke={style.color}
-                          strokeDasharray={style.dash}
-                          strokeWidth="2"
-                          x1="0"
-                          x2="28"
-                          y1="4"
-                          y2="4"
-                        />
-                      </svg>
-                      {seriesName}
-                    </Button>
-                  </li>
-                )
-              })}
-            </ul>
-            <ul className={styles.treatmentLegend} aria-label="Treatment chart legend">
-              <li>
-                <Button
-                  aria-pressed={selectedTreatment === 'recommended'}
-                  className={styles.legendButton}
-                  onBlur={() => setHoveredTreatment(null)}
-                  onClick={() => setSelectedTreatment(current => (current === 'recommended' ? null : 'recommended'))}
-                  onFocus={() => setHoveredTreatment('recommended')}
-                  onMouseEnter={() => setHoveredTreatment('recommended')}
-                  onMouseLeave={() => setHoveredTreatment(null)}
-                  size="small"
-                  variant="invisible"
-                >
-                  <svg aria-hidden="true" height="8" width="28">
-                    <line stroke="var(--fgColor-default)" strokeWidth="2" x1="0" x2="28" y1="4" y2="4" />
-                  </svg>
-                  Recommended
-                </Button>
-              </li>
-              <li>
-                <Button
-                  aria-pressed={selectedTreatment === 'control'}
-                  className={styles.legendButton}
-                  onBlur={() => setHoveredTreatment(null)}
-                  onClick={() => setSelectedTreatment(current => (current === 'control' ? null : 'control'))}
-                  onFocus={() => setHoveredTreatment('control')}
-                  onMouseEnter={() => setHoveredTreatment('control')}
-                  onMouseLeave={() => setHoveredTreatment(null)}
-                  size="small"
-                  variant="invisible"
-                >
-                  <svg aria-hidden="true" height="8" width="28">
-                    <line
-                      opacity="0.4"
-                      stroke="var(--fgColor-default)"
-                      strokeWidth="1.5"
-                      x1="0"
-                      x2="28"
-                      y1="4"
-                      y2="4"
-                    />
-                  </svg>
-                  Control
-                </Button>
-              </li>
-            </ul>
-          </div>
-          <div className={styles.chartGrid}>
-            {metrics.map(metric => (
-              <TrendChart
-                dates={dates}
-                highlightedSeries={highlightedSeries}
-                highlightedTreatment={highlightedTreatment}
-                key={metric.id}
-                metric={metric}
-                points={scenarioPoints}
-                series={series}
-              />
-            ))}
-          </div>
-        </Stack>
-        <Stack gap="normal">
-          <Table.Container>
-            <Table.Title as="h3" id="baseline-heatmap-heading">
-              Change from Control
-            </Table.Title>
-            <Table.Subtitle as="p" id="baseline-heatmap-description">
-              Recommended results are shown first, followed by the percent change from Control in parentheses.
-            </Table.Subtitle>
-            <Table.Actions>
-              <FormControl>
-                <FormControl.Label>Metric</FormControl.Label>
-                <Select
-                  onChange={event => setHeatmapMetricId(event.currentTarget.value as BaselineTrendMetricId)}
-                  value={heatmapMetricId}
-                >
-                  {metrics.map(metric => (
-                    <Select.Option key={metric.id} value={metric.id}>
-                      {metric.label}
-                    </Select.Option>
-                  ))}
-                </Select>
-              </FormControl>
-            </Table.Actions>
-            <ControlTable dates={dates} metric={heatmapMetric} points={scenarioPoints} series={series} />
-          </Table.Container>
-          <details className={styles.details}>
-            <summary className={styles.summary} id="raw-trend-data-heading">
-              View raw trend data
-            </summary>
-            <div className={styles.rawData}>
-              <DataTable
-                aria-labelledby="raw-trend-data-heading"
-                cellPadding="condensed"
-                columns={[
-                  {
-                    id: 'date',
-                    header: 'Date',
-                    field: 'date',
-                    renderCell: row => <time dateTime={row.date}>{row.date}</time>,
-                  },
-                  {
-                    id: 'model',
-                    header: 'Model',
-                    field: 'seriesName',
-                    rowHeader: true,
-                  },
-                  ...metrics.map(metric => ({
-                    id: metric.id,
-                    header: metric.label,
-                    field: 'metrics' as const,
-                    align: 'end' as const,
-                    renderCell: (row: (typeof rawTrendRows)[number]) => {
-                      const value = row.metrics[metric.id]
-                      return (
-                        <span className={styles.metricValue}>
-                          {value.raw} <span className={styles.change}>({formatChange(value.change)})</span>
-                        </span>
-                      )
-                    },
-                  })),
-                ]}
-                data={rawTrendRows}
-              />
-            </div>
-          </details>
-        </Stack>
-      </Stack>
-    </section>
-  )
-}
diff --git a/website/src/app/components/BenchmarkOverview.module.css b/website/src/app/components/BenchmarkOverview.module.css
new file mode 100644
index 00000000..95d258e3
--- /dev/null
+++ b/website/src/app/components/BenchmarkOverview.module.css
@@ -0,0 +1,8 @@
+.metric {
+  font: var(--text-codeInline-shorthand);
+  white-space: nowrap;
+}
+
+.change {
+  color: var(--fgColor-muted);
+}
diff --git a/website/src/app/components/BenchmarkOverview.tsx b/website/src/app/components/BenchmarkOverview.tsx
new file mode 100644
index 00000000..42a189c0
--- /dev/null
+++ b/website/src/app/components/BenchmarkOverview.tsx
@@ -0,0 +1,142 @@
+'use client'
+
+import {Stack} from '@primer/react'
+import {Blankslate, DataTable, Table} from '@primer/react/experimental'
+import type {Route} from 'next'
+import type {BenchmarkOverviewData} from '../../benchmark-results'
+import type {Benchmark} from '../../benchmarks'
+import {Link} from '../../components/Link'
+import {BenchmarkTrends} from './BenchmarkTrends'
+import styles from './BenchmarkOverview.module.css'
+
+type BenchmarkOverviewResult = BenchmarkOverviewData['results'][number]
+
+function Metric({value}: {value: string}) {
+  const match = /^(.*) \((.*)\)$/.exec(value)
+  if (!match) {
+    return value
+  }
+
+  return (
+    <span className={styles.metric}>
+      {match[1]} <span className={styles.change}>({match[2]})</span>
+    </span>
+  )
+}
+
+function BenchmarkResultsTable({
+  benchmark,
+  date,
+  results,
+}: {
+  benchmark: Benchmark
+  date: string | null
+  results: Array<BenchmarkOverviewResult>
+}) {
+  if (results.length === 0) {
+    return (
+      <>
+        <h1 className="text-title-medium">{benchmark.name} benchmark</h1>
+        <Blankslate border>
+          <Blankslate.Heading as="h2">No results</Blankslate.Heading>
+          <Blankslate.Description>No results have been recorded for this benchmark yet.</Blankslate.Description>
+        </Blankslate>
+      </>
+    )
+  }
+
+  return (
+    <Table.Container>
+      <Table.Title as="h1" className="text-title-medium" id="benchmark-overview-heading">
+        {benchmark.name} benchmark
+      </Table.Title>
+      <Table.Subtitle as="p" id="benchmark-overview-description">
+        Each metric is the Benchmark result across all capabilities, followed by the percent change from Control in
+        parentheses. Models are ranked by test performance, followed by resource usage.
+        {date ? (
+          <>
+            {' '}
+            Latest results:{' '}
+            <Link href={`/benchmarks/${benchmark.id}/runs/${date}` as Route}>
+              <time dateTime={date}>{date}</time>
+            </Link>
+            .
+          </>
+        ) : null}
+      </Table.Subtitle>
+      <DataTable
+        aria-describedby="benchmark-overview-description"
+        aria-labelledby="benchmark-overview-heading"
+        cellPadding="condensed"
+        columns={[
+          {
+            id: 'model',
+            header: 'Model',
+            field: 'model',
+            rowHeader: true,
+          },
+          {
+            id: 'reasoning-effort',
+            header: 'Effort',
+            field: 'reasoningEffort',
+          },
+          {
+            id: 'tests',
+            header: 'Tests',
+            field: 'comparison',
+            align: 'end',
+            renderCell: row => {
+              return <Metric value={row.comparison.tests} />
+            },
+          },
+          {
+            id: 'output-tokens',
+            header: 'Output tokens',
+            field: 'comparison',
+            align: 'end',
+            renderCell: row => {
+              return <Metric value={row.comparison.outputTokens} />
+            },
+          },
+          {
+            id: 'premium-requests',
+            header: 'Premium requests',
+            field: 'comparison',
+            align: 'end',
+            renderCell: row => {
+              return <Metric value={row.comparison.premiumRequests} />
+            },
+          },
+          {
+            id: 'session-time',
+            header: 'Session time',
+            field: 'comparison',
+            align: 'end',
+            renderCell: row => {
+              return <Metric value={row.comparison.sessionTime} />
+            },
+          },
+          {
+            id: 'api-time',
+            header: 'API time',
+            field: 'comparison',
+            align: 'end',
+            renderCell: row => {
+              return <Metric value={row.comparison.apiTime} />
+            },
+          },
+        ]}
+        data={results}
+      />
+    </Table.Container>
+  )
+}
+
+export function BenchmarkOverview({benchmark, overview}: {benchmark: Benchmark; overview: BenchmarkOverviewData}) {
+  return (
+    <Stack padding="normal" gap="spacious">
+      <BenchmarkResultsTable benchmark={benchmark} date={overview.date} results={overview.results} />
+      <BenchmarkTrends points={overview.trends} />
+    </Stack>
+  )
+}
diff --git a/website/src/app/components/BaselineTrends.module.css b/website/src/app/components/BenchmarkTrends.module.css
similarity index 50%
rename from website/src/app/components/BaselineTrends.module.css
rename to website/src/app/components/BenchmarkTrends.module.css
index fbe85381..2ab5bb48 100644
--- a/website/src/app/components/BaselineTrends.module.css
+++ b/website/src/app/components/BenchmarkTrends.module.css
@@ -3,10 +3,7 @@
 }
 
 .change {
-  display: inline-block;
-  min-width: 8ch;
   color: var(--fgColor-muted);
-  text-align: right;
 }
 
 .chartGrid {
@@ -43,43 +40,6 @@
   pointer-events: none;
 }
 
-.controlLine,
-.controlPoint {
-  pointer-events: none;
-}
-
-.lineHitTarget {
-  cursor: crosshair;
-  pointer-events: stroke;
-}
-
-.lineHitTarget:focus-visible {
-  outline: none;
-  stroke: var(--focus-outline-color);
-}
-
-.tooltip {
-  pointer-events: none;
-}
-
-.tooltipBackground {
-  fill: var(--bgColor-default);
-  stroke: var(--borderColor-muted);
-  stroke-width: 1;
-  rx: var(--borderRadius-medium);
-}
-
-.tooltipTitle {
-  fill: var(--fgColor-default);
-  font-size: var(--text-body-size-small);
-  font-weight: var(--base-text-weight-semibold);
-}
-
-.tooltipValue {
-  fill: var(--fgColor-muted);
-  font-size: var(--text-body-size-small);
-}
-
 .legend {
   display: flex;
   flex-wrap: wrap;
@@ -89,39 +49,6 @@
   list-style: none;
 }
 
-.legendGroups {
-  display: flex;
-  flex-wrap: wrap;
-  align-items: start;
-  justify-content: space-between;
-  gap: var(--stack-gap-normal);
-}
-
-.legendButton {
-  gap: var(--stack-gap-condensed);
-}
-
-.treatmentLegend {
-  display: flex;
-  flex-wrap: wrap;
-  gap: var(--stack-gap-normal);
-  padding: 0;
-  margin: 0;
-  list-style: none;
-}
-
-.seriesGroup,
-.treatmentGroup {
-  transition: opacity var(--motion-transition-hover);
-}
-
-@media (prefers-reduced-motion: reduce) {
-  .seriesGroup,
-  .treatmentGroup {
-    transition: none;
-  }
-}
-
 .legendItem {
   display: flex;
   align-items: center;
@@ -133,6 +60,7 @@
   display: inline-flex;
   width: 100%;
   justify-content: flex-end;
+  gap: 0.5ch;
   font: var(--text-codeInline-shorthand);
   white-space: nowrap;
 }
diff --git a/website/src/app/components/BenchmarkTrends.tsx b/website/src/app/components/BenchmarkTrends.tsx
new file mode 100644
index 00000000..fe3b8eca
--- /dev/null
+++ b/website/src/app/components/BenchmarkTrends.tsx
@@ -0,0 +1,368 @@
+'use client'
+
+import {FormControl, Heading, Select, Stack, Text} from '@primer/react'
+import {DataTable, Table} from '@primer/react/experimental'
+import {useState} from 'react'
+import type {BenchmarkTrendMetricId, BenchmarkTrendPoint} from '../../benchmark-results'
+import styles from './BenchmarkTrends.module.css'
+
+const metrics: Array<{
+  id: BenchmarkTrendMetricId
+  label: string
+}> = [
+  {id: 'tests', label: 'Tests passed'},
+  {id: 'outputTokens', label: 'Output tokens'},
+  {id: 'premiumRequests', label: 'Premium requests'},
+  {id: 'sessionTime', label: 'Session time'},
+  {id: 'apiTime', label: 'API time'},
+]
+
+const lineStyles = [
+  {color: 'var(--data-blue-color-emphasis, var(--data-blue-color))', dash: undefined},
+  {color: 'var(--data-green-color-emphasis, var(--data-green-color))', dash: '7 3'},
+  {color: 'var(--data-orange-color-emphasis, var(--data-orange-color))', dash: '2 3'},
+  {color: 'var(--data-pink-color-emphasis, var(--data-pink-color))', dash: '10 3 2 3'},
+  {color: 'var(--data-yellow-color-emphasis, var(--data-yellow-color))', dash: '4 4'},
+  {color: 'var(--data-red-color-emphasis, var(--data-red-color))', dash: '12 4'},
+]
+
+const numberFormatter = new Intl.NumberFormat('en-US', {
+  maximumFractionDigits: 1,
+})
+
+function getSeriesName(point: BenchmarkTrendPoint): string {
+  return point.reasoningEffort ? `${point.model} (${point.reasoningEffort})` : point.model
+}
+
+function formatChange(change: number | null): string {
+  if (change === null) {
+    return 'N/A'
+  }
+
+  return `${change > 0 ? '+' : ''}${numberFormatter.format(change)}%`
+}
+
+function TrendChart({
+  dates,
+  metric,
+  points,
+  series,
+}: {
+  dates: Array<string>
+  metric: (typeof metrics)[number]
+  points: Array<BenchmarkTrendPoint>
+  series: Array<string>
+}) {
+  const width = 520
+  const height = 240
+  const padding = {top: 16, right: 16, bottom: 52, left: 48}
+  const plotWidth = width - padding.left - padding.right
+  const plotHeight = height - padding.top - padding.bottom
+  const values = points.flatMap(point => {
+    const value = point.metrics[metric.id]
+    return [value.value, value.controlValue].filter((item): item is number => {
+      return item !== null
+    })
+  })
+  const rawMin = values.length > 0 ? Math.min(...values) : 0
+  const rawMax = values.length > 0 ? Math.max(...values) : 0
+  const range = rawMax - rawMin
+  const margin = (range || Math.max(rawMax, 1)) * 0.1
+  const min = Math.max(0, rawMin - margin)
+  const max = rawMax + margin
+  const getX = (date: string): number => {
+    return padding.left + (dates.length === 1 ? plotWidth / 2 : (dates.indexOf(date) / (dates.length - 1)) * plotWidth)
+  }
+  const getY = (value: number): number => {
+    return padding.top + (1 - (value - min) / (max - min)) * plotHeight
+  }
+
+  return (
+    <article className={styles.chart}>
+      <Heading as="h3" variant="small">
+        {metric.label}
+      </Heading>
+      <svg
+        aria-label={`${metric.label} by model over time`}
+        className={styles.chartSvg}
+        role="img"
+        viewBox={`0 0 ${width} ${height}`}
+      >
+        {[max, (max + min) / 2, min].map(value => {
+          const y = getY(value)
+          return (
+            <g key={value}>
+              <line className={styles.gridLine} x1={padding.left} x2={width - padding.right} y1={y} y2={y} />
+              <text className={styles.axisLabel} textAnchor="end" x={padding.left - 8} y={y + 4}>
+                {numberFormatter.format(value)}
+              </text>
+            </g>
+          )
+        })}
+        {dates.map(date => {
+          return (
+            <text className={styles.axisLabel} key={date} textAnchor="middle" x={getX(date)} y={height - 24}>
+              {date.slice(5)}
+            </text>
+          )
+        })}
+        <text className={styles.axisLabel} textAnchor="middle" x={padding.left + plotWidth / 2} y={height - 4}>
+          Run date
+        </text>
+        {series.map((seriesName, index) => {
+          const seriesPoints = points
+            .filter(point => {
+              return getSeriesName(point) === seriesName
+            })
+            .toSorted((a, b) => {
+              return a.date.localeCompare(b.date)
+            })
+          const style = lineStyles[index % lineStyles.length]
+
+          return (
+            <g key={seriesName}>
+              {(['controlValue', 'value'] as const).map((valueKey, treatmentIndex) => {
+                const treatmentPoints = seriesPoints.filter(point => {
+                  return point.metrics[metric.id][valueKey] !== null
+                })
+                const path = treatmentPoints
+                  .map((point, pointIndex) => {
+                    const value = point.metrics[metric.id][valueKey] ?? 0
+                    return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(value)}`
+                  })
+                  .join(' ')
+
+                return (
+                  <g key={valueKey} opacity={treatmentIndex === 0 ? 0.4 : 1}>
+                    <path
+                      className={styles.seriesLine}
+                      d={path}
+                      fill="none"
+                      stroke={style.color}
+                      strokeDasharray={style.dash}
+                      strokeLinecap="round"
+                      strokeLinejoin="round"
+                      strokeWidth={treatmentIndex === 0 ? '1.5' : '2'}
+                    />
+                    {treatmentPoints.map(point => {
+                      const value = point.metrics[metric.id][valueKey] ?? 0
+                      return (
+                        <circle
+                          cx={getX(point.date)}
+                          cy={getY(value)}
+                          fill="var(--bgColor-default)"
+                          key={point.id}
+                          r={treatmentIndex === 0 ? '3' : '3.5'}
+                          stroke={style.color}
+                          strokeWidth={treatmentIndex === 0 ? '1.5' : '2'}
+                        >
+                          <title>
+                            {seriesName}, {treatmentIndex === 0 ? 'Control' : 'Benchmark'}, {point.date}:{' '}
+                            {point.metrics[metric.id][treatmentIndex === 0 ? 'controlRaw' : 'raw']}
+                          </title>
+                        </circle>
+                      )
+                    })}
+                  </g>
+                )
+              })}
+            </g>
+          )
+        })}
+      </svg>
+    </article>
+  )
+}
+
+export function BenchmarkTrends({points}: {points: Array<BenchmarkTrendPoint>}) {
+  const [tableMetricId, setTableMetricId] = useState<BenchmarkTrendMetricId>('tests')
+  const dates = [
+    ...new Set(
+      points.map(point => {
+        return point.date
+      }),
+    ),
+  ].toSorted()
+  const series = [
+    ...new Set(
+      points.map(point => {
+        return getSeriesName(point)
+      }),
+    ),
+  ].toSorted()
+  const tableMetric =
+    metrics.find(metric => {
+      return metric.id === tableMetricId
+    }) ?? metrics[0]
+  const comparisonRows = series.map(seriesName => {
+    return {
+      id: seriesName,
+      model: seriesName,
+      values: Object.fromEntries(
+        dates.map(date => {
+          const point = points.find(candidate => {
+            return getSeriesName(candidate) === seriesName && candidate.date === date
+          })
+          return [date, point?.metrics[tableMetric.id]]
+        }),
+      ),
+    }
+  })
+  const rawRows = points
+    .toSorted((a, b) => {
+      return a.date.localeCompare(b.date) || getSeriesName(a).localeCompare(getSeriesName(b))
+    })
+    .map(point => {
+      return {...point, seriesName: getSeriesName(point)}
+    })
+
+  if (points.length === 0) {
+    return null
+  }
+
+  return (
+    <section aria-labelledby="benchmark-trends-heading">
+      <Stack gap="spacious">
+        <Stack gap="condensed">
+          <Heading as="h2" id="benchmark-trends-heading">
+            Trends
+          </Heading>
+          <Text as="p" className={styles.description}>
+            Strong lines show Benchmark results and muted lines show Control over time.
+          </Text>
+        </Stack>
+        <ul className={styles.legend} aria-label="Model chart legend">
+          {series.map((seriesName, index) => {
+            const style = lineStyles[index % lineStyles.length]
+            return (
+              <li className={styles.legendItem} key={seriesName}>
+                <svg aria-hidden="true" height="8" width="28">
+                  <line
+                    stroke={style.color}
+                    strokeDasharray={style.dash}
+                    strokeWidth="2"
+                    x1="0"
+                    x2="28"
+                    y1="4"
+                    y2="4"
+                  />
+                </svg>
+                {seriesName}
+              </li>
+            )
+          })}
+        </ul>
+        <div className={styles.chartGrid}>
+          {metrics.map(metric => {
+            return <TrendChart dates={dates} key={metric.id} metric={metric} points={points} series={series} />
+          })}
+        </div>
+        <Table.Container>
+          <Table.Title as="h3" id="benchmark-change-heading">
+            Change from Control
+          </Table.Title>
+          <Table.Subtitle as="p" id="benchmark-change-description">
+            Benchmark results are shown first, followed by the percent change from Control in parentheses.
+          </Table.Subtitle>
+          <Table.Actions>
+            <FormControl>
+              <FormControl.Label>Metric</FormControl.Label>
+              <Select
+                onChange={event => {
+                  setTableMetricId(event.currentTarget.value as BenchmarkTrendMetricId)
+                }}
+                value={tableMetricId}
+              >
+                {metrics.map(metric => {
+                  return (
+                    <Select.Option key={metric.id} value={metric.id}>
+                      {metric.label}
+                    </Select.Option>
+                  )
+                })}
+              </Select>
+            </FormControl>
+          </Table.Actions>
+          <DataTable
+            aria-describedby="benchmark-change-description"
+            aria-labelledby="benchmark-change-heading"
+            cellPadding="condensed"
+            columns={[
+              {
+                id: 'model',
+                header: 'Model',
+                field: 'model',
+                rowHeader: true,
+              },
+              ...dates.map(date => {
+                return {
+                  id: date,
+                  header: () => {
+                    return <time dateTime={date}>{date}</time>
+                  },
+                  field: 'values' as const,
+                  align: 'end' as const,
+                  renderCell: (row: (typeof comparisonRows)[number]) => {
+                    const value = row.values[date]
+                    return value ? (
+                      <span className={styles.metricValue}>
+                        {value.raw} <span className={styles.change}>({formatChange(value.change)})</span>
+                      </span>
+                    ) : (
+                      'N/A'
+                    )
+                  },
+                }
+              }),
+            ]}
+            data={comparisonRows}
+          />
+        </Table.Container>
+        <details className={styles.details}>
+          <summary className={styles.summary} id="raw-benchmark-trend-data-heading">
+            View raw trend data
+          </summary>
+          <div className={styles.rawData}>
+            <DataTable
+              aria-labelledby="raw-benchmark-trend-data-heading"
+              cellPadding="condensed"
+              columns={[
+                {
+                  id: 'date',
+                  header: 'Date',
+                  field: 'date',
+                  renderCell: row => {
+                    return <time dateTime={row.date}>{row.date}</time>
+                  },
+                },
+                {
+                  id: 'model',
+                  header: 'Model',
+                  field: 'seriesName',
+                  rowHeader: true,
+                },
+                ...metrics.map(metric => {
+                  return {
+                    id: metric.id,
+                    header: metric.label,
+                    field: 'metrics' as const,
+                    align: 'end' as const,
+                    renderCell: (row: (typeof rawRows)[number]) => {
+                      const value = row.metrics[metric.id]
+                      return (
+                        <span className={styles.metricValue}>
+                          {value.raw} <span className={styles.change}>({formatChange(value.change)})</span>
+                        </span>
+                      )
+                    },
+                  }
+                }),
+              ]}
+              data={rawRows}
+            />
+          </div>
+        </details>
+      </Stack>
+    </section>
+  )
+}
diff --git a/website/src/app/components/Index.module.css b/website/src/app/components/Index.module.css
deleted file mode 100644
index 7190d400..00000000
--- a/website/src/app/components/Index.module.css
+++ /dev/null
@@ -1,15 +0,0 @@
-.layout {
-  min-height: calc(100dvh - var(--header-height) - (2 * var(--stack-padding-normal)));
-}
-
-.metric {
-  font: var(--text-codeInline-shorthand);
-  white-space: nowrap;
-}
-
-.change {
-  display: inline-block;
-  min-width: 8ch;
-  color: var(--fgColor-muted);
-  text-align: right;
-}
diff --git a/website/src/app/components/Index.tsx b/website/src/app/components/Index.tsx
deleted file mode 100644
index 26998085..00000000
--- a/website/src/app/components/Index.tsx
+++ /dev/null
@@ -1,220 +0,0 @@
-'use client'
-
-import {NavList, PageLayout, Stack} from '@primer/react'
-import {DataTable, Table, Blankslate} from '@primer/react/experimental'
-import type {Route} from 'next'
-import NextLink from 'next/link'
-import {Link} from '../../components/Link'
-import {BaselineTrends, type BaselineTrendPoint} from './BaselineTrends'
-import styles from './Index.module.css'
-
-type BaselineMetric = {
-  raw: string
-  change: string | null
-}
-
-export type BaselineResult = {
-  id: string
-  model: string
-  reasoningEffort: string
-  tests: BaselineMetric
-  turns: BaselineMetric
-  outputTokens: BaselineMetric
-  premiumRequests: BaselineMetric
-  apiDuration: BaselineMetric
-  sessionDuration: BaselineMetric
-  toolCalls: BaselineMetric
-}
-
-export type BaselineComparison = {
-  id: string
-  scenarioId: string
-  results: Array<BaselineResult>
-}
-
-function Metric({value}: {value: BaselineMetric}) {
-  if (value.raw === '—') {
-    return '—'
-  }
-
-  return (
-    <span className={styles.metric}>
-      {value.raw} <span className={styles.change}>({value.change ?? '—'})</span>
-    </span>
-  )
-}
-
-export function BaselineResultsTable({
-  description,
-  heading,
-  headingId,
-  headingLevel = 'h2',
-  results,
-}: {
-  description?: React.ReactNode
-  heading: React.ReactNode
-  headingId: string
-  headingLevel?: 'h1' | 'h2'
-  results: Array<BaselineResult>
-}) {
-  const descriptionId = `${headingId}-description`
-
-  return (
-    <Table.Container>
-      <Table.Title as={headingLevel} id={headingId}>
-        {heading}
-      </Table.Title>
-      {description ? (
-        <Table.Subtitle as="p" id={descriptionId}>
-          {description}
-        </Table.Subtitle>
-      ) : null}
-      <DataTable
-        aria-labelledby={headingId}
-        aria-describedby={description ? descriptionId : undefined}
-        cellPadding="condensed"
-        columns={[
-          {
-            id: 'model',
-            header: 'Model',
-            field: 'model',
-            rowHeader: true,
-            width: 'auto',
-          },
-          {
-            id: 'reasoningEffort',
-            header: 'Effort',
-            field: 'reasoningEffort',
-            width: 'auto',
-          },
-          {
-            id: 'tests',
-            header: 'Tests passed',
-            field: 'tests',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.tests} />,
-          },
-          {
-            id: 'turns',
-            header: 'Turns',
-            field: 'turns',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.turns} />,
-          },
-          {
-            id: 'output-tokens',
-            header: 'Output tokens',
-            field: 'outputTokens',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.outputTokens} />,
-          },
-          {
-            id: 'premium-requests',
-            header: 'Premium requests',
-            field: 'premiumRequests',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.premiumRequests} />,
-          },
-          {
-            id: 'api-time',
-            header: 'API time',
-            field: 'apiDuration',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.apiDuration} />,
-          },
-          {
-            id: 'session-time',
-            header: 'Session time',
-            field: 'sessionDuration',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.sessionDuration} />,
-          },
-          {
-            id: 'tool-calls',
-            header: 'Tool calls',
-            field: 'toolCalls',
-            align: 'end',
-            width: 'auto',
-            renderCell: row => <Metric value={row.toolCalls} />,
-          },
-        ]}
-        data={results}
-      />
-    </Table.Container>
-  )
-}
-
-export function Index({
-  baseline,
-  baselineTrends,
-  selectedScenarioId,
-}: {
-  baseline: Array<BaselineComparison> | null
-  baselineTrends: Array<BaselineTrendPoint>
-  selectedScenarioId?: string
-}) {
-  const selectedScenario = baseline?.find(scenario => scenario.scenarioId === selectedScenarioId) ?? baseline?.[0]
-
-  return (
-    <Stack padding="normal" gap="spacious">
-      <section>
-        {baseline ? (
-          <PageLayout className={styles.layout} containerWidth="full" padding="none" rowGap="normal">
-            <PageLayout.Pane aria-label="Baseline scenarios" divider="line" padding="none" position="start" sticky>
-              <NavList aria-label="Baseline scenarios">
-                {baseline.map(scenario => (
-                  <NavList.Item
-                    as={NextLink}
-                    aria-current={scenario.scenarioId === selectedScenario?.scenarioId ? 'page' : undefined}
-                    href={`/baseline/${scenario.scenarioId}` as Route}
-                    key={scenario.id}
-                  >
-                    {scenario.scenarioId}
-                  </NavList.Item>
-                ))}
-              </NavList>
-            </PageLayout.Pane>
-            <PageLayout.Content as="div">
-              {selectedScenario ? (
-                <Stack gap="spacious">
-                  <BaselineResultsTable
-                    description={
-                      <>
-                        Recommended results are shown first, followed by the percent change from Control in parentheses.
-                        Positive means the raw value increased and negative means it decreased. Models are ranked by
-                        test pass rate within each scenario.
-                      </>
-                    }
-                    heading={
-                      <Link href={`/scenarios/${selectedScenario.scenarioId}`}>{selectedScenario.scenarioId}</Link>
-                    }
-                    headingId={`baseline-${selectedScenario.scenarioId}-heading`}
-                    headingLevel="h1"
-                    results={selectedScenario.results}
-                  />
-                  <BaselineTrends points={baselineTrends} scenarioId={selectedScenario.scenarioId} />
-                </Stack>
-              ) : null}
-            </PageLayout.Content>
-          </PageLayout>
-        ) : (
-          <>
-            <h1 className="text-title-medium pb-4">Baseline</h1>
-            <Blankslate border>
-              <Blankslate.Heading as="h2">No results</Blankslate.Heading>
-              <Blankslate.Description>
-                No baseline results have been recorded yet. Run the baseline tests to see results here.
-              </Blankslate.Description>
-            </Blankslate>
-          </>
-        )}
-      </section>
-    </Stack>
-  )
-}
diff --git a/website/src/app/components/PageHeader.tsx b/website/src/app/components/PageHeader.tsx
index b4b88ef1..cb210dbd 100644
--- a/website/src/app/components/PageHeader.tsx
+++ b/website/src/app/components/PageHeader.tsx
@@ -25,10 +25,10 @@ export function PageHeader() {
           </UnderlineNav.Item>
           <UnderlineNav.Item
             as={Link}
-            href="/baseline"
-            aria-current={pathname.startsWith('/baseline') ? 'page' : undefined}
+            href="/benchmarks"
+            aria-current={pathname.startsWith('/benchmarks') ? 'page' : undefined}
           >
-            Baseline
+            Benchmarks
           </UnderlineNav.Item>
           <UnderlineNav.Item
             as={Link}
@@ -37,13 +37,6 @@ export function PageHeader() {
           >
             Experiments
           </UnderlineNav.Item>
-          <UnderlineNav.Item
-            as={Link}
-            href="/benchmarks"
-            aria-current={pathname.startsWith('/benchmarks') ? 'page' : undefined}
-          >
-            Benchmarks
-          </UnderlineNav.Item>
           <UnderlineNav.Item
             as={Link}
             href="/scenarios"
diff --git a/website/src/app/experiments/[id]/runs/[date]/page.tsx b/website/src/app/experiments/[id]/runs/[date]/page.tsx
index 6593984a..02f524cd 100644
--- a/website/src/app/experiments/[id]/runs/[date]/page.tsx
+++ b/website/src/app/experiments/[id]/runs/[date]/page.tsx
@@ -22,10 +22,7 @@ export default async function RunPage(props: RunPageProps) {
     notFound()
   }
 
-  const [experiment, run] = await Promise.all([getExperiment(id), getRun(date)])
-  if (run.output.experiment.id !== id) {
-    notFound()
-  }
+  const [experiment, run] = await Promise.all([getExperiment(id), getRun(id, date)])
 
   return (
     <RunDetailsPage
diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx
index c48dc822..8f449a16 100644
--- a/website/src/app/layout.tsx
+++ b/website/src/app/layout.tsx
@@ -7,7 +7,7 @@ export const metadata = {
     default: 'primer / agent-eval',
     template: '%s · primer / agent-eval',
   },
-  description: 'View results of experiments with agents across different scenarios with Primer',
+  description: 'View design system benchmark and experiment results for coding agents',
 }
 
 export default function Layout({children}: {children: React.ReactNode}) {
diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx
index 3c812b3a..2473472a 100644
--- a/website/src/app/page.tsx
+++ b/website/src/app/page.tsx
@@ -1,7 +1,7 @@
-import {getBaselinePageData} from '../baseline-results'
-import {BaselineOverview} from './components/BaselineOverview'
+import {getBenchmarkPageData} from '../benchmark-page-data'
+import {BenchmarkOverview} from './components/BenchmarkOverview'
 
 export default async function IndexPage() {
-  const {aggregateResults, aggregateTrends} = await getBaselinePageData()
-  return <BaselineOverview results={aggregateResults} trends={aggregateTrends} />
+  const {benchmark, overview} = await getBenchmarkPageData('design-system')
+  return <BenchmarkOverview benchmark={benchmark} overview={overview} />
 }
diff --git a/website/src/baseline-results.ts b/website/src/baseline-results.ts
deleted file mode 100644
index d2699067..00000000
--- a/website/src/baseline-results.ts
+++ /dev/null
@@ -1,476 +0,0 @@
-import type {BaselineComparison, BaselineResult} from './app/components/Index'
-import type {BaselineTrendPoint} from './app/components/BaselineTrends'
-import {list as listRuns, type RunOutput, type RunOutputResult} from './runs'
-
-type TreatmentResults = {
-  control?: RunOutputResult
-  baseline?: RunOutputResult
-}
-
-type ModelTreatmentResults = TreatmentResults & {
-  id: string
-  model: string
-  reasoningEffort: string
-}
-
-type MetricValue = {
-  raw: string
-  change: string | null
-}
-
-const percentFormatter = new Intl.NumberFormat('en-US', {
-  maximumFractionDigits: 1,
-})
-
-const numberFormatter = new Intl.NumberFormat('en-US')
-
-function getResultKey(scenarioId: string, model: string, reasoningEffort: string | undefined) {
-  return JSON.stringify([scenarioId, model, reasoningEffort ?? ''])
-}
-
-function getModelResultKey(model: string, reasoningEffort: string | undefined) {
-  return JSON.stringify([model, reasoningEffort ?? ''])
-}
-
-function getPercentChange(control: number | undefined, baseline: number | undefined): string | null {
-  const change = getPercentChangeValue(control, baseline)
-  if (change === null) {
-    return null
-  }
-
-  return `${change > 0 ? '+' : ''}${percentFormatter.format(change)}%`
-}
-
-function getPercentChangeValue(control: number | undefined, baseline: number | undefined): number | null {
-  if (control === undefined || baseline === undefined || (control === 0 && baseline !== 0)) {
-    return null
-  }
-
-  if (control === baseline) {
-    return 0
-  }
-
-  return ((baseline - control) / Math.abs(control)) * 100
-}
-
-function getTestPassRate(result: RunOutputResult | undefined) {
-  if (!result) {
-    return undefined
-  }
-
-  if (result.testResults.numTotalTests === 0) {
-    return 0
-  }
-
-  return result.testResults.numPassedTests / result.testResults.numTotalTests
-}
-
-function countToolCalls(result: RunOutputResult | undefined) {
-  if (!result) {
-    return undefined
-  }
-
-  return Object.values(result.assistant.tools).reduce((total, count) => total + count, 0)
-}
-
-function average(values: Array<number | undefined>) {
-  const recorded = values.filter(value => value !== undefined)
-  if (recorded.length === 0) {
-    return undefined
-  }
-
-  return recorded.reduce((total, value) => total + value, 0) / recorded.length
-}
-
-function formatMetric(
-  control: number | undefined,
-  baseline: number | undefined,
-  format: (value: number) => string,
-): MetricValue {
-  return {
-    raw: baseline === undefined ? '—' : format(baseline),
-    change: getPercentChange(control, baseline),
-  }
-}
-
-function formatDuration(milliseconds: number): string {
-  if (milliseconds < 1000) {
-    return `${numberFormatter.format(milliseconds)}ms`
-  }
-
-  return `${percentFormatter.format(milliseconds / 1000)}s`
-}
-
-function getBaselineComparisons(output: RunOutput): Array<BaselineComparison> {
-  const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
-  const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
-
-  if (!controlTreatment || !baselineTreatment) {
-    throw new Error('The latest baseline run must include Control and Recommended treatments')
-  }
-
-  const resultsByScenario = new Map<string, Map<string, ModelTreatmentResults>>()
-  for (const scenario of output.scenarios) {
-    const scenarioResults = new Map<string, ModelTreatmentResults>()
-    for (const model of output.experiment.models) {
-      const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined]
-      for (const reasoningEffort of reasoningEfforts) {
-        scenarioResults.set(getModelResultKey(model.name, reasoningEffort), {
-          id: getResultKey(scenario.id, model.name, reasoningEffort),
-          model: model.name,
-          reasoningEffort: reasoningEffort ?? '—',
-        })
-      }
-    }
-    resultsByScenario.set(scenario.id, scenarioResults)
-  }
-
-  for (const result of output.results) {
-    const treatmentResults = resultsByScenario
-      .get(result.scenarioId)
-      ?.get(getModelResultKey(result.model, result.reasoningEffort))
-    if (!treatmentResults) {
-      throw new Error(`Result "${result.id}" does not match a configured scenario, model, and reasoning effort`)
-    }
-
-    if (result.treatmentId === controlTreatment.id) {
-      treatmentResults.control = result
-    } else if (result.treatmentId === baselineTreatment.id) {
-      treatmentResults.baseline = result
-    }
-  }
-
-  return output.scenarios.map(scenario => {
-    const comparisons = Array.from(resultsByScenario.get(scenario.id)?.values() ?? [], treatmentResults => {
-      const {id, model, reasoningEffort, control, baseline} = treatmentResults
-      const baselinePassRate = getTestPassRate(baseline)
-
-      return {
-        id,
-        model,
-        reasoningEffort,
-        passRate: baselinePassRate,
-        turnsValue: baseline?.assistant.turns,
-        tests: {
-          raw: baseline ? `${baseline.testResults.numPassedTests}/${baseline.testResults.numTotalTests}` : '—',
-          change: getPercentChange(control?.testResults.numPassedTests, baseline?.testResults.numPassedTests),
-        },
-        turns: formatMetric(control?.assistant.turns, baseline?.assistant.turns, value =>
-          numberFormatter.format(value),
-        ),
-        outputTokens: formatMetric(control?.assistant.outputTokens, baseline?.assistant.outputTokens, value =>
-          numberFormatter.format(value),
-        ),
-        premiumRequests: formatMetric(control?.assistant.premiumRequests, baseline?.assistant.premiumRequests, value =>
-          numberFormatter.format(value),
-        ),
-        apiDuration: formatMetric(
-          control?.assistant.totalApiDurationMs,
-          baseline?.assistant.totalApiDurationMs,
-          formatDuration,
-        ),
-        sessionDuration: formatMetric(
-          control?.assistant.sessionDurationMs,
-          baseline?.assistant.sessionDurationMs,
-          formatDuration,
-        ),
-        toolCalls: formatMetric(countToolCalls(control), countToolCalls(baseline), value =>
-          numberFormatter.format(value),
-        ),
-      }
-    })
-
-    return {
-      id: scenario.id,
-      scenarioId: scenario.id,
-      results: comparisons
-        .toSorted((a, b) => {
-          if (a.passRate !== b.passRate) {
-            return (b.passRate ?? -1) - (a.passRate ?? -1)
-          }
-
-          if (a.turnsValue !== b.turnsValue) {
-            return (a.turnsValue ?? Number.POSITIVE_INFINITY) - (b.turnsValue ?? Number.POSITIVE_INFINITY)
-          }
-
-          return a.model.localeCompare(b.model)
-        })
-        .map(comparison => ({
-          id: comparison.id,
-          model: comparison.model,
-          reasoningEffort: comparison.reasoningEffort,
-          tests: comparison.tests,
-          turns: comparison.turns,
-          outputTokens: comparison.outputTokens,
-          premiumRequests: comparison.premiumRequests,
-          apiDuration: comparison.apiDuration,
-          sessionDuration: comparison.sessionDuration,
-          toolCalls: comparison.toolCalls,
-        })),
-    }
-  })
-}
-
-function getAggregateBaselineResults(output: RunOutput): Array<BaselineResult> {
-  const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
-  const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
-
-  if (!controlTreatment || !baselineTreatment) {
-    throw new Error('The latest baseline run must include Control and Recommended treatments')
-  }
-
-  const results = output.experiment.models.flatMap(model => {
-    const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined]
-    return reasoningEfforts.map(reasoningEffort => {
-      const matches = (result: RunOutputResult) =>
-        result.model === model.name && result.reasoningEffort === reasoningEffort
-      const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result))
-      const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result))
-      const baselinePassRate = average(baselines.map(getTestPassRate))
-      const baselinePassedTests = average(baselines.map(result => result.testResults.numPassedTests))
-      const baselineTotalTests = average(baselines.map(result => result.testResults.numTotalTests))
-      const controlTurns = average(controls.map(result => result.assistant.turns))
-      const baselineTurns = average(baselines.map(result => result.assistant.turns))
-
-      return {
-        id: getResultKey('aggregate', model.name, reasoningEffort),
-        model: model.name,
-        reasoningEffort: reasoningEffort ?? '—',
-        passRate: baselinePassRate,
-        turnsValue: baselineTurns,
-        tests: {
-          raw:
-            baselinePassedTests === undefined || baselineTotalTests === undefined
-              ? '—'
-              : `${numberFormatter.format(baselinePassedTests)}/${numberFormatter.format(baselineTotalTests)}`,
-          change: getPercentChange(
-            average(controls.map(result => result.testResults.numPassedTests)),
-            baselinePassedTests,
-          ),
-        },
-        turns: formatMetric(controlTurns, baselineTurns, value => numberFormatter.format(value)),
-        outputTokens: formatMetric(
-          average(controls.map(result => result.assistant.outputTokens)),
-          average(baselines.map(result => result.assistant.outputTokens)),
-          value => numberFormatter.format(value),
-        ),
-        premiumRequests: formatMetric(
-          average(controls.map(result => result.assistant.premiumRequests)),
-          average(baselines.map(result => result.assistant.premiumRequests)),
-          value => numberFormatter.format(value),
-        ),
-        apiDuration: formatMetric(
-          average(controls.map(result => result.assistant.totalApiDurationMs)),
-          average(baselines.map(result => result.assistant.totalApiDurationMs)),
-          formatDuration,
-        ),
-        sessionDuration: formatMetric(
-          average(controls.map(result => result.assistant.sessionDurationMs)),
-          average(baselines.map(result => result.assistant.sessionDurationMs)),
-          formatDuration,
-        ),
-        toolCalls: formatMetric(average(controls.map(countToolCalls)), average(baselines.map(countToolCalls)), value =>
-          numberFormatter.format(value),
-        ),
-      }
-    })
-  })
-
-  return results
-    .toSorted((a, b) => {
-      if (a.passRate !== b.passRate) {
-        return (b.passRate ?? -1) - (a.passRate ?? -1)
-      }
-
-      if (a.turnsValue !== b.turnsValue) {
-        return (a.turnsValue ?? Number.POSITIVE_INFINITY) - (b.turnsValue ?? Number.POSITIVE_INFINITY)
-      }
-
-      return a.model.localeCompare(b.model)
-    })
-    .map(result => ({
-      id: result.id,
-      model: result.model,
-      reasoningEffort: result.reasoningEffort,
-      tests: result.tests,
-      turns: result.turns,
-      outputTokens: result.outputTokens,
-      premiumRequests: result.premiumRequests,
-      apiDuration: result.apiDuration,
-      sessionDuration: result.sessionDuration,
-      toolCalls: result.toolCalls,
-    }))
-}
-
-function getBaselineTrendPoints(date: string, output: RunOutput): Array<BaselineTrendPoint> {
-  const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
-  const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
-  if (!controlTreatment || !baselineTreatment) {
-    return []
-  }
-
-  const controls = new Map(
-    output.results
-      .filter(result => result.treatmentId === controlTreatment.id)
-      .map(result => [getResultKey(result.scenarioId, result.model, result.reasoningEffort), result]),
-  )
-
-  return output.results
-    .filter(result => result.treatmentId === baselineTreatment.id)
-    .map(result => {
-      const control = controls.get(getResultKey(result.scenarioId, result.model, result.reasoningEffort))
-      const controlToolCalls = countToolCalls(control)
-      const baselineToolCalls = countToolCalls(result)
-      const metric = (
-        value: number,
-        controlValue: number | undefined,
-        raw = numberFormatter.format(value),
-        controlRaw = controlValue === undefined ? null : numberFormatter.format(controlValue),
-      ) => ({
-        value,
-        raw,
-        change: getPercentChangeValue(controlValue, value),
-        controlValue: controlValue ?? null,
-        controlRaw,
-      })
-
-      return {
-        id: `${date}:${result.id}`,
-        date,
-        scenarioId: result.scenarioId,
-        model: result.model,
-        reasoningEffort: result.reasoningEffort ?? '—',
-        metrics: {
-          tests: {
-            value: result.testResults.numPassedTests,
-            raw: `${result.testResults.numPassedTests}/${result.testResults.numTotalTests}`,
-            change: getPercentChangeValue(control?.testResults.numPassedTests, result.testResults.numPassedTests),
-            controlValue: control?.testResults.numPassedTests ?? null,
-            controlRaw: control ? `${control.testResults.numPassedTests}/${control.testResults.numTotalTests}` : null,
-          },
-          turns: metric(result.assistant.turns, control?.assistant.turns),
-          outputTokens: metric(result.assistant.outputTokens, control?.assistant.outputTokens),
-          premiumRequests: metric(result.assistant.premiumRequests, control?.assistant.premiumRequests),
-          apiDuration: metric(
-            result.assistant.totalApiDurationMs / 1000,
-            control ? control.assistant.totalApiDurationMs / 1000 : undefined,
-            formatDuration(result.assistant.totalApiDurationMs),
-            control ? formatDuration(control.assistant.totalApiDurationMs) : null,
-          ),
-          sessionDuration: metric(
-            result.assistant.sessionDurationMs / 1000,
-            control ? control.assistant.sessionDurationMs / 1000 : undefined,
-            formatDuration(result.assistant.sessionDurationMs),
-            control ? formatDuration(control.assistant.sessionDurationMs) : null,
-          ),
-          toolCalls: metric(baselineToolCalls ?? 0, controlToolCalls),
-        },
-      }
-    })
-}
-
-function getAggregateTrendPoints(date: string, output: RunOutput): Array<BaselineTrendPoint> {
-  const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control')
-  const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended')
-  if (!controlTreatment || !baselineTreatment) {
-    return []
-  }
-
-  return output.experiment.models.flatMap(model => {
-    const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined]
-    return reasoningEfforts.map(reasoningEffort => {
-      const matches = (result: RunOutputResult) =>
-        result.model === model.name && result.reasoningEffort === reasoningEffort
-      const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result))
-      const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result))
-      const metric = (
-        baselineValue: number | undefined,
-        controlValue: number | undefined,
-        raw = baselineValue === undefined ? '—' : numberFormatter.format(baselineValue),
-        controlRaw = controlValue === undefined ? null : numberFormatter.format(controlValue),
-      ) => ({
-        value: baselineValue ?? null,
-        raw,
-        change: getPercentChangeValue(controlValue, baselineValue),
-        controlValue: controlValue ?? null,
-        controlRaw,
-      })
-      const baselinePassedTests = average(baselines.map(result => result.testResults.numPassedTests))
-      const baselineTotalTests = average(baselines.map(result => result.testResults.numTotalTests))
-      const controlPassedTests = average(controls.map(result => result.testResults.numPassedTests))
-
-      return {
-        id: `${date}:aggregate:${model.name}:${reasoningEffort ?? ''}`,
-        date,
-        scenarioId: 'aggregate',
-        model: model.name,
-        reasoningEffort: reasoningEffort ?? '—',
-        metrics: {
-          tests: {
-            value: baselinePassedTests ?? null,
-            raw:
-              baselinePassedTests === undefined || baselineTotalTests === undefined
-                ? '—'
-                : `${numberFormatter.format(baselinePassedTests)}/${numberFormatter.format(baselineTotalTests)}`,
-            change: getPercentChangeValue(controlPassedTests, baselinePassedTests),
-            controlValue: controlPassedTests ?? null,
-            controlRaw: (() => {
-              const passed = average(controls.map(result => result.testResults.numPassedTests))
-              const total = average(controls.map(result => result.testResults.numTotalTests))
-              return passed === undefined || total === undefined
-                ? null
-                : `${numberFormatter.format(passed)}/${numberFormatter.format(total)}`
-            })(),
-          },
-          turns: metric(
-            average(baselines.map(result => result.assistant.turns)),
-            average(controls.map(result => result.assistant.turns)),
-          ),
-          outputTokens: metric(
-            average(baselines.map(result => result.assistant.outputTokens)),
-            average(controls.map(result => result.assistant.outputTokens)),
-          ),
-          premiumRequests: metric(
-            average(baselines.map(result => result.assistant.premiumRequests)),
-            average(controls.map(result => result.assistant.premiumRequests)),
-          ),
-          apiDuration: (() => {
-            const baselineValue = average(baselines.map(result => result.assistant.totalApiDurationMs))
-            const controlValue = average(controls.map(result => result.assistant.totalApiDurationMs))
-            return metric(
-              baselineValue === undefined ? undefined : baselineValue / 1000,
-              controlValue === undefined ? undefined : controlValue / 1000,
-              baselineValue === undefined ? '—' : formatDuration(baselineValue),
-              controlValue === undefined ? null : formatDuration(controlValue),
-            )
-          })(),
-          sessionDuration: (() => {
-            const baselineValue = average(baselines.map(result => result.assistant.sessionDurationMs))
-            const controlValue = average(controls.map(result => result.assistant.sessionDurationMs))
-            return metric(
-              baselineValue === undefined ? undefined : baselineValue / 1000,
-              controlValue === undefined ? undefined : controlValue / 1000,
-              baselineValue === undefined ? '—' : formatDuration(baselineValue),
-              controlValue === undefined ? null : formatDuration(controlValue),
-            )
-          })(),
-          toolCalls: metric(average(baselines.map(countToolCalls)), average(controls.map(countToolCalls))),
-        },
-      }
-    })
-  })
-}
-
-async function getBaselinePageData() {
-  const runs = await listRuns()
-  const baselineRuns = runs.filter(run => run.output.experiment.id === 'baseline')
-  const latestRun = baselineRuns[0]
-  const baseline = latestRun ? getBaselineComparisons(latestRun.output) : null
-  const baselineTrends = baselineRuns.flatMap(run => getBaselineTrendPoints(run.name, run.output))
-  const aggregateResults = latestRun ? getAggregateBaselineResults(latestRun.output) : []
-  const aggregateTrends = baselineRuns.flatMap(run => getAggregateTrendPoints(run.name, run.output))
-
-  return {aggregateResults, aggregateTrends, baseline, baselineTrends}
-}
-
-export {getBaselinePageData}
diff --git a/website/src/benchmark-page-data.ts b/website/src/benchmark-page-data.ts
new file mode 100644
index 00000000..60208817
--- /dev/null
+++ b/website/src/benchmark-page-data.ts
@@ -0,0 +1,28 @@
+import {get as getBenchmark} from './benchmarks'
+import {getBenchmarkOverviewData, getBenchmarkPageResults, listBenchmarkRuns} from './benchmark-results'
+
+async function getBenchmarkPageData(id: string) {
+  const [benchmark, runs] = await Promise.all([getBenchmark(id), listBenchmarkRuns(id)])
+
+  return {
+    benchmark,
+    overview: getBenchmarkOverviewData(runs),
+    results: getBenchmarkPageResults(benchmark, runs[0]),
+    runs: runs.map(run => {
+      const trials = [...run.output.trials.values()]
+      return {
+        id: run.id,
+        name: run.name,
+        resultCount: trials.length,
+        passedTests: trials.reduce((total, trial) => {
+          return total + trial.testResults.numPassedTests
+        }, 0),
+        totalTests: trials.reduce((total, trial) => {
+          return total + trial.testResults.numTotalTests
+        }, 0),
+      }
+    }),
+  }
+}
+
+export {getBenchmarkPageData}
diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts
index 7aaa67fb..3e075f81 100644
--- a/website/src/benchmark-results.ts
+++ b/website/src/benchmark-results.ts
@@ -16,15 +16,18 @@ type BenchmarkOutputTrial = BenchmarkOutput['trials'] extends Map<string, infer
 type ResultTotals = {
   passed: number
   total: number
+  outputTokens: number
+  premiumRequests: number
+  sessionDurationMs: number
+  totalApiDurationMs: number
 }
 
 export type BenchmarkComparison = {
-  control: string
-  benchmark: string
-  delta: string
-  controlTests: string
-  benchmarkTests: string
-  deltaValue: number | null
+  tests: string
+  outputTokens: string
+  premiumRequests: string
+  sessionTime: string
+  apiTime: string
 }
 
 export type BenchmarkCapabilityResult = {
@@ -34,6 +37,12 @@ export type BenchmarkCapabilityResult = {
   scenarios: Array<{
     id: string
     comparison: BenchmarkComparison
+    models: Array<{
+      id: string
+      name: string
+      reasoningEffort: string
+      comparison: BenchmarkComparison
+    }>
   }>
 }
 
@@ -42,6 +51,37 @@ export type BenchmarkPageResults = {
   capabilities: Array<BenchmarkCapabilityResult>
 }
 
+export type BenchmarkOverviewResult = {
+  id: string
+  model: string
+  reasoningEffort: string
+  comparison: BenchmarkComparison
+}
+
+export type BenchmarkTrendMetricId = 'tests' | 'outputTokens' | 'premiumRequests' | 'sessionTime' | 'apiTime'
+
+export type BenchmarkTrendMetric = {
+  value: number | null
+  raw: string
+  change: number | null
+  controlValue: number | null
+  controlRaw: string | null
+}
+
+export type BenchmarkTrendPoint = {
+  id: string
+  date: string
+  model: string
+  reasoningEffort: string
+  metrics: Record<BenchmarkTrendMetricId, BenchmarkTrendMetric>
+}
+
+export type BenchmarkOverviewData = {
+  date: string | null
+  results: Array<BenchmarkOverviewResult>
+  trends: Array<BenchmarkTrendPoint>
+}
+
 type OutputCandidate = {
   filepath: string
   date: string
@@ -116,9 +156,22 @@ function getTotals(trials: Array<BenchmarkOutputTrial>): ResultTotals {
     (totals, trial) => {
       totals.passed += trial.testResults.numPassedTests
       totals.total += trial.testResults.numTotalTests
+      for (const session of trial.agent.sessions) {
+        totals.outputTokens += session.outputTokens
+        totals.premiumRequests += session.premiumRequests
+        totals.sessionDurationMs += session.sessionDurationMs
+        totals.totalApiDurationMs += session.totalApiDurationMs
+      }
       return totals
     },
-    {passed: 0, total: 0},
+    {
+      passed: 0,
+      total: 0,
+      outputTokens: 0,
+      premiumRequests: 0,
+      sessionDurationMs: 0,
+      totalApiDurationMs: 0,
+    },
   )
 }
 
@@ -130,20 +183,46 @@ function getPassRate(totals: ResultTotals): number | null {
   return totals.passed / totals.total
 }
 
-function formatRate(rate: number | null): string {
-  if (rate === null) {
-    return 'N/A'
+function getPercentDelta(control: number, benchmark: number): number | null {
+  if (control === 0) {
+    return benchmark === 0 ? 0 : null
   }
 
-  return `${(rate * 100).toFixed(1)}%`
+  return (benchmark - control) / control
 }
 
-function formatTests(totals: ResultTotals): string {
-  if (totals.total === 0) {
+function getPercentDeltaValue(control: number, benchmark: number): number | null {
+  const delta = getPercentDelta(control, benchmark)
+  return delta === null ? null : delta * 100
+}
+
+function formatPercentDelta(control: number, benchmark: number): string {
+  const delta = getPercentDelta(control, benchmark)
+  if (delta === null) {
     return 'N/A'
   }
 
-  return `${totals.passed}/${totals.total}`
+  const sign = delta > 0 ? '+' : ''
+  return `${sign}${(delta * 100).toFixed(1)}%`
+}
+
+function formatValue(value: string, control: number, benchmark: number): string {
+  return `${value} (${formatPercentDelta(control, benchmark)})`
+}
+
+function formatNumber(value: number): string {
+  return new Intl.NumberFormat('en-US').format(value)
+}
+
+function formatDuration(milliseconds: number): string {
+  const seconds = milliseconds / 1000
+  if (seconds < 60) {
+    return `${seconds.toFixed(1)}s`
+  }
+
+  const minutes = Math.floor(seconds / 60)
+  const remainingSeconds = seconds - minutes * 60
+  return `${minutes}m ${remainingSeconds.toFixed(1)}s`
 }
 
 function createComparison(
@@ -161,22 +240,40 @@ function createComparison(
       return trial.treatmentId === benchmarkTreatmentId
     }),
   )
-  const controlRate = getPassRate(controlTotals)
-  const benchmarkRate = getPassRate(benchmarkTotals)
-  const deltaValue = controlRate === null || benchmarkRate === null ? null : benchmarkRate - controlRate
 
   return {
-    control: formatRate(controlRate),
-    benchmark: formatRate(benchmarkRate),
-    delta:
-      deltaValue === null ? 'N/A' : `${deltaValue > 0 ? '+' : ''}${(deltaValue * 100).toFixed(1)} percentage points`,
-    controlTests: formatTests(controlTotals),
-    benchmarkTests: formatTests(benchmarkTotals),
-    deltaValue,
+    tests: formatValue(
+      `${benchmarkTotals.passed}/${benchmarkTotals.total}`,
+      controlTotals.passed,
+      benchmarkTotals.passed,
+    ),
+    outputTokens: formatValue(
+      formatNumber(benchmarkTotals.outputTokens),
+      controlTotals.outputTokens,
+      benchmarkTotals.outputTokens,
+    ),
+    premiumRequests: formatValue(
+      formatNumber(benchmarkTotals.premiumRequests),
+      controlTotals.premiumRequests,
+      benchmarkTotals.premiumRequests,
+    ),
+    sessionTime: formatValue(
+      formatDuration(benchmarkTotals.sessionDurationMs),
+      controlTotals.sessionDurationMs,
+      benchmarkTotals.sessionDurationMs,
+    ),
+    apiTime: formatValue(
+      formatDuration(benchmarkTotals.totalApiDurationMs),
+      controlTotals.totalApiDurationMs,
+      benchmarkTotals.totalApiDurationMs,
+    ),
   }
 }
 
-function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date: string): BenchmarkPageResults {
+function getTreatments(output: BenchmarkOutput): {
+  controlTreatmentId: string
+  benchmarkTreatmentId: string
+} {
   const controlTreatment = [...output.treatments].find(([, treatment]) => {
     return treatment.name === 'Control'
   })
@@ -185,9 +282,141 @@ function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date:
   })
 
   if (!controlTreatment || !benchmarkTreatment) {
-    throw new Error(`Benchmark "${benchmark.id}" results must include Control and Benchmark treatments`)
+    throw new Error(`Benchmark "${output.benchmarkId}" results must include Control and Benchmark treatments`)
   }
 
+  return {
+    controlTreatmentId: controlTreatment[0],
+    benchmarkTreatmentId: benchmarkTreatment[0],
+  }
+}
+
+function groupTrialsByModel(trials: Array<BenchmarkOutputTrial>): Array<Array<BenchmarkOutputTrial>> {
+  const modelTrials = new Map<string, Array<BenchmarkOutputTrial>>()
+  for (const trial of trials) {
+    const key = `${trial.model.name}\0${trial.model.reasoningEffort}`
+    const trialsForModel = modelTrials.get(key) ?? []
+    trialsForModel.push(trial)
+    modelTrials.set(key, trialsForModel)
+  }
+  return [...modelTrials.values()]
+}
+
+function createTrendMetric(
+  value: number | null,
+  raw: string,
+  controlValue: number | null,
+  controlRaw: string | null,
+  controlDeltaValue: number,
+  benchmarkDeltaValue: number,
+): BenchmarkTrendMetric {
+  return {
+    value,
+    raw,
+    change: getPercentDeltaValue(controlDeltaValue, benchmarkDeltaValue),
+    controlValue,
+    controlRaw,
+  }
+}
+
+function createTrendPoint(
+  date: string,
+  trials: Array<BenchmarkOutputTrial>,
+  controlTreatmentId: string,
+  benchmarkTreatmentId: string,
+): BenchmarkTrendPoint {
+  const trial = trials[0]
+  const controlTotals = getTotals(
+    trials.filter(candidate => {
+      return candidate.treatmentId === controlTreatmentId
+    }),
+  )
+  const benchmarkTotals = getTotals(
+    trials.filter(candidate => {
+      return candidate.treatmentId === benchmarkTreatmentId
+    }),
+  )
+  const controlPassRate = getPassRate(controlTotals)
+  const benchmarkPassRate = getPassRate(benchmarkTotals)
+
+  return {
+    id: `${date}:${trial.model.name}:${trial.model.reasoningEffort}`,
+    date,
+    model: trial.model.name,
+    reasoningEffort: trial.model.reasoningEffort,
+    metrics: {
+      tests: createTrendMetric(
+        benchmarkPassRate === null ? null : benchmarkPassRate * 100,
+        `${benchmarkTotals.passed}/${benchmarkTotals.total}`,
+        controlPassRate === null ? null : controlPassRate * 100,
+        `${controlTotals.passed}/${controlTotals.total}`,
+        controlTotals.passed,
+        benchmarkTotals.passed,
+      ),
+      outputTokens: createTrendMetric(
+        benchmarkTotals.outputTokens,
+        formatNumber(benchmarkTotals.outputTokens),
+        controlTotals.outputTokens,
+        formatNumber(controlTotals.outputTokens),
+        controlTotals.outputTokens,
+        benchmarkTotals.outputTokens,
+      ),
+      premiumRequests: createTrendMetric(
+        benchmarkTotals.premiumRequests,
+        formatNumber(benchmarkTotals.premiumRequests),
+        controlTotals.premiumRequests,
+        formatNumber(controlTotals.premiumRequests),
+        controlTotals.premiumRequests,
+        benchmarkTotals.premiumRequests,
+      ),
+      sessionTime: createTrendMetric(
+        benchmarkTotals.sessionDurationMs / 1000,
+        formatDuration(benchmarkTotals.sessionDurationMs),
+        controlTotals.sessionDurationMs / 1000,
+        formatDuration(controlTotals.sessionDurationMs),
+        controlTotals.sessionDurationMs,
+        benchmarkTotals.sessionDurationMs,
+      ),
+      apiTime: createTrendMetric(
+        benchmarkTotals.totalApiDurationMs / 1000,
+        formatDuration(benchmarkTotals.totalApiDurationMs),
+        controlTotals.totalApiDurationMs / 1000,
+        formatDuration(controlTotals.totalApiDurationMs),
+        controlTotals.totalApiDurationMs,
+        benchmarkTotals.totalApiDurationMs,
+      ),
+    },
+  }
+}
+
+function compareModelPerformance(
+  a: Array<BenchmarkOutputTrial>,
+  b: Array<BenchmarkOutputTrial>,
+  benchmarkTreatmentId: string,
+): number {
+  const aTotals = getTotals(
+    a.filter(trial => {
+      return trial.treatmentId === benchmarkTreatmentId
+    }),
+  )
+  const bTotals = getTotals(
+    b.filter(trial => {
+      return trial.treatmentId === benchmarkTreatmentId
+    }),
+  )
+
+  return (
+    (getPassRate(bTotals) ?? 0) - (getPassRate(aTotals) ?? 0) ||
+    aTotals.outputTokens - bTotals.outputTokens ||
+    aTotals.premiumRequests - bTotals.premiumRequests ||
+    aTotals.sessionDurationMs - bTotals.sessionDurationMs ||
+    aTotals.totalApiDurationMs - bTotals.totalApiDurationMs
+  )
+}
+
+function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date: string): BenchmarkPageResults {
+  const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(output)
+
   const trials = [...output.trials.values()]
   const capabilities = benchmark.capabilities.map((capability, capabilityIndex) => {
     const capabilityTrials = trials.filter(trial => {
@@ -197,15 +426,27 @@ function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date:
     return {
       id: `${capabilityIndex}-${capability.name}`,
       name: capability.name,
-      comparison: createComparison(capabilityTrials, controlTreatment[0], benchmarkTreatment[0]),
+      comparison: createComparison(capabilityTrials, controlTreatmentId, benchmarkTreatmentId),
       scenarios: capability.scenarios.map(scenario => {
         const scenarioTrials = capabilityTrials.filter(trial => {
           return trial.scenarioId === scenario.id
         })
-
         return {
           id: scenario.id,
-          comparison: createComparison(scenarioTrials, controlTreatment[0], benchmarkTreatment[0]),
+          comparison: createComparison(scenarioTrials, controlTreatmentId, benchmarkTreatmentId),
+          models: groupTrialsByModel(scenarioTrials)
+            .toSorted((a, b) => {
+              return compareModelPerformance(a, b, benchmarkTreatmentId)
+            })
+            .map(trialsForModel => {
+              const trial = trialsForModel[0]
+              return {
+                id: `${trial.model.name}\0${trial.model.reasoningEffort}`,
+                name: trial.model.name,
+                reasoningEffort: trial.model.reasoningEffort,
+                comparison: createComparison(trialsForModel, controlTreatmentId, benchmarkTreatmentId),
+              }
+            }),
         }
       }),
     }
@@ -214,6 +455,40 @@ function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date:
   return {date, capabilities}
 }
 
+export function getBenchmarkOverviewData(runs: Array<BenchmarkRun>): BenchmarkOverviewData {
+  const latestRun = runs[0]
+  const results = latestRun
+    ? (() => {
+        const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(latestRun.output)
+        return groupTrialsByModel([...latestRun.output.trials.values()])
+          .toSorted((a, b) => {
+            return compareModelPerformance(a, b, benchmarkTreatmentId)
+          })
+          .map(trials => {
+            const trial = trials[0]
+            return {
+              id: `${trial.model.name}\0${trial.model.reasoningEffort}`,
+              model: trial.model.name,
+              reasoningEffort: trial.model.reasoningEffort,
+              comparison: createComparison(trials, controlTreatmentId, benchmarkTreatmentId),
+            }
+          })
+      })()
+    : []
+  const trends = runs.flatMap(run => {
+    const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(run.output)
+    return groupTrialsByModel([...run.output.trials.values()]).map(trials => {
+      return createTrendPoint(run.name, trials, controlTreatmentId, benchmarkTreatmentId)
+    })
+  })
+
+  return {
+    date: latestRun?.name ?? null,
+    results,
+    trends,
+  }
+}
+
 export async function listBenchmarkRuns(benchmarkId: string): Promise<Array<BenchmarkRun>> {
   const candidates = (await getDatedCandidates(benchmarkId)).toSorted((a, b) => {
     return b.date.localeCompare(a.date)
diff --git a/website/src/runs.ts b/website/src/runs.ts
index e0c857b7..6e25417d 100644
--- a/website/src/runs.ts
+++ b/website/src/runs.ts
@@ -9,7 +9,7 @@ const {deserialize} = await import(
   '@primer/agent-eval/experiment'
 )
 
-const RESULTS_DIR = path.resolve(process.cwd(), '..', 'results')
+const RESULTS_DIR = path.resolve(process.cwd(), '..', 'results', 'experiments')
 
 type ExperimentOutputTrial = ExperimentOutput['trials'] extends Map<string, infer Trial> ? Trial : never
 
@@ -110,10 +110,12 @@ function isRunName(name: string): boolean {
   return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(name)
 }
 
-async function listResultDirectories(): Promise<Array<Dirent>> {
+async function listExperimentDirectories(): Promise<Array<Dirent>> {
   try {
     const entries = await fs.readdir(RESULTS_DIR, {withFileTypes: true})
-    return entries.filter(entry => entry.isDirectory() && isRunName(entry.name))
+    return entries.filter(entry => {
+      return entry.isDirectory()
+    })
   } catch (error) {
     if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
       return []
@@ -123,51 +125,57 @@ async function listResultDirectories(): Promise<Array<Dirent>> {
   }
 }
 
-async function list(): Promise<Array<Run>> {
-  const results = await listResultDirectories().then(entries => {
-    return Promise.all(
-      entries.map(async entry => {
-        const directory = path.join(RESULTS_DIR, entry.name)
-        const outputFile = path.join(directory, 'output.json')
-        const contents = await fs.readFile(outputFile, 'utf-8')
-        const output = parseOutput(contents)
-        return [directory, entry.name, output] as const
-      }),
-    )
-  })
-
-  return results
-    .map(([directory, name, output]) => {
-      const date = new Date(`${name}T00:00:00.000Z`)
-      return {id: name, name, directory, date, output}
+async function listRunDirectories(experimentId: string): Promise<Array<Dirent>> {
+  const experimentDirectory = path.join(RESULTS_DIR, experimentId)
+  try {
+    const entries = await fs.readdir(experimentDirectory, {withFileTypes: true})
+    return entries.filter(entry => {
+      return entry.isDirectory() && isRunName(entry.name)
     })
-    .toSorted((a, b) => b.date.getTime() - a.date.getTime())
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return []
+    }
+
+    throw error
+  }
 }
 
 async function listForExperiment(experimentId: string): Promise<Array<Run>> {
-  const runs = await list()
-  return runs.filter(run => run.output.experiment.id === experimentId)
+  const entries = await listRunDirectories(experimentId)
+  const runs = await Promise.all(
+    entries.map(entry => {
+      return find(experimentId, entry.name)
+    }),
+  )
+
+  return runs
+    .filter((run): run is Run => {
+      return run !== null
+    })
+    .toSorted((a, b) => {
+      return b.date.getTime() - a.date.getTime()
+    })
 }
 
-async function latest(): Promise<Run | null> {
-  const runs = await listResultDirectories().then(entries => {
-    return entries.map(entry => [new Date(`${entry.name}T00:00:00.000Z`), entry.name] as const)
+async function list(): Promise<Array<Run>> {
+  const experiments = await listExperimentDirectories()
+  const runs = await Promise.all(
+    experiments.map(experiment => {
+      return listForExperiment(experiment.name)
+    }),
+  )
+  return runs.flat().toSorted((a, b) => {
+    return b.date.getTime() - a.date.getTime()
   })
-  if (runs.length === 0) {
-    return null
-  }
-
-  const sorted = runs.toSorted((a, b) => b[0].getTime() - a[0].getTime())
-  const run = await find(sorted[0][1])
-  return run
 }
 
-async function find(name: string): Promise<Run | null> {
+async function find(experimentId: string, name: string): Promise<Run | null> {
   if (!isRunName(name)) {
     return null
   }
 
-  const directory = path.join(RESULTS_DIR, name)
+  const directory = path.join(RESULTS_DIR, experimentId, name)
   if (!existsSync(directory)) {
     return null
   }
@@ -184,6 +192,10 @@ async function find(name: string): Promise<Run | null> {
   const outputFile = path.join(directory, 'output.json')
   const contents = await fs.readFile(outputFile, 'utf-8')
   const output = parseOutput(contents)
+  if (output.experiment.id !== experimentId) {
+    return null
+  }
+
   return {
     id: name,
     name,
@@ -193,10 +205,10 @@ async function find(name: string): Promise<Run | null> {
   }
 }
 
-async function get(name: string): Promise<Run> {
-  const run = await find(name)
+async function get(experimentId: string, name: string): Promise<Run> {
+  const run = await find(experimentId, name)
   if (!run) {
-    throw new Error(`Run "${name}" was not found in: ${RESULTS_DIR}`)
+    throw new Error(`Run "${name}" for experiment "${experimentId}" was not found in: ${RESULTS_DIR}`)
   }
 
   return run
@@ -275,5 +287,5 @@ function normalizeOutput(output: ExperimentOutput): RunOutput {
   }
 }
 
-export {list, listForExperiment, latest, get}
+export {list, listForExperiment, get}
 export type {Run, RunOutput, RunOutputResult}

From cb59e187dff783817cbd55dbdd3206641ccd81e7 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 17:48:22 -0500
Subject: [PATCH 34/52] refactor: clean up benchmark models and sandbox cleanup

---
 benchmarks/design-system.ts                   |   9 +-
 packages/agent-eval/src/plan.ts               |   3 +-
 .../agent-eval/src/sandbox/system.test.ts     |  43 ++++-
 packages/agent-eval/src/sandbox/system.ts     | 165 ++++++++++--------
 4 files changed, 129 insertions(+), 91 deletions(-)

diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts
index 00955953..1e4422a5 100644
--- a/benchmarks/design-system.ts
+++ b/benchmarks/design-system.ts
@@ -3,14 +3,7 @@ import {defineConfig} from '@primer/agent-eval/benchmark'
 export const benchmark = defineConfig({
   name: 'Design System',
   description: 'Benchmark the performance of agents with different design system tasks.',
-  models: [
-    'gpt-5.6-sol',
-    'gpt-5.6-terra',
-    'claude-opus-5',
-    'claude-sonnet-5',
-    'gemini-3.1-pro-preview',
-    'gemini-3.5-flash',
-  ],
+  models: ['gpt-5.6-sol', 'gpt-5.6-terra', 'claude-opus-5', 'claude-sonnet-5', 'gemini-3.5-flash'],
   async setup({sandbox}) {
     // Setup the Primer MCP server locally
     await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest'])
diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
index 7425112c..6b1de65b 100644
--- a/packages/agent-eval/src/plan.ts
+++ b/packages/agent-eval/src/plan.ts
@@ -4,6 +4,7 @@ import {TrialSchema, run as runTrial} from './trial'
 import type {Trial, TrialResult} from './trial'
 import type {EnvironmentConfig} from './environment'
 import {DefaultHost, type Host} from './host'
+import {logger} from './logger'
 
 const PlanSchema = z.object({
   trials: z.array(TrialSchema),
@@ -71,7 +72,7 @@ async function retry<T>(fn: () => Promise<T>, retries: number = 3): Promise<T> {
     return await fn()
   } catch (error) {
     if (retries > 0) {
-      console.log('Retrying after error: %s', error)
+      logger.error({error}, 'Retrying')
       return retry(fn, retries - 1)
     }
     throw error
diff --git a/packages/agent-eval/src/sandbox/system.test.ts b/packages/agent-eval/src/sandbox/system.test.ts
index f7e3a8d6..238ed33c 100644
--- a/packages/agent-eval/src/sandbox/system.test.ts
+++ b/packages/agent-eval/src/sandbox/system.test.ts
@@ -2,13 +2,10 @@ import Docker from 'dockerode'
 import {beforeEach, describe, expect, test, vi} from 'vitest'
 import {VirtualHost} from '../host'
 import {MCP_CONFIG_PATH, NODE_USER, SKILLS_DIR} from './constants'
-import {SandboxSchema, SystemSandbox} from './system'
+import {createContainer, SandboxSchema, SystemSandbox} from './system'
 import {VirtualSandbox} from './virtual'
 
-function createSandbox() {
-  const container = {
-    stop: vi.fn(),
-  }
+function createSandbox(container = {remove: vi.fn()}) {
   // @ts-expect-error This test only exercises methods whose container operations are mocked.
   return new SystemSandbox(VirtualHost.create(), new Docker(), container)
 }
@@ -26,6 +23,42 @@ describe('SandboxSchema', () => {
   })
 })
 
+describe('SystemSandbox lifecycle', () => {
+  test('force removes the container when disposed', async () => {
+    const container = {
+      remove: vi.fn(),
+    }
+    const sandbox = createSandbox(container)
+
+    await sandbox[Symbol.asyncDispose]()
+
+    expect(container.remove).toHaveBeenCalledWith({force: true})
+  })
+
+  test('force removes the container when initialization fails', async () => {
+    const initializationError = new Error('Failed to start container')
+    const container = {
+      start: vi.fn().mockRejectedValue(initializationError),
+      remove: vi.fn(),
+    }
+    const docker = {
+      createContainer: vi.fn().mockResolvedValue(container),
+      pull: vi.fn((_name: string, callback: (error: Error | null, stream: NodeJS.ReadableStream) => void) => {
+        callback(null, {} as NodeJS.ReadableStream)
+      }),
+      modem: {
+        followProgress: vi.fn((_stream: NodeJS.ReadableStream, onFinished: (error: Error | null) => void) => {
+          onFinished(null)
+        }),
+      },
+    }
+
+    // @ts-expect-error This test only exercises the Docker methods used before container initialization.
+    await expect(createContainer(docker, 'test-image')).rejects.toBe(initializationError)
+    expect(container.remove).toHaveBeenCalledWith({force: true})
+  })
+})
+
 describe('SystemSandbox configuration helpers', () => {
   let sandbox: SystemSandbox
 
diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index a60030be..f7614198 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -71,7 +71,7 @@ class SystemSandbox implements Sandbox {
   }
 
   async [Symbol.asyncDispose]() {
-    await this.#container.stop()
+    await this.#container.remove({force: true})
   }
 
   async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise<void> {
@@ -334,90 +334,101 @@ async function createContainer(docker: Docker, dockerImage: string): Promise<Ini
     },
   })
 
-  await container.start()
+  try {
+    await container.start()
 
-  logger.debug('Creating workspace directory...')
-  await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, CONTAINER_WORKDIR], {
-    user: 'root',
-  })
+    logger.debug('Creating workspace directory...')
+    await execCommand(docker, container, 'mkdir', ['-p', CONTAINER_WORKDIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'chown', ['-R', NODE_USER, CONTAINER_WORKDIR], {
+      user: 'root',
+    })
 
-  logger.debug('Installing CA certificates...')
-  await execCommand(docker, container, 'apt-get', ['update'], {
-    user: 'root',
-  })
-  await execCommand(
-    docker,
-    container,
-    'apt-get',
-    ['install', '-y', '--no-install-recommends', 'ca-certificates', 'curl'],
-    {
+    logger.debug('Installing CA certificates...')
+    await execCommand(docker, container, 'apt-get', ['update'], {
       user: 'root',
-    },
-  )
-  await execCommand(docker, container, 'test', ['-d', '/etc/ssl/certs'], {
-    user: 'root',
-  })
+    })
+    await execCommand(
+      docker,
+      container,
+      'apt-get',
+      ['install', '-y', '--no-install-recommends', 'ca-certificates', 'curl'],
+      {
+        user: 'root',
+      },
+    )
+    await execCommand(docker, container, 'test', ['-d', '/etc/ssl/certs'], {
+      user: 'root',
+    })
 
-  logger.debug('Installing npm...')
-  await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
-    user: 'root',
-  })
-  const npmVersion = await execCommand(docker, container, 'npm', ['--version'], {
-    user: 'root',
-  })
-  if (npmVersion.stdout.trim() !== NPM_VERSION) {
-    throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
-  }
+    logger.debug('Installing npm...')
+    await execCommand(docker, container, 'npm', ['install', '--global', `npm@${NPM_VERSION}`], {
+      user: 'root',
+    })
+    const npmVersion = await execCommand(docker, container, 'npm', ['--version'], {
+      user: 'root',
+    })
+    if (npmVersion.stdout.trim() !== NPM_VERSION) {
+      throw new Error(`Expected npm ${NPM_VERSION}, received ${npmVersion.stdout.trim()}`)
+    }
 
-  logger.debug('Setting up npm for non-root global installs')
-  await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, NPM_GLOBAL_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'npm', ['config', 'set', 'prefix', NPM_GLOBAL_DIR], {
-    user: NODE_USER,
-  })
+    logger.debug('Setting up npm for non-root global installs')
+    await execCommand(docker, container, 'mkdir', ['-p', NPM_GLOBAL_DIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'chown', ['-R', NODE_USER, NPM_GLOBAL_DIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'npm', ['config', 'set', 'prefix', NPM_GLOBAL_DIR], {
+      user: NODE_USER,
+    })
 
-  logger.debug('Setting up copilot...')
-  await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, COPILOT_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'npm', ['install', '-g', `@github/copilot@${COPILOT_CLI_VERSION}`], {
-    user: NODE_USER,
-  })
-  await execCommand(docker, container, 'touch', [path.join(COPILOT_DIR, 'mcp-config.json')], {
-    user: NODE_USER,
-  })
-  await execCommand(
-    docker,
-    container,
-    'bash',
-    ['-c', `echo '${JSON.stringify(DEFAULT_MCP_CONFIG)}' > ${MCP_CONFIG_PATH}`],
-    {
+    logger.debug('Setting up copilot...')
+    await execCommand(docker, container, 'mkdir', ['-p', COPILOT_DIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'chown', ['-R', NODE_USER, COPILOT_DIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'npm', ['install', '-g', `@github/copilot@${COPILOT_CLI_VERSION}`], {
       user: NODE_USER,
-    },
-  )
-  await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], {
-    user: NODE_USER,
-  })
+    })
+    await execCommand(docker, container, 'touch', [path.join(COPILOT_DIR, 'mcp-config.json')], {
+      user: NODE_USER,
+    })
+    await execCommand(
+      docker,
+      container,
+      'bash',
+      ['-c', `echo '${JSON.stringify(DEFAULT_MCP_CONFIG)}' > ${MCP_CONFIG_PATH}`],
+      {
+        user: NODE_USER,
+      },
+    )
+    await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], {
+      user: NODE_USER,
+    })
 
-  logger.debug('Setting up agents config...')
-  await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
-    user: 'root',
-  })
-  await execCommand(docker, container, 'chown', ['-R', NODE_USER, AGENTS_DIR], {
-    user: 'root',
-  })
+    logger.debug('Setting up agents config...')
+    await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], {
+      user: 'root',
+    })
+    await execCommand(docker, container, 'chown', ['-R', NODE_USER, AGENTS_DIR], {
+      user: 'root',
+    })
 
-  return container as InitializedContainer
+    return container as InitializedContainer
+  } catch (error) {
+    try {
+      await container.remove({force: true})
+    } catch (cleanupError) {
+      throw new AggregateError([error, cleanupError], 'Failed to initialize and remove sandbox container', {
+        cause: cleanupError,
+      })
+    }
+    throw error
+  }
 }
 
 function mapCopiedHeader(header: Headers, sourceName: string, destinationName: string): Headers {
@@ -684,4 +695,4 @@ const SandboxSchema = z.custom<Sandbox>(value => {
   return value instanceof SystemSandbox || value instanceof VirtualSandbox
 })
 
-export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE}
+export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE, createContainer}

From aa64c16c1b74afd5aec8bbac4e5aad76e5e6cd5c Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 17:53:29 -0500
Subject: [PATCH 35/52] refactor: move benchmark scenarios to stacked change

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 18 ---------
 .../src/App.tsx                               | 19 ---------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 22 -----------
 .../src/App.tsx                               | 11 ------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../008-agent-infers-action-menu/index.html   | 12 ------
 .../008-agent-infers-action-menu/package.json | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 22 -----------
 .../008-agent-infers-action-menu/src/App.tsx  | 13 -------
 .../008-agent-infers-action-menu/src/main.tsx | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 21 ----------
 .../src/App.tsx                               | 11 ------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 22 -----------
 .../src/App.tsx                               | 13 -------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../011-agent-uses-motion-tokens/index.html   | 12 ------
 .../011-agent-uses-motion-tokens/package.json | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 17 --------
 .../011-agent-uses-motion-tokens/src/App.tsx  | 25 ------------
 .../011-agent-uses-motion-tokens/src/main.tsx | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../012-agent-infers-status-tokens/index.html | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 23 -----------
 .../src/App.tsx                               | 22 -----------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 17 --------
 .../src/App.tsx                               | 12 ------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 21 ----------
 .../src/App.tsx                               | 25 ------------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../015-agent-infers-copy-icon/index.html     | 12 ------
 .../015-agent-infers-copy-icon/package.json   | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 23 -----------
 .../015-agent-infers-copy-icon/src/App.tsx    | 10 -----
 .../015-agent-infers-copy-icon/src/main.tsx   | 16 --------
 .../015-agent-infers-copy-icon/src/styles.css |  7 ----
 .../015-agent-infers-copy-icon/tsconfig.json  | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 26 -------------
 .../src/App.tsx                               | 16 --------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 23 -----------
 .../src/App.tsx                               |  9 -----
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../018-agent-uses-filter-pattern/index.html  | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 23 -----------
 .../018-agent-uses-filter-pattern/src/App.tsx | 17 --------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 19 ---------
 .../src/App.tsx                               | 31 ---------------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 18 ---------
 .../src/App.tsx                               | 12 ------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 29 --------------
 .../src/App.tsx                               |  3 --
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 22 -----------
 .../src/App.tsx                               |  3 --
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../023-agent-adds-theme-switcher/index.html  | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 24 ------------
 .../023-agent-adds-theme-switcher/src/App.tsx |  3 --
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../024-agent-sets-up-tailwindcss/index.html  | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 39 -------------------
 .../024-agent-sets-up-tailwindcss/src/App.tsx |  3 --
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 .../index.html                                | 12 ------
 .../package.json                              | 24 ------------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 23 -----------
 .../src/App.tsx                               | 11 ------
 .../src/main.tsx                              | 16 --------
 .../src/styles.css                            |  9 -----
 .../tsconfig.json                             | 18 ---------
 .../vite.config.ts                            |  6 ---
 .../index.html                                | 12 ------
 .../package.json                              | 22 -----------
 .../scenario.config.ts                        |  7 ----
 .../scenario.test.ts                          | 22 -----------
 .../src/App.tsx                               | 11 ------
 .../src/main.tsx                              | 10 -----
 .../src/styles.css                            |  7 ----
 .../tsconfig.json                             | 18 ---------
 169 files changed, 2480 deletions(-)
 delete mode 100644 scenarios/006-agent-uses-pagination-component/index.html
 delete mode 100644 scenarios/006-agent-uses-pagination-component/package.json
 delete mode 100644 scenarios/006-agent-uses-pagination-component/scenario.config.ts
 delete mode 100644 scenarios/006-agent-uses-pagination-component/scenario.test.ts
 delete mode 100644 scenarios/006-agent-uses-pagination-component/src/App.tsx
 delete mode 100644 scenarios/006-agent-uses-pagination-component/src/main.tsx
 delete mode 100644 scenarios/006-agent-uses-pagination-component/src/styles.css
 delete mode 100644 scenarios/006-agent-uses-pagination-component/tsconfig.json
 delete mode 100644 scenarios/007-agent-infers-billing-banner/index.html
 delete mode 100644 scenarios/007-agent-infers-billing-banner/package.json
 delete mode 100644 scenarios/007-agent-infers-billing-banner/scenario.config.ts
 delete mode 100644 scenarios/007-agent-infers-billing-banner/scenario.test.ts
 delete mode 100644 scenarios/007-agent-infers-billing-banner/src/App.tsx
 delete mode 100644 scenarios/007-agent-infers-billing-banner/src/main.tsx
 delete mode 100644 scenarios/007-agent-infers-billing-banner/src/styles.css
 delete mode 100644 scenarios/007-agent-infers-billing-banner/tsconfig.json
 delete mode 100644 scenarios/008-agent-infers-action-menu/index.html
 delete mode 100644 scenarios/008-agent-infers-action-menu/package.json
 delete mode 100644 scenarios/008-agent-infers-action-menu/scenario.config.ts
 delete mode 100644 scenarios/008-agent-infers-action-menu/scenario.test.ts
 delete mode 100644 scenarios/008-agent-infers-action-menu/src/App.tsx
 delete mode 100644 scenarios/008-agent-infers-action-menu/src/main.tsx
 delete mode 100644 scenarios/008-agent-infers-action-menu/src/styles.css
 delete mode 100644 scenarios/008-agent-infers-action-menu/tsconfig.json
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/index.html
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/package.json
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
 delete mode 100644 scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/index.html
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/package.json
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/scenario.config.ts
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/scenario.test.ts
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/src/App.tsx
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/src/main.tsx
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/src/styles.css
 delete mode 100644 scenarios/010-agent-uses-typography-tokens/tsconfig.json
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/index.html
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/package.json
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/scenario.config.ts
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/scenario.test.ts
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/src/App.tsx
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/src/main.tsx
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/src/styles.css
 delete mode 100644 scenarios/011-agent-uses-motion-tokens/tsconfig.json
 delete mode 100644 scenarios/012-agent-infers-status-tokens/index.html
 delete mode 100644 scenarios/012-agent-infers-status-tokens/package.json
 delete mode 100644 scenarios/012-agent-infers-status-tokens/scenario.config.ts
 delete mode 100644 scenarios/012-agent-infers-status-tokens/scenario.test.ts
 delete mode 100644 scenarios/012-agent-infers-status-tokens/src/App.tsx
 delete mode 100644 scenarios/012-agent-infers-status-tokens/src/main.tsx
 delete mode 100644 scenarios/012-agent-infers-status-tokens/src/styles.css
 delete mode 100644 scenarios/012-agent-infers-status-tokens/tsconfig.json
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/index.html
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/package.json
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/src/styles.css
 delete mode 100644 scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
 delete mode 100644 scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
 delete mode 100644 scenarios/015-agent-infers-copy-icon/index.html
 delete mode 100644 scenarios/015-agent-infers-copy-icon/package.json
 delete mode 100644 scenarios/015-agent-infers-copy-icon/scenario.config.ts
 delete mode 100644 scenarios/015-agent-infers-copy-icon/scenario.test.ts
 delete mode 100644 scenarios/015-agent-infers-copy-icon/src/App.tsx
 delete mode 100644 scenarios/015-agent-infers-copy-icon/src/main.tsx
 delete mode 100644 scenarios/015-agent-infers-copy-icon/src/styles.css
 delete mode 100644 scenarios/015-agent-infers-copy-icon/tsconfig.json
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
 delete mode 100644 scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/index.html
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/package.json
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/src/styles.css
 delete mode 100644 scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/index.html
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/package.json
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/scenario.config.ts
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/scenario.test.ts
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/src/App.tsx
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/src/main.tsx
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/src/styles.css
 delete mode 100644 scenarios/018-agent-uses-filter-pattern/tsconfig.json
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/index.html
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/package.json
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/src/styles.css
 delete mode 100644 scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/index.html
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/package.json
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/src/styles.css
 delete mode 100644 scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/index.html
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/package.json
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
 delete mode 100644 scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/index.html
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/package.json
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/scenario.config.ts
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/scenario.test.ts
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/src/App.tsx
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/src/main.tsx
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/src/styles.css
 delete mode 100644 scenarios/022-agent-enables-automatic-theming/tsconfig.json
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/index.html
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/package.json
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/scenario.config.ts
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/scenario.test.ts
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/src/App.tsx
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/src/main.tsx
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/src/styles.css
 delete mode 100644 scenarios/023-agent-adds-theme-switcher/tsconfig.json
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/index.html
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/package.json
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/src/styles.css
 delete mode 100644 scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
 delete mode 100644 scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/index.html
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/package.json
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/src/styles.css
 delete mode 100644 scenarios/026-agent-avoids-deprecated-notification/tsconfig.json

diff --git a/scenarios/006-agent-uses-pagination-component/index.html b/scenarios/006-agent-uses-pagination-component/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/006-agent-uses-pagination-component/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/006-agent-uses-pagination-component/package.json b/scenarios/006-agent-uses-pagination-component/package.json
deleted file mode 100644
index 6c0786af..00000000
--- a/scenarios/006-agent-uses-pagination-component/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "006-agent-uses-pagination-component",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/006-agent-uses-pagination-component/scenario.config.ts b/scenarios/006-agent-uses-pagination-component/scenario.config.ts
deleted file mode 100644
index 59de7fc8..00000000
--- a/scenarios/006-agent-uses-pagination-component/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent uses an existing component when adding pagination.',
-  prompt: `Add pagination controls below the issue list. Show 25 issues per page and include previous and next navigation.`,
-  tags: ['component', 'pagination', 'vite'],
-})
diff --git a/scenarios/006-agent-uses-pagination-component/scenario.test.ts b/scenarios/006-agent-uses-pagination-component/scenario.test.ts
deleted file mode 100644
index 553184ff..00000000
--- a/scenarios/006-agent-uses-pagination-component/scenario.test.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports Pagination from the design system', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bPagination\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-})
-
-test('renders Pagination', () => {
-  expect(app).toMatch(/<Pagination(?:\s[^>]*)?>/)
-})
-
-test('configures the current page and page count', () => {
-  expect(app).toMatch(/<Pagination[^>]*\bcurrentPage=\{?[^}\s]+}?/)
-  expect(app).toMatch(/<Pagination[^>]*\bpageCount=\{?[^}\s]+}?/)
-})
diff --git a/scenarios/006-agent-uses-pagination-component/src/App.tsx b/scenarios/006-agent-uses-pagination-component/src/App.tsx
deleted file mode 100644
index dccd9e6d..00000000
--- a/scenarios/006-agent-uses-pagination-component/src/App.tsx
+++ /dev/null
@@ -1,19 +0,0 @@
-export function App() {
-  const issues = Array.from({length: 25}, (_, index) => {
-    return {
-      id: index + 1,
-      title: `Issue ${index + 1}`,
-    }
-  })
-
-  return (
-    <main>
-      <h1>Issues</h1>
-      <ul>
-        {issues.map(issue => {
-          return <li key={issue.id}>{issue.title}</li>
-        })}
-      </ul>
-    </main>
-  )
-}
diff --git a/scenarios/006-agent-uses-pagination-component/src/main.tsx b/scenarios/006-agent-uses-pagination-component/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/006-agent-uses-pagination-component/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/006-agent-uses-pagination-component/src/styles.css b/scenarios/006-agent-uses-pagination-component/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/006-agent-uses-pagination-component/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/006-agent-uses-pagination-component/tsconfig.json b/scenarios/006-agent-uses-pagination-component/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/006-agent-uses-pagination-component/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/007-agent-infers-billing-banner/index.html b/scenarios/007-agent-infers-billing-banner/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/007-agent-infers-billing-banner/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/007-agent-infers-billing-banner/package.json b/scenarios/007-agent-infers-billing-banner/package.json
deleted file mode 100644
index 3114537a..00000000
--- a/scenarios/007-agent-infers-billing-banner/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "007-agent-infers-billing-banner",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/007-agent-infers-billing-banner/scenario.config.ts b/scenarios/007-agent-infers-billing-banner/scenario.config.ts
deleted file mode 100644
index a1fb46aa..00000000
--- a/scenarios/007-agent-infers-billing-banner/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent selects an appropriate component for a persistent warning.',
-  prompt: `Show a persistent warning at the top of the page when an account has a past-due balance. Include a link to billing settings.`,
-  tags: ['banner', 'component', 'vite'],
-})
diff --git a/scenarios/007-agent-infers-billing-banner/scenario.test.ts b/scenarios/007-agent-infers-billing-banner/scenario.test.ts
deleted file mode 100644
index fc7ab5fc..00000000
--- a/scenarios/007-agent-infers-billing-banner/scenario.test.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports the current Banner component', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-})
-
-test('renders an attention Banner for the past-due state', () => {
-  expect(app).toMatch(/<Banner[^>]*variant=["'](?:warning|critical)["'][^>]*>/)
-  expect(app).toMatch(/past[- ]due/i)
-})
-
-test('does not use the deprecated Flash component', () => {
-  expect(app).not.toMatch(/\bFlash\b/)
-})
-
-test('links to billing settings from the warning', () => {
-  expect(app).toMatch(/<Banner[\s\S]*billing[\s\S]*<\/Banner>/i)
-})
diff --git a/scenarios/007-agent-infers-billing-banner/src/App.tsx b/scenarios/007-agent-infers-billing-banner/src/App.tsx
deleted file mode 100644
index 2f6f28f5..00000000
--- a/scenarios/007-agent-infers-billing-banner/src/App.tsx
+++ /dev/null
@@ -1,11 +0,0 @@
-export function App() {
-  const hasPastDueBalance = true
-
-  return (
-    <main>
-      <h1>Account</h1>
-      <p>{hasPastDueBalance ? 'Payment required' : 'Your account is in good standing'}</p>
-      <a href="/settings/billing">Billing settings</a>
-    </main>
-  )
-}
diff --git a/scenarios/007-agent-infers-billing-banner/src/main.tsx b/scenarios/007-agent-infers-billing-banner/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/007-agent-infers-billing-banner/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/007-agent-infers-billing-banner/src/styles.css b/scenarios/007-agent-infers-billing-banner/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/007-agent-infers-billing-banner/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/007-agent-infers-billing-banner/tsconfig.json b/scenarios/007-agent-infers-billing-banner/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/007-agent-infers-billing-banner/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/008-agent-infers-action-menu/index.html b/scenarios/008-agent-infers-action-menu/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/008-agent-infers-action-menu/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/008-agent-infers-action-menu/package.json b/scenarios/008-agent-infers-action-menu/package.json
deleted file mode 100644
index 539e8506..00000000
--- a/scenarios/008-agent-infers-action-menu/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "008-agent-infers-action-menu",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/008-agent-infers-action-menu/scenario.config.ts b/scenarios/008-agent-infers-action-menu/scenario.config.ts
deleted file mode 100644
index 654ef042..00000000
--- a/scenarios/008-agent-infers-action-menu/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent selects an appropriate component for secondary actions.',
-  prompt: `Add archive, transfer, and delete actions to the repository header without crowding the existing primary actions.`,
-  tags: ['component', 'menu', 'vite'],
-})
diff --git a/scenarios/008-agent-infers-action-menu/scenario.test.ts b/scenarios/008-agent-infers-action-menu/scenario.test.ts
deleted file mode 100644
index 961b51a5..00000000
--- a/scenarios/008-agent-infers-action-menu/scenario.test.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports ActionMenu and ActionList from the design system', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-})
-
-test('renders an ActionMenu', () => {
-  expect(app).toMatch(/<ActionMenu(?:\s[^>]*)?>[\s\S]*<\/ActionMenu>/)
-})
-
-test.each(['Archive', 'Transfer', 'Delete'])('includes the %s action', action => {
-  expect(app).toContain(action)
-})
-
-test('marks the delete action as destructive', () => {
-  expect(app).toMatch(/<ActionList\.Item[^>]*variant=["']danger["'][^>]*>[\s\S]*Delete/)
-})
diff --git a/scenarios/008-agent-infers-action-menu/src/App.tsx b/scenarios/008-agent-infers-action-menu/src/App.tsx
deleted file mode 100644
index edeaf481..00000000
--- a/scenarios/008-agent-infers-action-menu/src/App.tsx
+++ /dev/null
@@ -1,13 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <header>
-        <h1>octo-repo</h1>
-        <div>
-          <button type="button">Edit</button>
-        </div>
-      </header>
-      <p>Repository settings and activity.</p>
-    </main>
-  )
-}
diff --git a/scenarios/008-agent-infers-action-menu/src/main.tsx b/scenarios/008-agent-infers-action-menu/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/008-agent-infers-action-menu/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/008-agent-infers-action-menu/src/styles.css b/scenarios/008-agent-infers-action-menu/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/008-agent-infers-action-menu/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/008-agent-infers-action-menu/tsconfig.json b/scenarios/008-agent-infers-action-menu/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/008-agent-infers-action-menu/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/index.html b/scenarios/009-agent-uses-layout-and-color-tokens/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/package.json b/scenarios/009-agent-uses-layout-and-color-tokens/package.json
deleted file mode 100644
index 756b697a..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "009-agent-uses-layout-and-color-tokens",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
deleted file mode 100644
index 5cc514b6..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent uses design tokens for layout and color styling.',
-  prompt: `Style the status summary card so its content is clearly grouped and visually distinct from the page background.`,
-  tags: ['color', 'layout', 'tokens', 'vite'],
-})
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
deleted file mode 100644
index 728f1a32..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-
-test('uses semantic background or foreground color tokens', () => {
-  expect(styles).toMatch(/var\(--(?:bgColor|fgColor)-[A-Za-z0-9-]+\)/)
-})
-
-test('uses a semantic border token', () => {
-  expect(styles).toMatch(/var\(--border(?:Color|Width)?-[A-Za-z0-9-]+\)/)
-})
-
-test('uses stack tokens for layout spacing', () => {
-  expect(styles).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/)
-})
-
-test('does not introduce raw hexadecimal colors', () => {
-  expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i)
-})
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
deleted file mode 100644
index d270d223..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx
+++ /dev/null
@@ -1,11 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>System status</h1>
-      <section className="status-summary">
-        <h2>All systems operational</h2>
-        <p>Last checked one minute ago.</p>
-      </section>
-    </main>
-  )
-}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css b/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json b/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/010-agent-uses-typography-tokens/index.html b/scenarios/010-agent-uses-typography-tokens/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/010-agent-uses-typography-tokens/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/010-agent-uses-typography-tokens/package.json b/scenarios/010-agent-uses-typography-tokens/package.json
deleted file mode 100644
index f3d8033e..00000000
--- a/scenarios/010-agent-uses-typography-tokens/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "010-agent-uses-typography-tokens",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
deleted file mode 100644
index 7ef60583..00000000
--- a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent uses role-appropriate typography tokens.',
-  prompt: `Improve the typography of the documentation page. It contains a page title, introductory text, inline code, and a code example.`,
-  tags: ['tokens', 'typography', 'vite'],
-})
diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.test.ts b/scenarios/010-agent-uses-typography-tokens/scenario.test.ts
deleted file mode 100644
index 9d8ca2fd..00000000
--- a/scenarios/010-agent-uses-typography-tokens/scenario.test.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-
-test('uses a title typography shorthand token', () => {
-  expect(styles).toMatch(/font:\s*var\(--text-title-shorthand-(?:small|medium|large)\)/)
-})
-
-test('uses a body typography shorthand token', () => {
-  expect(styles).toMatch(/font:\s*var\(--text-body-shorthand-(?:small|medium|large)\)/)
-})
-
-test('uses code typography shorthand tokens', () => {
-  expect(styles).toMatch(/font:\s*var\(--text-codeInline-shorthand\)/)
-  expect(styles).toMatch(/font:\s*var\(--text-codeBlock-shorthand\)/)
-})
-
-test('does not set raw font sizes or line heights', () => {
-  expect(styles).not.toMatch(/(?:font-size|line-height):\s*(?:\d|calc\()/)
-})
diff --git a/scenarios/010-agent-uses-typography-tokens/src/App.tsx b/scenarios/010-agent-uses-typography-tokens/src/App.tsx
deleted file mode 100644
index beeddf48..00000000
--- a/scenarios/010-agent-uses-typography-tokens/src/App.tsx
+++ /dev/null
@@ -1,13 +0,0 @@
-export function App() {
-  return (
-    <main className="documentation">
-      <h1>Configure the CLI</h1>
-      <p>
-        Create a <code>config.json</code> file in your project directory.
-      </p>
-      <pre>
-        <code>{`{"theme": "system"}`}</code>
-      </pre>
-    </main>
-  )
-}
diff --git a/scenarios/010-agent-uses-typography-tokens/src/main.tsx b/scenarios/010-agent-uses-typography-tokens/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/010-agent-uses-typography-tokens/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/010-agent-uses-typography-tokens/src/styles.css b/scenarios/010-agent-uses-typography-tokens/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/010-agent-uses-typography-tokens/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/010-agent-uses-typography-tokens/tsconfig.json b/scenarios/010-agent-uses-typography-tokens/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/010-agent-uses-typography-tokens/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/011-agent-uses-motion-tokens/index.html b/scenarios/011-agent-uses-motion-tokens/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/011-agent-uses-motion-tokens/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/011-agent-uses-motion-tokens/package.json b/scenarios/011-agent-uses-motion-tokens/package.json
deleted file mode 100644
index d906c89c..00000000
--- a/scenarios/011-agent-uses-motion-tokens/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "011-agent-uses-motion-tokens",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
deleted file mode 100644
index 83f268e7..00000000
--- a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent uses motion tokens and respects reduced-motion preferences.',
-  prompt: `Add a short transition when the details panel expands or collapses. Keep the interaction comfortable for people who prefer reduced motion.`,
-  tags: ['accessibility', 'motion', 'tokens', 'vite'],
-})
diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.test.ts b/scenarios/011-agent-uses-motion-tokens/scenario.test.ts
deleted file mode 100644
index 861b5f15..00000000
--- a/scenarios/011-agent-uses-motion-tokens/scenario.test.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-
-test('uses a motion token for the transition', () => {
-  expect(styles).toMatch(/var\(--motion-(?:transition|duration|easing)-[A-Za-z0-9-]+\)/)
-})
-
-test('defines a reduced-motion alternative', () => {
-  expect(styles).toMatch(/@media\s*\(prefers-reduced-motion:\s*reduce\)/)
-})
-
-test('does not use raw transition timing values', () => {
-  expect(styles).not.toMatch(/(?:transition|animation)[^;]*(?:\d+m?s|ease(?:-in|-out|-in-out)?)/)
-})
diff --git a/scenarios/011-agent-uses-motion-tokens/src/App.tsx b/scenarios/011-agent-uses-motion-tokens/src/App.tsx
deleted file mode 100644
index cd0a07d8..00000000
--- a/scenarios/011-agent-uses-motion-tokens/src/App.tsx
+++ /dev/null
@@ -1,25 +0,0 @@
-import {useState} from 'react'
-
-export function App() {
-  const [isOpen, setIsOpen] = useState(false)
-
-  return (
-    <main>
-      <button
-        aria-expanded={isOpen}
-        aria-controls="details-panel"
-        type="button"
-        onClick={() => {
-          setIsOpen(current => {
-            return !current
-          })
-        }}
-      >
-        View details
-      </button>
-      <section className="details-panel" hidden={!isOpen} id="details-panel">
-        <p>The latest deployment completed successfully.</p>
-      </section>
-    </main>
-  )
-}
diff --git a/scenarios/011-agent-uses-motion-tokens/src/main.tsx b/scenarios/011-agent-uses-motion-tokens/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/011-agent-uses-motion-tokens/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/011-agent-uses-motion-tokens/src/styles.css b/scenarios/011-agent-uses-motion-tokens/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/011-agent-uses-motion-tokens/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/011-agent-uses-motion-tokens/tsconfig.json b/scenarios/011-agent-uses-motion-tokens/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/011-agent-uses-motion-tokens/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/012-agent-infers-status-tokens/index.html b/scenarios/012-agent-infers-status-tokens/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/012-agent-infers-status-tokens/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/012-agent-infers-status-tokens/package.json b/scenarios/012-agent-infers-status-tokens/package.json
deleted file mode 100644
index 7dec141f..00000000
--- a/scenarios/012-agent-infers-status-tokens/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "012-agent-infers-status-tokens",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/012-agent-infers-status-tokens/scenario.config.ts b/scenarios/012-agent-infers-status-tokens/scenario.config.ts
deleted file mode 100644
index 17fe6215..00000000
--- a/scenarios/012-agent-infers-status-tokens/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent chooses semantically correct status tokens.',
-  prompt: `Update the deployment list so successful and failed deployments are easy to distinguish without relying on text alone.`,
-  tags: ['color', 'status', 'tokens', 'vite'],
-})
diff --git a/scenarios/012-agent-infers-status-tokens/scenario.test.ts b/scenarios/012-agent-infers-status-tokens/scenario.test.ts
deleted file mode 100644
index 4b181bbe..00000000
--- a/scenarios/012-agent-infers-status-tokens/scenario.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-
-test('uses success tokens for successful deployments', () => {
-  expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/)
-})
-
-test('uses danger tokens for failed deployments', () => {
-  expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-danger(?:-[A-Za-z0-9-]+)?\)/)
-})
-
-test('uses a non-text status indicator', () => {
-  expect(app).toMatch(/@primer\/octicons-react/)
-  expect(app).toMatch(/(?:Check|Pass|X|Stop|Alert)[A-Za-z]*Icon/)
-})
-
-test('does not introduce raw hexadecimal colors', () => {
-  expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i)
-})
diff --git a/scenarios/012-agent-infers-status-tokens/src/App.tsx b/scenarios/012-agent-infers-status-tokens/src/App.tsx
deleted file mode 100644
index 012f7e30..00000000
--- a/scenarios/012-agent-infers-status-tokens/src/App.tsx
+++ /dev/null
@@ -1,22 +0,0 @@
-export function App() {
-  const deployments = [
-    {id: 1, name: 'Production', status: 'success'},
-    {id: 2, name: 'Staging', status: 'failure'},
-  ]
-
-  return (
-    <main>
-      <h1>Deployments</h1>
-      <ul>
-        {deployments.map(deployment => {
-          return (
-            <li className={`deployment deployment-${deployment.status}`} key={deployment.id}>
-              <span>{deployment.name}</span>
-              <span>{deployment.status}</span>
-            </li>
-          )
-        })}
-      </ul>
-    </main>
-  )
-}
diff --git a/scenarios/012-agent-infers-status-tokens/src/main.tsx b/scenarios/012-agent-infers-status-tokens/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/012-agent-infers-status-tokens/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/012-agent-infers-status-tokens/src/styles.css b/scenarios/012-agent-infers-status-tokens/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/012-agent-infers-status-tokens/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/012-agent-infers-status-tokens/tsconfig.json b/scenarios/012-agent-infers-status-tokens/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/012-agent-infers-status-tokens/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/index.html b/scenarios/013-agent-infers-compact-control-tokens/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/013-agent-infers-compact-control-tokens/package.json b/scenarios/013-agent-infers-compact-control-tokens/package.json
deleted file mode 100644
index 959cb638..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "013-agent-infers-compact-control-tokens",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
deleted file mode 100644
index 55f240e0..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent chooses appropriate tokens for a compact toolbar.',
-  prompt: `Make the repository file toolbar more compact while keeping its controls usable and consistently spaced.`,
-  tags: ['controls', 'layout', 'tokens', 'vite'],
-})
diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
deleted file mode 100644
index 57bab2ed..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-
-test('uses compact control size or padding tokens', () => {
-  expect(styles).toMatch(/var\(--control-(?:xsmall|small)-(?:size|padding(?:Block|Inline-[A-Za-z]+))\)/)
-})
-
-test('uses a token for spacing between toolbar controls', () => {
-  expect(styles).toMatch(/var\(--(?:controlStack|stack)-[A-Za-z0-9-]*gap[A-Za-z0-9-]*\)/)
-})
-
-test('does not use raw pixel values for control sizing', () => {
-  expect(styles).not.toMatch(/(?:gap|height|padding(?:-block|-inline)?):\s*\d+px/)
-})
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
deleted file mode 100644
index a78da00d..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>src/components</h1>
-      <div className="file-toolbar" role="toolbar" aria-label="File actions">
-        <button type="button">Add file</button>
-        <button type="button">Go to file</button>
-        <button type="button">Code</button>
-      </div>
-    </main>
-  )
-}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/styles.css b/scenarios/013-agent-infers-compact-control-tokens/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json b/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html b/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
deleted file mode 100644
index ce58381c..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "014-agent-replaces-custom-icons-with-octicons",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
deleted file mode 100644
index 070ea270..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent replaces custom icons with maintained design-system icons.',
-  prompt: `Replace the hand-drawn search, download, and trash icons in the toolbar with icons from the project's design system.`,
-  tags: ['icon', 'octicons', 'vite'],
-})
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
deleted file mode 100644
index a4a00106..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports icons directly from Primer Octicons', () => {
-  expect(app).toMatch(/from\s+['"]@primer\/octicons-react['"]/)
-})
-
-test.each(['SearchIcon', 'DownloadIcon', 'TrashIcon'])('uses %s', icon => {
-  expect(app).toMatch(new RegExp(`(?:<${icon}(?:\\s[^>]*)?\\/?>|icon=\\{${icon}\\})`))
-})
-
-test('removes the hand-drawn SVG elements', () => {
-  expect(app).not.toMatch(/<svg\b/)
-})
-
-test('does not use the deprecated Octicon wrapper', () => {
-  expect(app).not.toMatch(/<Octicon\b/)
-})
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
deleted file mode 100644
index 2a8dce2f..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/App.tsx
+++ /dev/null
@@ -1,25 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>Files</h1>
-      <div role="toolbar" aria-label="File actions">
-        <button type="button" aria-label="Search">
-          <svg aria-hidden="true" viewBox="0 0 16 16">
-            <circle cx="7" cy="7" r="5" />
-            <path d="m11 11 4 4" />
-          </svg>
-        </button>
-        <button type="button" aria-label="Download">
-          <svg aria-hidden="true" viewBox="0 0 16 16">
-            <path d="M8 1v10m-4-4 4 4 4-4M2 15h12" />
-          </svg>
-        </button>
-        <button type="button" aria-label="Delete">
-          <svg aria-hidden="true" viewBox="0 0 16 16">
-            <path d="M3 4h10M6 1h4l1 3H5l1-3Zm-1 5v8m3-8v8m3-8v8" />
-          </svg>
-        </button>
-      </div>
-    </main>
-  )
-}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/015-agent-infers-copy-icon/index.html b/scenarios/015-agent-infers-copy-icon/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/015-agent-infers-copy-icon/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/015-agent-infers-copy-icon/package.json b/scenarios/015-agent-infers-copy-icon/package.json
deleted file mode 100644
index 4bde6a34..00000000
--- a/scenarios/015-agent-infers-copy-icon/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "015-agent-infers-copy-icon",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/015-agent-infers-copy-icon/scenario.config.ts b/scenarios/015-agent-infers-copy-icon/scenario.config.ts
deleted file mode 100644
index 946fd008..00000000
--- a/scenarios/015-agent-infers-copy-icon/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent infers the appropriate icon for a compact action.',
-  prompt: `Add a compact control next to the commit SHA that copies it to the clipboard.`,
-  tags: ['component', 'icon', 'vite'],
-})
diff --git a/scenarios/015-agent-infers-copy-icon/scenario.test.ts b/scenarios/015-agent-infers-copy-icon/scenario.test.ts
deleted file mode 100644
index 4288cb4f..00000000
--- a/scenarios/015-agent-infers-copy-icon/scenario.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports and renders CopyIcon', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bCopyIcon\b[^}]*}\s+from\s+['"]@primer\/octicons-react['"]/)
-  expect(app).toMatch(/<CopyIcon(?:\s[^>]*)?\/?>|icon=\{CopyIcon\}/)
-})
-
-test('uses the current IconButton component', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bIconButton\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/<IconButton\b/)
-})
-
-test('gives the compact control an accessible label', () => {
-  expect(app).toMatch(/<IconButton[^>]*aria-label=["'][^"']*copy[^"']*["'][^>]*>/i)
-})
-
-test('copies the commit SHA to the clipboard', () => {
-  expect(app).toMatch(/navigator\.clipboard\.writeText\(/)
-})
diff --git a/scenarios/015-agent-infers-copy-icon/src/App.tsx b/scenarios/015-agent-infers-copy-icon/src/App.tsx
deleted file mode 100644
index e0070a47..00000000
--- a/scenarios/015-agent-infers-copy-icon/src/App.tsx
+++ /dev/null
@@ -1,10 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>Latest commit</h1>
-      <p>
-        <code>8f3c2a1</code>
-      </p>
-    </main>
-  )
-}
diff --git a/scenarios/015-agent-infers-copy-icon/src/main.tsx b/scenarios/015-agent-infers-copy-icon/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/015-agent-infers-copy-icon/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/015-agent-infers-copy-icon/src/styles.css b/scenarios/015-agent-infers-copy-icon/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/015-agent-infers-copy-icon/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/015-agent-infers-copy-icon/tsconfig.json b/scenarios/015-agent-infers-copy-icon/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/015-agent-infers-copy-icon/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html b/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
deleted file mode 100644
index 7d9a9548..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "016-agent-uses-loading-and-empty-state-patterns",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
deleted file mode 100644
index aa01276b..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent applies established loading and empty-state patterns.',
-  prompt: `Add appropriate loading and empty states to the repository list. The empty state should help the user create their first repository.`,
-  tags: ['empty-state', 'loading', 'pattern', 'vite'],
-})
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
deleted file mode 100644
index fb4742a0..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('uses skeleton components for the loading state', () => {
-  expect(app).toMatch(
-    /import\s+{[^}]*(?:SkeletonBox|SkeletonText)[^}]*}\s+from\s+['"]@primer\/react(?:\/experimental)?['"]/,
-  )
-  expect(app).toMatch(/<(?:SkeletonBox|SkeletonText)\b/)
-})
-
-test('uses Blankslate for the empty state', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bBlankslate\b[^}]*}\s+from\s+['"]@primer\/react\/experimental['"]/)
-  expect(app).toMatch(/<Blankslate(?:\s[^>]*)?>[\s\S]*<\/Blankslate>/)
-})
-
-test('gives the empty state a create-repository action', () => {
-  expect(app).toMatch(/<Blankslate[\s\S]*(?:Create|New) repository[\s\S]*<\/Blankslate>/i)
-})
-
-test('renders the states conditionally', () => {
-  expect(app).toMatch(/\bisLoading\b[\s\S]*(?:SkeletonBox|SkeletonText)/)
-  expect(app).toMatch(/repositories\.length[\s\S]*Blankslate|Blankslate[\s\S]*repositories\.length/)
-})
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
deleted file mode 100644
index d04ef697..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-export function App() {
-  const isLoading = false
-  const repositories = [{id: 1, name: 'octo-repo'}]
-
-  return (
-    <main>
-      <h1>Repositories</h1>
-      <p>{isLoading ? 'Loading repositories' : `${repositories.length} repositories`}</p>
-      <ul>
-        {repositories.map(repository => {
-          return <li key={repository.id}>{repository.name}</li>
-        })}
-      </ul>
-    </main>
-  )
-}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/index.html b/scenarios/017-agent-uses-confirmation-pattern/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/017-agent-uses-confirmation-pattern/package.json b/scenarios/017-agent-uses-confirmation-pattern/package.json
deleted file mode 100644
index d14fc47e..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "017-agent-uses-confirmation-pattern",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
deleted file mode 100644
index 6fefaca9..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent applies the established confirmation pattern to a destructive action.',
-  prompt: `Let an administrator delete the repository after confirming the destructive action. Include clear cancel and confirm paths.`,
-  tags: ['confirmation', 'dialog', 'pattern', 'vite'],
-})
diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
deleted file mode 100644
index 34f7e37d..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('uses the current ConfirmationDialog API', () => {
-  expect(app).toMatch(/import\s+{[^}]*(?:\bConfirmationDialog\b|\buseConfirm\b)[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/<ConfirmationDialog\b|\buseConfirm\(/)
-})
-
-test('does not use the deprecated Dialog component', () => {
-  expect(app).not.toMatch(/<Dialog\b/)
-})
-
-test('provides clear cancel and destructive confirmation labels', () => {
-  expect(app).toMatch(/cancel/i)
-  expect(app).toMatch(/delete repository/i)
-})
-
-test('uses a danger-styled confirmation action', () => {
-  expect(app).toMatch(/(?:confirmButtonType|variant)=["']danger["']/)
-})
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
deleted file mode 100644
index 975c0eab..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx
+++ /dev/null
@@ -1,9 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>Danger zone</h1>
-      <p>Deleting this repository removes its code, issues, and settings.</p>
-      <button type="button">Delete repository</button>
-    </main>
-  )
-}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/styles.css b/scenarios/017-agent-uses-confirmation-pattern/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json b/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/018-agent-uses-filter-pattern/index.html b/scenarios/018-agent-uses-filter-pattern/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/018-agent-uses-filter-pattern/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/018-agent-uses-filter-pattern/package.json b/scenarios/018-agent-uses-filter-pattern/package.json
deleted file mode 100644
index 38fceca3..00000000
--- a/scenarios/018-agent-uses-filter-pattern/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "018-agent-uses-filter-pattern",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
deleted file mode 100644
index ba9008ea..00000000
--- a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent composes current components into an issue-filtering pattern.',
-  prompt: `Add controls for filtering the issue list by author, label, and open or closed status.`,
-  tags: ['filter', 'pattern', 'vite'],
-})
diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.test.ts b/scenarios/018-agent-uses-filter-pattern/scenario.test.ts
deleted file mode 100644
index e3c57d5c..00000000
--- a/scenarios/018-agent-uses-filter-pattern/scenario.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('composes current menu components for the filters', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app.match(/<ActionMenu(?:\s[^>]*)?>/g)?.length).toBeGreaterThanOrEqual(3)
-})
-
-test.each(['author', 'label', 'status'])('provides a visible %s filter', filter => {
-  expect(app).toMatch(new RegExp(`>[^<]*${filter}[^<]*<`, 'i'))
-})
-
-test('announces the updated result count', () => {
-  expect(app).toMatch(/role=["']status["']/)
-})
-
-test('does not use deprecated filtering components', () => {
-  expect(app).not.toMatch(/\bFilteredSearch\b|\bSelectPanel\b/)
-})
diff --git a/scenarios/018-agent-uses-filter-pattern/src/App.tsx b/scenarios/018-agent-uses-filter-pattern/src/App.tsx
deleted file mode 100644
index 8b22478b..00000000
--- a/scenarios/018-agent-uses-filter-pattern/src/App.tsx
+++ /dev/null
@@ -1,17 +0,0 @@
-export function App() {
-  const issues = [
-    {id: 1, title: 'Improve keyboard navigation'},
-    {id: 2, title: 'Document release process'},
-  ]
-
-  return (
-    <main>
-      <h1>Issues</h1>
-      <ul>
-        {issues.map(issue => {
-          return <li key={issue.id}>{issue.title}</li>
-        })}
-      </ul>
-    </main>
-  )
-}
diff --git a/scenarios/018-agent-uses-filter-pattern/src/main.tsx b/scenarios/018-agent-uses-filter-pattern/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/018-agent-uses-filter-pattern/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/018-agent-uses-filter-pattern/src/styles.css b/scenarios/018-agent-uses-filter-pattern/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/018-agent-uses-filter-pattern/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/018-agent-uses-filter-pattern/tsconfig.json b/scenarios/018-agent-uses-filter-pattern/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/018-agent-uses-filter-pattern/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/index.html b/scenarios/019-agent-uses-dismissal-utilities/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/019-agent-uses-dismissal-utilities/package.json b/scenarios/019-agent-uses-dismissal-utilities/package.json
deleted file mode 100644
index 2b1372dd..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "019-agent-uses-dismissal-utilities",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
deleted file mode 100644
index 2954db45..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent reuses design-system utilities for common dismissal behavior.',
-  prompt: `Update the existing floating panel so it closes when the user clicks outside it or presses Escape. Preserve the current markup and positioning.`,
-  tags: ['hooks', 'interaction', 'utilities', 'vite'],
-})
diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
deleted file mode 100644
index 1e379243..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports the outside-click utility', () => {
-  expect(app).toMatch(/import\s+{[^}]*\buseOnOutsideClick\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/\buseOnOutsideClick\(/)
-})
-
-test('imports the Escape-key utility', () => {
-  expect(app).toMatch(/import\s+{[^}]*\buseOnEscapePress\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/\buseOnEscapePress\(/)
-})
-
-test('does not add global event listeners directly', () => {
-  expect(app).not.toMatch(/(?:window|document)\.addEventListener\(/)
-})
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
deleted file mode 100644
index 9fa93e5a..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx
+++ /dev/null
@@ -1,31 +0,0 @@
-import {useState} from 'react'
-
-export function App() {
-  const [isOpen, setIsOpen] = useState(false)
-
-  return (
-    <main>
-      <button
-        type="button"
-        onClick={() => {
-          setIsOpen(true)
-        }}
-      >
-        Open panel
-      </button>
-      {isOpen ? (
-        <aside className="floating-panel">
-          <h2>Notifications</h2>
-          <button
-            type="button"
-            onClick={() => {
-              setIsOpen(false)
-            }}
-          >
-            Close
-          </button>
-        </aside>
-      ) : null}
-    </main>
-  )
-}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/styles.css b/scenarios/019-agent-uses-dismissal-utilities/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json b/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/index.html b/scenarios/020-agent-uses-resize-observer-utility/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/020-agent-uses-resize-observer-utility/package.json b/scenarios/020-agent-uses-resize-observer-utility/package.json
deleted file mode 100644
index 77ab4a83..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "020-agent-uses-resize-observer-utility",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
deleted file mode 100644
index 37450d53..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent reuses the design-system resize observer utility.',
-  prompt: `Make the contribution chart update its dimensions whenever its container is resized.`,
-  tags: ['hooks', 'responsive', 'utilities', 'vite'],
-})
diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
deleted file mode 100644
index 65f7e04c..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('imports and calls useResizeObserver', () => {
-  expect(app).toMatch(/import\s+{[^}]*\buseResizeObserver\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/\buseResizeObserver\(/)
-})
-
-test('uses observed dimensions for the chart', () => {
-  expect(app).toMatch(/<(?:svg|rect)[^>]*(?:width|height)=\{[^}]+\}/)
-})
-
-test('does not instantiate ResizeObserver directly', () => {
-  expect(app).not.toMatch(/new\s+ResizeObserver\(/)
-})
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
deleted file mode 100644
index 0ff16788..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>Contributions</h1>
-      <section className="chart-container">
-        <svg aria-label="Contribution activity" role="img" height="120" width="720">
-          <rect height="80" width="680" x="20" y="20" />
-        </svg>
-      </section>
-    </main>
-  )
-}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/styles.css b/scenarios/020-agent-uses-resize-observer-utility/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json b/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/index.html b/scenarios/021-agent-sets-up-primer-in-vite/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/package.json b/scenarios/021-agent-sets-up-primer-in-vite/package.json
deleted file mode 100644
index 8081e4bd..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "021-agent-sets-up-primer-in-vite",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
deleted file mode 100644
index e36c9608..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent correctly configures a Vite application to use the design system.',
-  prompt: `Set up this Vite application to use our design system and update the default page to demonstrate that it is working.`,
-  tags: ['setup', 'vite'],
-})
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
deleted file mode 100644
index 302e353c..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8'))
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
-const source = `${app}\n${main}`
-const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies}
-
-test('installs the design-system packages', () => {
-  expect(dependencies).toHaveProperty('@primer/react')
-  expect(dependencies).toHaveProperty('@primer/primitives')
-})
-
-test('loads base primitives and light and dark themes', () => {
-  expect(source).toMatch(/@primer\/primitives\/dist\/css\/primitives\.css/)
-  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/)
-  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/)
-})
-
-test('wraps the application in BaseStyles', () => {
-  expect(source).toMatch(/import\s+{[^}]*\bBaseStyles\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(source).toMatch(/<BaseStyles(?:\s[^>]*)?>[\s\S]*<\/BaseStyles>/)
-})
-
-test('demonstrates a current design-system component', () => {
-  expect(app).toMatch(/import\s+{[^}]*(?:\bButton\b|\bBanner\b|\bCard\b)[^}]*}\s+from\s+['"]@primer\/react['"]/)
-})
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
deleted file mode 100644
index 15f8c284..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx
+++ /dev/null
@@ -1,3 +0,0 @@
-export function App() {
-  return <main>Hello world</main>
-}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css b/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json b/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/022-agent-enables-automatic-theming/index.html b/scenarios/022-agent-enables-automatic-theming/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/022-agent-enables-automatic-theming/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/022-agent-enables-automatic-theming/package.json b/scenarios/022-agent-enables-automatic-theming/package.json
deleted file mode 100644
index fd18ce95..00000000
--- a/scenarios/022-agent-enables-automatic-theming/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "022-agent-enables-automatic-theming",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
deleted file mode 100644
index b5f91a29..00000000
--- a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent configures automatic light and dark theme support.',
-  prompt: `Make the application follow the user's system light or dark appearance setting.`,
-  tags: ['theme', 'theming', 'vite'],
-})
diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.test.ts b/scenarios/022-agent-enables-automatic-theming/scenario.test.ts
deleted file mode 100644
index 7e79fe1e..00000000
--- a/scenarios/022-agent-enables-automatic-theming/scenario.test.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
-const html = await fs.readFile(path.resolve(import.meta.dirname, 'index.html'), 'utf8')
-const source = `${app}\n${main}\n${html}`
-
-test('loads the light and dark functional themes', () => {
-  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/)
-  expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/)
-})
-
-test('uses automatic color mode', () => {
-  expect(source).toMatch(/data-color-mode(?:=|["']\s*,\s*)["']auto["']/)
-})
-
-test('configures light and dark themes', () => {
-  expect(source).toMatch(/data-light-theme(?:=|["']\s*,\s*)["']light["']/)
-  expect(source).toMatch(/data-dark-theme(?:=|["']\s*,\s*)["']dark["']/)
-})
diff --git a/scenarios/022-agent-enables-automatic-theming/src/App.tsx b/scenarios/022-agent-enables-automatic-theming/src/App.tsx
deleted file mode 100644
index 15f8c284..00000000
--- a/scenarios/022-agent-enables-automatic-theming/src/App.tsx
+++ /dev/null
@@ -1,3 +0,0 @@
-export function App() {
-  return <main>Hello world</main>
-}
diff --git a/scenarios/022-agent-enables-automatic-theming/src/main.tsx b/scenarios/022-agent-enables-automatic-theming/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/022-agent-enables-automatic-theming/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/022-agent-enables-automatic-theming/src/styles.css b/scenarios/022-agent-enables-automatic-theming/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/022-agent-enables-automatic-theming/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/022-agent-enables-automatic-theming/tsconfig.json b/scenarios/022-agent-enables-automatic-theming/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/022-agent-enables-automatic-theming/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/023-agent-adds-theme-switcher/index.html b/scenarios/023-agent-adds-theme-switcher/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/023-agent-adds-theme-switcher/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/023-agent-adds-theme-switcher/package.json b/scenarios/023-agent-adds-theme-switcher/package.json
deleted file mode 100644
index f582719b..00000000
--- a/scenarios/023-agent-adds-theme-switcher/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "023-agent-adds-theme-switcher",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
deleted file mode 100644
index 286ecea8..00000000
--- a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent implements a persistent user-controlled theme preference.',
-  prompt: `Add an appearance setting with system, light, and dark choices. Apply the choice immediately and remember it across visits.`,
-  tags: ['interaction', 'theme', 'theming', 'vite'],
-})
diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.test.ts b/scenarios/023-agent-adds-theme-switcher/scenario.test.ts
deleted file mode 100644
index bc663247..00000000
--- a/scenarios/023-agent-adds-theme-switcher/scenario.test.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8')
-const source = `${app}\n${main}`
-
-test.each(['system', 'light', 'dark'])('offers the %s appearance choice', choice => {
-  expect(source).toMatch(new RegExp(`(?:value=["']${choice}["']|>${choice}<)`, 'i'))
-})
-
-test('persists the appearance preference', () => {
-  expect(source).toMatch(/localStorage\.setItem\(/)
-  expect(source).toMatch(/localStorage\.getItem\(/)
-})
-
-test('applies the selected theme using data attributes', () => {
-  expect(source).toMatch(/(?:dataset|setAttribute\()[\s\S]*(?:colorMode|data-color-mode)/)
-})
-
-test('supports the system appearance setting', () => {
-  expect(source).toMatch(/matchMedia\(['"]\(prefers-color-scheme:\s*dark\)['"]\)/)
-})
diff --git a/scenarios/023-agent-adds-theme-switcher/src/App.tsx b/scenarios/023-agent-adds-theme-switcher/src/App.tsx
deleted file mode 100644
index 15f8c284..00000000
--- a/scenarios/023-agent-adds-theme-switcher/src/App.tsx
+++ /dev/null
@@ -1,3 +0,0 @@
-export function App() {
-  return <main>Hello world</main>
-}
diff --git a/scenarios/023-agent-adds-theme-switcher/src/main.tsx b/scenarios/023-agent-adds-theme-switcher/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/023-agent-adds-theme-switcher/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/023-agent-adds-theme-switcher/src/styles.css b/scenarios/023-agent-adds-theme-switcher/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/023-agent-adds-theme-switcher/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/023-agent-adds-theme-switcher/tsconfig.json b/scenarios/023-agent-adds-theme-switcher/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/023-agent-adds-theme-switcher/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/index.html b/scenarios/024-agent-sets-up-tailwindcss/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/024-agent-sets-up-tailwindcss/package.json b/scenarios/024-agent-sets-up-tailwindcss/package.json
deleted file mode 100644
index f7daaaaf..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "024-agent-sets-up-tailwindcss",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
deleted file mode 100644
index a4350ac4..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent configures Tailwind CSS alongside the design system.',
-  prompt: `Add Tailwind CSS to this Vite application and use it to lay out the default page without breaking the existing design-system styles.`,
-  tags: ['setup', 'tailwindcss', 'vite'],
-})
diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
deleted file mode 100644
index 48d1fc52..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts
+++ /dev/null
@@ -1,39 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-async function readOptional(relativePath: string): Promise<string> {
-  try {
-    return await fs.readFile(path.resolve(import.meta.dirname, relativePath), 'utf8')
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return ''
-    }
-
-    throw error
-  }
-}
-
-const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8'))
-const viteConfig = await readOptional('vite.config.ts')
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies}
-
-test('installs Tailwind CSS and its Vite plugin', () => {
-  expect(dependencies).toHaveProperty('tailwindcss')
-  expect(dependencies).toHaveProperty('@tailwindcss/vite')
-})
-
-test('configures the Tailwind Vite plugin', () => {
-  expect(viteConfig).toMatch(/from\s+['"]@tailwindcss\/vite['"]/)
-  expect(viteConfig).toMatch(/\btailwindcss\(\)/)
-})
-
-test('loads Tailwind CSS', () => {
-  expect(styles).toMatch(/@import\s+['"]tailwindcss['"]/)
-})
-
-test('uses utility classes in the default page', () => {
-  expect(app).toMatch(/className=["'][^"']*(?:flex|grid|gap-|p-|m-)[^"']*["']/)
-})
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
deleted file mode 100644
index 15f8c284..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx
+++ /dev/null
@@ -1,3 +0,0 @@
-export function App() {
-  return <main>Hello world</main>
-}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/styles.css b/scenarios/024-agent-sets-up-tailwindcss/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json b/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html b/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
deleted file mode 100644
index 84f90292..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite template</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
deleted file mode 100644
index 0d371a78..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-  "name": "025-agent-uses-tokens-with-tailwindcss",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@tailwindcss/vite": "^4.3.3",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "tailwindcss": "^4.3.3",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
deleted file mode 100644
index 1f1f6c1c..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent uses design tokens when styling with Tailwind CSS.',
-  prompt: `Use Tailwind utility classes to style the deployment status panel while keeping its colors and spacing aligned with the design system.`,
-  tags: ['tailwindcss', 'tokens', 'vite'],
-})
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
deleted file mode 100644
index fd9e8dfd..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8')
-const source = `${app}\n${styles}`
-
-test('uses Tailwind utility classes on the deployment panel', () => {
-  expect(app).toMatch(/className=["'][^"']*(?:bg-|text-|border-|p-|gap-)[^"']*["']/)
-})
-
-test('uses semantic status tokens with Tailwind', () => {
-  expect(source).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/)
-})
-
-test('uses stack tokens for panel spacing', () => {
-  expect(source).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/)
-})
-
-test('does not introduce raw hexadecimal colors', () => {
-  expect(source).not.toMatch(/#[\da-f]{3,8}\b/i)
-})
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
deleted file mode 100644
index 93b598b9..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx
+++ /dev/null
@@ -1,11 +0,0 @@
-export function App() {
-  return (
-    <main>
-      <h1>Deployments</h1>
-      <section className="deployment-status">
-        <h2>Production</h2>
-        <p>Deployment succeeded</p>
-      </section>
-    </main>
-  )
-}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
deleted file mode 100644
index 4bf28b51..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx
+++ /dev/null
@@ -1,16 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App'
-import './styles.css'
-
-const root = document.getElementById('root')
-
-if (!root) {
-  throw new Error('Root element not found')
-}
-
-createRoot(root).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
deleted file mode 100644
index 43cacaf0..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css
+++ /dev/null
@@ -1,9 +0,0 @@
-@import 'tailwindcss';
-
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}
diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
deleted file mode 100644
index 6fb85ea5..00000000
--- a/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import tailwindcss from '@tailwindcss/vite'
-import {defineConfig} from 'vite'
-
-export default defineConfig({
-  plugins: [tailwindcss()],
-})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/index.html b/scenarios/026-agent-avoids-deprecated-notification/index.html
deleted file mode 100644
index 1bd99ac0..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/index.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite + React + TS</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>
diff --git a/scenarios/026-agent-avoids-deprecated-notification/package.json b/scenarios/026-agent-avoids-deprecated-notification/package.json
deleted file mode 100644
index 7d8b7dc0..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/package.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "026-agent-avoids-deprecated-notification",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "type-check": "tsc --noEmit"
-  },
-  "dependencies": {
-    "react": "19.2.8",
-    "react-dom": "19.2.8"
-  },
-  "devDependencies": {
-    "@primer/agent-eval": "workspace:*",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
-    "typescript": "^6",
-    "vite": "^8",
-    "vitest": "^4.1.8"
-  }
-}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
deleted file mode 100644
index f884cb24..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import {defineConfig} from '@primer/agent-eval/scenario'
-
-export default defineConfig({
-  description: 'Evaluate whether the agent avoids deprecated components when adding a page-level warning.',
-  prompt: `Add a dismissible page-level warning above the repository settings when branch protection is disabled. Include a link to enable branch protection.`,
-  tags: ['banner', 'component', 'deprecated', 'vite'],
-})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
deleted file mode 100644
index 99ed9b58..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import {expect, test} from 'vitest'
-
-const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8')
-
-test('uses the current Banner component', () => {
-  expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
-  expect(app).toMatch(/<Banner[^>]*variant=["']warning["'][^>]*>/)
-})
-
-test('makes the warning dismissible', () => {
-  expect(app).toMatch(/<Banner[^>]*\bonDismiss=/)
-})
-
-test('links to branch protection settings', () => {
-  expect(app).toMatch(/<Banner[\s\S]*branch protection[\s\S]*(?:href|Link)[\s\S]*<\/Banner>/i)
-})
-
-test('does not use the deprecated Flash component', () => {
-  expect(app).not.toMatch(/\bFlash\b/)
-})
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
deleted file mode 100644
index 14b146a4..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx
+++ /dev/null
@@ -1,11 +0,0 @@
-export function App() {
-  const isBranchProtectionEnabled = false
-
-  return (
-    <main>
-      <h1>Repository settings</h1>
-      <p>Branch protection is {isBranchProtectionEnabled ? 'enabled' : 'disabled'} for the default branch.</p>
-      <a href="/settings/branches">Branch protection settings</a>
-    </main>
-  )
-}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
deleted file mode 100644
index 83f3281d..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx
+++ /dev/null
@@ -1,10 +0,0 @@
-import {StrictMode} from 'react'
-import {createRoot} from 'react-dom/client'
-import {App} from './App.tsx'
-import './styles.css'
-
-createRoot(document.getElementById('root')!).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-)
diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/styles.css b/scenarios/026-agent-avoids-deprecated-notification/src/styles.css
deleted file mode 100644
index 2142ede2..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/src/styles.css
+++ /dev/null
@@ -1,7 +0,0 @@
-:root {
-  font-family: system-ui, sans-serif;
-}
-
-body {
-  margin: 0;
-}
diff --git a/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json b/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json
deleted file mode 100644
index d7265e99..00000000
--- a/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "useDefineForClassFields": true,
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "module": "ESNext",
-    "skipLibCheck": true,
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "isolatedModules": true,
-    "moduleDetection": "force",
-    "noEmit": true,
-    "jsx": "react-jsx",
-    "strict": true,
-    "types": ["vite/client"]
-  },
-  "include": ["src"]
-}

From 2e98455dad28ec6ecf61897e9c6d1e190d38f836 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 19:47:25 -0500
Subject: [PATCH 36/52] chore: log sandbox commands

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 packages/agent-eval/src/sandbox/system.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/agent-eval/src/sandbox/system.ts b/packages/agent-eval/src/sandbox/system.ts
index f7614198..6db12ba4 100644
--- a/packages/agent-eval/src/sandbox/system.ts
+++ b/packages/agent-eval/src/sandbox/system.ts
@@ -187,6 +187,7 @@ class SystemSandbox implements Sandbox {
   }
 
   async runCommand(command: string, args: Array<string> = [], options?: RunOptions): Promise<CommandResult> {
+    logger.debug('[sandbox] Running command: %s %s', command, args.join(' '))
     return execCommand(this.#docker, this.#container, command, args, {
       env: {
         HOME: options?.user === 'root' ? '/root' : '/home/node',

From b469cc343aff7caf5f8570d4d1ddbf23454e2664 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 19:47:25 -0500
Subject: [PATCH 37/52] chore: align benchmark support lockfile

Remove scenario package entries that are introduced by the final stack layer instead of the benchmark infrastructure branch.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 pnpm-lock.yaml | 606 -------------------------------------------------
 1 file changed, 606 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0105ec21..771973c5 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -374,600 +374,6 @@ importers:
         specifier: ^4.1.11
         version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
 
-  scenarios/006-agent-uses-pagination-component:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/007-agent-infers-billing-banner:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/008-agent-infers-action-menu:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/009-agent-uses-layout-and-color-tokens:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/010-agent-uses-typography-tokens:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/011-agent-uses-motion-tokens:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/012-agent-infers-status-tokens:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/013-agent-infers-compact-control-tokens:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/014-agent-replaces-custom-icons-with-octicons:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/015-agent-infers-copy-icon:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/016-agent-uses-loading-and-empty-state-patterns:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/017-agent-uses-confirmation-pattern:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/018-agent-uses-filter-pattern:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/019-agent-uses-dismissal-utilities:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/020-agent-uses-resize-observer-utility:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/021-agent-sets-up-primer-in-vite:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/022-agent-enables-automatic-theming:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/023-agent-adds-theme-switcher:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/024-agent-sets-up-tailwindcss:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/025-agent-uses-tokens-with-tailwindcss:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@tailwindcss/vite':
-        specifier: ^4.3.3
-        version: 4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      tailwindcss:
-        specifier: ^4.3.3
-        version: 4.3.3
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
-  scenarios/026-agent-avoids-deprecated-notification:
-    dependencies:
-      react:
-        specifier: 19.2.8
-        version: 19.2.8
-      react-dom:
-        specifier: 19.2.8
-        version: 19.2.8(react@19.2.8)
-    devDependencies:
-      '@primer/agent-eval':
-        specifier: workspace:*
-        version: link:../../packages/agent-eval
-      '@types/react':
-        specifier: ^19
-        version: 19.2.18
-      '@types/react-dom':
-        specifier: ^19
-        version: 19.2.5(@types/react@19.2.18)
-      typescript:
-        specifier: ^6
-        version: 6.0.3
-      vite:
-        specifier: ^8
-        version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-      vitest:
-        specifier: ^4.1.8
-        version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))
-
   website:
     dependencies:
       '@primer/agent-eval':
@@ -1891,11 +1297,6 @@ packages:
   '@tailwindcss/postcss@4.3.3':
     resolution: {integrity: sha512-JTSZZGQi1AyKirbLN3azmjVzef92tcX7h+iSqPdaeStyFpGpDlKvvpxeOE8njhbUanbRwr3z8DyzhICWnMtQeg==}
 
-  '@tailwindcss/vite@4.3.3':
-    resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==}
-    peerDependencies:
-      vite: ^5.2.0 || ^6 || ^7 || ^8
-
   '@tanstack/react-virtual@3.14.6':
     resolution: {integrity: sha512-4+Uq8m0/gzO4kMCHUEpTtGX1RnONK0C+g88b2ltwPMWUBiaVarBuWKoPJaz7gj1cKCVRAdyu+U8GcKhwCc2beA==}
     peerDependencies:
@@ -4924,13 +4325,6 @@ snapshots:
       postcss: 8.5.26
       tailwindcss: 4.3.3
 
-  '@tailwindcss/vite@4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))':
-    dependencies:
-      '@tailwindcss/node': 4.3.3
-      '@tailwindcss/oxide': 4.3.3
-      tailwindcss: 4.3.3
-      vite: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)
-
   '@tanstack/react-virtual@3.14.6(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
     dependencies:
       '@tanstack/virtual-core': 3.17.4

From 02edbcd9e5b8f3ae298b84d614d52286bad0a96a Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 19:50:31 -0500
Subject: [PATCH 38/52] fix: accept sub-agent user messages

Allow user.message events without agentMode and preserve the routing fields emitted for sub-agent prompts.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 .changeset/public-socks-smile.md            |  5 +++++
 packages/agent-eval/src/copilot-cli.test.ts | 23 +++++++++++++++++++++
 packages/agent-eval/src/copilot-cli.ts      |  6 +++++-
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/public-socks-smile.md

diff --git a/.changeset/public-socks-smile.md b/.changeset/public-socks-smile.md
new file mode 100644
index 00000000..9f17a6db
--- /dev/null
+++ b/.changeset/public-socks-smile.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Accept Copilot sub-agent user messages that omit agentMode and preserve their routing fields.
diff --git a/packages/agent-eval/src/copilot-cli.test.ts b/packages/agent-eval/src/copilot-cli.test.ts
index 878d33a3..350937c0 100644
--- a/packages/agent-eval/src/copilot-cli.test.ts
+++ b/packages/agent-eval/src/copilot-cli.test.ts
@@ -263,6 +263,29 @@ describe(parseMessage, () => {
     expect(parseMessage(message)).toEqual(message)
   })
 
+  test('parses sub-agent user messages without an agent mode', () => {
+    const message = {
+      type: 'user.message',
+      data: {
+        content: 'Run the build command `npm run build` to see if it succeeds.',
+        transformedContent:
+          '<current_datetime>2026-09-04T00:43:56.551+00:00</current_datetime>\n\nRun the build command `npm run build` to see if it succeeds.',
+        source: 'agent-f13fa250-5bee-4bfc-8356-5b91a7f72f05',
+        supportedNativeDocumentMimeTypes: [],
+        delivery: 'idle',
+        interactionId: '0c57cce9-3187-4397-b4ae-d6131b882626',
+        turnId: '0',
+        parentAgentTaskId: '5bbe4674-f7e7-4ef2-a532-29eaef862529',
+      },
+      agentId: 'f55e8634-e6a7-4b8c-a002-8fa22e9a55cd',
+      id: 'b8d674d7-f16c-455c-83f5-a7366407445a',
+      timestamp: '2026-09-04T00:43:56.551Z',
+      parentId: 'b390bc41-c964-4e13-b3a3-12fb8f2e6303',
+    }
+
+    expect(parseMessage(message)).toMatchObject(message)
+  })
+
   test('does not treat malformed known messages as unrecognized', () => {
     expect(() => parseMessage({type: 'assistant.turn_start', data: {}})).toThrow()
   })
diff --git a/packages/agent-eval/src/copilot-cli.ts b/packages/agent-eval/src/copilot-cli.ts
index b1bb53fd..9bb949e2 100644
--- a/packages/agent-eval/src/copilot-cli.ts
+++ b/packages/agent-eval/src/copilot-cli.ts
@@ -46,6 +46,7 @@ const EventFieldsSchema = {
   id: z.string(),
   timestamp: z.string(),
   parentId: z.string(),
+  agentId: z.optional(z.string()),
 }
 
 const EphemeralEventFieldsSchema = {
@@ -103,8 +104,11 @@ const UserMessageSchema = z.object({
     transformedContent: z.string(),
     attachments: z.optional(z.array(z.unknown())),
     supportedNativeDocumentMimeTypes: z.array(z.string()),
-    agentMode: z.string(),
+    agentMode: z.optional(z.string()),
+    source: z.optional(z.string()),
+    delivery: z.optional(z.string()),
     interactionId: z.string(),
+    turnId: z.optional(z.string()),
     parentAgentTaskId: z.string(),
   }),
 })

From 3bc0e0c09f763ec537867018ad62cb563a9fcf8d Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 19:52:24 -0500
Subject: [PATCH 39/52] fix: remove stale package exports

Remove package export declarations for legacy entry points that are no longer built, allowing publint to validate a clean package build.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 .changeset/calm-benchmarks-compare.md |  2 +-
 packages/agent-eval/package.json      | 12 ------------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/.changeset/calm-benchmarks-compare.md b/.changeset/calm-benchmarks-compare.md
index 4c2c3de5..30a46a04 100644
--- a/.changeset/calm-benchmarks-compare.md
+++ b/.changeset/calm-benchmarks-compare.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add support for benchmarks
+Add support for benchmarks and replace the legacy `experiments`, `scenarios`, and `output` entry points. Import experiment APIs from `@primer/agent-eval/experiment`, scenario APIs from `@primer/agent-eval/scenario`, and their output types and serialization helpers from the corresponding singular entry point.
diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index 2ea4d9d3..636404f6 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -22,14 +22,6 @@
       "types": "./dist/experiment.d.ts",
       "default": "./dist/experiment.js"
     },
-    "./experiments": {
-      "types": "./dist/experiments.d.ts",
-      "default": "./dist/experiments.js"
-    },
-    "./output": {
-      "types": "./dist/output.d.ts",
-      "default": "./dist/output.js"
-    },
     "./scenario": {
       "types": "./dist/scenario.d.ts",
       "default": "./dist/scenario.js"
@@ -37,10 +29,6 @@
     "./sandbox": {
       "types": "./dist/sandbox/index.d.ts",
       "default": "./dist/sandbox/index.js"
-    },
-    "./scenarios": {
-      "types": "./dist/scenarios.d.ts",
-      "default": "./dist/scenarios.js"
     }
   },
   "engines": {

From 8a88259dedb6c9774200777c3a84e3435a89f891 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 20:19:37 -0500
Subject: [PATCH 40/52] fix: exclude walkthrough skill from artifacts

Remove the temporary agent-browser skill after walkthrough capture so downloaded skill artifacts contain only the evaluation configuration.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 .changeset/social-moons-nail.md       |  5 +++
 packages/agent-eval/src/trial.test.ts | 51 +++++++++++++++++++++++++--
 packages/agent-eval/src/trial.ts      |  8 ++++-
 3 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 .changeset/social-moons-nail.md

diff --git a/.changeset/social-moons-nail.md b/.changeset/social-moons-nail.md
new file mode 100644
index 00000000..9da288ad
--- /dev/null
+++ b/.changeset/social-moons-nail.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Exclude the temporary agent-browser walkthrough skill from downloaded trial artifacts.
diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts
index ce2e0c5d..0d06b5ce 100644
--- a/packages/agent-eval/src/trial.test.ts
+++ b/packages/agent-eval/src/trial.test.ts
@@ -1,7 +1,15 @@
 import path from 'node:path'
 import {describe, expect, test, vi} from 'vitest'
-import {VirtualHost} from './host'
-import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type CommandResult, type Sandbox} from './sandbox'
+import {VirtualHost, type Host} from './host'
+import {
+  AGENTS_DIR,
+  CONTAINER_WORKDIR,
+  COPILOT_DIR,
+  NODE_USER,
+  SKILLS_DIR,
+  type CommandResult,
+  type Sandbox,
+} from './sandbox'
 import {run} from './trial'
 import type {Trial} from './trial'
 import type {ResultMessage} from './copilot-cli'
@@ -147,6 +155,21 @@ function writeWalkthroughArtifact(filepath: string, contents = ''): RunCommandMo
   }
 }
 
+function manageAgentBrowserSkill(host: Host): RunCommandMock {
+  return async ({params, sandbox}) => {
+    const [command, args] = params
+    const skillDirectory = path.posix.join(SKILLS_DIR, 'agent-browser')
+
+    if (command === 'npx' && Array.isArray(args) && args[0] === 'skills' && args[1] === 'add') {
+      await sandbox.writeFile(path.posix.join(skillDirectory, 'SKILL.md'), 'agent browser skill')
+    }
+
+    if (command === 'rm' && Array.isArray(args) && args[0] === '-rf' && args[1] === skillDirectory) {
+      await host.fs.rm(skillDirectory, {recursive: true, force: true})
+    }
+  }
+}
+
 describe('run', () => {
   test('copies scenario files into the container workdir', async () => {
     const trial: Trial = {
@@ -784,6 +807,30 @@ describe('run', () => {
       ).resolves.toBe('agent config')
     })
 
+    test('excludes the walkthrough skill from the agent configuration', async () => {
+      const trial = createTrial()
+      const {sandbox, host, ...runOptions} = await setup(trial)
+      const treatmentSkillPath = path.posix.join(SKILLS_DIR, 'treatment-skill', 'SKILL.md')
+      await host.fs.mkdir(path.posix.dirname(treatmentSkillPath), {recursive: true})
+      await host.fs.writeFile(treatmentSkillPath, 'treatment skill')
+      mockRunCommand(sandbox, [manageAgentBrowserSkill(host)])
+
+      const result = await run({
+        ...runOptions,
+        host,
+        sandbox,
+        trial,
+      })
+
+      expect(host.existsSync(path.join(result.artifacts.skillsConfigDirectory, 'skills', 'agent-browser'))).toBe(false)
+      await expect(
+        host.fs.readFile(
+          path.join(result.artifacts.skillsConfigDirectory, 'skills', 'treatment-skill', 'SKILL.md'),
+          'utf8',
+        ),
+      ).resolves.toBe('treatment skill')
+    })
+
     test('downloads the walkthrough', async () => {
       const trial = createTrial()
       const {sandbox, host, ...runOptions} = await setup(trial)
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index 1dd4ed3f..4e454b1b 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -1,7 +1,7 @@
 import path from 'node:path'
 import {isMessageType, MessageSchema, parseMessage, type Message} from './copilot-cli'
 import {DefaultHost, type Host} from './host'
-import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, type Sandbox} from './sandbox'
+import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, SKILLS_DIR, type Sandbox} from './sandbox'
 import {parseTestResults, TestResultsSchema} from './vitest'
 import {logger} from './logger'
 import * as z from 'zod/mini'
@@ -20,6 +20,7 @@ const TrialSchema = z.object({
 type Trial = z.infer<typeof TrialSchema>
 
 const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
+const AGENT_BROWSER_SKILL_DIRECTORY = path.posix.join(SKILLS_DIR, 'agent-browser')
 
 const WalkthroughSchema = z.discriminatedUnion('type', [
   z.object({type: z.literal('Unavailable')}),
@@ -278,6 +279,11 @@ Only capture the walkthrough, do not make any further code changes.`
     logger.warn('%s Unable to capture walkthrough: %s', logPrefix, walkthroughResult.stderr)
   }
 
+  logger.debug('%s Removing walkthrough skill...', logPrefix)
+  await sandbox.runCommand('rm', ['-rf', AGENT_BROWSER_SKILL_DIRECTORY], {
+    user: NODE_USER,
+  })
+
   const artifactDirectory = path.join(artifactsDirectory, trial.id)
   const workspaceDirectory = path.join(artifactDirectory, 'workspace')
   const walkthroughPath = path.join(artifactDirectory, 'walkthrough')

From f6be5ce2e875613ed43d87ecc985a1ac1465ef57 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 20:38:09 -0500
Subject: [PATCH 41/52] fix: collect benchmark output token metrics

Read output token counts from model message events while retaining compatibility with older assistant message output. Compare test success rates across benchmark treatments and report equal values as 0%.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 87b13945-ea60-40ed-b1b1-1c4e9ee24723
---
 .changeset/bright-tokens-report.md          |  5 ++
 packages/agent-eval/src/copilot-cli.test.ts | 16 +++++
 packages/agent-eval/src/copilot-cli.ts      | 13 ++++
 packages/agent-eval/src/report.test.ts      | 57 ++++++++++++++++-
 packages/agent-eval/src/report.ts           | 10 ++-
 packages/agent-eval/src/trial.test.ts       | 70 +++++++++++++++++++++
 packages/agent-eval/src/trial.ts            | 13 +++-
 website/src/benchmark-results.ts            | 16 ++---
 8 files changed, 186 insertions(+), 14 deletions(-)
 create mode 100644 .changeset/bright-tokens-report.md

diff --git a/.changeset/bright-tokens-report.md b/.changeset/bright-tokens-report.md
new file mode 100644
index 00000000..f6a851b2
--- /dev/null
+++ b/.changeset/bright-tokens-report.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Fix benchmark output token collection and compare benchmark test success rates instead of passed-test totals.
diff --git a/packages/agent-eval/src/copilot-cli.test.ts b/packages/agent-eval/src/copilot-cli.test.ts
index 350937c0..0c4f1c94 100644
--- a/packages/agent-eval/src/copilot-cli.test.ts
+++ b/packages/agent-eval/src/copilot-cli.test.ts
@@ -59,6 +59,22 @@ describe(parseMessage, () => {
       timestamp: '2026-07-24T01:57:27.191Z',
       parentId: '080d2918-29d2-4da6-b115-3fe37825b5d2',
     },
+    {
+      type: 'model.message',
+      data: {
+        kind: 'message',
+        turn: 0,
+        message: {
+          role: 'assistant',
+          content: 'Done.',
+          outputTokens: 42,
+        },
+      },
+      ephemeral: true,
+      id: 'dbf7cd98-649a-44c2-b013-680c902773ac',
+      timestamp: '2026-09-04T01:06:13.385Z',
+      parentId: '7e3943af-9aca-4042-b1fd-92a390203597',
+    },
     {
       type: 'user.message',
       data: {
diff --git a/packages/agent-eval/src/copilot-cli.ts b/packages/agent-eval/src/copilot-cli.ts
index 9bb949e2..6f730824 100644
--- a/packages/agent-eval/src/copilot-cli.ts
+++ b/packages/agent-eval/src/copilot-cli.ts
@@ -96,6 +96,17 @@ const ModelCallStartMessageSchema = z.object({
   }),
 })
 
+const ModelMessageSchema = z.looseObject({
+  type: z.literal('model.message'),
+  ...EphemeralEventFieldsSchema,
+  data: z.looseObject({
+    message: z.looseObject({
+      role: z.string(),
+      outputTokens: z.optional(z.number()),
+    }),
+  }),
+})
+
 const UserMessageSchema = z.object({
   type: z.literal('user.message'),
   ...EventFieldsSchema,
@@ -310,6 +321,7 @@ const KnownMessageSchema = z.discriminatedUnion('type', [
   SessionSkillsLoadedMessageSchema,
   SessionToolsUpdatedMessageSchema,
   ModelCallStartMessageSchema,
+  ModelMessageSchema,
   UserMessageSchema,
   AssistantTurnStartMessageSchema,
   AssistantMessageStartMessageSchema,
@@ -336,6 +348,7 @@ const KNOWN_MESSAGE_TYPES = new Set([
   'session.skills_loaded',
   'session.tools_updated',
   'model.call_start',
+  'model.message',
   'user.message',
   'assistant.turn_start',
   'assistant.message_start',
diff --git a/packages/agent-eval/src/report.test.ts b/packages/agent-eval/src/report.test.ts
index 19d85876..c99317e1 100644
--- a/packages/agent-eval/src/report.test.ts
+++ b/packages/agent-eval/src/report.test.ts
@@ -321,6 +321,61 @@ test('formats benchmark results as capability comparisons by scenario', () => {
     Design system  Authoring   All scenarios        All models                             2/4 (+100.0%)  1,500 (-11.8%)  3 (+50.0%)        1m 0.0s (-20.0%)   35.0s (-30.0%)
                                  create-component   All models                             2/4 (+100.0%)  1,500 (-11.8%)  3 (+50.0%)        1m 0.0s (-20.0%)   35.0s (-30.0%)
                                                         gpt-5.6-sol      medium            2/2 (+100.0%)  800 (-20.0%)    2 (+100.0%)       35.0s (-22.2%)     20.0s (-33.3%)
-                                                        claude-sonnet-5  high              0/2 (0.0%)     700 (0.0%)      1 (0.0%)          25.0s (-16.7%)     15.0s (-25.0%)"
+                                                        claude-sonnet-5  high              0/2 (0%)       700 (0%)        1 (0%)            25.0s (-16.7%)     15.0s (-25.0%)"
   `)
 })
+
+test('compares benchmark test success rates when test totals differ', () => {
+  const capability: Capability = {
+    name: 'Authoring',
+    scenarios: [
+      {
+        id: 'create-component',
+        directory: '/scenarios/create-component',
+        prompt: 'Complete the task',
+        tags: [],
+        testPath: '/scenarios/create-component/scenario.test.ts',
+      },
+    ],
+  }
+  const results: Array<BenchmarkTrialResult> = [
+    {
+      ...createResult({
+        treatment: 'Control',
+        scenario: 'create-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 1,
+        numTotalTests: 2,
+        sessions: [],
+      }),
+      capability,
+    },
+    {
+      ...createResult({
+        treatment: 'Benchmark',
+        scenario: 'create-component',
+        model: {
+          name: 'gpt-5.6-sol',
+          reasoningEffort: 'medium',
+        },
+        numPassedTests: 2,
+        numTotalTests: 4,
+        sessions: [],
+      }),
+      capability,
+    },
+  ]
+
+  expect(
+    formatBenchmarkResults(
+      {
+        name: 'Design system',
+        capabilities: [capability],
+      },
+      results,
+    ),
+  ).toContain('2/4 (0%)')
+})
diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts
index cc7be24e..f978d6cf 100644
--- a/packages/agent-eval/src/report.ts
+++ b/packages/agent-eval/src/report.ts
@@ -237,8 +237,12 @@ function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario
 }
 
 function formatPercentDelta(control: number, benchmark: number): string {
+  if (control === benchmark) {
+    return '0%'
+  }
+
   if (control === 0) {
-    return benchmark === 0 ? '0.0%' : 'N/A'
+    return 'N/A'
   }
 
   const delta = (benchmark - control) / control
@@ -370,8 +374,8 @@ function formatBenchmarkComparison(comparison: BenchmarkComparison): TableRow {
     'Reasoning Effort': comparison.reasoningEffort ?? '',
     Tests: formatBenchmarkValue(
       `${comparison.benchmarkTreatment.numPassedTests}/${comparison.benchmarkTreatment.numTotalTests}`,
-      comparison.control.numPassedTests,
-      comparison.benchmarkTreatment.numPassedTests,
+      getSuccessRate(comparison.control),
+      getSuccessRate(comparison.benchmarkTreatment),
     ),
     'Output Tokens': formatBenchmarkValue(
       formatNumber(comparison.benchmarkTreatment.outputTokens),
diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts
index 0d06b5ce..e5383ef6 100644
--- a/packages/agent-eval/src/trial.test.ts
+++ b/packages/agent-eval/src/trial.test.ts
@@ -171,6 +171,76 @@ function manageAgentBrowserSkill(host: Host): RunCommandMock {
 }
 
 describe('run', () => {
+  test('collects output tokens from model messages without double counting assistant messages', async () => {
+    const trial = createTrial()
+    const {sandbox, ...runOptions} = await setup(trial)
+    const writeTokenOutput: RunCommandMock = async ({params}) => {
+      const [command, args] = params
+      if (command !== 'copilot' || !Array.isArray(args) || args[0] !== '--prompt') {
+        return
+      }
+
+      return {
+        stdout: [
+          JSON.stringify({
+            type: 'model.message',
+            data: {
+              message: {
+                role: 'assistant',
+                outputTokens: 42,
+              },
+            },
+            ephemeral: true,
+            id: 'model-message',
+            timestamp: '',
+            parentId: '',
+          }),
+          JSON.stringify({
+            type: 'assistant.message',
+            data: {
+              messageId: 'assistant-message',
+              content: 'Done.',
+              toolRequests: [],
+              interactionId: 'interaction',
+              turnId: 'turn',
+              outputTokens: 42,
+            },
+            id: 'assistant-message',
+            timestamp: '',
+            parentId: '',
+          }),
+          JSON.stringify({
+            type: 'result',
+            timestamp: '',
+            sessionId: '',
+            exitCode: 0,
+            usage: {
+              premiumRequests: 0,
+              totalApiDurationMs: 0,
+              sessionDurationMs: 0,
+              codeChanges: {
+                linesAdded: 0,
+                linesRemoved: 0,
+                filesModified: [],
+              },
+            },
+          }),
+        ].join('\n'),
+        stderr: '',
+        exitCode: 0,
+      }
+    }
+    mockRunCommand(sandbox, [writeTokenOutput])
+
+    const result = await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(result.agent.sessions[0].outputTokens).toBe(42)
+  })
+
   test('copies scenario files into the container workdir', async () => {
     const trial: Trial = {
       id: 'test-id',
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index 4e454b1b..ad4ca274 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -372,7 +372,9 @@ Only capture the walkthrough, do not make any further code changes.`
 function getAgentSession(messages: Array<Message>): AgentSession {
   const turns = new Set()
   const toolCalls = new Map()
-  let outputTokens = 0
+  let assistantOutputTokens = 0
+  let modelOutputTokens = 0
+  let hasModelOutput = false
 
   for (const message of messages) {
     if (isMessageType(message, 'assistant.turn_start')) {
@@ -380,7 +382,12 @@ function getAgentSession(messages: Array<Message>): AgentSession {
     }
 
     if (isMessageType(message, 'assistant.message')) {
-      outputTokens += message.data.outputTokens ?? 0
+      assistantOutputTokens += message.data.outputTokens ?? 0
+    }
+
+    if (isMessageType(message, 'model.message') && message.data.message.role === 'assistant') {
+      hasModelOutput = true
+      modelOutputTokens += message.data.message.outputTokens ?? 0
     }
 
     if (isMessageType(message, 'tool.execution_start')) {
@@ -396,7 +403,7 @@ function getAgentSession(messages: Array<Message>): AgentSession {
 
   return {
     messages,
-    outputTokens,
+    outputTokens: hasModelOutput ? modelOutputTokens : assistantOutputTokens,
     premiumRequests: result.usage.premiumRequests,
     sessionDurationMs: result.usage.sessionDurationMs,
     tools: Object.fromEntries(toolCalls),
diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts
index 3e075f81..ed9687fa 100644
--- a/website/src/benchmark-results.ts
+++ b/website/src/benchmark-results.ts
@@ -197,6 +197,10 @@ function getPercentDeltaValue(control: number, benchmark: number): number | null
 }
 
 function formatPercentDelta(control: number, benchmark: number): string {
+  if (control === benchmark) {
+    return '0%'
+  }
+
   const delta = getPercentDelta(control, benchmark)
   if (delta === null) {
     return 'N/A'
@@ -240,13 +244,11 @@ function createComparison(
       return trial.treatmentId === benchmarkTreatmentId
     }),
   )
+  const controlPassRate = getPassRate(controlTotals) ?? 0
+  const benchmarkPassRate = getPassRate(benchmarkTotals) ?? 0
 
   return {
-    tests: formatValue(
-      `${benchmarkTotals.passed}/${benchmarkTotals.total}`,
-      controlTotals.passed,
-      benchmarkTotals.passed,
-    ),
+    tests: formatValue(`${benchmarkTotals.passed}/${benchmarkTotals.total}`, controlPassRate, benchmarkPassRate),
     outputTokens: formatValue(
       formatNumber(benchmarkTotals.outputTokens),
       controlTotals.outputTokens,
@@ -350,8 +352,8 @@ function createTrendPoint(
         `${benchmarkTotals.passed}/${benchmarkTotals.total}`,
         controlPassRate === null ? null : controlPassRate * 100,
         `${controlTotals.passed}/${controlTotals.total}`,
-        controlTotals.passed,
-        benchmarkTotals.passed,
+        controlPassRate ?? 0,
+        benchmarkPassRate ?? 0,
       ),
       outputTokens: createTrendMetric(
         benchmarkTotals.outputTokens,

From 24dfd15ecdceda46e7332567b1e3304471fc962f Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 21:05:46 -0500
Subject: [PATCH 42/52] fix: restore benchmark overview interactions

Fix chart hydration errors, restore interactive trend tooltips and highlighting, and add capability and scenario filters with metric-aware axes.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705
---
 .../src/app/components/BenchmarkOverview.tsx  |   2 +-
 .../app/components/BenchmarkTrends.module.css |  89 ++-
 .../src/app/components/BenchmarkTrends.tsx    | 584 +++++++++++++++---
 website/src/benchmark-results.ts              |  70 ++-
 4 files changed, 650 insertions(+), 95 deletions(-)

diff --git a/website/src/app/components/BenchmarkOverview.tsx b/website/src/app/components/BenchmarkOverview.tsx
index 42a189c0..fab67598 100644
--- a/website/src/app/components/BenchmarkOverview.tsx
+++ b/website/src/app/components/BenchmarkOverview.tsx
@@ -136,7 +136,7 @@ export function BenchmarkOverview({benchmark, overview}: {benchmark: Benchmark;
   return (
     <Stack padding="normal" gap="spacious">
       <BenchmarkResultsTable benchmark={benchmark} date={overview.date} results={overview.results} />
-      <BenchmarkTrends points={overview.trends} />
+      <BenchmarkTrends capabilities={benchmark.capabilities} points={overview.trends} />
     </Stack>
   )
 }
diff --git a/website/src/app/components/BenchmarkTrends.module.css b/website/src/app/components/BenchmarkTrends.module.css
index 2ab5bb48..993cfdb8 100644
--- a/website/src/app/components/BenchmarkTrends.module.css
+++ b/website/src/app/components/BenchmarkTrends.module.css
@@ -40,6 +40,48 @@
   pointer-events: none;
 }
 
+.controlLine,
+.controlPoint {
+  pointer-events: none;
+}
+
+.lineHitTarget {
+  cursor: crosshair;
+  pointer-events: stroke;
+}
+
+.lineHitTarget:focus-visible {
+  outline: none;
+  stroke: var(--focus-outline-color);
+}
+
+.pointHitTarget {
+  cursor: crosshair;
+  fill: transparent;
+}
+
+.tooltip {
+  pointer-events: none;
+}
+
+.tooltipBackground {
+  fill: var(--bgColor-default);
+  stroke: var(--borderColor-muted);
+  stroke-width: 1;
+  rx: var(--borderRadius-medium);
+}
+
+.tooltipTitle {
+  fill: var(--fgColor-default);
+  font-size: var(--text-body-size-small);
+  font-weight: var(--base-text-weight-semibold);
+}
+
+.tooltipValue {
+  fill: var(--fgColor-muted);
+  font-size: var(--text-body-size-small);
+}
+
 .legend {
   display: flex;
   flex-wrap: wrap;
@@ -49,11 +91,52 @@
   list-style: none;
 }
 
-.legendItem {
+.filters {
   display: flex;
-  align-items: center;
+  flex-wrap: wrap;
+  align-items: start;
+  gap: var(--stack-gap-normal);
+}
+
+.trendsHeader {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: start;
+  justify-content: space-between;
+  gap: var(--stack-gap-normal);
+}
+
+.legendGroups {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: start;
+  justify-content: space-between;
+  gap: var(--stack-gap-normal);
+}
+
+.legendButton {
   gap: var(--stack-gap-condensed);
-  font: var(--text-body-shorthand-small);
+}
+
+.treatmentLegend {
+  display: flex;
+  flex-wrap: wrap;
+  gap: var(--stack-gap-normal);
+  padding: 0;
+  margin: 0;
+  list-style: none;
+}
+
+.seriesGroup,
+.treatmentGroup {
+  transition: opacity var(--motion-transition-hover);
+}
+
+@media (prefers-reduced-motion: reduce) {
+  .seriesGroup,
+  .treatmentGroup {
+    transition: none;
+  }
 }
 
 .metricValue {
diff --git a/website/src/app/components/BenchmarkTrends.tsx b/website/src/app/components/BenchmarkTrends.tsx
index fe3b8eca..a1fa0d41 100644
--- a/website/src/app/components/BenchmarkTrends.tsx
+++ b/website/src/app/components/BenchmarkTrends.tsx
@@ -1,9 +1,10 @@
 'use client'
 
-import {FormControl, Heading, Select, Stack, Text} from '@primer/react'
+import {Button, FormControl, Heading, Select, Stack, Text} from '@primer/react'
 import {DataTable, Table} from '@primer/react/experimental'
-import {useState} from 'react'
+import {type MouseEvent, useState} from 'react'
 import type {BenchmarkTrendMetricId, BenchmarkTrendPoint} from '../../benchmark-results'
+import type {Benchmark} from '../../benchmarks'
 import styles from './BenchmarkTrends.module.css'
 
 const metrics: Array<{
@@ -30,6 +31,8 @@ const numberFormatter = new Intl.NumberFormat('en-US', {
   maximumFractionDigits: 1,
 })
 
+type TrendTreatment = 'benchmark' | 'control'
+
 function getSeriesName(point: BenchmarkTrendPoint): string {
   return point.reasoningEffort ? `${point.model} (${point.reasoningEffort})` : point.model
 }
@@ -42,17 +45,61 @@ function formatChange(change: number | null): string {
   return `${change > 0 ? '+' : ''}${numberFormatter.format(change)}%`
 }
 
+function formatAxisValue(metricId: BenchmarkTrendMetricId, value: number): string {
+  if (metricId === 'tests') {
+    return `${Math.round(value)}%`
+  }
+
+  if (metricId === 'sessionTime' || metricId === 'apiTime') {
+    return `${numberFormatter.format(value)}s`
+  }
+
+  return numberFormatter.format(value)
+}
+
+function getAxisDomain(
+  metricId: BenchmarkTrendMetricId,
+  rawMin: number,
+  rawMax: number,
+): {
+  min: number
+  max: number
+} {
+  const range = rawMax - rawMin
+  const margin = (range || Math.max(rawMax, 1)) * 0.1
+
+  if (metricId !== 'tests') {
+    return {
+      min: Math.max(0, rawMin - margin),
+      max: rawMax + margin,
+    }
+  }
+
+  return {
+    min: 0,
+    max: 100,
+  }
+}
+
 function TrendChart({
   dates,
   metric,
   points,
   series,
+  highlightedSeries,
+  highlightedTreatment,
 }: {
   dates: Array<string>
   metric: (typeof metrics)[number]
   points: Array<BenchmarkTrendPoint>
   series: Array<string>
+  highlightedSeries: string | null
+  highlightedTreatment: TrendTreatment | null
 }) {
+  const [activeTrend, setActiveTrend] = useState<{
+    pointId: string
+    treatment: TrendTreatment
+  } | null>(null)
   const width = 520
   const height = 240
   const padding = {top: 16, right: 16, bottom: 52, left: 48}
@@ -66,16 +113,52 @@ function TrendChart({
   })
   const rawMin = values.length > 0 ? Math.min(...values) : 0
   const rawMax = values.length > 0 ? Math.max(...values) : 0
-  const range = rawMax - rawMin
-  const margin = (range || Math.max(rawMax, 1)) * 0.1
-  const min = Math.max(0, rawMin - margin)
-  const max = rawMax + margin
+  const {min, max} = getAxisDomain(metric.id, rawMin, rawMax)
+  const axisValues = metric.id === 'tests' ? [max, Math.round((max + min) / 2), min] : [max, (max + min) / 2, min]
   const getX = (date: string): number => {
     return padding.left + (dates.length === 1 ? plotWidth / 2 : (dates.indexOf(date) / (dates.length - 1)) * plotWidth)
   }
   const getY = (value: number): number => {
     return padding.top + (1 - (value - min) / (max - min)) * plotHeight
   }
+  const activePoint = points.find(point => {
+    return point.id === activeTrend?.pointId
+  })
+  const activeMetric = activePoint?.metrics[metric.id]
+  const activeValue =
+    activeTrend?.treatment === 'control'
+      ? (activeMetric?.controlValue ?? undefined)
+      : (activeMetric?.value ?? undefined)
+  const activeRaw = activeTrend?.treatment === 'control' ? activeMetric?.controlRaw : activeMetric?.raw
+  const tooltipWidth = 248
+  const tooltipHeight = 52
+  const tooltipX = activePoint
+    ? Math.min(Math.max(getX(activePoint.date) - tooltipWidth / 2, padding.left), width - padding.right - tooltipWidth)
+    : 0
+  const tooltipY =
+    activePoint && activeValue !== undefined
+      ? Math.max(padding.top, Math.min(getY(activeValue) - tooltipHeight - 10, height - padding.bottom - tooltipHeight))
+      : 0
+  const tooltipId = `${metric.id}-chart-tooltip`
+  const controlOpacity = highlightedTreatment === 'benchmark' ? 0.15 : highlightedTreatment === 'control' ? 1 : 0.4
+  const benchmarkOpacity = highlightedTreatment === 'control' ? 0.15 : 1
+  const activateClosestPoint = (
+    event: MouseEvent<SVGPathElement>,
+    seriesPoints: Array<BenchmarkTrendPoint>,
+    treatment: TrendTreatment,
+  ) => {
+    const svg = event.currentTarget.ownerSVGElement
+    if (!svg || seriesPoints.length === 0) {
+      return
+    }
+
+    const bounds = svg.getBoundingClientRect()
+    const pointerX = ((event.clientX - bounds.left) / bounds.width) * width
+    const closestPoint = seriesPoints.reduce((closest, point) => {
+      return Math.abs(getX(point.date) - pointerX) < Math.abs(getX(closest.date) - pointerX) ? point : closest
+    })
+    setActiveTrend({pointId: closestPoint.id, treatment})
+  }
 
   return (
     <article className={styles.chart}>
@@ -88,13 +171,13 @@ function TrendChart({
         role="img"
         viewBox={`0 0 ${width} ${height}`}
       >
-        {[max, (max + min) / 2, min].map(value => {
+        {axisValues.map(value => {
           const y = getY(value)
           return (
             <g key={value}>
               <line className={styles.gridLine} x1={padding.left} x2={width - padding.right} y1={y} y2={y} />
               <text className={styles.axisLabel} textAnchor="end" x={padding.left - 8} y={y + 4}>
-                {numberFormatter.format(value)}
+                {formatAxisValue(metric.id, value)}
               </text>
             </g>
           )
@@ -117,75 +200,237 @@ function TrendChart({
             .toSorted((a, b) => {
               return a.date.localeCompare(b.date)
             })
+          const controlPoints = seriesPoints.filter(point => {
+            return point.metrics[metric.id].controlValue !== null
+          })
+          const benchmarkPoints = seriesPoints.filter(point => {
+            return point.metrics[metric.id].value !== null
+          })
           const style = lineStyles[index % lineStyles.length]
+          const controlPath = controlPoints
+            .map((point, pointIndex) => {
+              return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(
+                point.metrics[metric.id].controlValue ?? 0,
+              )}`
+            })
+            .join(' ')
+          const benchmarkPath = benchmarkPoints
+            .map((point, pointIndex) => {
+              return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(point.metrics[metric.id].value ?? 0)}`
+            })
+            .join(' ')
 
           return (
-            <g key={seriesName}>
-              {(['controlValue', 'value'] as const).map((valueKey, treatmentIndex) => {
-                const treatmentPoints = seriesPoints.filter(point => {
-                  return point.metrics[metric.id][valueKey] !== null
-                })
-                const path = treatmentPoints
-                  .map((point, pointIndex) => {
-                    const value = point.metrics[metric.id][valueKey] ?? 0
-                    return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(value)}`
-                  })
-                  .join(' ')
-
+            <g
+              className={styles.seriesGroup}
+              key={seriesName}
+              opacity={highlightedSeries && highlightedSeries !== seriesName ? 0.15 : 1}
+            >
+              {controlPoints.length > 0 ? (
+                <>
+                  <path
+                    className={`${styles.controlLine} ${styles.treatmentGroup}`}
+                    d={controlPath}
+                    fill="none"
+                    opacity={controlOpacity}
+                    stroke={style.color}
+                    strokeDasharray={style.dash}
+                    strokeLinecap="round"
+                    strokeLinejoin="round"
+                    strokeWidth="1.5"
+                  />
+                  <path
+                    aria-describedby={
+                      activePoint && getSeriesName(activePoint) === seriesName && activeTrend?.treatment === 'control'
+                        ? tooltipId
+                        : undefined
+                    }
+                    aria-label={`${seriesName}, Control ${metric.label} trend`}
+                    className={styles.lineHitTarget}
+                    d={controlPath}
+                    fill="none"
+                    onBlur={() => {
+                      setActiveTrend(null)
+                    }}
+                    onFocus={() => {
+                      const pointId = controlPoints.at(-1)?.id
+                      setActiveTrend(pointId ? {pointId, treatment: 'control'} : null)
+                    }}
+                    onMouseLeave={() => {
+                      setActiveTrend(null)
+                    }}
+                    onMouseMove={event => {
+                      activateClosestPoint(event, controlPoints, 'control')
+                    }}
+                    stroke="transparent"
+                    strokeWidth="16"
+                    tabIndex={0}
+                  />
+                </>
+              ) : null}
+              {benchmarkPoints.length > 0 ? (
+                <>
+                  <path
+                    className={`${styles.seriesLine} ${styles.treatmentGroup}`}
+                    d={benchmarkPath}
+                    fill="none"
+                    opacity={benchmarkOpacity}
+                    stroke={style.color}
+                    strokeDasharray={style.dash}
+                    strokeLinecap="round"
+                    strokeLinejoin="round"
+                    strokeWidth="2"
+                  />
+                  <path
+                    aria-describedby={
+                      activePoint && getSeriesName(activePoint) === seriesName && activeTrend?.treatment === 'benchmark'
+                        ? tooltipId
+                        : undefined
+                    }
+                    aria-label={`${seriesName}, Benchmark ${metric.label} trend`}
+                    className={styles.lineHitTarget}
+                    d={benchmarkPath}
+                    fill="none"
+                    onBlur={() => {
+                      setActiveTrend(null)
+                    }}
+                    onFocus={() => {
+                      const pointId = benchmarkPoints.at(-1)?.id
+                      setActiveTrend(pointId ? {pointId, treatment: 'benchmark'} : null)
+                    }}
+                    onMouseLeave={() => {
+                      setActiveTrend(null)
+                    }}
+                    onMouseMove={event => {
+                      activateClosestPoint(event, benchmarkPoints, 'benchmark')
+                    }}
+                    stroke="transparent"
+                    strokeWidth="16"
+                    tabIndex={0}
+                  />
+                </>
+              ) : null}
+              {seriesPoints.map(point => {
+                const value = point.metrics[metric.id]
                 return (
-                  <g key={valueKey} opacity={treatmentIndex === 0 ? 0.4 : 1}>
-                    <path
-                      className={styles.seriesLine}
-                      d={path}
-                      fill="none"
-                      stroke={style.color}
-                      strokeDasharray={style.dash}
-                      strokeLinecap="round"
-                      strokeLinejoin="round"
-                      strokeWidth={treatmentIndex === 0 ? '1.5' : '2'}
-                    />
-                    {treatmentPoints.map(point => {
-                      const value = point.metrics[metric.id][valueKey] ?? 0
-                      return (
-                        <circle
-                          cx={getX(point.date)}
-                          cy={getY(value)}
-                          fill="var(--bgColor-default)"
-                          key={point.id}
-                          r={treatmentIndex === 0 ? '3' : '3.5'}
-                          stroke={style.color}
-                          strokeWidth={treatmentIndex === 0 ? '1.5' : '2'}
-                        >
-                          <title>
-                            {seriesName}, {treatmentIndex === 0 ? 'Control' : 'Benchmark'}, {point.date}:{' '}
-                            {point.metrics[metric.id][treatmentIndex === 0 ? 'controlRaw' : 'raw']}
-                          </title>
-                        </circle>
-                      )
-                    })}
+                  <g aria-hidden="true" key={point.id}>
+                    {value.controlValue !== null ? (
+                      <circle
+                        className={`${styles.controlPoint} ${styles.treatmentGroup}`}
+                        cx={getX(point.date)}
+                        cy={getY(value.controlValue)}
+                        fill="var(--bgColor-default)"
+                        opacity={controlOpacity}
+                        r={activeTrend?.pointId === point.id && activeTrend.treatment === 'control' ? '4.5' : '3'}
+                        stroke={style.color}
+                        strokeWidth="1.5"
+                      />
+                    ) : null}
+                    {value.value !== null ? (
+                      <circle
+                        className={styles.treatmentGroup}
+                        cx={getX(point.date)}
+                        cy={getY(value.value)}
+                        fill="var(--bgColor-default)"
+                        opacity={benchmarkOpacity}
+                        r={activeTrend?.pointId === point.id && activeTrend.treatment === 'benchmark' ? '5' : '3.5'}
+                        stroke={style.color}
+                        strokeWidth="2"
+                      />
+                    ) : null}
+                    {value.controlValue !== null ? (
+                      <circle
+                        className={styles.pointHitTarget}
+                        cx={getX(point.date)}
+                        cy={getY(value.controlValue)}
+                        onMouseEnter={() => {
+                          setActiveTrend({pointId: point.id, treatment: 'control'})
+                        }}
+                        onMouseLeave={() => {
+                          setActiveTrend(null)
+                        }}
+                        r="10"
+                      />
+                    ) : null}
+                    {value.value !== null ? (
+                      <circle
+                        className={styles.pointHitTarget}
+                        cx={getX(point.date)}
+                        cy={getY(value.value)}
+                        onMouseEnter={() => {
+                          setActiveTrend({pointId: point.id, treatment: 'benchmark'})
+                        }}
+                        onMouseLeave={() => {
+                          setActiveTrend(null)
+                        }}
+                        r="10"
+                      />
+                    ) : null}
                   </g>
                 )
               })}
             </g>
           )
         })}
+        {activePoint && activeMetric && activeRaw ? (
+          <g className={styles.tooltip} id={tooltipId} role="tooltip" transform={`translate(${tooltipX} ${tooltipY})`}>
+            <rect className={styles.tooltipBackground} height={tooltipHeight} width={tooltipWidth} />
+            <text className={styles.tooltipTitle} x="10" y="20">
+              {`${getSeriesName(activePoint)} - ${activeTrend?.treatment === 'control' ? 'Control' : 'Benchmark'}`}
+            </text>
+            <text className={styles.tooltipValue} x="10" y="40">
+              {`${activePoint.date}: ${activeRaw}${
+                activeTrend?.treatment === 'benchmark' ? ` (${formatChange(activeMetric.change)})` : ''
+              }`}
+            </text>
+          </g>
+        ) : null}
       </svg>
     </article>
   )
 }
 
-export function BenchmarkTrends({points}: {points: Array<BenchmarkTrendPoint>}) {
+export function BenchmarkTrends({
+  capabilities,
+  points,
+}: {
+  capabilities: Benchmark['capabilities']
+  points: Array<BenchmarkTrendPoint>
+}) {
   const [tableMetricId, setTableMetricId] = useState<BenchmarkTrendMetricId>('tests')
+  const [selectedCapabilityId, setSelectedCapabilityId] = useState('')
+  const [selectedScenarioId, setSelectedScenarioId] = useState('')
+  const [selectedSeries, setSelectedSeries] = useState<string | null>(null)
+  const [hoveredSeries, setHoveredSeries] = useState<string | null>(null)
+  const [selectedTreatment, setSelectedTreatment] = useState<TrendTreatment | null>(null)
+  const [hoveredTreatment, setHoveredTreatment] = useState<TrendTreatment | null>(null)
+  const scenarios = [
+    ...new Set(
+      (selectedCapabilityId
+        ? (capabilities.find(capability => {
+            return capability.name === selectedCapabilityId
+          })?.scenarios ?? [])
+        : capabilities.flatMap(capability => {
+            return capability.scenarios
+          })
+      ).map(scenario => {
+        return scenario.id
+      }),
+    ),
+  ].toSorted()
+  const filteredPoints = points.filter(point => {
+    return point.capabilityId === (selectedCapabilityId || null) && point.scenarioId === (selectedScenarioId || null)
+  })
   const dates = [
     ...new Set(
-      points.map(point => {
+      filteredPoints.map(point => {
         return point.date
       }),
     ),
   ].toSorted()
   const series = [
     ...new Set(
-      points.map(point => {
+      filteredPoints.map(point => {
         return getSeriesName(point)
       }),
     ),
@@ -194,13 +439,15 @@ export function BenchmarkTrends({points}: {points: Array<BenchmarkTrendPoint>})
     metrics.find(metric => {
       return metric.id === tableMetricId
     }) ?? metrics[0]
+  const highlightedSeries = hoveredSeries ?? selectedSeries
+  const highlightedTreatment = hoveredTreatment ?? selectedTreatment
   const comparisonRows = series.map(seriesName => {
     return {
       id: seriesName,
       model: seriesName,
       values: Object.fromEntries(
         dates.map(date => {
-          const point = points.find(candidate => {
+          const point = filteredPoints.find(candidate => {
             return getSeriesName(candidate) === seriesName && candidate.date === date
           })
           return [date, point?.metrics[tableMetric.id]]
@@ -208,7 +455,7 @@ export function BenchmarkTrends({points}: {points: Array<BenchmarkTrendPoint>})
       ),
     }
   })
-  const rawRows = points
+  const rawRows = filteredPoints
     .toSorted((a, b) => {
       return a.date.localeCompare(b.date) || getSeriesName(a).localeCompare(getSeriesName(b))
     })
@@ -223,40 +470,201 @@ export function BenchmarkTrends({points}: {points: Array<BenchmarkTrendPoint>})
   return (
     <section aria-labelledby="benchmark-trends-heading">
       <Stack gap="spacious">
-        <Stack gap="condensed">
-          <Heading as="h2" id="benchmark-trends-heading">
-            Trends
-          </Heading>
-          <Text as="p" className={styles.description}>
-            Strong lines show Benchmark results and muted lines show Control over time.
-          </Text>
-        </Stack>
-        <ul className={styles.legend} aria-label="Model chart legend">
-          {series.map((seriesName, index) => {
-            const style = lineStyles[index % lineStyles.length]
-            return (
-              <li className={styles.legendItem} key={seriesName}>
-                <svg aria-hidden="true" height="8" width="28">
-                  <line
-                    stroke={style.color}
-                    strokeDasharray={style.dash}
-                    strokeWidth="2"
-                    x1="0"
-                    x2="28"
-                    y1="4"
-                    y2="4"
-                  />
-                </svg>
-                {seriesName}
-              </li>
-            )
-          })}
-        </ul>
-        <div className={styles.chartGrid}>
-          {metrics.map(metric => {
-            return <TrendChart dates={dates} key={metric.id} metric={metric} points={points} series={series} />
-          })}
+        <div className={styles.trendsHeader}>
+          <Stack gap="condensed">
+            <Heading as="h2" id="benchmark-trends-heading">
+              Trends
+            </Heading>
+            <Text as="p" className={styles.description}>
+              Strong lines show Benchmark results and muted lines show Control over time.
+            </Text>
+          </Stack>
+          <div className={styles.filters}>
+            <FormControl>
+              <FormControl.Label>Capability</FormControl.Label>
+              <Select
+                onChange={event => {
+                  const capabilityId = event.currentTarget.value
+                  const capabilityScenarios =
+                    capabilities.find(capability => {
+                      return capability.name === capabilityId
+                    })?.scenarios ?? []
+                  setSelectedCapabilityId(capabilityId)
+                  if (
+                    selectedScenarioId &&
+                    !capabilityScenarios.some(scenario => {
+                      return scenario.id === selectedScenarioId
+                    })
+                  ) {
+                    setSelectedScenarioId('')
+                  }
+                }}
+                value={selectedCapabilityId}
+              >
+                <Select.Option value="">All capabilities</Select.Option>
+                {capabilities.map(capability => {
+                  return (
+                    <Select.Option key={capability.name} value={capability.name}>
+                      {capability.name}
+                    </Select.Option>
+                  )
+                })}
+              </Select>
+            </FormControl>
+            <FormControl>
+              <FormControl.Label>Scenario</FormControl.Label>
+              <Select
+                onChange={event => {
+                  setSelectedScenarioId(event.currentTarget.value)
+                }}
+                value={selectedScenarioId}
+              >
+                <Select.Option value="">All scenarios</Select.Option>
+                {scenarios.map(scenarioId => {
+                  return (
+                    <Select.Option key={scenarioId} value={scenarioId}>
+                      {scenarioId}
+                    </Select.Option>
+                  )
+                })}
+              </Select>
+            </FormControl>
+          </div>
         </div>
+        <Stack gap="normal">
+          <div className={styles.legendGroups}>
+            <ul className={styles.legend} aria-label="Model chart legend">
+              {series.map((seriesName, index) => {
+                const style = lineStyles[index % lineStyles.length]
+                return (
+                  <li key={seriesName}>
+                    <Button
+                      aria-pressed={selectedSeries === seriesName}
+                      className={styles.legendButton}
+                      onBlur={() => {
+                        setHoveredSeries(null)
+                      }}
+                      onClick={() => {
+                        setSelectedSeries(current => {
+                          return current === seriesName ? null : seriesName
+                        })
+                      }}
+                      onFocus={() => {
+                        setHoveredSeries(seriesName)
+                      }}
+                      onMouseEnter={() => {
+                        setHoveredSeries(seriesName)
+                      }}
+                      onMouseLeave={() => {
+                        setHoveredSeries(null)
+                      }}
+                      size="small"
+                      variant="invisible"
+                    >
+                      <svg aria-hidden="true" height="8" width="28">
+                        <line
+                          stroke={style.color}
+                          strokeDasharray={style.dash}
+                          strokeWidth="2"
+                          x1="0"
+                          x2="28"
+                          y1="4"
+                          y2="4"
+                        />
+                      </svg>
+                      {seriesName}
+                    </Button>
+                  </li>
+                )
+              })}
+            </ul>
+            <ul className={styles.treatmentLegend} aria-label="Treatment chart legend">
+              <li>
+                <Button
+                  aria-pressed={selectedTreatment === 'benchmark'}
+                  className={styles.legendButton}
+                  onBlur={() => {
+                    setHoveredTreatment(null)
+                  }}
+                  onClick={() => {
+                    setSelectedTreatment(current => {
+                      return current === 'benchmark' ? null : 'benchmark'
+                    })
+                  }}
+                  onFocus={() => {
+                    setHoveredTreatment('benchmark')
+                  }}
+                  onMouseEnter={() => {
+                    setHoveredTreatment('benchmark')
+                  }}
+                  onMouseLeave={() => {
+                    setHoveredTreatment(null)
+                  }}
+                  size="small"
+                  variant="invisible"
+                >
+                  <svg aria-hidden="true" height="8" width="28">
+                    <line stroke="var(--fgColor-default)" strokeWidth="2" x1="0" x2="28" y1="4" y2="4" />
+                  </svg>
+                  Benchmark
+                </Button>
+              </li>
+              <li>
+                <Button
+                  aria-pressed={selectedTreatment === 'control'}
+                  className={styles.legendButton}
+                  onBlur={() => {
+                    setHoveredTreatment(null)
+                  }}
+                  onClick={() => {
+                    setSelectedTreatment(current => {
+                      return current === 'control' ? null : 'control'
+                    })
+                  }}
+                  onFocus={() => {
+                    setHoveredTreatment('control')
+                  }}
+                  onMouseEnter={() => {
+                    setHoveredTreatment('control')
+                  }}
+                  onMouseLeave={() => {
+                    setHoveredTreatment(null)
+                  }}
+                  size="small"
+                  variant="invisible"
+                >
+                  <svg aria-hidden="true" height="8" width="28">
+                    <line
+                      opacity="0.4"
+                      stroke="var(--fgColor-default)"
+                      strokeWidth="1.5"
+                      x1="0"
+                      x2="28"
+                      y1="4"
+                      y2="4"
+                    />
+                  </svg>
+                  Control
+                </Button>
+              </li>
+            </ul>
+          </div>
+          <div className={styles.chartGrid}>
+            {metrics.map(metric => {
+              return (
+                <TrendChart
+                  dates={dates}
+                  highlightedSeries={highlightedSeries}
+                  highlightedTreatment={highlightedTreatment}
+                  key={metric.id}
+                  metric={metric}
+                  points={filteredPoints}
+                  series={series}
+                />
+              )
+            })}
+          </div>
+        </Stack>
         <Table.Container>
           <Table.Title as="h3" id="benchmark-change-heading">
             Change from Control
diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts
index ed9687fa..2dcc4dae 100644
--- a/website/src/benchmark-results.ts
+++ b/website/src/benchmark-results.ts
@@ -71,6 +71,8 @@ export type BenchmarkTrendMetric = {
 export type BenchmarkTrendPoint = {
   id: string
   date: string
+  capabilityId: string | null
+  scenarioId: string | null
   model: string
   reasoningEffort: string
   metrics: Record<BenchmarkTrendMetricId, BenchmarkTrendMetric>
@@ -326,6 +328,8 @@ function createTrendPoint(
   trials: Array<BenchmarkOutputTrial>,
   controlTreatmentId: string,
   benchmarkTreatmentId: string,
+  capabilityId: string | null,
+  scenarioId: string | null,
 ): BenchmarkTrendPoint {
   const trial = trials[0]
   const controlTotals = getTotals(
@@ -342,8 +346,10 @@ function createTrendPoint(
   const benchmarkPassRate = getPassRate(benchmarkTotals)
 
   return {
-    id: `${date}:${trial.model.name}:${trial.model.reasoningEffort}`,
+    id: `${date}:${capabilityId ?? 'all'}:${scenarioId ?? 'all'}:${trial.model.name}:${trial.model.reasoningEffort}`,
     date,
+    capabilityId,
+    scenarioId,
     model: trial.model.name,
     reasoningEffort: trial.model.reasoningEffort,
     metrics: {
@@ -391,6 +397,19 @@ function createTrendPoint(
   }
 }
 
+function createTrendPoints(
+  date: string,
+  trials: Array<BenchmarkOutputTrial>,
+  controlTreatmentId: string,
+  benchmarkTreatmentId: string,
+  capabilityId: string | null,
+  scenarioId: string | null,
+): Array<BenchmarkTrendPoint> {
+  return groupTrialsByModel(trials).map(modelTrials => {
+    return createTrendPoint(date, modelTrials, controlTreatmentId, benchmarkTreatmentId, capabilityId, scenarioId)
+  })
+}
+
 function compareModelPerformance(
   a: Array<BenchmarkOutputTrial>,
   b: Array<BenchmarkOutputTrial>,
@@ -479,9 +498,54 @@ export function getBenchmarkOverviewData(runs: Array<BenchmarkRun>): BenchmarkOv
     : []
   const trends = runs.flatMap(run => {
     const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(run.output)
-    return groupTrialsByModel([...run.output.trials.values()]).map(trials => {
-      return createTrendPoint(run.name, trials, controlTreatmentId, benchmarkTreatmentId)
+    const trials = [...run.output.trials.values()]
+    const scenarioIds = new Set(
+      trials.map(trial => {
+        return trial.scenarioId
+      }),
+    )
+    const allTrendPoints = createTrendPoints(run.name, trials, controlTreatmentId, benchmarkTreatmentId, null, null)
+    const scenarioTrendPoints = [...scenarioIds].flatMap(scenarioId => {
+      return createTrendPoints(
+        run.name,
+        trials.filter(trial => {
+          return trial.scenarioId === scenarioId
+        }),
+        controlTreatmentId,
+        benchmarkTreatmentId,
+        null,
+        scenarioId,
+      )
     })
+    const capabilityTrendPoints = [...run.output.capabilities.values()].flatMap(capability => {
+      const capabilityTrials = trials.filter(trial => {
+        return trial.capabilityId === capability.name
+      })
+      const capabilityPoints = createTrendPoints(
+        run.name,
+        capabilityTrials,
+        controlTreatmentId,
+        benchmarkTreatmentId,
+        capability.name,
+        null,
+      )
+      const capabilityScenarioPoints = capability.scenarioIds.flatMap(scenarioId => {
+        return createTrendPoints(
+          run.name,
+          capabilityTrials.filter(trial => {
+            return trial.scenarioId === scenarioId
+          }),
+          controlTreatmentId,
+          benchmarkTreatmentId,
+          capability.name,
+          scenarioId,
+        )
+      })
+
+      return [...capabilityPoints, ...capabilityScenarioPoints]
+    })
+
+    return [...allTrendPoints, ...scenarioTrendPoints, ...capabilityTrendPoints]
   })
 
   return {

From 3897fc6273eac35068e68abc78f88dafe618e66a Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 21:42:23 -0500
Subject: [PATCH 43/52] fix: preserve scenario path and browser tests

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .changeset/bright-benchmarks-run.md        |   9 ++
 .changeset/bright-tokens-report.md         |   5 -
 .changeset/calm-benchmarks-compare.md      |   5 -
 .changeset/calm-experiments-unify.md       |  11 ++
 .changeset/fresh-experiment-configs.md     |   7 ++
 .changeset/gentle-scenarios-unify.md       |  15 +++
 .changeset/happy-output-maps.md            |  12 ++
 .changeset/kind-trials-compare.md          |   9 ++
 .changeset/lucky-sandboxes-open.md         |   7 ++
 .changeset/neat-models-vary.md             |   7 ++
 .changeset/portable-results-bundle.md      |   7 ++
 .changeset/public-socks-smile.md           |   5 -
 .changeset/quick-copilot-metrics.md        |   5 +
 .changeset/sixty-ways-taste.md             |   5 -
 .changeset/social-moons-nail.md            |   5 -
 .changeset/steady-trial-results.md         |   5 +
 packages/agent-eval/src/experiment.test.ts |  68 +++++++++++
 packages/agent-eval/src/experiment.ts      |  34 +++++-
 packages/agent-eval/src/scenario.ts        |  76 +++++++-----
 packages/agent-eval/src/trial.test.ts      | 104 ++++++++++++++++-
 packages/agent-eval/src/trial.ts           | 127 ++++++++++++++++++---
 21 files changed, 451 insertions(+), 77 deletions(-)
 create mode 100644 .changeset/bright-benchmarks-run.md
 delete mode 100644 .changeset/bright-tokens-report.md
 delete mode 100644 .changeset/calm-benchmarks-compare.md
 create mode 100644 .changeset/calm-experiments-unify.md
 create mode 100644 .changeset/fresh-experiment-configs.md
 create mode 100644 .changeset/gentle-scenarios-unify.md
 create mode 100644 .changeset/happy-output-maps.md
 create mode 100644 .changeset/kind-trials-compare.md
 create mode 100644 .changeset/lucky-sandboxes-open.md
 create mode 100644 .changeset/neat-models-vary.md
 create mode 100644 .changeset/portable-results-bundle.md
 delete mode 100644 .changeset/public-socks-smile.md
 create mode 100644 .changeset/quick-copilot-metrics.md
 delete mode 100644 .changeset/sixty-ways-taste.md
 delete mode 100644 .changeset/social-moons-nail.md
 create mode 100644 .changeset/steady-trial-results.md

diff --git a/.changeset/bright-benchmarks-run.md b/.changeset/bright-benchmarks-run.md
new file mode 100644
index 00000000..e77f0a03
--- /dev/null
+++ b/.changeset/bright-benchmarks-run.md
@@ -0,0 +1,9 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add benchmark configuration, discovery, execution, and output APIs through the new `@primer/agent-eval/benchmark` entry point.
+
+The package root also exports benchmark APIs with explicit names, including `defineBenchmarkConfig`, `getBenchmark`, `listBenchmarks`, `runBenchmark`, `getBenchmarkOutput`, `serializeBenchmarkOutput`, and `deserializeBenchmarkOutput`, along with benchmark configuration, result, capability, and output types.
+
+The CLI can now select and run benchmarks from a benchmarks directory.
diff --git a/.changeset/bright-tokens-report.md b/.changeset/bright-tokens-report.md
deleted file mode 100644
index f6a851b2..00000000
--- a/.changeset/bright-tokens-report.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@primer/agent-eval': patch
----
-
-Fix benchmark output token collection and compare benchmark test success rates instead of passed-test totals.
diff --git a/.changeset/calm-benchmarks-compare.md b/.changeset/calm-benchmarks-compare.md
deleted file mode 100644
index 30a46a04..00000000
--- a/.changeset/calm-benchmarks-compare.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@primer/agent-eval': minor
----
-
-Add support for benchmarks and replace the legacy `experiments`, `scenarios`, and `output` entry points. Import experiment APIs from `@primer/agent-eval/experiment`, scenario APIs from `@primer/agent-eval/scenario`, and their output types and serialization helpers from the corresponding singular entry point.
diff --git a/.changeset/calm-experiments-unify.md b/.changeset/calm-experiments-unify.md
new file mode 100644
index 00000000..36fc7c30
--- /dev/null
+++ b/.changeset/calm-experiments-unify.md
@@ -0,0 +1,11 @@
+---
+'@primer/agent-eval': minor
+---
+
+Consolidate experiment configuration, discovery, execution, and output helpers under `@primer/agent-eval/experiment`.
+
+- Replace `findExperiment` with `getExperiment`.
+- Replace the package-root `run` export with `runExperiment`.
+- Replace the package-root `defineConfig` export with `defineExperimentConfig`.
+- Keep `listExperiments` under the singular experiment entry point and package root.
+- Remove `loadExperimentConfigs`, `ExperimentSourceOptions`, and `LoadExperimentOptions`; pass explicit experiment and scenario directories to the new discovery helpers instead.
diff --git a/.changeset/fresh-experiment-configs.md b/.changeset/fresh-experiment-configs.md
new file mode 100644
index 00000000..7e01d2fd
--- /dev/null
+++ b/.changeset/fresh-experiment-configs.md
@@ -0,0 +1,7 @@
+---
+'@primer/agent-eval': minor
+---
+
+Replace the experiment configuration model with schema-backed configuration for resolved trials.
+
+`ExperimentConfigSchema` is now exported, scenarios can be configured by ID or with `{path, name?}` inline path objects, models use model variant configuration, and experiment and treatment setup callbacks use the new sandbox interface. `InlineScenarioConfig` and `ExperimentScenarioConfig` remain available from `@primer/agent-eval/experiment`; `TreatmentConfig` is replaced by the new treatment API.
diff --git a/.changeset/gentle-scenarios-unify.md b/.changeset/gentle-scenarios-unify.md
new file mode 100644
index 00000000..a2169f4c
--- /dev/null
+++ b/.changeset/gentle-scenarios-unify.md
@@ -0,0 +1,15 @@
+---
+'@primer/agent-eval': minor
+---
+
+Consolidate scenario configuration and discovery under `@primer/agent-eval/scenario` and remove the legacy `@primer/agent-eval/scenarios` entry point.
+
+- Replace `defineScenario` with `defineConfig` from the scenario entry point or `defineScenarioConfig` from the package root.
+- Replace `findScenario` with `getScenario`.
+- Replace `ResolvedScenario` with the flattened `Scenario` type.
+- Remove `loadScenarioDirectory`.
+- Add `ScenarioConfigSchema` and `ScenarioSchema`.
+
+Scenario discovery now accepts an explicit directory and optional host, ignores template directories prefixed with `000`, requires scenario packages to include `package.json`, and resolves browser tests from `browser.test.ts`.
+
+Experiments can continue loading scenarios directly from `{path, name?}` entries without the package discovery requirements, including scenarios that use the legacy `scenario.browser.test.ts` filename.
diff --git a/.changeset/happy-output-maps.md b/.changeset/happy-output-maps.md
new file mode 100644
index 00000000..43942e8b
--- /dev/null
+++ b/.changeset/happy-output-maps.md
@@ -0,0 +1,12 @@
+---
+'@primer/agent-eval': minor
+---
+
+Replace the legacy agent-eval output API and remove the `@primer/agent-eval/output` entry point.
+
+- Replace `createAgentEvalOutput` with `output` from `@primer/agent-eval/experiment` or `getExperimentOutput` from the package root.
+- Replace `parseAgentEvalOutput` with `deserialize` from `@primer/agent-eval/experiment` or `deserializeExperimentOutput` from the package root.
+- Replace `AgentEvalOutput` with `ExperimentOutput`.
+- Replace `AgentEvalOutputResult` with trial records represented by `TrialResult`.
+
+Experiment output now stores scenarios, treatments, and trials in keyed maps, records model variants and per-session agent metrics, and uses directory-oriented artifact fields.
diff --git a/.changeset/kind-trials-compare.md b/.changeset/kind-trials-compare.md
new file mode 100644
index 00000000..9140b3d5
--- /dev/null
+++ b/.changeset/kind-trials-compare.md
@@ -0,0 +1,9 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add public trial and treatment APIs for composing and inspecting individual evaluation runs.
+
+The package root now exports `TrialSchema`, `TrialResultSchema`, `runTrial`, `compareTrial`, `Trial`, and `TrialResult`, plus `TreatmentSchema`, `ControlTreatment`, and `Treatment`. The previous `TreatmentResult` type is replaced by `TrialResult`, and treatments now use a schema-backed `{name, setup?}` shape.
+
+Trials continue to run optional scenario browser tests with Playwright and combine their results with the standard scenario test results.
diff --git a/.changeset/lucky-sandboxes-open.md b/.changeset/lucky-sandboxes-open.md
new file mode 100644
index 00000000..973dbed2
--- /dev/null
+++ b/.changeset/lucky-sandboxes-open.md
@@ -0,0 +1,7 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add the `@primer/agent-eval/sandbox` entry point for sandbox runtimes, configuration, and constants.
+
+Sandbox, Copilot plugin, and MCP types previously re-exported from `@primer/agent-eval/experiment` now come from this entry point. The concrete `Sandbox` class is replaced by a `Sandbox` interface with `SystemSandbox` and `VirtualSandbox` implementations. The entry point also exports host-aware creation options, command and copy result types, and sandbox path and user constants.
diff --git a/.changeset/neat-models-vary.md b/.changeset/neat-models-vary.md
new file mode 100644
index 00000000..37477c54
--- /dev/null
+++ b/.changeset/neat-models-vary.md
@@ -0,0 +1,7 @@
+---
+'@primer/agent-eval': minor
+---
+
+Replace the previous model configuration helpers with model variants.
+
+`ExperimentModelConfig`, `ModelInfo`, and `resolveModelConfigs` are removed. Experiment and benchmark configurations now use `ModelVariantConfig`; resolved runs use `ModelVariant`; and validation and expansion are provided by `ModelVariantSchema`, `ModelVariantConfigSchema`, and `getModelVariants`. Low-level model helpers and types are no longer exported from the package root or experiment entry point.
diff --git a/.changeset/portable-results-bundle.md b/.changeset/portable-results-bundle.md
new file mode 100644
index 00000000..5604df67
--- /dev/null
+++ b/.changeset/portable-results-bundle.md
@@ -0,0 +1,7 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add `--output-dir` for creating portable experiment and benchmark result bundles containing `output.json` and an `artifacts` directory.
+
+Artifact and walkthrough paths in bundled output are relative to `output.json`, so the complete directory can move between machines without path rewriting. `--output-dir` cannot be combined with `--output` or `--artifacts`.
diff --git a/.changeset/public-socks-smile.md b/.changeset/public-socks-smile.md
deleted file mode 100644
index 9f17a6db..00000000
--- a/.changeset/public-socks-smile.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@primer/agent-eval': patch
----
-
-Accept Copilot sub-agent user messages that omit agentMode and preserve their routing fields.
diff --git a/.changeset/quick-copilot-metrics.md b/.changeset/quick-copilot-metrics.md
new file mode 100644
index 00000000..6ef2549c
--- /dev/null
+++ b/.changeset/quick-copilot-metrics.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Improve Copilot result collection by accepting sub-agent `user.message` events without `agentMode`, preserving their routing fields, and collecting output token counts from `model.message` events while retaining compatibility with older Copilot output.
diff --git a/.changeset/sixty-ways-taste.md b/.changeset/sixty-ways-taste.md
deleted file mode 100644
index f27b0da1..00000000
--- a/.changeset/sixty-ways-taste.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@primer/agent-eval': minor
----
-
-Add `--output-dir` for creating portable experiment and benchmark bundles containing `output.json` and its artifacts.
diff --git a/.changeset/social-moons-nail.md b/.changeset/social-moons-nail.md
deleted file mode 100644
index 9da288ad..00000000
--- a/.changeset/social-moons-nail.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@primer/agent-eval': patch
----
-
-Exclude the temporary agent-browser walkthrough skill from downloaded trial artifacts.
diff --git a/.changeset/steady-trial-results.md b/.changeset/steady-trial-results.md
new file mode 100644
index 00000000..c96f2099
--- /dev/null
+++ b/.changeset/steady-trial-results.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Correct trial artifacts and benchmark comparisons by excluding the temporary `agent-browser` walkthrough skill from downloaded artifacts, comparing test success rates instead of passed-test totals, and reporting equal benchmark metrics as a 0% change.
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
index 3d83ade6..5ea1ee29 100644
--- a/packages/agent-eval/src/experiment.test.ts
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -1,3 +1,4 @@
+import path from 'node:path'
 import {expect, test} from 'vitest'
 import {defineConfig, getExperiment, listExperiments} from './experiment'
 import {VirtualHost} from './host'
@@ -104,6 +105,73 @@ test('listExperiments prefers the named experiment export', async () => {
   ])
 })
 
+test('listExperiments resolves inline scenario paths with optional names', async () => {
+  const unnamedDirectory = path.resolve('/fixtures/unnamed-scenario')
+  const namedDirectory = path.resolve('/fixtures/named-scenario')
+  const inlineConfig = {
+    ...config,
+    scenarios: [
+      {
+        path: unnamedDirectory,
+      },
+      {
+        name: 'custom-name',
+        path: namedDirectory,
+      },
+    ],
+  }
+  const scenarioConfig = JSON.stringify({
+    prompt: 'Complete the task',
+  })
+  const host = VirtualHost.create({
+    '/experiments': {
+      'inline.ts': `export default ${JSON.stringify(inlineConfig)}`,
+    },
+    '/fixtures': {
+      'unnamed-scenario': {
+        'scenario.browser.test.ts': '',
+        'scenario.config.ts': `export default ${scenarioConfig}`,
+        'scenario.test.ts': '',
+      },
+      'named-scenario': {
+        'scenario.config.ts': `export default ${scenarioConfig}`,
+        'scenario.test.ts': '',
+      },
+    },
+  })
+
+  await expect(
+    listExperiments({
+      host,
+      experimentsDirectory: '/experiments',
+      scenariosDirectory: '/scenarios',
+    }),
+  ).resolves.toEqual([
+    {
+      ...resolveConfig(inlineConfig),
+      id: 'inline',
+      filepath: '/experiments/inline.ts',
+      scenarios: [
+        {
+          id: 'unnamed-scenario',
+          directory: unnamedDirectory,
+          prompt: 'Complete the task',
+          tags: [],
+          testPath: path.join(unnamedDirectory, 'scenario.test.ts'),
+          browserTestPath: path.join(unnamedDirectory, 'scenario.browser.test.ts'),
+        },
+        {
+          id: 'custom-name',
+          directory: namedDirectory,
+          prompt: 'Complete the task',
+          tags: [],
+          testPath: path.join(namedDirectory, 'scenario.test.ts'),
+        },
+      ],
+    },
+  ])
+})
+
 test('listExperiments ignores unsupported, reserved, missing, and invalid configs', async () => {
   const serializedConfig = JSON.stringify(config)
   const host = VirtualHost.create({
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index aa6dce49..65a838c8 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -12,7 +12,7 @@ import {
 import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
 import {create as createPlan, run as runPlan} from './plan'
-import {getScenario, ScenarioSchema, type Scenario} from './scenario'
+import {getScenario, loadScenario, ScenarioSchema, type Scenario} from './scenario'
 import {selectShard, type Shard} from './shard'
 import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
 import {
@@ -25,20 +25,32 @@ import {
 } from './trial'
 import {TestResultsSchema} from './vitest'
 
+type InlineScenarioConfig = {
+  name?: string
+  path: string
+}
+
+type ExperimentScenarioConfig = string | InlineScenarioConfig
+
 type ExperimentConfig = {
   name: string
   description: string
   models: ModelVariantConfig
-  scenarios: Array<string>
+  scenarios: Array<ExperimentScenarioConfig>
   setup?: TreatmentSetup
   treatments: Array<Treatment>
 }
 
+const InlineScenarioConfigSchema = z.object({
+  name: z.optional(z.string()),
+  path: z.string(),
+})
+
 const ExperimentConfigSchema = z.object({
   name: z.string(),
   description: z.string(),
   models: ModelVariantConfigSchema,
-  scenarios: z.array(z.string()),
+  scenarios: z.array(z.union([z.string(), InlineScenarioConfigSchema])),
   setup: z.optional(TreatmentSetupSchema),
   treatments: z.array(TreatmentSchema),
 }) satisfies z.ZodMiniType<ExperimentConfig>
@@ -113,7 +125,12 @@ async function listExperiments({
     const {data: config} = parseResult
     const scenarios = await Promise.all(
       config.scenarios.map(scenario => {
-        return getScenario(host, scenariosDirectory, scenario)
+        if (typeof scenario === 'string') {
+          return getScenario(host, scenariosDirectory, scenario)
+        }
+
+        const directory = path.resolve(scenario.path)
+        return loadScenario(host, directory, scenario.name ?? path.basename(directory))
       }),
     )
     const id = path.basename(filename, path.extname(filename))
@@ -314,4 +331,11 @@ function deserialize(input: unknown): ExperimentOutput {
 }
 
 export {ExperimentConfigSchema, defineConfig, deserialize, getExperiment, listExperiments, output, run, serialize}
-export type {ExperimentConfig, Experiment, ExperimentOutput, ExperimentOutputOptions}
+export type {
+  ExperimentConfig,
+  Experiment,
+  ExperimentOutput,
+  ExperimentOutputOptions,
+  ExperimentScenarioConfig,
+  InlineScenarioConfig,
+}
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index c54b2eb3..75e47a15 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -35,6 +35,50 @@ type ScenarioSourceOptions = {
   directory: string
 }
 
+async function loadScenario(host: Host, directory: string, id = path.basename(directory)): Promise<Scenario> {
+  if (!host.existsSync(directory)) {
+    throw new Error(`Scenario "${id}" directory was not found: ${directory}`)
+  }
+
+  const stats = await host.fs.stat(directory)
+  if (!stats.isDirectory()) {
+    throw new Error(`Scenario "${id}" directory was not found: ${directory}`)
+  }
+
+  const configPath = path.join(directory, 'scenario.config.ts')
+  if (!host.existsSync(configPath)) {
+    throw new Error(`Scenario "${id}" config file was not found: ${configPath}`)
+  }
+
+  const testPath = path.join(directory, 'scenario.test.ts')
+  if (!host.existsSync(testPath)) {
+    throw new Error(`Scenario "${id}" test file was not found: ${testPath}`)
+  }
+
+  const data: ScenarioConfigModule = await host.loadModule(configPath)
+  const config = ScenarioConfigSchema.parse(data.default)
+  const scenario: Scenario = {
+    id,
+    directory,
+    prompt: config.prompt,
+    tags: config.tags ?? [],
+    testPath,
+  }
+
+  if (config.description) {
+    scenario.description = config.description
+  }
+
+  const browserTestPath = ['browser.test.ts', 'scenario.browser.test.ts']
+    .map(filename => path.join(directory, filename))
+    .find(filepath => host.existsSync(filepath))
+  if (browserTestPath) {
+    scenario.browserTestPath = browserTestPath
+  }
+
+  return scenario
+}
+
 function getScenarioSource(
   hostOrOptions: Host | ScenarioSourceOptions,
   directory?: string,
@@ -103,35 +147,13 @@ async function listScenarios(
   const scenarios: Array<Scenario> = []
 
   for (const entry of candidates) {
-    const data: ScenarioConfigModule = await host.loadModule(path.join(directory, entry.name, 'scenario.config.ts'))
-    if (!data.default) {
-      continue
-    }
-
-    const parseResult = ScenarioConfigSchema.safeParse(data.default)
-    if (!parseResult.success) {
+    const scenarioDirectory = path.join(directory, entry.name)
+    const data: ScenarioConfigModule = await host.loadModule(path.join(scenarioDirectory, 'scenario.config.ts'))
+    if (!ScenarioConfigSchema.safeParse(data.default).success) {
       continue
     }
 
-    const {data: config} = parseResult
-    const scenario: Scenario = {
-      id: entry.name,
-      directory: path.join(directory, entry.name),
-      prompt: config.prompt,
-      tags: config.tags ?? [],
-      testPath: path.join(directory, entry.name, 'scenario.test.ts'),
-    }
-
-    if (config.description) {
-      scenario.description = config.description
-    }
-
-    const browserTestPath = path.join(directory, entry.name, 'browser.test.ts')
-    if (host.existsSync(browserTestPath)) {
-      scenario.browserTestPath = browserTestPath
-    }
-
-    scenarios.push(scenario)
+    scenarios.push(await loadScenario(host, scenarioDirectory, entry.name))
   }
 
   return scenarios
@@ -155,5 +177,5 @@ async function getScenario(
   throw new Error(`Scenario "${id}" was not found in: ${source.directory}`)
 }
 
-export {defineConfig, listScenarios, getScenario, ScenarioSchema, ScenarioConfigSchema}
+export {defineConfig, listScenarios, getScenario, loadScenario, ScenarioSchema, ScenarioConfigSchema}
 export type {ScenarioConfig, Scenario, ScenarioSourceOptions}
diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts
index e5383ef6..6d6e6552 100644
--- a/packages/agent-eval/src/trial.test.ts
+++ b/packages/agent-eval/src/trial.test.ts
@@ -29,6 +29,7 @@ async function setup(trial: Trial) {
       },
       'scenario.config.ts': '',
       'scenario.test.ts': '',
+      ...(trial.scenario.browserTestPath ? {[path.basename(trial.scenario.browserTestPath)]: ''} : {}),
     },
   })
   const sandbox = await host.createSandbox()
@@ -91,8 +92,14 @@ const writeCopilotResult: RunCommandMock = async ({params}) => {
 const writeTestFile: RunCommandMock = async ({params, sandbox}) => {
   const [command, args] = params
   if (command === 'sh' && Array.isArray(args) && args[0] === '-c' && args[1].startsWith('npx vitest run')) {
+    const config = await sandbox.readFile('vitest.agent-eval.config.ts')
+    const outputFile = config.match(/outputFile: "([^"]+)"/)?.[1]
+    if (!outputFile) {
+      throw new Error('Vitest output file was not configured')
+    }
+
     await sandbox.writeFile(
-      'test-results.json',
+      outputFile,
       JSON.stringify({
         numTotalTests: 0,
         numPassedTests: 0,
@@ -280,7 +287,15 @@ describe('run', () => {
     })
 
     expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.directory, CONTAINER_WORKDIR, {
-      exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next', 'dist'],
+      exclude: [
+        'scenario.config.ts',
+        'scenario.test.ts',
+        'browser.test.ts',
+        'scenario.browser.test.ts',
+        'node_modules',
+        '.next',
+        'dist',
+      ],
     })
   })
 
@@ -661,6 +676,91 @@ describe('run', () => {
     )
   })
 
+  test('runs and combines browser tests with scenario tests', async () => {
+    const trial = createTrial()
+    trial.scenario.browserTestPath = '/scenarios/test/scenario.browser.test.ts'
+    const {sandbox, ...runOptions} = await setup(trial)
+    const writeTestResults: RunCommandMock = async ({params, sandbox: testSandbox}) => {
+      const [command, args] = params
+      if (command !== 'sh' || !Array.isArray(args) || args[0] !== '-c') {
+        return
+      }
+
+      const browser = args.at(-1) === 'scenario.browser.test.ts'
+      const outputFile = browser ? 'browser-test-results.json' : 'test-results.json'
+      await testSandbox.writeFile(
+        outputFile,
+        JSON.stringify({
+          numTotalTests: 1,
+          numPassedTests: browser ? 0 : 1,
+          numFailedTests: browser ? 1 : 0,
+          numPendingTests: 0,
+          numTodoTests: 0,
+          success: !browser,
+          testResults: [
+            {
+              assertionResults: [],
+            },
+          ],
+        }),
+      )
+
+      return {
+        stdout: '',
+        stderr: '',
+        exitCode: 0,
+      }
+    }
+    mockRunCommand(sandbox, [writeTestResults])
+
+    const result = await run({
+      ...runOptions,
+      sandbox,
+      trial,
+    })
+
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      'npm',
+      ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'],
+      {
+        user: NODE_USER,
+      },
+    )
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      './node_modules/.bin/playwright',
+      ['install', '--with-deps', 'chromium'],
+      {
+        user: 'root',
+        env: {
+          PLAYWRIGHT_BROWSERS_PATH: '/ms-playwright',
+        },
+      },
+    )
+    expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.browserTestPath, 'scenario.browser.test.ts')
+    expect(sandbox.runCommand).toHaveBeenCalledWith(
+      'sh',
+      [
+        '-c',
+        'npx vitest run --config "$1" "$2" || true',
+        'vitest-run',
+        'vitest.agent-eval.config.ts',
+        'scenario.browser.test.ts',
+      ],
+      {
+        user: NODE_USER,
+        env: {
+          PLAYWRIGHT_BROWSERS_PATH: '/ms-playwright',
+        },
+      },
+    )
+    expect(result.testResults).toMatchObject({
+      numTotalTests: 2,
+      numPassedTests: 1,
+      numFailedTests: 1,
+      success: false,
+    })
+  })
+
   test('runs Copilot with the walkthrough arguments', async () => {
     const trial = createTrial()
     const {sandbox, ...runOptions} = await setup(trial)
diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts
index ad4ca274..90b04335 100644
--- a/packages/agent-eval/src/trial.ts
+++ b/packages/agent-eval/src/trial.ts
@@ -21,6 +21,7 @@ type Trial = z.infer<typeof TrialSchema>
 
 const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg'])
 const AGENT_BROWSER_SKILL_DIRECTORY = path.posix.join(SKILLS_DIR, 'agent-browser')
+const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright'
 
 const WalkthroughSchema = z.discriminatedUnion('type', [
   z.object({type: z.literal('Unavailable')}),
@@ -123,7 +124,15 @@ async function run({
   logger.info('%s Copying files from: %s...', logPrefix, trial.scenario.directory)
 
   await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, {
-    exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next', 'dist'],
+    exclude: [
+      'scenario.config.ts',
+      'scenario.test.ts',
+      'browser.test.ts',
+      'scenario.browser.test.ts',
+      'node_modules',
+      '.next',
+      'dist',
+    ],
   })
   await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], {
     user: 'root',
@@ -163,6 +172,24 @@ async function run({
     user: NODE_USER,
   })
 
+  if (trial.scenario.browserTestPath) {
+    logger.info('%s Installing browser test dependencies...', logPrefix)
+    await sandbox.runCommand(
+      'npm',
+      ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'],
+      {
+        user: NODE_USER,
+      },
+    )
+    logger.info('%s Installing Playwright browser...', logPrefix)
+    await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], {
+      user: 'root',
+      env: {
+        PLAYWRIGHT_BROWSERS_PATH,
+      },
+    })
+  }
+
   logger.info('%s Running copilot...', logPrefix)
   const copilotOutput = await sandbox.runCommand(
     'copilot',
@@ -197,24 +224,72 @@ async function run({
   logger.info('%s Running tests...', logPrefix)
 
   const TEST_PATH = 'scenario.test.ts'
+  const BROWSER_TEST_PATH = 'scenario.browser.test.ts'
   const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts'
   const TEST_RESULTS_PATH = 'test-results.json'
-
-  await sandbox.copy(trial.scenario.testPath, TEST_PATH)
-  await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(TEST_RESULTS_PATH))
-  await sandbox.runCommand(
-    'sh',
-    ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, TEST_PATH],
+  const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json'
+  const scenarioTests = [
     {
-      user: NODE_USER,
-      env: {},
+      sourcePath: trial.scenario.testPath,
+      testPath: TEST_PATH,
+      resultsPath: TEST_RESULTS_PATH,
+      browser: false,
     },
-  )
-  const testResultsContent = await sandbox.readFile(TEST_RESULTS_PATH)
-  const rawTestResult: unknown = JSON.parse(testResultsContent)
-  const testResults = parseTestResults(rawTestResult)
-  if (!testResults.success) {
-    throw new Error(`Failed to parse test results: ${testResults.error}`)
+  ]
+
+  if (trial.scenario.browserTestPath) {
+    scenarioTests.push({
+      sourcePath: trial.scenario.browserTestPath,
+      testPath: BROWSER_TEST_PATH,
+      resultsPath: BROWSER_TEST_RESULTS_PATH,
+      browser: true,
+    })
+  }
+
+  const testRuns: Array<z.infer<typeof TestResultsSchema>> = []
+  for (const scenarioTest of scenarioTests) {
+    await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath)
+    await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser))
+    await sandbox.runCommand(
+      'sh',
+      ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath],
+      {
+        user: NODE_USER,
+        env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : {},
+      },
+    )
+
+    const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath)
+    const rawTestResult: unknown = JSON.parse(testResultsContent)
+    const testResults = parseTestResults(rawTestResult)
+    if (!testResults.success) {
+      throw new Error(`Failed to parse test results: ${testResults.error}`)
+    }
+
+    testRuns.push(testResults.data)
+  }
+
+  const firstTestRun = testRuns[0]
+  if (!firstTestRun) {
+    throw new Error('No test results were collected')
+  }
+
+  const testResults =
+    testRuns.length === 1
+      ? firstTestRun
+      : {
+          ...firstTestRun,
+          numFailedTests: testRuns.reduce((total, result) => total + result.numFailedTests, 0),
+          numPassedTests: testRuns.reduce((total, result) => total + result.numPassedTests, 0),
+          numPendingTests: testRuns.reduce((total, result) => total + result.numPendingTests, 0),
+          numTodoTests: testRuns.reduce((total, result) => total + result.numTodoTests, 0),
+          numTotalTests: testRuns.reduce((total, result) => total + result.numTotalTests, 0),
+          success: testRuns.every(result => result.success),
+          testResults: testRuns.flatMap(result => result.testResults),
+        }
+
+  if (testRuns.length > 1) {
+    await sandbox.writeFile(TEST_RESULTS_PATH, JSON.stringify(testResults))
   }
 
   const WALKTHROUGH_DIR = 'walkthrough'
@@ -364,7 +439,7 @@ Only capture the walkthrough, do not make any further code changes.`
     agent: {
       sessions: [getAgentSession(messages)],
     },
-    testResults: testResults.data,
+    testResults,
     walkthrough,
   }
 }
@@ -412,11 +487,27 @@ function getAgentSession(messages: Array<Message>): AgentSession {
   }
 }
 
-function getVitestConfig(outputFile: string) {
-  return `import {defineConfig} from 'vitest/config';
+function getVitestConfig(outputFile: string, browser = false) {
+  const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright';\n` : ''
+  const browserConfig = browser
+    ? `    browser: {
+      enabled: true,
+      headless: true,
+      instances: [
+        {
+          browser: 'chromium',
+        },
+      ],
+      provider: playwright(),
+    },
+`
+    : ''
+
+  return `${browserImport}import {defineConfig} from 'vitest/config';
 
 export default defineConfig({
   test: {
+${browserConfig}    include: ['**/*.test.ts'],
     reporters: [
       [
         'json',

From 5ca85067ef2d8b545edb7f71b65a4684607c244b Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 21:43:41 -0500
Subject: [PATCH 44/52] refactor: remove cli package entrypoint

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .changeset/tidy-cli-entrypoint.md | 5 +++++
 packages/agent-eval/package.json  | 4 ----
 2 files changed, 5 insertions(+), 4 deletions(-)
 create mode 100644 .changeset/tidy-cli-entrypoint.md

diff --git a/.changeset/tidy-cli-entrypoint.md b/.changeset/tidy-cli-entrypoint.md
new file mode 100644
index 00000000..244c630c
--- /dev/null
+++ b/.changeset/tidy-cli-entrypoint.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Remove the `@primer/agent-eval/cli` package entry point. Use the `agent-eval` executable to run the CLI; programmatic APIs remain available from the package root and the benchmark, experiment, scenario, and sandbox entry points.
diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json
index 636404f6..0344e8a5 100644
--- a/packages/agent-eval/package.json
+++ b/packages/agent-eval/package.json
@@ -14,10 +14,6 @@
       "types": "./dist/benchmark.d.ts",
       "default": "./dist/benchmark.js"
     },
-    "./cli": {
-      "types": "./dist/cli.d.ts",
-      "default": "./dist/cli.js"
-    },
     "./experiment": {
       "types": "./dist/experiment.d.ts",
       "default": "./dist/experiment.js"

From 031cf7210e6ac0bd4e222e9672daaf6a824d9c37 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 21:47:57 -0500
Subject: [PATCH 45/52] docs: shorten benchmark changesets

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .changeset/bright-benchmarks-run.md     |  6 +-----
 .changeset/calm-experiments-unify.md    |  8 +-------
 .changeset/fresh-experiment-configs.md  |  4 +---
 .changeset/gentle-scenarios-unify.md    | 12 +-----------
 .changeset/happy-output-maps.md         |  9 +--------
 .changeset/kind-trials-compare.md       |  6 +-----
 .changeset/lucky-sandboxes-open.md      |  4 +---
 .changeset/neat-models-vary.md          |  4 +---
 .changeset/plain-scenarios-discover.md  |  5 +++++
 .changeset/portable-results-bundle.md   |  4 +---
 .changeset/quick-copilot-metrics.md     |  2 +-
 .changeset/steady-trial-results.md      |  2 +-
 .changeset/tender-benchmarks-compare.md |  5 +++++
 .changeset/tidy-cli-entrypoint.md       |  2 +-
 14 files changed, 22 insertions(+), 51 deletions(-)
 create mode 100644 .changeset/plain-scenarios-discover.md
 create mode 100644 .changeset/tender-benchmarks-compare.md

diff --git a/.changeset/bright-benchmarks-run.md b/.changeset/bright-benchmarks-run.md
index e77f0a03..0cee7d26 100644
--- a/.changeset/bright-benchmarks-run.md
+++ b/.changeset/bright-benchmarks-run.md
@@ -2,8 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add benchmark configuration, discovery, execution, and output APIs through the new `@primer/agent-eval/benchmark` entry point.
-
-The package root also exports benchmark APIs with explicit names, including `defineBenchmarkConfig`, `getBenchmark`, `listBenchmarks`, `runBenchmark`, `getBenchmarkOutput`, `serializeBenchmarkOutput`, and `deserializeBenchmarkOutput`, along with benchmark configuration, result, capability, and output types.
-
-The CLI can now select and run benchmarks from a benchmarks directory.
+Add benchmark configuration, discovery, execution, and output APIs through `@primer/agent-eval/benchmark` and explicitly named package-root exports. The CLI can now select and run benchmarks from a benchmarks directory.
diff --git a/.changeset/calm-experiments-unify.md b/.changeset/calm-experiments-unify.md
index 36fc7c30..647e5dca 100644
--- a/.changeset/calm-experiments-unify.md
+++ b/.changeset/calm-experiments-unify.md
@@ -2,10 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Consolidate experiment configuration, discovery, execution, and output helpers under `@primer/agent-eval/experiment`.
-
-- Replace `findExperiment` with `getExperiment`.
-- Replace the package-root `run` export with `runExperiment`.
-- Replace the package-root `defineConfig` export with `defineExperimentConfig`.
-- Keep `listExperiments` under the singular experiment entry point and package root.
-- Remove `loadExperimentConfigs`, `ExperimentSourceOptions`, and `LoadExperimentOptions`; pass explicit experiment and scenario directories to the new discovery helpers instead.
+Move experiment APIs to `@primer/agent-eval/experiment`, replacing `findExperiment`, package-root `run`, and package-root `defineConfig` with `getExperiment`, `runExperiment`, and `defineExperimentConfig`. Remove the legacy experiment loading helpers in favor of `getExperiment` and `listExperiments` with explicit source directories.
diff --git a/.changeset/fresh-experiment-configs.md b/.changeset/fresh-experiment-configs.md
index 7e01d2fd..518205c9 100644
--- a/.changeset/fresh-experiment-configs.md
+++ b/.changeset/fresh-experiment-configs.md
@@ -2,6 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Replace the experiment configuration model with schema-backed configuration for resolved trials.
-
-`ExperimentConfigSchema` is now exported, scenarios can be configured by ID or with `{path, name?}` inline path objects, models use model variant configuration, and experiment and treatment setup callbacks use the new sandbox interface. `InlineScenarioConfig` and `ExperimentScenarioConfig` remain available from `@primer/agent-eval/experiment`; `TreatmentConfig` is replaced by the new treatment API.
+Add `ExperimentConfigSchema` and update experiment configuration to use model variants, the new treatment API, and the new sandbox interface. Scenarios continue to support IDs and `{path, name?}` entries through the exported `ExperimentScenarioConfig` and `InlineScenarioConfig` types.
diff --git a/.changeset/gentle-scenarios-unify.md b/.changeset/gentle-scenarios-unify.md
index a2169f4c..ac210883 100644
--- a/.changeset/gentle-scenarios-unify.md
+++ b/.changeset/gentle-scenarios-unify.md
@@ -2,14 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Consolidate scenario configuration and discovery under `@primer/agent-eval/scenario` and remove the legacy `@primer/agent-eval/scenarios` entry point.
-
-- Replace `defineScenario` with `defineConfig` from the scenario entry point or `defineScenarioConfig` from the package root.
-- Replace `findScenario` with `getScenario`.
-- Replace `ResolvedScenario` with the flattened `Scenario` type.
-- Remove `loadScenarioDirectory`.
-- Add `ScenarioConfigSchema` and `ScenarioSchema`.
-
-Scenario discovery now accepts an explicit directory and optional host, ignores template directories prefixed with `000`, requires scenario packages to include `package.json`, and resolves browser tests from `browser.test.ts`.
-
-Experiments can continue loading scenarios directly from `{path, name?}` entries without the package discovery requirements, including scenarios that use the legacy `scenario.browser.test.ts` filename.
+Move scenario APIs to `@primer/agent-eval/scenario`, replacing `defineScenario`, `findScenario`, and `ResolvedScenario` with `defineConfig`, `getScenario`, and `Scenario`. Add scenario schemas and remove the legacy `@primer/agent-eval/scenarios` entry point and `loadScenarioDirectory`.
diff --git a/.changeset/happy-output-maps.md b/.changeset/happy-output-maps.md
index 43942e8b..7b37fdef 100644
--- a/.changeset/happy-output-maps.md
+++ b/.changeset/happy-output-maps.md
@@ -2,11 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Replace the legacy agent-eval output API and remove the `@primer/agent-eval/output` entry point.
-
-- Replace `createAgentEvalOutput` with `output` from `@primer/agent-eval/experiment` or `getExperimentOutput` from the package root.
-- Replace `parseAgentEvalOutput` with `deserialize` from `@primer/agent-eval/experiment` or `deserializeExperimentOutput` from the package root.
-- Replace `AgentEvalOutput` with `ExperimentOutput`.
-- Replace `AgentEvalOutputResult` with trial records represented by `TrialResult`.
-
-Experiment output now stores scenarios, treatments, and trials in keyed maps, records model variants and per-session agent metrics, and uses directory-oriented artifact fields.
+Replace `createAgentEvalOutput`, `parseAgentEvalOutput`, `AgentEvalOutput`, and `AgentEvalOutputResult` with the experiment `output` and `deserialize` helpers, `ExperimentOutput`, and `TrialResult`. Experiment output now uses keyed maps, model variants, per-session agent metrics, and directory-oriented artifact fields.
diff --git a/.changeset/kind-trials-compare.md b/.changeset/kind-trials-compare.md
index 9140b3d5..a973361a 100644
--- a/.changeset/kind-trials-compare.md
+++ b/.changeset/kind-trials-compare.md
@@ -2,8 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add public trial and treatment APIs for composing and inspecting individual evaluation runs.
-
-The package root now exports `TrialSchema`, `TrialResultSchema`, `runTrial`, `compareTrial`, `Trial`, and `TrialResult`, plus `TreatmentSchema`, `ControlTreatment`, and `Treatment`. The previous `TreatmentResult` type is replaced by `TrialResult`, and treatments now use a schema-backed `{name, setup?}` shape.
-
-Trials continue to run optional scenario browser tests with Playwright and combine their results with the standard scenario test results.
+Add package-root trial and treatment schemas, types, execution helpers, and comparison helpers, replacing `TreatmentResult` with `TrialResult`. Trials continue to run optional Playwright browser tests and combine them with standard scenario test results.
diff --git a/.changeset/lucky-sandboxes-open.md b/.changeset/lucky-sandboxes-open.md
index 973dbed2..07755484 100644
--- a/.changeset/lucky-sandboxes-open.md
+++ b/.changeset/lucky-sandboxes-open.md
@@ -2,6 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add the `@primer/agent-eval/sandbox` entry point for sandbox runtimes, configuration, and constants.
-
-Sandbox, Copilot plugin, and MCP types previously re-exported from `@primer/agent-eval/experiment` now come from this entry point. The concrete `Sandbox` class is replaced by a `Sandbox` interface with `SystemSandbox` and `VirtualSandbox` implementations. The entry point also exports host-aware creation options, command and copy result types, and sandbox path and user constants.
+Add `@primer/agent-eval/sandbox` for sandbox runtimes, configuration, constants, and the plugin and MCP types previously exported from the experiment entry point. Replace the concrete `Sandbox` class with the `Sandbox` interface plus `SystemSandbox` and `VirtualSandbox`.
diff --git a/.changeset/neat-models-vary.md b/.changeset/neat-models-vary.md
index 37477c54..58d3fe49 100644
--- a/.changeset/neat-models-vary.md
+++ b/.changeset/neat-models-vary.md
@@ -2,6 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Replace the previous model configuration helpers with model variants.
-
-`ExperimentModelConfig`, `ModelInfo`, and `resolveModelConfigs` are removed. Experiment and benchmark configurations now use `ModelVariantConfig`; resolved runs use `ModelVariant`; and validation and expansion are provided by `ModelVariantSchema`, `ModelVariantConfigSchema`, and `getModelVariants`. Low-level model helpers and types are no longer exported from the package root or experiment entry point.
+Replace `ExperimentModelConfig`, `ModelInfo`, and `resolveModelConfigs` with model variant configuration, schemas, and expansion helpers. Low-level model helpers and types are no longer exported from the package root or experiment entry point.
diff --git a/.changeset/plain-scenarios-discover.md b/.changeset/plain-scenarios-discover.md
new file mode 100644
index 00000000..bc3ec7e5
--- /dev/null
+++ b/.changeset/plain-scenarios-discover.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Update scenario discovery to support explicit hosts, skip template directories, and recognize `browser.test.ts`. Inline scenario paths remain supported without package discovery requirements and continue to recognize `scenario.browser.test.ts`.
diff --git a/.changeset/portable-results-bundle.md b/.changeset/portable-results-bundle.md
index 5604df67..d5747ee7 100644
--- a/.changeset/portable-results-bundle.md
+++ b/.changeset/portable-results-bundle.md
@@ -2,6 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add `--output-dir` for creating portable experiment and benchmark result bundles containing `output.json` and an `artifacts` directory.
-
-Artifact and walkthrough paths in bundled output are relative to `output.json`, so the complete directory can move between machines without path rewriting. `--output-dir` cannot be combined with `--output` or `--artifacts`.
+Add `--output-dir` for portable experiment and benchmark bundles with artifact and walkthrough paths relative to `output.json`. This option cannot be combined with `--output` or `--artifacts`.
diff --git a/.changeset/quick-copilot-metrics.md b/.changeset/quick-copilot-metrics.md
index 6ef2549c..5b340c1a 100644
--- a/.changeset/quick-copilot-metrics.md
+++ b/.changeset/quick-copilot-metrics.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': patch
 ---
 
-Improve Copilot result collection by accepting sub-agent `user.message` events without `agentMode`, preserving their routing fields, and collecting output token counts from `model.message` events while retaining compatibility with older Copilot output.
+Accept sub-agent `user.message` events without `agentMode` and preserve their routing fields. Collect output token counts from `model.message` events while retaining compatibility with older Copilot output.
diff --git a/.changeset/steady-trial-results.md b/.changeset/steady-trial-results.md
index c96f2099..dfc532b5 100644
--- a/.changeset/steady-trial-results.md
+++ b/.changeset/steady-trial-results.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': patch
 ---
 
-Correct trial artifacts and benchmark comparisons by excluding the temporary `agent-browser` walkthrough skill from downloaded artifacts, comparing test success rates instead of passed-test totals, and reporting equal benchmark metrics as a 0% change.
+Exclude the temporary `agent-browser` walkthrough skill from downloaded trial artifacts.
diff --git a/.changeset/tender-benchmarks-compare.md b/.changeset/tender-benchmarks-compare.md
new file mode 100644
index 00000000..b46d1535
--- /dev/null
+++ b/.changeset/tender-benchmarks-compare.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Compare benchmark test success rates instead of passed-test totals and report equal metrics as a 0% change.
diff --git a/.changeset/tidy-cli-entrypoint.md b/.changeset/tidy-cli-entrypoint.md
index 244c630c..89038be2 100644
--- a/.changeset/tidy-cli-entrypoint.md
+++ b/.changeset/tidy-cli-entrypoint.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Remove the `@primer/agent-eval/cli` package entry point. Use the `agent-eval` executable to run the CLI; programmatic APIs remain available from the package root and the benchmark, experiment, scenario, and sandbox entry points.
+Remove the `@primer/agent-eval/cli` package entry point. Use the `agent-eval` executable for CLI usage and the package root or API entry points for programmatic usage.

From 1c3bc63e9eb9ad361cf113384d55d1d50a6838e5 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 21:49:19 -0500
Subject: [PATCH 46/52] docs: update benchmark API documentation

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 README.md                     |  71 +++++++++++++++++-------
 packages/agent-eval/README.md | 101 ++++++++++++++++++++++++++--------
 website/README.md             |   3 +
 3 files changed, 131 insertions(+), 44 deletions(-)

diff --git a/README.md b/README.md
index f624b4d9..df58d500 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,8 @@
 ## What this project does
 
 This project provides a framework for using agent-eval to evaluate scenarios.
-Each experiment defines treatments that set up the agent's environment before
-it completes one or more scenarios.
+Experiments compare treatments across selected scenarios, while benchmarks
+group scenarios into capabilities to measure broader agent performance.
 
 Scenarios represent tasks where, given a prompt, we measure how the agent
 behaves. This framework helps compare the effectiveness of different treatments
@@ -22,12 +22,14 @@ Results are scored by:
 
 - **Scenarios** describe the task and tests used to grade the agent's output.
 - **Experiments** select the models, scenarios, and treatments to run together.
+- **Benchmarks** group scenarios by capability and compare them against the
+  control treatment.
 - **Treatments** define the conditions for a run, such as adding an MCP server,
   custom sub-agent, or skill before the scenario starts.
 
-## Running experiments
+## Running evaluations
 
-Run local experiments with the `agent-eval` CLI:
+Run a local experiment with the `agent-eval` CLI:
 
 ```sh
 COPILOT_GITHUB_TOKEN=... agent-eval \
@@ -36,30 +38,35 @@ COPILOT_GITHUB_TOKEN=... agent-eval \
   --experiment mcp
 ```
 
-You can also provide a path directly to `--experiment`:
+Run a benchmark by selecting a file from the benchmarks directory:
 
 ```sh
 COPILOT_GITHUB_TOKEN=... agent-eval \
-  --experiment ./experiments/mcp.ts \
+  --benchmarks ./benchmarks \
+  --benchmark design-system \
   --scenarios ./scenarios
 ```
 
-The experiment and scenario directories default to `./experiments` and
-`./scenarios`, respectively. Results are written to `./output.json` by default.
-The output includes a run ID, the selected experiment ID, and an array of
-treatment results.
+The benchmark, experiment, and scenario directories default to `./benchmarks`,
+`./experiments`, and `./scenarios`. Results are written to `./output.json` by
+default, with trial artifacts stored in `./artifacts`.
+
+Use `--output-dir <directory>` to keep `output.json` and its artifacts together
+with portable relative paths. It cannot be combined with `--output` or
+`--artifacts`.
 
 ## Authoring scenarios
 
 Scenarios live in [`./scenarios`](./scenarios/). Each scenario has a
 `scenario.config.ts` file that defines the agent prompt and a `scenario.test.ts`
 file that grades the agent's output. Scenarios can also include an optional
-`scenario.browser.test.ts` file for checks that need a browser:
+`browser.test.ts` file for checks that need a browser. The legacy
+`scenario.browser.test.ts` filename remains supported:
 
 ```ts
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses a Primer button correctly',
   prompt: 'Update the index page to use a primary button',
   tags: ['baseline', 'button', 'primer'],
@@ -100,10 +107,8 @@ export const experiment = defineConfig({
 })
 ```
 
-Each model config has a `name` and a `reasoningEfforts` array. The experiment
-runs once for each configured effort. Model information, including each model's
-supported reasoning efforts, is exported as `models` from
-`@primer/agent-eval`.
+Models can be specified by name to use the default `medium` reasoning effort or
+with a `name` and `reasoningEfforts` array to run multiple variants.
 
 The experiment config specifies:
 
@@ -138,8 +143,9 @@ export const experiment = defineConfig({
 
 Scenarios can also be defined inline in an experiment. Inline scenario paths
 resolve from the directory where the CLI is run, and use the same
-`scenario.config.ts`, `scenario.test.ts`, and optional `scenario.browser.test.ts`
-files as repository scenarios:
+`scenario.config.ts`, `scenario.test.ts`, and optional browser test files as
+repository scenarios. Use `name` to override the scenario ID derived from the
+directory name:
 
 ```ts
 export const experiment = defineConfig({
@@ -148,6 +154,7 @@ export const experiment = defineConfig({
   models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
   scenarios: [
     {
+      name: 'local-button',
       path: './scenarios/local-button-scenario',
     },
   ],
@@ -155,7 +162,31 @@ export const experiment = defineConfig({
 })
 ```
 
-### Describe eval tests
+## Authoring benchmarks
+
+Benchmarks live in [`./benchmarks`](./benchmarks/) and group scenarios into
+capabilities:
+
+```ts
+import {defineConfig} from '@primer/agent-eval/benchmark'
+
+export const benchmark = defineConfig({
+  name: 'Design system',
+  description: 'Measure agent performance across design system tasks',
+  models: ['gpt-5.6-sol'],
+  capabilities: [
+    {
+      name: 'Uses components',
+      scenarios: ['001-agent-uses-button-from-primer'],
+    },
+  ],
+})
+```
+
+Each benchmark runs the control treatment for every configured model variant
+and scenario.
+
+## Describing eval tests
 
 Individual test titles and statuses are included in each run's `testResults`.
 Add a JSDoc, block, or consecutive line comment immediately before a test to
diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md
index da242a60..fe54e300 100644
--- a/packages/agent-eval/README.md
+++ b/packages/agent-eval/README.md
@@ -1,7 +1,7 @@
 # @primer/agent-eval
 
-A library and cli tool for creating and running experiments in order to evaluate
-agent behavior across different scenarios.
+A library and CLI tool for creating and running experiments and benchmarks that
+evaluate agent behavior across different scenarios.
 
 ## Getting started
 
@@ -12,7 +12,7 @@ command using [npm](https://www.npmjs.com/):
 npm install -S @primer/agent-eval
 ```
 
-This will provide both the cli and library for creating and running experiments.
+This provides the `agent-eval` executable and the package's programmatic APIs.
 Typically, you'll first create an experiment:
 
 ```tsx
@@ -71,9 +71,9 @@ against:
 ```tsx
 // scenarios/uses-button-from-primer/scenario.config.ts
 
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent completes the example task',
   prompt: `Example scenario prompt that will instruct the agent to perform a task`,
   tags: ['baseline', 'button', 'primer'],
@@ -96,13 +96,12 @@ agent sees the prompt for the scenario.
 
 ### Browser tests
 
-Add an optional `scenario.browser.test.ts` file when a scenario needs tests in a
-real browser. Agent eval installs Vitest, Playwright, Vitest's Playwright browser
-provider, Chromium, and its system dependencies, then runs the browser test in
-Vitest browser mode after the required `scenario.test.ts` file. Results from
-both files are combined in the scenario score and test-results artifact.
+Add an optional `browser.test.ts` file when a scenario needs tests in a real
+browser. The legacy `scenario.browser.test.ts` filename remains supported.
+Agent eval runs browser tests with Playwright after `scenario.test.ts` and
+combines both results in the scenario score and test-results artifact.
 
-With everything in place, you can now use the `@primer/agent-eval` cli to run
+With everything in place, you can now use the `agent-eval` executable to run
 the experiment:
 
 ```bash
@@ -123,10 +122,19 @@ COPILOT_GITHUB_TOKEN=... agent-eval \
 
 Use `--experiments` to load experiment files from a local directory. Experiment
 files may export an `experiment` named export or a default export. `--experiment`
-may also be a path to a local experiment file when you only want to run one
-experiment. The experiments directory defaults to `./experiments`. Use
-`--scenarios` to set the directory containing scenario directories; it defaults
-to `./scenarios`.
+selects an experiment by its filename without the extension. The experiments
+directory defaults to `./experiments`. Use `--scenarios` to set the directory
+containing scenario directories; it defaults to `./scenarios`.
+
+Use `--benchmark` to select a benchmark by filename and `--benchmarks` to set
+the benchmark directory:
+
+```sh
+COPILOT_GITHUB_TOKEN=... agent-eval \
+  --benchmarks ./benchmarks \
+  --scenarios ./scenarios \
+  --benchmark design-system
+```
 
 ### Result bundles
 
@@ -153,13 +161,13 @@ and `artifacts/` within the selected directory. Use the existing `--output` and
 
 ## Scenario config authoring
 
-Use `defineScenario` from `@primer/agent-eval/scenario` in each
+Use `defineConfig` from `@primer/agent-eval/scenario` in each
 `scenario.config.ts` file:
 
 ```ts
-import {defineScenario} from '@primer/agent-eval/scenario'
+import {defineConfig} from '@primer/agent-eval/scenario'
 
-export default defineScenario({
+export default defineConfig({
   description: 'Evaluate whether the agent uses a Primer button correctly',
   prompt: 'Update the index page to use a primary button',
   tags: ['baseline', 'button', 'primer'],
@@ -167,8 +175,7 @@ export default defineScenario({
 ```
 
 Scenario descriptions and tags are optional. Use `description` to explain what
-the scenario tests. Pass `tags` to `listScenarios` to return only scenarios that
-include every requested tag.
+the scenario tests.
 
 ## Experiment config authoring
 
@@ -187,10 +194,41 @@ export const experiment = defineConfig({
 })
 ```
 
-Each model config has a `name` and a `reasoningEfforts` array. The experiment
-runs once for each configured effort. Model information, including each model's
-supported reasoning efforts, is exported as `models` from
-`@primer/agent-eval`.
+Models can be specified by name to use the default `medium` reasoning effort or
+with a `name` and `reasoningEfforts` array to run multiple variants.
+
+Scenarios can be selected by ID or loaded directly from a path:
+
+```ts
+scenarios: [
+  '001-agent-uses-button-from-primer',
+  {
+    name: 'local-button',
+    path: './scenarios/local-button-scenario',
+  },
+]
+```
+
+## Benchmark config authoring
+
+Use `defineConfig` from `@primer/agent-eval/benchmark` to group scenarios into
+capabilities:
+
+```ts
+import {defineConfig} from '@primer/agent-eval/benchmark'
+
+export const benchmark = defineConfig({
+  name: 'Design system',
+  description: 'Measure agent performance across design system tasks',
+  models: ['gpt-5.6-sol'],
+  capabilities: [
+    {
+      name: 'Uses components',
+      scenarios: ['001-agent-uses-button-from-primer'],
+    },
+  ],
+})
+```
 
 Treatment setup can add custom Copilot sub-agents to `~/.copilot/agents`:
 
@@ -247,3 +285,18 @@ await sandbox.addCopilotPlugin({
 
 Use `{type: 'local', sourcePath: './plugins/local-marketplace'}` as the
 marketplace `source` to install from a local marketplace.
+
+## Programmatic APIs
+
+The package root exports explicitly named benchmark, experiment, scenario,
+treatment, and trial APIs. Domain entry points are available from
+`@primer/agent-eval/benchmark`, `@primer/agent-eval/experiment`,
+`@primer/agent-eval/scenario`, and `@primer/agent-eval/sandbox`.
+
+Use the benchmark and experiment entry points for configuration, discovery,
+execution, output creation, serialization, and deserialization. Use the
+sandbox entry point for `Sandbox`, `SystemSandbox`, `VirtualSandbox`, plugin and
+MCP configuration types, and sandbox constants.
+
+The CLI is available through the `agent-eval` executable rather than a
+`@primer/agent-eval/cli` package entry point.
diff --git a/website/README.md b/website/README.md
index cafef617..2ad28f84 100644
--- a/website/README.md
+++ b/website/README.md
@@ -27,3 +27,6 @@ results/
     ├── output.json
     └── artifacts/
 ```
+
+Artifact and walkthrough paths are relative to each `output.json`, so result
+directories should be moved or uploaded as complete bundles.

From 3f367437266b71b78539ffad50f1f76378f13b30 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 22:29:07 -0500
Subject: [PATCH 47/52] refactor: remove baseline experiment

---
 experiments/baseline.ts | 37 -------------------------------------
 1 file changed, 37 deletions(-)
 delete mode 100644 experiments/baseline.ts

diff --git a/experiments/baseline.ts b/experiments/baseline.ts
deleted file mode 100644
index bfd4f20e..00000000
--- a/experiments/baseline.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-// import path from 'node:path'
-// import {defineConfig} from '@primer/agent-eval/experiment'
-// import {listScenarios} from '@primer/agent-eval'
-//
-// const scenarios = await listScenarios({
-//   directory: path.resolve(import.meta.dirname, '..', 'scenarios'),
-//   tags: ['baseline'],
-// })
-//
-// export const experiment = defineConfig({
-//   name: 'Baseline',
-//   description: 'Baseline experiment to evaluate the performance of the agent with our recommended setup.',
-//   models: [
-//     {name: 'gpt-5.6-sol', reasoningEfforts: []},
-//     {name: 'gpt-5.6-terra', reasoningEfforts: []},
-//     {name: 'claude-opus-5', reasoningEfforts: []},
-//     {name: 'claude-sonnet-5', reasoningEfforts: []},
-//     {name: 'gemini-3.1-pro-preview', reasoningEfforts: []},
-//     {name: 'gemini-3.6-flash', reasoningEfforts: []},
-//   ],
-//   scenarios: scenarios.filter(scenario => !scenario.id.startsWith('000')).map(scenario => scenario.id),
-//   treatments: [
-//     {
-//       name: 'Recommended',
-//       async setup({sandbox}) {
-//         // Setup the Primer MCP server locally
-//         await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest'])
-//         await sandbox.addMcpServer('primer', {
-//           type: 'local',
-//           command: 'npx',
-//           args: ['--no-install', '@primer/mcp'],
-//           tools: ['*'],
-//         })
-//       },
-//     },
-//   ],
-// })

From f57a0f5304ef2922279097f3c216e5693d2e806c Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 22:44:23 -0500
Subject: [PATCH 48/52] test: consolidate experiment coverage

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 7eb8999b-2277-4d41-915f-fb35cac43373
---
 .../agent-eval/src/experiment-output.test.ts  | 86 -------------------
 packages/agent-eval/src/experiment.test.ts    | 86 ++++++++++++++++++-
 2 files changed, 85 insertions(+), 87 deletions(-)
 delete mode 100644 packages/agent-eval/src/experiment-output.test.ts

diff --git a/packages/agent-eval/src/experiment-output.test.ts b/packages/agent-eval/src/experiment-output.test.ts
deleted file mode 100644
index e93dc809..00000000
--- a/packages/agent-eval/src/experiment-output.test.ts
+++ /dev/null
@@ -1,86 +0,0 @@
-import {expect, test} from 'vitest'
-import {deserialize, output, serialize} from './experiment'
-import type {TrialResult} from './trial'
-
-test('serializes and deserializes experiment identity and result maps', () => {
-  const experimentOutput = output('baseline', [])
-  const serialized = serialize(experimentOutput)
-
-  expect(JSON.parse(serialized)).toEqual({
-    experimentId: 'baseline',
-    scenarios: {},
-    treatments: {},
-    trials: {},
-  })
-  expect(deserialize(serialized)).toEqual({
-    experimentId: 'baseline',
-    scenarios: new Map(),
-    treatments: new Map(),
-    trials: new Map(),
-  })
-})
-
-test('creates portable artifact paths relative to the output directory', () => {
-  const trialResult: TrialResult = {
-    artifacts: {
-      directory: '/bundle/artifacts/trial',
-      copilotConfigDirectory: '/bundle/artifacts/trial/.copilot',
-      skillsConfigDirectory: '/bundle/artifacts/trial/.agents',
-      testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json',
-      workspaceDirectory: '/bundle/artifacts/trial/workspace',
-    },
-    trial: {
-      id: 'trial',
-      scenario: {
-        id: 'scenario',
-        directory: '/scenarios/scenario',
-        prompt: 'Complete the task',
-        tags: [],
-        testPath: '/scenarios/scenario/scenario.test.ts',
-      },
-      treatment: {
-        name: 'Control',
-      },
-      model: {
-        name: 'gpt-5.6-sol',
-        reasoningEffort: 'medium',
-      },
-    },
-    agent: {
-      sessions: [],
-    },
-    testResults: {
-      numTotalTests: 1,
-      numPassedTests: 1,
-      numFailedTests: 0,
-      numPendingTests: 0,
-      numTodoTests: 0,
-      success: true,
-      testResults: [],
-    },
-    walkthrough: {
-      type: 'Screenshot',
-      filepath: '/bundle/artifacts/trial/walkthrough/screenshot.png',
-    },
-  }
-
-  const portableOutput = output('baseline', [trialResult], {
-    baseDirectory: '/bundle',
-  })
-
-  expect(portableOutput.trials.get('trial')).toEqual(
-    expect.objectContaining({
-      artifacts: {
-        directory: 'artifacts/trial',
-        copilotConfigDirectory: 'artifacts/trial/.copilot',
-        skillsConfigDirectory: 'artifacts/trial/.agents',
-        testResultsPath: 'artifacts/trial/workspace/test-results.json',
-        workspaceDirectory: 'artifacts/trial/workspace',
-      },
-      walkthrough: {
-        type: 'Screenshot',
-        filepath: 'artifacts/trial/walkthrough/screenshot.png',
-      },
-    }),
-  )
-})
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
index 5ea1ee29..826f6c5a 100644
--- a/packages/agent-eval/src/experiment.test.ts
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -1,7 +1,8 @@
 import path from 'node:path'
 import {expect, test} from 'vitest'
-import {defineConfig, getExperiment, listExperiments} from './experiment'
+import {defineConfig, deserialize, getExperiment, listExperiments, output, serialize} from './experiment'
 import {VirtualHost} from './host'
+import type {TrialResult} from './trial'
 
 const config = defineConfig({
   name: 'Test experiment',
@@ -263,3 +264,86 @@ test('getExperiment throws when the experiment is not found', async () => {
     }),
   ).rejects.toThrow('Experiment "missing" was not found in: /experiments')
 })
+
+test('serializes and deserializes experiment identity and result maps', () => {
+  const experimentOutput = output('baseline', [])
+  const serialized = serialize(experimentOutput)
+
+  expect(JSON.parse(serialized)).toEqual({
+    experimentId: 'baseline',
+    scenarios: {},
+    treatments: {},
+    trials: {},
+  })
+  expect(deserialize(serialized)).toEqual({
+    experimentId: 'baseline',
+    scenarios: new Map(),
+    treatments: new Map(),
+    trials: new Map(),
+  })
+})
+
+test('creates portable artifact paths relative to the output directory', () => {
+  const trialResult: TrialResult = {
+    artifacts: {
+      directory: '/bundle/artifacts/trial',
+      copilotConfigDirectory: '/bundle/artifacts/trial/.copilot',
+      skillsConfigDirectory: '/bundle/artifacts/trial/.agents',
+      testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json',
+      workspaceDirectory: '/bundle/artifacts/trial/workspace',
+    },
+    trial: {
+      id: 'trial',
+      scenario: {
+        id: 'scenario',
+        directory: '/scenarios/scenario',
+        prompt: 'Complete the task',
+        tags: [],
+        testPath: '/scenarios/scenario/scenario.test.ts',
+      },
+      treatment: {
+        name: 'Control',
+      },
+      model: {
+        name: 'gpt-5.6-sol',
+        reasoningEffort: 'medium',
+      },
+    },
+    agent: {
+      sessions: [],
+    },
+    testResults: {
+      numTotalTests: 1,
+      numPassedTests: 1,
+      numFailedTests: 0,
+      numPendingTests: 0,
+      numTodoTests: 0,
+      success: true,
+      testResults: [],
+    },
+    walkthrough: {
+      type: 'Screenshot',
+      filepath: '/bundle/artifacts/trial/walkthrough/screenshot.png',
+    },
+  }
+
+  const portableOutput = output('baseline', [trialResult], {
+    baseDirectory: '/bundle',
+  })
+
+  expect(portableOutput.trials.get('trial')).toEqual(
+    expect.objectContaining({
+      artifacts: {
+        directory: 'artifacts/trial',
+        copilotConfigDirectory: 'artifacts/trial/.copilot',
+        skillsConfigDirectory: 'artifacts/trial/.agents',
+        testResultsPath: 'artifacts/trial/workspace/test-results.json',
+        workspaceDirectory: 'artifacts/trial/workspace',
+      },
+      walkthrough: {
+        type: 'Screenshot',
+        filepath: 'artifacts/trial/walkthrough/screenshot.png',
+      },
+    }),
+  )
+})

From ac5679302a11d7adbf8b53d7fbefba86ac34bc33 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 22:47:43 -0500
Subject: [PATCH 49/52] fix: resolve benchmark support integration

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 7eb8999b-2277-4d41-915f-fb35cac43373
---
 packages/agent-eval/src/host.ts   |  6 +++---
 packages/agent-eval/src/plan.ts   | 11 ++++-------
 packages/agent-eval/src/report.ts |  4 ++--
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
index 8e386719..e70f7c7a 100644
--- a/packages/agent-eval/src/host.ts
+++ b/packages/agent-eval/src/host.ts
@@ -44,12 +44,12 @@ class VirtualHost implements Host {
   vol: Volume
 
   constructor(json?: NestedDirectoryJSON) {
-    const {fs, vol} = memfs(json)
+    const {fs: virtualFs, vol} = memfs(json)
     this.vol = vol
-    this.existsSync = fs.existsSync
+    this.existsSync = virtualFs.existsSync
     // @ts-expect-error - not every constant is exposed but memfs should match
     // for our test cases
-    this.fs = fs.promises
+    this.fs = virtualFs.promises
   }
 
   async loadModule<T = unknown>(filepath: string): Promise<T> {
diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts
index 6b1de65b..9d503cc8 100644
--- a/packages/agent-eval/src/plan.ts
+++ b/packages/agent-eval/src/plan.ts
@@ -1,19 +1,16 @@
 import Queue from 'p-queue'
-import * as z from 'zod/mini'
-import {TrialSchema, run as runTrial} from './trial'
+import {run as runTrial} from './trial'
 import type {Trial, TrialResult} from './trial'
 import type {EnvironmentConfig} from './environment'
 import {DefaultHost, type Host} from './host'
 import {logger} from './logger'
 
-const PlanSchema = z.object({
-  trials: z.array(TrialSchema),
-})
-
 /**
  * A plan is an ordered list of trials to be ran.
  */
-type Plan = z.infer<typeof PlanSchema>
+type Plan = {
+  trials: Array<Trial>
+}
 
 // TODO: support plan with sharding
 async function create(trials: Array<Trial>): Promise<Plan> {
diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts
index f978d6cf..c6bdd9ba 100644
--- a/packages/agent-eval/src/report.ts
+++ b/packages/agent-eval/src/report.ts
@@ -300,8 +300,8 @@ function getBenchmarkComparisons(
         scenario: value.scenario,
         model: value.model,
         reasoningEffort: value.reasoningEffort,
-        control: createResultSummary(benchmark),
-        benchmarkTreatment: createResultSummary(benchmark),
+        control: createResultSummary(benchmark.name),
+        benchmarkTreatment: createResultSummary(benchmark.name),
       }
       const summary = result.trial.treatment.name === 'Control' ? comparison.control : comparison.benchmarkTreatment
       addResultToSummary(summary, result)

From 47d2719351d2f02471f3f490ecf6bdf7fde97d3b Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 22:47:43 -0500
Subject: [PATCH 50/52] refactor: derive artifact directory from output

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 7eb8999b-2277-4d41-915f-fb35cac43373
---
 .changeset/portable-results-bundle.md       |  2 +-
 packages/agent-eval/README.md               |  4 ++--
 packages/agent-eval/src/cli.test.ts         |  6 +++---
 packages/agent-eval/src/cli.ts              |  7 -------
 packages/agent-eval/src/environment.test.ts | 13 ++-----------
 packages/agent-eval/src/environment.ts      |  9 +++------
 6 files changed, 11 insertions(+), 30 deletions(-)

diff --git a/.changeset/portable-results-bundle.md b/.changeset/portable-results-bundle.md
index d5747ee7..074ebd31 100644
--- a/.changeset/portable-results-bundle.md
+++ b/.changeset/portable-results-bundle.md
@@ -2,4 +2,4 @@
 '@primer/agent-eval': minor
 ---
 
-Add `--output-dir` for portable experiment and benchmark bundles with artifact and walkthrough paths relative to `output.json`. This option cannot be combined with `--output` or `--artifacts`.
+Add `--output-dir` for portable experiment and benchmark bundles with artifact and walkthrough paths relative to `output.json`. Remove `--artifacts` and derive the artifact directory from `--output-dir` or the directory containing `--output`.
diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md
index fe54e300..01f0854b 100644
--- a/packages/agent-eval/README.md
+++ b/packages/agent-eval/README.md
@@ -156,8 +156,8 @@ agent-eval \
 Artifact and walkthrough references written by the CLI are relative to the
 directory containing `output.json`. Upload or download the complete `run`
 directory to preserve those references. `--output-dir` creates `output.json`
-and `artifacts/` within the selected directory. Use the existing `--output` and
-`--artifacts` options separately when a custom layout is required.
+and `artifacts/` within the selected directory. When using `--output`, artifacts
+are written to an `artifacts/` directory beside the selected file.
 
 ## Scenario config authoring
 
diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts
index 6c2427a9..7f42fdba 100644
--- a/packages/agent-eval/src/cli.test.ts
+++ b/packages/agent-eval/src/cli.test.ts
@@ -46,10 +46,10 @@ describe('cli', () => {
     expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]'))
   })
 
-  test('rejects output directory combinations with explicit artifact paths', async () => {
-    process.argv = ['node', 'agent-eval', '--output-dir', 'results/run', '--artifacts', 'artifacts']
+  test('rejects output directory combinations with explicit output paths', async () => {
+    process.argv = ['node', 'agent-eval', '--output-dir', 'results/run', '--output', 'output.json']
     process.env.COPILOT_GITHUB_TOKEN = 'token'
 
-    await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --artifacts or --output')
+    await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --output')
   })
 })
diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts
index b77460fd..ed9d8501 100644
--- a/packages/agent-eval/src/cli.ts
+++ b/packages/agent-eval/src/cli.ts
@@ -24,11 +24,6 @@ import {compare as compareTrial} from './trial'
 
 const {values} = parseArgs({
   options: {
-    artifacts: {
-      type: 'string',
-      short: 'a',
-      description: 'The directory to save artifacts to',
-    },
     benchmark: {
       type: 'string',
       short: 'b',
@@ -90,7 +85,6 @@ function displayHelp() {
 Usage: agent-eval [options]
 
 Options:
-  -a, --artifacts <dir>      The directory to save artifacts to (default: ./artifacts)
   -b, --benchmark <file>     The file name of the benchmark to run
       --benchmarks <dir>     The directory containing local benchmark files (default: ./benchmarks)
   -c, --concurrency <num>    The number of treatments to run in parallel
@@ -123,7 +117,6 @@ if (!COPILOT_GITHUB_TOKEN) {
 }
 
 const env = getEnvironmentConfig({
-  artifactsDirectory: values.artifacts,
   benchmarksDirectory: values.benchmarks,
   concurrency: values.concurrency,
   copilotToken: COPILOT_GITHUB_TOKEN,
diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts
index 33a1c0f0..b6ded377 100644
--- a/packages/agent-eval/src/environment.test.ts
+++ b/packages/agent-eval/src/environment.test.ts
@@ -24,7 +24,6 @@ describe('getEnvironmentConfig', () => {
   test('uses valid custom values', () => {
     expect(
       getEnvironmentConfig({
-        artifactsDirectory: './custom-artifacts',
         benchmarksDirectory: './custom-benchmarks',
         concurrency: '4',
         copilotToken: 'token',
@@ -34,7 +33,7 @@ describe('getEnvironmentConfig', () => {
         scenariosDirectory: './custom-scenarios',
       }),
     ).toEqual({
-      artifactsDirectory: path.resolve('custom-artifacts'),
+      artifactsDirectory: path.resolve('results/artifacts'),
       benchmarksDirectory: path.resolve('custom-benchmarks'),
       concurrency: 4,
       copilotToken: 'token',
@@ -64,21 +63,13 @@ describe('getEnvironmentConfig', () => {
   })
 
   test('rejects output directory combinations with explicit output paths', () => {
-    expect(() => {
-      getEnvironmentConfig({
-        artifactsDirectory: './artifacts',
-        copilotToken: 'token',
-        outputDirectory: './results/run',
-      })
-    }).toThrow('--output-dir cannot be combined with --artifacts or --output')
-
     expect(() => {
       getEnvironmentConfig({
         copilotToken: 'token',
         outputDirectory: './results/run',
         outputPath: './output.json',
       })
-    }).toThrow('--output-dir cannot be combined with --artifacts or --output')
+    }).toThrow('--output-dir cannot be combined with --output')
   })
 
   test.each(['0', '-1', 'invalid', '1.5'])('falls back to one for invalid concurrency %s', concurrency => {
diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts
index 66b77ec9..8ce00d19 100644
--- a/packages/agent-eval/src/environment.ts
+++ b/packages/agent-eval/src/environment.ts
@@ -13,7 +13,6 @@ type EnvironmentConfig = {
 }
 
 type EnvironmentOptions = {
-  artifactsDirectory?: string
   benchmarksDirectory?: string
   concurrency?: string
   copilotToken: string
@@ -25,14 +24,11 @@ type EnvironmentOptions = {
 }
 
 function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig {
-  if (options.outputDirectory && (options.artifactsDirectory || options.outputPath)) {
-    throw new Error('--output-dir cannot be combined with --artifacts or --output')
+  if (options.outputDirectory && options.outputPath) {
+    throw new Error('--output-dir cannot be combined with --output')
   }
 
   const outputDirectory = options.outputDirectory ? path.resolve(options.outputDirectory) : undefined
-  const artifactsDirectory = outputDirectory
-    ? path.join(outputDirectory, 'artifacts')
-    : path.resolve(options.artifactsDirectory ?? 'artifacts')
   const benchmarksDirectory = path.resolve(options.benchmarksDirectory ?? 'benchmarks')
   const parsedConcurrency = options.concurrency ? parseInt(options.concurrency, 10) : 1
   const concurrency =
@@ -43,6 +39,7 @@ function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig {
   const outputPath = outputDirectory
     ? path.join(outputDirectory, 'output.json')
     : path.resolve(options.outputPath ?? 'output.json')
+  const artifactsDirectory = path.join(path.dirname(outputPath), 'artifacts')
   const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios')
 
   return {

From e58491ff63c4af4f8ea99462a1802bd245809f03 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 22:52:33 -0500
Subject: [PATCH 51/52] fix: address benchmark review feedback

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 packages/agent-eval/src/benchmark.test.ts  | 20 +++++++++++++++
 packages/agent-eval/src/benchmark.ts       |  2 +-
 packages/agent-eval/src/experiment.test.ts | 22 ++++++++++++++++
 packages/agent-eval/src/experiment.ts      |  2 +-
 packages/agent-eval/src/scenario.test.ts   | 30 ++++++++++++++++++++++
 packages/agent-eval/src/scenario.ts        |  8 ++++--
 script/run-benchmark.sh                    |  2 +-
 7 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 4c764730..97ffa187 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -122,6 +122,26 @@ test('listBenchmarks prefers the named benchmark export', async () => {
   ])
 })
 
+test('listBenchmarks sorts benchmarks by filename', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = createHost({
+    'z-last.ts': `export const benchmark = ${serializedConfig}`,
+    'a-first.ts': `export const benchmark = ${serializedConfig}`,
+  })
+
+  const benchmarks = await listBenchmarks({
+    host,
+    benchmarksDirectory: '/benchmarks',
+    scenariosDirectory: '/scenarios',
+  })
+
+  expect(
+    benchmarks.map(benchmark => {
+      return benchmark.id
+    }),
+  ).toEqual(['a-first', 'z-last'])
+})
+
 test('listBenchmarks validates the benchmarks directory', async () => {
   const host = VirtualHost.create()
 
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index 4dfc5d9c..bd2bae37 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -77,7 +77,7 @@ async function listBenchmarks({
     throw new Error(`Benchmarks path is not a directory: ${benchmarksDirectory}`)
   }
 
-  const filenames = await host.fs.readdir(benchmarksDirectory)
+  const filenames = (await host.fs.readdir(benchmarksDirectory)).sort()
   const benchmarks: Array<Benchmark> = []
 
   for (const filename of filenames) {
diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts
index 826f6c5a..3b5fad20 100644
--- a/packages/agent-eval/src/experiment.test.ts
+++ b/packages/agent-eval/src/experiment.test.ts
@@ -106,6 +106,28 @@ test('listExperiments prefers the named experiment export', async () => {
   ])
 })
 
+test('listExperiments sorts experiments by filename', async () => {
+  const serializedConfig = JSON.stringify(config)
+  const host = VirtualHost.create({
+    '/experiments': {
+      'z-last.ts': `export const experiment = ${serializedConfig}`,
+      'a-first.ts': `export const experiment = ${serializedConfig}`,
+    },
+  })
+
+  const experiments = await listExperiments({
+    host,
+    experimentsDirectory: '/experiments',
+    scenariosDirectory: '/scenarios',
+  })
+
+  expect(
+    experiments.map(experiment => {
+      return experiment.id
+    }),
+  ).toEqual(['a-first', 'z-last'])
+})
+
 test('listExperiments resolves inline scenario paths with optional names', async () => {
   const unnamedDirectory = path.resolve('/fixtures/unnamed-scenario')
   const namedDirectory = path.resolve('/fixtures/named-scenario')
diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts
index 65a838c8..1cd3875a 100644
--- a/packages/agent-eval/src/experiment.ts
+++ b/packages/agent-eval/src/experiment.ts
@@ -95,7 +95,7 @@ async function listExperiments({
     throw new Error(`Experiments path is not a directory: ${experimentsDirectory}`)
   }
 
-  const filenames = await host.fs.readdir(experimentsDirectory)
+  const filenames = (await host.fs.readdir(experimentsDirectory)).sort()
   const experiments: Array<Experiment> = []
 
   for (const filename of filenames) {
diff --git a/packages/agent-eval/src/scenario.test.ts b/packages/agent-eval/src/scenario.test.ts
index 2bdc8062..809d493a 100644
--- a/packages/agent-eval/src/scenario.test.ts
+++ b/packages/agent-eval/src/scenario.test.ts
@@ -86,6 +86,36 @@ test('listScenarios includes optional metadata and browser tests', async () => {
   ])
 })
 
+test('listScenarios sorts scenarios by directory name', async () => {
+  const config = JSON.stringify(
+    defineConfig({
+      prompt: 'test',
+    }),
+  )
+  const host = VirtualHost.create({
+    '/scenarios': {
+      '002-last': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+      '001-first': {
+        'package.json': '{}',
+        'scenario.config.ts': `export default ${config}`,
+        'scenario.test.ts': '',
+      },
+    },
+  })
+
+  const scenarios = await listScenarios(host, '/scenarios')
+
+  expect(
+    scenarios.map(scenario => {
+      return scenario.id
+    }),
+  ).toEqual(['001-first', '002-last'])
+})
+
 test('listScenarios ignores configs without a default export', async () => {
   const config = JSON.stringify(
     defineConfig({
diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts
index 75e47a15..1b691ef6 100644
--- a/packages/agent-eval/src/scenario.ts
+++ b/packages/agent-eval/src/scenario.ts
@@ -111,8 +111,12 @@ async function listScenarios(
     throw new Error('Expected scenarios path to be a directory')
   }
 
-  const entries = await host.fs.readdir(directory, {
-    withFileTypes: true,
+  const entries = (
+    await host.fs.readdir(directory, {
+      withFileTypes: true,
+    })
+  ).sort((a, b) => {
+    return a.name.localeCompare(b.name)
   })
   const candidates = entries.filter(entry => {
     if (!entry.isDirectory()) {
diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh
index dbf10527..687d7486 100755
--- a/script/run-benchmark.sh
+++ b/script/run-benchmark.sh
@@ -3,7 +3,7 @@
 set -euo pipefail
 
 repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-benchmark_name='design-system'
+benchmark_name="${BENCHMARK_NAME:-design-system}"
 run_date="${RUN_DATE:-$(date -u +%F)}"
 run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date"
 

From 76ed0daa04f3a52c30f29780e1fd47dbf137bfff Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Thu, 3 Sep 2026 23:10:58 -0500
Subject: [PATCH 52/52] fix: address remaining benchmark review feedback

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 33a4d86e-9f6e-4031-b77c-499d3530b6fc
---
 .github/workflows/experiment.yml          |  1 -
 packages/agent-eval/README.md             | 10 +++
 packages/agent-eval/src/benchmark.test.ts | 85 ++++++++++++++++++++++-
 packages/agent-eval/src/benchmark.ts      | 25 +++++--
 packages/agent-eval/src/host.test.ts      |  3 +
 packages/agent-eval/src/host.ts           |  5 +-
 6 files changed, 121 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml
index bbfed459..43cb8956 100644
--- a/.github/workflows/experiment.yml
+++ b/.github/workflows/experiment.yml
@@ -74,7 +74,6 @@ jobs:
             --experiment "$EXPERIMENT_NAME" \
             --experiments experiments \
             --scenarios scenarios \
-            --artifacts "$run_directory/artifacts" \
             --output "$run_directory/output-${{ matrix.order }}.json" \
             --shard "$SHARD"
       - name: Prepare experiment artifact
diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md
index 01f0854b..bd02ff20 100644
--- a/packages/agent-eval/README.md
+++ b/packages/agent-eval/README.md
@@ -221,15 +221,25 @@ export const benchmark = defineConfig({
   name: 'Design system',
   description: 'Measure agent performance across design system tasks',
   models: ['gpt-5.6-sol'],
+  async setup({sandbox}) {
+    await sandbox.addAgentSkill('design-system', 'Uses the design system', 'Follow the design system guidance.')
+  },
   capabilities: [
     {
       name: 'Uses components',
       scenarios: ['001-agent-uses-button-from-primer'],
+      async setup({sandbox}) {
+        await sandbox.writeFile('/root/.copilot/component-guidance.md', 'Prefer existing components.')
+      },
     },
   ],
 })
 ```
 
+The top-level setup runs first for every benchmark treatment trial. A
+capability setup runs next for treatment trials in that capability. Control
+trials do not run either setup.
+
 Treatment setup can add custom Copilot sub-agents to `~/.copilot/agents`:
 
 ```ts
diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts
index 97ffa187..6d9e74ce 100644
--- a/packages/agent-eval/src/benchmark.test.ts
+++ b/packages/agent-eval/src/benchmark.test.ts
@@ -1,4 +1,4 @@
-import {expect, test} from 'vitest'
+import {afterEach, expect, test, vi} from 'vitest'
 import {
   defineConfig,
   deserialize,
@@ -10,8 +10,22 @@ import {
   type BenchmarkTrialResult,
 } from './benchmark'
 import {VirtualHost} from './host'
+import {run as runPlan} from './plan'
 import {defineConfig as defineScenarioConfig} from './scenario'
 
+vi.mock('./plan', async importOriginal => {
+  const original = await importOriginal<typeof import('./plan')>()
+  return {
+    ...original,
+    run: vi.fn(original.run),
+  }
+})
+
+afterEach(() => {
+  vi.clearAllMocks()
+  vi.restoreAllMocks()
+})
+
 const config = defineConfig({
   name: 'Test benchmark',
   description: 'Tests a benchmark',
@@ -231,6 +245,75 @@ test('run returns an empty result when the benchmark has no trials', async () =>
   ).resolves.toEqual([])
 })
 
+test('run applies global and capability setup to benchmark treatment trials', async () => {
+  const setupOrder: Array<string> = []
+  const benchmarkConfig = defineConfig({
+    name: 'Benchmark with setup',
+    description: 'Runs global and capability setup',
+    models: ['gpt-5.6-sol'],
+    async setup() {
+      setupOrder.push('global')
+    },
+    capabilities: [
+      {
+        name: 'Capability with setup',
+        scenarios: ['001-scenario'],
+        async setup() {
+          setupOrder.push('capability')
+        },
+      },
+    ],
+  })
+  const host = createHost({
+    'with-setup.ts': '',
+  })
+  const loadModule = host.loadModule.bind(host)
+  vi.spyOn(host, 'loadModule').mockImplementation(async filepath => {
+    if (filepath === '/benchmarks/with-setup.ts') {
+      return {
+        benchmark: benchmarkConfig,
+      }
+    }
+
+    return loadModule(filepath)
+  })
+  vi.mocked(runPlan).mockImplementationOnce(async ({plan}) => {
+    const controlTrial = plan.trials.find(trial => {
+      return trial.treatment.name === 'Control'
+    })
+    const benchmarkTrial = plan.trials.find(trial => {
+      return trial.treatment.name === 'Benchmark'
+    })
+
+    expect(controlTrial?.treatment.setup).toBeUndefined()
+    expect(benchmarkTrial?.treatment.setup).toBeDefined()
+
+    const sandbox = await host.createSandbox()
+    await benchmarkTrial?.treatment.setup?.({sandbox})
+
+    return []
+  })
+
+  await expect(
+    run({
+      env: {
+        artifactsDirectory: '/artifacts',
+        benchmarksDirectory: '/benchmarks',
+        concurrency: 1,
+        copilotToken: 'token',
+        dockerImage: 'node:26-slim',
+        experimentsDirectory: '/experiments',
+        outputPath: '/output.json',
+        scenariosDirectory: '/scenarios',
+      },
+      host,
+      id: 'with-setup',
+    }),
+  ).resolves.toEqual([])
+
+  expect(setupOrder).toEqual(['global', 'capability'])
+})
+
 test('output serializes and deserializes benchmark capability metadata', () => {
   const capability = {
     name: 'Test capability',
diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts
index bd2bae37..fafd9ecf 100644
--- a/packages/agent-eval/src/benchmark.ts
+++ b/packages/agent-eval/src/benchmark.ts
@@ -7,7 +7,7 @@ import {logger} from './logger'
 import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model'
 import {create as createPlan, run as runPlan} from './plan'
 import {getScenario, ScenarioSchema, type Scenario} from './scenario'
-import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type TreatmentSetup} from './treatment'
+import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment'
 import {
   getPortableTrialPaths,
   TrialAgentSchema,
@@ -21,6 +21,7 @@ import {TestResultsSchema} from './vitest'
 const CapabilityConfigSchema = z.object({
   name: z.string(),
   scenarios: z.array(z.string()),
+  setup: z.optional(TreatmentSetupSchema),
 })
 
 const BenchmarkConfigSchema = z.object({
@@ -40,6 +41,7 @@ function defineConfig<const Config extends BenchmarkConfig>(config: Config): Con
 type Capability = {
   name: string
   scenarios: Array<Scenario>
+  setup?: TreatmentSetup
 }
 
 type Benchmark = {
@@ -112,6 +114,7 @@ async function listBenchmarks({
         return {
           name: capability.name,
           scenarios,
+          setup: capability.setup,
         }
       }),
     )
@@ -183,13 +186,10 @@ async function run({
     scenariosDirectory: env.scenariosDirectory,
     id,
   })
-  const benchmarkTreatment = {
-    name: 'Benchmark',
-    setup: benchmark.setup,
-  }
   const trialCapabilities = new Map<string, Capability>()
   const trials: Array<Trial> = benchmark.models.flatMap(model => {
     return benchmark.capabilities.flatMap(capability => {
+      const benchmarkTreatment = createBenchmarkTreatment(benchmark, capability)
       return capability.scenarios.flatMap(scenario => {
         return [ControlTreatment, benchmarkTreatment].map(treatment => {
           const trial = {
@@ -224,6 +224,21 @@ async function run({
   })
 }
 
+function createBenchmarkTreatment(benchmark: Benchmark, capability: Capability): Treatment {
+  const setup =
+    benchmark.setup || capability.setup
+      ? async ({sandbox}: Parameters<TreatmentSetup>[0]) => {
+          await benchmark.setup?.({sandbox})
+          await capability.setup?.({sandbox})
+        }
+      : undefined
+
+  return {
+    name: 'Benchmark',
+    setup,
+  }
+}
+
 const CapabilityOutputSchema = z.object({
   name: z.string(),
   scenarioIds: z.array(z.string()),
diff --git a/packages/agent-eval/src/host.test.ts b/packages/agent-eval/src/host.test.ts
index ffb5e99b..8ba72da0 100644
--- a/packages/agent-eval/src/host.test.ts
+++ b/packages/agent-eval/src/host.test.ts
@@ -18,6 +18,9 @@ test('SystemHost provides access to the system filesystem and module loader', as
 
     expect(host.existsSync(textFilepath)).toBe(true)
     await expect(host.fs.readFile(textFilepath, 'utf8')).resolves.toBe('example')
+    await expect(host.loadModule<{value: string}>(moduleFilepath)).resolves.toMatchObject({
+      value: 'loaded',
+    })
     await expect(host.loadModule<{value: string}>(pathToFileURL(moduleFilepath).href)).resolves.toMatchObject({
       value: 'loaded',
     })
diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts
index e70f7c7a..cbf23747 100644
--- a/packages/agent-eval/src/host.ts
+++ b/packages/agent-eval/src/host.ts
@@ -1,5 +1,6 @@
 import {existsSync} from 'node:fs'
 import fs from 'node:fs/promises'
+import {pathToFileURL} from 'node:url'
 import {memfs, Volume, type NestedDirectoryJSON} from 'memfs'
 import {SystemSandbox, VirtualSandbox, type Sandbox, type SandboxCreateOptions} from './sandbox'
 
@@ -26,7 +27,9 @@ class SystemHost implements Host {
   }
 
   loadModule<T = unknown>(filepath: string): Promise<T> {
-    return import(filepath)
+    const specifier =
+      filepath.startsWith('file:') || filepath.startsWith('data:') ? filepath : pathToFileURL(filepath).href
+    return import(specifier)
   }
 
   createSandbox(options?: SandboxCreateOptions): Promise<Sandbox> {