feat: support multi-turn scenarios

.changeset/clear-beans-read.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add support for defining, running, and reporting multi-turn scenarios.

README.md

@@ -66,6 +66,21 @@ export default defineScenario({
 })
 ```
 
+To test context across a conversation, add follow-up `turns`. Each turn names
+the test that must pass before the next prompt is sent:
+
+```ts
+export default defineScenario({
+  prompt: 'Change the button color to blue',
+  turns: [
+    {
+      prompt: 'Actually, make it red instead',
+      test: 'red.test.ts',
+    },
+  ],
+})
+```
+
 ## Authoring experiments
 
 Experiments live in [`./experiments`](./experiments/). Each experiment is a

packages/agent-eval/README.md

@@ -147,6 +147,27 @@ Scenario descriptions and tags are optional. Use `description` to explain what
 the scenario tests. Pass `tags` to `listScenarios` to return only scenarios that
 include every requested tag.
 
+### Multi-turn scenarios
+
+Add `turns` to continue the initial conversation after `scenario.test.ts`
+passes. Each follow-up turn provides a prompt and the test file that verifies
+the agent's changes. An optional browser test can also run for that turn:
+
+```ts
+export default defineScenario({
+  prompt: 'Change the button color to blue',
+  turns: [
+    {
+      prompt: 'Actually, make it red instead',
+      test: 'red.test.ts',
+      browserTest: 'red.browser.test.ts',
+    },
+  ],
+})
+```
+
+The evaluator stops after a failed turn instead of sending later prompts.
+
 ## Experiment config authoring
 
 Use `defineConfig` from `@primer/agent-eval/experiment` to keep local experiment

packages/agent-eval/src/experiment-config.ts

@@ -5,6 +5,13 @@ type ScenarioConfig = {
   description?: string
   prompt: string
   tags?: Array<string>
+  turns?: Array<ScenarioTurnConfig>
+}
+
+type ScenarioTurnConfig = {
+  prompt: string
+  test: string
+  browserTest?: string
 }
 
 type InlineScenarioConfig = {
@@ -35,4 +42,11 @@ const ControlTreatment: TreatmentConfig = {
 }
 
 export {ControlTreatment}
-export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig}
+export type {
+  ExperimentConfig,
+  ExperimentScenarioConfig,
+  InlineScenarioConfig,
+  ScenarioConfig,
+  ScenarioTurnConfig,
+  TreatmentConfig,
+}

packages/agent-eval/src/output.ts

@@ -107,9 +107,27 @@ const ResolvedScenarioSchema = z.object({
   config: z.object({
     description: z.optional(z.string()),
     prompt: z.string(),
+    turns: z.optional(
+      z.array(
+        z.object({
+          prompt: z.string(),
+          test: z.string(),
+          browserTest: z.optional(z.string()),
+        }),
+      ),
+    ),
   }),
   testPath: z.string(),
   browserTestPath: z.optional(z.string()),
+  turns: z.optional(
+    z.array(
+      z.object({
+        prompt: z.string(),
+        testPath: z.string(),
+        browserTestPath: z.optional(z.string()),
+      }),
+    ),
+  ),
 })
 
 const AgentEvalOutputResultSchema = z.object({

packages/agent-eval/src/run.test.ts

@@ -32,6 +32,28 @@ describe('getCopilotArgs', () => {
       'json',
     ])
   })
+
+  test('resumes the previous session for a follow-up turn', () => {
+    expect(
+      getCopilotArgs({
+        prompt: 'Actually, make it red',
+        model: 'gpt-5.5',
+        sessionId: 'session-id',
+      }),
+    ).toEqual([
+      '-p',
+      'Actually, make it red',
+      '--model',
+      'gpt-5.5',
+      '--allow-all',
+      '--resume',
+      'session-id',
+      '--mode',
+      'autopilot',
+      '--output-format',
+      'json',
+    ])
+  })
 })
 
 describe('getVitestConfig', () => {