feat: add configurable Copilot runner

packages/agent-eval/README.md

@@ -37,6 +37,10 @@ export const experiment = defineConfig({
   // for you to evaluate their performance
   scenarios: ['uses-button-from-primer'],
 
+  // Optional runner dimension. Defaults to ['copilot-cli'] when omitted.
+  // Include 'copilot-sdk' to compare SDK-driven runs against CLI-driven runs.
+  runners: ['copilot-cli', 'copilot-sdk'],
+
   // An array of treatments. Each treatment is tested and compared against
   // each other and to the control for the experiment. A treatment represents a
   // series of steps to setup the environment that an agent runs within. For
@@ -159,6 +163,7 @@ export const experiment = defineConfig({
   name: 'Example experiment',
   description: 'Compare treatment behavior',
   models: [{name: 'gpt-5.5', reasoningEfforts: ['low', 'medium', 'high']}],
+  runners: ['copilot-cli', 'copilot-sdk'],
   scenarios: ['001-agent-uses-button-from-primer'],
   treatments: [],
 })
@@ -169,6 +174,11 @@ runs once for each configured effort. Model information, including each model's
 supported reasoning efforts, is exported as `models` from
 `@primer/agent-eval`.
 
+Experiment configs may also specify a `runners` array to run each
+model/scenario/treatment combination through multiple Copilot runtimes. When
+omitted, experiments use `copilot-cli`. Add `copilot-sdk` to run through the
+Copilot SDK instead.
+
 Treatment setup can add custom Copilot sub-agents to `~/.copilot/agents`:
 
 ```ts

packages/agent-eval/src/cli.ts

@@ -4,7 +4,7 @@ import {existsSync} from 'node:fs'
 import path from 'node:path'
 import fs from 'node:fs/promises'
 import {parseArgs} from 'node:util'
-import {ControlTreatment, type ExperimentConfig} from './experiment-config'
+import {ControlTreatment, type CopilotRunner, type ExperimentConfig} from './experiment-config'
 import {resolveModelConfigs, type Model, type ReasoningEffort} from './model'
 import {createAgentEvalOutput} from './output'
 import type {Treatment, TreatmentResult} from './treatment'
@@ -167,6 +167,7 @@ function compareResults(a: TreatmentResult, b: TreatmentResult): number {
     a.assistant.premiumRequests - b.assistant.premiumRequests ||
     a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) ||
     a.treatment.config.name.localeCompare(b.treatment.config.name) ||
+    a.treatment.runner.localeCompare(b.treatment.runner) ||
     a.treatment.model.localeCompare(b.treatment.model) ||
     (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') ||
     a.treatment.scenario.id.localeCompare(b.treatment.scenario.id)
@@ -176,6 +177,7 @@ function compareResults(a: TreatmentResult, b: TreatmentResult): number {
 type ResultSummary = {
   experiment: string
   treatment?: string
+  runner?: string
   scenario?: string
   model?: Model
   reasoningEffort?: ReasoningEffort
@@ -190,6 +192,7 @@ type ResultSummary = {
 
 type ResultSummaryValues = {
   treatment?: string
+  runner?: string
   scenario?: string
   model?: Model
   reasoningEffort?: ReasoningEffort
@@ -199,6 +202,7 @@ function createResultSummary(result: TreatmentResult, summaryValues: ResultSumma
   return {
     experiment: result.treatment.experiment.name,
     treatment: summaryValues.treatment,
+    runner: summaryValues.runner,
     scenario: summaryValues.scenario,
     model: summaryValues.model,
     reasoningEffort: summaryValues.reasoningEffort,
@@ -238,6 +242,7 @@ function compareSummaries(a: ResultSummary, b: ResultSummary): number {
     a.premiumRequests - b.premiumRequests ||
     a.experiment.localeCompare(b.experiment) ||
     (a.treatment ?? '').localeCompare(b.treatment ?? '') ||
+    (a.runner ?? '').localeCompare(b.runner ?? '') ||
     (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
     (a.model ?? '').localeCompare(b.model ?? '') ||
     (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '')
@@ -296,6 +301,7 @@ function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValu
   return [
     result.treatment.experiment.name,
     summaryValues.treatment ?? '',
+    summaryValues.runner ?? '',
     summaryValues.scenario ?? '',
     summaryValues.model ?? '',
     summaryValues.reasoningEffort ?? '',
@@ -324,6 +330,7 @@ function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
 
     const treatmentValues = {
       treatment: result.treatment.config.name,
+      runner: result.treatment.runner,
     }
     const treatmentKey = getSummaryKey(result, treatmentValues)
     const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues)
@@ -332,6 +339,7 @@ function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
 
     const scenarioValues = {
       treatment: result.treatment.config.name,
+      runner: result.treatment.runner,
       scenario: result.treatment.scenario.id,
     }
     const scenarioKey = getSummaryKey(result, scenarioValues)
@@ -341,6 +349,7 @@ function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
 
     const modelValues = {
       treatment: result.treatment.config.name,
+      runner: result.treatment.runner,
       scenario: result.treatment.scenario.id,
       model: result.treatment.model,
       reasoningEffort: result.treatment.reasoningEffort,
@@ -364,7 +373,11 @@ function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
             summary,
             scenarios: [...scenarioSummaries.values()]
               .filter(scenarioSummary => {
-                return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment
+                return (
+                  scenarioSummary.experiment === experiment &&
+                  scenarioSummary.treatment === summary.treatment &&
+                  scenarioSummary.runner === summary.runner
+                )
               })
               .toSorted(compareSummaries)
               .map(scenarioSummary => {
@@ -375,6 +388,7 @@ function getResultSummaries(results: Array<TreatmentResult>): ResultHierarchy {
                       return (
                         modelSummary.experiment === experiment &&
                         modelSummary.treatment === summary.treatment &&
+                        modelSummary.runner === summary.runner &&
                         modelSummary.scenario === scenarioSummary.scenario
                       )
                     })
@@ -391,6 +405,7 @@ function formatResultSummaries(results: Array<TreatmentResult>): string {
   const columns = [
     'Experiment',
     'Treatment',
+    'Runner',
     'Scenario',
     'Model',
     'Reasoning Effort',
@@ -433,6 +448,7 @@ function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario
   return {
     Experiment: level === 'treatment' ? summary.experiment : '',
     Treatment: level === 'treatment' ? (summary.treatment ?? '') : '',
+    Runner: level === 'treatment' ? (summary.runner ?? '') : '',
     Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario ?? ''}` : '',
     Model: level === 'model' ? `    ${summary.model ?? ''}` : 'All models',
     'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '',
@@ -458,29 +474,34 @@ const scenarios = await Promise.all(
   }),
 )
 
+const runners: Array<CopilotRunner> = config.runners ?? ['copilot-cli']
 const treatments: Array<Treatment> = config.models.flatMap(modelConfig => {
   return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => {
-    return scenarios.flatMap(scenarioConfig => {
-      return [
-        {
-          config: ControlTreatment,
-          scenario: scenarioConfig,
-          experiment: config,
-          id: randomUUID(),
-          model,
-          reasoningEffort,
-        },
-        ...config.treatments.map(treatment => {
-          return {
-            config: treatment,
+    return runners.flatMap(runner => {
+      return scenarios.flatMap(scenarioConfig => {
+        return [
+          {
+            config: ControlTreatment,
             scenario: scenarioConfig,
             experiment: config,
             id: randomUUID(),
             model,
             reasoningEffort,
-          }
-        }),
-      ]
+            runner,
+          },
+          ...config.treatments.map(treatment => {
+            return {
+              config: treatment,
+              scenario: scenarioConfig,
+              experiment: config,
+              id: randomUUID(),
+              model,
+              reasoningEffort,
+              runner,
+            }
+          }),
+        ]
+      })
     })
   })
 })

packages/agent-eval/src/experiment-config.ts

@@ -18,11 +18,14 @@ type ExperimentConfig = {
   name: string
   description: string
   models: Array<ExperimentModelConfig>
+  runners?: Array<CopilotRunner>
   scenarios: Array<ExperimentScenarioConfig>
   setup?: Setup
   treatments: Array<TreatmentConfig>
 }
 
+type CopilotRunner = 'copilot-cli' | 'copilot-sdk'
+
 type TreatmentConfig = {
   name: string
   setup?: Setup
@@ -35,4 +38,11 @@ const ControlTreatment: TreatmentConfig = {
 }
 
 export {ControlTreatment}
-export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig}
+export type {
+  CopilotRunner,
+  ExperimentConfig,
+  ExperimentScenarioConfig,
+  InlineScenarioConfig,
+  ScenarioConfig,
+  TreatmentConfig,
+}

packages/agent-eval/src/experiment.ts

@@ -1,4 +1,4 @@
-import type {ExperimentConfig, TreatmentConfig} from './experiment-config'
+import type {CopilotRunner, ExperimentConfig, TreatmentConfig} from './experiment-config'
 import type {
   CopilotPluginConfig,
   CopilotPluginSource,
@@ -27,5 +27,6 @@ export type {
   RemoteCopilotPluginSource,
   Sandbox,
   TreatmentConfig,
+  CopilotRunner,
 }
 export {defineConfig}

packages/agent-eval/src/index.ts

@@ -9,6 +9,7 @@ export {models} from './model'
 export type {
   ExperimentConfig,
   ExperimentModelConfig,
+  CopilotRunner,
   Model,
   ModelConfig,
   ModelInfo,

packages/agent-eval/src/output.test.ts

@@ -15,6 +15,7 @@ const output: AgentEvalOutput = {
     name: 'Example',
     description: 'An example experiment',
     models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
+    runners: ['copilot-cli'],
     scenarios: ['example'],
   },
   scenarios: [
@@ -43,6 +44,7 @@ const output: AgentEvalOutput = {
       treatmentId: 'treatment-id',
       model: 'gpt-5.5',
       reasoningEffort: 'high',
+      runner: 'copilot-cli',
       scenarioId: 'example',
       artifacts: {
         copilotConfigPath: '/artifacts/.copilot',
@@ -86,6 +88,7 @@ describe(createAgentEvalOutput, () => {
       name: 'Example',
       description: 'An example experiment',
       models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}],
+      runners: ['copilot-cli'],
       scenarios: ['example'],
       treatments: [],
     }
@@ -101,6 +104,7 @@ describe(createAgentEvalOutput, () => {
         id: 'treatment-id',
         model: 'gpt-5.5',
         reasoningEffort: 'high',
+        runner: 'copilot-cli',
       },
     }
     const duplicateTreatmentResult: TreatmentResult = {

packages/agent-eval/src/output.ts

@@ -1,6 +1,6 @@
 import * as z from 'zod/mini'
 import {MessageSchema, type Message} from './copilot-cli'
-import type {ExperimentConfig, ExperimentScenarioConfig} from './experiment-config'
+import type {CopilotRunner, ExperimentConfig, ExperimentScenarioConfig} from './experiment-config'
 import {models} from './model'
 import type {Model, ReasoningEffort} from './model'
 import type {ResolvedScenario} from './resolve-experiment-scenario'
@@ -11,6 +11,7 @@ type AgentEvalOutputResult = {
   treatmentId: string
   model: Model
   reasoningEffort?: ReasoningEffort
+  runner?: CopilotRunner
   scenarioId: string
   artifacts: {
     copilotConfigPath: string
@@ -53,6 +54,7 @@ type AgentEvalOutput = {
       name: Model
       reasoningEfforts: Array<ReasoningEffort>
     }>
+    runners?: Array<CopilotRunner>
     scenarios: Array<ExperimentScenarioConfig>
   }
   scenarios: Array<ResolvedScenario>
@@ -75,6 +77,7 @@ const ReasoningEffortSchema = z.custom<ReasoningEffort, string>(
   value => typeof value === 'string' && reasoningEfforts.has(value),
   'Expected a supported reasoning effort',
 )
+const CopilotRunnerSchema = z.enum(['copilot-cli', 'copilot-sdk'])
 
 const ExperimentScenarioSchema = z.union([
   z.string(),
@@ -98,6 +101,7 @@ const AgentEvalOutputExperimentSchema = z.object({
   name: z.string(),
   description: z.string(),
   models: z.array(ExperimentModelConfigSchema),
+  runners: z.optional(z.array(CopilotRunnerSchema)),
   scenarios: z.array(ExperimentScenarioSchema),
 })
 
@@ -117,6 +121,7 @@ const AgentEvalOutputResultSchema = z.object({
   treatmentId: z.string(),
   model: ModelSchema,
   reasoningEffort: z.optional(ReasoningEffortSchema),
+  runner: z.optional(CopilotRunnerSchema),
   scenarioId: z.string(),
   artifacts: z.object({
     copilotConfigPath: z.string(),
@@ -207,6 +212,7 @@ function createAgentEvalOutput({
       name: experiment.name,
       description: experiment.description,
       models: experiment.models,
+      runners: experiment.runners,
       scenarios: experiment.scenarios,
     },
     scenarios,
@@ -222,6 +228,7 @@ function createAgentEvalOutput({
         treatmentId: treatment.id,
         model: result.treatment.model,
         reasoningEffort: result.treatment.reasoningEffort,
+        runner: result.treatment.runner,
         scenarioId: result.treatment.scenario.id,
         artifacts: result.artifacts,
         assistant: result.assistant,

packages/agent-eval/src/run.test.ts

@@ -1,5 +1,5 @@
 import {describe, expect, test} from 'vitest'
-import {getCopilotArgs, getVitestConfig} from './run'
+import {getCopilotArgs, getCopilotSdkRunnerScript, getVitestConfig, normalizeCopilotMessage} from './run'
 
 describe('getCopilotArgs', () => {
   test('omits reasoning effort when not configured', () => {
@@ -34,6 +34,42 @@ describe('getCopilotArgs', () => {
   })
 })
 
+describe('getCopilotSdkRunnerScript', () => {
+  test('runs prompts in autopilot mode', () => {
+    expect(getCopilotSdkRunnerScript()).toContain(`agentMode: 'autopilot'`)
+  })
+})
+
+describe('normalizeCopilotMessage', () => {
+  test('normalizes SDK messages for existing Copilot log parsing', () => {
+    expect(
+      normalizeCopilotMessage({
+        type: 'assistant.message',
+        id: 'message-id',
+        timestamp: '2026-01-01T00:00:00.000Z',
+        parentId: null,
+        data: {
+          messageId: 'assistant-message-id',
+          content: 'Done',
+        },
+      }),
+    ).toEqual({
+      type: 'assistant.message',
+      id: 'message-id',
+      timestamp: '2026-01-01T00:00:00.000Z',
+      parentId: '',
+      data: {
+        messageId: 'assistant-message-id',
+        content: 'Done',
+        toolRequests: [],
+        interactionId: '',
+        turnId: '',
+        requestId: '',
+      },
+    })
+  })
+})
+
 describe('getVitestConfig', () => {
   test('configures node tests by default', () => {
     const config = getVitestConfig('test-results.json')