From 63a3444b2d918e0a519d1b6f0a2f06e258adcaf9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:20 +0000 Subject: [PATCH 1/4] Apply remaining changes Co-authored-by: joshblack <3901764+joshblack@users.noreply.github.com> --- scenarios/001-agent-uses-button-from-primer/next-env.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scenarios/001-agent-uses-button-from-primer/next-env.d.ts b/scenarios/001-agent-uses-button-from-primer/next-env.d.ts index 9edff1c..ce4e94a 100644 --- a/scenarios/001-agent-uses-button-from-primer/next-env.d.ts +++ b/scenarios/001-agent-uses-button-from-primer/next-env.d.ts @@ -1,6 +1,7 @@ /// /// import "./.next/types/routes.d.ts"; +import "./.next/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. From 05887ddfd85fb28eeb4dbdf2f50fb9bb55c7d385 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:01:51 +0000 Subject: [PATCH 2/4] Add Copilot runner dimension Co-authored-by: joshblack <3901764+joshblack@users.noreply.github.com> --- packages/agent-eval/README.md | 10 + packages/agent-eval/src/cli.ts | 57 ++-- packages/agent-eval/src/experiment-config.ts | 5 +- packages/agent-eval/src/experiment.ts | 3 +- packages/agent-eval/src/index.ts | 1 + packages/agent-eval/src/output.test.ts | 4 + packages/agent-eval/src/output.ts | 9 +- packages/agent-eval/src/run.test.ts | 38 ++- packages/agent-eval/src/run.ts | 306 ++++++++++++++++++- packages/agent-eval/src/sandbox.ts | 2 +- packages/agent-eval/src/treatment.ts | 3 +- 11 files changed, 398 insertions(+), 40 deletions(-) diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 3556a3f..0e7fa37 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -37,6 +37,10 @@ export const experiment = defineConfig({ // for you to evaluate their performance scenarios: ['uses-button-from-primer'], + // Optional runner dimension. Defaults to ['copilot-cli'] when omitted. + // Include 'copilot-sdk' to compare SDK-driven runs against CLI-driven runs. + runners: ['copilot-cli', 'copilot-sdk'], + // An array of treatments. Each treatment is tested and compared against // each other and to the control for the experiment. A treatment represents a // series of steps to setup the environment that an agent runs within. For @@ -159,6 +163,7 @@ export const experiment = defineConfig({ name: 'Example experiment', description: 'Compare treatment behavior', models: [{name: 'gpt-5.5', reasoningEfforts: ['low', 'medium', 'high']}], + runners: ['copilot-cli', 'copilot-sdk'], scenarios: ['001-agent-uses-button-from-primer'], treatments: [], }) @@ -169,6 +174,11 @@ runs once for each configured effort. Model information, including each model's supported reasoning efforts, is exported as `models` from `@primer/agent-eval`. +Experiment configs may also specify a `runners` array to run each +model/scenario/treatment combination through multiple Copilot runtimes. When +omitted, experiments use `copilot-cli`. Add `copilot-sdk` to run through the +Copilot SDK instead. + Treatment setup can add custom Copilot sub-agents to `~/.copilot/agents`: ```ts diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index e56a7ff..350ca0b 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -4,7 +4,7 @@ import {existsSync} from 'node:fs' import path from 'node:path' import fs from 'node:fs/promises' import {parseArgs} from 'node:util' -import {ControlTreatment, type ExperimentConfig} from './experiment-config' +import {ControlTreatment, type CopilotRunner, type ExperimentConfig} from './experiment-config' import {resolveModelConfigs, type Model, type ReasoningEffort} from './model' import {createAgentEvalOutput} from './output' import type {Treatment, TreatmentResult} from './treatment' @@ -167,6 +167,7 @@ function compareResults(a: TreatmentResult, b: TreatmentResult): number { a.assistant.premiumRequests - b.assistant.premiumRequests || a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) || a.treatment.config.name.localeCompare(b.treatment.config.name) || + a.treatment.runner.localeCompare(b.treatment.runner) || a.treatment.model.localeCompare(b.treatment.model) || (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') || a.treatment.scenario.id.localeCompare(b.treatment.scenario.id) @@ -176,6 +177,7 @@ function compareResults(a: TreatmentResult, b: TreatmentResult): number { type ResultSummary = { experiment: string treatment?: string + runner?: string scenario?: string model?: Model reasoningEffort?: ReasoningEffort @@ -190,6 +192,7 @@ type ResultSummary = { type ResultSummaryValues = { treatment?: string + runner?: string scenario?: string model?: Model reasoningEffort?: ReasoningEffort @@ -199,6 +202,7 @@ function createResultSummary(result: TreatmentResult, summaryValues: ResultSumma return { experiment: result.treatment.experiment.name, treatment: summaryValues.treatment, + runner: summaryValues.runner, scenario: summaryValues.scenario, model: summaryValues.model, reasoningEffort: summaryValues.reasoningEffort, @@ -238,6 +242,7 @@ function compareSummaries(a: ResultSummary, b: ResultSummary): number { a.premiumRequests - b.premiumRequests || a.experiment.localeCompare(b.experiment) || (a.treatment ?? '').localeCompare(b.treatment ?? '') || + (a.runner ?? '').localeCompare(b.runner ?? '') || (a.scenario ?? '').localeCompare(b.scenario ?? '') || (a.model ?? '').localeCompare(b.model ?? '') || (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '') @@ -296,6 +301,7 @@ function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValu return [ result.treatment.experiment.name, summaryValues.treatment ?? '', + summaryValues.runner ?? '', summaryValues.scenario ?? '', summaryValues.model ?? '', summaryValues.reasoningEffort ?? '', @@ -324,6 +330,7 @@ function getResultSummaries(results: Array): ResultHierarchy { const treatmentValues = { treatment: result.treatment.config.name, + runner: result.treatment.runner, } const treatmentKey = getSummaryKey(result, treatmentValues) const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues) @@ -332,6 +339,7 @@ function getResultSummaries(results: Array): ResultHierarchy { const scenarioValues = { treatment: result.treatment.config.name, + runner: result.treatment.runner, scenario: result.treatment.scenario.id, } const scenarioKey = getSummaryKey(result, scenarioValues) @@ -341,6 +349,7 @@ function getResultSummaries(results: Array): ResultHierarchy { const modelValues = { treatment: result.treatment.config.name, + runner: result.treatment.runner, scenario: result.treatment.scenario.id, model: result.treatment.model, reasoningEffort: result.treatment.reasoningEffort, @@ -364,7 +373,11 @@ function getResultSummaries(results: Array): ResultHierarchy { summary, scenarios: [...scenarioSummaries.values()] .filter(scenarioSummary => { - return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment + return ( + scenarioSummary.experiment === experiment && + scenarioSummary.treatment === summary.treatment && + scenarioSummary.runner === summary.runner + ) }) .toSorted(compareSummaries) .map(scenarioSummary => { @@ -375,6 +388,7 @@ function getResultSummaries(results: Array): ResultHierarchy { return ( modelSummary.experiment === experiment && modelSummary.treatment === summary.treatment && + modelSummary.runner === summary.runner && modelSummary.scenario === scenarioSummary.scenario ) }) @@ -391,6 +405,7 @@ function formatResultSummaries(results: Array): string { const columns = [ 'Experiment', 'Treatment', + 'Runner', 'Scenario', 'Model', 'Reasoning Effort', @@ -433,6 +448,7 @@ function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario return { Experiment: level === 'treatment' ? summary.experiment : '', Treatment: level === 'treatment' ? (summary.treatment ?? '') : '', + Runner: level === 'treatment' ? (summary.runner ?? '') : '', Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? ` ${summary.scenario ?? ''}` : '', Model: level === 'model' ? ` ${summary.model ?? ''}` : 'All models', 'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '', @@ -458,29 +474,34 @@ const scenarios = await Promise.all( }), ) +const runners: Array = config.runners ?? ['copilot-cli'] const treatments: Array = config.models.flatMap(modelConfig => { return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => { - return scenarios.flatMap(scenarioConfig => { - return [ - { - config: ControlTreatment, - scenario: scenarioConfig, - experiment: config, - id: randomUUID(), - model, - reasoningEffort, - }, - ...config.treatments.map(treatment => { - return { - config: treatment, + return runners.flatMap(runner => { + return scenarios.flatMap(scenarioConfig => { + return [ + { + config: ControlTreatment, scenario: scenarioConfig, experiment: config, id: randomUUID(), model, reasoningEffort, - } - }), - ] + runner, + }, + ...config.treatments.map(treatment => { + return { + config: treatment, + scenario: scenarioConfig, + experiment: config, + id: randomUUID(), + model, + reasoningEffort, + runner, + } + }), + ] + }) }) }) }) diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts index 6ccbea1..b012512 100644 --- a/packages/agent-eval/src/experiment-config.ts +++ b/packages/agent-eval/src/experiment-config.ts @@ -18,11 +18,14 @@ type ExperimentConfig = { name: string description: string models: Array + runners?: Array scenarios: Array setup?: Setup treatments: Array } +type CopilotRunner = 'copilot-cli' | 'copilot-sdk' + type TreatmentConfig = { name: string setup?: Setup @@ -35,4 +38,4 @@ const ControlTreatment: TreatmentConfig = { } export {ControlTreatment} -export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig} +export type {CopilotRunner, ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig} diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts index 5cbf7f2..f6c8ee3 100644 --- a/packages/agent-eval/src/experiment.ts +++ b/packages/agent-eval/src/experiment.ts @@ -1,4 +1,4 @@ -import type {ExperimentConfig, TreatmentConfig} from './experiment-config' +import type {CopilotRunner, ExperimentConfig, TreatmentConfig} from './experiment-config' import type { CopilotPluginConfig, CopilotPluginSource, @@ -27,5 +27,6 @@ export type { RemoteCopilotPluginSource, Sandbox, TreatmentConfig, + CopilotRunner, } export {defineConfig} diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index 8e0a2f1..7a96dcf 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -9,6 +9,7 @@ export {models} from './model' export type { ExperimentConfig, ExperimentModelConfig, + CopilotRunner, Model, ModelConfig, ModelInfo, diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts index eba25fa..31a037d 100644 --- a/packages/agent-eval/src/output.test.ts +++ b/packages/agent-eval/src/output.test.ts @@ -15,6 +15,7 @@ const output: AgentEvalOutput = { name: 'Example', description: 'An example experiment', models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], + runners: ['copilot-cli'], scenarios: ['example'], }, scenarios: [ @@ -43,6 +44,7 @@ const output: AgentEvalOutput = { treatmentId: 'treatment-id', model: 'gpt-5.5', reasoningEffort: 'high', + runner: 'copilot-cli', scenarioId: 'example', artifacts: { copilotConfigPath: '/artifacts/.copilot', @@ -86,6 +88,7 @@ describe(createAgentEvalOutput, () => { name: 'Example', description: 'An example experiment', models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], + runners: ['copilot-cli'], scenarios: ['example'], treatments: [], } @@ -101,6 +104,7 @@ describe(createAgentEvalOutput, () => { id: 'treatment-id', model: 'gpt-5.5', reasoningEffort: 'high', + runner: 'copilot-cli', }, } const duplicateTreatmentResult: TreatmentResult = { diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts index f321971..bc13177 100644 --- a/packages/agent-eval/src/output.ts +++ b/packages/agent-eval/src/output.ts @@ -1,6 +1,6 @@ import * as z from 'zod/mini' import {MessageSchema, type Message} from './copilot-cli' -import type {ExperimentConfig, ExperimentScenarioConfig} from './experiment-config' +import type {CopilotRunner, ExperimentConfig, ExperimentScenarioConfig} from './experiment-config' import {models} from './model' import type {Model, ReasoningEffort} from './model' import type {ResolvedScenario} from './resolve-experiment-scenario' @@ -11,6 +11,7 @@ type AgentEvalOutputResult = { treatmentId: string model: Model reasoningEffort?: ReasoningEffort + runner?: CopilotRunner scenarioId: string artifacts: { copilotConfigPath: string @@ -53,6 +54,7 @@ type AgentEvalOutput = { name: Model reasoningEfforts: Array }> + runners?: Array scenarios: Array } scenarios: Array @@ -75,6 +77,7 @@ const ReasoningEffortSchema = z.custom( value => typeof value === 'string' && reasoningEfforts.has(value), 'Expected a supported reasoning effort', ) +const CopilotRunnerSchema = z.enum(['copilot-cli', 'copilot-sdk']) const ExperimentScenarioSchema = z.union([ z.string(), @@ -98,6 +101,7 @@ const AgentEvalOutputExperimentSchema = z.object({ name: z.string(), description: z.string(), models: z.array(ExperimentModelConfigSchema), + runners: z.optional(z.array(CopilotRunnerSchema)), scenarios: z.array(ExperimentScenarioSchema), }) @@ -117,6 +121,7 @@ const AgentEvalOutputResultSchema = z.object({ treatmentId: z.string(), model: ModelSchema, reasoningEffort: z.optional(ReasoningEffortSchema), + runner: z.optional(CopilotRunnerSchema), scenarioId: z.string(), artifacts: z.object({ copilotConfigPath: z.string(), @@ -207,6 +212,7 @@ function createAgentEvalOutput({ name: experiment.name, description: experiment.description, models: experiment.models, + runners: experiment.runners, scenarios: experiment.scenarios, }, scenarios, @@ -222,6 +228,7 @@ function createAgentEvalOutput({ treatmentId: treatment.id, model: result.treatment.model, reasoningEffort: result.treatment.reasoningEffort, + runner: result.treatment.runner, scenarioId: result.treatment.scenario.id, artifacts: result.artifacts, assistant: result.assistant, diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts index fb9abf6..aa1b45c 100644 --- a/packages/agent-eval/src/run.test.ts +++ b/packages/agent-eval/src/run.test.ts @@ -1,5 +1,5 @@ import {describe, expect, test} from 'vitest' -import {getCopilotArgs, getVitestConfig} from './run' +import {getCopilotArgs, getCopilotSdkRunnerScript, getVitestConfig, normalizeCopilotMessage} from './run' describe('getCopilotArgs', () => { test('omits reasoning effort when not configured', () => { @@ -34,6 +34,42 @@ describe('getCopilotArgs', () => { }) }) +describe('getCopilotSdkRunnerScript', () => { + test('runs prompts in autopilot mode', () => { + expect(getCopilotSdkRunnerScript()).toContain(`agentMode: 'autopilot'`) + }) +}) + +describe('normalizeCopilotMessage', () => { + test('normalizes SDK messages for existing Copilot log parsing', () => { + expect( + normalizeCopilotMessage({ + type: 'assistant.message', + id: 'message-id', + timestamp: '2026-01-01T00:00:00.000Z', + parentId: null, + data: { + messageId: 'assistant-message-id', + content: 'Done', + }, + }), + ).toEqual({ + type: 'assistant.message', + id: 'message-id', + timestamp: '2026-01-01T00:00:00.000Z', + parentId: '', + data: { + messageId: 'assistant-message-id', + content: 'Done', + toolRequests: [], + interactionId: '', + turnId: '', + requestId: '', + }, + }) + }) +}) + describe('getVitestConfig', () => { test('configures node tests by default', () => { const config = getVitestConfig('test-results.json') diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts index 384a73d..6483b56 100644 --- a/packages/agent-eval/src/run.ts +++ b/packages/agent-eval/src/run.ts @@ -1,13 +1,18 @@ import {randomUUID} from 'node:crypto' import path from 'node:path' import fs from 'node:fs/promises' -import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, Sandbox} from './sandbox' +import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, NPM_GLOBAL_DIR, Sandbox} from './sandbox' import type {Treatment, TreatmentResult} from './treatment' +import type {CopilotRunner} from './experiment-config' import type {Model, ReasoningEffort} from './model' import {isMessageType, parseMessage, type Message} from './copilot-cli' import {getTestMetadata, parseTestResults} from './vitest' const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright' +const COPILOT_SDK_VERSION = '1.0.11' +const COPILOT_SDK_RUNNER_PATH = '/tmp/agent-eval-copilot-sdk-runner.cjs' +const COPILOT_SDK_RUNNER_CONFIG_PATH = '/tmp/agent-eval-copilot-sdk-runner-config.json' +const NPM_GLOBAL_NODE_MODULES = path.posix.join(NPM_GLOBAL_DIR, 'lib/node_modules') type RunOptions = { artifactsDirectory: string @@ -140,6 +145,286 @@ function getCopilotArgs({ return [...args, '--mode', 'autopilot', '--output-format', 'json'] } +function normalizeCopilotMessage(message: Record): Record { + const normalized = { + ...message, + parentId: message.parentId ?? '', + } + const data = (typeof normalized.data === 'object' && normalized.data !== null ? normalized.data : {}) as Record< + string, + unknown + > + + switch (normalized.type) { + case 'user.message': + normalized.data = { + content: '', + transformedContent: '', + attachments: [], + supportedNativeDocumentMimeTypes: [], + agentMode: '', + interactionId: '', + parentAgentTaskId: '', + ...data, + } + break + case 'assistant.message': + normalized.data = { + toolRequests: [], + interactionId: '', + turnId: '', + requestId: '', + ...data, + } + break + case 'assistant.turn_start': + normalized.data = { + interactionId: '', + ...data, + } + break + case 'assistant.tool_call_delta': + normalized.data = { + toolName: '', + ...data, + } + break + case 'tool.execution_start': + normalized.data = { + arguments: {}, + turnId: '', + model: '', + ...data, + } + break + case 'tool.execution_complete': + normalized.data = data.success + ? { + interactionId: '', + turnId: '', + model: '', + result: { + content: '', + detailedContent: '', + ...((typeof data.result === 'object' && data.result !== null ? data.result : {}) as Record), + }, + toolTelemetry: {}, + ...data, + } + : { + interactionId: '', + turnId: '', + model: '', + error: { + message: '', + code: '', + ...((typeof data.error === 'object' && data.error !== null ? data.error : {}) as Record), + }, + toolTelemetry: {}, + ...data, + } + break + case 'session.task_complete': + normalized.data = { + summary: '', + success: false, + ...data, + } + break + } + + return normalized +} + +function parseCopilotOutput(output: string): Array { + return output.split('\n').flatMap(line => { + const trimmed = line.trim() + if (trimmed.length === 0) { + return [] + } + return parseMessage(normalizeCopilotMessage(JSON.parse(trimmed))) + }) +} + +function getCopilotSdkRunnerScript(): string { + return ` +const fs = require('node:fs/promises') +const {CopilotClient, approveAll} = require('@github/copilot-sdk') + +function normalizeEvent(event) { + return { + ...event, + parentId: event.parentId ?? '', + } +} + +function emit(event) { + console.log(JSON.stringify(normalizeEvent(event))) +} + +async function main() { + const config = JSON.parse(await fs.readFile(process.argv[2], 'utf8')) + const startedAt = Date.now() + let sessionId = '' + let totalApiDurationMs = 0 + let codeChanges = { + linesAdded: 0, + linesRemoved: 0, + filesModified: [], + } + + const client = new CopilotClient({ + workingDirectory: process.cwd(), + baseDirectory: config.copilotHome, + gitHubToken: process.env.COPILOT_GITHUB_TOKEN, + useLoggedInUser: false, + logLevel: 'none', + }) + + await client.start() + try { + const sessionConfig = { + model: config.model, + onPermissionRequest: approveAll, + } + + if (config.reasoningEffort) { + sessionConfig.reasoningEffort = config.reasoningEffort + } + + const session = await client.createSession(sessionConfig) + sessionId = session.sessionId + session.on(event => { + if (event.type === 'assistant.usage') { + totalApiDurationMs += event.data.duration ?? 0 + } + + if (event.type === 'session.shutdown') { + totalApiDurationMs = event.data.totalApiDurationMs ?? totalApiDurationMs + codeChanges = event.data.codeChanges ?? codeChanges + } + + emit(event) + }) + + await session.sendAndWait({ + prompt: config.prompt, + agentMode: 'autopilot', + }, config.timeoutMs) + await session.disconnect() + } finally { + const errors = await client.stop() + if (errors.length > 0) { + throw new Error(errors.map(error => error.message).join('\\n')) + } + } + + emit({ + type: 'result', + timestamp: new Date().toISOString(), + sessionId, + exitCode: 0, + usage: { + premiumRequests: 0, + totalApiDurationMs, + sessionDurationMs: Date.now() - startedAt, + codeChanges, + }, + }) +} + +main().catch(error => { + console.error(error?.stack ?? String(error)) + process.exit(1) +}) +` +} + +async function runCopilotSdk({ + sandbox, + prompt, + model, + reasoningEffort, + copilotToken, +}: { + sandbox: Sandbox + prompt: string + model: Model + reasoningEffort?: ReasoningEffort + copilotToken: string +}): Promise> { + console.log('Installing copilot sdk...') + await sandbox.runCommand('npm', ['install', '-g', `@github/copilot-sdk@${COPILOT_SDK_VERSION}`], { + user: NODE_USER, + }) + await sandbox.writeFile(COPILOT_SDK_RUNNER_PATH, getCopilotSdkRunnerScript()) + await sandbox.writeFile( + COPILOT_SDK_RUNNER_CONFIG_PATH, + JSON.stringify({ + copilotHome: COPILOT_DIR, + model, + prompt, + reasoningEffort, + timeoutMs: 60 * 60 * 1000, + }), + ) + const copilotOutput = await sandbox.runCommand('node', [COPILOT_SDK_RUNNER_PATH, COPILOT_SDK_RUNNER_CONFIG_PATH], { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + NODE_PATH: NPM_GLOBAL_NODE_MODULES, + }, + }) + + return parseCopilotOutput(copilotOutput.stdout) +} + +async function runCopilotCli({ + sandbox, + prompt, + model, + reasoningEffort, + copilotToken, +}: { + sandbox: Sandbox + prompt: string + model: Model + reasoningEffort?: ReasoningEffort + copilotToken: string +}): Promise> { + const args = getCopilotArgs({ + prompt, + model, + reasoningEffort, + }) + const copilotOutput = await sandbox.runCommand('copilot', args, { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + }) + + return parseCopilotOutput(copilotOutput.stdout) +} + +async function runCopilot( + runner: CopilotRunner, + options: { + sandbox: Sandbox + prompt: string + model: Model + reasoningEffort?: ReasoningEffort + copilotToken: string + }, +): Promise> { + switch (runner) { + case 'copilot-cli': + return runCopilotCli(options) + case 'copilot-sdk': + return runCopilotSdk(options) + } +} + async function runTreatment( treatment: Treatment, {artifactsDirectory, copilotToken, dockerImage}: RunTreatmentOptions, @@ -209,23 +494,12 @@ async function runTreatment( console.log('Running copilot...') const {prompt} = treatment.scenario.config - const args = getCopilotArgs({ + const messages = await runCopilot(treatment.runner, { + sandbox, prompt, model: treatment.model, reasoningEffort: treatment.reasoningEffort, - }) - const copilotOutput = await sandbox.runCommand('copilot', args, { - user: NODE_USER, - env: { - COPILOT_GITHUB_TOKEN: copilotToken, - }, - }) - const messages: Array = copilotOutput.stdout.split('\n').flatMap(line => { - const trimmed = line.trim() - if (trimmed.length === 0) { - return [] - } - return parseMessage(JSON.parse(trimmed)) + copilotToken, }) const TEST_PATH = 'scenario.test.ts' @@ -384,4 +658,4 @@ async function runTreatment( } } -export {getCopilotArgs, getVitestConfig, run} +export {getCopilotArgs, getCopilotSdkRunnerScript, getVitestConfig, normalizeCopilotMessage, run} diff --git a/packages/agent-eval/src/sandbox.ts b/packages/agent-eval/src/sandbox.ts index 477a956..2933ac7 100644 --- a/packages/agent-eval/src/sandbox.ts +++ b/packages/agent-eval/src/sandbox.ts @@ -772,7 +772,7 @@ function captureStream(destination: NodeJS.WritableStream): {stream: Writable; r } } -export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox} +export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, NPM_GLOBAL_DIR, Sandbox} export type { AgentSkillCopiedFile, AgentSkillFile, diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts index 5463dc1..6843fe2 100644 --- a/packages/agent-eval/src/treatment.ts +++ b/packages/agent-eval/src/treatment.ts @@ -1,4 +1,4 @@ -import type {ExperimentConfig, TreatmentConfig} from './experiment-config' +import type {CopilotRunner, ExperimentConfig, TreatmentConfig} from './experiment-config' import type {Model, ReasoningEffort} from './model' import type {Message} from './copilot-cli' import type {ResolvedScenario} from './resolve-experiment-scenario' @@ -10,6 +10,7 @@ type Treatment = { id: string model: Model reasoningEffort?: ReasoningEffort + runner: CopilotRunner } type TreatmentResult = { From 60ffdca8a904f44c428a04e6d1364cd3ce53eac8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:02:26 +0000 Subject: [PATCH 3/4] Implement Copilot SDK runner Co-authored-by: joshblack <3901764+joshblack@users.noreply.github.com> --- packages/agent-eval/src/experiment-config.ts | 9 ++++++++- packages/agent-eval/src/run.ts | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts index b012512..1b54cf8 100644 --- a/packages/agent-eval/src/experiment-config.ts +++ b/packages/agent-eval/src/experiment-config.ts @@ -38,4 +38,11 @@ const ControlTreatment: TreatmentConfig = { } export {ControlTreatment} -export type {CopilotRunner, ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig} +export type { + CopilotRunner, + ExperimentConfig, + ExperimentScenarioConfig, + InlineScenarioConfig, + ScenarioConfig, + TreatmentConfig, +} diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts index 6483b56..64dfa38 100644 --- a/packages/agent-eval/src/run.ts +++ b/packages/agent-eval/src/run.ts @@ -206,7 +206,10 @@ function normalizeCopilotMessage(message: Record): Record), + ...((typeof data.result === 'object' && data.result !== null ? data.result : {}) as Record< + string, + unknown + >), }, toolTelemetry: {}, ...data, From 52236ba00b65704487ef5228a1f5324d7f3a570c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:18:42 +0000 Subject: [PATCH 4/4] Fix SDK runner type checks Co-authored-by: joshblack <3901764+joshblack@users.noreply.github.com> --- packages/agent-eval/src/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts index 64dfa38..7989727 100644 --- a/packages/agent-eval/src/run.ts +++ b/packages/agent-eval/src/run.ts @@ -146,7 +146,7 @@ function getCopilotArgs({ } function normalizeCopilotMessage(message: Record): Record { - const normalized = { + const normalized: Record = { ...message, parentId: message.parentId ?? '', }