diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py new file mode 100644 index 000000000..7f1b28428 --- /dev/null +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -0,0 +1,58 @@ +"""Harness-neutral contract between evaluation pipelines and the agents they run.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from pydantic import BaseModel, ConfigDict + +from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.metrics import AgentMetrics + +__all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness"] + + +class AgentIdentity[N: str = str](BaseModel): + """What is being evaluated, resolved before any task runs. + + ``N`` lets applications type the harness name, e.g. with their own ``StrEnum`` of harnesses. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + harness: N + # CLI version, engine commit, or hosted agent version + version: str + # Configured model; None when the harness pins or delegates model selection + model: str | None = None + + +@dataclass(frozen=True) +class AgentRequest[T]: + task: T + # The agent's workspace: a prepared repository, or an empty directory for answer-only tasks + work_dir: Path + # Receives logs and raw captures + output_dir: Path + # None uses the harness's default + timeout: int | None = None + + +@dataclass(frozen=True) +class AgentResult: + # Final answer; None when the output is left in the workspace + response: str | None = None + metrics: AgentMetrics | None = None + experiment: ExperimentConfiguration | None = None + + +class Harness[T, N: str = str](Protocol): + """Runs an agent on tasks of type ``T``, identified by a harness name of type ``N``. + + ``identity`` checks the harness is usable and raises ``AgentError`` otherwise. ``run`` raises + ``AgentTimeoutError`` when the timeout is exceeded and ``AgentError`` for any other failure. + """ + + def identity(self) -> AgentIdentity[N]: ... + + def run(self, request: AgentRequest[T], /) -> AgentResult: ... diff --git a/src/bcbench/agent/__init__.py b/src/bcbench/agent/__init__.py index 4f2a4e0b6..9a08648e6 100644 --- a/src/bcbench/agent/__init__.py +++ b/src/bcbench/agent/__init__.py @@ -2,7 +2,7 @@ from bcbench.agent.bcal import BCalBackendConfig, get_bcal_version, run_bcal_agent from bcbench.agent.claude import get_claude_version, run_claude_code -from bcbench.agent.copilot import run_copilot_agent +from bcbench.agent.copilot import CopilotHarness from bcbench.agent.pr_review import get_pr_review_version, run_pr_review_agent -__all__ = ["BCalBackendConfig", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_copilot_agent", "run_pr_review_agent"] +__all__ = ["BCalBackendConfig", "CopilotHarness", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_pr_review_agent"] diff --git a/src/bcbench/agent/bcal/agent.py b/src/bcbench/agent/bcal/agent.py index 24ff0005a..8764ddb46 100644 --- a/src/bcbench/agent/bcal/agent.py +++ b/src/bcbench/agent/bcal/agent.py @@ -7,6 +7,7 @@ from pathlib import Path from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.artifacts import ALPACKAGES_DIRNAME @@ -124,7 +125,7 @@ def run_bcal_agent( repo_path: Path, backend_config: BCalBackendConfig, timeout: int = 25 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: +) -> AgentResult: logger.info(f"Running bcal CLI on: {entry.instance_id}") # The .alpackages dir is created by the NL2AL pipeline setup step @@ -151,7 +152,7 @@ def run_bcal_agent( execution_time = time.monotonic() - start logger.info(f"bcal CLI run complete for: {entry.instance_id}") - return AgentMetrics(execution_time=execution_time), ExperimentConfiguration() + return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration()) except subprocess.TimeoutExpired: logger.exception(f"bcal CLI timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 8c9f34d24..c04cd5888 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -5,6 +5,7 @@ from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.exceptions import AgentError, AgentTimeoutError @@ -39,12 +40,8 @@ def run_claude_code( plugin_root: Path, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run Claude Code on a single dataset entry. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used - """ +) -> AgentResult: + """Run Claude Code on a single dataset entry.""" claude_cmd = shutil.which("claude") if not claude_cmd: raise AgentError("Claude Code not found in PATH. Please ensure it is installed and available.") @@ -145,7 +142,7 @@ def run_claude_code( stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else "" logger.debug(f"Claude Code raw output: {stdout}") - metrics, _ = parse_stream_output(stdout.splitlines(), log_transcript=True) + metrics, response = parse_stream_output(stdout.splitlines(), log_transcript=True) except subprocess.TimeoutExpired: logger.exception(f"Claude Code timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) @@ -157,7 +154,7 @@ def run_claude_code( logger.exception("Unexpected error running Claude Code") raise else: - return metrics, config + return AgentResult(response=response, metrics=metrics, experiment=config) finally: if bc_gateway is not None: bc_gateway.stop() diff --git a/src/bcbench/agent/copilot/__init__.py b/src/bcbench/agent/copilot/__init__.py index 40648ec39..340415886 100644 --- a/src/bcbench/agent/copilot/__init__.py +++ b/src/bcbench/agent/copilot/__init__.py @@ -1,5 +1,5 @@ """GitHub Copilot CLI agent module.""" -from bcbench.agent.copilot.agent import run_copilot_agent +from bcbench.agent.copilot.agent import CopilotHarness -__all__ = ["run_copilot_agent"] +__all__ = ["CopilotHarness"] diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index c95a530bb..2d23d72e0 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -1,12 +1,14 @@ """GitHub Copilot CLI Agent implementation.""" import logging +from dataclasses import dataclass from pathlib import Path +from typing import override -from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot +from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, get_copilot_version, invoke_copilot from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult, Harness from bcbench_core.exceptions import AgentTimeoutError from bcbench.agent.shared import ( @@ -24,87 +26,90 @@ logger = logging.getLogger(__name__) -def run_copilot_agent( - entry: BaseDatasetEntry, - model: str, - category: EvaluationCategory, - repo_path: Path, - output_dir: Path, - agent_config: AgentConfig, - plugin_root: Path, - runtime: AgentRuntimeConfig | None = None, - timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run GitHub Copilot CLI agent on a single dataset entry. +@dataclass(frozen=True) +class CopilotHarness(Harness[BaseDatasetEntry, AgentHarness]): + """GitHub Copilot CLI on any category, customized through the agent config.""" - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment - """ - logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") + model: str + category: EvaluationCategory + agent_config: AgentConfig + plugin_root: Path + runtime: AgentRuntimeConfig | None = None + timeout: int = 60 * 60 - prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) - bc_gateway = start_bc_mcp_gateway(runtime) - mcp_config_json, mcp_server_names = build_mcp_config( - agent_config, - entry, - repo_path, - runtime=runtime, - bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, - ) - lsp_plugin_dir: Path | None = build_al_lsp_plugin( - entry, - category, - repo_path, - AgentHarness.COPILOT, - plugin_root, - runtime=runtime, - ) - profile_dir: Path = instructions_profile_dir(entry.customization_profile) - target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path) - instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename) - skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir) - custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir) - plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True) + @override + def identity(self) -> AgentIdentity[AgentHarness]: + return AgentIdentity[AgentHarness](harness=AgentHarness.COPILOT, version=get_copilot_version(), model=self.model) - config = ExperimentConfiguration( - mcp_servers=mcp_server_names, - al_lsp_enabled=lsp_plugin_dir is not None, - custom_instructions=instructions_enabled, - skills_enabled=skills_enabled, - custom_agent=custom_agent, - plugins=[plugin.record for plugin, _ in plugins] or None, - ) + @override + def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: + entry, repo_path, output_dir = request.task, request.work_dir, request.output_dir + agent_config, category, runtime, plugin_root = self.agent_config, self.category, self.runtime, self.plugin_root + logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") - try: - lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else () - # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it - # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for - # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not - # silently widen the agent's sandbox access. - options = CopilotOptions( - allow_all_tools=True, + prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) + bc_gateway = start_bc_mcp_gateway(runtime) + mcp_config_json, mcp_server_names = build_mcp_config( + agent_config, + entry, + repo_path, + runtime=runtime, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) + lsp_plugin_dir: Path | None = build_al_lsp_plugin( + entry, + category, + repo_path, + AgentHarness.COPILOT, + plugin_root, + runtime=runtime, + ) + profile_dir: Path = instructions_profile_dir(entry.customization_profile) + target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path) + instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename) + skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir) + custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir) + plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True) + + config = ExperimentConfiguration( + mcp_servers=mcp_server_names, + al_lsp_enabled=lsp_plugin_dir is not None, custom_instructions=instructions_enabled, - log_dir=output_dir, - mcp_config_json=mcp_config_json, - plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)), - granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access), + skills_enabled=skills_enabled, custom_agent=custom_agent, - workspace_mcp=True, + plugins=[plugin.record for plugin, _ in plugins] or None, ) - metrics, _ = invoke_copilot( - prompt=prompt, - model=model, - work_dir=repo_path, - timeout=timeout, - env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), - options=options, - ) - logger.info(f"Copilot CLI run complete for: {entry.instance_id}") - except CopilotTimeoutError as exc: - raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc - else: - return metrics, config - finally: - if bc_gateway is not None: - bc_gateway.stop() + try: + lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else () + # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it + # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for + # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not + # silently widen the agent's sandbox access. + options = CopilotOptions( + allow_all_tools=True, + custom_instructions=instructions_enabled, + log_dir=output_dir, + mcp_config_json=mcp_config_json, + plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)), + granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access), + custom_agent=custom_agent, + workspace_mcp=True, + ) + + metrics, response = invoke_copilot( + prompt=prompt, + model=self.model, + work_dir=repo_path, + timeout=self.timeout if request.timeout is None else request.timeout, + env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), + options=options, + ) + logger.info(f"Copilot CLI run complete for: {entry.instance_id}") + except CopilotTimeoutError as exc: + raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc + else: + return AgentResult(response=response, metrics=metrics, experiment=config) + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/pr_review/agent.py b/src/bcbench/agent/pr_review/agent.py index a68fc2c1d..87f62f81d 100644 --- a/src/bcbench/agent/pr_review/agent.py +++ b/src/bcbench/agent/pr_review/agent.py @@ -22,6 +22,7 @@ import yaml from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentError, AgentTimeoutError from bcbench_core.git import commit_changes, has_changes, init_repo @@ -161,16 +162,13 @@ def run_pr_review_agent( engine_path: Path | None = None, min_severity: str | None = None, timeout: int = 60 * 60, -) -> tuple[PRReviewMetrics, ExperimentConfiguration]: +) -> AgentResult: """Run the engine's complete local review pipeline and write review.json. Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific inputs (BC-ALAgents source, min severity) for the code-review category only. - - Returns: - Tuple of (PRReviewMetrics, ExperimentConfiguration). """ if category is not EvaluationCategory.CODE_REVIEW: raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") @@ -235,4 +233,4 @@ def run_pr_review_agent( logger.exception("Unexpected error running engine review") raise else: - return build_pr_review_metrics(output_dir, time.monotonic() - start), config + return AgentResult(metrics=build_pr_review_metrics(output_dir, time.monotonic() - start), experiment=config) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 963fbac77..925249d03 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -4,13 +4,13 @@ from typing import Annotated, cast, override import typer -from bcbench_core.agent.copilot import get_copilot_version from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentIdentity, AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentError from bcbench_core.filesystem import prepare_run_dir -from bcbench.agent import BCalBackendConfig, get_bcal_version, get_claude_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.agent import BCalBackendConfig, CopilotHarness, get_bcal_version, get_claude_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_pr_review_agent from bcbench.cli_options import ( AgentConfigPath, ClaudeCodeModel, @@ -33,7 +33,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.evaluate import AgentRunner, CodeReviewPipeline, EvaluationPipeline, NL2ALPipeline +from bcbench.evaluate import AgentRunner, CodeReviewPipeline, EvaluationPipeline, NL2ALPipeline, harness_runner from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.paths import SHARED_CONFIG_FILE from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult @@ -86,6 +86,7 @@ def evaluate_copilot( ) entry = category.entry_class.load(category.dataset_path(tier), entry_id=entry_id)[0] agent_config = AgentConfig.from_file(config_path) + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI") @@ -95,27 +96,13 @@ def evaluate_copilot( repo_path=repo_path, result_dir=run_dir, container=runtime.container if runtime else None, - model=model, - agent_name=AgentHarness.COPILOT, - agent_version=get_copilot_version(), + agent=harness.identity(), category=category, tier=tier, ) pipeline = category.pipeline(agent_config) - pipeline.execute( - context, - lambda ctx: run_copilot_agent( - entry=ctx.entry, - repo_path=ctx.repo_path, - category=category, - model=ctx.model, - output_dir=ctx.result_dir, - agent_config=agent_config, - plugin_root=_config.paths.plugin_root, - runtime=runtime, - ), - ) + pipeline.execute(context, harness_runner(harness)) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") @@ -171,9 +158,7 @@ def evaluate_claude_code( repo_path=repo_path, result_dir=run_dir, container=runtime.container if runtime else None, - model=model, - agent_name=AgentHarness.CLAUDE, - agent_version=get_claude_version(), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.CLAUDE, version=get_claude_version(), model=model), category=category, tier=tier, ) @@ -185,7 +170,7 @@ def evaluate_claude_code( entry=ctx.entry, repo_path=ctx.repo_path, category=category, - model=ctx.model, + model=model, output_dir=ctx.result_dir, agent_config=agent_config, plugin_root=_config.paths.plugin_root, @@ -228,9 +213,7 @@ def evaluate_pr_review( repo_path=repo_path, result_dir=run_dir, container=None, - model=model, - agent_name=AgentHarness.PR_REVIEW, - agent_version=get_pr_review_version(engine_path), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.PR_REVIEW, version=get_pr_review_version(engine_path), model=model), category=category, ) @@ -240,7 +223,7 @@ def evaluate_pr_review( entry=ctx.entry, repo_path=ctx.repo_path, category=category, - model=ctx.model, + model=model, output_dir=ctx.result_dir, engine_path=engine_path, ), @@ -280,9 +263,7 @@ def evaluate_bcal( repo_path=repo_path, result_dir=run_dir, container=None, - model=llm_model or "external-command", - agent_name=AgentHarness.BCAL, - agent_version=get_bcal_version(), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.BCAL, version=get_bcal_version(), model=llm_model or "external-command"), category=category, tier=tier, ) @@ -352,14 +333,13 @@ def evaluate_mock( entry=entry, repo_path=Path(), result_dir=run_dir, - model="mock-model", - agent_name=AgentHarness.MOCK, + agent=AgentIdentity[AgentHarness](harness=AgentHarness.MOCK, version="mock", model="mock-model"), category=category, tier=tier, ) pipeline = MockEvaluationPipeline() - pipeline.execute(context, lambda ctx: (None, None)) + pipeline.execute(context, lambda _ctx: AgentResult()) logger.info("Mock evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 1f4e024fa..c259c52fe 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -4,8 +4,9 @@ from typing import Annotated, cast import typer +from bcbench_core.agent.harness import AgentRequest -from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.agent import BCalBackendConfig, CopilotHarness, run_bcal_agent, run_claude_code, run_pr_review_agent from bcbench.cli_options import ( AgentConfigPath, ClaudeCodeModel, @@ -81,16 +82,8 @@ def run_copilot( agent_config = AgentConfig.from_file(config_path) category.pipeline(agent_config).setup_workspace(entry, repo_path) - run_copilot_agent( - entry=entry, - repo_path=repo_path, - model=model, - category=category, - output_dir=output_dir, - agent_config=agent_config, - plugin_root=_config.paths.plugin_root, - runtime=runtime, - ) + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) + harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir)) @run_app.command("claude") diff --git a/src/bcbench/evaluate/__init__.py b/src/bcbench/evaluate/__init__.py index a0e7f1fd4..49ee8eb15 100644 --- a/src/bcbench/evaluate/__init__.py +++ b/src/bcbench/evaluate/__init__.py @@ -1,6 +1,6 @@ """Evaluation module for running pipelines and creating results.""" -from bcbench.evaluate.base import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import AgentRunner, EvaluationPipeline, harness_runner from bcbench.evaluate.bugfix import BugFixPipeline from bcbench.evaluate.codereview import CodeReviewPipeline from bcbench.evaluate.dataquery import DataQueryPipeline @@ -21,4 +21,5 @@ "ExtRequestTriagePipeline", "NL2ALPipeline", "TestGenerationPipeline", + "harness_runner", ] diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 8d2750619..8b037a6b2 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -5,22 +5,30 @@ from pathlib import Path from typing import Protocol -from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentRequest, AgentResult, Harness from bcbench_core.exceptions import AgentTimeoutError from bcbench.dataset import BaseDatasetEntry from bcbench.results import BaseEvaluationResult from bcbench.results.base import RESULT_FILE_SUFFIX -from bcbench.types import EvaluationContext +from bcbench.types import AgentHarness, EvaluationContext logger = logging.getLogger(__name__) -__all__ = ["AgentRunner", "EvaluationPipeline"] +__all__ = ["AgentRunner", "EvaluationPipeline", "harness_runner"] class AgentRunner[E: BaseDatasetEntry](Protocol): - def __call__(self, context: EvaluationContext[E], /) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: ... + def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... + + +def harness_runner[E: BaseDatasetEntry](harness: Harness[E, AgentHarness]) -> AgentRunner[E]: + """Adapt ``harness`` to the pipelines' ``AgentRunner`` until every harness implements ``Harness``.""" + + def run(context: EvaluationContext[E], /) -> AgentResult: + return harness.run(AgentRequest(task=context.entry, work_dir=context.repo_path, output_dir=context.result_dir)) + + return run class EvaluationPipeline[E: BaseDatasetEntry](ABC): diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index 4f513f40f..8dded4c63 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -44,8 +44,9 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None: @override def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[BugFixEntry]) -> None: diff --git a/src/bcbench/evaluate/codereview.py b/src/bcbench/evaluate/codereview.py index 8ad821415..7fa87f7f8 100644 --- a/src/bcbench/evaluate/codereview.py +++ b/src/bcbench/evaluate/codereview.py @@ -56,8 +56,9 @@ def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None: @override def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None: diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 77ba43705..2665e1a5a 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -106,8 +106,9 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: @override def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: AgentRunner[DataQueryEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_advisor.py b/src/bcbench/evaluate/ext_request_advisor.py index 7c98a12d0..58e0e23ad 100644 --- a/src/bcbench/evaluate/ext_request_advisor.py +++ b/src/bcbench/evaluate/ext_request_advisor.py @@ -30,8 +30,9 @@ def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_implement.py b/src/bcbench/evaluate/ext_request_implement.py index 79d6ee20c..d439d9aee 100644 --- a/src/bcbench/evaluate/ext_request_implement.py +++ b/src/bcbench/evaluate/ext_request_implement.py @@ -37,8 +37,9 @@ def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_triage.py b/src/bcbench/evaluate/ext_request_triage.py index 1743916e8..5366bef71 100644 --- a/src/bcbench/evaluate/ext_request_triage.py +++ b/src/bcbench/evaluate/ext_request_triage.py @@ -40,8 +40,9 @@ def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index e04e4c65b..40cb7a99d 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -71,8 +71,9 @@ def setup(self, context: EvaluationContext[NL2ALEntry]) -> None: def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentRunner[NL2ALEntry]) -> None: # Single attempt — retries are disabled. An empty diff (the agent asked for clarification # instead of editing) is scored as a failure in evaluate(), not re-run. - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[NL2ALEntry]) -> None: diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index a72b6f526..12e7704d9 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -66,8 +66,9 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None: @override def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[TestGenEntry]) -> None: diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 765617267..16b32237b 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -38,22 +38,22 @@ class BaseEvaluationResult(BaseModel): @classmethod def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dict[str, Any]: - metrics_contract = context.agent_name.metrics_contract + metrics_contract = context.agent.harness.metrics_contract if not context.metrics: logger.warning(f"Creating result for {context.entry.instance_id} with no agent metrics - performance data will be unavailable") elif type(context.metrics) is not metrics_contract.metrics_type: - raise TypeError(f"{context.agent_name} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") + raise TypeError(f"{context.agent.harness} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") elif missing_metrics := sorted(name for name in metrics_contract.required_fields if getattr(context.metrics, name) is None): logger.warning(f"Result for {context.entry.instance_id} missing metrics: {', '.join(missing_metrics)}") return { "instance_id": context.entry.instance_id, "project": context.entry.extract_project_name(), - "model": context.model.replace(".", "-"), + "model": (context.agent.model or "").replace(".", "-"), "category": context.category, "tier": context.tier, - "agent_name": context.agent_name, - "agent_version": context.agent_version, + "agent_name": context.agent.harness, + "agent_version": context.agent.version, "metrics": context.metrics, "experiment": context.experiment, } diff --git a/src/bcbench/types.py b/src/bcbench/types.py index c75a1e733..0500a2c08 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -10,6 +10,7 @@ import yaml from bcbench_core.agent.customization import CustomAgentsConfig, ToggleConfig from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.container import ContainerConfig from pydantic import BaseModel, ConfigDict, Field, StrictBool, StringConstraints, model_validator @@ -629,9 +630,8 @@ class EvaluationContext[E: BaseDatasetEntry]: repo_path: Path result_dir: Path - # Agent metadata - agent_name: AgentHarness - model: str + # The agent under evaluation, resolved before running + agent: AgentIdentity[AgentHarness] # Evaluation category category: EvaluationCategory @@ -640,8 +640,6 @@ class EvaluationContext[E: BaseDatasetEntry]: # BC Container configuration (optional — not all categories require a container) container: ContainerConfig | None = None - agent_version: str | None = None - # Agent execution metrics metrics: AgentMetrics | None = None diff --git a/tests/conftest.py b/tests/conftest.py index 0aae5ea5f..1ac1d615e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ import pytest from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.container import ContainerConfig from bcbench_core.dataset import TestEntry @@ -85,6 +86,7 @@ def create_evaluation_context[EntryT: BaseDatasetEntry]( entry: EntryT | None = None, agent_name: AgentHarness = AgentHarness.COPILOT, model: str = "test-model", + agent_version: str = "1.0.0", category: EvaluationCategory = EvaluationCategory.BUG_FIX, container_name: str = "test-container", password: str = "test-password", @@ -97,8 +99,7 @@ def create_evaluation_context[EntryT: BaseDatasetEntry]( repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig(name=container_name, username=username, password=password, company="CRONUS"), - agent_name=agent_name, - model=model, + agent=AgentIdentity[AgentHarness](harness=agent_name, version=agent_version, model=model), category=category, ) @@ -218,8 +219,7 @@ def create_codereview_result( repo_path=Path(), result_dir=Path(), container=ContainerConfig(name="t", username="t", password="t", company="CRONUS"), - agent_name=agent_name, - model=model, + agent=AgentIdentity[AgentHarness](harness=agent_name, version="1.0.0", model=model), category=EvaluationCategory.CODE_REVIEW, ) context.metrics = metrics diff --git a/tests/test_agent_version_results.py b/tests/test_agent_version_results.py index 1af54016f..e785746f2 100644 --- a/tests/test_agent_version_results.py +++ b/tests/test_agent_version_results.py @@ -17,9 +17,7 @@ @pytest.mark.parametrize("version", ["1.2.3", "a" * 40, None]) def test_agent_version_round_trips_through_results_and_summaries(tmp_path: Path, version: str | None) -> None: - context = create_evaluation_context(tmp_path) - context.agent_version = version - result = BugFixResult.create_success(context, "patch") + result = BugFixResult.create_success(create_evaluation_context(tmp_path), "patch").model_copy(update={"agent_version": version}) result.save(tmp_path, "result.jsonl") restored = BaseEvaluationResult.from_json(json.loads((tmp_path / "result.jsonl").read_text())) summary = EvaluationResultSummary.from_results([restored], "run") @@ -32,8 +30,7 @@ def test_agent_version_round_trips_through_results_and_summaries(tmp_path: Path, def test_timeout_result_keeps_version_resolved_before_execution(tmp_path: Path) -> None: - context = create_evaluation_context(tmp_path) - context.agent_version = "1.2.3" + context = create_evaluation_context(tmp_path, agent_version="1.2.3") pipeline = context.category.pipeline(AgentConfig()) with ( patch.object(type(pipeline), "setup"), diff --git a/tests/test_bcal_agent_provider.py b/tests/test_bcal_agent_provider.py index 062d9d31d..29a364b30 100644 --- a/tests/test_bcal_agent_provider.py +++ b/tests/test_bcal_agent_provider.py @@ -63,7 +63,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), patch.object(subprocess, "run", side_effect=fake_run), ): - metrics, _ = bcal_agent.run_bcal_agent( + result = bcal_agent.run_bcal_agent( entry=entry, repo_path=workspace, backend_config=BCalBackendConfig( @@ -72,7 +72,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: ), ) - assert metrics is not None + assert result.metrics is not None args = captured["args"] assert "--deployment=gpt-5" in args assert "--llm-backend=external-command" in args diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index b1a4d004d..0060be941 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -142,8 +142,8 @@ def execute(self, context, _agent_runner): llm_model=llm_model, ) - assert captured["context"].model == expected_model - assert captured["context"].agent_version == "18.0.1.2-beta" + assert captured["context"].agent.model == expected_model + assert captured["context"].agent.version == "18.0.1.2-beta" def _option_names(command: click.Command) -> set[str]: @@ -190,9 +190,9 @@ def execute(self, context, agent_runner): @pytest.mark.parametrize( ("commands", "command_name", "agent_name", "extra_args"), [ - (run_commands, "run_copilot", "run_copilot_agent", {}), + (run_commands, "run_copilot", "CopilotHarness", {}), (run_commands, "run_claude", "run_claude_code", {}), - (evaluate_commands, "evaluate_copilot", "run_copilot_agent", {"run_id": "test-run"}), + (evaluate_commands, "evaluate_copilot", "CopilotHarness", {"run_id": "test-run"}), (evaluate_commands, "evaluate_claude_code", "run_claude_code", {"run_id": "test-run"}), ], ) @@ -201,7 +201,6 @@ def test_agent_commands_preserve_requested_mcp_flags(tmp_path, agent_command_cat with ( patch.object(commands, agent_name) as run_agent, - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): getattr(commands, command_name)( @@ -226,9 +225,9 @@ def test_agent_commands_preserve_requested_mcp_flags(tmp_path, agent_command_cat @pytest.mark.parametrize( ("commands", "command_name", "agent_name", "extra_args"), [ - (run_commands, "run_copilot", "run_copilot_agent", {}), + (run_commands, "run_copilot", "CopilotHarness", {}), (run_commands, "run_claude", "run_claude_code", {}), - (evaluate_commands, "evaluate_copilot", "run_copilot_agent", {"run_id": "test-run"}), + (evaluate_commands, "evaluate_copilot", "CopilotHarness", {"run_id": "test-run"}), (evaluate_commands, "evaluate_claude_code", "run_claude_code", {"run_id": "test-run"}), ], ) @@ -239,7 +238,6 @@ def test_agent_commands_pass_the_given_agent_config(tmp_path, agent_command_cate with ( patch.object(commands, agent_name) as run_agent, - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): getattr(commands, command_name)( @@ -324,7 +322,7 @@ def test_result_summarize_creates_all_outputs(sample_results_directory, problem_ with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), patch.object(EvaluationCategory, "dataset_path", return_value=dataset_path), - patch.object(evaluate_commands, "get_copilot_version", side_effect=AssertionError("Must use artifact version")), + patch("bcbench.agent.copilot.agent.get_copilot_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_claude_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_pr_review_version", side_effect=AssertionError("Must use artifact version")), ): diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index ab4bd4f74..4a47b8c55 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -5,14 +5,30 @@ import pytest from bcbench_core.agent.copilot import CopilotProcessError, CopilotTimeoutError from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentRequest, AgentResult from bcbench_core.exceptions import AgentTimeoutError -from bcbench.agent.copilot.agent import run_copilot_agent +from bcbench.agent.copilot.agent import CopilotHarness +from bcbench.dataset import BaseDatasetEntry from bcbench.operations import instructions_profile_dir from bcbench.types import AgentConfig, EvaluationCategory, PluginConfig from tests.conftest import create_dataset_entry +def run_copilot_harness( + entry: BaseDatasetEntry, + model: str, + category: EvaluationCategory, + repo_path: Path, + output_dir: Path, + agent_config: AgentConfig, + plugin_root: Path, + timeout: int | None = None, +) -> AgentResult: + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=plugin_root) + return harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, timeout=timeout)) + + def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Path, monkeypatch): repo_path = tmp_path / "repo" output_dir = tmp_path / "output" @@ -30,13 +46,13 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat patch("bcbench.agent.copilot.agent.setup_skills", return_value=False), patch("bcbench.agent.copilot.agent.setup_custom_agent", return_value=None), patch("bcbench.agent.copilot.agent.resolve_config_plugins", return_value=[]), - patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, None)) as mock_parse_output, + patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, "final answer")) as mock_parse_output, patch( "bcbench_core.agent.copilot.agent.subprocess.run", return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - run_copilot_agent( + result = run_copilot_harness( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -46,6 +62,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat plugin_root=tmp_path / "plugins", ) + assert result.response == "final answer" assert mock_run.call_args.args[0] == [ "copilot", "--output-format=json", @@ -95,7 +112,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - _, config = run_copilot_agent( + config = run_copilot_harness( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -103,7 +120,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr output_dir=output_dir, agent_config=AgentConfig(), plugin_root=tmp_path / "plugins", - ) + ).experiment assert mock_run.call_args.args[0] == [ "copilot", @@ -121,6 +138,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr "--agent=al-dev", "--prompt=do the task", ] + assert config is not None assert config.mcp_servers == ["probe"] assert config.al_lsp_enabled is lsp_enabled assert config.custom_instructions is True @@ -161,7 +179,7 @@ def test_copilot_timeout_adapts_core_metrics_and_preserves_experiment_metadata(c run.side_effect = subprocess.TimeoutExpired(["copilot"], timeout) with pytest.raises(AgentTimeoutError, match="Copilot CLI timed out") as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins", timeout=timeout) + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins", timeout=timeout) assert error.value.metrics is not None assert error.value.metrics.execution_time == timeout @@ -177,7 +195,7 @@ def test_copilot_process_errors_propagate_and_stop_gateway(configured_copilot_ru run.side_effect = failure with pytest.raises(CopilotProcessError) as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") assert error.value.__cause__ is failure gateway.stop.assert_called_once_with() @@ -187,7 +205,7 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop repo_path, output_dir, gateway, run = configured_copilot_run with patch("bcbench_core.agent.copilot.agent._find_copilot", return_value=None), pytest.raises(CopilotProcessError, match="not found"): - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") run.assert_not_called() gateway.stop.assert_called_once_with() @@ -196,7 +214,7 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop def test_successful_copilot_run_stops_gateway(configured_copilot_run): repo_path, output_dir, gateway, _ = configured_copilot_run - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") gateway.stop.assert_called_once_with() @@ -210,7 +228,7 @@ def test_copilot_sets_up_the_session_from_the_given_agent_config(configured_copi mocks["build_mcp_config"].return_value = (None, None) mocks["setup_custom_agent"].return_value = None mocks["resolve_config_plugins"].return_value = [] - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, agent_config, repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, agent_config, repo_path.parent / "plugins") for name in ("build_prompt", "build_mcp_config", "resolve_config_plugins"): assert agent_config in mocks[name].call_args.args, name @@ -225,7 +243,7 @@ def test_copilot_sets_up_customizations_from_the_entry_profile_into_github(confi with patch.multiple("bcbench.agent.copilot.agent", **dict.fromkeys(helpers, DEFAULT)) as mocks: mocks["setup_custom_agent"].return_value = None - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") profile_dir = instructions_profile_dir(create_dataset_entry().customization_profile) for name in helpers: @@ -240,7 +258,7 @@ def test_copilot_keeps_plugins_in_the_given_plugin_root(configured_copilot_run): with patch.multiple("bcbench.agent.copilot.agent", build_al_lsp_plugin=DEFAULT, resolve_config_plugins=DEFAULT) as mocks: mocks["build_al_lsp_plugin"].return_value = None mocks["resolve_config_plugins"].return_value = [] - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), plugin_root) + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), plugin_root) assert plugin_root in mocks["build_al_lsp_plugin"].call_args.args assert plugin_root in mocks["resolve_config_plugins"].call_args.args diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 28783bb85..d2d2a89c7 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.container import ContainerConfig from bcbench_core.exceptions import BuildError @@ -87,8 +88,7 @@ def _context(self, tmp_path: Path) -> EvaluationContext[DataQueryEntry]: repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig("bcbench", "admin", "secret", company="CRONUS"), - model="test-model", - agent_name=AgentHarness.COPILOT, + agent=AgentIdentity[AgentHarness](harness=AgentHarness.COPILOT, version="1.0.0", model="test-model"), category=EvaluationCategory.DATA_QUERY, ) diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 17ea4e049..e3bd2bed5 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -7,6 +7,7 @@ import pytest from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentTimeoutError @@ -39,7 +40,8 @@ def run_agent(self, context: EvaluationContext[E], agent_runner: AgentRunner[E]) self.run_agent_called = True if self.raise_in_run_agent is not None: raise self.raise_in_run_agent - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[E]) -> None: @@ -48,8 +50,8 @@ def evaluate(self, context: EvaluationContext[E]) -> None: raise self.raise_in_evaluate -def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: - return AgentMetrics(execution_time=1.0), ExperimentConfiguration() +def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> AgentResult: + return AgentResult(metrics=AgentMetrics(execution_time=1.0), experiment=ExperimentConfiguration()) def _read_only_result[E: BaseDatasetEntry](ctx: EvaluationContext[E]) -> BaseEvaluationResult: @@ -93,7 +95,7 @@ def test_persists_category_specific_timeout_result(self, tmp_path): ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL) pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=AgentTimeoutError("test timeout")) - pipeline.execute(ctx, lambda _: (None, None)) + pipeline.execute(ctx, lambda _: AgentResult()) result_file = ctx.result_dir / f"{entry.instance_id}{RESULT_FILE_SUFFIX}" payload = json.loads(result_file.read_text(encoding="utf-8")) diff --git a/tests/test_nl2al_pipeline.py b/tests/test_nl2al_pipeline.py index 06457d34e..ffc899dea 100644 --- a/tests/test_nl2al_pipeline.py +++ b/tests/test_nl2al_pipeline.py @@ -1,6 +1,7 @@ """Tests for NL2ALPipeline.evaluate() — empty-diff handling is nl2al-specific.""" import pytest +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import EmptyDiffError from bcbench.evaluate.nl2al import NL2ALPipeline @@ -85,7 +86,7 @@ def test_runs_agent_once_and_never_retries(self, tmp_path, monkeypatch): def agent_runner(_ctx): agent_calls["n"] += 1 - return (None, None) + return AgentResult() reset_calls = {"n": 0} monkeypatch.setattr(pipeline, "setup_workspace", lambda *_args, **_kw: reset_calls.__setitem__("n", reset_calls["n"] + 1)) diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index e294a47b4..11def48bc 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -163,7 +163,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: patch("bcbench.agent.pr_review.agent.time.monotonic", side_effect=[10.0, 12.5]), patch("bcbench.agent.pr_review.agent.subprocess.run", return_value=completed) as run_process, ): - metrics, config = run_pr_review_agent( + result = run_pr_review_agent( entry=entry, model="gpt-5.6-luna", category=EvaluationCategory.CODE_REVIEW, @@ -172,6 +172,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: engine_path=tmp_path / "engine", ) + metrics = result.metrics assert isinstance(metrics, PRReviewMetrics) assert metrics.execution_time == 2.5 assert metrics.prompt_tokens == 100 @@ -180,7 +181,8 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: assert metrics.ai_credits == 0.25 assert metrics.api_calls == 2 assert metrics.copilot_cli_version == "1.0.81-0" - assert config.is_empty() + assert result.experiment is not None + assert result.experiment.is_empty() resolve_engine.assert_called_once_with(tmp_path / "engine") prepare_bcquality.assert_called_once_with(tmp_path / "engine", "pwsh", (tmp_path / "output" / "bcquality").resolve()) assert run_process.call_args.kwargs["encoding"] == "utf-8" diff --git a/tests/test_review_runners.py b/tests/test_review_runners.py index 9f2bba7f2..0d1ebe225 100644 --- a/tests/test_review_runners.py +++ b/tests/test_review_runners.py @@ -12,21 +12,27 @@ from bcbench.types import AgentHarness, EvaluationCategory -@pytest.mark.parametrize( - ("command", "runner_name"), - [ - (run_commands.run_copilot, "run_copilot_agent"), - (run_commands.run_claude, "run_claude_code"), - ], -) -def test_generic_run_commands_accept_code_review(tmp_path: Path, command, runner_name: str) -> None: +def test_copilot_run_command_accepts_code_review(tmp_path: Path) -> None: + entry = object() + with ( + patch.object(CodeReviewEntry, "load", return_value=[entry]), + patch.object(CodeReviewPipeline, "setup_workspace"), + patch.object(run_commands, "CopilotHarness") as harness, + ): + run_commands.run_copilot("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") + + assert harness.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW + assert harness.return_value.run.call_args.args[0].task is entry + + +def test_claude_run_command_accepts_code_review(tmp_path: Path) -> None: entry = object() with ( patch.object(CodeReviewEntry, "load", return_value=[entry]), patch.object(CodeReviewPipeline, "setup_workspace"), - patch.object(run_commands, runner_name) as agent_runner, + patch.object(run_commands, "run_claude_code") as agent_runner, ): - command("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") + run_commands.run_claude("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") assert agent_runner.call_args.kwargs["entry"] is entry assert agent_runner.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW @@ -44,7 +50,7 @@ def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, comm with ( patch.object(CodeReviewEntry, "load", return_value=[object()]), patch.object(CodeReviewPipeline, "execute", side_effect=lambda context, runner: contexts.append(context)), - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), + patch("bcbench.agent.copilot.agent.get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): command( @@ -56,8 +62,8 @@ def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, comm ) assert len(contexts) == 1 - assert contexts[0].agent_name is agent_name - assert contexts[0].agent_version == "1.2.3" + assert contexts[0].agent.harness is agent_name + assert contexts[0].agent.version == "1.2.3" assert contexts[0].category is EvaluationCategory.CODE_REVIEW @@ -88,10 +94,10 @@ def test_pr_review_evaluation_is_fixed_to_runner_and_category(tmp_path: Path) -> assert result.exit_code == 0, result.exception assert len(contexts) == 1 - assert contexts[0].agent_name is AgentHarness.PR_REVIEW - assert contexts[0].agent_version == "a" * 40 + assert contexts[0].agent.harness is AgentHarness.PR_REVIEW + assert contexts[0].agent.version == "a" * 40 assert contexts[0].category is EvaluationCategory.CODE_REVIEW - assert contexts[0].model == "gpt-5.6-luna" + assert contexts[0].agent.model == "gpt-5.6-luna" assert agent_runner.call_args.kwargs["engine_path"] == tmp_path get_version.assert_called_once_with(tmp_path)