From ac3e21c68d66052f292f52f9c8f0bdcd074d399f Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 11:13:42 +0100 Subject: [PATCH 1/7] Add a harness-neutral agent contract to bcbench-core Harness[T] reports an AgentIdentity (harness, version, configured model, provenance) before running and turns an AgentRequest[T] into an AgentResult (response, transcript, metrics, experiment configuration, observed model). It is generic over the task type so specialised harnesses (PR Review, BCal, hosted agents) take their own typed tasks, and AgentResult is a dataclass so harness-specific metrics subclasses pass through unchanged. The transcript uses the user/assistant turn shape LMChecklist already scores. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/bcbench-core/README.md | 4 + .../src/bcbench_core/agent/harness.py | 75 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 packages/bcbench-core/src/bcbench_core/agent/harness.py diff --git a/packages/bcbench-core/README.md b/packages/bcbench-core/README.md index d762163ab..14a1d09a6 100644 --- a/packages/bcbench-core/README.md +++ b/packages/bcbench-core/README.md @@ -21,6 +21,10 @@ The import and environment rules are enforced by ruff (`banned-api` in [`pyproje - For Copilot invocation: an installed and authenticated [GitHub Copilot CLI](https://github.com/github/copilot-cli). - For BC container operations: PowerShell 7 with the latest [BcContainerHelper](https://github.com/microsoft/navcontainerhelper) module. The PowerShell modules ship in [`src/bcbench_core/powershell`](src/bcbench_core/powershell). +## Agent harnesses + +[`bcbench_core.agent.harness`](src/bcbench_core/agent/harness.py) defines the harness-neutral contract. A `Harness[T]` reports its `AgentIdentity` before any run and turns an `AgentRequest[T]` into an `AgentResult`. Output is a final response, a transcript, or changes left in the workspace. Applications construct harnesses with their own configuration and pass them to evaluation pipelines, which depend only on this contract. + ## Logging Modules log through `logging.getLogger(__name__)` and never configure logging on import. Applications call `bcbench_core.logs.setup_logging` once from their entry point. diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py new file mode 100644 index 000000000..f434b6b64 --- /dev/null +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -0,0 +1,75 @@ +"""Harness-neutral contract between evaluation applications and the agents they run. + +A harness wraps one way of producing an answer (a CLI agent, a review engine, a hosted agent). +Applications construct harnesses with harness-specific configuration and only use this contract, +so evaluation pipelines never depend on a concrete harness. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.metrics import AgentMetrics + +__all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness", "TranscriptTurn"] + + +class AgentIdentity(BaseModel): + """What is being evaluated, resolved before any task runs.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + harness: str + # CLI version, engine commit, or hosted agent version + version: str + # Configured model; None when the harness pins or delegates model selection + model: str | None = None + # Further pins needed to reproduce a run, e.g. a knowledge-base commit or a definition hash + provenance: dict[str, JsonValue] = Field(default_factory=dict) + + +class TranscriptTurn(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + role: Literal["user", "assistant"] + content: str + + +@dataclass(frozen=True) +class AgentRequest[T]: + task: T + # The agent's workspace: a prepared repository, or an empty directory for answer-only tasks + work_dir: Path + # Receives logs and raw captures + output_dir: Path + # Passed explicitly; harnesses never read the process environment + env: Mapping[str, str] + # None uses the harness's default + timeout: int | None = None + + +@dataclass(frozen=True) +class AgentResult: + # Final answer; None when the output is left in the workspace + response: str | None = None + transcript: tuple[TranscriptTurn, ...] = () + metrics: AgentMetrics | None = None + experiment: ExperimentConfiguration | None = None + # Model reported by the agent, when it differs from or replaces the configured one + observed_model: str | None = None + + +class Harness[T](Protocol): + """Runs an agent on tasks of type ``T``. + + ``identity`` checks the harness is usable and raises ``AgentError`` otherwise. ``run`` raises + ``AgentTimeoutError`` when the timeout is exceeded and ``AgentError`` for any other failure. + """ + + def identity(self) -> AgentIdentity: ... + + def run(self, request: AgentRequest[T], /) -> AgentResult: ... From 266df189e47c6ad8eb4c5e0c782e8159a828bdb9 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 11:17:38 +0100 Subject: [PATCH 2/7] Return AgentResult from BC-Bench agent harnesses The Copilot, Claude, PR Review and BCal harness functions now return the core AgentResult instead of a (metrics, experiment) tuple, and pipelines read it through AgentRunner. Copilot and Claude also return their final message as the response, which was previously discarded. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/bcbench/agent/bcal/agent.py | 5 +++-- src/bcbench/agent/claude/agent.py | 13 +++++-------- src/bcbench/agent/copilot/agent.py | 14 +++++--------- src/bcbench/agent/pr_review/agent.py | 8 +++----- src/bcbench/commands/evaluate.py | 3 ++- src/bcbench/evaluate/base.py | 5 ++--- src/bcbench/evaluate/bugfix.py | 3 ++- src/bcbench/evaluate/codereview.py | 3 ++- src/bcbench/evaluate/dataquery.py | 3 ++- src/bcbench/evaluate/ext_request_advisor.py | 3 ++- src/bcbench/evaluate/ext_request_implement.py | 3 ++- src/bcbench/evaluate/ext_request_triage.py | 3 ++- src/bcbench/evaluate/nl2al.py | 3 ++- src/bcbench/evaluate/testgeneration.py | 3 ++- tests/test_bcal_agent_provider.py | 4 ++-- tests/test_copilot_agent.py | 10 ++++++---- tests/test_evaluate_pipeline.py | 10 ++++++---- tests/test_nl2al_pipeline.py | 3 ++- tests/test_pr_review_agent.py | 6 ++++-- 19 files changed, 56 insertions(+), 49 deletions(-) diff --git a/src/bcbench/agent/bcal/agent.py b/src/bcbench/agent/bcal/agent.py index 24ff0005a..8764ddb46 100644 --- a/src/bcbench/agent/bcal/agent.py +++ b/src/bcbench/agent/bcal/agent.py @@ -7,6 +7,7 @@ from pathlib import Path from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.artifacts import ALPACKAGES_DIRNAME @@ -124,7 +125,7 @@ def run_bcal_agent( repo_path: Path, backend_config: BCalBackendConfig, timeout: int = 25 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: +) -> AgentResult: logger.info(f"Running bcal CLI on: {entry.instance_id}") # The .alpackages dir is created by the NL2AL pipeline setup step @@ -151,7 +152,7 @@ def run_bcal_agent( execution_time = time.monotonic() - start logger.info(f"bcal CLI run complete for: {entry.instance_id}") - return AgentMetrics(execution_time=execution_time), ExperimentConfiguration() + return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration()) except subprocess.TimeoutExpired: logger.exception(f"bcal CLI timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 8c9f34d24..c04cd5888 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -5,6 +5,7 @@ from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.exceptions import AgentError, AgentTimeoutError @@ -39,12 +40,8 @@ def run_claude_code( plugin_root: Path, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run Claude Code on a single dataset entry. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used - """ +) -> AgentResult: + """Run Claude Code on a single dataset entry.""" claude_cmd = shutil.which("claude") if not claude_cmd: raise AgentError("Claude Code not found in PATH. Please ensure it is installed and available.") @@ -145,7 +142,7 @@ def run_claude_code( stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else "" logger.debug(f"Claude Code raw output: {stdout}") - metrics, _ = parse_stream_output(stdout.splitlines(), log_transcript=True) + metrics, response = parse_stream_output(stdout.splitlines(), log_transcript=True) except subprocess.TimeoutExpired: logger.exception(f"Claude Code timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) @@ -157,7 +154,7 @@ def run_claude_code( logger.exception("Unexpected error running Claude Code") raise else: - return metrics, config + return AgentResult(response=response, metrics=metrics, experiment=config) finally: if bc_gateway is not None: bc_gateway.stop() diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index c95a530bb..d277c7b52 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -6,7 +6,7 @@ from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentTimeoutError from bcbench.agent.shared import ( @@ -34,12 +34,8 @@ def run_copilot_agent( plugin_root: Path, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run GitHub Copilot CLI agent on a single dataset entry. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment - """ +) -> AgentResult: + """Run GitHub Copilot CLI agent on a single dataset entry.""" logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) @@ -92,7 +88,7 @@ def run_copilot_agent( workspace_mcp=True, ) - metrics, _ = invoke_copilot( + metrics, response = invoke_copilot( prompt=prompt, model=model, work_dir=repo_path, @@ -104,7 +100,7 @@ def run_copilot_agent( except CopilotTimeoutError as exc: raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc else: - return metrics, config + return AgentResult(response=response, metrics=metrics, experiment=config) finally: if bc_gateway is not None: bc_gateway.stop() diff --git a/src/bcbench/agent/pr_review/agent.py b/src/bcbench/agent/pr_review/agent.py index a68fc2c1d..87f62f81d 100644 --- a/src/bcbench/agent/pr_review/agent.py +++ b/src/bcbench/agent/pr_review/agent.py @@ -22,6 +22,7 @@ import yaml from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentError, AgentTimeoutError from bcbench_core.git import commit_changes, has_changes, init_repo @@ -161,16 +162,13 @@ def run_pr_review_agent( engine_path: Path | None = None, min_severity: str | None = None, timeout: int = 60 * 60, -) -> tuple[PRReviewMetrics, ExperimentConfiguration]: +) -> AgentResult: """Run the engine's complete local review pipeline and write review.json. Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific inputs (BC-ALAgents source, min severity) for the code-review category only. - - Returns: - Tuple of (PRReviewMetrics, ExperimentConfiguration). """ if category is not EvaluationCategory.CODE_REVIEW: raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") @@ -235,4 +233,4 @@ def run_pr_review_agent( logger.exception("Unexpected error running engine review") raise else: - return build_pr_review_metrics(output_dir, time.monotonic() - start), config + return AgentResult(metrics=build_pr_review_metrics(output_dir, time.monotonic() - start), experiment=config) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 963fbac77..fd17d54bc 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -6,6 +6,7 @@ import typer from bcbench_core.agent.copilot import get_copilot_version from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentError from bcbench_core.filesystem import prepare_run_dir @@ -359,7 +360,7 @@ def evaluate_mock( ) pipeline = MockEvaluationPipeline() - pipeline.execute(context, lambda ctx: (None, None)) + pipeline.execute(context, lambda _ctx: AgentResult()) logger.info("Mock evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 8d2750619..590840677 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -5,8 +5,7 @@ from pathlib import Path from typing import Protocol -from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentTimeoutError from bcbench.dataset import BaseDatasetEntry @@ -20,7 +19,7 @@ class AgentRunner[E: BaseDatasetEntry](Protocol): - def __call__(self, context: EvaluationContext[E], /) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: ... + def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... class EvaluationPipeline[E: BaseDatasetEntry](ABC): diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index 4f513f40f..f34c113e7 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -45,7 +45,8 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None: @override def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[BugFixEntry]) -> None: diff --git a/src/bcbench/evaluate/codereview.py b/src/bcbench/evaluate/codereview.py index 8ad821415..4e574e387 100644 --- a/src/bcbench/evaluate/codereview.py +++ b/src/bcbench/evaluate/codereview.py @@ -57,7 +57,8 @@ def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None: @override def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None: diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 77ba43705..af618f436 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -107,7 +107,8 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: @override def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: AgentRunner[DataQueryEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_advisor.py b/src/bcbench/evaluate/ext_request_advisor.py index 7c98a12d0..0bffb5d3d 100644 --- a/src/bcbench/evaluate/ext_request_advisor.py +++ b/src/bcbench/evaluate/ext_request_advisor.py @@ -31,7 +31,8 @@ def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_implement.py b/src/bcbench/evaluate/ext_request_implement.py index 79d6ee20c..ae4368537 100644 --- a/src/bcbench/evaluate/ext_request_implement.py +++ b/src/bcbench/evaluate/ext_request_implement.py @@ -38,7 +38,8 @@ def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_triage.py b/src/bcbench/evaluate/ext_request_triage.py index 1743916e8..f310fd9c9 100644 --- a/src/bcbench/evaluate/ext_request_triage.py +++ b/src/bcbench/evaluate/ext_request_triage.py @@ -41,7 +41,8 @@ def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index e04e4c65b..54f567f00 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -72,7 +72,8 @@ def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentR # Single attempt — retries are disabled. An empty diff (the agent asked for clarification # instead of editing) is scored as a failure in evaluate(), not re-run. with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[NL2ALEntry]) -> None: diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index a72b6f526..51a7c597b 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -67,7 +67,8 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None: @override def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[TestGenEntry]) -> None: diff --git a/tests/test_bcal_agent_provider.py b/tests/test_bcal_agent_provider.py index 062d9d31d..29a364b30 100644 --- a/tests/test_bcal_agent_provider.py +++ b/tests/test_bcal_agent_provider.py @@ -63,7 +63,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), patch.object(subprocess, "run", side_effect=fake_run), ): - metrics, _ = bcal_agent.run_bcal_agent( + result = bcal_agent.run_bcal_agent( entry=entry, repo_path=workspace, backend_config=BCalBackendConfig( @@ -72,7 +72,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: ), ) - assert metrics is not None + assert result.metrics is not None args = captured["args"] assert "--deployment=gpt-5" in args assert "--llm-backend=external-command" in args diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index ab4bd4f74..d7d2d5e7d 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -30,13 +30,13 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat patch("bcbench.agent.copilot.agent.setup_skills", return_value=False), patch("bcbench.agent.copilot.agent.setup_custom_agent", return_value=None), patch("bcbench.agent.copilot.agent.resolve_config_plugins", return_value=[]), - patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, None)) as mock_parse_output, + patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, "final answer")) as mock_parse_output, patch( "bcbench_core.agent.copilot.agent.subprocess.run", return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - run_copilot_agent( + result = run_copilot_agent( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -46,6 +46,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat plugin_root=tmp_path / "plugins", ) + assert result.response == "final answer" assert mock_run.call_args.args[0] == [ "copilot", "--output-format=json", @@ -95,7 +96,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - _, config = run_copilot_agent( + config = run_copilot_agent( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -103,7 +104,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr output_dir=output_dir, agent_config=AgentConfig(), plugin_root=tmp_path / "plugins", - ) + ).experiment assert mock_run.call_args.args[0] == [ "copilot", @@ -121,6 +122,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr "--agent=al-dev", "--prompt=do the task", ] + assert config is not None assert config.mcp_servers == ["probe"] assert config.al_lsp_enabled is lsp_enabled assert config.custom_instructions is True diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 17ea4e049..e3bd2bed5 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -7,6 +7,7 @@ import pytest from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentTimeoutError @@ -39,7 +40,8 @@ def run_agent(self, context: EvaluationContext[E], agent_runner: AgentRunner[E]) self.run_agent_called = True if self.raise_in_run_agent is not None: raise self.raise_in_run_agent - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[E]) -> None: @@ -48,8 +50,8 @@ def evaluate(self, context: EvaluationContext[E]) -> None: raise self.raise_in_evaluate -def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: - return AgentMetrics(execution_time=1.0), ExperimentConfiguration() +def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> AgentResult: + return AgentResult(metrics=AgentMetrics(execution_time=1.0), experiment=ExperimentConfiguration()) def _read_only_result[E: BaseDatasetEntry](ctx: EvaluationContext[E]) -> BaseEvaluationResult: @@ -93,7 +95,7 @@ def test_persists_category_specific_timeout_result(self, tmp_path): ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL) pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=AgentTimeoutError("test timeout")) - pipeline.execute(ctx, lambda _: (None, None)) + pipeline.execute(ctx, lambda _: AgentResult()) result_file = ctx.result_dir / f"{entry.instance_id}{RESULT_FILE_SUFFIX}" payload = json.loads(result_file.read_text(encoding="utf-8")) diff --git a/tests/test_nl2al_pipeline.py b/tests/test_nl2al_pipeline.py index 06457d34e..ffc899dea 100644 --- a/tests/test_nl2al_pipeline.py +++ b/tests/test_nl2al_pipeline.py @@ -1,6 +1,7 @@ """Tests for NL2ALPipeline.evaluate() — empty-diff handling is nl2al-specific.""" import pytest +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import EmptyDiffError from bcbench.evaluate.nl2al import NL2ALPipeline @@ -85,7 +86,7 @@ def test_runs_agent_once_and_never_retries(self, tmp_path, monkeypatch): def agent_runner(_ctx): agent_calls["n"] += 1 - return (None, None) + return AgentResult() reset_calls = {"n": 0} monkeypatch.setattr(pipeline, "setup_workspace", lambda *_args, **_kw: reset_calls.__setitem__("n", reset_calls["n"] + 1)) diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index e294a47b4..11def48bc 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -163,7 +163,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: patch("bcbench.agent.pr_review.agent.time.monotonic", side_effect=[10.0, 12.5]), patch("bcbench.agent.pr_review.agent.subprocess.run", return_value=completed) as run_process, ): - metrics, config = run_pr_review_agent( + result = run_pr_review_agent( entry=entry, model="gpt-5.6-luna", category=EvaluationCategory.CODE_REVIEW, @@ -172,6 +172,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: engine_path=tmp_path / "engine", ) + metrics = result.metrics assert isinstance(metrics, PRReviewMetrics) assert metrics.execution_time == 2.5 assert metrics.prompt_tokens == 100 @@ -180,7 +181,8 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: assert metrics.ai_credits == 0.25 assert metrics.api_calls == 2 assert metrics.copilot_cli_version == "1.0.81-0" - assert config.is_empty() + assert result.experiment is not None + assert result.experiment.is_empty() resolve_engine.assert_called_once_with(tmp_path / "engine") prepare_bcquality.assert_called_once_with(tmp_path / "engine", "pwsh", (tmp_path / "output" / "bcquality").resolve()) assert run_process.call_args.kwargs["encoding"] == "utf-8" From 5fa922419805c1fcda04ef6ced909adde4458a66 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 11:42:04 +0100 Subject: [PATCH 3/7] Implement the harness contract for GitHub Copilot CLI and drop the transcript type CopilotHarness replaces run_copilot_agent: model, category, agent config, plugin root and runtime are construction-time settings, while the entry, workspace, output directory, environment and timeout arrive per run as an AgentRequest. identity() resolves the Copilot CLI version, replacing the CLI's direct get_copilot_version call. harness_runner adapts any Harness to the pipeline's AgentRunner, so other harnesses can follow the same path. The harness now scrubs BC credentials from the request's environment instead of reading the process environment. The transcript type is deferred until a harness produces one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/bcbench-core/README.md | 2 +- .../src/bcbench_core/agent/harness.py | 12 +- src/bcbench/agent/__init__.py | 4 +- src/bcbench/agent/copilot/__init__.py | 4 +- src/bcbench/agent/copilot/agent.py | 157 +++++++++--------- src/bcbench/agent/shared/env.py | 5 +- src/bcbench/commands/evaluate.py | 24 +-- src/bcbench/commands/run.py | 16 +- src/bcbench/evaluate/__init__.py | 3 +- src/bcbench/evaluate/base.py | 14 +- tests/test_agent_env.py | 8 + tests/test_cli_commands.py | 12 +- tests/test_copilot_agent.py | 37 +++-- tests/test_review_runners.py | 28 ++-- 14 files changed, 175 insertions(+), 151 deletions(-) diff --git a/packages/bcbench-core/README.md b/packages/bcbench-core/README.md index 14a1d09a6..206891ac3 100644 --- a/packages/bcbench-core/README.md +++ b/packages/bcbench-core/README.md @@ -23,7 +23,7 @@ The import and environment rules are enforced by ruff (`banned-api` in [`pyproje ## Agent harnesses -[`bcbench_core.agent.harness`](src/bcbench_core/agent/harness.py) defines the harness-neutral contract. A `Harness[T]` reports its `AgentIdentity` before any run and turns an `AgentRequest[T]` into an `AgentResult`. Output is a final response, a transcript, or changes left in the workspace. Applications construct harnesses with their own configuration and pass them to evaluation pipelines, which depend only on this contract. +[`bcbench_core.agent.harness`](src/bcbench_core/agent/harness.py) defines the harness-neutral contract. A `Harness[T]` reports its `AgentIdentity` before any run and turns an `AgentRequest[T]` into an `AgentResult`. Output is a final response or changes left in the workspace. Applications construct harnesses with their own configuration and pass them to evaluation pipelines, which depend only on this contract. ## Logging diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py index f434b6b64..1d51b93d2 100644 --- a/packages/bcbench-core/src/bcbench_core/agent/harness.py +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -8,14 +8,14 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Literal, Protocol +from typing import Protocol from pydantic import BaseModel, ConfigDict, Field, JsonValue from bcbench_core.agent.experiment import ExperimentConfiguration from bcbench_core.agent.metrics import AgentMetrics -__all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness", "TranscriptTurn"] +__all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness"] class AgentIdentity(BaseModel): @@ -32,13 +32,6 @@ class AgentIdentity(BaseModel): provenance: dict[str, JsonValue] = Field(default_factory=dict) -class TranscriptTurn(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - role: Literal["user", "assistant"] - content: str - - @dataclass(frozen=True) class AgentRequest[T]: task: T @@ -56,7 +49,6 @@ class AgentRequest[T]: class AgentResult: # Final answer; None when the output is left in the workspace response: str | None = None - transcript: tuple[TranscriptTurn, ...] = () metrics: AgentMetrics | None = None experiment: ExperimentConfiguration | None = None # Model reported by the agent, when it differs from or replaces the configured one diff --git a/src/bcbench/agent/__init__.py b/src/bcbench/agent/__init__.py index 4f2a4e0b6..9a08648e6 100644 --- a/src/bcbench/agent/__init__.py +++ b/src/bcbench/agent/__init__.py @@ -2,7 +2,7 @@ from bcbench.agent.bcal import BCalBackendConfig, get_bcal_version, run_bcal_agent from bcbench.agent.claude import get_claude_version, run_claude_code -from bcbench.agent.copilot import run_copilot_agent +from bcbench.agent.copilot import CopilotHarness from bcbench.agent.pr_review import get_pr_review_version, run_pr_review_agent -__all__ = ["BCalBackendConfig", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_copilot_agent", "run_pr_review_agent"] +__all__ = ["BCalBackendConfig", "CopilotHarness", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_pr_review_agent"] diff --git a/src/bcbench/agent/copilot/__init__.py b/src/bcbench/agent/copilot/__init__.py index 40648ec39..340415886 100644 --- a/src/bcbench/agent/copilot/__init__.py +++ b/src/bcbench/agent/copilot/__init__.py @@ -1,5 +1,5 @@ """GitHub Copilot CLI agent module.""" -from bcbench.agent.copilot.agent import run_copilot_agent +from bcbench.agent.copilot.agent import CopilotHarness -__all__ = ["run_copilot_agent"] +__all__ = ["CopilotHarness"] diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index d277c7b52..b82acd5f5 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -1,12 +1,13 @@ """GitHub Copilot CLI Agent implementation.""" import logging +from dataclasses import dataclass from pathlib import Path -from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot +from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, get_copilot_version, invoke_copilot from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.harness import AgentResult +from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult from bcbench_core.exceptions import AgentTimeoutError from bcbench.agent.shared import ( @@ -23,84 +24,90 @@ logger = logging.getLogger(__name__) +DEFAULT_TIMEOUT = 60 * 60 -def run_copilot_agent( - entry: BaseDatasetEntry, - model: str, - category: EvaluationCategory, - repo_path: Path, - output_dir: Path, - agent_config: AgentConfig, - plugin_root: Path, - runtime: AgentRuntimeConfig | None = None, - timeout: int = 60 * 60, -) -> AgentResult: - """Run GitHub Copilot CLI agent on a single dataset entry.""" - logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") - prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) - bc_gateway = start_bc_mcp_gateway(runtime) - mcp_config_json, mcp_server_names = build_mcp_config( - agent_config, - entry, - repo_path, - runtime=runtime, - bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, - ) - lsp_plugin_dir: Path | None = build_al_lsp_plugin( - entry, - category, - repo_path, - AgentHarness.COPILOT, - plugin_root, - runtime=runtime, - ) - profile_dir: Path = instructions_profile_dir(entry.customization_profile) - target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path) - instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename) - skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir) - custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir) - plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True) +@dataclass(frozen=True) +class CopilotHarness: + """GitHub Copilot CLI on any category, customized through the agent config.""" - config = ExperimentConfiguration( - mcp_servers=mcp_server_names, - al_lsp_enabled=lsp_plugin_dir is not None, - custom_instructions=instructions_enabled, - skills_enabled=skills_enabled, - custom_agent=custom_agent, - plugins=[plugin.record for plugin, _ in plugins] or None, - ) + model: str + category: EvaluationCategory + agent_config: AgentConfig + plugin_root: Path + runtime: AgentRuntimeConfig | None = None - try: - lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else () - # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it - # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for - # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not - # silently widen the agent's sandbox access. - options = CopilotOptions( - allow_all_tools=True, + def identity(self) -> AgentIdentity: + return AgentIdentity(harness=AgentHarness.COPILOT, version=get_copilot_version(), model=self.model) + + def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: + entry, repo_path, output_dir = request.task, request.work_dir, request.output_dir + agent_config, category, runtime, plugin_root = self.agent_config, self.category, self.runtime, self.plugin_root + logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") + + prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) + bc_gateway = start_bc_mcp_gateway(runtime) + mcp_config_json, mcp_server_names = build_mcp_config( + agent_config, + entry, + repo_path, + runtime=runtime, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) + lsp_plugin_dir: Path | None = build_al_lsp_plugin( + entry, + category, + repo_path, + AgentHarness.COPILOT, + plugin_root, + runtime=runtime, + ) + profile_dir: Path = instructions_profile_dir(entry.customization_profile) + target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path) + instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename) + skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir) + custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir) + plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True) + + config = ExperimentConfiguration( + mcp_servers=mcp_server_names, + al_lsp_enabled=lsp_plugin_dir is not None, custom_instructions=instructions_enabled, - log_dir=output_dir, - mcp_config_json=mcp_config_json, - plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)), - granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access), + skills_enabled=skills_enabled, custom_agent=custom_agent, - workspace_mcp=True, + plugins=[plugin.record for plugin, _ in plugins] or None, ) - metrics, response = invoke_copilot( - prompt=prompt, - model=model, - work_dir=repo_path, - timeout=timeout, - env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), - options=options, - ) - logger.info(f"Copilot CLI run complete for: {entry.instance_id}") - except CopilotTimeoutError as exc: - raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc - else: - return AgentResult(response=response, metrics=metrics, experiment=config) - finally: - if bc_gateway is not None: - bc_gateway.stop() + try: + lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else () + # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it + # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for + # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not + # silently widen the agent's sandbox access. + options = CopilotOptions( + allow_all_tools=True, + custom_instructions=instructions_enabled, + log_dir=output_dir, + mcp_config_json=mcp_config_json, + plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)), + granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access), + custom_agent=custom_agent, + workspace_mcp=True, + ) + + metrics, response = invoke_copilot( + prompt=prompt, + model=self.model, + work_dir=repo_path, + timeout=request.timeout or DEFAULT_TIMEOUT, + env=agent_subprocess_env(base=request.env, pass_bc_credentials=category.pass_on_bc_container_credentials), + options=options, + ) + logger.info(f"Copilot CLI run complete for: {entry.instance_id}") + except CopilotTimeoutError as exc: + raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc + else: + return AgentResult(response=response, metrics=metrics, experiment=config) + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/shared/env.py b/src/bcbench/agent/shared/env.py index 7a529e21d..b47fad8ea 100644 --- a/src/bcbench/agent/shared/env.py +++ b/src/bcbench/agent/shared/env.py @@ -1,4 +1,5 @@ import os +from collections.abc import Mapping # BC container connection details/credentials the harness uses to build the MCP config and to reach the # container. They must NOT leak into a launched agent's own process environment: otherwise the agent can @@ -12,8 +13,8 @@ _WITHHELD_ENV_VARS = frozenset({"BC_COMPANY", "BC_CONTAINER_NAME"}) -def agent_subprocess_env(overrides: dict[str, str] | None = None, *, pass_bc_credentials: bool = False) -> dict[str, str]: - env = dict(os.environ) +def agent_subprocess_env(overrides: dict[str, str] | None = None, *, base: Mapping[str, str] | None = None, pass_bc_credentials: bool = False) -> dict[str, str]: + env = dict(os.environ if base is None else base) if not pass_bc_credentials: env = {k: v for k, v in env.items() if not k.startswith(_WITHHELD_ENV_PREFIXES) and k not in _WITHHELD_ENV_VARS} if overrides: diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index fd17d54bc..848c9b254 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -1,17 +1,17 @@ import logging +import os import random from pathlib import Path from typing import Annotated, cast, override import typer -from bcbench_core.agent.copilot import get_copilot_version from bcbench_core.agent.experiment import ExperimentConfiguration from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentError from bcbench_core.filesystem import prepare_run_dir -from bcbench.agent import BCalBackendConfig, get_bcal_version, get_claude_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.agent import BCalBackendConfig, CopilotHarness, get_bcal_version, get_claude_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_pr_review_agent from bcbench.cli_options import ( AgentConfigPath, ClaudeCodeModel, @@ -34,7 +34,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.evaluate import AgentRunner, CodeReviewPipeline, EvaluationPipeline, NL2ALPipeline +from bcbench.evaluate import AgentRunner, CodeReviewPipeline, EvaluationPipeline, NL2ALPipeline, harness_runner from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.paths import SHARED_CONFIG_FILE from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult @@ -87,6 +87,8 @@ def evaluate_copilot( ) entry = category.entry_class.load(category.dataset_path(tier), entry_id=entry_id)[0] agent_config = AgentConfig.from_file(config_path) + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) + identity = harness.identity() run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI") @@ -98,25 +100,13 @@ def evaluate_copilot( container=runtime.container if runtime else None, model=model, agent_name=AgentHarness.COPILOT, - agent_version=get_copilot_version(), + agent_version=identity.version, category=category, tier=tier, ) pipeline = category.pipeline(agent_config) - pipeline.execute( - context, - lambda ctx: run_copilot_agent( - entry=ctx.entry, - repo_path=ctx.repo_path, - category=category, - model=ctx.model, - output_dir=ctx.result_dir, - agent_config=agent_config, - plugin_root=_config.paths.plugin_root, - runtime=runtime, - ), - ) + pipeline.execute(context, harness_runner(harness, os.environ)) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 1f4e024fa..78b564cfc 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -1,11 +1,13 @@ """CLI commands for running agents.""" import logging +import os from typing import Annotated, cast import typer +from bcbench_core.agent.harness import AgentRequest -from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.agent import BCalBackendConfig, CopilotHarness, run_bcal_agent, run_claude_code, run_pr_review_agent from bcbench.cli_options import ( AgentConfigPath, ClaudeCodeModel, @@ -81,16 +83,8 @@ def run_copilot( agent_config = AgentConfig.from_file(config_path) category.pipeline(agent_config).setup_workspace(entry, repo_path) - run_copilot_agent( - entry=entry, - repo_path=repo_path, - model=model, - category=category, - output_dir=output_dir, - agent_config=agent_config, - plugin_root=_config.paths.plugin_root, - runtime=runtime, - ) + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) + harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, env=os.environ)) @run_app.command("claude") diff --git a/src/bcbench/evaluate/__init__.py b/src/bcbench/evaluate/__init__.py index a0e7f1fd4..49ee8eb15 100644 --- a/src/bcbench/evaluate/__init__.py +++ b/src/bcbench/evaluate/__init__.py @@ -1,6 +1,6 @@ """Evaluation module for running pipelines and creating results.""" -from bcbench.evaluate.base import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import AgentRunner, EvaluationPipeline, harness_runner from bcbench.evaluate.bugfix import BugFixPipeline from bcbench.evaluate.codereview import CodeReviewPipeline from bcbench.evaluate.dataquery import DataQueryPipeline @@ -21,4 +21,5 @@ "ExtRequestTriagePipeline", "NL2ALPipeline", "TestGenerationPipeline", + "harness_runner", ] diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 590840677..23d9b3a80 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -2,10 +2,11 @@ import logging from abc import ABC, abstractmethod +from collections.abc import Mapping from pathlib import Path from typing import Protocol -from bcbench_core.agent.harness import AgentResult +from bcbench_core.agent.harness import AgentRequest, AgentResult, Harness from bcbench_core.exceptions import AgentTimeoutError from bcbench.dataset import BaseDatasetEntry @@ -15,13 +16,22 @@ logger = logging.getLogger(__name__) -__all__ = ["AgentRunner", "EvaluationPipeline"] +__all__ = ["AgentRunner", "EvaluationPipeline", "harness_runner"] class AgentRunner[E: BaseDatasetEntry](Protocol): def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... +def harness_runner[E: BaseDatasetEntry](harness: Harness[E], env: Mapping[str, str]) -> AgentRunner[E]: + """Run ``harness`` on the context's entry, in its repository, writing logs to its result directory.""" + + def run(context: EvaluationContext[E], /) -> AgentResult: + return harness.run(AgentRequest(task=context.entry, work_dir=context.repo_path, output_dir=context.result_dir, env=env)) + + return run + + class EvaluationPipeline[E: BaseDatasetEntry](ABC): """Abstract base class for evaluation pipelines. diff --git a/tests/test_agent_env.py b/tests/test_agent_env.py index 66f350963..c611bb61d 100644 --- a/tests/test_agent_env.py +++ b/tests/test_agent_env.py @@ -28,6 +28,14 @@ def test_preserves_other_vars(monkeypatch): assert "BC_SERVER_PASSWORD" not in env +def test_scrubs_the_given_base_instead_of_the_process_env(monkeypatch): + monkeypatch.setenv("FROM_PROCESS", "1") + + env = agent_subprocess_env(base={"PATH": "/usr/bin", "BC_SERVER_PASSWORD": "secret"}) + + assert env == {"PATH": "/usr/bin"} + + def test_preserves_bc_connection_vars_when_allowed(monkeypatch): monkeypatch.setenv("BC_SERVER_USERNAME", "admin") monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index b1a4d004d..e8d8568ee 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -190,9 +190,9 @@ def execute(self, context, agent_runner): @pytest.mark.parametrize( ("commands", "command_name", "agent_name", "extra_args"), [ - (run_commands, "run_copilot", "run_copilot_agent", {}), + (run_commands, "run_copilot", "CopilotHarness", {}), (run_commands, "run_claude", "run_claude_code", {}), - (evaluate_commands, "evaluate_copilot", "run_copilot_agent", {"run_id": "test-run"}), + (evaluate_commands, "evaluate_copilot", "CopilotHarness", {"run_id": "test-run"}), (evaluate_commands, "evaluate_claude_code", "run_claude_code", {"run_id": "test-run"}), ], ) @@ -201,7 +201,6 @@ def test_agent_commands_preserve_requested_mcp_flags(tmp_path, agent_command_cat with ( patch.object(commands, agent_name) as run_agent, - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): getattr(commands, command_name)( @@ -226,9 +225,9 @@ def test_agent_commands_preserve_requested_mcp_flags(tmp_path, agent_command_cat @pytest.mark.parametrize( ("commands", "command_name", "agent_name", "extra_args"), [ - (run_commands, "run_copilot", "run_copilot_agent", {}), + (run_commands, "run_copilot", "CopilotHarness", {}), (run_commands, "run_claude", "run_claude_code", {}), - (evaluate_commands, "evaluate_copilot", "run_copilot_agent", {"run_id": "test-run"}), + (evaluate_commands, "evaluate_copilot", "CopilotHarness", {"run_id": "test-run"}), (evaluate_commands, "evaluate_claude_code", "run_claude_code", {"run_id": "test-run"}), ], ) @@ -239,7 +238,6 @@ def test_agent_commands_pass_the_given_agent_config(tmp_path, agent_command_cate with ( patch.object(commands, agent_name) as run_agent, - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): getattr(commands, command_name)( @@ -324,7 +322,7 @@ def test_result_summarize_creates_all_outputs(sample_results_directory, problem_ with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), patch.object(EvaluationCategory, "dataset_path", return_value=dataset_path), - patch.object(evaluate_commands, "get_copilot_version", side_effect=AssertionError("Must use artifact version")), + patch("bcbench.agent.copilot.agent.get_copilot_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_claude_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_pr_review_version", side_effect=AssertionError("Must use artifact version")), ): diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index d7d2d5e7d..ed0ca3fbf 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -1,3 +1,4 @@ +import os import subprocess from pathlib import Path from unittest.mock import DEFAULT, Mock, patch @@ -5,14 +6,30 @@ import pytest from bcbench_core.agent.copilot import CopilotProcessError, CopilotTimeoutError from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentRequest, AgentResult from bcbench_core.exceptions import AgentTimeoutError -from bcbench.agent.copilot.agent import run_copilot_agent +from bcbench.agent.copilot.agent import CopilotHarness +from bcbench.dataset import BaseDatasetEntry from bcbench.operations import instructions_profile_dir from bcbench.types import AgentConfig, EvaluationCategory, PluginConfig from tests.conftest import create_dataset_entry +def run_copilot_harness( + entry: BaseDatasetEntry, + model: str, + category: EvaluationCategory, + repo_path: Path, + output_dir: Path, + agent_config: AgentConfig, + plugin_root: Path, + timeout: int | None = None, +) -> AgentResult: + harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=plugin_root) + return harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, env=os.environ, timeout=timeout)) + + def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Path, monkeypatch): repo_path = tmp_path / "repo" output_dir = tmp_path / "output" @@ -36,7 +53,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - result = run_copilot_agent( + result = run_copilot_harness( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -96,7 +113,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - config = run_copilot_agent( + config = run_copilot_harness( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -163,7 +180,7 @@ def test_copilot_timeout_adapts_core_metrics_and_preserves_experiment_metadata(c run.side_effect = subprocess.TimeoutExpired(["copilot"], timeout) with pytest.raises(AgentTimeoutError, match="Copilot CLI timed out") as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins", timeout=timeout) + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins", timeout=timeout) assert error.value.metrics is not None assert error.value.metrics.execution_time == timeout @@ -179,7 +196,7 @@ def test_copilot_process_errors_propagate_and_stop_gateway(configured_copilot_ru run.side_effect = failure with pytest.raises(CopilotProcessError) as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") assert error.value.__cause__ is failure gateway.stop.assert_called_once_with() @@ -189,7 +206,7 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop repo_path, output_dir, gateway, run = configured_copilot_run with patch("bcbench_core.agent.copilot.agent._find_copilot", return_value=None), pytest.raises(CopilotProcessError, match="not found"): - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") run.assert_not_called() gateway.stop.assert_called_once_with() @@ -198,7 +215,7 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop def test_successful_copilot_run_stops_gateway(configured_copilot_run): repo_path, output_dir, gateway, _ = configured_copilot_run - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") gateway.stop.assert_called_once_with() @@ -212,7 +229,7 @@ def test_copilot_sets_up_the_session_from_the_given_agent_config(configured_copi mocks["build_mcp_config"].return_value = (None, None) mocks["setup_custom_agent"].return_value = None mocks["resolve_config_plugins"].return_value = [] - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, agent_config, repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, agent_config, repo_path.parent / "plugins") for name in ("build_prompt", "build_mcp_config", "resolve_config_plugins"): assert agent_config in mocks[name].call_args.args, name @@ -227,7 +244,7 @@ def test_copilot_sets_up_customizations_from_the_entry_profile_into_github(confi with patch.multiple("bcbench.agent.copilot.agent", **dict.fromkeys(helpers, DEFAULT)) as mocks: mocks["setup_custom_agent"].return_value = None - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), repo_path.parent / "plugins") profile_dir = instructions_profile_dir(create_dataset_entry().customization_profile) for name in helpers: @@ -242,7 +259,7 @@ def test_copilot_keeps_plugins_in_the_given_plugin_root(configured_copilot_run): with patch.multiple("bcbench.agent.copilot.agent", build_al_lsp_plugin=DEFAULT, resolve_config_plugins=DEFAULT) as mocks: mocks["build_al_lsp_plugin"].return_value = None mocks["resolve_config_plugins"].return_value = [] - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), plugin_root) + run_copilot_harness(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, AgentConfig(), plugin_root) assert plugin_root in mocks["build_al_lsp_plugin"].call_args.args assert plugin_root in mocks["resolve_config_plugins"].call_args.args diff --git a/tests/test_review_runners.py b/tests/test_review_runners.py index 9f2bba7f2..21fba629c 100644 --- a/tests/test_review_runners.py +++ b/tests/test_review_runners.py @@ -12,21 +12,27 @@ from bcbench.types import AgentHarness, EvaluationCategory -@pytest.mark.parametrize( - ("command", "runner_name"), - [ - (run_commands.run_copilot, "run_copilot_agent"), - (run_commands.run_claude, "run_claude_code"), - ], -) -def test_generic_run_commands_accept_code_review(tmp_path: Path, command, runner_name: str) -> None: +def test_copilot_run_command_accepts_code_review(tmp_path: Path) -> None: + entry = object() + with ( + patch.object(CodeReviewEntry, "load", return_value=[entry]), + patch.object(CodeReviewPipeline, "setup_workspace"), + patch.object(run_commands, "CopilotHarness") as harness, + ): + run_commands.run_copilot("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") + + assert harness.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW + assert harness.return_value.run.call_args.args[0].task is entry + + +def test_claude_run_command_accepts_code_review(tmp_path: Path) -> None: entry = object() with ( patch.object(CodeReviewEntry, "load", return_value=[entry]), patch.object(CodeReviewPipeline, "setup_workspace"), - patch.object(run_commands, runner_name) as agent_runner, + patch.object(run_commands, "run_claude_code") as agent_runner, ): - command("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") + run_commands.run_claude("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") assert agent_runner.call_args.kwargs["entry"] is entry assert agent_runner.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW @@ -44,7 +50,7 @@ def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, comm with ( patch.object(CodeReviewEntry, "load", return_value=[object()]), patch.object(CodeReviewPipeline, "execute", side_effect=lambda context, runner: contexts.append(context)), - patch.object(evaluate_commands, "get_copilot_version", return_value="1.2.3"), + patch("bcbench.agent.copilot.agent.get_copilot_version", return_value="1.2.3"), patch.object(evaluate_commands, "get_claude_version", return_value="1.2.3"), ): command( From d5343981e76537c27a8bf4a4770e4ba834f5731f Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 12:02:03 +0100 Subject: [PATCH 4/7] Carry the AgentIdentity on EvaluationContext and keep env out of the contract EvaluationContext now holds the resolved AgentIdentity instead of separate agent_name, model and agent_version fields; results read harness, model and version from it. The environment is a harness concern, not per-run input, so AgentRequest no longer carries it and the Copilot harness reads it as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/bcbench_core/agent/harness.py | 3 -- src/bcbench/agent/copilot/agent.py | 2 +- src/bcbench/agent/shared/env.py | 5 ++-- src/bcbench/commands/evaluate.py | 29 ++++++------------- src/bcbench/commands/run.py | 3 +- src/bcbench/evaluate/base.py | 7 ++--- src/bcbench/evaluate/bugfix.py | 2 +- src/bcbench/evaluate/codereview.py | 2 +- src/bcbench/evaluate/dataquery.py | 2 +- src/bcbench/evaluate/ext_request_advisor.py | 2 +- src/bcbench/evaluate/ext_request_implement.py | 2 +- src/bcbench/evaluate/ext_request_triage.py | 2 +- src/bcbench/evaluate/nl2al.py | 2 +- src/bcbench/evaluate/testgeneration.py | 2 +- src/bcbench/results/base.py | 13 +++++---- src/bcbench/types.py | 8 ++--- tests/conftest.py | 8 ++--- tests/test_agent_env.py | 8 ----- tests/test_agent_version_results.py | 7 ++--- tests/test_cli_commands.py | 4 +-- tests/test_copilot_agent.py | 3 +- tests/test_dataquery_evaluation.py | 4 +-- tests/test_review_runners.py | 10 +++---- 23 files changed, 50 insertions(+), 80 deletions(-) diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py index 1d51b93d2..264c50e02 100644 --- a/packages/bcbench-core/src/bcbench_core/agent/harness.py +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -5,7 +5,6 @@ so evaluation pipelines never depend on a concrete harness. """ -from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from typing import Protocol @@ -39,8 +38,6 @@ class AgentRequest[T]: work_dir: Path # Receives logs and raw captures output_dir: Path - # Passed explicitly; harnesses never read the process environment - env: Mapping[str, str] # None uses the harness's default timeout: int | None = None diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index b82acd5f5..f2cd8bdba 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -100,7 +100,7 @@ def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: model=self.model, work_dir=repo_path, timeout=request.timeout or DEFAULT_TIMEOUT, - env=agent_subprocess_env(base=request.env, pass_bc_credentials=category.pass_on_bc_container_credentials), + env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), options=options, ) logger.info(f"Copilot CLI run complete for: {entry.instance_id}") diff --git a/src/bcbench/agent/shared/env.py b/src/bcbench/agent/shared/env.py index b47fad8ea..7a529e21d 100644 --- a/src/bcbench/agent/shared/env.py +++ b/src/bcbench/agent/shared/env.py @@ -1,5 +1,4 @@ import os -from collections.abc import Mapping # BC container connection details/credentials the harness uses to build the MCP config and to reach the # container. They must NOT leak into a launched agent's own process environment: otherwise the agent can @@ -13,8 +12,8 @@ _WITHHELD_ENV_VARS = frozenset({"BC_COMPANY", "BC_CONTAINER_NAME"}) -def agent_subprocess_env(overrides: dict[str, str] | None = None, *, base: Mapping[str, str] | None = None, pass_bc_credentials: bool = False) -> dict[str, str]: - env = dict(os.environ if base is None else base) +def agent_subprocess_env(overrides: dict[str, str] | None = None, *, pass_bc_credentials: bool = False) -> dict[str, str]: + env = dict(os.environ) if not pass_bc_credentials: env = {k: v for k, v in env.items() if not k.startswith(_WITHHELD_ENV_PREFIXES) and k not in _WITHHELD_ENV_VARS} if overrides: diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 848c9b254..f4ebe7f11 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -1,12 +1,11 @@ import logging -import os import random from pathlib import Path from typing import Annotated, cast, override import typer from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.harness import AgentResult +from bcbench_core.agent.harness import AgentIdentity, AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentError from bcbench_core.filesystem import prepare_run_dir @@ -88,7 +87,6 @@ def evaluate_copilot( entry = category.entry_class.load(category.dataset_path(tier), entry_id=entry_id)[0] agent_config = AgentConfig.from_file(config_path) harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) - identity = harness.identity() run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI") @@ -98,15 +96,13 @@ def evaluate_copilot( repo_path=repo_path, result_dir=run_dir, container=runtime.container if runtime else None, - model=model, - agent_name=AgentHarness.COPILOT, - agent_version=identity.version, + agent=harness.identity(), category=category, tier=tier, ) pipeline = category.pipeline(agent_config) - pipeline.execute(context, harness_runner(harness, os.environ)) + pipeline.execute(context, harness_runner(harness)) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") @@ -162,9 +158,7 @@ def evaluate_claude_code( repo_path=repo_path, result_dir=run_dir, container=runtime.container if runtime else None, - model=model, - agent_name=AgentHarness.CLAUDE, - agent_version=get_claude_version(), + agent=AgentIdentity(harness=AgentHarness.CLAUDE, version=get_claude_version(), model=model), category=category, tier=tier, ) @@ -176,7 +170,7 @@ def evaluate_claude_code( entry=ctx.entry, repo_path=ctx.repo_path, category=category, - model=ctx.model, + model=model, output_dir=ctx.result_dir, agent_config=agent_config, plugin_root=_config.paths.plugin_root, @@ -219,9 +213,7 @@ def evaluate_pr_review( repo_path=repo_path, result_dir=run_dir, container=None, - model=model, - agent_name=AgentHarness.PR_REVIEW, - agent_version=get_pr_review_version(engine_path), + agent=AgentIdentity(harness=AgentHarness.PR_REVIEW, version=get_pr_review_version(engine_path), model=model), category=category, ) @@ -231,7 +223,7 @@ def evaluate_pr_review( entry=ctx.entry, repo_path=ctx.repo_path, category=category, - model=ctx.model, + model=model, output_dir=ctx.result_dir, engine_path=engine_path, ), @@ -271,9 +263,7 @@ def evaluate_bcal( repo_path=repo_path, result_dir=run_dir, container=None, - model=llm_model or "external-command", - agent_name=AgentHarness.BCAL, - agent_version=get_bcal_version(), + agent=AgentIdentity(harness=AgentHarness.BCAL, version=get_bcal_version(), model=llm_model or "external-command"), category=category, tier=tier, ) @@ -343,8 +333,7 @@ def evaluate_mock( entry=entry, repo_path=Path(), result_dir=run_dir, - model="mock-model", - agent_name=AgentHarness.MOCK, + agent=AgentIdentity(harness=AgentHarness.MOCK, version="mock", model="mock-model"), category=category, tier=tier, ) diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 78b564cfc..c259c52fe 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -1,7 +1,6 @@ """CLI commands for running agents.""" import logging -import os from typing import Annotated, cast import typer @@ -84,7 +83,7 @@ def run_copilot( category.pipeline(agent_config).setup_workspace(entry, repo_path) harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=_config.paths.plugin_root, runtime=runtime) - harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, env=os.environ)) + harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir)) @run_app.command("claude") diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 23d9b3a80..aca891b2e 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -2,7 +2,6 @@ import logging from abc import ABC, abstractmethod -from collections.abc import Mapping from pathlib import Path from typing import Protocol @@ -23,11 +22,11 @@ class AgentRunner[E: BaseDatasetEntry](Protocol): def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... -def harness_runner[E: BaseDatasetEntry](harness: Harness[E], env: Mapping[str, str]) -> AgentRunner[E]: - """Run ``harness`` on the context's entry, in its repository, writing logs to its result directory.""" +def harness_runner[E: BaseDatasetEntry](harness: Harness[E]) -> AgentRunner[E]: + """Adapt ``harness`` to the pipelines' ``AgentRunner`` until every harness implements ``Harness``.""" def run(context: EvaluationContext[E], /) -> AgentResult: - return harness.run(AgentRequest(task=context.entry, work_dir=context.repo_path, output_dir=context.result_dir, env=env)) + return harness.run(AgentRequest(task=context.entry, work_dir=context.repo_path, output_dir=context.result_dir)) return run diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index f34c113e7..8dded4c63 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -44,7 +44,7 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None: @override def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/codereview.py b/src/bcbench/evaluate/codereview.py index 4e574e387..7fa87f7f8 100644 --- a/src/bcbench/evaluate/codereview.py +++ b/src/bcbench/evaluate/codereview.py @@ -56,7 +56,7 @@ def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None: @override def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index af618f436..2665e1a5a 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -106,7 +106,7 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: @override def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: AgentRunner[DataQueryEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/ext_request_advisor.py b/src/bcbench/evaluate/ext_request_advisor.py index 0bffb5d3d..58e0e23ad 100644 --- a/src/bcbench/evaluate/ext_request_advisor.py +++ b/src/bcbench/evaluate/ext_request_advisor.py @@ -30,7 +30,7 @@ def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/ext_request_implement.py b/src/bcbench/evaluate/ext_request_implement.py index ae4368537..d439d9aee 100644 --- a/src/bcbench/evaluate/ext_request_implement.py +++ b/src/bcbench/evaluate/ext_request_implement.py @@ -37,7 +37,7 @@ def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/ext_request_triage.py b/src/bcbench/evaluate/ext_request_triage.py index f310fd9c9..5366bef71 100644 --- a/src/bcbench/evaluate/ext_request_triage.py +++ b/src/bcbench/evaluate/ext_request_triage.py @@ -40,7 +40,7 @@ def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index 54f567f00..40cb7a99d 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -71,7 +71,7 @@ def setup(self, context: EvaluationContext[NL2ALEntry]) -> None: def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentRunner[NL2ALEntry]) -> None: # Single attempt — retries are disabled. An empty diff (the agent asked for clarification # instead of editing) is scored as a failure in evaluate(), not re-run. - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index 51a7c597b..12e7704d9 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -66,7 +66,7 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None: @override def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + with github_log_group(f"{context.agent.harness} -- Entry: {context.entry.instance_id}"): result = agent_runner(context) context.metrics, context.experiment = result.metrics, result.experiment diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 765617267..46fdef699 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, model_validator from bcbench.dataset import BaseDatasetEntry -from bcbench.types import AnyAgentMetrics, DatasetTier, EvaluationCategory, EvaluationContext +from bcbench.types import AgentHarness, AnyAgentMetrics, DatasetTier, EvaluationCategory, EvaluationContext logger = logging.getLogger(__name__) @@ -38,22 +38,23 @@ class BaseEvaluationResult(BaseModel): @classmethod def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dict[str, Any]: - metrics_contract = context.agent_name.metrics_contract + agent_name = AgentHarness(context.agent.harness) + metrics_contract = agent_name.metrics_contract if not context.metrics: logger.warning(f"Creating result for {context.entry.instance_id} with no agent metrics - performance data will be unavailable") elif type(context.metrics) is not metrics_contract.metrics_type: - raise TypeError(f"{context.agent_name} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") + raise TypeError(f"{agent_name} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") elif missing_metrics := sorted(name for name in metrics_contract.required_fields if getattr(context.metrics, name) is None): logger.warning(f"Result for {context.entry.instance_id} missing metrics: {', '.join(missing_metrics)}") return { "instance_id": context.entry.instance_id, "project": context.entry.extract_project_name(), - "model": context.model.replace(".", "-"), + "model": (context.agent.model or "").replace(".", "-"), "category": context.category, "tier": context.tier, - "agent_name": context.agent_name, - "agent_version": context.agent_version, + "agent_name": agent_name, + "agent_version": context.agent.version, "metrics": context.metrics, "experiment": context.experiment, } diff --git a/src/bcbench/types.py b/src/bcbench/types.py index c75a1e733..275a32130 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -10,6 +10,7 @@ import yaml from bcbench_core.agent.customization import CustomAgentsConfig, ToggleConfig from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.container import ContainerConfig from pydantic import BaseModel, ConfigDict, Field, StrictBool, StringConstraints, model_validator @@ -629,9 +630,8 @@ class EvaluationContext[E: BaseDatasetEntry]: repo_path: Path result_dir: Path - # Agent metadata - agent_name: AgentHarness - model: str + # The agent under evaluation, resolved before running + agent: AgentIdentity # Evaluation category category: EvaluationCategory @@ -640,8 +640,6 @@ class EvaluationContext[E: BaseDatasetEntry]: # BC Container configuration (optional — not all categories require a container) container: ContainerConfig | None = None - agent_version: str | None = None - # Agent execution metrics metrics: AgentMetrics | None = None diff --git a/tests/conftest.py b/tests/conftest.py index 0aae5ea5f..dd5d24dd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ import pytest from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.container import ContainerConfig from bcbench_core.dataset import TestEntry @@ -85,6 +86,7 @@ def create_evaluation_context[EntryT: BaseDatasetEntry]( entry: EntryT | None = None, agent_name: AgentHarness = AgentHarness.COPILOT, model: str = "test-model", + agent_version: str = "1.0.0", category: EvaluationCategory = EvaluationCategory.BUG_FIX, container_name: str = "test-container", password: str = "test-password", @@ -97,8 +99,7 @@ def create_evaluation_context[EntryT: BaseDatasetEntry]( repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig(name=container_name, username=username, password=password, company="CRONUS"), - agent_name=agent_name, - model=model, + agent=AgentIdentity(harness=agent_name, version=agent_version, model=model), category=category, ) @@ -218,8 +219,7 @@ def create_codereview_result( repo_path=Path(), result_dir=Path(), container=ContainerConfig(name="t", username="t", password="t", company="CRONUS"), - agent_name=agent_name, - model=model, + agent=AgentIdentity(harness=agent_name, version="1.0.0", model=model), category=EvaluationCategory.CODE_REVIEW, ) context.metrics = metrics diff --git a/tests/test_agent_env.py b/tests/test_agent_env.py index c611bb61d..66f350963 100644 --- a/tests/test_agent_env.py +++ b/tests/test_agent_env.py @@ -28,14 +28,6 @@ def test_preserves_other_vars(monkeypatch): assert "BC_SERVER_PASSWORD" not in env -def test_scrubs_the_given_base_instead_of_the_process_env(monkeypatch): - monkeypatch.setenv("FROM_PROCESS", "1") - - env = agent_subprocess_env(base={"PATH": "/usr/bin", "BC_SERVER_PASSWORD": "secret"}) - - assert env == {"PATH": "/usr/bin"} - - def test_preserves_bc_connection_vars_when_allowed(monkeypatch): monkeypatch.setenv("BC_SERVER_USERNAME", "admin") monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") diff --git a/tests/test_agent_version_results.py b/tests/test_agent_version_results.py index 1af54016f..e785746f2 100644 --- a/tests/test_agent_version_results.py +++ b/tests/test_agent_version_results.py @@ -17,9 +17,7 @@ @pytest.mark.parametrize("version", ["1.2.3", "a" * 40, None]) def test_agent_version_round_trips_through_results_and_summaries(tmp_path: Path, version: str | None) -> None: - context = create_evaluation_context(tmp_path) - context.agent_version = version - result = BugFixResult.create_success(context, "patch") + result = BugFixResult.create_success(create_evaluation_context(tmp_path), "patch").model_copy(update={"agent_version": version}) result.save(tmp_path, "result.jsonl") restored = BaseEvaluationResult.from_json(json.loads((tmp_path / "result.jsonl").read_text())) summary = EvaluationResultSummary.from_results([restored], "run") @@ -32,8 +30,7 @@ def test_agent_version_round_trips_through_results_and_summaries(tmp_path: Path, def test_timeout_result_keeps_version_resolved_before_execution(tmp_path: Path) -> None: - context = create_evaluation_context(tmp_path) - context.agent_version = "1.2.3" + context = create_evaluation_context(tmp_path, agent_version="1.2.3") pipeline = context.category.pipeline(AgentConfig()) with ( patch.object(type(pipeline), "setup"), diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index e8d8568ee..0060be941 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -142,8 +142,8 @@ def execute(self, context, _agent_runner): llm_model=llm_model, ) - assert captured["context"].model == expected_model - assert captured["context"].agent_version == "18.0.1.2-beta" + assert captured["context"].agent.model == expected_model + assert captured["context"].agent.version == "18.0.1.2-beta" def _option_names(command: click.Command) -> set[str]: diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index ed0ca3fbf..4a47b8c55 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -1,4 +1,3 @@ -import os import subprocess from pathlib import Path from unittest.mock import DEFAULT, Mock, patch @@ -27,7 +26,7 @@ def run_copilot_harness( timeout: int | None = None, ) -> AgentResult: harness = CopilotHarness(model=model, category=category, agent_config=agent_config, plugin_root=plugin_root) - return harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, env=os.environ, timeout=timeout)) + return harness.run(AgentRequest(task=entry, work_dir=repo_path, output_dir=output_dir, timeout=timeout)) def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Path, monkeypatch): diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 28783bb85..c477acc33 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from bcbench_core.agent.harness import AgentIdentity from bcbench_core.container import ContainerConfig from bcbench_core.exceptions import BuildError @@ -87,8 +88,7 @@ def _context(self, tmp_path: Path) -> EvaluationContext[DataQueryEntry]: repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig("bcbench", "admin", "secret", company="CRONUS"), - model="test-model", - agent_name=AgentHarness.COPILOT, + agent=AgentIdentity(harness=AgentHarness.COPILOT, version="1.0.0", model="test-model"), category=EvaluationCategory.DATA_QUERY, ) diff --git a/tests/test_review_runners.py b/tests/test_review_runners.py index 21fba629c..fc934e98d 100644 --- a/tests/test_review_runners.py +++ b/tests/test_review_runners.py @@ -62,8 +62,8 @@ def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, comm ) assert len(contexts) == 1 - assert contexts[0].agent_name is agent_name - assert contexts[0].agent_version == "1.2.3" + assert contexts[0].agent.harness == agent_name + assert contexts[0].agent.version == "1.2.3" assert contexts[0].category is EvaluationCategory.CODE_REVIEW @@ -94,10 +94,10 @@ def test_pr_review_evaluation_is_fixed_to_runner_and_category(tmp_path: Path) -> assert result.exit_code == 0, result.exception assert len(contexts) == 1 - assert contexts[0].agent_name is AgentHarness.PR_REVIEW - assert contexts[0].agent_version == "a" * 40 + assert contexts[0].agent.harness == AgentHarness.PR_REVIEW + assert contexts[0].agent.version == "a" * 40 assert contexts[0].category is EvaluationCategory.CODE_REVIEW - assert contexts[0].model == "gpt-5.6-luna" + assert contexts[0].agent.model == "gpt-5.6-luna" assert agent_runner.call_args.kwargs["engine_path"] == tmp_path get_version.assert_called_once_with(tmp_path) From 9be520efd92c9d52d9617d38b4b67df6ecc8ab07 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 13:41:56 +0100 Subject: [PATCH 5/7] Type the harness name, declare CopilotHarness as a Harness, and make its timeout a field AgentIdentity and Harness take an optional harness-name type, so BC-Bench uses AgentIdentity[AgentHarness] and reads the enum (and its metrics contract) without converting from str. CopilotHarness explicitly inherits Harness so ty checks it at the class definition, and its default timeout is a dataclass field instead of a module constant. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/bcbench_core/agent/harness.py | 15 +++++++++------ src/bcbench/agent/copilot/agent.py | 16 +++++++++------- src/bcbench/commands/evaluate.py | 8 ++++---- src/bcbench/evaluate/base.py | 4 ++-- src/bcbench/results/base.py | 9 ++++----- src/bcbench/types.py | 2 +- tests/conftest.py | 4 ++-- tests/test_dataquery_evaluation.py | 2 +- tests/test_review_runners.py | 4 ++-- 9 files changed, 34 insertions(+), 30 deletions(-) diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py index 264c50e02..407bdf64f 100644 --- a/packages/bcbench-core/src/bcbench_core/agent/harness.py +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -17,12 +17,15 @@ __all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness"] -class AgentIdentity(BaseModel): - """What is being evaluated, resolved before any task runs.""" +class AgentIdentity[N: str = str](BaseModel): + """What is being evaluated, resolved before any task runs. + + ``N`` lets applications type the harness name, e.g. with their own ``StrEnum`` of harnesses. + """ model_config = ConfigDict(frozen=True, extra="forbid") - harness: str + harness: N # CLI version, engine commit, or hosted agent version version: str # Configured model; None when the harness pins or delegates model selection @@ -52,13 +55,13 @@ class AgentResult: observed_model: str | None = None -class Harness[T](Protocol): - """Runs an agent on tasks of type ``T``. +class Harness[T, N: str = str](Protocol): + """Runs an agent on tasks of type ``T``, identified by a harness name of type ``N``. ``identity`` checks the harness is usable and raises ``AgentError`` otherwise. ``run`` raises ``AgentTimeoutError`` when the timeout is exceeded and ``AgentError`` for any other failure. """ - def identity(self) -> AgentIdentity: ... + def identity(self) -> AgentIdentity[N]: ... def run(self, request: AgentRequest[T], /) -> AgentResult: ... diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index f2cd8bdba..83d14b907 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -3,11 +3,12 @@ import logging from dataclasses import dataclass from pathlib import Path +from typing import override from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, get_copilot_version, invoke_copilot from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult +from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult, Harness from bcbench_core.exceptions import AgentTimeoutError from bcbench.agent.shared import ( @@ -24,11 +25,9 @@ logger = logging.getLogger(__name__) -DEFAULT_TIMEOUT = 60 * 60 - @dataclass(frozen=True) -class CopilotHarness: +class CopilotHarness(Harness[BaseDatasetEntry, AgentHarness]): """GitHub Copilot CLI on any category, customized through the agent config.""" model: str @@ -36,10 +35,13 @@ class CopilotHarness: agent_config: AgentConfig plugin_root: Path runtime: AgentRuntimeConfig | None = None + timeout: int = 60 * 60 - def identity(self) -> AgentIdentity: - return AgentIdentity(harness=AgentHarness.COPILOT, version=get_copilot_version(), model=self.model) + @override + def identity(self) -> AgentIdentity[AgentHarness]: + return AgentIdentity[AgentHarness](harness=AgentHarness.COPILOT, version=get_copilot_version(), model=self.model) + @override def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: entry, repo_path, output_dir = request.task, request.work_dir, request.output_dir agent_config, category, runtime, plugin_root = self.agent_config, self.category, self.runtime, self.plugin_root @@ -99,7 +101,7 @@ def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: prompt=prompt, model=self.model, work_dir=repo_path, - timeout=request.timeout or DEFAULT_TIMEOUT, + timeout=request.timeout or self.timeout, env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), options=options, ) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index f4ebe7f11..925249d03 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -158,7 +158,7 @@ def evaluate_claude_code( repo_path=repo_path, result_dir=run_dir, container=runtime.container if runtime else None, - agent=AgentIdentity(harness=AgentHarness.CLAUDE, version=get_claude_version(), model=model), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.CLAUDE, version=get_claude_version(), model=model), category=category, tier=tier, ) @@ -213,7 +213,7 @@ def evaluate_pr_review( repo_path=repo_path, result_dir=run_dir, container=None, - agent=AgentIdentity(harness=AgentHarness.PR_REVIEW, version=get_pr_review_version(engine_path), model=model), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.PR_REVIEW, version=get_pr_review_version(engine_path), model=model), category=category, ) @@ -263,7 +263,7 @@ def evaluate_bcal( repo_path=repo_path, result_dir=run_dir, container=None, - agent=AgentIdentity(harness=AgentHarness.BCAL, version=get_bcal_version(), model=llm_model or "external-command"), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.BCAL, version=get_bcal_version(), model=llm_model or "external-command"), category=category, tier=tier, ) @@ -333,7 +333,7 @@ def evaluate_mock( entry=entry, repo_path=Path(), result_dir=run_dir, - agent=AgentIdentity(harness=AgentHarness.MOCK, version="mock", model="mock-model"), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.MOCK, version="mock", model="mock-model"), category=category, tier=tier, ) diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index aca891b2e..8b037a6b2 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -11,7 +11,7 @@ from bcbench.dataset import BaseDatasetEntry from bcbench.results import BaseEvaluationResult from bcbench.results.base import RESULT_FILE_SUFFIX -from bcbench.types import EvaluationContext +from bcbench.types import AgentHarness, EvaluationContext logger = logging.getLogger(__name__) @@ -22,7 +22,7 @@ class AgentRunner[E: BaseDatasetEntry](Protocol): def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... -def harness_runner[E: BaseDatasetEntry](harness: Harness[E]) -> AgentRunner[E]: +def harness_runner[E: BaseDatasetEntry](harness: Harness[E, AgentHarness]) -> AgentRunner[E]: """Adapt ``harness`` to the pipelines' ``AgentRunner`` until every harness implements ``Harness``.""" def run(context: EvaluationContext[E], /) -> AgentResult: diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 46fdef699..16b32237b 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, model_validator from bcbench.dataset import BaseDatasetEntry -from bcbench.types import AgentHarness, AnyAgentMetrics, DatasetTier, EvaluationCategory, EvaluationContext +from bcbench.types import AnyAgentMetrics, DatasetTier, EvaluationCategory, EvaluationContext logger = logging.getLogger(__name__) @@ -38,12 +38,11 @@ class BaseEvaluationResult(BaseModel): @classmethod def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dict[str, Any]: - agent_name = AgentHarness(context.agent.harness) - metrics_contract = agent_name.metrics_contract + metrics_contract = context.agent.harness.metrics_contract if not context.metrics: logger.warning(f"Creating result for {context.entry.instance_id} with no agent metrics - performance data will be unavailable") elif type(context.metrics) is not metrics_contract.metrics_type: - raise TypeError(f"{agent_name} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") + raise TypeError(f"{context.agent.harness} must use {metrics_contract.metrics_type.__name__}, got {type(context.metrics).__name__}") elif missing_metrics := sorted(name for name in metrics_contract.required_fields if getattr(context.metrics, name) is None): logger.warning(f"Result for {context.entry.instance_id} missing metrics: {', '.join(missing_metrics)}") @@ -53,7 +52,7 @@ def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dic "model": (context.agent.model or "").replace(".", "-"), "category": context.category, "tier": context.tier, - "agent_name": agent_name, + "agent_name": context.agent.harness, "agent_version": context.agent.version, "metrics": context.metrics, "experiment": context.experiment, diff --git a/src/bcbench/types.py b/src/bcbench/types.py index 275a32130..0500a2c08 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -631,7 +631,7 @@ class EvaluationContext[E: BaseDatasetEntry]: result_dir: Path # The agent under evaluation, resolved before running - agent: AgentIdentity + agent: AgentIdentity[AgentHarness] # Evaluation category category: EvaluationCategory diff --git a/tests/conftest.py b/tests/conftest.py index dd5d24dd1..1ac1d615e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,7 +99,7 @@ def create_evaluation_context[EntryT: BaseDatasetEntry]( repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig(name=container_name, username=username, password=password, company="CRONUS"), - agent=AgentIdentity(harness=agent_name, version=agent_version, model=model), + agent=AgentIdentity[AgentHarness](harness=agent_name, version=agent_version, model=model), category=category, ) @@ -219,7 +219,7 @@ def create_codereview_result( repo_path=Path(), result_dir=Path(), container=ContainerConfig(name="t", username="t", password="t", company="CRONUS"), - agent=AgentIdentity(harness=agent_name, version="1.0.0", model=model), + agent=AgentIdentity[AgentHarness](harness=agent_name, version="1.0.0", model=model), category=EvaluationCategory.CODE_REVIEW, ) context.metrics = metrics diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index c477acc33..d2d2a89c7 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -88,7 +88,7 @@ def _context(self, tmp_path: Path) -> EvaluationContext[DataQueryEntry]: repo_path=tmp_path / "repo", result_dir=tmp_path / "results", container=ContainerConfig("bcbench", "admin", "secret", company="CRONUS"), - agent=AgentIdentity(harness=AgentHarness.COPILOT, version="1.0.0", model="test-model"), + agent=AgentIdentity[AgentHarness](harness=AgentHarness.COPILOT, version="1.0.0", model="test-model"), category=EvaluationCategory.DATA_QUERY, ) diff --git a/tests/test_review_runners.py b/tests/test_review_runners.py index fc934e98d..0d1ebe225 100644 --- a/tests/test_review_runners.py +++ b/tests/test_review_runners.py @@ -62,7 +62,7 @@ def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, comm ) assert len(contexts) == 1 - assert contexts[0].agent.harness == agent_name + assert contexts[0].agent.harness is agent_name assert contexts[0].agent.version == "1.2.3" assert contexts[0].category is EvaluationCategory.CODE_REVIEW @@ -94,7 +94,7 @@ def test_pr_review_evaluation_is_fixed_to_runner_and_category(tmp_path: Path) -> assert result.exit_code == 0, result.exception assert len(contexts) == 1 - assert contexts[0].agent.harness == AgentHarness.PR_REVIEW + assert contexts[0].agent.harness is AgentHarness.PR_REVIEW assert contexts[0].agent.version == "a" * 40 assert contexts[0].category is EvaluationCategory.CODE_REVIEW assert contexts[0].agent.model == "gpt-5.6-luna" From 9996d8b54a3e02fc0ff86a3826118d3942d92d54 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 13:59:42 +0100 Subject: [PATCH 6/7] Trim the harness contract to what harnesses use today Drop AgentIdentity.provenance and AgentResult.observed_model until a harness needs them, simplify the module docstring, and remove the README section. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/bcbench-core/README.md | 4 ---- .../bcbench-core/src/bcbench_core/agent/harness.py | 13 ++----------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/packages/bcbench-core/README.md b/packages/bcbench-core/README.md index 206891ac3..d762163ab 100644 --- a/packages/bcbench-core/README.md +++ b/packages/bcbench-core/README.md @@ -21,10 +21,6 @@ The import and environment rules are enforced by ruff (`banned-api` in [`pyproje - For Copilot invocation: an installed and authenticated [GitHub Copilot CLI](https://github.com/github/copilot-cli). - For BC container operations: PowerShell 7 with the latest [BcContainerHelper](https://github.com/microsoft/navcontainerhelper) module. The PowerShell modules ship in [`src/bcbench_core/powershell`](src/bcbench_core/powershell). -## Agent harnesses - -[`bcbench_core.agent.harness`](src/bcbench_core/agent/harness.py) defines the harness-neutral contract. A `Harness[T]` reports its `AgentIdentity` before any run and turns an `AgentRequest[T]` into an `AgentResult`. Output is a final response or changes left in the workspace. Applications construct harnesses with their own configuration and pass them to evaluation pipelines, which depend only on this contract. - ## Logging Modules log through `logging.getLogger(__name__)` and never configure logging on import. Applications call `bcbench_core.logs.setup_logging` once from their entry point. diff --git a/packages/bcbench-core/src/bcbench_core/agent/harness.py b/packages/bcbench-core/src/bcbench_core/agent/harness.py index 407bdf64f..7f1b28428 100644 --- a/packages/bcbench-core/src/bcbench_core/agent/harness.py +++ b/packages/bcbench-core/src/bcbench_core/agent/harness.py @@ -1,15 +1,10 @@ -"""Harness-neutral contract between evaluation applications and the agents they run. - -A harness wraps one way of producing an answer (a CLI agent, a review engine, a hosted agent). -Applications construct harnesses with harness-specific configuration and only use this contract, -so evaluation pipelines never depend on a concrete harness. -""" +"""Harness-neutral contract between evaluation pipelines and the agents they run.""" from dataclasses import dataclass from pathlib import Path from typing import Protocol -from pydantic import BaseModel, ConfigDict, Field, JsonValue +from pydantic import BaseModel, ConfigDict from bcbench_core.agent.experiment import ExperimentConfiguration from bcbench_core.agent.metrics import AgentMetrics @@ -30,8 +25,6 @@ class AgentIdentity[N: str = str](BaseModel): version: str # Configured model; None when the harness pins or delegates model selection model: str | None = None - # Further pins needed to reproduce a run, e.g. a knowledge-base commit or a definition hash - provenance: dict[str, JsonValue] = Field(default_factory=dict) @dataclass(frozen=True) @@ -51,8 +44,6 @@ class AgentResult: response: str | None = None metrics: AgentMetrics | None = None experiment: ExperimentConfiguration | None = None - # Model reported by the agent, when it differs from or replaces the configured one - observed_model: str | None = None class Harness[T, N: str = str](Protocol): From 44cd5ec7431576269f1e2ea4bd869252def24e3c Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 14:59:41 +0100 Subject: [PATCH 7/7] Fall back to the default Copilot timeout only when none is requested Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/bcbench/agent/copilot/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 83d14b907..2d23d72e0 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -101,7 +101,7 @@ def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult: prompt=prompt, model=self.model, work_dir=repo_path, - timeout=request.timeout or self.timeout, + timeout=self.timeout if request.timeout is None else request.timeout, env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), options=options, )