Return AgentResult from BC-Bench agent harnesses

src/bcbench/agent/bcal/agent.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.agent.version import get_cli_version
 from bcbench_core.artifacts import ALPACKAGES_DIRNAME
@@ -124,7 +125,7 @@ def run_bcal_agent(
     repo_path: Path,
     backend_config: BCalBackendConfig,
     timeout: int = 25 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
+) -> AgentResult:
     logger.info(f"Running bcal CLI on: {entry.instance_id}")
 
     # The .alpackages dir is created by the NL2AL pipeline setup step
@@ -151,7 +152,7 @@ def run_bcal_agent(
         execution_time = time.monotonic() - start
 
         logger.info(f"bcal CLI run complete for: {entry.instance_id}")
-        return AgentMetrics(execution_time=execution_time), ExperimentConfiguration()
+        return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration())
     except subprocess.TimeoutExpired:
         logger.exception(f"bcal CLI timed out after {timeout} seconds")
         metrics = AgentMetrics(execution_time=timeout)

src/bcbench/agent/claude/agent.py

@@ -5,6 +5,7 @@
 
 from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.agent.version import get_cli_version
 from bcbench_core.exceptions import AgentError, AgentTimeoutError
@@ -39,12 +40,8 @@ def run_claude_code(
     plugin_root: Path,
     runtime: AgentRuntimeConfig | None = None,
     timeout: int = 60 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
-    """Run Claude Code on a single dataset entry.
-
-    Returns:
-        Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used
-    """
+) -> AgentResult:
+    """Run Claude Code on a single dataset entry."""
     claude_cmd = shutil.which("claude")
     if not claude_cmd:
         raise AgentError("Claude Code not found in PATH. Please ensure it is installed and available.")
@@ -145,7 +142,7 @@ def run_claude_code(
         stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else ""
         logger.debug(f"Claude Code raw output: {stdout}")
 
-        metrics, _ = parse_stream_output(stdout.splitlines(), log_transcript=True)
+        metrics, response = parse_stream_output(stdout.splitlines(), log_transcript=True)
     except subprocess.TimeoutExpired:
         logger.exception(f"Claude Code timed out after {timeout} seconds")
         metrics = AgentMetrics(execution_time=timeout)
@@ -157,7 +154,7 @@ def run_claude_code(
         logger.exception("Unexpected error running Claude Code")
         raise
     else:
-        return metrics, config
+        return AgentResult(response=response, metrics=metrics, experiment=config)
     finally:
         if bc_gateway is not None:
             bc_gateway.stop()

src/bcbench/agent/copilot/agent.py

@@ -6,7 +6,7 @@
 from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot
 from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills
 from bcbench_core.agent.experiment import ExperimentConfiguration
-from bcbench_core.agent.metrics import AgentMetrics
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.exceptions import AgentTimeoutError
 
 from bcbench.agent.shared import (
@@ -34,12 +34,8 @@ def run_copilot_agent(
     plugin_root: Path,
     runtime: AgentRuntimeConfig | None = None,
     timeout: int = 60 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
-    """Run GitHub Copilot CLI agent on a single dataset entry.
-
-    Returns:
-        Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment
-    """
+) -> AgentResult:
+    """Run GitHub Copilot CLI agent on a single dataset entry."""
     logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}")
 
     prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp))
@@ -92,7 +88,7 @@ def run_copilot_agent(
             workspace_mcp=True,
         )
 
-        metrics, _ = invoke_copilot(
+        metrics, response = invoke_copilot(
             prompt=prompt,
             model=model,
             work_dir=repo_path,
@@ -104,7 +100,7 @@ def run_copilot_agent(
     except CopilotTimeoutError as exc:
         raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc
     else:
-        return metrics, config
+        return AgentResult(response=response, metrics=metrics, experiment=config)
     finally:
         if bc_gateway is not None:
             bc_gateway.stop()

src/bcbench/agent/pr_review/agent.py

@@ -22,6 +22,7 @@
 
 import yaml
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.exceptions import AgentError, AgentTimeoutError
 from bcbench_core.git import commit_changes, has_changes, init_repo
 
@@ -161,16 +162,13 @@ def run_pr_review_agent(
     engine_path: Path | None = None,
     min_severity: str | None = None,
     timeout: int = 60 * 60,
-) -> tuple[PRReviewMetrics, ExperimentConfiguration]:
+) -> AgentResult:
     """Run the engine's complete local review pipeline and write review.json.
 
     Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents
     PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it
     owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific
     inputs (BC-ALAgents source, min severity) for the code-review category only.
-
-    Returns:
-        Tuple of (PRReviewMetrics, ExperimentConfiguration).
     """
     if category is not EvaluationCategory.CODE_REVIEW:
         raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.")
@@ -235,4 +233,4 @@ def run_pr_review_agent(
         logger.exception("Unexpected error running engine review")
         raise
     else:
-        return build_pr_review_metrics(output_dir, time.monotonic() - start), config
+        return AgentResult(metrics=build_pr_review_metrics(output_dir, time.monotonic() - start), experiment=config)

src/bcbench/commands/evaluate.py

@@ -6,6 +6,7 @@
 import typer
 from bcbench_core.agent.copilot import get_copilot_version
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.exceptions import AgentError
 from bcbench_core.filesystem import prepare_run_dir
@@ -359,7 +360,7 @@ def evaluate_mock(
     )
 
     pipeline = MockEvaluationPipeline()
-    pipeline.execute(context, lambda ctx: (None, None))
+    pipeline.execute(context, lambda _ctx: AgentResult())
 
     logger.info("Mock evaluation complete!")
     logger.info(f"Results saved to: {run_dir}")

src/bcbench/evaluate/base.py

@@ -5,8 +5,7 @@
 from pathlib import Path
 from typing import Protocol
 
-from bcbench_core.agent.experiment import ExperimentConfiguration
-from bcbench_core.agent.metrics import AgentMetrics
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.exceptions import AgentTimeoutError
 
 from bcbench.dataset import BaseDatasetEntry
@@ -20,7 +19,7 @@
 
 
 class AgentRunner[E: BaseDatasetEntry](Protocol):
-    def __call__(self, context: EvaluationContext[E], /) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: ...
+    def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ...
 
 
 class EvaluationPipeline[E: BaseDatasetEntry](ABC):

src/bcbench/evaluate/bugfix.py

@@ -45,7 +45,8 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[BugFixEntry]) -> None:

src/bcbench/evaluate/codereview.py

@@ -57,7 +57,8 @@ def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None:

src/bcbench/evaluate/dataquery.py

@@ -107,7 +107,8 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: AgentRunner[DataQueryEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None:

src/bcbench/evaluate/ext_request_advisor.py

@@ -31,7 +31,8 @@ def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None:

src/bcbench/evaluate/ext_request_implement.py

@@ -38,7 +38,8 @@ def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None:

src/bcbench/evaluate/ext_request_triage.py

@@ -41,7 +41,8 @@ def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None:

src/bcbench/evaluate/nl2al.py

@@ -72,7 +72,8 @@ def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentR
         # Single attempt — retries are disabled. An empty diff (the agent asked for clarification
         # instead of editing) is scored as a failure in evaluate(), not re-run.
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[NL2ALEntry]) -> None:

src/bcbench/evaluate/testgeneration.py

@@ -67,7 +67,8 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None:
     @override
     def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None:
         with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"):
-            context.metrics, context.experiment = agent_runner(context)
+            result = agent_runner(context)
+            context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[TestGenEntry]) -> None:

tests/test_bcal_agent_provider.py

@@ -63,7 +63,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock:
             patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"),
             patch.object(subprocess, "run", side_effect=fake_run),
         ):
-            metrics, _ = bcal_agent.run_bcal_agent(
+            result = bcal_agent.run_bcal_agent(
                 entry=entry,
                 repo_path=workspace,
                 backend_config=BCalBackendConfig(
@@ -72,7 +72,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock:
                 ),
             )
 
-        assert metrics is not None
+        assert result.metrics is not None
         args = captured["args"]
         assert "--deployment=gpt-5" in args
         assert "--llm-backend=external-command" in args

tests/test_copilot_agent.py

@@ -30,13 +30,13 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat
         patch("bcbench.agent.copilot.agent.setup_skills", return_value=False),
         patch("bcbench.agent.copilot.agent.setup_custom_agent", return_value=None),
         patch("bcbench.agent.copilot.agent.resolve_config_plugins", return_value=[]),
-        patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, None)) as mock_parse_output,
+        patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, "final answer")) as mock_parse_output,
         patch(
             "bcbench_core.agent.copilot.agent.subprocess.run",
             return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""),
         ) as mock_run,
     ):
-        run_copilot_agent(
+        result = run_copilot_agent(
             entry=create_dataset_entry(),
             model="copilot-test-model",
             category=EvaluationCategory.BUG_FIX,
@@ -46,6 +46,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat
             plugin_root=tmp_path / "plugins",
         )
 
+    assert result.response == "final answer"
     assert mock_run.call_args.args[0] == [
         "copilot",
         "--output-format=json",
@@ -95,15 +96,15 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr
             return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""),
         ) as mock_run,
     ):
-        _, config = run_copilot_agent(
+        config = run_copilot_agent(
             entry=create_dataset_entry(),
             model="copilot-test-model",
             category=EvaluationCategory.BUG_FIX,
             repo_path=repo_path,
             output_dir=output_dir,
             agent_config=AgentConfig(),
             plugin_root=tmp_path / "plugins",
-        )
+        ).experiment
 
     assert mock_run.call_args.args[0] == [
         "copilot",
@@ -121,6 +122,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr
         "--agent=al-dev",
         "--prompt=do the task",
     ]
+    assert config is not None
     assert config.mcp_servers == ["probe"]
     assert config.al_lsp_enabled is lsp_enabled
     assert config.custom_instructions is True

tests/test_evaluate_pipeline.py

@@ -7,6 +7,7 @@
 
 import pytest
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.exceptions import AgentTimeoutError
 
@@ -39,7 +40,8 @@ def run_agent(self, context: EvaluationContext[E], agent_runner: AgentRunner[E])
         self.run_agent_called = True
         if self.raise_in_run_agent is not None:
             raise self.raise_in_run_agent
-        context.metrics, context.experiment = agent_runner(context)
+        result = agent_runner(context)
+        context.metrics, context.experiment = result.metrics, result.experiment
 
     @override
     def evaluate(self, context: EvaluationContext[E]) -> None:
@@ -48,8 +50,8 @@ def evaluate(self, context: EvaluationContext[E]) -> None:
             raise self.raise_in_evaluate
 
 
-def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]:
-    return AgentMetrics(execution_time=1.0), ExperimentConfiguration()
+def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> AgentResult:
+    return AgentResult(metrics=AgentMetrics(execution_time=1.0), experiment=ExperimentConfiguration())
 
 
 def _read_only_result[E: BaseDatasetEntry](ctx: EvaluationContext[E]) -> BaseEvaluationResult:
@@ -93,7 +95,7 @@ def test_persists_category_specific_timeout_result(self, tmp_path):
         ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL)
         pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=AgentTimeoutError("test timeout"))
 
-        pipeline.execute(ctx, lambda _: (None, None))
+        pipeline.execute(ctx, lambda _: AgentResult())
 
         result_file = ctx.result_dir / f"{entry.instance_id}{RESULT_FILE_SUFFIX}"
         payload = json.loads(result_file.read_text(encoding="utf-8"))

tests/test_nl2al_pipeline.py

@@ -1,6 +1,7 @@
 """Tests for NL2ALPipeline.evaluate() — empty-diff handling is nl2al-specific."""
 
 import pytest
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.exceptions import EmptyDiffError
 
 from bcbench.evaluate.nl2al import NL2ALPipeline
@@ -85,7 +86,7 @@ def test_runs_agent_once_and_never_retries(self, tmp_path, monkeypatch):
 
         def agent_runner(_ctx):
             agent_calls["n"] += 1
-            return (None, None)
+            return AgentResult()
 
         reset_calls = {"n": 0}
         monkeypatch.setattr(pipeline, "setup_workspace", lambda *_args, **_kw: reset_calls.__setitem__("n", reset_calls["n"] + 1))