From 266df189e47c6ad8eb4c5e0c782e8159a828bdb9 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 3 Nov 2026 11:17:38 +0100 Subject: [PATCH] Return AgentResult from BC-Bench agent harnesses The Copilot, Claude, PR Review and BCal harness functions now return the core AgentResult instead of a (metrics, experiment) tuple, and pipelines read it through AgentRunner. Copilot and Claude also return their final message as the response, which was previously discarded. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/bcbench/agent/bcal/agent.py | 5 +++-- src/bcbench/agent/claude/agent.py | 13 +++++-------- src/bcbench/agent/copilot/agent.py | 14 +++++--------- src/bcbench/agent/pr_review/agent.py | 8 +++----- src/bcbench/commands/evaluate.py | 3 ++- src/bcbench/evaluate/base.py | 5 ++--- src/bcbench/evaluate/bugfix.py | 3 ++- src/bcbench/evaluate/codereview.py | 3 ++- src/bcbench/evaluate/dataquery.py | 3 ++- src/bcbench/evaluate/ext_request_advisor.py | 3 ++- src/bcbench/evaluate/ext_request_implement.py | 3 ++- src/bcbench/evaluate/ext_request_triage.py | 3 ++- src/bcbench/evaluate/nl2al.py | 3 ++- src/bcbench/evaluate/testgeneration.py | 3 ++- tests/test_bcal_agent_provider.py | 4 ++-- tests/test_copilot_agent.py | 10 ++++++---- tests/test_evaluate_pipeline.py | 10 ++++++---- tests/test_nl2al_pipeline.py | 3 ++- tests/test_pr_review_agent.py | 6 ++++-- 19 files changed, 56 insertions(+), 49 deletions(-) diff --git a/src/bcbench/agent/bcal/agent.py b/src/bcbench/agent/bcal/agent.py index 24ff0005a..8764ddb46 100644 --- a/src/bcbench/agent/bcal/agent.py +++ b/src/bcbench/agent/bcal/agent.py @@ -7,6 +7,7 @@ from pathlib import Path from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.artifacts import ALPACKAGES_DIRNAME @@ -124,7 +125,7 @@ def run_bcal_agent( repo_path: Path, backend_config: BCalBackendConfig, timeout: int = 25 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: +) -> AgentResult: logger.info(f"Running bcal CLI on: {entry.instance_id}") # The .alpackages dir is created by the NL2AL pipeline setup step @@ -151,7 +152,7 @@ def run_bcal_agent( execution_time = time.monotonic() - start logger.info(f"bcal CLI run complete for: {entry.instance_id}") - return AgentMetrics(execution_time=execution_time), ExperimentConfiguration() + return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration()) except subprocess.TimeoutExpired: logger.exception(f"bcal CLI timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 8c9f34d24..c04cd5888 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -5,6 +5,7 @@ from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.agent.version import get_cli_version from bcbench_core.exceptions import AgentError, AgentTimeoutError @@ -39,12 +40,8 @@ def run_claude_code( plugin_root: Path, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run Claude Code on a single dataset entry. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used - """ +) -> AgentResult: + """Run Claude Code on a single dataset entry.""" claude_cmd = shutil.which("claude") if not claude_cmd: raise AgentError("Claude Code not found in PATH. Please ensure it is installed and available.") @@ -145,7 +142,7 @@ def run_claude_code( stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else "" logger.debug(f"Claude Code raw output: {stdout}") - metrics, _ = parse_stream_output(stdout.splitlines(), log_transcript=True) + metrics, response = parse_stream_output(stdout.splitlines(), log_transcript=True) except subprocess.TimeoutExpired: logger.exception(f"Claude Code timed out after {timeout} seconds") metrics = AgentMetrics(execution_time=timeout) @@ -157,7 +154,7 @@ def run_claude_code( logger.exception("Unexpected error running Claude Code") raise else: - return metrics, config + return AgentResult(response=response, metrics=metrics, experiment=config) finally: if bc_gateway is not None: bc_gateway.stop() diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index c95a530bb..d277c7b52 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -6,7 +6,7 @@ from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentTimeoutError from bcbench.agent.shared import ( @@ -34,12 +34,8 @@ def run_copilot_agent( plugin_root: Path, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run GitHub Copilot CLI agent on a single dataset entry. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment - """ +) -> AgentResult: + """Run GitHub Copilot CLI agent on a single dataset entry.""" logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp)) @@ -92,7 +88,7 @@ def run_copilot_agent( workspace_mcp=True, ) - metrics, _ = invoke_copilot( + metrics, response = invoke_copilot( prompt=prompt, model=model, work_dir=repo_path, @@ -104,7 +100,7 @@ def run_copilot_agent( except CopilotTimeoutError as exc: raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc else: - return metrics, config + return AgentResult(response=response, metrics=metrics, experiment=config) finally: if bc_gateway is not None: bc_gateway.stop() diff --git a/src/bcbench/agent/pr_review/agent.py b/src/bcbench/agent/pr_review/agent.py index a68fc2c1d..87f62f81d 100644 --- a/src/bcbench/agent/pr_review/agent.py +++ b/src/bcbench/agent/pr_review/agent.py @@ -22,6 +22,7 @@ import yaml from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentError, AgentTimeoutError from bcbench_core.git import commit_changes, has_changes, init_repo @@ -161,16 +162,13 @@ def run_pr_review_agent( engine_path: Path | None = None, min_severity: str | None = None, timeout: int = 60 * 60, -) -> tuple[PRReviewMetrics, ExperimentConfiguration]: +) -> AgentResult: """Run the engine's complete local review pipeline and write review.json. Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific inputs (BC-ALAgents source, min severity) for the code-review category only. - - Returns: - Tuple of (PRReviewMetrics, ExperimentConfiguration). """ if category is not EvaluationCategory.CODE_REVIEW: raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") @@ -235,4 +233,4 @@ def run_pr_review_agent( logger.exception("Unexpected error running engine review") raise else: - return build_pr_review_metrics(output_dir, time.monotonic() - start), config + return AgentResult(metrics=build_pr_review_metrics(output_dir, time.monotonic() - start), experiment=config) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 963fbac77..fd17d54bc 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -6,6 +6,7 @@ import typer from bcbench_core.agent.copilot import get_copilot_version from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentError from bcbench_core.filesystem import prepare_run_dir @@ -359,7 +360,7 @@ def evaluate_mock( ) pipeline = MockEvaluationPipeline() - pipeline.execute(context, lambda ctx: (None, None)) + pipeline.execute(context, lambda _ctx: AgentResult()) logger.info("Mock evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 8d2750619..590840677 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -5,8 +5,7 @@ from pathlib import Path from typing import Protocol -from bcbench_core.agent.experiment import ExperimentConfiguration -from bcbench_core.agent.metrics import AgentMetrics +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import AgentTimeoutError from bcbench.dataset import BaseDatasetEntry @@ -20,7 +19,7 @@ class AgentRunner[E: BaseDatasetEntry](Protocol): - def __call__(self, context: EvaluationContext[E], /) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: ... + def __call__(self, context: EvaluationContext[E], /) -> AgentResult: ... class EvaluationPipeline[E: BaseDatasetEntry](ABC): diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index 4f513f40f..f34c113e7 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -45,7 +45,8 @@ def setup(self, context: EvaluationContext[BugFixEntry]) -> None: @override def run_agent(self, context: EvaluationContext[BugFixEntry], agent_runner: AgentRunner[BugFixEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[BugFixEntry]) -> None: diff --git a/src/bcbench/evaluate/codereview.py b/src/bcbench/evaluate/codereview.py index 8ad821415..4e574e387 100644 --- a/src/bcbench/evaluate/codereview.py +++ b/src/bcbench/evaluate/codereview.py @@ -57,7 +57,8 @@ def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None: @override def run_agent(self, context: EvaluationContext[CodeReviewEntry], agent_runner: AgentRunner[CodeReviewEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None: diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 77ba43705..af618f436 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -107,7 +107,8 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: @override def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: AgentRunner[DataQueryEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_advisor.py b/src/bcbench/evaluate/ext_request_advisor.py index 7c98a12d0..0bffb5d3d 100644 --- a/src/bcbench/evaluate/ext_request_advisor.py +++ b/src/bcbench/evaluate/ext_request_advisor.py @@ -31,7 +31,8 @@ def setup(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestAdvisorEntry], agent_runner: AgentRunner[ExtRequestAdvisorEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestAdvisorEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_implement.py b/src/bcbench/evaluate/ext_request_implement.py index 79d6ee20c..ae4368537 100644 --- a/src/bcbench/evaluate/ext_request_implement.py +++ b/src/bcbench/evaluate/ext_request_implement.py @@ -38,7 +38,8 @@ def setup(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestImplementEntry], agent_runner: AgentRunner[ExtRequestImplementEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestImplementEntry]) -> None: diff --git a/src/bcbench/evaluate/ext_request_triage.py b/src/bcbench/evaluate/ext_request_triage.py index 1743916e8..f310fd9c9 100644 --- a/src/bcbench/evaluate/ext_request_triage.py +++ b/src/bcbench/evaluate/ext_request_triage.py @@ -41,7 +41,8 @@ def setup(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: @override def run_agent(self, context: EvaluationContext[ExtRequestTriageEntry], agent_runner: AgentRunner[ExtRequestTriageEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[ExtRequestTriageEntry]) -> None: diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index e04e4c65b..54f567f00 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -72,7 +72,8 @@ def run_agent(self, context: EvaluationContext[NL2ALEntry], agent_runner: AgentR # Single attempt — retries are disabled. An empty diff (the agent asked for clarification # instead of editing) is scored as a failure in evaluate(), not re-run. with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[NL2ALEntry]) -> None: diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index a72b6f526..51a7c597b 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -67,7 +67,8 @@ def setup(self, context: EvaluationContext[TestGenEntry]) -> None: @override def run_agent(self, context: EvaluationContext[TestGenEntry], agent_runner: AgentRunner[TestGenEntry]) -> None: with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[TestGenEntry]) -> None: diff --git a/tests/test_bcal_agent_provider.py b/tests/test_bcal_agent_provider.py index 062d9d31d..29a364b30 100644 --- a/tests/test_bcal_agent_provider.py +++ b/tests/test_bcal_agent_provider.py @@ -63,7 +63,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), patch.object(subprocess, "run", side_effect=fake_run), ): - metrics, _ = bcal_agent.run_bcal_agent( + result = bcal_agent.run_bcal_agent( entry=entry, repo_path=workspace, backend_config=BCalBackendConfig( @@ -72,7 +72,7 @@ def fake_run(args: list[str], **_: object) -> MagicMock: ), ) - assert metrics is not None + assert result.metrics is not None args = captured["args"] assert "--deployment=gpt-5" in args assert "--llm-backend=external-command" in args diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index ab4bd4f74..d7d2d5e7d 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -30,13 +30,13 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat patch("bcbench.agent.copilot.agent.setup_skills", return_value=False), patch("bcbench.agent.copilot.agent.setup_custom_agent", return_value=None), patch("bcbench.agent.copilot.agent.resolve_config_plugins", return_value=[]), - patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, None)) as mock_parse_output, + patch("bcbench_core.agent.copilot.agent.parse_output", return_value=(None, "final answer")) as mock_parse_output, patch( "bcbench_core.agent.copilot.agent.subprocess.run", return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - run_copilot_agent( + result = run_copilot_agent( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -46,6 +46,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat plugin_root=tmp_path / "plugins", ) + assert result.response == "final answer" assert mock_run.call_args.args[0] == [ "copilot", "--output-format=json", @@ -95,7 +96,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout='{"type":"result"}\n', stderr=""), ) as mock_run, ): - _, config = run_copilot_agent( + config = run_copilot_agent( entry=create_dataset_entry(), model="copilot-test-model", category=EvaluationCategory.BUG_FIX, @@ -103,7 +104,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr output_dir=output_dir, agent_config=AgentConfig(), plugin_root=tmp_path / "plugins", - ) + ).experiment assert mock_run.call_args.args[0] == [ "copilot", @@ -121,6 +122,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr "--agent=al-dev", "--prompt=do the task", ] + assert config is not None assert config.mcp_servers == ["probe"] assert config.al_lsp_enabled is lsp_enabled assert config.custom_instructions is True diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 17ea4e049..e3bd2bed5 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -7,6 +7,7 @@ import pytest from bcbench_core.agent.experiment import ExperimentConfiguration +from bcbench_core.agent.harness import AgentResult from bcbench_core.agent.metrics import AgentMetrics from bcbench_core.exceptions import AgentTimeoutError @@ -39,7 +40,8 @@ def run_agent(self, context: EvaluationContext[E], agent_runner: AgentRunner[E]) self.run_agent_called = True if self.raise_in_run_agent is not None: raise self.raise_in_run_agent - context.metrics, context.experiment = agent_runner(context) + result = agent_runner(context) + context.metrics, context.experiment = result.metrics, result.experiment @override def evaluate(self, context: EvaluationContext[E]) -> None: @@ -48,8 +50,8 @@ def evaluate(self, context: EvaluationContext[E]) -> None: raise self.raise_in_evaluate -def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> tuple[AgentMetrics | None, ExperimentConfiguration | None]: - return AgentMetrics(execution_time=1.0), ExperimentConfiguration() +def _noop_runner(_ctx: EvaluationContext[BugFixEntry]) -> AgentResult: + return AgentResult(metrics=AgentMetrics(execution_time=1.0), experiment=ExperimentConfiguration()) def _read_only_result[E: BaseDatasetEntry](ctx: EvaluationContext[E]) -> BaseEvaluationResult: @@ -93,7 +95,7 @@ def test_persists_category_specific_timeout_result(self, tmp_path): ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL) pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=AgentTimeoutError("test timeout")) - pipeline.execute(ctx, lambda _: (None, None)) + pipeline.execute(ctx, lambda _: AgentResult()) result_file = ctx.result_dir / f"{entry.instance_id}{RESULT_FILE_SUFFIX}" payload = json.loads(result_file.read_text(encoding="utf-8")) diff --git a/tests/test_nl2al_pipeline.py b/tests/test_nl2al_pipeline.py index 06457d34e..ffc899dea 100644 --- a/tests/test_nl2al_pipeline.py +++ b/tests/test_nl2al_pipeline.py @@ -1,6 +1,7 @@ """Tests for NL2ALPipeline.evaluate() — empty-diff handling is nl2al-specific.""" import pytest +from bcbench_core.agent.harness import AgentResult from bcbench_core.exceptions import EmptyDiffError from bcbench.evaluate.nl2al import NL2ALPipeline @@ -85,7 +86,7 @@ def test_runs_agent_once_and_never_retries(self, tmp_path, monkeypatch): def agent_runner(_ctx): agent_calls["n"] += 1 - return (None, None) + return AgentResult() reset_calls = {"n": 0} monkeypatch.setattr(pipeline, "setup_workspace", lambda *_args, **_kw: reset_calls.__setitem__("n", reset_calls["n"] + 1)) diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index e294a47b4..11def48bc 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -163,7 +163,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: patch("bcbench.agent.pr_review.agent.time.monotonic", side_effect=[10.0, 12.5]), patch("bcbench.agent.pr_review.agent.subprocess.run", return_value=completed) as run_process, ): - metrics, config = run_pr_review_agent( + result = run_pr_review_agent( entry=entry, model="gpt-5.6-luna", category=EvaluationCategory.CODE_REVIEW, @@ -172,6 +172,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: engine_path=tmp_path / "engine", ) + metrics = result.metrics assert isinstance(metrics, PRReviewMetrics) assert metrics.execution_time == 2.5 assert metrics.prompt_tokens == 100 @@ -180,7 +181,8 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: assert metrics.ai_credits == 0.25 assert metrics.api_calls == 2 assert metrics.copilot_cli_version == "1.0.81-0" - assert config.is_empty() + assert result.experiment is not None + assert result.experiment.is_empty() resolve_engine.assert_called_once_with(tmp_path / "engine") prepare_bcquality.assert_called_once_with(tmp_path / "engine", "pwsh", (tmp_path / "output" / "bcquality").resolve()) assert run_process.call_args.kwargs["encoding"] == "utf-8"