Add a harness-neutral agent contract and adopt it for Copilot CLI

packages/bcbench-core/src/bcbench_core/agent/harness.py

@@ -0,0 +1,58 @@
+"""Harness-neutral contract between evaluation pipelines and the agents they run."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol
+
+from pydantic import BaseModel, ConfigDict
+
+from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.metrics import AgentMetrics
+
+__all__ = ["AgentIdentity", "AgentRequest", "AgentResult", "Harness"]
+
+
+class AgentIdentity[N: str = str](BaseModel):
+    """What is being evaluated, resolved before any task runs.
+
+    ``N`` lets applications type the harness name, e.g. with their own ``StrEnum`` of harnesses.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    harness: N
+    # CLI version, engine commit, or hosted agent version
+    version: str
+    # Configured model; None when the harness pins or delegates model selection
+    model: str | None = None
+
+
+@dataclass(frozen=True)
+class AgentRequest[T]:
+    task: T
+    # The agent's workspace: a prepared repository, or an empty directory for answer-only tasks
+    work_dir: Path
+    # Receives logs and raw captures
+    output_dir: Path
+    # None uses the harness's default
+    timeout: int | None = None
+
+
+@dataclass(frozen=True)
+class AgentResult:
+    # Final answer; None when the output is left in the workspace
+    response: str | None = None
+    metrics: AgentMetrics | None = None
+    experiment: ExperimentConfiguration | None = None
+
+
+class Harness[T, N: str = str](Protocol):
+    """Runs an agent on tasks of type ``T``, identified by a harness name of type ``N``.
+
+    ``identity`` checks the harness is usable and raises ``AgentError`` otherwise. ``run`` raises
+    ``AgentTimeoutError`` when the timeout is exceeded and ``AgentError`` for any other failure.
+    """
+
+    def identity(self) -> AgentIdentity[N]: ...
+
+    def run(self, request: AgentRequest[T], /) -> AgentResult: ...

src/bcbench/agent/__init__.py

@@ -2,7 +2,7 @@
 
 from bcbench.agent.bcal import BCalBackendConfig, get_bcal_version, run_bcal_agent
 from bcbench.agent.claude import get_claude_version, run_claude_code
-from bcbench.agent.copilot import run_copilot_agent
+from bcbench.agent.copilot import CopilotHarness
 from bcbench.agent.pr_review import get_pr_review_version, run_pr_review_agent
 
-__all__ = ["BCalBackendConfig", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_copilot_agent", "run_pr_review_agent"]
+__all__ = ["BCalBackendConfig", "CopilotHarness", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_pr_review_agent"]

src/bcbench/agent/bcal/agent.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.agent.version import get_cli_version
 from bcbench_core.artifacts import ALPACKAGES_DIRNAME
@@ -124,7 +125,7 @@ def run_bcal_agent(
     repo_path: Path,
     backend_config: BCalBackendConfig,
     timeout: int = 25 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
+) -> AgentResult:
     logger.info(f"Running bcal CLI on: {entry.instance_id}")
 
     # The .alpackages dir is created by the NL2AL pipeline setup step
@@ -151,7 +152,7 @@ def run_bcal_agent(
         execution_time = time.monotonic() - start
 
         logger.info(f"bcal CLI run complete for: {entry.instance_id}")
-        return AgentMetrics(execution_time=execution_time), ExperimentConfiguration()
+        return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration())
     except subprocess.TimeoutExpired:
         logger.exception(f"bcal CLI timed out after {timeout} seconds")
         metrics = AgentMetrics(execution_time=timeout)

src/bcbench/agent/claude/agent.py

@@ -5,6 +5,7 @@
 
 from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.agent.version import get_cli_version
 from bcbench_core.exceptions import AgentError, AgentTimeoutError
@@ -39,12 +40,8 @@ def run_claude_code(
     plugin_root: Path,
     runtime: AgentRuntimeConfig | None = None,
     timeout: int = 60 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
-    """Run Claude Code on a single dataset entry.
-
-    Returns:
-        Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used
-    """
+) -> AgentResult:
+    """Run Claude Code on a single dataset entry."""
     claude_cmd = shutil.which("claude")
     if not claude_cmd:
         raise AgentError("Claude Code not found in PATH. Please ensure it is installed and available.")
@@ -145,7 +142,7 @@ def run_claude_code(
         stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else ""
         logger.debug(f"Claude Code raw output: {stdout}")
 
-        metrics, _ = parse_stream_output(stdout.splitlines(), log_transcript=True)
+        metrics, response = parse_stream_output(stdout.splitlines(), log_transcript=True)
     except subprocess.TimeoutExpired:
         logger.exception(f"Claude Code timed out after {timeout} seconds")
         metrics = AgentMetrics(execution_time=timeout)
@@ -157,7 +154,7 @@ def run_claude_code(
         logger.exception("Unexpected error running Claude Code")
         raise
     else:
-        return metrics, config
+        return AgentResult(response=response, metrics=metrics, experiment=config)
     finally:
         if bc_gateway is not None:
             bc_gateway.stop()

src/bcbench/agent/copilot/__init__.py

@@ -1,5 +1,5 @@
 """GitHub Copilot CLI agent module."""
 
-from bcbench.agent.copilot.agent import run_copilot_agent
+from bcbench.agent.copilot.agent import CopilotHarness
 
-__all__ = ["run_copilot_agent"]
+__all__ = ["CopilotHarness"]

src/bcbench/agent/copilot/agent.py

@@ -1,12 +1,14 @@
 """GitHub Copilot CLI Agent implementation."""
 
 import logging
+from dataclasses import dataclass
 from pathlib import Path
+from typing import override
 
-from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, invoke_copilot
+from bcbench_core.agent.copilot import CopilotOptions, CopilotTimeoutError, get_copilot_version, invoke_copilot
 from bcbench_core.agent.customization import setup_custom_agent, setup_instructions, setup_skills
 from bcbench_core.agent.experiment import ExperimentConfiguration
-from bcbench_core.agent.metrics import AgentMetrics
+from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult, Harness
 from bcbench_core.exceptions import AgentTimeoutError
 
 from bcbench.agent.shared import (
@@ -24,87 +26,90 @@
 logger = logging.getLogger(__name__)
 
 
-def run_copilot_agent(
-    entry: BaseDatasetEntry,
-    model: str,
-    category: EvaluationCategory,
-    repo_path: Path,
-    output_dir: Path,
-    agent_config: AgentConfig,
-    plugin_root: Path,
-    runtime: AgentRuntimeConfig | None = None,
-    timeout: int = 60 * 60,
-) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
-    """Run GitHub Copilot CLI agent on a single dataset entry.
+@dataclass(frozen=True)
+class CopilotHarness(Harness[BaseDatasetEntry, AgentHarness]):
+    """GitHub Copilot CLI on any category, customized through the agent config."""
 
-    Returns:
-        Tuple of (AgentMetrics, ExperimentConfiguration) with metrics and configuration used during the experiment
-    """
-    logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}")
+    model: str
+    category: EvaluationCategory
+    agent_config: AgentConfig
+    plugin_root: Path
+    runtime: AgentRuntimeConfig | None = None
+    timeout: int = 60 * 60
 
-    prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp))
-    bc_gateway = start_bc_mcp_gateway(runtime)
-    mcp_config_json, mcp_server_names = build_mcp_config(
-        agent_config,
-        entry,
-        repo_path,
-        runtime=runtime,
-        bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None,
-    )
-    lsp_plugin_dir: Path | None = build_al_lsp_plugin(
-        entry,
-        category,
-        repo_path,
-        AgentHarness.COPILOT,
-        plugin_root,
-        runtime=runtime,
-    )
-    profile_dir: Path = instructions_profile_dir(entry.customization_profile)
-    target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path)
-    instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename)
-    skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir)
-    custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir)
-    plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True)
+    @override
+    def identity(self) -> AgentIdentity[AgentHarness]:
+        return AgentIdentity[AgentHarness](harness=AgentHarness.COPILOT, version=get_copilot_version(), model=self.model)
 
-    config = ExperimentConfiguration(
-        mcp_servers=mcp_server_names,
-        al_lsp_enabled=lsp_plugin_dir is not None,
-        custom_instructions=instructions_enabled,
-        skills_enabled=skills_enabled,
-        custom_agent=custom_agent,
-        plugins=[plugin.record for plugin, _ in plugins] or None,
-    )
+    @override
+    def run(self, request: AgentRequest[BaseDatasetEntry], /) -> AgentResult:
+        entry, repo_path, output_dir = request.task, request.work_dir, request.output_dir
+        agent_config, category, runtime, plugin_root = self.agent_config, self.category, self.runtime, self.plugin_root
+        logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}")
 
-    try:
-        lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else ()
-        # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it
-        # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for
-        # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not
-        # silently widen the agent's sandbox access.
-        options = CopilotOptions(
-            allow_all_tools=True,
+        prompt: str = build_prompt(entry, repo_path, agent_config, category, al_mcp=bool(runtime and runtime.al_mcp))
+        bc_gateway = start_bc_mcp_gateway(runtime)
+        mcp_config_json, mcp_server_names = build_mcp_config(
+            agent_config,
+            entry,
+            repo_path,
+            runtime=runtime,
+            bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None,
+        )
+        lsp_plugin_dir: Path | None = build_al_lsp_plugin(
+            entry,
+            category,
+            repo_path,
+            AgentHarness.COPILOT,
+            plugin_root,
+            runtime=runtime,
+        )
+        profile_dir: Path = instructions_profile_dir(entry.customization_profile)
+        target_dir: Path = AgentHarness.COPILOT.get_target_dir(repo_path)
+        instructions_enabled: bool = setup_instructions(agent_config.instructions, profile_dir, target_dir, AgentHarness.COPILOT.instruction_filename)
+        skills_enabled: bool = setup_skills(agent_config.skills, profile_dir, target_dir)
+        custom_agent: str | None = setup_custom_agent(agent_config.agents, profile_dir, target_dir)
+        plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(agent_config, plugin_root, allow_copilot_manifest=True)
+
+        config = ExperimentConfiguration(
+            mcp_servers=mcp_server_names,
+            al_lsp_enabled=lsp_plugin_dir is not None,
             custom_instructions=instructions_enabled,
-            log_dir=output_dir,
-            mcp_config_json=mcp_config_json,
-            plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)),
-            granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access),
+            skills_enabled=skills_enabled,
             custom_agent=custom_agent,
-            workspace_mcp=True,
+            plugins=[plugin.record for plugin, _ in plugins] or None,
         )
 
-        metrics, _ = invoke_copilot(
-            prompt=prompt,
-            model=model,
-            work_dir=repo_path,
-            timeout=timeout,
-            env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials),
-            options=options,
-        )
-        logger.info(f"Copilot CLI run complete for: {entry.instance_id}")
-    except CopilotTimeoutError as exc:
-        raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc
-    else:
-        return metrics, config
-    finally:
-        if bc_gateway is not None:
-            bc_gateway.stop()
+        try:
+            lsp_plugin_dirs = (lsp_plugin_dir,) if lsp_plugin_dir is not None else ()
+            # --add-dir grants read+write (unlike --plugin-dir, which only registers a plugin), so hand it
+            # only to plugins that opt in via grant_dir_access - currently a temporary accommodation for
+            # BCQuality, whose skill reads its own knowledge files at runtime. Enabling a plugin must not
+            # silently widen the agent's sandbox access.
+            options = CopilotOptions(
+                allow_all_tools=True,
+                custom_instructions=instructions_enabled,
+                log_dir=output_dir,
+                mcp_config_json=mcp_config_json,
+                plugin_dirs=(*lsp_plugin_dirs, *(plugin_dir for _, plugin_dir in plugins)),
+                granted_dirs=tuple(plugin_dir for plugin, plugin_dir in plugins if plugin.grant_dir_access),
+                custom_agent=custom_agent,
+                workspace_mcp=True,
+            )
+
+            metrics, response = invoke_copilot(
+                prompt=prompt,
+                model=self.model,
+                work_dir=repo_path,
+                timeout=self.timeout if request.timeout is None else request.timeout,
+                env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials),
+                options=options,
+            )
+            logger.info(f"Copilot CLI run complete for: {entry.instance_id}")
+        except CopilotTimeoutError as exc:
+            raise AgentTimeoutError("Copilot CLI timed out", metrics=exc.metrics, config=config) from exc
+        else:
+            return AgentResult(response=response, metrics=metrics, experiment=config)
+        finally:
+            if bc_gateway is not None:
+                bc_gateway.stop()

src/bcbench/agent/pr_review/agent.py

@@ -22,6 +22,7 @@
 
 import yaml
 from bcbench_core.agent.experiment import ExperimentConfiguration
+from bcbench_core.agent.harness import AgentResult
 from bcbench_core.exceptions import AgentError, AgentTimeoutError
 from bcbench_core.git import commit_changes, has_changes, init_repo
 
@@ -161,16 +162,13 @@ def run_pr_review_agent(
     engine_path: Path | None = None,
     min_severity: str | None = None,
     timeout: int = 60 * 60,
-) -> tuple[PRReviewMetrics, ExperimentConfiguration]:
+) -> AgentResult:
     """Run the engine's complete local review pipeline and write review.json.
 
     Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents
     PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it
     owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific
     inputs (BC-ALAgents source, min severity) for the code-review category only.
-
-    Returns:
-        Tuple of (PRReviewMetrics, ExperimentConfiguration).
     """
     if category is not EvaluationCategory.CODE_REVIEW:
         raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.")
@@ -235,4 +233,4 @@ def run_pr_review_agent(
         logger.exception("Unexpected error running engine review")
         raise
     else:
-        return build_pr_review_metrics(output_dir, time.monotonic() - start), config
+        return AgentResult(metrics=build_pr_review_metrics(output_dir, time.monotonic() - start), experiment=config)