From cbf81187310c989018c4e20cf95e48915de75084 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 17:51:54 -0800 Subject: [PATCH 1/7] feat(agents): select an agent's own tools at deploy, and claude-code's A task can now say which of an agent's own tools it may use: deploy_agent takes builtin_tools, {"disable": "*"}, {"disable": [names]} or {"only": [names]}. It travels on agent-config/v1, as configuration does. The protocol's BuiltinTools types the field over a Literal of the agent's tools, so the card's schema names them, and its enabled() says what a selection leaves on. deploy_agent refuses, before any request, an agent whose agent-config has no builtin_tools or a selection naming a tool its schema doesn't. claude-code names the pinned CLI's 24 tools and turns a selection into --tools with the tools left on, plus --strict-mcp-config, so its model is offered only those and the task env's MCP servers. The bench needs {"disable": "*"} for its GUI-only rule. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 17 ++++++++ .../agentenv_protocol/a2a_agent/__init__.py | 2 + .../agentenv_protocol/a2a_agent/tasks/v1.py | 28 +++++++++++- .../agentenv-protocol/tests/test_a2a_agent.py | 27 +++++++++++- src/agent_env/agents/README.md | 5 ++- .../agents/agents/claude-code/a2a_server.py | 14 +++++- .../agents/claude-code/claude_bridge.py | 14 ++++-- .../task_step/task_steps/deploy_agent.py | 28 ++++++++++++ .../cli/run_bundle_claude_code_test.py | 14 +++++- .../agents/claude_code/test_card_contract.py | 2 +- .../test_claude_bridge_builtin_tools.py | 40 +++++++++++++++++ .../test_deploy_agent_role_delivery.py | 43 ++++++++++++++++++- 12 files changed, 222 insertions(+), 12 deletions(-) create mode 100644 tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 5462c89e6..7999d9b78 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1007,6 +1007,23 @@ class MyAgentConfig(AgentConfig): provider_token: WriteOnly[str | None] = None ``` +An agent that can turn its own tools off, such as its shell, file or web tools, takes `builtin_tools`, typed +`BuiltinTools` over a `Literal` of its tools' names, so its card's schema names them. A task selects them on +`deploy_agent`: `{"disable": "*"}` turns them all off, `{"disable": [...]}` the ones named, and `{"only": [...]}` keeps +the ones named. `request.config.builtin_tools.enabled()` is what the selection leaves on, for the agent to +pass to its runtime. The MCP tools AgentEnv gives it are not its own and stay. A selection for an agent whose config has +no `builtin_tools`, or that names a tool its schema doesn't, is refused before the deploy sends it anything. + +```python +from typing import Literal + +from agentenv_protocol.a2a_agent import AgentConfig, BuiltinTools + + +class MyAgentConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["shell", "read_file", "web_fetch"]] | None = None +``` + `output_format` is author-owned in v1 rather than a field on the base `AgentConfig`. An agent that supports structured output must declare the field on its config subclass, apply it to its model or runtime, and return the value diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 67a1b904c..8ccde546d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -102,6 +102,7 @@ from .staging import STAGING_ENDPOINT, STAGING_V1_URI, StagingStore, staging_routes from .tasks.v1 import ( AgentConfig, + BuiltinTools, AgentRunResult, DataPart, FilePart, @@ -154,6 +155,7 @@ "AgentIdentity", "AgentRunResult", "AttributionProbeResponse", + "BuiltinTools", "BundleSkillRequest", "ChangelogIncrement", "ContextEventsTrajectoryRequest", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 90fd4bc45..c65ac84ec 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -17,7 +17,7 @@ from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union from a2a.types import TaskStatusUpdateEvent -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator logger = logging.getLogger(__name__) @@ -127,6 +127,32 @@ class AgentConfig(BaseModel): timeout_seconds: int = 600 +ToolName = TypeVar("ToolName", bound=str) + + +class BuiltinTools(BaseModel, Generic[ToolName]): + """Which of an agent's own tools it may use, as a task selects them: ``disable`` turns off the ones it names, or + all of them with ``"*"``; ``only`` keeps the ones it names. An agent's config takes it as + ``builtin_tools: BuiltinTools[] | None``, so its card's agent-config schema names them.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + disable: Literal["*"] | list[ToolName] | None = None + only: list[ToolName] | None = None + + @model_validator(mode="after") + def _one_of(self) -> BuiltinTools: + if (self.disable is None) == (self.only is None): + raise ValueError("builtin_tools takes exactly one of disable and only") + return self + + def enabled(self, tools: Iterable[str]) -> list[str]: + """Those of ``tools`` the selection leaves on, in their order.""" + if self.only is not None: + return [tool for tool in tools if tool in self.only] + return [] if self.disable == "*" else [tool for tool in tools if tool not in self.disable] + + ConfigT = TypeVar("ConfigT", bound=AgentConfig) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent.py b/packages/agentenv-protocol/tests/test_a2a_agent.py index 422b4e334..8e84a01ab 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent.py @@ -6,7 +6,7 @@ import time from dataclasses import FrozenInstanceError from functools import wraps -from typing import Any, Union +from typing import Any, Literal, Union from unittest.mock import MagicMock import httpx @@ -44,6 +44,7 @@ AgentEnvAgent, AgentIdentity, AttributionProbeResponse, + BuiltinTools, BundleSkillRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, @@ -1774,6 +1775,30 @@ async def run(self, request: TaskRequest[RuntimeConfig]) -> TaskResult: assert "default" not in schema["$defs"]["NestedConfig"]["properties"]["token"] +def test_builtin_tools_names_the_agents_own_tools_on_its_card_and_takes_one_selection() -> None: + class RuntimeConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["Bash", "Read", "WebFetch"]] | None = None + + @a2a_agent(identity=AgentIdentity(name="tooled", description="test", version="1"), config=RuntimeConfig) + class Agent(AgentEnvAgent): + async def run(self, request: TaskRequest[RuntimeConfig]) -> TaskResult: + return TaskResult.text("ok") + + with TestClient(Agent().create_app()) as client: + card = client.get("/.well-known/agent-card.json").json() + statuses = [_operation(client, card, AGENT_CONFIG_V1.uri, "set", {"builtin_tools": selection}).status_code + for selection in ({"disable": "*"}, {"only": ["Read"]}, {"disable": ["Glob"]}, + {"disable": "*", "only": ["Read"]}, {})] + + request = _extension(card, AGENT_CONFIG_V1.uri)["params"]["methods"]["set"]["request"] + tools = ["Bash", "Read", "WebFetch"] + assert "builtin_tools" in request["supported"] and '"enum": ["Bash", "Read", "WebFetch"]' in json.dumps(request) + assert statuses == [200, 200, 400, 400, 400] + assert [BuiltinTools[str].model_validate(selection).enabled(tools) + for selection in ({"disable": "*"}, {"disable": ["WebFetch"]}, {"only": ["WebFetch", "Read"]})] == [ + [], ["Bash", "Read"], ["Read", "WebFetch"]] + + def test_validation_errors_do_not_echo_rejected_values() -> None: secret = "sk-live-SUPERSECRET" diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index 3ec6562c2..674ae6578 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -21,7 +21,10 @@ the folder changes. A run that deploys the agent registers that image when it's pulls it by its digest and builds it from this folder only when it can't, and a provider that runs images by name can run it. The CLI's version is the Dockerfile's `CLAUDE_CODE_VERSION`. The image gives the scripts the model runs -`node`, without npm or npx, and the Python packages `model-requirements.txt` pins. +`node`, without npm or npx, and the Python packages `model-requirements.txt` pins. A `deploy_agent` `builtin_tools` +selection, such as `{"disable": "*"}`, becomes the CLI's `--tools` list with `--strict-mcp-config`, so the model is +offered only the CLI tools it leaves on and the task env's MCP servers; the names are `ClaudeCodeTool`'s in +`a2a_server.py`, the pinned CLI's. `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 082ee207b..8a143ba33 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -20,7 +20,7 @@ from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager from pathlib import Path -from typing import Any +from typing import Any, Literal, get_args from a2a.types import AgentSkill from agentenv_protocol.a2a_agent import ( @@ -34,6 +34,7 @@ AgentConfig, AgentEnvAgent, AgentIdentity, + BuiltinTools, BundleSkillRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, @@ -121,6 +122,13 @@ _INSTALL_COMMANDS_HOST = [_UNPACK, _ENDPOINT + f"bash {{work_dir}}/agent/install.sh host {{agent_name}} {{a2a_port}} {_HOST_WORKDIR}"] +# The CLI's own tools, by the names its model sees, in the pinned version. +ClaudeCodeTool = Literal["Agent", "Bash", "CronCreate", "CronDelete", "CronList", "Edit", "EnterWorktree", + "ExitWorktree", "ListAgents", "NotebookEdit", "Read", "ReportFindings", "ScheduleWakeup", + "SendMessage", "Skill", "TaskCreate", "TaskGet", "TaskList", "TaskStop", "TaskUpdate", + "WebFetch", "WebSearch", "Workflow", "Write"] + + class ClaudeCodeConfig(AgentConfig): timeout_seconds: int = 1800 model: str | None = None @@ -134,6 +142,8 @@ class ClaudeCodeConfig(AgentConfig): effort: str | None = None max_turns: int | None = None max_thinking_tokens: int | None = None + # Which of the CLI's own tools it may use. With any selection, its MCP servers are only the ones agent-env gives it. + builtin_tools: BuiltinTools[ClaudeCodeTool] | None = None def _rewrite_session_ids(jsonl_text: str, session_id: str) -> tuple[list[str], int]: @@ -488,6 +498,8 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon effort=cfg.effort, max_turns=cfg.max_turns, max_thinking_tokens=cfg.max_thinking_tokens, + builtin_tools=(None if cfg.builtin_tools is None + else cfg.builtin_tools.enabled(get_args(ClaudeCodeTool))), event_callback=request.trajectory.append, ) except Exception as e: diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index c7163188e..2e3a2c440 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -383,6 +383,7 @@ async def run_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, + builtin_tools: list[str] | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: """Run one turn, resuming the CLI session when a transient upstream failure cuts it short; a resume spends what @@ -395,7 +396,7 @@ async def run_claude( _spawn_claude, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, system_prompt=system_prompt, output_format=output_format, project_id=project_id, task_id=task_id, role=role, effort=effort, max_turns=max_turns, max_thinking_tokens=max_thinking_tokens, - event_callback=event_callback, + builtin_tools=builtin_tools, event_callback=event_callback, ) attempt_prompt, attempt_resume, attempt_session = prompt, resume, session_id trajectory: list[dict] = [] @@ -572,8 +573,10 @@ def _with_1m_window(model: str) -> str: def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: list[str] | None, mcp_servers: dict[str, dict] | None, system_prompt: str | None, output_format: dict | None, - role: str | None, effort: str | None, max_turns: int | None) -> list[str]: - """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec.""" + role: str | None, effort: str | None, max_turns: int | None, + builtin_tools: list[str] | None = None) -> list[str]: + """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec. ``builtin_tools``, when + given, are the only CLI tools the model is offered, and its MCP servers are only ``mcp_servers``.""" cmd = ["claude", "-p", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions", "--setting-sources", "user"] cmd += ["--resume", session_id] if resume else ["--session-id", session_id] @@ -593,6 +596,8 @@ def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: cmd += ["--json-schema", json.dumps(output_format["schema"])] if allowed_tools: cmd += ["--allowedTools", ",".join(allowed_tools)] + if builtin_tools is not None: + cmd += ["--tools", ",".join(builtin_tools), "--strict-mcp-config"] if mcp_servers: def entry(info: dict) -> dict: server: dict = {"type": "http", "url": info["url"]} @@ -622,11 +627,12 @@ async def _spawn_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, + builtin_tools: list[str] | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: cmd = _command(session_id, resume, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, system_prompt=system_prompt, output_format=output_format, role=role, effort=effort, - max_turns=max_turns) + max_turns=max_turns, builtin_tools=builtin_tools) logger.info(f"Running claude CLI (resume={resume}, session_id={session_id}, model={model})") WORKSPACE_DIR.mkdir(parents=True, exist_ok=True) proc = await _started(asyncio.create_subprocess_exec( diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index 8e763e702..6ad0b0322 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -8,11 +8,14 @@ import httpx from agentenv_protocol.a2a_agent import ( + BuiltinTools, NamespaceChangelogEnableResponse, ObjectChangelogApplyResponse, ObjectSnapshotLoadResponse, card_request_accepts, ) +from jsonschema import Draft202012Validator +from jsonschema.exceptions import best_match from agent_env.a2a_agent import protocol from agent_env.a2a_agent.object_transfer import ( @@ -146,6 +149,22 @@ def _skill_fields(skill: dict) -> dict: } + +def _check_builtin_tools(card: dict, agent_name: str, selection: dict) -> None: + """Refuse a selection of the agent's own tools that its card can't take: its agent-config has no ``builtin_tools``, + or its schema doesn't name a tool the selection does.""" + extension = A2AAgent.find_extension(card, A2AAgent.EXT_AGENT_CONFIG) or {} + request = (extension.get("params") or {}).get("methods", {}).get("set", {}).get("request", {}) + if "builtin_tools" not in request.get("supported", []): + raise RuntimeError(f"Agent '{agent_name}' cannot have its own tools selected: its card's " + f"{A2AAgent.EXT_AGENT_CONFIG} takes no 'builtin_tools'") + schema = request.get("schema") or {} + field = (schema.get("properties") or {}).get("builtin_tools") + if field is not None: + error = best_match(Draft202012Validator({**field, "$defs": schema.get("$defs", {})}).iter_errors(selection)) + if error is not None: + raise RuntimeError(f"Agent '{agent_name}' can't take builtin_tools {selection}: {error.message}") + class DeployAgentTaskStep(TaskStep): type: ClassVar[str] = "deploy_agent" entity_refs = ( @@ -186,6 +205,7 @@ def __init__( agent_snapshot_target_context_id: Optional[str] = None, network_policy: Optional[dict] = None, role: Optional[str] = None, + builtin_tools: Optional[dict] = None, enable_agent_changelog: bool = False, agent_changelog_toolcall_position_exclusive: Optional[int] = None, sandbox_name: Optional[str] = None, @@ -217,6 +237,8 @@ def __init__( self.metadata = metadata or {} self.network_policy = NetworkPolicy.from_dict(network_policy).to_dict() if network_policy else None self.role = role or None + self.builtin_tools = (None if builtin_tools is None + else BuiltinTools[str].model_validate(builtin_tools).model_dump(exclude_none=True)) self.enable_agent_changelog = enable_agent_changelog self.agent_changelog_object_url = agent_changelog_object_url self.agent_changelog_toolcall_position_exclusive = agent_changelog_toolcall_position_exclusive @@ -254,6 +276,7 @@ def to_dict(self) -> dict: base["metadata"] = self.metadata base["network_policy"] = self.network_policy base["role"] = self.role + base["builtin_tools"] = self.builtin_tools base["enable_agent_changelog"] = self.enable_agent_changelog base["agent_changelog_object_url"] = self.agent_changelog_object_url base["agent_changelog_toolcall_position_exclusive"] = self.agent_changelog_toolcall_position_exclusive @@ -286,6 +309,7 @@ def from_dict(cls, data: dict) -> DeployAgentTaskStep: metadata=dict(data.get("metadata") or {}), network_policy=data.get("network_policy"), role=data.get("role"), + builtin_tools=data.get("builtin_tools"), enable_agent_changelog=data.get("enable_agent_changelog", False), agent_changelog_object_url=data.get("agent_changelog_object_url"), agent_changelog_toolcall_position_exclusive=data.get("agent_changelog_toolcall_position_exclusive"), @@ -417,6 +441,10 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas identity_payload["description"] = self.agent_description if self.role is not None: identity_payload["role"] = self.role + if self.builtin_tools is not None: + # Before the first request to the agent, so an agent that would keep its own tools never runs the task. + _check_builtin_tools(card, self.agent_name, self.builtin_tools) + identity_payload["builtin_tools"] = self.builtin_tools identity = A2AAgent.negotiate_agent_config(card, identity_payload) if self.role is not None: carried_by = [via for via, carried in ( diff --git a/tst/integration/cli/run_bundle_claude_code_test.py b/tst/integration/cli/run_bundle_claude_code_test.py index 019d0728b..a6742b9ee 100644 --- a/tst/integration/cli/run_bundle_claude_code_test.py +++ b/tst/integration/cli/run_bundle_claude_code_test.py @@ -6,6 +6,7 @@ import json import logging import uuid +from typing import get_args import pytest from click.testing import CliRunner @@ -26,6 +27,7 @@ from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.agents.claude_code.agent import a2a_server from tst.util.fake_anthropic import FakeAnthropic, answer, delegate_then_answer, tool_then_answer pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -156,14 +158,21 @@ def test_the_alias_claude_code_names_the_shipped_agent_until_the_store_has_an_ag assert named == [shipped, "claude-code"] -def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with(state, endpoint): +@pytest.mark.parametrize("builtin_tools, kept", [ + (None, set(get_args(a2a_server.ClaudeCodeTool))), + ({"disable": "*"}, set()), + ({"disable": ["WebFetch"]}, set(get_args(a2a_server.ClaudeCodeTool)) - {"WebFetch"}), +], ids=["own-tools", "none-of-its-own", "all-but-webfetch"]) +def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with(state, endpoint, builtin_tools, kept): + """With no selection, the CLI offers exactly the tools claude-code's card names; a selection leaves the rest off.""" endpoint.respond = tool_then_answer("__echo", {"text": "ping-from-tool"}, prefix="the tool said: ") root = _bundle(state / "tools", "echo", [ {"id": "env", "type": "deploy_env", "env_id": ECHO_ENV, "sandbox_type": "local"}, *_ask("Call the echo tool with the text ping-from-tool.", "ping-from-tool", env_ids=[ECHO_ENV]), ]) + selected = {} if builtin_tools is None else {"builtin_tools": builtin_tools} root.joinpath("tasks/echo.json").write_text(json.dumps( - [{**step, "depends_on": ["env"]} if step["id"] == "agent" else step + [{**step, "depends_on": ["env"], **selected} if step["id"] == "agent" else step for step in json.loads(root.joinpath("tasks/echo.json").read_text())])) result = _run(root) @@ -171,6 +180,7 @@ def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with( offered = sorted({tool["name"] for call in endpoint.messages() for tool in call.body.get("tools") or []}) seen = (result.output, offered, endpoint.tool_results()) assert result.exit_code == 0 and "tasks/echo.json v1: passed" in result.output, seen + assert {name for name in offered if not name.startswith("mcp__")} == kept, seen # The fake never repeats the prompt, so the text reached the answer through the tool. assert any(name.endswith("__echo") for name in offered), seen assert ["ping-from-tool" in result for result in endpoint.tool_results()] == [True], seen diff --git a/tst/unit/agents/claude_code/test_card_contract.py b/tst/unit/agents/claude_code/test_card_contract.py index 510c8965b..0ace58b4f 100644 --- a/tst/unit/agents/claude_code/test_card_contract.py +++ b/tst/unit/agents/claude_code/test_card_contract.py @@ -67,7 +67,7 @@ def test_extensions_and_their_routes(app): def test_the_config_takes_every_field_prompt_agent_sends_that_the_cli_has(app): config = _extensions(app)["urn:agentenv:agent-config/v1"]["params"]["methods"] assert config["set"]["request"]["supported"] == [ - "allowed_tools", "description", "effort", "max_thinking_tokens", "max_turns", "model", "name", + "allowed_tools", "builtin_tools", "description", "effort", "max_thinking_tokens", "max_turns", "model", "name", "output_format", "project_id", "role", "system_prompt", "task_id", "timeout_seconds", ] assert set(config["set"]["request"]["schema"]["properties"]) == set(config["set"]["request"]["supported"]) diff --git a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py new file mode 100644 index 000000000..3a1f17ca9 --- /dev/null +++ b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py @@ -0,0 +1,40 @@ +"""A selection of the CLI's own tools: the CLI is offered only the tools it leaves on, by the names its card lists, +and only the MCP servers agent-env gives it; with none, the CLI keeps its own.""" + +from typing import get_args + +import pytest + +from tst.unit.agents.claude_code.agent import a2a_server, claude_bridge + +TOOLS = get_args(a2a_server.ClaudeCodeTool) + + +def _command(selection: dict | None) -> list[str]: + config = a2a_server.ClaudeCodeConfig(builtin_tools=selection) + return claude_bridge._command( + "sess-1", False, model=None, allowed_tools=None, mcp_servers=None, system_prompt=None, output_format=None, + role=None, effort=None, max_turns=None, + builtin_tools=None if config.builtin_tools is None else config.builtin_tools.enabled(TOOLS)) + + +def test_no_selection_leaves_the_clis_tools_and_mcp_servers_alone(): + command = _command(None) + + assert "--tools" not in command and "--strict-mcp-config" not in command + + +@pytest.mark.parametrize("selection, offered", [ + ({"disable": "*"}, []), + ({"disable": ["WebFetch", "WebSearch"]}, [tool for tool in TOOLS if tool not in ("WebFetch", "WebSearch")]), + ({"only": ["Read", "Agent"]}, ["Agent", "Read"]), +]) +def test_a_selection_is_the_clis_tool_list_with_only_agent_envs_mcp_servers(selection, offered): + command = _command(selection) + + assert command[command.index("--tools") + 1] == ",".join(offered) and "--strict-mcp-config" in command + + +def test_a_tool_the_cli_does_not_have_is_refused(): + with pytest.raises(ValueError, match="Glob"): + a2a_server.ClaudeCodeConfig(builtin_tools={"only": ["Glob"]}) diff --git a/tst/unit/task_step/test_deploy_agent_role_delivery.py b/tst/unit/task_step/test_deploy_agent_role_delivery.py index d9e0707ae..e642fd0de 100644 --- a/tst/unit/task_step/test_deploy_agent_role_delivery.py +++ b/tst/unit/task_step/test_deploy_agent_role_delivery.py @@ -1,13 +1,16 @@ """deploy_agent delivers the step's role to the agent, or fails the step before any request reaches it; it never -records a role the agent did not get.""" +records a role the agent did not get. Its selection of the agent's own tools goes the same way: refused before any +request by an agent whose agent-config doesn't take it or names no such tool.""" import json import logging from types import SimpleNamespace +from typing import Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from agentenv_protocol.a2a_agent import AgentConfig, BuiltinTools from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.env.env import DeployedGatewayEnv @@ -46,6 +49,14 @@ def _card(*extensions: dict) -> dict: ROLELESS_CONFIG = _agent_config(["name", "description"]) +class _ToolConfig(AgentConfig): + builtin_tools: BuiltinTools[Literal["Bash", "Read"]] | None = None + + +TOOLS_CONFIG = _agent_config(["name", "description", "builtin_tools"]) +TOOLS_CONFIG["params"]["methods"]["set"]["request"]["schema"] = _ToolConfig.model_json_schema() + + @pytest.mark.asyncio async def test_a_card_with_both_paths_gets_the_role_on_both_and_the_agent_records_its_envs(wiring, caplog): with caplog.at_level(logging.WARNING, logger=_LOGGER): @@ -186,6 +197,36 @@ async def test_an_explicit_cli_role_env_var_is_kept(wiring): assert wiring.env_vars == {ROLE_ENV_VAR: "cli"} +@pytest.mark.asyncio +async def test_a_selection_of_the_agents_own_tools_is_sent_once_with_its_identity(wiring): + context = await wiring.deploy(_card(HEADERS_MCP, TOOLS_CONFIG), role=None, builtin_tools={"disable": "*"}) + + assert wiring.bodies("/ext/agent-config") == [{"name": "solver", "builtin_tools": {"disable": "*"}}] + assert len(context.deployed_agents) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card, refusal", [ + (_card(HEADERS_MCP, ROLE_CONFIG), "takes no 'builtin_tools'"), + (_card(HEADERS_MCP), "takes no 'builtin_tools'"), + (_card(HEADERS_MCP, TOOLS_CONFIG), "'Glob' is not one of"), +], ids=["config-without-it", "no-agent-config", "unknown-tool"]) +async def test_a_selection_the_agent_cannot_take_fails_before_any_request(wiring, card, refusal): + with pytest.raises(RuntimeError, match=refusal): + await wiring.deploy(card, role=None, builtin_tools={"only": ["Read", "Glob"]}) + + assert wiring.requests == [] + assert wiring.agent.close.await_count == 1 + + +def test_a_selection_takes_exactly_one_of_disable_and_only(): + for selection in ({}, {"disable": "*", "only": ["Read"]}, {"enable": ["Read"]}): + with pytest.raises(ValueError): + DeployAgentTaskStep(id="ag", version=None, builtin_tools=selection) + step = DeployAgentTaskStep(id="ag", version=None, builtin_tools={"disable": ["Bash"]}) + assert DeployAgentTaskStep.from_dict({**step.to_dict(), "version": 1}).builtin_tools == {"disable": ["Bash"]} + + class _Wiring: """One deploy_agent run: the requests the agent received, the agent double, and the step's context.""" From 92cac7043bea650caa961f3fc5a6fad2fa7ce7da Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 18:40:45 -0800 Subject: [PATCH 2/7] fix(agents): claude-code takes its own copy of the selection type; deploy_agent's check needs no A2AAgent The claude-code image installs the newest released agentenv-framework-protocol, which has no BuiltinTools yet, so the agent declares the same fields and checks itself until its floor can be raised. The builtin_tools check takes the agent-config extension from its caller, since deploy_agent imports A2AAgent only inside execute. Co-Authored-By: Claude Opus 5.5 --- .../agents/agents/claude-code/a2a_server.py | 26 +++++++++++++++++-- .../task_step/task_steps/deploy_agent.py | 15 ++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 8a143ba33..eb1d85ef4 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -34,7 +34,6 @@ AgentConfig, AgentEnvAgent, AgentIdentity, - BuiltinTools, BundleSkillRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, @@ -59,6 +58,7 @@ extension, ) from agentenv_protocol.transfers import Uploaded, download, upload +from pydantic import BaseModel, ConfigDict, model_validator from starlette.applications import Starlette from starlette.exceptions import HTTPException @@ -129,6 +129,28 @@ "WebFetch", "WebSearch", "Workflow", "Write"] +class ClaudeCodeTools(BaseModel): + """The protocol's ``BuiltinTools[ClaudeCodeTool]``, which the image's protocol floor doesn't have yet: the same + fields and checks, so the card's schema is the same.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + disable: Literal["*"] | list[ClaudeCodeTool] | None = None + only: list[ClaudeCodeTool] | None = None + + @model_validator(mode="after") + def _one_of(self) -> ClaudeCodeTools: + if (self.disable is None) == (self.only is None): + raise ValueError("builtin_tools takes exactly one of disable and only") + return self + + def enabled(self, tools: Iterable[str]) -> list[str]: + """Those of ``tools`` the selection leaves on, in their order.""" + if self.only is not None: + return [tool for tool in tools if tool in self.only] + return [] if self.disable == "*" else [tool for tool in tools if tool not in self.disable] + + class ClaudeCodeConfig(AgentConfig): timeout_seconds: int = 1800 model: str | None = None @@ -143,7 +165,7 @@ class ClaudeCodeConfig(AgentConfig): max_turns: int | None = None max_thinking_tokens: int | None = None # Which of the CLI's own tools it may use. With any selection, its MCP servers are only the ones agent-env gives it. - builtin_tools: BuiltinTools[ClaudeCodeTool] | None = None + builtin_tools: ClaudeCodeTools | None = None def _rewrite_session_ids(jsonl_text: str, session_id: str) -> tuple[list[str], int]: diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index 6ad0b0322..cbacdc804 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -8,6 +8,7 @@ import httpx from agentenv_protocol.a2a_agent import ( + AGENT_CONFIG_V1, BuiltinTools, NamespaceChangelogEnableResponse, ObjectChangelogApplyResponse, @@ -150,14 +151,13 @@ def _skill_fields(skill: dict) -> dict: -def _check_builtin_tools(card: dict, agent_name: str, selection: dict) -> None: - """Refuse a selection of the agent's own tools that its card can't take: its agent-config has no ``builtin_tools``, - or its schema doesn't name a tool the selection does.""" - extension = A2AAgent.find_extension(card, A2AAgent.EXT_AGENT_CONFIG) or {} - request = (extension.get("params") or {}).get("methods", {}).get("set", {}).get("request", {}) +def _check_builtin_tools(agent_config: dict | None, agent_name: str, selection: dict) -> None: + """Refuse a selection of the agent's own tools that its card's agent-config extension can't take: it has no + ``builtin_tools``, or its schema doesn't name a tool the selection does.""" + request = ((agent_config or {}).get("params") or {}).get("methods", {}).get("set", {}).get("request", {}) if "builtin_tools" not in request.get("supported", []): raise RuntimeError(f"Agent '{agent_name}' cannot have its own tools selected: its card's " - f"{A2AAgent.EXT_AGENT_CONFIG} takes no 'builtin_tools'") + f"{AGENT_CONFIG_V1.uri} takes no 'builtin_tools'") schema = request.get("schema") or {} field = (schema.get("properties") or {}).get("builtin_tools") if field is not None: @@ -443,7 +443,8 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas identity_payload["role"] = self.role if self.builtin_tools is not None: # Before the first request to the agent, so an agent that would keep its own tools never runs the task. - _check_builtin_tools(card, self.agent_name, self.builtin_tools) + _check_builtin_tools(A2AAgent.find_extension(card, A2AAgent.EXT_AGENT_CONFIG), self.agent_name, + self.builtin_tools) identity_payload["builtin_tools"] = self.builtin_tools identity = A2AAgent.negotiate_agent_config(card, identity_payload) if self.role is not None: From 9834d78f3d4bcf3ec1bb10341bf497847a645d4c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 18:47:21 -0800 Subject: [PATCH 3/7] fix(agents): claude-code names all 27 of its CLI's tools, and a list to turn off keeps the rest at their defaults In the agent's container the CLI's default tools depend on the model (the fake one gets no Task tools) and gain the three MCP resource tools when MCP servers are attached, so the card names every tool the CLI has, and the test checks the CLI offers nothing it doesn't name. A disable list now goes to --disallowedTools, so the rest keep the CLI's defaults instead of every named tool being turned on; "*" and only stay a --tools list. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/agents/README.md | 7 +-- .../agents/agents/claude-code/a2a_server.py | 17 +++--- .../agents/claude-code/claude_bridge.py | 18 ++++--- .../cli/run_bundle_claude_code_test.py | 53 +++++++++++-------- .../test_claude_bridge_builtin_tools.py | 28 +++++----- 5 files changed, 67 insertions(+), 56 deletions(-) diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index 674ae6578..31fd7ef23 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -22,9 +22,10 @@ pulls it by its digest and builds it from this folder only when it can't, and a run it. The CLI's version is the Dockerfile's `CLAUDE_CODE_VERSION`. The image gives the scripts the model runs `node`, without npm or npx, and the Python packages `model-requirements.txt` pins. A `deploy_agent` `builtin_tools` -selection, such as `{"disable": "*"}`, becomes the CLI's `--tools` list with `--strict-mcp-config`, so the model is -offered only the CLI tools it leaves on and the task env's MCP servers; the names are `ClaudeCodeTool`'s in -`a2a_server.py`, the pinned CLI's. +selection picks the CLI's own tools: `{"disable": "*"}` and `{"only": [...]}` become its `--tools` list, a list to +turn off becomes `--disallowedTools`, which keeps the CLI's defaults for the rest, and any selection adds +`--strict-mcp-config`, so its MCP servers are only the task env's. The names are `ClaudeCodeTool`'s in +`a2a_server.py`: every tool the pinned CLI has, though which it offers by default depends on the model. `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index eb1d85ef4..37e532a45 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -20,7 +20,7 @@ from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager from pathlib import Path -from typing import Any, Literal, get_args +from typing import Any, Literal from a2a.types import AgentSkill from agentenv_protocol.a2a_agent import ( @@ -122,9 +122,11 @@ _INSTALL_COMMANDS_HOST = [_UNPACK, _ENDPOINT + f"bash {{work_dir}}/agent/install.sh host {{agent_name}} {{a2a_port}} {_HOST_WORKDIR}"] -# The CLI's own tools, by the names its model sees, in the pinned version. +# The CLI's own tools, by the names its model sees, in the pinned version. Which of them it offers by default depends +# on the model, and its MCP resource tools come only with MCP servers. ClaudeCodeTool = Literal["Agent", "Bash", "CronCreate", "CronDelete", "CronList", "Edit", "EnterWorktree", - "ExitWorktree", "ListAgents", "NotebookEdit", "Read", "ReportFindings", "ScheduleWakeup", + "ExitWorktree", "ListAgents", "ListMcpResourcesTool", "NotebookEdit", "Read", + "ReadMcpResourceDirTool", "ReadMcpResourceTool", "ReportFindings", "ScheduleWakeup", "SendMessage", "Skill", "TaskCreate", "TaskGet", "TaskList", "TaskStop", "TaskUpdate", "WebFetch", "WebSearch", "Workflow", "Write"] @@ -144,12 +146,6 @@ def _one_of(self) -> ClaudeCodeTools: raise ValueError("builtin_tools takes exactly one of disable and only") return self - def enabled(self, tools: Iterable[str]) -> list[str]: - """Those of ``tools`` the selection leaves on, in their order.""" - if self.only is not None: - return [tool for tool in tools if tool in self.only] - return [] if self.disable == "*" else [tool for tool in tools if tool not in self.disable] - class ClaudeCodeConfig(AgentConfig): timeout_seconds: int = 1800 @@ -520,8 +516,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon effort=cfg.effort, max_turns=cfg.max_turns, max_thinking_tokens=cfg.max_thinking_tokens, - builtin_tools=(None if cfg.builtin_tools is None - else cfg.builtin_tools.enabled(get_args(ClaudeCodeTool))), + builtin_tools=None if cfg.builtin_tools is None else cfg.builtin_tools.model_dump(exclude_none=True), event_callback=request.trajectory.append, ) except Exception as e: diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index 2e3a2c440..beb7f6aa4 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -383,7 +383,7 @@ async def run_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, - builtin_tools: list[str] | None = None, + builtin_tools: dict | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: """Run one turn, resuming the CLI session when a transient upstream failure cuts it short; a resume spends what @@ -574,9 +574,10 @@ def _with_1m_window(model: str) -> str: def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: list[str] | None, mcp_servers: dict[str, dict] | None, system_prompt: str | None, output_format: dict | None, role: str | None, effort: str | None, max_turns: int | None, - builtin_tools: list[str] | None = None) -> list[str]: - """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec. ``builtin_tools``, when - given, are the only CLI tools the model is offered, and its MCP servers are only ``mcp_servers``.""" + builtin_tools: dict | None = None) -> list[str]: + """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec. ``builtin_tools`` selects + the CLI's own tools: ``{"disable": "*"}`` offers none, ``{"only": [...]}`` just those, and ``{"disable": [...]}`` + the CLI's defaults but those; with any selection, its MCP servers are only ``mcp_servers``.""" cmd = ["claude", "-p", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions", "--setting-sources", "user"] cmd += ["--resume", session_id] if resume else ["--session-id", session_id] @@ -597,7 +598,12 @@ def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: if allowed_tools: cmd += ["--allowedTools", ",".join(allowed_tools)] if builtin_tools is not None: - cmd += ["--tools", ",".join(builtin_tools), "--strict-mcp-config"] + only, disable = builtin_tools.get("only"), builtin_tools.get("disable") + if only is not None or disable == "*": + cmd += ["--tools", ",".join(only or [])] + elif disable: + cmd += ["--disallowedTools", ",".join(disable)] + cmd.append("--strict-mcp-config") if mcp_servers: def entry(info: dict) -> dict: server: dict = {"type": "http", "url": info["url"]} @@ -627,7 +633,7 @@ async def _spawn_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, - builtin_tools: list[str] | None = None, + builtin_tools: dict | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: cmd = _command(session_id, resume, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, diff --git a/tst/integration/cli/run_bundle_claude_code_test.py b/tst/integration/cli/run_bundle_claude_code_test.py index a6742b9ee..5b202ba7d 100644 --- a/tst/integration/cli/run_bundle_claude_code_test.py +++ b/tst/integration/cli/run_bundle_claude_code_test.py @@ -159,31 +159,40 @@ def test_the_alias_claude_code_names_the_shipped_agent_until_the_store_has_an_ag @pytest.mark.parametrize("builtin_tools, kept", [ - (None, set(get_args(a2a_server.ClaudeCodeTool))), - ({"disable": "*"}, set()), - ({"disable": ["WebFetch"]}, set(get_args(a2a_server.ClaudeCodeTool)) - {"WebFetch"}), -], ids=["own-tools", "none-of-its-own", "all-but-webfetch"]) + (None, lambda defaults: defaults), + ({"disable": "*"}, lambda defaults: set()), + ({"disable": ["WebFetch"]}, lambda defaults: defaults - {"WebFetch"}), + ({"only": ["Read", "TaskList"]}, lambda defaults: {"Read", "TaskList"}), +], ids=["own-tools", "none-of-its-own", "all-but-webfetch", "only-two"]) def test_the_shipped_claude_code_calls_the_tools_of_the_env_it_is_deployed_with(state, endpoint, builtin_tools, kept): - """With no selection, the CLI offers exactly the tools claude-code's card names; a selection leaves the rest off.""" + """The CLI offers only tools claude-code's card names, and a selection offers what it leaves of them: ``only`` + the tools it names, a list to turn off the CLI's defaults but those.""" endpoint.respond = tool_then_answer("__echo", {"text": "ping-from-tool"}, prefix="the tool said: ") - root = _bundle(state / "tools", "echo", [ - {"id": "env", "type": "deploy_env", "env_id": ECHO_ENV, "sandbox_type": "local"}, - *_ask("Call the echo tool with the text ping-from-tool.", "ping-from-tool", env_ids=[ECHO_ENV]), - ]) - selected = {} if builtin_tools is None else {"builtin_tools": builtin_tools} - root.joinpath("tasks/echo.json").write_text(json.dumps( - [{**step, "depends_on": ["env"], **selected} if step["id"] == "agent" else step - for step in json.loads(root.joinpath("tasks/echo.json").read_text())])) - - result = _run(root) - offered = sorted({tool["name"] for call in endpoint.messages() for tool in call.body.get("tools") or []}) - seen = (result.output, offered, endpoint.tool_results()) - assert result.exit_code == 0 and "tasks/echo.json v1: passed" in result.output, seen - assert {name for name in offered if not name.startswith("mcp__")} == kept, seen - # The fake never repeats the prompt, so the text reached the answer through the tool. - assert any(name.endswith("__echo") for name in offered), seen - assert ["ping-from-tool" in result for result in endpoint.tool_results()] == [True], seen + def run(name, selection): + root = _bundle(state / name, "echo", [ + {"id": "env", "type": "deploy_env", "env_id": ECHO_ENV, "sandbox_type": "local"}, + *_ask("Call the echo tool with the text ping-from-tool.", "ping-from-tool", env_ids=[ECHO_ENV]), + ]) + selected = {} if selection is None else {"builtin_tools": selection} + root.joinpath("tasks/echo.json").write_text(json.dumps( + [{**step, "depends_on": ["env"], **selected} if step["id"] == "agent" else step + for step in json.loads(root.joinpath("tasks/echo.json").read_text())])) + before = len(endpoint.messages()) + result = _run(root) + offered = {tool["name"] for call in endpoint.messages()[before:] for tool in call.body.get("tools") or []} + seen = (result.output, sorted(offered), endpoint.tool_results()) + assert result.exit_code == 0 and "tasks/echo.json v1: passed" in result.output, seen + # The fake never repeats the prompt, so the text reached the answer through the tool. + assert any(name.endswith("__echo") for name in offered), seen + return {name for name in offered if not name.startswith("mcp__")} + + defaults = run("tools", None) + offered = defaults if builtin_tools is None else run("selected", builtin_tools) + + assert defaults <= set(get_args(a2a_server.ClaudeCodeTool)), sorted(defaults) + assert offered == kept(defaults), sorted(offered) + assert all("ping-from-tool" in result for result in endpoint.tool_results()) async def _send_project_id(step, context): diff --git a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py index 3a1f17ca9..61bf136e3 100644 --- a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py +++ b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py @@ -1,38 +1,38 @@ -"""A selection of the CLI's own tools: the CLI is offered only the tools it leaves on, by the names its card lists, -and only the MCP servers agent-env gives it; with none, the CLI keeps its own.""" - -from typing import get_args +"""A selection of the CLI's own tools: ``"*"`` and ``only`` name the tools the CLI may offer, a list to turn off leaves +the CLI's defaults but those, and with any selection its MCP servers are only agent-env's; with none, the CLI keeps +its own. A tool the CLI doesn't have is refused.""" import pytest from tst.unit.agents.claude_code.agent import a2a_server, claude_bridge -TOOLS = get_args(a2a_server.ClaudeCodeTool) - def _command(selection: dict | None) -> list[str]: config = a2a_server.ClaudeCodeConfig(builtin_tools=selection) return claude_bridge._command( "sess-1", False, model=None, allowed_tools=None, mcp_servers=None, system_prompt=None, output_format=None, role=None, effort=None, max_turns=None, - builtin_tools=None if config.builtin_tools is None else config.builtin_tools.enabled(TOOLS)) + builtin_tools=None if config.builtin_tools is None else config.builtin_tools.model_dump(exclude_none=True)) def test_no_selection_leaves_the_clis_tools_and_mcp_servers_alone(): command = _command(None) - assert "--tools" not in command and "--strict-mcp-config" not in command + assert not {"--tools", "--disallowedTools", "--strict-mcp-config"} & set(command) -@pytest.mark.parametrize("selection, offered", [ - ({"disable": "*"}, []), - ({"disable": ["WebFetch", "WebSearch"]}, [tool for tool in TOOLS if tool not in ("WebFetch", "WebSearch")]), - ({"only": ["Read", "Agent"]}, ["Agent", "Read"]), +@pytest.mark.parametrize("selection, flags", [ + ({"disable": "*"}, ["--tools", ""]), + ({"only": ["Read", "Agent"]}, ["--tools", "Read,Agent"]), + ({"only": []}, ["--tools", ""]), + ({"disable": ["WebFetch", "WebSearch"]}, ["--disallowedTools", "WebFetch,WebSearch"]), ]) -def test_a_selection_is_the_clis_tool_list_with_only_agent_envs_mcp_servers(selection, offered): +def test_a_selection_is_the_clis_tool_flags_with_only_agent_envs_mcp_servers(selection, flags): command = _command(selection) + at = command.index(flags[0]) - assert command[command.index("--tools") + 1] == ",".join(offered) and "--strict-mcp-config" in command + assert command[at:at + 2] == flags and "--strict-mcp-config" in command + assert not ({"--tools", "--disallowedTools"} - {flags[0]}) & set(command) def test_a_tool_the_cli_does_not_have_is_refused(): From ba518585c55f4e5fe13f13170b98a5032f2d5f04 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 18:55:20 -0800 Subject: [PATCH 4/7] fix(task-steps): prompt_agent sends the agent's tool selection again every turn Chaos testing restarted the claude-code container between two turns: the second turn offered all the CLI's own tools again, since the selection lived only in the agent's memory. prompt_agent now sends the selection deploy_agent recorded on the agent's card with each turn's config, so a restarted agent has it before the turn runs. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/prompt_agent.py | 2 + .../test_prompt_agent_builtin_tools.py | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tst/unit/task_step/test_prompt_agent_builtin_tools.py diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 145fd5e2d..c11fb716b 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -486,6 +486,8 @@ async def _converse( "output_format": self.output_format, "timeout_seconds": self.timeout_seconds, "task_id": context.metadata.get("task_id"), "agentenv_tools": self.agentenv_tools, + # Again each turn, so an agent that restarted since deploy_agent sent it doesn't get its own tools back. + "builtin_tools": card.get("builtin_tools"), } desired_config = {k: v for k, v in desired_config.items() if v is not None} model_params = get_config().get_model_params(self.model_params) diff --git a/tst/unit/task_step/test_prompt_agent_builtin_tools.py b/tst/unit/task_step/test_prompt_agent_builtin_tools.py new file mode 100644 index 000000000..4066f94a3 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_builtin_tools.py @@ -0,0 +1,67 @@ +"""prompt_agent sends the agent's selection of its own tools again on every turn, as deploy_agent recorded it on the +agent's card, so an agent that restarted since its deploy doesn't get its own tools back; an agent deployed without +one is sent none.""" + +from unittest.mock import MagicMock + +import pytest +from a2a.types import TaskState + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import prompt_agent as pa_mod +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +SUPPORTED = ["model", "builtin_tools"] + + +@pytest.fixture +def posted(monkeypatch): + """Mocks the turn's HTTP and store boundary, and collects each agent config posted.""" + cfg = MagicMock() + cfg.get_model_params.return_value = {} + cfg.get_model_for_role.return_value = None + monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) + configs = [] + + async def post_agent_config(url, fields): + configs.append(fields) + + async def send(url, parts, message_id, context_id, timeout): + return f"task-{message_id}", None + + async def poll(url, task_id, timeout, interval, **_): + return {"status": {"state": TaskState.completed, "message": {"parts": [{"kind": "text", "text": "done"}]}}} + + monkeypatch.setattr(pa_mod.protocol, "post_agent_config", post_agent_config) + monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send) + monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll) + return configs + + +def _context(**stamped) -> TaskStepContext: + card = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", "methods": {"set": {"request": {"supported": SUPPORTED}}}}}]}, **stamped} + context = TaskStepContext(instance_id="ti-1") + context.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://solver", a2a_url="http://solver", + a2a_card=card)) + return context + + +@pytest.mark.asyncio +async def test_every_turn_sends_the_selection_deploy_agent_recorded(posted): + context = _context(builtin_tools={"disable": "*"}) + + for turn in ("one", "two"): + await PromptAgentTaskStep(id=turn, version=None, prompt=turn, agent_name="solver").execute(context) + + assert [fields.get("builtin_tools") for fields in posted] == [{"disable": "*"}, {"disable": "*"}] + + +@pytest.mark.asyncio +async def test_an_agent_deployed_without_a_selection_is_sent_none(posted): + await PromptAgentTaskStep(id="one", version=None, prompt="one", agent_name="solver").execute(_context()) + + assert all("builtin_tools" not in fields for fields in posted) From 6dd7aaa4367e7cb2c173f9c6e2cf62630038f123 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 20:49:03 -0800 Subject: [PATCH 5/7] feat(validator): check that an agent honours builtin_tools An agent whose agent-config takes builtin_tools could advertise it and never enforce it, and validation would pass. The new verify_a2a_builtin_tools step deploys a second agent with {"disable": "*"} and no envs, so any tool call it makes is one of its own: asked to run a shell command it must make none (or, reporting no count, say it has no tool), an unknown tool must be refused, and agent-config must read the selection back. The validation agent itself is never limited, since the other checks need its tools. An agent without the field is recorded as not supporting it. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/validator.py | 2 + src/agent_env/task_step/registry.py | 2 + .../a2a_agent_validator/__init__.py | 2 + .../verify_a2a_builtin_tools.py | 112 +++++++++++++++++ .../test_verify_a2a_builtin_tools.py | 118 ++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py create mode 100644 tst/unit/task_step/test_verify_a2a_builtin_tools.py diff --git a/src/agent_env/a2a_agent/validator.py b/src/agent_env/a2a_agent/validator.py index 42ccac326..094f1dee6 100644 --- a/src/agent_env/a2a_agent/validator.py +++ b/src/agent_env/a2a_agent/validator.py @@ -185,6 +185,7 @@ async def _run( VerifyA2AAgentCardStep, VerifyA2AAgentConfigIdentityStep, VerifyA2AAgentMCPStep, + VerifyA2ABuiltinToolsStep, VerifyA2AModalitiesStep, VerifyA2APeerAgentsStep, VerifyA2ARoleStep, @@ -279,6 +280,7 @@ async def _run( VerifyA2AAgentCardStep(id=f"{task_id}-card", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2AAgentConfigIdentityStep(id=f"{task_id}-agent-config-identity", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2ARoleStep(id=f"{task_id}-role", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), + VerifyA2ABuiltinToolsStep(id=f"{task_id}-builtin-tools", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyA2ASystemPromptStep(id=f"{task_id}-system-prompt", version=None, a2a_agent_id=agent.id, a2a_agent_version=agent.version, depends_on=deploy_agent_dep, fail_task_on_error=False), PromptAgentTaskStep(id=protocol_prompt_id, version=None, prompt="Hello, respond with OK.", prompt_id=protocol_prompt_id, timeout_seconds=120, depends_on=deploy_agent_dep, fail_task_on_error=False), VerifyCoreA2AProtocolStep(id=f"{task_id}-protocol", version=None, a2a_agent_id=agent.id, prompt_id=protocol_prompt_id, depends_on=[TaskStepDependency(task_step_id=protocol_prompt_id)], fail_task_on_error=False), diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 83821a632..d343df2c2 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -69,6 +69,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_skill_config import VerifyA2ASkillConfigStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_snapshot import VerifyA2ASnapshotStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_trajectory import VerifyA2ATrajectoryStep + from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import VerifyA2ABuiltinToolsStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_role import VerifyA2ARoleStep from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_system_prompt import VerifyA2ASystemPromptStep from agent_env.task_step.task_steps.add_skills import AddSkillsTaskStep @@ -135,6 +136,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(VerifyA2AModalitiesStep): VerifyA2AModalitiesStep, _get_type(VerifySandboxTaskStep): VerifySandboxTaskStep, _get_type(AggregateVerifiersTaskStep): AggregateVerifiersTaskStep, + _get_type(VerifyA2ABuiltinToolsStep): VerifyA2ABuiltinToolsStep, _get_type(VerifyA2ARoleStep): VerifyA2ARoleStep, _get_type(VerifyA2ASystemPromptStep): VerifyA2ASystemPromptStep, } diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py index b879cad97..a386ca654 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/__init__.py @@ -1,6 +1,7 @@ from .verify_a2a_agent_card import VerifyA2AAgentCardStep from .verify_a2a_agent_config_identity import VerifyA2AAgentConfigIdentityStep from .verify_a2a_agent_mcp import VerifyA2AAgentMCPStep +from .verify_a2a_builtin_tools import VerifyA2ABuiltinToolsStep from .verify_a2a_core_protocol import VerifyCoreA2AProtocolStep from .verify_a2a_install import VerifyA2AInstallStep from .verify_a2a_modalities import VerifyA2AModalitiesStep @@ -15,6 +16,7 @@ "VerifyA2AAgentCardStep", "VerifyA2AAgentConfigIdentityStep", "VerifyA2AAgentMCPStep", + "VerifyA2ABuiltinToolsStep", "VerifyCoreA2AProtocolStep", "VerifyA2AInstallStep", "VerifyA2AModalitiesStep", diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py new file mode 100644 index 000000000..ac540e02b --- /dev/null +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py @@ -0,0 +1,112 @@ +"""Verify an agent whose agent-config takes ``builtin_tools`` honours it. A second agent is deployed with +``{"disable": "*"}`` and no envs, so it has no MCP tools and any tool call it makes is one of its own: asked to run a +shell command, it must make none. On that agent, a tool its card doesn't name is refused, and agent-config reads the +selection back. The validation agent itself is never limited, since the other checks need its tools. + +An agent whose agent-config doesn't take ``builtin_tools`` is recorded as not supporting it, which isn't a failure. + +Does NOT catch: a model that wouldn't have used a tool it has, which reads as honoured. +""" +from __future__ import annotations + +import logging +from typing import ClassVar, Optional + +import httpx + +from agent_env.a2a_agent import A2AAgent +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +logger = logging.getLogger(__name__) + +AGENT_NAME = "builtin-tools-agent" +NO_SUCH_TOOL = "agentenv-validator-no-such-tool" +NO_TOOLS = "NO-TOOLS" +PROMPT = ("Run the shell command `uname -a` with a tool you have and reply with its exact output. If you have no tool " + f"that can run a shell command, reply with exactly {NO_TOOLS} and nothing else.") + + +class VerifyA2ABuiltinToolsStep(TaskStep): + type: ClassVar[str] = "verify_a2a_builtin_tools" + entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) + + def __init__( + self, + id: str, + version: Optional[int], + a2a_agent_id: str, + a2a_agent_version: Optional[int] = None, + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + ): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.a2a_agent_id = a2a_agent_id + self.a2a_agent_version = a2a_agent_version + + def to_dict(self) -> dict: + base = super().to_dict() + base["a2a_agent_id"] = self.a2a_agent_id + base["a2a_agent_version"] = self.a2a_agent_version + return base + + @classmethod + def from_dict(cls, data: dict) -> VerifyA2ABuiltinToolsStep: + return cls( + **cls._base_from_dict(data), + a2a_agent_id=data["a2a_agent_id"], + a2a_agent_version=data.get("a2a_agent_version"), + ) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + main = next((a for a in context.deployed_agents), None) + if main is None: + raise RuntimeError("No deployed agent found in context") + params = (A2AAgent.find_extension(main.a2a_card or {}, A2AAgent.EXT_AGENT_CONFIG) or {}).get("params") or {} + methods = params.get("methods") or {} + if "builtin_tools" not in (methods.get("set") or {}).get("request", {}).get("supported", []): + self._record(context, {"supported": False, "reason": "agent-config takes no builtin_tools"}) + return context + + prompt_id = f"{self.id}-prompt" + try: + await DeployAgentTaskStep(id=f"{self.id}-deploy", version=None, a2a_agent_id=self.a2a_agent_id, + a2a_agent_version=self.a2a_agent_version, agent_name=AGENT_NAME, + builtin_tools={"disable": "*"}).execute(context) + limited = next(a for a in context.deployed_agents if a.agent_name == AGENT_NAME) + url = (limited.a2a_url or limited.api_url) + params.get("endpoint", "/ext/agent-config") + async with httpx.AsyncClient(timeout=30.0) as client: + refused = (await client.post(url, json={"builtin_tools": {"only": [NO_SUCH_TOOL]}})).status_code + config = (((await client.get(url)).json() or {}).get("config") or {}) if "get" in methods else None + await PromptAgentTaskStep(id=prompt_id, version=None, prompt=PROMPT, prompt_id=prompt_id, + agent_name=AGENT_NAME, timeout_seconds=180).execute(context) + except Exception as e: + self._record(context, {"supported": True, "passed": False, "reason": f"{type(e).__name__}: {e}"[:500]}) + return context + + response = next(r for r in reversed(context.prompt_responses) if r.prompt_id == prompt_id) + count = response.tool_call_count + checks = { + "unknown_tool_refused": refused == 400, + "read_back": None if config is None else config.get("builtin_tools") == {"disable": "*"}, + "no_tool_calls": count == 0 if count is not None else response.response.strip() == NO_TOOLS, + } + passed = all(check is not False for check in checks.values()) + if not passed: + logger.warning("builtin_tools not honoured by '%s': %s", self.a2a_agent_id, checks) + self._record(context, {"supported": True, "passed": passed, **checks, "tool_call_count": count, + "response": response.response[:500]}) + return context + + def _record(self, context: TaskStepContext, validated: dict) -> None: + # As the role check: the context's record first, then the agent's metadata, whose CAS can lose to the other + # validator steps running beside this one. + context.metadata.setdefault("verifications", {})["a2a_builtin_tools"] = validated + try: + fresh = A2AAgent.get(self.a2a_agent_id, self.a2a_agent_version) + fresh.update_metadata({**fresh.metadata, "validated_builtin_tools_via_agent_config": validated}) + except Exception as e: + logger.warning("Failed to persist validated_builtin_tools_via_agent_config (CAS conflict?): %s", e) diff --git a/tst/unit/task_step/test_verify_a2a_builtin_tools.py b/tst/unit/task_step/test_verify_a2a_builtin_tools.py new file mode 100644 index 000000000..78b5b5fc7 --- /dev/null +++ b/tst/unit/task_step/test_verify_a2a_builtin_tools.py @@ -0,0 +1,118 @@ +"""The validator's builtin_tools check: an agent whose agent-config doesn't take the field is recorded as not +supporting it and nothing is deployed; otherwise a second agent, deployed with {"disable": "*"} and no envs, must +refuse an unknown tool, read the selection back, and make no tool call when asked to run a shell command, or say it +has no tool when it reports no count.""" + +import json + +import httpx +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.a2a_agent_validator import verify_a2a_builtin_tools as module +from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import ( + AGENT_NAME, + NO_SUCH_TOOL, + VerifyA2ABuiltinToolsStep, +) + + +def _card(*supported: str) -> dict: + return {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", + "methods": {"set": {"request": {"supported": list(supported)}}, "get": {"method": "GET"}}}}]}} + + +class _Agent: + """The limited agent's agent-config: a selection it was deployed with, and the 400 an unknown tool gets.""" + + def __init__(self): + self.config, self.deploys = {}, [] + + def handle(self, request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response(200, json={"config": self.config}) + body = json.loads(request.content) + if NO_SUCH_TOOL in str(body): + return httpx.Response(400, json={"detail": "not one of the agent's tools"}) + self.config.update(body) + return httpx.Response(200, json={}) + + +@pytest.fixture +def wired(monkeypatch): + def wire(tool_call_count, reply="Linux box 6.8.0"): + agent = _Agent() + real_client = httpx.AsyncClient + monkeypatch.setattr(module.httpx, "AsyncClient", + lambda **kw: real_client(transport=httpx.MockTransport(agent.handle), **kw)) + monkeypatch.setattr(A2AAgent, "get", lambda *a, **kw: type("Stored", (), { + "metadata": {}, "update_metadata": lambda self, m: None})()) + + async def deploy(step, context): + agent.deploys.append(step.builtin_tools) + agent.config["builtin_tools"] = step.builtin_tools + context.deployed_agents.append(DeployedAgent(agent_name=step.agent_name, api_url="http://limited", + a2a_url="http://limited", a2a_card={})) + return context + + async def prompt(step, context): + context.prompt_responses.append(PromptResponse(prompt_id=step.prompt_id, response=reply, + tool_call_count=tool_call_count)) + return context + + monkeypatch.setattr(module.DeployAgentTaskStep, "execute", deploy) + monkeypatch.setattr(module.PromptAgentTaskStep, "execute", prompt) + return agent + + return wire + + +def _context(card: dict) -> TaskStepContext: + context = TaskStepContext(instance_id="ti-1") + context.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://main", a2a_url="http://main", + a2a_card=card)) + return context + + +async def _verify(card: dict) -> dict: + step = VerifyA2ABuiltinToolsStep(id="builtin-tools", version=None, a2a_agent_id="agent-1") + return (await step.execute(_context(card))).metadata["verifications"]["a2a_builtin_tools"] + + +@pytest.mark.asyncio +async def test_an_agent_whose_config_takes_no_builtin_tools_is_recorded_as_not_supporting_it(wired): + agent = wired(0) + + assert await _verify(_card("name", "role")) == {"supported": False, + "reason": "agent-config takes no builtin_tools"} + assert agent.deploys == [] + + +@pytest.mark.asyncio +async def test_an_agent_that_honours_it_passes(wired): + agent = wired(0) + + verified = await _verify(_card("name", "builtin_tools")) + + assert agent.deploys == [{"disable": "*"}] + assert {k: verified[k] for k in ("supported", "passed", "unknown_tool_refused", "read_back", "no_tool_calls")} == { + "supported": True, "passed": True, "unknown_tool_refused": True, "read_back": True, "no_tool_calls": True} + + +@pytest.mark.asyncio +async def test_an_agent_whose_limited_turn_calls_a_tool_fails(wired): + wired(2) + + verified = await _verify(_card("name", "builtin_tools")) + + assert (verified["passed"], verified["no_tool_calls"], verified["tool_call_count"]) == (False, False, 2) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reply, passed", [("NO-TOOLS", True), ("Linux box 6.8.0", False)]) +async def test_an_agent_that_reports_no_count_must_say_it_has_no_tool(wired, reply, passed): + wired(None, reply) + + assert (await _verify(_card("name", "builtin_tools")))["passed"] is passed From 5ea4765f761893afcbbf96e047faebb7b4820ef9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 21:00:05 -0800 Subject: [PATCH 6/7] refactor(agents): claude-code's tool selection typed end to end and held to the protocol and the pinned CLI - ClaudeCodeTool and ClaudeCodeTools move into claude_bridge, so the CLI command takes the typed selection instead of a dict a2a_server dumped and the bridge read back by key. - A test holds ClaudeCodeTools to the protocol's BuiltinTools[ClaudeCodeTool] schema, so the copy can't drift. - TOOLS_CLI_VERSION records the CLI the tool names were taken from; a test fails when the Dockerfile's CLAUDE_CODE_VERSION moves without it. - BuiltinTools loses enabled(), which nothing called. Also from the validator e2e on a VM: the builtin_tools check now reads a selection back without the fields an agent leaves unset (the protocol's readback returns them as null), and the episode rules count the new step among the agent validator's. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 4 +- .../agentenv_protocol/a2a_agent/tasks/v1.py | 6 --- .../agentenv-protocol/tests/test_a2a_agent.py | 4 -- src/agent_env/agents/README.md | 3 +- .../agents/agents/claude-code/a2a_server.py | 31 ++----------- .../agents/claude-code/claude_bridge.py | 45 +++++++++++++++---- src/agent_env/task/episode/rules.py | 7 +-- .../verify_a2a_builtin_tools.py | 7 ++- .../cli/run_bundle_claude_code_test.py | 4 +- .../test_claude_bridge_builtin_tools.py | 29 ++++++++++-- .../test_verify_a2a_builtin_tools.py | 4 +- 11 files changed, 84 insertions(+), 60 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 7999d9b78..a6ecae16b 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1010,8 +1010,8 @@ class MyAgentConfig(AgentConfig): An agent that can turn its own tools off, such as its shell, file or web tools, takes `builtin_tools`, typed `BuiltinTools` over a `Literal` of its tools' names, so its card's schema names them. A task selects them on `deploy_agent`: `{"disable": "*"}` turns them all off, `{"disable": [...]}` the ones named, and `{"only": [...]}` keeps -the ones named. `request.config.builtin_tools.enabled()` is what the selection leaves on, for the agent to -pass to its runtime. The MCP tools AgentEnv gives it are not its own and stay. A selection for an agent whose config has +the ones named; the agent applies `request.config.builtin_tools` to its runtime. The MCP tools AgentEnv gives it are not +its own and stay. A selection for an agent whose config has no `builtin_tools`, or that names a tool its schema doesn't, is refused before the deploy sends it anything. ```python diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index c65ac84ec..a2eb5d3b4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -146,12 +146,6 @@ def _one_of(self) -> BuiltinTools: raise ValueError("builtin_tools takes exactly one of disable and only") return self - def enabled(self, tools: Iterable[str]) -> list[str]: - """Those of ``tools`` the selection leaves on, in their order.""" - if self.only is not None: - return [tool for tool in tools if tool in self.only] - return [] if self.disable == "*" else [tool for tool in tools if tool not in self.disable] - ConfigT = TypeVar("ConfigT", bound=AgentConfig) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent.py b/packages/agentenv-protocol/tests/test_a2a_agent.py index 8e84a01ab..5e82d5501 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent.py @@ -1791,12 +1791,8 @@ async def run(self, request: TaskRequest[RuntimeConfig]) -> TaskResult: {"disable": "*", "only": ["Read"]}, {})] request = _extension(card, AGENT_CONFIG_V1.uri)["params"]["methods"]["set"]["request"] - tools = ["Bash", "Read", "WebFetch"] assert "builtin_tools" in request["supported"] and '"enum": ["Bash", "Read", "WebFetch"]' in json.dumps(request) assert statuses == [200, 200, 400, 400, 400] - assert [BuiltinTools[str].model_validate(selection).enabled(tools) - for selection in ({"disable": "*"}, {"disable": ["WebFetch"]}, {"only": ["WebFetch", "Read"]})] == [ - [], ["Bash", "Read"], ["Read", "WebFetch"]] def test_validation_errors_do_not_echo_rejected_values() -> None: diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index 31fd7ef23..b2c29aeca 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -25,7 +25,8 @@ run it. The CLI's version is the Dockerfile's selection picks the CLI's own tools: `{"disable": "*"}` and `{"only": [...]}` become its `--tools` list, a list to turn off becomes `--disallowedTools`, which keeps the CLI's defaults for the rest, and any selection adds `--strict-mcp-config`, so its MCP servers are only the task env's. The names are `ClaudeCodeTool`'s in -`a2a_server.py`: every tool the pinned CLI has, though which it offers by default depends on the model. +`claude_bridge.py`: every tool the pinned CLI has, though which it offers by default depends on the model. A test fails +when the Dockerfile's `CLAUDE_CODE_VERSION` moves, until `ClaudeCodeTool` and `TOOLS_CLI_VERSION` move with it. `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 37e532a45..7dc27625e 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -20,7 +20,7 @@ from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager from pathlib import Path -from typing import Any, Literal +from typing import Any from a2a.types import AgentSkill from agentenv_protocol.a2a_agent import ( @@ -58,7 +58,6 @@ extension, ) from agentenv_protocol.transfers import Uploaded, download, upload -from pydantic import BaseModel, ConfigDict, model_validator from starlette.applications import Starlette from starlette.exceptions import HTTPException @@ -74,6 +73,7 @@ ) from claude_bridge import ( CLAUDE_STATE_DIR, + ClaudeCodeTools, SESSION_JSONL_DIR, WORKSPACE_DIR, kill_running_clis, @@ -122,31 +122,6 @@ _INSTALL_COMMANDS_HOST = [_UNPACK, _ENDPOINT + f"bash {{work_dir}}/agent/install.sh host {{agent_name}} {{a2a_port}} {_HOST_WORKDIR}"] -# The CLI's own tools, by the names its model sees, in the pinned version. Which of them it offers by default depends -# on the model, and its MCP resource tools come only with MCP servers. -ClaudeCodeTool = Literal["Agent", "Bash", "CronCreate", "CronDelete", "CronList", "Edit", "EnterWorktree", - "ExitWorktree", "ListAgents", "ListMcpResourcesTool", "NotebookEdit", "Read", - "ReadMcpResourceDirTool", "ReadMcpResourceTool", "ReportFindings", "ScheduleWakeup", - "SendMessage", "Skill", "TaskCreate", "TaskGet", "TaskList", "TaskStop", "TaskUpdate", - "WebFetch", "WebSearch", "Workflow", "Write"] - - -class ClaudeCodeTools(BaseModel): - """The protocol's ``BuiltinTools[ClaudeCodeTool]``, which the image's protocol floor doesn't have yet: the same - fields and checks, so the card's schema is the same.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - disable: Literal["*"] | list[ClaudeCodeTool] | None = None - only: list[ClaudeCodeTool] | None = None - - @model_validator(mode="after") - def _one_of(self) -> ClaudeCodeTools: - if (self.disable is None) == (self.only is None): - raise ValueError("builtin_tools takes exactly one of disable and only") - return self - - class ClaudeCodeConfig(AgentConfig): timeout_seconds: int = 1800 model: str | None = None @@ -516,7 +491,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon effort=cfg.effort, max_turns=cfg.max_turns, max_thinking_tokens=cfg.max_thinking_tokens, - builtin_tools=None if cfg.builtin_tools is None else cfg.builtin_tools.model_dump(exclude_none=True), + builtin_tools=cfg.builtin_tools, event_callback=request.trajectory.append, ) except Exception as e: diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index beb7f6aa4..cecd8286f 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -16,6 +16,9 @@ from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, model_validator from kit import in_thread_to_its_end @@ -52,6 +55,33 @@ def project_dir(workspace: Path) -> Path: MODEL_HEADERS_ENV = "LITELLM_EXTRA_HEADERS" +# The CLI version ClaudeCodeTool names the tools of; a test holds it to the Dockerfile's CLAUDE_CODE_VERSION. +TOOLS_CLI_VERSION = "2.1.280" +# The CLI's own tools, by the names its model sees. Which of them it offers by default depends on the model, and its MCP +# resource tools come only with MCP servers. +ClaudeCodeTool = Literal["Agent", "Bash", "CronCreate", "CronDelete", "CronList", "Edit", "EnterWorktree", + "ExitWorktree", "ListAgents", "ListMcpResourcesTool", "NotebookEdit", "Read", + "ReadMcpResourceDirTool", "ReadMcpResourceTool", "ReportFindings", "ScheduleWakeup", + "SendMessage", "Skill", "TaskCreate", "TaskGet", "TaskList", "TaskStop", "TaskUpdate", + "WebFetch", "WebSearch", "Workflow", "Write"] + + +class ClaudeCodeTools(BaseModel): + """The protocol's ``BuiltinTools[ClaudeCodeTool]``, which the image's protocol floor doesn't have yet: the same + fields and checks, so the card's schema is the same.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + disable: Literal["*"] | list[ClaudeCodeTool] | None = None + only: list[ClaudeCodeTool] | None = None + + @model_validator(mode="after") + def _one_of(self) -> ClaudeCodeTools: + if (self.disable is None) == (self.only is None): + raise ValueError("builtin_tools takes exactly one of disable and only") + return self + + def session_jsonl_path(session_id: str) -> Path: return SESSION_JSONL_DIR / f"{session_id}.jsonl" @@ -383,7 +413,7 @@ async def run_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, - builtin_tools: dict | None = None, + builtin_tools: ClaudeCodeTools | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: """Run one turn, resuming the CLI session when a transient upstream failure cuts it short; a resume spends what @@ -574,7 +604,7 @@ def _with_1m_window(model: str) -> str: def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: list[str] | None, mcp_servers: dict[str, dict] | None, system_prompt: str | None, output_format: dict | None, role: str | None, effort: str | None, max_turns: int | None, - builtin_tools: dict | None = None) -> list[str]: + builtin_tools: ClaudeCodeTools | None = None) -> list[str]: """The CLI's argv. The prompt goes on stdin, since one argument over 128 KiB fails exec. ``builtin_tools`` selects the CLI's own tools: ``{"disable": "*"}`` offers none, ``{"only": [...]}`` just those, and ``{"disable": [...]}`` the CLI's defaults but those; with any selection, its MCP servers are only ``mcp_servers``.""" @@ -598,11 +628,10 @@ def _command(session_id: str, resume: bool, *, model: str | None, allowed_tools: if allowed_tools: cmd += ["--allowedTools", ",".join(allowed_tools)] if builtin_tools is not None: - only, disable = builtin_tools.get("only"), builtin_tools.get("disable") - if only is not None or disable == "*": - cmd += ["--tools", ",".join(only or [])] - elif disable: - cmd += ["--disallowedTools", ",".join(disable)] + if builtin_tools.only is not None or builtin_tools.disable == "*": + cmd += ["--tools", ",".join(builtin_tools.only or [])] + elif builtin_tools.disable: + cmd += ["--disallowedTools", ",".join(builtin_tools.disable)] cmd.append("--strict-mcp-config") if mcp_servers: def entry(info: dict) -> dict: @@ -633,7 +662,7 @@ async def _spawn_claude( effort: str | None = None, max_turns: int | None = None, max_thinking_tokens: int | None = None, - builtin_tools: dict | None = None, + builtin_tools: ClaudeCodeTools | None = None, event_callback: Callable[[dict], None] | None = None, ) -> ClaudeResult: cmd = _command(session_id, resume, model=model, allowed_tools=allowed_tools, mcp_servers=mcp_servers, diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 86d84c921..d7409bb28 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -139,9 +139,10 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "peer_agents": _peer_agents, "rubrics_verifier": _rubrics_verifier, **dict.fromkeys(( - "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_core_protocol", - "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", "verify_a2a_role", - "verify_a2a_skill_config", "verify_a2a_snapshot", "verify_a2a_system_prompt", "verify_a2a_trajectory", + "verify_a2a_agent_card", "verify_a2a_agent_config_identity", "verify_a2a_agent_mcp", "verify_a2a_builtin_tools", + "verify_a2a_core_protocol", "verify_a2a_install", "verify_a2a_modalities", "verify_a2a_peer_agents", + "verify_a2a_role", "verify_a2a_skill_config", "verify_a2a_snapshot", "verify_a2a_system_prompt", + "verify_a2a_trajectory", ), _validates_an_agent), } diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py index ac540e02b..b3aa1d457 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py @@ -91,7 +91,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: count = response.tool_call_count checks = { "unknown_tool_refused": refused == 400, - "read_back": None if config is None else config.get("builtin_tools") == {"disable": "*"}, + "read_back": None if config is None else _set(config.get("builtin_tools")) == {"disable": "*"}, "no_tool_calls": count == 0 if count is not None else response.response.strip() == NO_TOOLS, } passed = all(check is not False for check in checks.values()) @@ -110,3 +110,8 @@ def _record(self, context: TaskStepContext, validated: dict) -> None: fresh.update_metadata({**fresh.metadata, "validated_builtin_tools_via_agent_config": validated}) except Exception as e: logger.warning("Failed to persist validated_builtin_tools_via_agent_config (CAS conflict?): %s", e) + + +def _set(selection: dict | None) -> dict: + """A selection as read back, without the fields it leaves unset, which an agent may return as null.""" + return {key: value for key, value in (selection or {}).items() if value is not None} diff --git a/tst/integration/cli/run_bundle_claude_code_test.py b/tst/integration/cli/run_bundle_claude_code_test.py index 5b202ba7d..c4f59b81e 100644 --- a/tst/integration/cli/run_bundle_claude_code_test.py +++ b/tst/integration/cli/run_bundle_claude_code_test.py @@ -27,7 +27,7 @@ from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from tst.unit.agents.claude_code.agent import a2a_server +from tst.unit.agents.claude_code.agent import claude_bridge from tst.util.fake_anthropic import FakeAnthropic, answer, delegate_then_answer, tool_then_answer pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -190,7 +190,7 @@ def run(name, selection): defaults = run("tools", None) offered = defaults if builtin_tools is None else run("selected", builtin_tools) - assert defaults <= set(get_args(a2a_server.ClaudeCodeTool)), sorted(defaults) + assert defaults <= set(get_args(claude_bridge.ClaudeCodeTool)), sorted(defaults) assert offered == kept(defaults), sorted(offered) assert all("ping-from-tool" in result for result in endpoint.tool_results()) diff --git a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py index 61bf136e3..b5d324052 100644 --- a/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py +++ b/tst/unit/agents/claude_code/test_claude_bridge_builtin_tools.py @@ -1,18 +1,22 @@ """A selection of the CLI's own tools: ``"*"`` and ``only`` name the tools the CLI may offer, a list to turn off leaves the CLI's defaults but those, and with any selection its MCP servers are only agent-env's; with none, the CLI keeps -its own. A tool the CLI doesn't have is refused.""" +its own. A tool the CLI doesn't have is refused. The selection's type has the protocol's schema, and its tool names +are those of the CLI version the Dockerfile pins.""" + +import re +from typing import Any import pytest +from agentenv_protocol.a2a_agent import BuiltinTools -from tst.unit.agents.claude_code.agent import a2a_server, claude_bridge +from tst.unit.agents.claude_code.agent import AGENT_DIR, a2a_server, claude_bridge def _command(selection: dict | None) -> list[str]: - config = a2a_server.ClaudeCodeConfig(builtin_tools=selection) return claude_bridge._command( "sess-1", False, model=None, allowed_tools=None, mcp_servers=None, system_prompt=None, output_format=None, role=None, effort=None, max_turns=None, - builtin_tools=None if config.builtin_tools is None else config.builtin_tools.model_dump(exclude_none=True)) + builtin_tools=a2a_server.ClaudeCodeConfig(builtin_tools=selection).builtin_tools) def test_no_selection_leaves_the_clis_tools_and_mcp_servers_alone(): @@ -38,3 +42,20 @@ def test_a_selection_is_the_clis_tool_flags_with_only_agent_envs_mcp_servers(sel def test_a_tool_the_cli_does_not_have_is_refused(): with pytest.raises(ValueError, match="Glob"): a2a_server.ClaudeCodeConfig(builtin_tools={"only": ["Glob"]}) + + +def test_the_selection_has_the_protocols_schema(): + def untitled(schema: Any) -> Any: + if isinstance(schema, dict): + return {k: untitled(v) for k, v in schema.items() if k not in ("title", "description")} + return [untitled(v) for v in schema] if isinstance(schema, list) else schema + + assert untitled(claude_bridge.ClaudeCodeTools.model_json_schema()) == untitled( + BuiltinTools[claude_bridge.ClaudeCodeTool].model_json_schema()) + + +def test_the_tool_names_are_those_of_the_cli_version_the_dockerfile_pins(): + pinned = re.search(r"^ARG CLAUDE_CODE_VERSION=(\S+)$", (AGENT_DIR / "Dockerfile").read_text(), re.M).group(1) + + assert claude_bridge.TOOLS_CLI_VERSION == pinned, ( + "the CLI version changed: record the tools the new CLI offers in ClaudeCodeTool, then TOOLS_CLI_VERSION") diff --git a/tst/unit/task_step/test_verify_a2a_builtin_tools.py b/tst/unit/task_step/test_verify_a2a_builtin_tools.py index 78b5b5fc7..8886af245 100644 --- a/tst/unit/task_step/test_verify_a2a_builtin_tools.py +++ b/tst/unit/task_step/test_verify_a2a_builtin_tools.py @@ -32,7 +32,9 @@ def __init__(self): def handle(self, request: httpx.Request) -> httpx.Response: if request.method == "GET": - return httpx.Response(200, json={"config": self.config}) + # As the protocol's readback writes a selection: with the field it leaves unset as null. + return httpx.Response(200, json={"config": {**self.config, "builtin_tools": { + "disable": None, "only": None, **self.config.get("builtin_tools", {})}}}) body = json.loads(request.content) if NO_SUCH_TOOL in str(body): return httpx.Response(400, json={"detail": "not one of the agent's tools"}) From 4cda7059acbdb30b3f410a6d3f99fb4c11b29ebd Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 1 Nov 2026 21:16:24 -0800 Subject: [PATCH 7/7] test: drop an unused import from the builtin_tools validator test Co-Authored-By: Claude Opus 5.5 --- tst/unit/task_step/test_verify_a2a_builtin_tools.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tst/unit/task_step/test_verify_a2a_builtin_tools.py b/tst/unit/task_step/test_verify_a2a_builtin_tools.py index 8886af245..de3c59987 100644 --- a/tst/unit/task_step/test_verify_a2a_builtin_tools.py +++ b/tst/unit/task_step/test_verify_a2a_builtin_tools.py @@ -11,11 +11,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext from agent_env.task_step.task_steps.a2a_agent_validator import verify_a2a_builtin_tools as module -from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import ( - AGENT_NAME, - NO_SUCH_TOOL, - VerifyA2ABuiltinToolsStep, -) +from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_builtin_tools import NO_SUCH_TOOL, VerifyA2ABuiltinToolsStep def _card(*supported: str) -> dict: