From 4fe84634748388f42a7e7903380bea92875b6c00 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:09:45 -0700 Subject: [PATCH 01/13] feat(computer): cua-driver as a computer driver trycua's cua-driver 0.38.0, from its PyPI wheel, runs on the computer as a daemon, and agents get its own 66 tools beside the computer_* tools, forwarded on each caller's connection. The computer/v1 operations are its desktop-scope calls in a session of the framework's own, computer-ready checks the pinned executable and tools, and the skill leads with cua-driver's own skill pack. The protocol SDK gains forwarded tools, served with their own schemas, and skills with files beside their SKILL.md, which add_skills sends as a stored bundle. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/__init__.py | 2 + .../agent_env_environment.py | 82 +- .../src/agentenv_protocol/computer/driver.py | 5 + .../agentenv_protocol/computer/extensions.py | 2 +- .../src/agentenv_protocol/computer/skill.py | 17 +- .../tests/test_agent_env_environment.py | 13 + .../tests/test_computer_driver.py | 15 + .../tests/test_tool_access.py | 28 +- .../tests/test_tool_annotations.py | 58 +- src/agent_env/drivers/README.md | 13 + src/agent_env/drivers/aliases.toml | 1 + .../drivers/envs/cua-driver/daemon.sh | 8 + .../drivers/envs/cua-driver/driver.py | 248 ++ .../drivers/envs/cua-driver/env.toml | 10 + .../drivers/envs/cua-driver/requirements.txt | 5 + .../drivers/envs/cua-driver/tools.json | 3832 +++++++++++++++++ .../task_step/task_steps/add_skills.py | 22 +- .../computer/cua-driver/envs/desktop/env.toml | 3 + tst/unit/bundle/shipped_bundles_test.py | 5 +- tst/unit/drivers/cua_driver_test.py | 254 ++ tst/unit/task_step/add_skills_env_test.py | 22 +- 21 files changed, 4624 insertions(+), 21 deletions(-) create mode 100644 src/agent_env/drivers/envs/cua-driver/daemon.sh create mode 100644 src/agent_env/drivers/envs/cua-driver/driver.py create mode 100644 src/agent_env/drivers/envs/cua-driver/env.toml create mode 100644 src/agent_env/drivers/envs/cua-driver/requirements.txt create mode 100644 src/agent_env/drivers/envs/cua-driver/tools.json create mode 100644 tst/integration/computer/cua-driver/envs/desktop/env.toml create mode 100644 tst/unit/drivers/cua_driver_test.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index a22d66607..0b9f68f17 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -11,6 +11,7 @@ AgentEnvEnvironment, AgentEnvFastMCPApplication, AgentEnvStarletteApplication, + ForwardedTool, add_data, agent_skill, create_fastmcp_app, @@ -88,6 +89,7 @@ "AgentEnvEnvironment", "AgentEnvFastMCPApplication", "AgentEnvStarletteApplication", + "ForwardedTool", "create_fastmcp_app", "environment_card", "reset_data", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index b8b522a5b..119b93fae 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -38,6 +38,7 @@ _TOOL_ATTR = "_aee_tool" _HIDDEN_BY_DEFAULT_ATTR = "_aee_hidden_by_default" _TOOL_OPTIONS_ATTR = "_aee_tool_options" +_FORWARDED_ATTR = "_aee_forwarded" _CARD_CONFIG_ATTR = "_aee_card_config" #: Set on the tool dispatch installed by ``_bind_tool_calls``; a guard downstream checks it at boot. _TOOL_CONTEXT_BOUND_ATTR = "__agentenv_tool_context__" @@ -129,20 +130,23 @@ def deco(fn: Callable) -> Callable: def agent_skill(fn: Callable) -> Callable: - """Mark the method that writes this env's skill for agents, returning its description and its markdown body; it - is served under ``SKILL_EXTENSION_URI`` as ``get {name}``, which writes the SKILL.md under the name asked for.""" + """Mark the method that writes this env's skill for agents, returning its description, its markdown body and, + optionally, its other files by relative path; it is served under ``SKILL_EXTENSION_URI`` as ``get {name}``, which + writes the SKILL.md under the name asked for and returns the files beside it.""" async def get(self, name: str) -> dict: if len(name) > 64 or not _SKILL_NAME.fullmatch(name): raise AgentEnvError("invalid_argument", f"a skill's name is lowercase letters, digits and single hyphens, up to 64, not {name!r}") - description, body = await fn(self) if inspect.iscoroutinefunction(fn) else fn(self) + description, body, *files = await fn(self) if inspect.iscoroutinefunction(fn) else fn(self) frontmatter = f"---\nname: {name}\ndescription: {json.dumps(description)}\n---\n" - return {"name": name, "description": description, "skill_md": f"{frontmatter}\n{body}"} + return {"name": name, "description": description, "skill_md": f"{frontmatter}\n{body}", + **({"files": files[0]} if files and files[0] else {})} get.__name__ = fn.__name__ return extension(SKILL_EXTENSION_URI, op="get", method="GET", - description="get returns a SKILL.md for agents under the name it's given.")(get) + description="get returns a SKILL.md for agents under the name it's given, and the skill's other " + "files when it has any.")(get) def _extension_slug(uri: str) -> str: @@ -202,6 +206,16 @@ def deco(fn: Callable) -> Callable: return deco +class ForwardedTool(NamedTuple): + """A tool an environment serves on another server's behalf, as that server lists it: its descriptor, its own + schema included, and its MCP ``annotations``; ``call`` takes the call's arguments as keywords and returns the + other server's ``CallToolResult``, which reaches the caller as it is.""" + + tool: EnvironmentTool + call: Callable[..., Any] + annotations: dict | None = None + + class _ToolOptions(NamedTuple): annotations: dict | None structured_output: bool | None @@ -239,7 +253,7 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: self._app: Any = None ext_handlers = _served_extensions(handler, _discover_extensions(handler)) environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers)) - tool_handlers = _served_tools(handler, _discover_tools(handler, environment_card.name)) + tool_handlers = _served_tools(handler, _with_forwarded(_discover_tools(handler, environment_card.name), handler)) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") environment_card = _merge_tools(environment_card, tool_handlers) @@ -329,6 +343,8 @@ def _add_route(self, app: Any, path: str, methods: list, handler: Callable) -> N app.custom_route(path, methods=methods)(handler) def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> None: + if (forwarded := getattr(fn, _FORWARDED_ATTR, None)) is not None: + return self._register_forwarded(app, descriptor, fn, forwarded) # passed only when set, so a FastMCP-shaped target without the parameters still mounts plain tools options = getattr(fn, _TOOL_OPTIONS_ATTR, None) or _ToolOptions(None, None, None) fastmcp = self._binding == "middleware" @@ -348,6 +364,21 @@ def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> extra["output_schema"] = None if options.structured_output is False else schema app.tool(name=descriptor.name, description=descriptor.description, **extra)(registered) + def _register_forwarded(self, app: Any, descriptor: EnvironmentTool, fn: Callable, forwarded: ForwardedTool) -> None: + """Put a forwarded tool into mcp's FastMCP tool manager as it is, with its own schema and its arguments passed + through untouched, since FastMCP's ``tool`` builds a schema from a signature.""" + from mcp.server.fastmcp.tools.base import Tool + from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata + + if self._binding != "dispatch": + raise RuntimeError(f"the forwarded tool {descriptor.name!r} needs mcp's FastMCP, whose tool manager takes a " + "tool with its own schema") + app._tool_manager._tools[descriptor.name] = Tool( + fn=fn, name=descriptor.name, description=descriptor.description or "", + parameters=descriptor.inputSchema or {"type": "object"}, fn_metadata=FuncMetadata(arg_model=_passed_through()), + is_async=True, + annotations=_mcp_tool_annotations(forwarded.annotations) if forwarded.annotations is not None else None) + def _check_tool_binding(self, app: Any) -> None: """Decide, before any route or tool is added, how every tools/call on ``app`` gets its ToolContext, and refuse what cannot be honoured. The standalone ``fastmcp`` package (a ``middleware`` list with @@ -564,6 +595,11 @@ def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]: """The ``@tool`` methods to serve, chosen at mount; a returned descriptor's description replaces the marked one.""" return tools + def forwarded_tools(self) -> list[ForwardedTool]: + """Tools served on another server's behalf, each with its own schema, beside the ``@tool`` methods: listed on + the card, chosen by ``served_tools`` and under tool access like them. Needs mcp's FastMCP.""" + return [] + def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -1080,6 +1116,40 @@ def _served_operations(handler: Any, methods: dict) -> dict: return {name: entry for name, entry in methods.items() if name in chosen} +def _with_forwarded(tool_handlers: list, handler: Any) -> list: + """The ``@tool`` handlers and then the handler's forwarded tools, each as (descriptor, call), the call marked so it + registers with its own schema; a forwarded tool named like another tool is refused.""" + hook = getattr(handler, "forwarded_tools", None) + forwarded = [(entry.tool, _forwarding(entry)) for entry in (hook() if hook is not None else [])] + names = [descriptor.name for descriptor, _fn in tool_handlers + forwarded] + if len(set(names)) != len(names): + raise ValueError(f"forwarded tools named like another tool: {sorted({n for n in names if names.count(n) > 1})}") + return tool_handlers + forwarded + + +def _forwarding(forwarded: ForwardedTool) -> Callable: + async def call(**arguments: Any) -> Any: + return await forwarded.call(**arguments) + + setattr(call, _FORWARDED_ATTR, forwarded) + return call + + +@functools.cache +def _passed_through() -> type: + """mcp's FastMCP argument model for a forwarded tool: any arguments, each passed to the call as it came.""" + from mcp.server.fastmcp.utilities.func_metadata import ArgModelBase + from pydantic import ConfigDict + + class PassedThrough(ArgModelBase): + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) + + def model_dump_one_level(self) -> dict[str, Any]: + return dict(self.__pydantic_extra__ or {}) + + return PassedThrough + + def _served_tools(handler: Any, tool_handlers: list) -> list: """The tools the handler's ``served_tools`` keeps, in its order and with its descriptors; all of them without one.""" hook = getattr(handler, "served_tools", None) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 172b57703..068c22517 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -111,6 +111,11 @@ async def ready_checks(self) -> list[ReadyCheck]: """Checks of the driver's own, run with the base class's by computer-ready/v1; never sends input.""" return [] + async def bundled_skill(self) -> tuple[str, str, dict[str, str]] | None: + """The skill the software this driver drives ships for agents, as its description, its SKILL.md body without + frontmatter, and its other files by relative path; the computer skill leads with it. None by default.""" + return None + def create_app(self) -> Any: self._computer_require_init() return super().create_app() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 52a522112..ff12c3b36 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -38,7 +38,7 @@ class Extensions: mounted_card: Callable[[], EnvironmentCard] @agent_skill - async def _computer_skill(self) -> tuple[str, str]: + async def _computer_skill(self) -> tuple[str, str, dict[str, str]]: return await skill(self) @extension(COMPUTER_READY, op="status", method="GET", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py index 5c31f55cb..416499ca6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py @@ -1,6 +1,7 @@ """The computer skill: a SKILL.md an agent reads before it uses the computer, written from the computer as it booted: -its platform, its session user, its screen, its OS's shortcuts, what its card declares, and the driver's notes. It -says nothing about any task.""" +its platform, its session user, its screen, its OS's shortcuts, what its card declares, and the driver's notes. A +driver whose software ships a skill of its own (``bundled_skill``) leads with it, its files beside the SKILL.md, and +the computer's part follows. It says nothing about any task.""" from __future__ import annotations import getpass @@ -24,8 +25,8 @@ _SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"} -async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str]: - """The skill's description and its markdown body, as ``agent_skill`` takes them.""" +async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, str]]: + """The skill's description, its markdown body and its other files, as ``agent_skill`` takes them.""" display = await driver._computer.display_now() platform = driver.platform system = " ".join(part for part in ((platform.distribution or platform.os or "").title(), platform.version, @@ -49,7 +50,13 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str]: sections += ["## Shortcuts", shortcuts.rstrip()] if notes := driver.notes.strip(): sections += ["## Notes", notes] - return description, "\n\n".join(sections) + "\n" + if (bundled := await driver.bundled_skill()) is None: + return description, "\n\n".join(sections) + "\n", {} + own_description, own_body, files = bundled + sections[:2] = ["# This computer", "Beside the tools above, this computer serves its `computer_*` tools: " + "`computer_screenshot` shows the screen, and the others click, type and press keys on what it shows."] + return (f"{own_description} This computer: {system}, {display.width}x{display.height}, and its computer_* tools.", + own_body.rstrip() + "\n\n" + "\n\n".join(sections) + "\n", files) def _input(driver: ComputerDriverEnvironment) -> str: diff --git a/packages/agentenv-protocol/tests/test_agent_env_environment.py b/packages/agentenv-protocol/tests/test_agent_env_environment.py index 8c26a270b..7b8897e46 100644 --- a/packages/agentenv-protocol/tests/test_agent_env_environment.py +++ b/packages/agentenv-protocol/tests/test_agent_env_environment.py @@ -184,6 +184,19 @@ async def test_an_env_serves_its_skill_under_the_name_its_asked_for_and_refuses_ assert refused.status_code == 400 and "lowercase letters, digits and single hyphens" in refused.text +class _SkillWithFilesHandler: + @agent_skill + async def _guide(self): + return "Drive the desktop.", "# Desktop\n\nSee LINUX.md.\n", {"LINUX.md": "# Linux\n"} + + +@pytest.mark.asyncio +async def test_a_skill_with_other_files_serves_them_beside_its_skill_md(): + async with _client(_SkillWithFilesHandler) as client: + answer = (await client.get("/agentenv/ext/skill/get", params={"name": "desktop"})).json() + assert answer["files"] == {"LINUX.md": "# Linux\n"} and answer["skill_md"].endswith("# Desktop\n\nSee LINUX.md.\n") + + @pytest.mark.asyncio async def test_unknown_method_returns_error(): async with _client(_ItemsHandler) as client: diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index c6ceec511..9c6e0874e 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -201,6 +201,21 @@ async def test_the_skill_is_written_from_the_computer_as_it_booted_under_the_nam assert refused.status_code == 400 and "lowercase letters, digits and single hyphens" in refused.text +@pytest.mark.asyncio +async def test_a_driver_whose_software_ships_a_skill_leads_with_it_and_the_computers_part_follows(monkeypatch): + class Bundling(FakeDriver): + async def bundled_skill(self): + return "Drive any app through its own tools.", "# Desktop driver\n\nRead LINUX.md first.\n", { + "LINUX.md": "# Linux\n"} + + async with http_client(Bundling()) as http: + answer = (await http.get("/agentenv/ext/skill/get", params={"name": "desktop"})).json() + body = answer["skill_md"].partition("---\n\n")[2] + assert body.startswith("# Desktop driver\n\nRead LINUX.md first.\n\n# This computer\n\nBeside the tools above") + assert "## Input" in body and "# Using this computer" not in body and answer["files"] == {"LINUX.md": "# Linux\n"} + assert answer["description"].startswith("Drive any app through its own tools. This computer: Ubuntu 24.04 (amd64)") + + @pytest.mark.asyncio async def test_a_minimal_driver_declares_only_what_it_has(): async with http_client(MinimalDriver(platform=MACOS)) as http: diff --git a/packages/agentenv-protocol/tests/test_tool_access.py b/packages/agentenv-protocol/tests/test_tool_access.py index ac04104a2..c2d520e27 100644 --- a/packages/agentenv-protocol/tests/test_tool_access.py +++ b/packages/agentenv-protocol/tests/test_tool_access.py @@ -14,6 +14,7 @@ from mcp import ClientSession # noqa: E402 from mcp.client import streamable_http as _streamable_http # noqa: E402 from mcp.server.fastmcp import FastMCP # noqa: E402 +from mcp.types import CallToolResult, TextContent # noqa: E402 from agentenv_protocol import ( # noqa: E402 ROLE_HEADER, @@ -21,6 +22,8 @@ AgentEnvEnvironment, AgentEnvFastMCPApplication, EnvironmentCard, + EnvironmentTool, + ForwardedTool, environment_card, tool, ) @@ -50,10 +53,10 @@ def grade(self) -> str: @contextlib.asynccontextmanager -async def _served(): - """The env on mcp's FastMCP over streamable HTTP, and a way to open a session as a role.""" +async def _served(env: AgentEnvEnvironment | None = None): + """``env``, by default an ``_Env``, on mcp's FastMCP over streamable HTTP, and a way to open a session as a role.""" app = FastMCP("desk", json_response=True) - _Env().mount(app) + (env or _Env()).mount(app) asgi = app.streamable_http_app() @contextlib.asynccontextmanager @@ -118,6 +121,25 @@ async def test_roles_see_and_call_only_what_their_rules_allow(): assert grader_tools == ["grade", "read", "write"] and default_tools == ["read", "write"] +@pytest.mark.asyncio +async def test_a_forwarded_tool_is_listed_and_refused_by_role_like_any_tool(): + class Forwarding(_Env): + def forwarded_tools(self): + async def click(**arguments): + return CallToolResult(content=[TextContent(type="text", text="clicked")]) + return [ForwardedTool(EnvironmentTool(name="click", inputSchema={"type": "object"}), click)] + + with anyio.fail_after(30): + async with _served(Forwarding()) as (rest, session): + await rest.post("/tools/disable", json={"role": "agent", "tools": ["click"]}) + async with session() as anyone: + listed, called = await _names(anyone), await anyone.call_tool("click", {}) + async with session("agent") as agent: + agent_tools, refused = await _names(agent), await agent.call_tool("click", {}) + assert listed == ["click", "read", "write"] and called.content[0].text == "clicked" + assert agent_tools == ["read", "write"] and refused.isError + + @pytest.mark.asyncio async def test_everything_off_then_one_tool_on_for_one_role(): with anyio.fail_after(30): diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index feeb1c58e..fbbdee94d 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -1,4 +1,4 @@ -"""@tool annotations, structured output and _meta on both FastMCPs, served_tools choosing what a handler serves, and\nupdate_tool changing a served description.""" +"""@tool annotations, structured output and _meta on both FastMCPs, served_tools choosing what a handler serves,\nupdate_tool changing a served description, and forwarded tools served with their own schemas.""" from __future__ import annotations from typing import Annotated @@ -20,6 +20,7 @@ AgentEnvFastMCPApplication, EnvironmentCard, EnvironmentTool, + ForwardedTool, environment_card, tool, ) @@ -209,3 +210,58 @@ async def test_update_tool_needs_a_mounted_env_and_a_served_tool(): env.create_app() with pytest.raises(ValueError, match="no served tool is named 'secret'"): await env.update_tool("secret", description="x") + + +_CLICK = {"type": "object", "additionalProperties": False, "properties": { + "element_token": {"type": "string"}, "x": {"type": "integer"}, "count": {"type": "integer", "minimum": 1}}} + + +@environment_card(name="forwarding") +class _Forwarding(AgentEnvEnvironment): + def __init__(self, calls: list) -> None: + self.calls = calls + + @tool() + def look(self) -> str: + return "looked" + + def forwarded_tools(self) -> list[ForwardedTool]: + async def click(**arguments) -> CallToolResult: + self.calls.append(arguments) + return CallToolResult(content=[TextContent(type="text", text="clicked"), + ImageContent(type="image", data="aGk=", mimeType="image/png")], + structuredContent={"hit": "button OK"}) + + return [ForwardedTool(EnvironmentTool(name="click", description="Click an element.", inputSchema=_CLICK), click, + {"readOnlyHint": False, "destructiveHint": False})] + + +@pytest.mark.asyncio +async def test_a_forwarded_tool_is_listed_with_its_own_schema_and_called_with_its_arguments_as_they_came(): + calls = [] + env = _Forwarding(calls) + with anyio.fail_after(30): + async with create_connected_server_and_client_session(env.create_app()) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + result = await session.call_tool("click", {"element_token": "s1:4", "count": 2}) + + assert listed["click"].inputSchema == _CLICK and listed["click"].description == "Click an element." + assert listed["click"].annotations.readOnlyHint is False and listed["click"].outputSchema is None + assert calls == [{"element_token": "s1:4", "count": 2}] + assert [c.type for c in result.content] == ["text", "image"] and result.structuredContent == {"hit": "button OK"} + assert {t.name: t.inputSchema for t in env.mounted_card().capabilities.tools}["click"] == _CLICK + + +def test_a_forwarded_tool_named_like_a_tool_is_refused(): + class Clashing(_Forwarding): + @tool() + def click(self) -> str: + return "mine" + + with pytest.raises(ValueError, match="forwarded tools named like another tool: \\['click'\\]"): + Clashing([]).create_app() + + +def test_a_forwarded_tool_needs_mcps_fastmcp(): + with pytest.raises(RuntimeError, match="the forwarded tool 'click' needs mcp's FastMCP"): + _Forwarding([]).mount(fastmcp.FastMCP("forwarding")) diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 349b45d22..ec7b58e00 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -14,5 +14,18 @@ there, and run `agentenv_protocol.computer.serve driver.py:ReferenceDriver` as a the shipped `desktop` example's driver. `python -m agentenv_protocol.computer.conformance ` checks it, or any driver, where it runs. +`envs/cua-driver/` is trycua's [cua-driver](https://github.com/trycua/cua/tree/main/libs/cua-driver) as a driver, +for the same X11 desktop on Linux (`amd64` and `arm64`). cua-driver 0.38.0 runs on the computer as a daemon, +`cua-driver serve`, started by `daemon.sh` once X answers, with approvals off and no telemetry. Agents get its own 66 +tools beside the `computer_*` tools, as `tools.json` lists them: element tokens from `get_window_state`, clicks and +typing by element or pixel, `run_script` and its browser tools, each call forwarded on its caller's own connection to the +daemon. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on its desktop-scope calls (XTest), +in a session of their own, so they never make an agent's element tokens stale. It has no pointer or keyboard hold group, +drags from a start to an end, and scrolls with no keys held. Its skill leads with cua-driver's own skill pack, as the +executable embeds it, its other files beside the SKILL.md. An env narrows which tools its agent sees with `[tools]`. +`requirements.txt` installs `cua-driver==0.38.0` from PyPI, whose wheel carries the executable, and `computer-ready` +checks that executable's sha256 and that the daemon lists the tools `tools.json` pins. Name it as `cua-driver`, the +alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/cua-driver`. + This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index 7d5edd691..43b2f3510 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -1,3 +1,4 @@ # Ids a task, or --driver, can name these drivers by; a store's env of the id wins over an alias. [envs] reference = ["reference"] +cua-driver = ["cua-driver"] diff --git a/src/agent_env/drivers/envs/cua-driver/daemon.sh b/src/agent_env/drivers/envs/cua-driver/daemon.sh new file mode 100644 index 000000000..11f478c23 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/daemon.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# cua-driver's daemon, once X answers: on the socket the driver reaches it at, with approvals off, since nobody is at +# the computer to give them, and nothing sent to trycua. +set -eu +export DO_NOT_TRACK=1 CUA_DRIVER_RS_TELEMETRY_ENABLED=false CUA_DRIVER_RS_UPDATE_CHECK=false +display="${DISPLAY#:}" +while [ ! -S "/tmp/.X11-unix/X${display%%.*}" ]; do sleep 0.2; done +exec venv/bin/cua-driver serve --socket /run/agentenv/cua-driver.sock --dangerously-bypass-approvals --no-overlay diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py new file mode 100644 index 000000000..3f9519a27 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -0,0 +1,248 @@ +"""cua-driver as a computer driver, for an X11 desktop on Linux: trycua's cua-driver runs on the computer as a daemon +(``daemon.sh``), agents get its own tools as its pinned version lists them (``tools.json``), beside the ``computer_*`` +tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes line-delimited JSON on a Unix +socket; the framework's calls and each caller's run on connections of their own, the framework's in a session of its +own. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" +from __future__ import annotations + +import asyncio +import base64 +import functools +import hashlib +import importlib.util +import json +import os +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +from mcp.types import CallToolResult, TextContent + +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + DeliveryFailed, + DeviceUnavailable, + Display, + Image, + ReadyCheck, + Unsupported, + capture, + click, + computer_driver, + display, + drag, + press, + scroll, + type_text, +) + +#: Where ``daemon.sh`` serves cua-driver's daemon. +SOCKET = Path("/run/agentenv/cua-driver.sock") +#: cua-driver's tools as its pinned version lists them, which the running daemon must list the same. +MANIFEST = Path(__file__).with_name("tools.json") +#: The sha256 of the cua-driver executable each arch's pinned wheel installs. +EXECUTABLES = { + "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", + "arm64": "26e9eee8165e9ce7b614b5e3d68e5d963231385a82b65236406718b5ce1ff936", +} +#: The session the framework's own calls run in, apart from every caller's. +FRAMEWORK_SESSION = "agentenv" +#: W3C modifier keys and the names cua-driver holds them by. +MODIFIERS = {"Control": "ctrl", "Shift": "shift", "Alt": "alt", "Meta": "super", "Super": "super"} +#: W3C key values and the names cua-driver presses them by; any printable ASCII character is pressed as itself. +NAMED = { + "Enter": "enter", "Tab": "tab", "Escape": "escape", " ": "space", "Backspace": "backspace", "Delete": "delete", + "Insert": "insert", "Home": "home", "End": "end", "PageUp": "pageup", "PageDown": "pagedown", "ArrowUp": "up", + "ArrowDown": "down", "ArrowLeft": "left", "ArrowRight": "right", **{f"F{n}": f"f{n}" for n in range(1, 13)}, +} +PRINTABLE = tuple(chr(code) for code in range(0x21, 0x7F)) +#: How long the skill pack's writing may take. +SKILL_TIMEOUT_S = 60 +#: The largest reply the daemon sends: a screenshot of a large display, base64. +REPLY_LIMIT = 64 << 20 + + +@computer_driver(name="cua-driver", + supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class CuaDriver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle") + max_click_count = 3 + max_scroll_amount = 50 + keys = (*MODIFIERS, *NAMED, *PRINTABLE) + + def __init__(self) -> None: + super().__init__() + self._framework = _Connection() + self._callers: dict[str, _Connection] = {} + self._pinned: list[ReadyCheck] | None = None + self._skill: tuple[str, str, dict[str, str]] | None = None + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]), + functools.partial(self._forward, t["name"]), t["annotations"] or None) + for t in _manifest()] + + async def ready_checks(self) -> list[ReadyCheck]: + """That the daemon answers as the pinned cua-driver does: its executable's sha256 and its tools, checked + until they pass once.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return self._pinned + + async def bundled_skill(self) -> tuple[str, str, dict[str, str]]: + if self._skill is None: + self._skill = await asyncio.to_thread(_skill_pack) + return self._skill + + @display + async def screen(self) -> Display: + size = (await self._call("get_screen_size", DeviceUnavailable))["structuredContent"] + return Display(size["width"], size["height"], size.get("scale_factor") or 1.0) + + @capture + async def grab(self) -> Image: + result = await self._call("get_desktop_state", DeviceUnavailable) + shot, state = next(c for c in result["content"] if c["type"] == "image"), result["structuredContent"] + return Image(state["screenshot_width"], state["screenshot_height"], base64.b64decode(shot["data"]), "png") + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + await self._send("click", x=x, y=y, button=button, count=count, modifier=_modifiers(modifiers)) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + (x0, y0), (x1, y1) = path[0], path[-1] + await self._send("drag", from_x=x0, from_y=y0, to_x=x1, to_y=y1, button=button, duration_ms=duration_ms, + modifier=_modifiers(modifiers)) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + if modifiers: + raise Unsupported("cua-driver scrolls with no keys held") + await self._send("scroll", x=x, y=y, direction=direction, amount=amount) + + @type_text + async def type_in(self, text) -> None: + await self._send("type_text", text=text) + + @press + async def press_keys(self, keys) -> None: + *held, key = keys + if any(k not in MODIFIERS for k in held): + raise Unsupported(f"cua-driver holds only modifier keys in a chord, not {[k for k in held if k not in MODIFIERS]}") + await self._send("press_key", key=MODIFIERS.get(key) or NAMED.get(key, key), modifiers=_modifiers(held)) + + async def _forward(self, name: str, **arguments: Any) -> CallToolResult: + """A call to one of cua-driver's own tools, on its caller's connection, answered as cua-driver answers it.""" + connection = self._callers.setdefault(self.caller_identity(ToolContext.current()), _Connection()) + try: + reply = await connection.request({"method": "call", "name": name, "args": arguments}) + except DeviceUnavailable as e: + return _refused(e.message) + if not reply.get("ok"): + return _refused(reply.get("error") or f"cua-driver refused {name}") + return CallToolResult.model_validate(reply["result"]) + + async def _send(self, name: str, **args: Any) -> None: + await self._call(name, DeliveryFailed, scope="desktop", **args) + + async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: + """The result of one of cua-driver's tools called in the framework's session; ``failure`` names what went + wrong when it fails or refuses.""" + reply = await self._framework.request({"method": "call", "name": name, + "args": {**args, "session": FRAMEWORK_SESSION}}) + if not reply.get("ok"): + raise failure(f"cua-driver {name}: {reply.get('error')}") + result = reply["result"] + if result.get("isError"): + raise failure(f"cua-driver {name}: {_text(result)}") + return result + + async def _pinned_checks(self) -> list[ReadyCheck]: + try: + listed = (await self._framework.request({"method": "list"}))["result"]["tools"] + except DeviceUnavailable: + return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")] + digest, expected = await asyncio.to_thread(_executable_sha256), EXECUTABLES.get(self.platform.arch or "") + running = {t["name"]: t["input_schema"] for t in listed} + pinned = {t["name"]: t["inputSchema"] for t in _manifest()} + differ = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name)) + return [ReadyCheck("cua-driver", digest == expected, "the pinned executable" if digest == expected else + f"its executable's sha256 is {digest}, not the pinned {expected}"), + ReadyCheck("cua-driver tools", not differ, f"{len(running)} tools, as pinned" if not differ else + f"these tools differ from tools.json: {', '.join(differ)}")] + + +class _Connection: + """One connection to cua-driver's daemon, opened on first use and again after it drops, taking one request at a + time.""" + + def __init__(self) -> None: + self._stream: tuple[asyncio.StreamReader, asyncio.StreamWriter] | None = None + self._lock = asyncio.Lock() + + async def request(self, payload: dict) -> dict: + async with self._lock: + try: + if self._stream is None: + self._stream = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) + reader, writer = self._stream + writer.write((json.dumps(payload) + "\n").encode()) + await writer.drain() + line = await reader.readline() + if not line: + raise ConnectionError("the daemon closed the connection") + except (OSError, ValueError) as e: + self._close() + raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None + return json.loads(line) + + def _close(self) -> None: + if self._stream is not None: + self._stream[1].close() + self._stream = None + + +@functools.cache +def _manifest() -> list[dict]: + return json.loads(MANIFEST.read_text()) + + +def _modifiers(keys) -> list[str]: + return [MODIFIERS[key] for key in keys] + + +def _refused(message: str) -> CallToolResult: + return CallToolResult(isError=True, content=[TextContent(type="text", text=message)]) + + +def _text(result: dict) -> str: + return " ".join(c.get("text", "") for c in result.get("content", []) if c.get("type") == "text") + + +def _executable() -> Path: + """cua-driver's executable in the driver's venv, found without importing its package, which loads its SDK.""" + return Path(importlib.util.find_spec("cua_driver").submodule_search_locations[0]) / "bin" / "cua-driver" + + +def _executable_sha256() -> str: + return hashlib.sha256(_executable().read_bytes()).hexdigest() + + +def _skill_pack() -> tuple[str, str, dict[str, str]]: + """cua-driver's skill pack as its executable embeds it, written offline by ``mcp-config`` into a home of its own: + its description, its SKILL.md body and its other files.""" + with tempfile.TemporaryDirectory(prefix="cua-driver-skill-") as home: + (Path(home) / ".claude").mkdir() + subprocess.run([str(_executable()), "mcp-config", "--client", "claude"], capture_output=True, check=True, + timeout=SKILL_TIMEOUT_S, env={**os.environ, "HOME": home, "DO_NOT_TRACK": "1", + "CUA_DRIVER_RS_TELEMETRY_ENABLED": "false", + "CUA_DRIVER_RS_UPDATE_CHECK": "false"}) + folder = Path(home) / ".cua-driver" / "skills" / "cua-driver" + files = {path.name: path.read_text() for path in sorted(folder.iterdir()) if path.is_file()} + _, frontmatter, body = files.pop("SKILL.md").split("---\n", 2) + description = next(line.partition(":")[2].strip() for line in frontmatter.splitlines() + if line.startswith("description:")) + return description, body.lstrip("\n"), files diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml new file mode 100644 index 000000000..c5247c8b0 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -0,0 +1,10 @@ +type = "computer_driver" +module = "driver.py:CuaDriver" +port = 18765 + +[install.linux] +python = "requirements.txt" +services = [ + { name = "daemon", command = "./daemon.sh" }, + { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, +] diff --git a/src/agent_env/drivers/envs/cua-driver/requirements.txt b/src/agent_env/drivers/envs/cua-driver/requirements.txt new file mode 100644 index 000000000..bfda61a57 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/requirements.txt @@ -0,0 +1,5 @@ +# What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version +# it ships, which serves the driver with agentenv_protocol.computer.serve. cua-driver's wheel carries its executable for +# each arch, whose sha256 the driver checks. +agentenv-framework-protocol[computer]>=0.1.381 +cua-driver==0.38.0 diff --git a/src/agent_env/drivers/envs/cua-driver/tools.json b/src/agent_env/drivers/envs/cua-driver/tools.json new file mode 100644 index 000000000..c65229a17 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/tools.json @@ -0,0 +1,3832 @@ +[ + { + "name": "list_apps", + "description": "List Linux apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: reserved (Linux X11/Wayland focus model differs from frontmost-app); always false.\n- kind: `\"desktop\"` for XDG `.desktop` launcher entries.\n- launch_path: the launcher command from `Exec=` (field codes stripped). Pass to `launch_app(launch_path=...)`.\n- bundle_id: the XDG \"desktop file id\" — the `.desktop` file's path relative to its XDG `applications/` root with the `.desktop` suffix stripped and path separators replaced with `-` (e.g. `kde4/konqbrowser.desktop` → `kde4-konqbrowser`).\n- last_used: RFC3339 mtime of the `.desktop` file, when readable.\n- windows: the app's current top-level windows (same records as list_windows); empty for a process without one.\n\nRunning apps come from `/proc`. Installed apps come from XDG Desktop Entry files in $XDG_DATA_HOME/applications and each $XDG_DATA_DIRS entry's applications/ subdir. Entries with `NoDisplay=true` or `Hidden=true` are filtered. A `.desktop` file whose launcher matches a running process (by basename), or whose WM_CLASS a top-level window carries, is merged into a single entry with `running: true`.\n\nUse this for \"is X installed?\" as well as \"is X running?\". For per-window state — visibility, geometry, titles — call list_windows instead.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "list_windows", + "description": "List top-level windows. Each record includes z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one). To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order. Filter with `app` or `pid`: an unfiltered call lists every window on the host.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", + "type": "string" + }, + "on_screen_only": { + "description": "When true, filter to visible windows only. Default false.", + "type": "boolean" + }, + "pid": { + "description": "Only list windows owned by this process ID.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_window_state", + "description": "Walk a running app's AT-SPI tree and return it ONCE, as compact Markdown by default (`tree_format:\"markdown\"`). Every actionable element is tagged `[N]` (an element_index); its element_token is `:N`, where `snapshot_id` is printed in the response header and returned in structuredContent. Pass `tree_format:\"elements\"` for the structured `elements` array instead (explicit `element_token`, `role`, `label`, `value`, `enabled`, `selected`, `actions`, `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`), or `\"both\"` for both (about twice the size). Set `query` to project the tree to matching rows plus their ancestor chain while preserving original indices; `total_element_count` reports the complete snapshot.\n\nSize: a read is bounded by default (`max_elements` 250 actionable nodes). When the walk stops at that budget the response says `Tree truncated at max_elements=…` and `truncated: true`; pass a larger `max_elements`, or `query` / `max_depth`, to reach the rest. `_note` is omitted unless `verbose:true`. `full_output:true` restores the previous full response: both representations, all metadata, and the walker's own limits (≤5 000 nodes).\n\nDIFF READS: pass `since: ` from an earlier read of the same window to get only what changed: `+` added rows, `~` changed rows, `-` removed rows (their ids are the old snapshot's), or `no change since …`. Rows not listed keep their previous `[N]` unless a `reindexed:` line says otherwise. The response carries a NEW snapshot_id: use it in element_tokens. An unknown, expired, other-window or differently-scoped (query/max_elements/max_depth) `since` falls back to a full read and `since_status` says why. The focused element is not reported on Linux yet.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error. Optional `max_dimension` caps the returned screenshot's long edge in pixels for a cheap thumbnail.\n\nOptional `max_elements` / `max_depth` bound the AT-SPI walk (Electron / large web apps produce 10k+ element trees). Markdown and structured elements are truncated identically. Default: `max_elements` 250.\n\nTIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. When omitted, a window's first snapshot gets a 2000 ms cold-start budget so Chromium/Electron can finish publishing its initial tree. Explicit values are always honored. Large apps (LibreOffice, GIMP, file managers) can exceed the budget; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.\n\nSCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.\n\nPOPUP MENUS: a context menu / popover / combo list is an override-redirect window that list_windows never shows. A click or right_click that opened one names it in its result (`popup: {window_id, bounds, title}`); pass that window_id here to walk the popup's own AT-SPI toplevel so its menu items get element indices (then click them by element_token). Omitting window_id while a popup of this pid is open walks that popup.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "capture_mode": { + "description": "DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.", + "enum": [ + "ax", + "vision" + ], + "type": "string" + }, + "display_only": { + "description": "Default false. With include_accessibility_tree:false, returns pixels for display without registering an action capture or replacing the agent snapshot. The image cannot ground input actions.", + "type": "boolean" + }, + "full_output": { + "description": "Default false. Set true to restore the previous full response: both `elements` and `tree_markdown`, all metadata, and the platform's own walk limits unless max_elements / max_depth are passed. Does not combine with `since`.", + "type": "boolean" + }, + "include_accessibility_tree": { + "description": "Default true — walk the AT-SPI tree and return it (per `tree_format`) alongside the screenshot. Set false to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return).", + "type": "boolean" + }, + "include_screenshot": { + "description": "Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action).", + "type": "boolean" + }, + "max_depth": { + "description": "Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps.", + "minimum": 1, + "type": "integer" + }, + "max_dimension": { + "description": "Legacy optional cap on the returned screenshot's long edge. Applied on top of the configured max_image_dimension ceiling when max_image_dimension is omitted.", + "minimum": 1, + "type": "integer" + }, + "max_elements": { + "description": "Cap on total AT-SPI nodes walked. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a large window.", + "minimum": 1, + "type": "integer" + }, + "max_image_dimension": { + "description": "Per-call long-edge override. This value wins over configured and legacy limits; 0 returns native-resolution PNG bytes. Omit to preserve configured behavior.", + "minimum": 0, + "type": "integer" + }, + "pid": { + "description": "Process ID that owns the window.", + "type": "integer" + }, + "query": { + "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", + "type": "string" + }, + "screenshot_out_file": { + "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output carries screenshot_file_path instead.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "since": { + "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that snapshot: `+` added rows, `~` changed rows, `-` removed rows (removed ids are the old snapshot's), or `no change`. Falls back to a full read when the snapshot is unknown, expired, belongs to another window, or was read with a different query/max_elements/max_depth; `since_status` says which. The response still carries a NEW snapshot_id and its element_tokens; rows not listed keep their previous [N] unless a `reindexed:` line says otherwise.", + "type": "string" + }, + "timeout_ms": { + "default": 1000, + "description": "Wall-clock budget in milliseconds for the accessibility-tree walk (default 1000, min 100, max 120000). Bounds the WHOLE walk. When the budget runs out the tool returns the PARTIAL tree it has, flagged with `truncated: true`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `elements_complete: false`; retry with a larger value (e.g. 5000) or narrow with `query` / `max_depth`.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + }, + "tree_format": { + "description": "Which tree representation to return. Default \"markdown\": the compact `tree_markdown` only, where a row `[N]` is addressed with element_token `:N`. \"elements\": the structured `elements` array only (per-element element_token, frame, parent_index, value). \"both\": the two together, as before; roughly doubles the size.", + "enum": [ + "markdown", + "elements", + "both" + ], + "type": "string" + }, + "verbose": { + "description": "Default false. Set true to include the `_note` string and the full `background_input` capability report (macOS), which are omitted by default to save context.", + "type": "boolean" + }, + "window_id": { + "description": "Native window identifier from list_windows, or the `popup.window_id` a click / right_click result named (an open context menu / popover; its menu items then get element indices). Omitted: the pid's open popup menu when one is mapped, else its focused / active / largest window.", + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "verify_state", + "description": "Deterministically verify bounded predicates against one exact window. The driver evaluates structured window/accessibility state and may return the final screenshot as uninterpreted visual evidence for a multimodal caller. Predicate results are satisfied, unsatisfied, or unknown; unknown never implies success. Accessibility projections are conservative: absence remains unknown unless the observed search domain is proven exhaustive.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "expect": { + "description": "One to eight predicates, combined with logical AND.", + "items": { + "additionalProperties": false, + "properties": { + "element": { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "exists": { + "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", + "type": "boolean" + }, + "selected": { + "type": "boolean" + }, + "selector": { + "additionalProperties": false, + "properties": { + "label_contains": { + "minLength": 1, + "type": "string" + }, + "role": { + "minLength": 1, + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "value_equals": { + "type": "string" + } + }, + "required": [ + "selector" + ], + "type": "object" + }, + "window": { + "additionalProperties": false, + "properties": { + "bounds": { + "additionalProperties": false, + "properties": { + "height": { + "type": "number" + }, + "tolerance_px": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "exists": { + "type": "boolean" + } + }, + "required": [], + "type": "object" + } + }, + "required": [], + "type": "object" + }, + "maxItems": 8, + "minItems": 1, + "type": "array" + }, + "include_screenshot": { + "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", + "type": "boolean" + }, + "pid": { + "description": "Exact process whose window may be observed.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session. This\nfield never selects capture modality or authorization.", + "type": "string" + }, + "stable_samples": { + "default": 2, + "description": "Consecutive satisfied samples required before returning success.", + "maximum": 5, + "minimum": 1, + "type": "integer" + }, + "timeout_ms": { + "default": 5000, + "description": "Bounded wait. Zero performs one sample.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Exact native window identifier.", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "expect" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "launch_app", + "description": "Launch a Linux app in the background. Provide launch_path (preferred — round-trip the value from list_apps), name (tried as a direct command, then matched against installed .desktop applications, then handed to xdg-open if it is a URL or existing file path), bundle_id (ignored on Linux), or urls (list of URLs to open). Resolution precedence: launch_path > name > bundle_id. Errors when the name resolves to nothing launchable.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "additional_arguments": { + "description": "Extra command-line arguments passed to the launched process.", + "items": { + "type": "string" + }, + "type": "array" + }, + "bundle_id": { + "description": "Ignored on Linux (macOS/Windows concept).", + "type": "string" + }, + "launch_path": { + "description": "Round-trip the `launch_path` returned by `list_apps` — the Exec= command from the .desktop file with XDG field codes already stripped. Highest precedence on Linux; spawned directly via the system shell.", + "type": "string" + }, + "name": { + "description": "App name or command to launch. Tried as a direct command first, then matched against installed .desktop applications (exact display name, desktop-file id, or Exec basename; else an unambiguous display-name substring).", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "urls": { + "description": "URLs to open via xdg-open.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "kill_app", + "description": "Force-terminate a process by pid (kill -9 equivalent on Linux). Use as escalation when the cooperative close path failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "PID of the process to terminate.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "bring_to_front", + "description": "Persistently activate a window so subsequent input lands on it. This deliberately breaks the no-foreground contract and is not part of the normal input ladder. For an ordinary `background_unavailable` response, retry only the refused action with `delivery_mode:\"foreground\"`; the input tool performs its own activate, act, and restore sequence. Use `bring_to_front` only for a focus-proxy surface that must remain foreground across multiple calls, such as a remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, with proper timestamp handling to beat focus-stealing prevention). Wayland: activates through a target-addressable compositor adapter (wlroots foreign-toplevel or the GNOME Shell helper) and refuses when the compositor offers no safe adapter. Matches the macOS / Windows bring_to_front rung.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "Process ID of the app to activate.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "X11 window id (xid) to activate. If omitted, the first window of `pid` is used.", + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "set_window_frame", + "description": "Set one exact top-level window's frame in the desktop-coordinate space reported by list_windows and verify the resulting geometry through an independent readback.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "height": { + "description": "New height, in the same units as list_windows bounds.", + "minimum": 1, + "type": "number" + }, + "pid": { + "description": "Process ID that owns the window.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "width": { + "description": "New width, in the same units as list_windows bounds.", + "minimum": 1, + "type": "number" + }, + "window_id": { + "description": "Window ID from list_windows.", + "minimum": 1, + "type": "integer" + }, + "x": { + "description": "New left edge in the desktop coordinate space reported by list_windows.", + "type": "number" + }, + "y": { + "description": "New top edge in the desktop coordinate space reported by list_windows.", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "invoke_menu", + "description": "Resolve an exact application-menu path one live native level at a time and invoke its final item through accessibility APIs. Missing, ambiguous, disabled, or structurally mismatched segments fail closed; this tool never falls back to pixels. Menu paths and refusals: `get_guide({topic:\"menus\"})`.", + "inputSchema": { + "additionalProperties": false, + "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).", + "properties": { + "path": { + "description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).", + "items": { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + "maxItems": 16, + "minItems": 1, + "type": "array" + }, + "pid": { + "description": "Process ID of the application that owns the menu.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window ID from list_windows whose menu is invoked.", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "path" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "click", + "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error.", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "capture_id": { + "description": "Optional one-shot binding to the exact PNG returned by get_window_state or get_desktop_state. When present, x/y are interpreted in that capture and admitted before native dispatch.", + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "count": { + "description": "Number of clicks in one press train: 1 (default), 2 = double-click, 3 = triple-click (line/paragraph selection). There is no separate triple_click tool — pass count: 3 here.", + "minimum": 1, + "type": "integer" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Coordinate frame (default \"window\"). Pass \"desktop\" with x,y and no pid/window_id for a screen-absolute click in get_desktop_state coordinates.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Exact window (kind \"window\", with pid and window_id) or desktop (kind \"desktop\", with display_id). \"primary\" is the portable desktop target.", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "double_click", + "description": "Double-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to). No focus steal. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "right_click", + "description": "Right-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to), one outside the window is refused (point_outside_window; x/y are window-local pixels, the result reports window_point / screen_point). No focus steal. When a context menu opens, the result names it (`popup: {window_id, bounds, title}`): call get_window_state(pid, window_id=) to index its menu items and click them by element_token. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "drag", + "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. Background delivery on X11 is one real held gesture on the session's virtual master pointer (path mpx_pointer: press, interpolated motion, release) — it lands on GTK/VCL/Qt/Chromium; a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it. Headless servers fall back to XSendEvent. duration_ms (default 500), steps (default 20). No focus steal. Limits and fallbacks: `get_guide({topic:\"drag\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "duration_ms": { + "description": "Total drag duration. Default: 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "from_x": { + "description": "Drag start X: window-local pixels of the target window's own get_window_state screenshot; pass scope:\"desktop\" for get_desktop_state pixels.", + "type": "number" + }, + "from_y": { + "description": "Drag start Y (same frame as from_x).", + "type": "number" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\".", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "steps": { + "description": "Intermediate MotionNotify events. Default: 20.", + "maximum": 200, + "minimum": 1, + "type": "integer" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "to_x": { + "description": "Drag end X (same frame as from_x).", + "type": "number" + }, + "to_y": { + "description": "Drag end Y (same frame as from_x).", + "type": "number" + }, + "window_id": { + "description": "Target window XID. Required.", + "type": "integer" + } + }, + "required": [ + "from_x", + "from_y", + "to_x", + "to_y" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "mouse_button_down", + "description": "Press and hold a mouse button at (x,y) via background X11 delivery. Does not release the button; pair with mouse_drag / mouse_button_up. Returns the current held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "mouse_drag", + "description": "Move a previously-held mouse button to a new point via background X11 delivery. Requires an active mouse_button_down state; does not release the button. Returns the updated held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "duration_ms": { + "description": "Total drag duration. Default: 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Optional process ID; must match the held button's target.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "steps": { + "description": "Intermediate MotionNotify events. Default: 20.", + "maximum": 200, + "minimum": 1, + "type": "integer" + }, + "window_id": { + "description": "Optional window id; must match the held button's target.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "mouse_button_up", + "description": "Release a previously-held mouse button via background X11 delivery. If x/y are omitted, releases at the last held position. Returns the current held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Optional process ID; must match the held button's target.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "window_id": { + "description": "Optional window id; must match the held button's target.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "parallel_mouse_drag", + "description": "Run multiple mouse drag gestures concurrently via Linux MPX/XI2 virtual master pointers. Each drag item runs on its own session-scoped master pointer (true same-window concurrent draws on X11). Each item presses once, glides continuously through its whole path, and releases once — one smooth held drag, not a chain of clicks. A path is given either as a straight segment (from_x/from_y → to_x/to_y) or as a function `fn` = y(x) sampled over [x_from, x_to] in window-local pixels (e.g. fn:\"x\" is a diagonal, fn:\"300+120*sin(x/40)\" a sine wave). Functions support + - * / ^, sin/cos/tan, sqrt, abs, exp, ln, pi, e.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "drags": { + "description": "Two or more drag gestures to run concurrently, each on its own virtual master pointer.", + "items": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "duration_ms": { + "description": "Default: 1500 for fn paths, 500 for straight.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "fn": { + "description": "Expression y(x) in window-local pixels; sampled over [x_from,x_to]. Mutually exclusive with from_x/to_x.", + "type": "string" + }, + "from_x": { + "description": "Straight-segment start X (window-local pixels).", + "type": "number" + }, + "from_y": { + "description": "Straight-segment start Y (window-local pixels).", + "type": "number" + }, + "path": { + "description": "Explicit window-local waypoints [[x,y],...] (>=2); pressed once, glided through, released once. Takes precedence over fn/from-to.", + "items": { + "items": { + "type": "number" + }, + "type": "array" + }, + "type": "array" + }, + "samples": { + "description": "Waypoints sampled along `fn`. Default: 80.", + "maximum": 400, + "minimum": 2, + "type": "integer" + }, + "session": { + "description": "Session/cursor id; also keys the virtual master pointer.", + "type": "string" + }, + "steps": { + "description": "Motion sub-steps along the whole path. Default: scaled to path length.", + "maximum": 300, + "minimum": 1, + "type": "integer" + }, + "to_x": { + "description": "Straight-segment end X (window-local pixels).", + "type": "number" + }, + "to_y": { + "description": "Straight-segment end Y (window-local pixels).", + "type": "number" + }, + "window_id": { + "description": "Window id from list_windows.", + "type": "integer" + }, + "x_from": { + "description": "Domain start (window-local x) when `fn` is used.", + "type": "number" + }, + "x_to": { + "description": "Domain end (window-local x) when `fn` is used.", + "type": "number" + } + }, + "required": [ + "session", + "window_id" + ], + "type": "object" + }, + "minItems": 2, + "type": "array" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "drags" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "type_text", + "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to type into the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to type into the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "text": { + "description": "Text to type.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Window-local pixels of the target window's own get_window_state screenshot by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\") — a bare pid+x/y is otherwise misinterpreted as window-local and can focus the wrong widget.", + "type": "number" + }, + "y": { + "description": "Pixel Y of the field (see x).", + "type": "number" + } + }, + "required": [ + "text" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "press_key", + "description": "Press a key via XSendEvent to a window. No focus steal.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "key": { + "description": "Key name: enter/return, tab, escape, space, backspace, delete, insert, home, end, pageup, pagedown, up, down, left, right, f1-f12, or any single ASCII character.", + "type": "string" + }, + "modifiers": { + "description": "Modifier keys held while the key is pressed: ctrl, shift, alt, super.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to send the key to the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to send the key to the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_token. Window-local pixels by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Pixel Y (see x).", + "type": "number" + } + }, + "required": [ + "key" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "hotkey", + "description": "Press a combination of keys simultaneously, e.g. [\"ctrl\",\"c\"] for Copy. Delivered to the target window without focus steal (real key events from a virtual master keyboard on X11); target does NOT need to be frontmost. Window-manager chords cannot reach the WM that way: alt+F4 on a window target closes that window through the WM (_NET_CLOSE_WINDOW, effect confirmed when it unmaps); super+*, alt+tab, ctrl+alt+* and other WM bindings are refused with code wm_chord_unavailable — use bring_to_front / list_windows / kill_app or delivery_mode:\"foreground\" instead.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "keys": { + "description": "Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"].", + "items": { + "type": "string" + }, + "minItems": 2, + "type": "array" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to send the chord to the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to send the chord to the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.", + "type": "number" + }, + "y": { + "description": "Screenshot-pixel Y (see x).", + "type": "number" + } + }, + "required": [ + "keys" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "set_value", + "description": "Set value of an AT-SPI element via SetValue action.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "value": { + "description": "New value. Written through AT-SPI EditableText/Value when the element exposes them; otherwise (GTK2 spin scales, VCL spin buttons) the field is clicked with the session's real pointer, its text selected and replaced through the virtual keyboard and committed with Tab, then read back.", + "type": "string" + }, + "window_id": { + "description": "Omit when element_token is supplied (the token carries it).", + "type": "integer" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "scroll", + "description": "Scroll the target pid's focused region via XSendEvent Button4/5. direction required; by defaults to line, amount defaults to 3.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "amount": { + "description": "Number of scroll steps. Default: 3.", + "maximum": 50, + "minimum": 1, + "type": "integer" + }, + "by": { + "description": "Scroll granularity. Default: line.", + "enum": [ + "line", + "page" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "direction": { + "description": "Scroll direction.", + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with x,y and no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local screenshot-pixel X of the scroll target. Pass with y and without element_token.", + "type": "number" + }, + "y": { + "description": "Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_token.", + "type": "number" + } + }, + "required": [ + "direction" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "clipboard_read", + "description": "List available system clipboard types and optionally return privacy-sensitive plain text. Clipboard content is never retained in telemetry.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include_text": { + "default": false, + "description": "Return plain-text clipboard content in addition to the available types.\nClipboard content is privacy-sensitive and is never retained in telemetry.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "clipboard_write", + "description": "Replace the system clipboard with exactly one value: plain text, an image from an absolute local path, or a file URL from an absolute local path. Returns the available types for read-back before paste.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "file_path": { + "description": "Absolute path to a local file to place on the clipboard as a file URL.", + "type": "string" + }, + "image_path": { + "description": "Absolute path to a local image to place on the clipboard.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "text": { + "description": "Plain text to place on the clipboard.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_screen_size", + "description": "Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_desktop_state", + "description": "Capture the full display in the desktop action coordinate frame. Use the returned PNG directly as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. No AT-SPI walk.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "max_image_dimension": { + "description": "Optional long-edge cap for the returned PNG, in pixels (aspect ratio preserved). Omitted or 0 returns the full-size capture. When the cap downsizes the image, the response reports `screenshot_original_width/height`, and x/y read off it for this session's later scope:\"desktop\" actions (or passed with its `capture_id`) are mapped back to the full-size frame automatically.", + "minimum": 0, + "type": "integer" + }, + "screenshot_out_file": { + "description": "Write PNG here instead of base64.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "get_cursor_position", + "description": "Return the current mouse cursor position in screen points (origin top-left).", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "move_cursor", + "description": "Move the synthetic agent cursor without changing the user's pointer. Only an explicit scope=desktop request moves the real OS pointer in get_desktop_state coordinates.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "cursor_id": { + "description": "Cursor instance to move. Default: 'default'.", + "type": "string" + }, + "scope": { + "default": "window", + "description": "\"window\" (default) moves only the agent cursor overlay; \"desktop\" moves the real OS pointer.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "x": { + "description": "Destination X. Window scope: screen coordinates of the agent cursor overlay. Desktop scope: get_desktop_state screenshot pixels.", + "type": "number" + }, + "y": { + "description": "Destination Y, in the same space as x.", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "set_agent_cursor_enabled", + "description": "Show or hide the agent cursor owned by a session.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "enabled": { + "description": "`true` shows the session's agent cursor overlay; `false` hides it.", + "type": "boolean" + }, + "session": { + "description": "Public label of the session that owns the cursor.", + "type": "string" + } + }, + "required": [ + "session", + "enabled" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "set_agent_cursor_motion", + "description": "Configure the movement style, timing, effects and visibility timing for a session cursor.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "arc_flow": { + "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", + "type": "number" + }, + "arc_size": { + "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", + "type": "number" + }, + "dwell_after_click_ms": { + "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", + "type": "number" + }, + "effects": { + "additionalProperties": false, + "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", + "properties": { + "glow": { + "description": "Soft glow around the cursor that grows with speed.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "magnet": { + "description": "Target glow when the `magnetic` style locks on.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "ripple": { + "description": "Ring that expands from the hotspot on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "squish": { + "description": "Brief scale-down of the cursor on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "trail": { + "description": "Short fading trail behind the cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "end_handle": { + "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "glide_duration_ms": { + "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", + "type": "number" + }, + "idle_hide_ms": { + "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", + "type": "number" + }, + "session": { + "description": "Public label of the session that owns the cursor. Omitted or null motion fields keep\ntheir current value.", + "type": "string" + }, + "spring": { + "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", + "type": "number" + }, + "start_handle": { + "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "style": { + "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", + "type": "number" + } + }, + "required": [ + "session" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_agent_cursor_state", + "description": "Return the session cursor's theme, semantic playback, position, visibility, and motion.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "Public label of the session whose cursor to inspect.", + "type": "string" + } + }, + "required": [ + "session" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "set_agent_cursor_theme", + "description": "Select an already-installed cursor theme for a session.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "reduced_motion": { + "default": "auto", + "description": "Theme animation policy: `on` uses the theme's reduced-motion frames, `off` always\nanimates, `auto` (default) leaves the choice to the host.", + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "session": { + "description": "Public label of the session that owns the cursor.", + "type": "string" + }, + "theme_id": { + "description": "Id of an installed cursor theme (see `cua-driver cursor-theme list`).", + "maxLength": 200, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "session", + "theme_id" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "check_permissions", + "description": "Check required permissions for cua-driver-rs on Linux. How to get them granted: `get_guide({topic:\"permissions\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "prompt": { + "description": "Ignored: Linux has no permission prompts.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "health_report", + "description": "Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped. For standalone CuaDriver, use `cua-driver permissions grant` to verify it explicitly. For an embedded or linked driver, use the host's permission flow for the app or executable macOS attributes this process's permission request to; the standalone command does not verify the host's capture readiness. cua-driver owns the health model; consumers stay thin.\n\nInput — all optional:\n {\n \"include\": [\"\", ...], // run only these\n \"skip\": [\"\", ...] // skip these\n }\nIf both are given, `include` wins.\n\nCanonical check names:\n macOS : binary_version, platform_supported, session_active,\n bundle_identity, tcc_accessibility, tcc_screen_recording,\n ax_capability, screen_capture_capability, windowserver_connection\n Windows: binary_version, platform_supported, session_active,\n ax_capability (via UIA), screen_capture_capability (via DXGI)\n Linux : binary_version, platform_supported, session_active,\n ax_capability (via AT-SPI), screen_capture_capability (via X11)\n\nOutput — stable contract, schema_version=\"1\":\n {\n \"schema_version\": \"1\",\n \"platform\": \"darwin\" | \"win32\" | \"linux\",\n \"driver_version\": \"\",\n \"overall\": \"ok\" | \"degraded\" | \"failed\",\n \"checks\": [\n {\n \"name\": \"\",\n \"status\": \"pass\" | \"fail\" | \"skip\",\n \"message\": \"\",\n \"hint\": \"\",\n \"data\": { /* check-specific structured fields */ }\n },\n ...\n ]\n }\n\n`overall` rules:\n - `ok` — every non-skipped check passes\n - `degraded` — at least one non-core check fails (binary is still usable)\n - `failed` — any core check fails (binary_version, platform_supported, session_active)\n\nStability: schema_version=\"1\" is the contract. Future breaking changes will be `\"2\"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include": { + "description": "Only run these checks (canonical names). Wins over `skip`.", + "items": { + "type": "string" + }, + "type": "array" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "skip": { + "description": "Skip these checks (canonical names). Ignored when `include` is set.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_config", + "description": "Return current cua-driver-rs configuration.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "set_config", + "description": "Update cua-driver-rs configuration. capture_mode / max_image_dimension take effect immediately.\n\nPass each setting as its own field: `{\"capture_mode\": \"som\", \"max_image_dimension\": 0}`.\n\nThe experimental_pip keys persist to ~/.cua-driver/config.json and apply on next daemon restart (the PiP backend is initialised once at startup; Linux ships only the trait stub today — see issue #1729).\n\n`cursor.motion.style`, `cursor.motion.timing` and `cursor.motion.effects.` (`on`, `off`, or `default`) save the default cursor motion for sessions started afterwards. `cursor.motion: \"default\"` clears every saved cursor motion default.\n\n`cursor.badges` (`minimal` default, `verbose`, or `off`) sets the agent cursor badge for every cursor at once.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "capture_mode": { + "description": "Default capture mode for get_window_state. (\"som\"/\"screenshot\" still decode as deprecated aliases.)", + "enum": [ + "ax", + "vision" + ], + "type": "string" + }, + "cursor.badges": { + "description": "Agent cursor badge. minimal (default): no badge at rest; a brief chip for a foreground moment or a key combo, and the session name on hover or when two or more agent cursors are on screen. verbose: the earlier badge with the session name and delivery/target chips on every action. off: no badge. Applies immediately to every cursor.", + "enum": [ + "minimal", + "verbose", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion": { + "description": "Pass `default` to clear every saved cursor motion default.", + "enum": [ + "default" + ], + "type": "string" + }, + "cursor.motion.effects.glow": { + "description": "Saved default for the `glow` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.magnet": { + "description": "Saved default for the `magnet` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.ripple": { + "description": "Saved default for the `ripple` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.squish": { + "description": "Saved default for the `squish` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.trail": { + "description": "Saved default for the `trail` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.style": { + "description": "Saved default cursor motion style for new sessions. Overridden by start_session cursor_motion and set_agent_cursor_motion; reduced motion always wins. Built-in default: signature_arc.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "cursor.motion.timing": { + "description": "Saved default cursor move timing for new sessions. Built-in default: steady (a 600 ms floor on every move).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "experimental_pip": { + "description": "Enable the experimental PiP preview window (applies next restart; Linux backend stubbed).", + "type": "boolean" + }, + "experimental_pip_geometry": { + "description": "PiP window size + optional position in `WxH` or `WxH+X+Y` form.", + "type": "string" + }, + "max_image_dimension": { + "description": "Max dimension for screenshot resizing (0 = no limit).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_accessibility_tree", + "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "zoom", + "description": "Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixels, with 20% padding. Output is at most 500 px wide. Corners, not a size: x1,y1 is the top-left and x2,y2 the bottom-right (x,y,width,height is also accepted and converted).\n\nAfter a zoom, pass from_zoom=true to click/type_text to auto-translate coordinates back to full-window space. Coordinate actions return `screenshot_context_missing` when no current snapshot contains a screenshot owned by this session. `from_zoom` actions return `zoom_context_missing` when the zoom was never created or was replaced; call `get_window_state`, then `zoom`, again on the same connection.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "Optional target pid. When omitted, the driver resolves the unique current snapshot for this session and window.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id of the window captured by get_window_state.", + "type": "integer" + }, + "x1": { + "description": "Left edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "x2": { + "description": "Right edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "y1": { + "description": "Top edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "y2": { + "description": "Bottom edge of the region in window-local screenshot pixels.", + "type": "number" + } + }, + "required": [ + "window_id", + "x1", + "y1", + "x2", + "y2" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "page", + "description": "Legacy browser compatibility tool. Prefer get_browser_state and the typed browser_* tools for exact targeting, endpoint ownership, and consent. Read-only get_text and query_dom remain available by default. Mutating actions require the daemon operator to set CUA_DRIVER_ENABLE_LEGACY_PAGE_MUTATIONS=1 before daemon startup (restart the daemon after changing it); this escape hatch does not provide the typed browser surface's exact binding or existing-profile grant guarantees. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page.\n- query_dom: Find elements matching a CSS selector.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- insert_text: Insert `text` at whatever currently holds DOM focus in one native operation (CDP Input.insertText) — no synthesized key events, but more durable than a one-shot execute_javascript write since rich-text editors already have to treat it like an IME commit. Try this before type_keystrokes on a contenteditable that discarded an execute_javascript write. Click/focus the target field first.\n- type_keystrokes: Type `text` via real per-character keystroke events into whatever currently holds DOM focus. Slower than insert_text but the most durable rung — use it when insert_text also gets discarded, or the editor's own keydown/keyup handlers need to see real keys. Click/focus the target field first.\n- enable_javascript_apple_events: macOS-only — patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", + "inputSchema": { + "additionalProperties": false, + "properties": { + "action": { + "description": "Action to perform.", + "enum": [ + "execute_javascript", + "get_text", + "query_dom", + "click_element", + "insert_text", + "type_keystrokes", + "enable_javascript_apple_events" + ], + "type": "string" + }, + "attributes": { + "description": "Element attributes to include in query_dom results.", + "items": { + "type": "string" + }, + "type": "array" + }, + "bundle_id": { + "description": "Bundle ID of the browser. Required for enable_javascript_apple_events (macOS only).", + "type": "string" + }, + "cdp_port": { + "description": "Optional, for execute_javascript/insert_text/type_keystrokes: use this exact CDP port instead of auto-discovering one from pid. Needed when the port was opened via the browser's own remote-debugging toggle rather than a launch-time flag, since that path may not answer the auto-discovery probe.", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "css_selector": { + "description": "CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*').", + "type": "string" + }, + "javascript": { + "description": "JavaScript to execute. Required for execute_javascript.", + "type": "string" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "selector": { + "description": "CSS selector for click_element (e.g. 'button.submit', '#login a').", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target_url_contains": { + "description": "Optional, for execute_javascript/insert_text/type_keystrokes: require exactly one browser tab whose URL contains this substring. Use this on a multi-tab browser — there's no built-in link between window_id and which tab a CDP call reaches.", + "type": "string" + }, + "text": { + "description": "Text to insert or type. Required for insert_text and type_keystrokes. The target field must already have DOM focus (click/focus it first).", + "type": "string" + }, + "user_has_confirmed_enabling": { + "description": "Must be true to proceed with enable_javascript_apple_events. This will quit and relaunch the browser.", + "type": "boolean" + }, + "window_id": { + "description": "Target window ID from list_windows.", + "type": "integer" + } + }, + "required": [ + "action" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "get_browser_state", + "description": "Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native browser window to classify it, correlate it to a CDP target (exact-or-refuse), and mint a session-scoped target id plus tab ids. Mode 2 (snapshot): pass target_id + tab_id. The dom_refs_v1 compatibility format returns composed DOM refs. semantic_v2 joins accessibility, DOM, layout, and viewport state; ranks visible content before retained/offscreen state; and returns a semantic outline, typed action refs, content refs, scoped reads, and opaque continuation. Never performs setup: a browser that is not attached yet refuses with browser_consent_required or browser_requires_setup, and detail.next_call names the browser_prepare call that attaches it. Use this route for any web page in Chrome, Edge or Brave; desktop pointer tools move the user's pointer. Setup and recovery: `get_guide({topic:\"browser\"})`.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "continuation": { + "description": "Opaque continuation minted by an earlier semantic_v2 response.", + "type": "string" + }, + "include_screenshot": { + "default": false, + "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.", + "type": "boolean" + }, + "pid": { + "description": "Native browser process id (bind mode).", + "type": "integer" + }, + "query": { + "description": "Read-only semantic match over role, accessible name, and visible text.", + "type": "string" + }, + "scope_ref": { + "description": "Current semantic/content ref whose subtree should be observed.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "snapshot_format": { + "description": "Versioned snapshot contract. dom_refs_v1 remains the compatibility default.", + "enum": [ + "dom_refs_v1", + "semantic_v2" + ], + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "window_id": { + "description": "Native window id owned by pid (bind mode).", + "type": "integer" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "browser_prepare", + "description": "Attach the browser tools to a browser. For a page in the user's running Chrome, Edge or Brave, call it once with pid, window_id, session and strategy {\"kind\":\"existing_profile\"} when get_browser_state refuses with browser_consent_required or browser_requires_setup; then call get_browser_state again and act with browser_click, browser_type or browser_pointer. It attaches, or refuses with nothing changed and says why. Explicitly prepare an owned DevTools endpoint for a browser. pid is required for an existing process or existing-profile attachment, and optional only for allow_launch=true with an isolated profile. Existing endpoints are detected without side effects. Acting setup for an isolated profile follows the runtime permission mode and optional capability manifest. It requires allow_launch=true, launches a separate browser, and never copies, modifies, or terminates the requested user profile. Without pid, only a platform-attested system Chrome/Edge installation (or a root-owned package payload on Linux) is eligible; redirects and user-controlled locations fail closed. Existing-profile attachment is explicit and follows the runtime's immutable permission mode: standard requires an explicit --grant existing-profile launch grant or an embedding authorization host, bounded requires a launch-approved exact resource manifest, and unrestricted requires explicit trusted startup risk acceptance. Ordinary MCP transport approval never proves profile authorization. On proven platforms, an authorized request also permits one bounded exact-window setup: open the recognized browser product's fixed remote-debugging page, toggle its uniquely matched per-instance checkbox, prove the PID-owned loopback endpoint, and close the temporary tab. Every visible effect is reported; ambiguity is refused. Where attaching would bring the browser in front of the user's app (macOS Chrome shows its own remote-debugging prompt in front whenever the driver opens a new browser connection; sessions for the same browser share one approved connection, kept for up to 30 minutes idle and 8 hours in total, so later sessions do not prompt again), existing-profile attachment refuses with background_unavailable unless delivery_mode is \"foreground\" (the app that was in front is brought back afterwards) or the browser window is on another Space than the user's, where it attaches in the background while the driver reverts the browser's own brief activations. A driver-owned isolated launch result carries an advisory lifecycle object: the browser process ends with the calling session after idle_expires_in_seconds without session activity, an isolated_named profile persists after the session and an isolated_new profile does not, and explicit end, disconnect, revocation or shutdown end it earlier. Setup and recovery: `get_guide({topic:\"browser\"})`.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "allow_launch": { + "description": "Allow a separate driver-owned isolated Chromium process to be launched (default false).", + "type": "boolean" + }, + "delivery_mode": { + "default": "background", + "description": "Existing-profile attachment only. background (default) attaches in the background when the browser window is on another Space than the user's, and otherwise refuses with background_unavailable, before any visible effect, where attaching would bring the browser in front of the user's app. foreground accepts that, for a screen nobody else is using or when the user agreed; the app that was in front is brought back afterwards.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "pid": { + "description": "Browser process id to prepare. Required except for a driver-owned isolated_new/isolated_named launch with allow_launch=true.", + "type": "integer" + }, + "profile": { + "additionalProperties": false, + "description": "Driver-owned isolated Chromium profile to launch with allow_launch=true. mode=isolated_new creates a fresh throwaway profile; mode=isolated_named reuses the named driver-owned profile. Never an existing user profile.", + "properties": { + "mode": { + "enum": [ + "isolated_new", + "isolated_named" + ], + "type": "string" + }, + "name": { + "description": "Required only for isolated_named; 1-64 path-safe ASCII characters.", + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": "object" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "strategy": { + "additionalProperties": false, + "description": "Attach to an already-running browser instead of launching one. Pass an object, {\"kind\":\"existing_profile\"}, not a string. kind=existing_profile attaches to the user's running profile at pid/window_id under the runtime's profile authorization.", + "properties": { + "kind": { + "enum": [ + "existing_profile" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Exact native window approval anchor; required for strategy.kind=existing_profile.", + "type": "integer" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_navigate", + "description": "Navigate one tab of an exactly-bound browser target to a new URL (http/https/about only). Refused for heuristic bindings. Navigation invalidates all p: refs for the tab.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "url": { + "description": "Destination URL (http:, https:, or about:).", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "url" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_click", + "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "delivery_mode": { + "default": "background", + "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "input_route": { + "description": "\"trusted\" (default): Input.dispatchMouseEvent. It refuses rather than foregrounding a standalone browser. \"dom_event\": synthetic full-background DOM click, only when explicitly requested. Dispatch does not prove the control activated; refresh page state and verify the expected postcondition.", + "enum": [ + "trusted", + "dom_event" + ], + "type": "string" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "x": { + "description": "Viewport x (CSS px) — alternative to ref.", + "type": "number" + }, + "y": { + "description": "Viewport y (CSS px) — alternative to ref.", + "type": "number" + } + }, + "required": [ + "target_id", + "tab_id" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_type", + "description": "Type text into an exactly-bound tab via the Input domain. mode=\"insert_text\" (default) uses Input.insertText; mode=\"keystrokes\" dispatches per-character key events. Both insert at the caret, so typing into a field that already holds text appends to it; pass replace=true to set the field instead, or to clear it by typing an empty string. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "mode": { + "description": "insert_text (default): bulk Input.insertText. keystrokes: per-character Input.dispatchKeyEvent.", + "enum": [ + "insert_text", + "keystrokes" + ], + "type": "string" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "replace": { + "description": "false (default): insert at the caret, appending to whatever the field already holds. true: select the element's whole content first so the text replaces it — with an empty text this clears the field. Replacement goes through the selection, so beforeinput/input still fire and framework state stays consistent.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "text": { + "description": "Text to type.", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "ref", + "text" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_select", + "description": "Select one option of a native HTML ref through CDP. This bypasses native file pickers, rejects symlinks and non-regular files, and never returns local paths.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "files": { + "description": "Absolute paths of 1 to 32 local regular files to assign to the input.", + "items": { + "description": "Absolute path to one local regular file.", + "type": "string" + }, + "maxItems": 32, + "minItems": 1, + "type": "array" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "ref", + "files" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_download", + "description": "Trigger one download through an exact live browser ref and save it inside an explicitly approved directory. Requires MCP-host destructive-tool approval, refuses ambiguous or stale capabilities, and never returns the source URL, filename, or destination path.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "destination_root": { + "description": "Absolute, existing, canonical directory approved to receive the download.", + "type": "string" + }, + "ref": { + "description": "Live page ref whose activation initiates the download.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", + "type": "string" + }, + "tab_id": { + "description": "Opaque exact tab id from get_browser_state.", + "type": "string" + }, + "target_id": { + "description": "Opaque exact browser target id from get_browser_state.", + "type": "string" + } + }, + "required": [ + "session", + "target_id", + "tab_id", + "ref", + "destination_root" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "browser_pointer", + "description": "Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. Semantic refs must declare pointer for hover, right-click, double-click, and drag; scroll accepts a scroll or pointer capability. The trusted route uses CDP Input events and refuses if standalone background posture cannot be preserved. The explicit dom_event route requires a page ref and synthesizes full-background DOM events. Never activates or brings a tab to the foreground.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "action": { + "description": "Pointer gesture. scroll needs delta_x or delta_y; drag needs destination_ref or to_x/to_y.", + "enum": [ + "hover", + "right_click", + "double_click", + "scroll", + "drag" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "delta_x": { + "description": "Horizontal scroll delta in CSS pixels.", + "type": "number" + }, + "delta_y": { + "description": "Vertical scroll delta in CSS pixels.", + "type": "number" + }, + "destination_ref": { + "description": "Drag destination page ref in the exact same frame.", + "type": "string" + }, + "input_route": { + "default": "trusted", + "description": "trusted sends CDP Input events; dom_event synthesizes DOM events in the page and requires ref.", + "enum": [ + "trusted", + "dom_event" + ], + "type": "string" + }, + "ref": { + "description": "Origin page ref. Alternative to x/y.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id minted by get_browser_state.", + "type": "string" + }, + "target_id": { + "description": "Opaque target id minted by get_browser_state.", + "type": "string" + }, + "to_x": { + "description": "Drag destination viewport x in CSS pixels.", + "type": "number" + }, + "to_y": { + "description": "Drag destination viewport y in CSS pixels.", + "type": "number" + }, + "x": { + "description": "Origin viewport x in CSS pixels.", + "type": "number" + }, + "y": { + "description": "Origin viewport y in CSS pixels.", + "type": "number" + } + }, + "required": [ + "target_id", + "tab_id", + "session", + "action" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "start_recording", + "description": "Start trajectory recording for the calling session. Each action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) from that session writes a turn folder under `output_dir`. Without `session`, every call on the same connection is recorded, including named-session calls. Other connections and session lifecycle calls (`start_session` / `end_session`) are not recorded. CLI recordings started with `cua-driver recording start` are daemon-wide.\n\nEach turn folder holds:\n\n- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI state immediately before and after the action.\n- `before.png` / `after.png` — target-window screenshots immediately before and after the action.\n- `evidence.json` — capture status and a stable classification when an expected artifact could not be captured.\n- `app_state.json` — post-action AX/UIA snapshot for the target pid.\n- `screenshot.png` — compatibility alias of `after.png`.\n- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.\n- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.\n\nThe per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.\n\nTurn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.\n\n**Video is off by default.** Pass `record_video: true` to also capture the main display to `/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.\n\n**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) under the daemon's Screen Recording grant, with no ffmpeg subprocess. Requires macOS 15.0+. On macOS 26 (Tahoe), the first direct capture can also show a one-time consent asking to let Cua Driver bypass the system private window picker and directly access your screen and audio; choose Allow, or run `cua-driver permissions grant` beforehand to answer it up front. The recorder captures screen video only and does not enable system-audio capture.\n\n**Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.\n\nState persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include_accessibility_tree": { + "default": true, + "description": "Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`.", + "type": "boolean" + }, + "output_dir": { + "description": "Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.", + "type": "string" + }, + "record_video": { + "description": "Capture the main display to /recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "state_timeout_ms": { + "default": 1000, + "description": "Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + } + }, + "required": [ + "output_dir" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "stop_recording", + "description": "Stop trajectory recording. Disables further per-turn capture and, when video was enabled, gracefully terminates the ffmpeg subprocess so the mp4's moov atom is finalized (the file is playable). Calling stop on an already-stopped session is a no-op. The response carries `last_video_path` pointing at the finalized mp4 (when video was on).\n\nA manual `stop_recording` is **unconditional** — it stops whatever recording is active regardless of which session started it. Ownership-scoped teardown (so one client disconnecting can't stop a recording a later client started) is handled by the registry's `session_end` lifecycle hook, not by this tool.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_recording_state", + "description": "Report the current trajectory recorder state: whether recording is enabled, the output directory (when enabled), and the 1-based counter for the next turn folder that will be written. Counter increments on every recorded action tool call and resets to 1 each time recording is (re-)enabled.\n\nPure read-only.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "replay_trajectory", + "description": "Replay a recorded trajectory by re-invoking every turn's tool call in lexical order. `dir` must point at a directory previously written by `start_recording`. Each `turn-NNNNN/` is parsed for `action.json`, and the recorded tool is called with its recorded `arguments` via the same dispatch path an MCP / CLI call uses.\n\nCaveats:\n- Element-token actions (`click({pid, element_token})` etc.) will fail because element tokens are per-snapshot and don't survive across sessions. Pixel clicks (`click({pid, x, y})`) and all keyboard tools replay cleanly. Failures are reported but don't stop replay unless `stop_on_error` is true.\n- `get_window_state` and other read-only tools are NOT currently recorded, so replays do not re-populate the per-(pid, window_id) element cache.\n- If recording is ENABLED while replay runs, the replay itself is recorded into the currently configured output directory. That's deliberate: recording a replay against a new build and diffing the two trajectories is the regression-test workflow.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delay_ms": { + "description": "Milliseconds to sleep between turns, for human-observable pacing. Default 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "dir": { + "description": "Trajectory directory previously written by `start_recording`. Absolute or ~-rooted.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "stop_on_error": { + "description": "Stop replay on the first tool-call error. Default true — set false to best-effort through the full trajectory.", + "type": "boolean" + } + }, + "required": [ + "dir" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "install_ffmpeg", + "description": "Install the ffmpeg binary used by start_recording's video capture (Linux/Windows; macOS records natively and needs no ffmpeg). Two-step and confirmed: called without `confirm` it only REPORTS the exact install command for this platform's package manager; pass `confirm: true` to actually run it. No-op if ffmpeg is already on PATH. ffmpeg is run as a separate process, never linked into the driver.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "confirm": { + "description": "Run the install command. Without it, only the planned command is reported.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "start_session", + "description": "Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "capture_scope": { + "description": "Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.", + "enum": [ + "auto", + "window", + "desktop" + ], + "type": "string" + }, + "cursor_motion": { + "additionalProperties": false, + "description": "Optional initial cursor motion (style, timing, effects and tuning). The host applies it\nbefore the cursor is first made visible. A later `set_agent_cursor_motion` call wins;\nthis wins over the saved default (`cursor.motion.*` in the driver config) and the\nbuilt-in `signature_arc`. Reduced motion always wins.", + "properties": { + "arc_flow": { + "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", + "type": "number" + }, + "arc_size": { + "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", + "type": "number" + }, + "dwell_after_click_ms": { + "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", + "type": "number" + }, + "effects": { + "additionalProperties": false, + "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", + "properties": { + "glow": { + "description": "Soft glow around the cursor that grows with speed.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "magnet": { + "description": "Target glow when the `magnetic` style locks on.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "ripple": { + "description": "Ring that expands from the hotspot on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "squish": { + "description": "Brief scale-down of the cursor on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "trail": { + "description": "Short fading trail behind the cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "end_handle": { + "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "glide_duration_ms": { + "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", + "type": "number" + }, + "idle_hide_ms": { + "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", + "type": "number" + }, + "spring": { + "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", + "type": "number" + }, + "start_handle": { + "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "style": { + "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "cursor_theme": { + "additionalProperties": true, + "description": "Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.", + "properties": { + "reduced_motion": { + "default": "auto", + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "theme_id": { + "type": "string" + } + }, + "required": [ + "theme_id" + ], + "type": "object" + }, + "session": { + "description": "Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "escalate_session", + "description": "Deprecated compatibility tool for legacy capture-scope sessions. New callers select window or desktop modality on each action. No deescalate_session tool exists.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "detail": { + "description": "Optional bounded diagnostic detail. Never use secrets or page content.", + "maxLength": 200, + "type": "string" + }, + "reason": { + "description": "Why the window-scoped attempt failed and desktop capture is needed.", + "enum": [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other" + ], + "type": "string" + }, + "session": { + "description": "Public label of the legacy capture-scope session to escalate.", + "type": "string" + } + }, + "required": [ + "session", + "reason" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }, + { + "name": "get_session", + "description": "Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "list_sessions", + "description": "List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "cursor": { + "description": "Opaque continuation cursor returned by a previous call.", + "type": "string" + }, + "limit": { + "description": "Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.", + "minimum": 0, + "type": "integer" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_session_state", + "description": "Deprecated compatibility alias that reads a live legacy session's capture policy. Use get_session for lifecycle state.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "end_session", + "description": "End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label to end. When omitted, end the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "get_guide", + "description": "Read Cua Driver guidance on demand, taken from the guide bundled with this driver (no skill install needed). Topics: overview, drag, browser, permissions, batching, menus. Call with no topic to list them. Read-only.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "topic": { + "description": "One of: overview, drag, browser, permissions, batching, menus. Omit to list the topics.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }, + { + "name": "run_steps", + "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Default app (name or bundle id) for steps that name no window.", + "type": "string" + }, + "delay_ms": { + "description": "Pause between steps in milliseconds (not after the last). Default 0.", + "maximum": 2000, + "minimum": 0, + "type": "integer" + }, + "delivery_mode": { + "description": "Default delivery_mode for steps whose tool takes one and that set none.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "foreground_fallback": { + "description": "When a step is refused because its target ignores background input (Electron, LibreOffice and other toolkits), retry it once with delivery_mode:\"foreground\": the window is activated and the pointer may move. Default false.", + "type": "boolean" + }, + "observe": { + "description": "Optional end-of-batch observation: arguments for ONE get_window_state call. `pid` and `window_id` default to those of the last step that names both. Defaults to since=\"latest\" (only what changed since your last read of that window with the same query/max_elements/max_depth; a full read when there is none), include_screenshot=false and max_elements=250; pass include_screenshot=true to see the window, or since=null for a full read. Omit to read nothing. {} observes the last window.", + "type": "object" + }, + "pid": { + "description": "Default pid for steps that name no window (a step's own pid/window_id/app/window wins).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "steps": { + "description": "Ordered actions. Execution stops at the first failing step. Each step is {tool, args} or {: args}, plus optional wait_for / expect / timeout_ms; a step may also be only a wait_for or expect check.", + "items": { + "additionalProperties": true, + "properties": { + "args": { + "description": "Arguments for that tool, as in a direct call, optionally with role/name/nth/app/window to name the target instead of pid/window_id/element_token.", + "type": "object" + }, + "expect": { + "description": "Up to 4 checks that must hold after the action: {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}. Default timeout 2000 ms.", + "items": { + "type": "object" + }, + "maxItems": 4, + "type": "array" + }, + "timeout_ms": { + "description": "How long a named target may take to appear. Default 3000.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "tool": { + "description": "Action tool to run.", + "enum": [ + "click", + "double_click", + "right_click", + "set_value", + "type_text", + "press_key", + "hotkey", + "scroll", + "drag", + "move_cursor" + ], + "type": "string" + }, + "wait_for": { + "description": "Check that must hold before the action: {role?, name?, text?, gone?, app?, window?, timeout_ms?}. Default timeout 5000 ms.", + "type": "object" + } + }, + "type": "object" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "window": { + "description": "Default window title substring for steps that name no window.", + "type": "string" + }, + "window_id": { + "description": "Default window_id for steps that name no window, with `pid`.", + "type": "integer" + } + }, + "required": [ + "steps" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "run_script", + "description": "Run a JavaScript program that drives apps through Cua Driver, in ONE call: loops, conditions, reads and checks between actions, and only the result comes back. Use it when the flow needs logic (a loop over rows, a branch on what the app shows, a retry); use run_steps for a fixed list of steps.\n\nThe script is the body of an async function: use `await`, and `return` a JSON value. It runs in a sandbox with no filesystem, network, process, timers or environment; the only API is `cua`:\n- `await cua.getApp(\"TextEdit\" | \"com.apple.TextEdit\" | {pid, windowId})` → app (follows the app's frontmost window unless a windowId is given)\n- `await cua.launch(nameOrBundleId)` → app; `cua.listApps()`, `cua.listWindows({pid?})`, `cua.sleep(ms)`, `cua.computer.target`\n- app.getState(opts?) → get_window_state result (`tree_markdown` or `tree_diff`, `snapshot_id`; pass get_window_state options such as {since:\"latest\"})\n- app.query({role?, name?, text?}) → matching elements across the app's windows {element_token, role, label, value, enabled, selected, frame}, then matching static-text rows {role, label, value, text, display_only} (read text such as a status line from their `value`)\n- app.waitFor({role?, name?, text?, gone?, value?, timeout_ms?}) and app.verify(predicate | predicates) (verify_state)\n- app.click(t), doubleClick(t), rightClick(t), typeText(text, t?), pressKey(\"return\" | \"cmd+s\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, direction, amount?), setValue(t, value), drag([x,y], [x,y]), zoom([x1,y1,x2,y2])\nwhere a target `t` is {role, name, nth?} (found fresh, waits up to 3 s), an element_token string, a row number from this app's last getState(), or [x, y] window pixels.\n- `cua.call(tool, args)` for any other allowed tool: click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, launch_app, list_apps, list_windows, get_window_state, verify_state, zoom.\nA failed driver call throws a CuaError (catch it to recover). Every call goes through the same session, permission and approval checks as a direct call.\n\nLimits: `timeout_ms` wall time (default 30000, max 120000), `max_calls` driver calls (default 100, max 500), 32 MiB heap. Returns the return value, console output, and a per-call log; an error names the script line, the call and the reason.\n\nExample:\nconst app = await cua.getApp(\"Calculator\");\nfor (const key of [\"7\", \"multiply\", \"6\", \"equals\"]) await app.click({role: \"button\", name: key});\nconst [display] = await app.query({role: \"statictext\"});\nreturn display.value; Batching rules: `get_guide({topic:\"batching\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "max_calls": { + "description": "Most driver calls the script may make. Default 100.", + "maximum": 500, + "minimum": 1, + "type": "integer" + }, + "script": { + "description": "JavaScript: the body of an async function. `return` the result.", + "maxLength": 65536, + "minLength": 1, + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "timeout_ms": { + "description": "Wall-time limit. Default 30000.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + } + }, + "required": [ + "script" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "check_for_update", + "description": "Check the saved stable/nightly Cua Driver channel for a release on GitHub. Returns current and selected channels, current and latest versions, an `update_available` boolean, the install one-liner, and the release notes URL. Read-only — never installs. Pacman-owned Linux executables return package-manager guidance without checking GitHub. Mirror of `cua-driver check-update --json`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": true + } + }, + { + "name": "install_extension", + "description": "Preview or install one Driver-managed optional extension. The first call without confirm returns the exact signed artifact, destination, license, source, and trust plan without mutation. Re-call with confirm=true to perform that exact verified installation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "confirm": { + "description": "Install the previewed extension. Omit or false for a read-only plan.", + "type": "boolean" + }, + "name": { + "description": "Extension to preview or install. Only perception (the local visual-region parser used by parse_visual_regions) is available.", + "enum": [ + "perception" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + } + }, + { + "name": "parse_visual_regions", + "description": "Parse one immutable registered capture into model-neutral text and icon regions.", + "inputSchema": { + "additionalProperties": false, + "description": "Parse one immutable screenshot capture. Pass the `capture_id` returned by\n`get_window_state` or `get_desktop_state`; the Driver capture registry\nresolves it to the exact pixels and action-coordinate transform.", + "properties": { + "capture_id": { + "description": "`capture_id` of a screenshot returned by get_window_state or get_desktop_state.", + "minLength": 1, + "type": "string" + }, + "options": { + "additionalProperties": false, + "default": {}, + "description": "Optional, model-neutral controls for one bounded parse.", + "properties": { + "kinds": { + "description": "Region kinds to return (`text`, `icon`). Omit for all kinds.", + "items": { + "enum": [ + "text", + "icon" + ], + "type": "string" + }, + "maxItems": 2, + "minItems": 1, + "type": "array" + }, + "max_regions": { + "description": "Return at most this many regions.", + "minimum": 1, + "type": "integer" + }, + "min_confidence": { + "description": "Drop regions below this confidence (0 to 1).", + "maximum": 1, + "minimum": 0, + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "capture_id" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + } +] diff --git a/src/agent_env/task_step/task_steps/add_skills.py b/src/agent_env/task_step/task_steps/add_skills.py index 2a44c5b98..5437548f8 100644 --- a/src/agent_env/task_step/task_steps/add_skills.py +++ b/src/agent_env/task_step/task_steps/add_skills.py @@ -2,7 +2,9 @@ from __future__ import annotations +import asyncio import hashlib +import json import logging import re from dataclasses import dataclass @@ -11,6 +13,7 @@ from agentenv_protocol import SKILL_EXTENSION_URI from agentenv_protocol.client import find_extension_method +from agent_env.config import get_config from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope @@ -21,6 +24,9 @@ #: ``context.metadata[ENV_SKILLS_KEY][agent_name][env_id][skill_name]``: the SKILL.md add_skills installed from an env, as #: the env, or one of its child envs, wrote it for that deploy. ENV_SKILLS_KEY = "env_skills" +#: Where an env's skill with files beside its SKILL.md is stored, under the artifact key prefix, by a digest of its +#: files, to be sent to the agent as a skill bundle. +_ENV_SKILL_PREFIX = "agentenv-env-skills" @dataclass @@ -291,8 +297,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.info(f"Registered skill: {result}") for env_id, skills in env_skills.items(): for skill in skills: + files = skill.get("files") + sent = ({"object_url": await asyncio.to_thread(_stored_skill, skill["skill_md"], files)} if files else + {"skill_md": skill["skill_md"]}) result = await A2AAgent.register_skill(deployed, name=skill["name"], description=skill["description"], - skill_md=skill["skill_md"]) + **sent) installed = context.metadata.setdefault(ENV_SKILLS_KEY, {}).setdefault(self.agent_name, {}) installed.setdefault(env_id, {})[skill["name"]] = skill["skill_md"] logger.info(f"Registered the skill {skill['name']} of env {env_id}: {result}") @@ -300,6 +309,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context +def _stored_skill(skill_md: str, files: dict[str, str]) -> str: + """The object-store prefix holding an env skill's SKILL.md and its other files, one prefix per content.""" + config = get_config() + store, contents = config.get_object_store(), {"SKILL.md": skill_md, **files} + digest = hashlib.sha256(json.dumps(contents, sort_keys=True).encode()).hexdigest() + prefix = f"{config.get_artifact_key_prefix()}{_ENV_SKILL_PREFIX}/{digest}" + for path, text in contents.items(): + store.put(f"{prefix}/{path}", text.encode(), content_type="text/markdown", allow_overwrite=True) + return store.object_url(prefix) + + async def _env_skills(context: TaskStepContext, agent: DeployedAgent, env_id: str) -> list[dict]: """The skills the env ``env_id`` serves, each under its card's environment name: its own card's, or else each of its child envs' (a gateway's servers); refused for an env this run didn't deploy or bind to ``agent``, or that serves diff --git a/tst/integration/computer/cua-driver/envs/desktop/env.toml b/tst/integration/computer/cua-driver/envs/desktop/env.toml new file mode 100644 index 000000000..4b8291877 --- /dev/null +++ b/tst/integration/computer/cua-driver/envs/desktop/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "cua-driver" diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 315f76591..070250e5b 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -52,12 +52,13 @@ def test_the_shipped_aliases_are_agent_envs_alone_and_claude_code_names_the_defa -def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_and_the_reference_driver(): +def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_and_the_drivers(): table = installed_aliases(tuple(_shipped().values()), ENV_ALIASES) assert table.refused == {} and table.refused_claims == {} assert {alias: named.target_id for alias, named in table.aliases.items()} == { - "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference"} + "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference", + "cua-driver": "@local/agentenv-framework/drivers/cua-driver"} assert A2AAgent.VALIDATION_ENV_ID == "mail" def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py new file mode 100644 index 000000000..6731c17a5 --- /dev/null +++ b/tst/unit/drivers/cua_driver_test.py @@ -0,0 +1,254 @@ +"""The cua-driver driver against a fake cua-driver daemon on a Unix socket: the computer/v1 operations become its +desktop-scope calls in the framework's session, its own tools are listed as pinned and forwarded on their caller's +connection as they came, readiness checks the pinned executable and tools, and the skill leads with cua-driver's pack.""" +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import importlib.util +import io +import json +import stat +import tempfile +from pathlib import Path + +import httpx +import pytest +import pytest_asyncio +from mcp.shared.memory import create_connected_server_and_client_session +from PIL import Image as PILImage + +import agent_env +from agentenv_protocol.computer import DeviceUnavailable, Unsupported + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" + + +def _png(width: int, height: int) -> str: + out = io.BytesIO() + PILImage.new("RGB", (width, height), (255, 0, 0)).save(out, "PNG") + return base64.b64encode(out.getvalue()).decode() + + +class FakeDaemon: + """cua-driver's daemon as the driver sees it, on ``path``: each request by the connection it came on, ``refuse`` + naming tools whose results are errors, ``fail`` tools it can't call, and ``tools`` what ``list`` lists.""" + + def __init__(self, manifest: list[dict], path: Path) -> None: + self.path = path + self.requests: list[tuple[int, dict]] = [] + self.refuse: set[str] = set() + self.fail: set[str] = set() + self.tools = [{"name": t["name"], "input_schema": t["inputSchema"]} for t in manifest] + self.connections = 0 + self.writers: list[asyncio.StreamWriter] = [] + + async def start(self) -> None: + self.server = await asyncio.start_unix_server(self.serve, path=str(self.path)) + + async def stop(self) -> None: + """As the daemon dying: its socket gone and every connection closed.""" + self.server.close() + for writer in self.writers: + writer.close() + await self.server.wait_closed() + self.path.unlink(missing_ok=True) + + async def serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self.connections += 1 + connection = self.connections + self.writers.append(writer) + while line := await reader.readline(): + request = json.loads(line) + self.requests.append((connection, request)) + writer.write((json.dumps(self.reply(request)) + "\n").encode()) + await writer.drain() + writer.close() + + def reply(self, request: dict) -> dict: + if request["method"] == "list": + return {"ok": True, "result": {"tools": self.tools}} + name = request["name"] + if name in self.fail: + return {"ok": False, "error": f"unknown tool {name}", "exit_code": 1} + if name in self.refuse: + return {"ok": True, "result": {"isError": True, "content": [{"type": "text", "text": f"{name} refused"}]}} + if name == "get_screen_size": + return {"ok": True, "result": {"content": [{"type": "text", "text": "1280x800"}], + "structuredContent": {"width": 1280, "height": 800, "scale_factor": 1.0}}} + if name == "get_desktop_state": + return {"ok": True, "result": { + "content": [{"type": "image", "data": _png(1280, 800), "mimeType": "image/png"}], + "structuredContent": {"screenshot_width": 1280, "screenshot_height": 800}}} + return {"ok": True, "result": {"content": [{"type": "text", "text": f"did {name}"}], + "structuredContent": {"tool": name}}} + + def calls(self, connection: int | None = None) -> list[tuple[str, dict]]: + return [(r["name"], r["args"]) for c, r in self.requests if r["method"] == "call" + and (connection is None or c == connection)] + + +@pytest.fixture +def module(monkeypatch): + monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + spec = importlib.util.spec_from_file_location("cua_driver_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +@pytest_asyncio.fixture +async def daemon(module, monkeypatch): + """A fake daemon on a socket the driver is pointed at, under a short /tmp path for macOS's socket path limit.""" + with tempfile.TemporaryDirectory(dir="/tmp", prefix="cd-") as folder: + fake = FakeDaemon(module._manifest(), Path(folder) / "cua-driver.sock") + monkeypatch.setattr(module, "SOCKET", fake.path) + await fake.start() + yield fake + await fake.stop() + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +@pytest.mark.asyncio +async def test_the_operations_become_desktop_scope_calls_in_the_frameworks_session(module, daemon): + driver = module.CuaDriver() + + size, shot = await driver.screen(), await driver.grab() + await driver.click_at(100, 200, "right", 2, ["Control"]) + await driver.drag_along([(10, 20), (30, 40)], "left", 300, ["Shift"]) + await driver.scroll_at(5, 6, "down", 3, []) + await driver.type_in("Hello, Wörld! 東京") + await driver.press_keys(["Control", "/"]) + await driver.press_keys(["Enter"]) + + assert (size.width, size.height, shot.width, shot.height, shot.mode) == (1280, 800, 1280, 800, "png") + framework = {"session": "agentenv"} + assert daemon.calls() == [ + ("get_screen_size", framework), ("get_desktop_state", framework), + ("click", {"scope": "desktop", "x": 100, "y": 200, "button": "right", "count": 2, "modifier": ["ctrl"], **framework}), + ("drag", {"scope": "desktop", "from_x": 10, "from_y": 20, "to_x": 30, "to_y": 40, "button": "left", + "duration_ms": 300, "modifier": ["shift"], **framework}), + ("scroll", {"scope": "desktop", "x": 5, "y": 6, "direction": "down", "amount": 3, **framework}), + ("type_text", {"scope": "desktop", "text": "Hello, Wörld! 東京", **framework}), + ("press_key", {"scope": "desktop", "key": "/", "modifiers": ["ctrl"], **framework}), + ("press_key", {"scope": "desktop", "key": "enter", "modifiers": [], **framework}), + ] + assert daemon.connections == 1 + + +@pytest.mark.asyncio +async def test_what_cua_driver_cant_do_or_refuses_fails_naming_it(module, daemon): + driver = module.CuaDriver() + daemon.refuse.add("click") + + with pytest.raises(Unsupported, match="holds only modifier keys"): + await driver.press_keys(["Control", "a", "b"]) + with pytest.raises(Unsupported, match="no keys held"): + await driver.scroll_at(1, 1, "up", 1, ["Shift"]) + with pytest.raises(Exception, match="cua-driver click: click refused") as refused: + await driver.click_at(1, 1, "left", 1, []) + assert type(refused.value).__name__ == "DeliveryFailed" + + +@pytest.mark.asyncio +async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_on_their_callers_connection(module, daemon): + driver = module.CuaDriver() + async with mcp_session(driver) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + await session.call_tool("computer_screenshot", {}) + answered = await session.call_tool("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}) + + manifest = {t["name"]: t for t in module._manifest()} + assert set(manifest) <= set(listed) and {"computer_click", "computer_screenshot"} <= set(listed) + assert listed["get_window_state"].inputSchema == manifest["get_window_state"]["inputSchema"] + assert listed["get_window_state"].annotations.readOnlyHint == manifest["get_window_state"]["annotations"]["readOnlyHint"] + framework, caller = 1, max(c for c, _ in daemon.requests) + assert caller != framework and daemon.calls(caller) == [("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"})] + assert answered.structuredContent == {"tool": "get_window_state"} and answered.content[0].text == "did get_window_state" + + +@pytest.mark.asyncio +async def test_a_failed_call_to_one_of_cua_drivers_tools_reaches_its_caller_as_an_error(module, daemon): + daemon.fail.add("list_apps") + async with mcp_session(module.CuaDriver()) as session: + failed = await session.call_tool("list_apps", {}) + await daemon.stop() + down = await session.call_tool("list_windows", {}) + assert failed.isError and failed.content[0].text == "unknown tool list_apps" + assert down.isError and "cua-driver's daemon isn't answering" in down.content[0].text + + +@pytest.mark.asyncio +async def test_readiness_checks_the_pinned_executable_and_tools_until_they_pass_once(module, daemon, monkeypatch): + driver = module.CuaDriver() + pinned = module.EXECUTABLES[driver.platform.arch] + digests = iter(["0" * 64, pinned]) + monkeypatch.setattr(module, "_executable_sha256", lambda: next(digests)) + daemon.tools[0] = {**daemon.tools[0], "input_schema": {"type": "object"}} + + wrong = await driver.ready_checks() + daemon.tools[0] = {"name": daemon.tools[0]["name"], "input_schema": module._manifest()[0]["inputSchema"]} + right = await driver.ready_checks() + lists = sum(1 for _, r in daemon.requests if r["method"] == "list") + again = await driver.ready_checks() + + assert [(c.name, c.ok) for c in wrong] == [("cua-driver", False), ("cua-driver tools", False)] + assert wrong[0].detail.startswith("its executable's sha256 is 0000") and module._manifest()[0]["name"] in wrong[1].detail + assert [(c.name, c.ok) for c in right] == [("cua-driver", True), ("cua-driver tools", True)] + assert again == right and sum(1 for _, r in daemon.requests if r["method"] == "list") == lists + + +@pytest.mark.asyncio +async def test_readiness_waits_for_a_daemon_that_isnt_answering(module, monkeypatch): + monkeypatch.setattr(module, "SOCKET", Path("/tmp/cd-nothing-here.sock")) + assert [(c.name, c.ok, c.detail) for c in await module.CuaDriver().ready_checks()] == [ + ("cua-driver", False, "its daemon isn't answering yet")] + + +@pytest.mark.asyncio +async def test_a_dropped_daemon_connection_is_device_unavailable_and_reopened(module, daemon): + driver = module.CuaDriver() + await driver.screen() + await daemon.stop() + + with pytest.raises(DeviceUnavailable, match="cua-driver's daemon isn't answering"): + await driver.screen() + await daemon.start() + assert (await driver.screen()).width == 1280 and daemon.connections == 2 + + +def test_the_skill_pack_is_written_offline_from_the_executable(module, monkeypatch, tmp_path): + executable = tmp_path / "cua-driver" + executable.write_text("""#!/bin/sh +[ "$1 $2 $3" = "mcp-config --client claude" ] && [ "$DO_NOT_TRACK" = 1 ] && [ -d "$HOME/.claude" ] || exit 9 +mkdir -p "$HOME/.cua-driver/skills/cua-driver" +printf -- '---\\nname: cua-driver\\ndescription: Drive a native GUI app.\\nversion: 0.38.0\\n---\\n\\n# cua-driver\\n\\nSee LINUX.md.\\n' > "$HOME/.cua-driver/skills/cua-driver/SKILL.md" +printf '# Linux\\n' > "$HOME/.cua-driver/skills/cua-driver/LINUX.md" +""") + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + monkeypatch.setattr(module, "_executable", lambda: executable) + + assert module._skill_pack() == ("Drive a native GUI app.", "# cua-driver\n\nSee LINUX.md.\n", {"LINUX.md": "# Linux\n"}) + + +@pytest.mark.asyncio +async def test_the_computer_skill_leads_with_cua_drivers_pack(module, daemon, monkeypatch): + calls = [] + monkeypatch.setattr(module, "_skill_pack", lambda: calls.append(1) or ( + "Drive a native GUI app.", "# cua-driver\n", {"LINUX.md": "# Linux\n"})) + driver = module.CuaDriver() + asgi = driver.create_app().streamable_http_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url="http://driver") as http: + first = (await http.get("/agentenv/ext/skill/get", params={"name": "desktop"})).json() + again = (await http.get("/agentenv/ext/skill/get", params={"name": "desktop"})).json() + + assert first == again and calls == [1] and first["files"] == {"LINUX.md": "# Linux\n"} + assert first["skill_md"].partition("---\n\n")[2].startswith("# cua-driver\n\n# This computer\n") + assert first["description"].startswith("Drive a native GUI app. This computer: ") diff --git a/tst/unit/task_step/add_skills_env_test.py b/tst/unit/task_step/add_skills_env_test.py index 0ba651fe0..1d741dd02 100644 --- a/tst/unit/task_step/add_skills_env_test.py +++ b/tst/unit/task_step/add_skills_env_test.py @@ -1,6 +1,6 @@ """``add_skills`` ``env_ids``: each env's skill, or each of its child envs' behind a gateway, written under that card's -environment name, installed on the agent and recorded in the run; refused for an env the run didn't deploy or bind to -the agent, or that serves no skill.""" +environment name, installed on the agent and recorded in the run, a skill with files beside its SKILL.md sent as a +stored bundle; refused for an env the run didn't deploy or bind to the agent, or that serves no skill.""" import hashlib from types import SimpleNamespace @@ -64,6 +64,24 @@ async def test_each_envs_skill_is_installed_under_its_environment_name_and_recor assert context.metadata[ENV_SKILLS_KEY] == {"solver": {"shop": {"my-shop": "---\nname: my-shop\n"}}} +@pytest.mark.asyncio +async def test_a_skill_with_files_is_stored_once_and_installed_as_a_bundle(agent, local_stores, monkeypatch): + files = {"LINUX.md": "# Linux\n", "WORKFLOW.md": "# Workflow\n"} + + async def invoke(base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + return {"name": params["name"], "description": "Drive the desktop", "skill_md": "# Skill\n", "files": files} + + monkeypatch.setattr(env_module.protocol_v1, "invoke_extension", invoke) + await _step("shop").execute(_context(_env())) + await _step("shop").execute(_context(_env())) + + (first, again) = [call.kwargs for call in agent.register.await_args_list] + store = local_stores.get_object_store() + assert first == again and set(first) == {"name", "description", "object_url"} + assert sorted(store.get_object_key(url).rpartition("/")[2] for url in store.list_at(first["object_url"])) == [ + "LINUX.md", "SKILL.md", "WORKFLOW.md"] + + @pytest.mark.asyncio async def test_an_env_behind_a_gateway_installs_each_child_envs_skill_under_the_childs_name(agent): gateway = _env("office", name="office", extensions=(), children=(("mail", (_SKILL,)), ("Chat Room", (_SKILL,)), From 6a6a9ae6faa943426b9bcbf3abaa65eb5a0b9864 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:11:51 -0700 Subject: [PATCH 02/13] test(computer): a task for the cua-driver test bundle Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/cua-driver/tasks/desktop.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tst/integration/computer/cua-driver/tasks/desktop.json diff --git a/tst/integration/computer/cua-driver/tasks/desktop.json b/tst/integration/computer/cua-driver/tasks/desktop.json new file mode 100644 index 000000000..67f3b3154 --- /dev/null +++ b/tst/integration/computer/cua-driver/tasks/desktop.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"} +] From 8b108c85a8ff51f30260244f7091545667d5d856 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:14:13 -0700 Subject: [PATCH 03/13] fix(computer): forward a cua-driver tool's argument named name Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/envs/cua-driver/driver.py | 6 +++--- tst/unit/drivers/cua_driver_test.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 3f9519a27..73920a74f 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -134,15 +134,15 @@ async def press_keys(self, keys) -> None: raise Unsupported(f"cua-driver holds only modifier keys in a chord, not {[k for k in held if k not in MODIFIERS]}") await self._send("press_key", key=MODIFIERS.get(key) or NAMED.get(key, key), modifiers=_modifiers(held)) - async def _forward(self, name: str, **arguments: Any) -> CallToolResult: + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: """A call to one of cua-driver's own tools, on its caller's connection, answered as cua-driver answers it.""" connection = self._callers.setdefault(self.caller_identity(ToolContext.current()), _Connection()) try: - reply = await connection.request({"method": "call", "name": name, "args": arguments}) + reply = await connection.request({"method": "call", "name": tool, "args": arguments}) except DeviceUnavailable as e: return _refused(e.message) if not reply.get("ok"): - return _refused(reply.get("error") or f"cua-driver refused {name}") + return _refused(reply.get("error") or f"cua-driver refused {tool}") return CallToolResult.model_validate(reply["result"]) async def _send(self, name: str, **args: Any) -> None: diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index 6731c17a5..7f1ac3314 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -164,13 +164,16 @@ async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_on_their_cal listed = {t.name: t for t in (await session.list_tools()).tools} await session.call_tool("computer_screenshot", {}) answered = await session.call_tool("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}) + launched = await session.call_tool("launch_app", {"name": "thunar"}) manifest = {t["name"]: t for t in module._manifest()} assert set(manifest) <= set(listed) and {"computer_click", "computer_screenshot"} <= set(listed) assert listed["get_window_state"].inputSchema == manifest["get_window_state"]["inputSchema"] assert listed["get_window_state"].annotations.readOnlyHint == manifest["get_window_state"]["annotations"]["readOnlyHint"] framework, caller = 1, max(c for c, _ in daemon.requests) - assert caller != framework and daemon.calls(caller) == [("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"})] + assert caller != framework and daemon.calls(caller) == [ + ("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}), ("launch_app", {"name": "thunar"})] + assert not launched.isError assert answered.structuredContent == {"tool": "get_window_state"} and answered.content[0].text == "did get_window_state" From 3f80135c71c2a2f2667b661d8f9badf583036b5d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:15:38 -0700 Subject: [PATCH 04/13] fix(computer): cua-driver's daemon starts in the session user's home Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/envs/cua-driver/daemon.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/agent_env/drivers/envs/cua-driver/daemon.sh b/src/agent_env/drivers/envs/cua-driver/daemon.sh index 11f478c23..6be1a0447 100644 --- a/src/agent_env/drivers/envs/cua-driver/daemon.sh +++ b/src/agent_env/drivers/envs/cua-driver/daemon.sh @@ -1,8 +1,11 @@ #!/bin/sh -# cua-driver's daemon, once X answers: on the socket the driver reaches it at, with approvals off, since nobody is at -# the computer to give them, and nothing sent to trycua. +# cua-driver's daemon, once X answers: from the session user's home, where the apps it launches start too, on the +# socket the driver reaches it at, with approvals off, since nobody is at the computer to give them, and nothing sent +# to trycua. set -eu export DO_NOT_TRACK=1 CUA_DRIVER_RS_TELEMETRY_ENABLED=false CUA_DRIVER_RS_UPDATE_CHECK=false +venv="$(cd "$(dirname "$0")" && pwd)/venv" display="${DISPLAY#:}" while [ ! -S "/tmp/.X11-unix/X${display%%.*}" ]; do sleep 0.2; done -exec venv/bin/cua-driver serve --socket /run/agentenv/cua-driver.sock --dangerously-bypass-approvals --no-overlay +cd "$HOME" +exec "$venv/bin/cua-driver" serve --socket /run/agentenv/cua-driver.sock --dangerously-bypass-approvals --no-overlay From 00c4519047eba054687532bd435926952bb31e31 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:19:51 -0700 Subject: [PATCH 05/13] test(computer): the cua-driver driver end to end, and one line for printable keys in the skill Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/skill.py | 10 ++- .../tests/test_computer_driver.py | 3 + tst/integration/computer/conftest.py | 21 +++-- tst/integration/computer/cua_driver_test.py | 84 +++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tst/integration/computer/cua_driver_test.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py index 416499ca6..829bffe56 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py @@ -23,6 +23,7 @@ """, } _SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"} +_PRINTABLE = tuple(chr(code) for code in range(0x21, 0x7F)) async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, str]]: @@ -61,7 +62,7 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, def _input(driver: ComputerDriverEnvironment) -> str: keys = ("any W3C key name, such as `'Enter'`, `'Control'`, `'ArrowLeft'` or `'a'`" if driver.keys == "all" else - "only " + ", ".join(f"`{key!r}`" for key in driver.keys)) + _keys(driver.keys)) space, scale = driver.settings.default_space, driver.settings.default_norm_scale points = ("pixels of the latest screenshot" if space == "frame" else f"0 to {scale} across and down the latest screenshot") @@ -73,3 +74,10 @@ def _input(driver: ComputerDriverEnvironment) -> str: f"- Points are {points}, unless a call names another space.", "- Input is foreground: it goes to the focused window and moves the pointer, as a person's would.", ]) + + +def _keys(keys) -> str: + """The keys a driver takes, every printable ASCII character said once when it takes them all.""" + every = set(_PRINTABLE) <= set(keys) + named = [key for key in keys if not (every and key in _PRINTABLE)] + return "only " + ", ".join(f"`{key!r}`" for key in named) + (", and any printable ASCII character" if every else "") diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index 9c6e0874e..c0196ffad 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -204,6 +204,8 @@ async def test_the_skill_is_written_from_the_computer_as_it_booted_under_the_nam @pytest.mark.asyncio async def test_a_driver_whose_software_ships_a_skill_leads_with_it_and_the_computers_part_follows(monkeypatch): class Bundling(FakeDriver): + keys = ("Enter", "Escape", *(chr(code) for code in range(0x21, 0x7F))) + async def bundled_skill(self): return "Drive any app through its own tools.", "# Desktop driver\n\nRead LINUX.md first.\n", { "LINUX.md": "# Linux\n"} @@ -213,6 +215,7 @@ async def bundled_skill(self): body = answer["skill_md"].partition("---\n\n")[2] assert body.startswith("# Desktop driver\n\nRead LINUX.md first.\n\n# This computer\n\nBeside the tools above") assert "## Input" in body and "# Using this computer" not in body and answer["files"] == {"LINUX.md": "# Linux\n"} + assert "- Keys: only `'Enter'`, `'Escape'`, and any printable ASCII character." in body assert answer["description"].startswith("Drive any app through its own tools. This computer: Ubuntu 24.04 (amd64)") diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 64b9ac3e2..4bfef907c 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,6 +1,6 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, composed once for a module's tests; ``state``, local stores for a test that -runs a bundle; and ``docker``, which runs the docker CLI for them.""" +shipped desktop example's computer env, and the cua-driver test bundle's, each composed once for a module's tests; +``state``, local stores for a test that runs a bundle; and ``docker``, which runs the docker CLI for them.""" import logging import re @@ -24,6 +24,7 @@ from agent_env.task.store import get_task_instance_store BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" +CUA_DRIVER = Path(__file__).parent / "cua-driver" @pytest.fixture(scope="session") @@ -38,7 +39,18 @@ def base(): def desktop(tmp_path_factory): """The shipped desktop example's computer env, composed for the module in stores of the module's own: an image for each arch this machine's Docker builds, the native one probed by gate 3.""" - root = tmp_path_factory.mktemp("desktop") + bundle = find_bundle("desktop") + yield from _composed(tmp_path_factory.mktemp("desktop"), parse_bundle(bundle.root, id_root=bundle.id_root)) + + +@pytest.fixture(scope="module") +def cua_desktop(tmp_path_factory): + """The cua-driver test bundle's desktop, the ubuntu-24.04 base with the cua-driver driver, composed as ``desktop`` + is.""" + yield from _composed(tmp_path_factory.mktemp("cua-desktop"), parse_bundle(CUA_DRIVER)) + + +def _composed(root: Path, bundle): patch = pytest.MonkeyPatch() patch.chdir(root) patch.setenv("XDG_STATE_HOME", str(root / "state")) @@ -46,9 +58,8 @@ def desktop(tmp_path_factory): patch.setenv("AGENT_ENV_OBJECT_STORE", "local") configure() reset_artifact_store() - bundle = find_bundle("desktop") with namespace_routing(): - plan = plan_bundle(resolve_bundle(parse_bundle(bundle.root, id_root=bundle.id_root)), installed=installed_bundles) + plan = plan_bundle(resolve_bundle(bundle), installed=installed_bundles) materialize(plan, build_here=frozenset(w.id for w in plan.writes if isinstance(w.source, BuiltImage))) yield Env.get(f"{bundle.id_root}/desktop") reset_artifact_store() diff --git a/tst/integration/computer/cua_driver_test.py b/tst/integration/computer/cua_driver_test.py new file mode 100644 index 000000000..e6a78dc20 --- /dev/null +++ b/tst/integration/computer/cua_driver_test.py @@ -0,0 +1,84 @@ +"""The cua-driver driver on the ubuntu-24.04 base, as the cua-driver test bundle's desktop names it: its put composes +an image for each arch this machine's Docker builds, gate 3 makes this machine's ready at full with cua-driver's own tools +beside the computer_* tools; deployed, an agent's session drives Thunar by element token through cua-driver's tools, +computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local +registry on localhost:5000. It runs cua-driver, so it runs on the Linux test VMs.""" + +import asyncio +import json +import platform +import re +from pathlib import Path + +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +import agent_env +from agentenv_protocol import SKILL_EXTENSION_URI +from agentenv_protocol.computer import COMPUTER_READY + +from agent_env.env import Env +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] +MANIFEST = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "tools.json" +COMPUTER_TOOLS = {"computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", + "computer_key", "computer_wait"} + + +def test_gate_3_makes_this_machines_arch_ready_with_cua_drivers_tools_beside_the_computers(cua_desktop): + results = cua_desktop.metadata["composed"]["container"] + pinned = {tool["name"] for tool in json.loads(MANIFEST.read_text())} + + assert results[ARCH]["status"] == "ready" and results[ARCH]["emulated"] is False + assert results[ARCH]["probe"]["passed"] and results[ARCH]["probe"]["level"] == "full" + assert results[ARCH]["installed"]["cua-driver"] == "0.38.0" + assert {tool["name"] for tool in results[ARCH]["card"]["capabilities"]["tools"]} == pinned | COMPUTER_TOOLS + for arch, result in results.items(): + if arch != ARCH: + assert result["status"] == "unverified" or "its Docker has no emulation for it" in result["reason"], result + + +@pytest.mark.asyncio +async def test_an_agent_drives_thunar_by_element_token_and_its_skill_leads_with_cua_drivers_pack(cua_desktop): + with namespace_routing(): + deployed = await Env.get(cua_desktop.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + launched = await session.call_tool("launch_app", {"name": "thunar"}) + window = await _window(session, "user - Thunar") + state = await session.call_tool("get_window_state", {"pid": window["pid"], + "window_id": window["window_id"]}) + tree, snapshot = state.structuredContent["tree_markdown"], state.structuredContent["snapshot_id"] + row = re.search(r'\[(\d+)\] push button "Open Parent"', tree).group(1) + clicked = await session.call_tool("click", {"element_token": f"{snapshot}:{row}"}) + parent = await _window(session, "home - Thunar") + ready = await deployed.invoke(COMPUTER_READY, "status", None) + skill = await deployed.invoke(SKILL_EXTENSION_URI, "get", {"name": "desktop"}) + finally: + report = await teardown_run(context) + + assert not launched.isError and not clicked.isError and parent["window_id"] == window["window_id"] + checks = {check["name"]: check for check in ready["checks"]} + assert ready["ready"] and checks["cua-driver"]["ok"] and checks["cua-driver tools"]["detail"] == "66 tools, as pinned" + assert {"LINUX.md", "WORKFLOW.md", "RUNTIME.md"} <= set(skill["files"]) + assert "# This computer" in skill["skill_md"] and skill["description"].startswith("Drive a native GUI app") + assert not report.failed + + +async def _window(session: ClientSession, title: str) -> dict: + """Thunar's window once its title is ``title``.""" + for _ in range(40): + listed = await session.call_tool("list_windows", {"app": "thunar"}) + if found := [w for w in listed.structuredContent["windows"] if w["title"] == title]: + return found[0] + await asyncio.sleep(0.5) + raise TimeoutError(f"no Thunar window titled {title!r}: {listed.structuredContent['windows']}") From 8ff597071ea8fcc52049f177dc6bfef25c18200f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:31:01 -0700 Subject: [PATCH 06/13] fix(computer): resend a read once on a new connection when cua-driver's daemon restarts Co-Authored-By: Claude Opus 5.5 --- .../drivers/envs/cua-driver/driver.py | 58 +++++++++++++------ tst/unit/drivers/cua_driver_test.py | 30 +++++++++- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 73920a74f..d696fff7a 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -2,7 +2,8 @@ (``daemon.sh``), agents get its own tools as its pinned version lists them (``tools.json``), beside the ``computer_*`` tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes line-delimited JSON on a Unix socket; the framework's calls and each caller's run on connections of their own, the framework's in a session of its -own. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" +own. When a connection drops, as when the daemon restarts, a read is sent once more on a new one; input is not, since +some of it may have arrived. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" from __future__ import annotations import asyncio @@ -138,7 +139,8 @@ async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: """A call to one of cua-driver's own tools, on its caller's connection, answered as cua-driver answers it.""" connection = self._callers.setdefault(self.caller_identity(ToolContext.current()), _Connection()) try: - reply = await connection.request({"method": "call", "name": tool, "args": arguments}) + reply = await connection.request({"method": "call", "name": tool, "args": arguments}, + retry=tool in _read_only()) except DeviceUnavailable as e: return _refused(e.message) if not reply.get("ok"): @@ -149,10 +151,12 @@ async def _send(self, name: str, **args: Any) -> None: await self._call(name, DeliveryFailed, scope="desktop", **args) async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: - """The result of one of cua-driver's tools called in the framework's session; ``failure`` names what went - wrong when it fails or refuses.""" + """The result of one of cua-driver's tools called in the framework's session, a read (``DeviceUnavailable`` + when it fails) sent again on a new connection when this one dropped; ``failure`` names what went wrong when + it fails or refuses.""" reply = await self._framework.request({"method": "call", "name": name, - "args": {**args, "session": FRAMEWORK_SESSION}}) + "args": {**args, "session": FRAMEWORK_SESSION}}, + retry=failure is DeviceUnavailable) if not reply.get("ok"): raise failure(f"cua-driver {name}: {reply.get('error')}") result = reply["result"] @@ -162,7 +166,7 @@ async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: async def _pinned_checks(self) -> list[ReadyCheck]: try: - listed = (await self._framework.request({"method": "list"}))["result"]["tools"] + listed = (await self._framework.request({"method": "list"}, retry=True))["result"]["tools"] except DeviceUnavailable: return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")] digest, expected = await asyncio.to_thread(_executable_sha256), EXECUTABLES.get(self.platform.arch or "") @@ -183,21 +187,32 @@ def __init__(self) -> None: self._stream: tuple[asyncio.StreamReader, asyncio.StreamWriter] | None = None self._lock = asyncio.Lock() - async def request(self, payload: dict) -> dict: + async def request(self, payload: dict, *, retry: bool = False) -> dict: + """The daemon's reply to ``payload``; with ``retry``, sent once more on a new connection when an open one has + dropped.""" async with self._lock: + was_open = self._stream is not None try: - if self._stream is None: - self._stream = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) - reader, writer = self._stream - writer.write((json.dumps(payload) + "\n").encode()) - await writer.drain() - line = await reader.readline() - if not line: - raise ConnectionError("the daemon closed the connection") - except (OSError, ValueError) as e: - self._close() - raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None - return json.loads(line) + return await self._exchange(payload) + except DeviceUnavailable: + if not (retry and was_open): + raise + return await self._exchange(payload) + + async def _exchange(self, payload: dict) -> dict: + try: + if self._stream is None: + self._stream = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) + reader, writer = self._stream + writer.write((json.dumps(payload) + "\n").encode()) + await writer.drain() + line = await reader.readline() + if not line: + raise ConnectionError("the daemon closed the connection") + except (OSError, ValueError) as e: + self._close() + raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None + return json.loads(line) def _close(self) -> None: if self._stream is not None: @@ -210,6 +225,11 @@ def _manifest() -> list[dict]: return json.loads(MANIFEST.read_text()) +@functools.cache +def _read_only() -> frozenset[str]: + return frozenset(t["name"] for t in _manifest() if (t["annotations"] or {}).get("readOnlyHint")) + + def _modifiers(keys) -> list[str]: return [MODIFIERS[key] for key in keys] diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index 7f1ac3314..0a11473e6 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -216,15 +216,39 @@ async def test_readiness_waits_for_a_daemon_that_isnt_answering(module, monkeypa @pytest.mark.asyncio -async def test_a_dropped_daemon_connection_is_device_unavailable_and_reopened(module, daemon): +async def test_a_read_is_sent_again_after_the_daemon_restarts_but_input_is_not(module, daemon): driver = module.CuaDriver() await driver.screen() await daemon.stop() - with pytest.raises(DeviceUnavailable, match="cua-driver's daemon isn't answering"): await driver.screen() await daemon.start() - assert (await driver.screen()).width == 1280 and daemon.connections == 2 + assert (await driver.screen()).width == 1280 + + await daemon.stop() + await daemon.start() + assert (await driver.screen()).width == 1280 + await daemon.stop() + await daemon.start() + with pytest.raises(DeviceUnavailable): + await driver.click_at(1, 1, "left", 1, []) + await driver.click_at(2, 2, "left", 1, []) + assert [args["x"] for name, args in daemon.calls() if name == "click"] == [2] + + +@pytest.mark.asyncio +async def test_one_of_cua_drivers_read_only_tools_is_sent_again_after_a_restart_and_its_input_tools_are_not(module, + daemon): + async with mcp_session(module.CuaDriver()) as session: + await session.call_tool("list_windows", {}) + await daemon.stop() + await daemon.start() + read = await session.call_tool("list_windows", {}) + await daemon.stop() + await daemon.start() + clicked = await session.call_tool("click", {"element_token": "s1:1"}) + assert not read.isError and clicked.isError and "isn't answering" in clicked.content[0].text + assert [name for name, _ in daemon.calls()] == ["list_windows", "list_windows"] def test_the_skill_pack_is_written_offline_from_the_executable(module, monkeypatch, tmp_path): From 99c7b15f39861fada9187fe3164295ea2fca9d75 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:34:38 -0700 Subject: [PATCH 07/13] docs(computer): forwarded tools, skill files and the cua-driver driver Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- packages/agentenv-protocol/README.md | 22 +++++++++++++++++---- tst/integration/computer/cua_driver_test.py | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 855895d24..57521db1e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | @@ -124,7 +124,7 @@ and deserialized through a registry. | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | | `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`. It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own tools forwarded to agents through `forwarded_tools` beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c7be5820a..d21dd5c67 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -240,6 +240,16 @@ at the gateway. Where the binding lives depends on the app. On mcp's own FastMCP it wraps the tool manager's dispatch; on the standalone `fastmcp` package it joins the app's middleware chain as the outermost layer, so a handler's own middleware runs under it too. An app with neither seam mounts unbound, with a warning, unless a tool declares a `ToolContext` parameter or the handler defines `on_tool_call`: then the mount is refused rather than serving an empty caller. +### Tools served on another server's behalf + +An environment can serve tools another server defines, each with that server's own schema: +`forwarded_tools()` returns `ForwardedTool(tool, call, annotations)` entries, where `tool` is an +`EnvironmentTool` carrying the other server's `inputSchema`, and `call` takes the call's arguments +as keywords, untouched, and returns that server's `CallToolResult`, which reaches the caller as it +is. They're listed on the card beside the `@tool` methods, chosen by `served_tools` and under tool +access like them; one named like another tool is refused. They need mcp's FastMCP. The cua-driver +driver serves cua-driver's own tools this way. + ### Getting part of the data A `@get_data` handler that takes a `paths` parameter returns only the data under the paths a caller @@ -262,9 +272,11 @@ def guide(self): The card then declares `urn:agentenv:skill/v1` (`SKILL_EXTENSION_URI`), whose `get {name}` answers `{name, description, skill_md}`, the shape `skill-config/v1` takes, with the frontmatter written under the name the caller gives; a name that isn't lowercase letters, digits and single -hyphens, up to 64, is refused `invalid_argument`. agent-env's `add_skills` step installs it on -an agent through `env_ids`, named after the env's environment name; for an env behind a gateway, -whose card nests its servers' cards, it installs each server's skill under that server's name. +hyphens, up to 64, is refused `invalid_argument`. A skill with other files beside its SKILL.md +returns them as a third value, `{relative path: text}`, and `get` answers them as `files`. +agent-env's `add_skills` step installs it on an agent through `env_ids`, named after the env's +environment name, a skill with files as a bundle of them; for an env behind a gateway, whose card +nests its servers' cards, it installs each server's skill under that server's name. ### Uploading an export through a grant @@ -426,7 +438,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope features. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base class): a SKILL.md written from the computer as it booted (its platform, session user and home, the live screen size, what its input takes, its OS's shortcuts) and ending with the driver - class's `notes`, markdown for what its card can't say. + class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of + its own returns it from `bundled_skill()` (its description, body and other files), and the + computer skill leads with it, the computer's part following. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool diff --git a/tst/integration/computer/cua_driver_test.py b/tst/integration/computer/cua_driver_test.py index e6a78dc20..fa20a1331 100644 --- a/tst/integration/computer/cua_driver_test.py +++ b/tst/integration/computer/cua_driver_test.py @@ -2,7 +2,7 @@ an image for each arch this machine's Docker builds, gate 3 makes this machine's ready at full with cua-driver's own tools beside the computer_* tools; deployed, an agent's session drives Thunar by element token through cua-driver's tools, computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local -registry on localhost:5000. It runs cua-driver, so it runs on the Linux test VMs.""" +registry on localhost:5000.""" import asyncio import json From 5bc5935fed59157b81448a75520797ee56f759e7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 14:54:38 -0700 Subject: [PATCH 08/13] fix(computer): keep skill files in their folder, and cua-driver's lost and cancelled requests honest add_skills refuses an env skill file whose path leaves the skill's folder, before anything is stored. The cua-driver driver closes a connection whose request is cancelled in flight, so its reply can't answer the next request; fails input whose reply is lost after it was sent as DeliveryFailed, its outcome unknown; and refuses a click, drag or chord holding a key that isn't a modifier as unsupported. Co-Authored-By: Claude Opus 5.5 --- .../drivers/envs/cua-driver/driver.py | 36 ++++++++++++----- .../task_step/task_steps/add_skills.py | 7 +++- tst/unit/drivers/cua_driver_test.py | 40 +++++++++++++++++-- tst/unit/task_step/add_skills_env_test.py | 12 ++++++ 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index d696fff7a..1e28f3746 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -2,8 +2,9 @@ (``daemon.sh``), agents get its own tools as its pinned version lists them (``tools.json``), beside the ``computer_*`` tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes line-delimited JSON on a Unix socket; the framework's calls and each caller's run on connections of their own, the framework's in a session of its -own. When a connection drops, as when the daemon restarts, a read is sent once more on a new one; input is not, since -some of it may have arrived. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" +own. When a connection drops, as when the daemon restarts, a read is sent once more on a new one; input is not, and +input whose reply is lost fails with its outcome unknown, since it may have arrived. It needs +``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" from __future__ import annotations import asyncio @@ -131,9 +132,7 @@ async def type_in(self, text) -> None: @press async def press_keys(self, keys) -> None: *held, key = keys - if any(k not in MODIFIERS for k in held): - raise Unsupported(f"cua-driver holds only modifier keys in a chord, not {[k for k in held if k not in MODIFIERS]}") - await self._send("press_key", key=MODIFIERS.get(key) or NAMED.get(key, key), modifiers=_modifiers(held)) + await self._send("press_key", modifiers=_modifiers(held), key=MODIFIERS.get(key) or NAMED.get(key, key)) async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: """A call to one of cua-driver's own tools, on its caller's connection, answered as cua-driver answers it.""" @@ -153,10 +152,13 @@ async def _send(self, name: str, **args: Any) -> None: async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: """The result of one of cua-driver's tools called in the framework's session, a read (``DeviceUnavailable`` when it fails) sent again on a new connection when this one dropped; ``failure`` names what went wrong when - it fails or refuses.""" - reply = await self._framework.request({"method": "call", "name": name, - "args": {**args, "session": FRAMEWORK_SESSION}}, - retry=failure is DeviceUnavailable) + it fails, refuses or loses its reply after it was sent.""" + try: + reply = await self._framework.request({"method": "call", "name": name, + "args": {**args, "session": FRAMEWORK_SESSION}}, + retry=failure is DeviceUnavailable) + except _ReplyLost as e: + raise failure(e.message) from None if not reply.get("ok"): raise failure(f"cua-driver {name}: {reply.get('error')}") result = reply["result"] @@ -179,6 +181,10 @@ async def _pinned_checks(self) -> list[ReadyCheck]: f"these tools differ from tools.json: {', '.join(differ)}")] +class _ReplyLost(DeviceUnavailable): + """The connection dropped after a request was sent, so the daemon may have acted on it.""" + + class _Connection: """One connection to cua-driver's daemon, opened on first use and again after it drops, taking one request at a time.""" @@ -200,10 +206,14 @@ async def request(self, payload: dict, *, retry: bool = False) -> dict: return await self._exchange(payload) async def _exchange(self, payload: dict) -> dict: + """One request and its reply; a request cancelled in flight closes the connection, so its reply can't be read + as the next one's.""" + sent = False try: if self._stream is None: self._stream = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) reader, writer = self._stream + sent = True writer.write((json.dumps(payload) + "\n").encode()) await writer.drain() line = await reader.readline() @@ -211,7 +221,13 @@ async def _exchange(self, payload: dict) -> dict: raise ConnectionError("the daemon closed the connection") except (OSError, ValueError) as e: self._close() + if sent: + raise _ReplyLost(f"cua-driver's daemon stopped answering after the request was sent, which it may " + f"have acted on: {e}") from None raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None + except asyncio.CancelledError: + self._close() + raise return json.loads(line) def _close(self) -> None: @@ -231,6 +247,8 @@ def _read_only() -> frozenset[str]: def _modifiers(keys) -> list[str]: + if held := [key for key in keys if key not in MODIFIERS]: + raise Unsupported(f"cua-driver holds only modifier keys, not {held}") return [MODIFIERS[key] for key in keys] diff --git a/src/agent_env/task_step/task_steps/add_skills.py b/src/agent_env/task_step/task_steps/add_skills.py index 5437548f8..ccb16b8a9 100644 --- a/src/agent_env/task_step/task_steps/add_skills.py +++ b/src/agent_env/task_step/task_steps/add_skills.py @@ -8,6 +8,7 @@ import logging import re from dataclasses import dataclass +from pathlib import PurePosixPath from typing import ClassVar, Optional from agentenv_protocol import SKILL_EXTENSION_URI @@ -310,7 +311,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: def _stored_skill(skill_md: str, files: dict[str, str]) -> str: - """The object-store prefix holding an env skill's SKILL.md and its other files, one prefix per content.""" + """The object-store prefix holding an env skill's SKILL.md and its other files, one prefix per content; a file + whose path leaves the skill's folder is refused.""" + if outside := [path for path in files if not path or PurePosixPath(path).is_absolute() + or ".." in PurePosixPath(path).parts]: + raise ValueError(f"an env skill's files must stay inside its folder, not {outside}") config = get_config() store, contents = config.get_object_store(), {"SKILL.md": skill_md, **files} digest = hashlib.sha256(json.dumps(contents, sort_keys=True).encode()).hexdigest() diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index 0a11473e6..37545ea7b 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -20,7 +20,7 @@ from PIL import Image as PILImage import agent_env -from agentenv_protocol.computer import DeviceUnavailable, Unsupported +from agentenv_protocol.computer import DeliveryFailed, DeviceUnavailable, Unsupported DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" @@ -33,13 +33,16 @@ def _png(width: int, height: int) -> str: class FakeDaemon: """cua-driver's daemon as the driver sees it, on ``path``: each request by the connection it came on, ``refuse`` - naming tools whose results are errors, ``fail`` tools it can't call, and ``tools`` what ``list`` lists.""" + naming tools whose results are errors, ``fail`` tools it can't call, ``slow`` tools it answers after a second, + ``drop`` tools whose connection it closes without answering, and ``tools`` what ``list`` lists.""" def __init__(self, manifest: list[dict], path: Path) -> None: self.path = path self.requests: list[tuple[int, dict]] = [] self.refuse: set[str] = set() self.fail: set[str] = set() + self.slow: set[str] = set() + self.drop: set[str] = set() self.tools = [{"name": t["name"], "input_schema": t["inputSchema"]} for t in manifest] self.connections = 0 self.writers: list[asyncio.StreamWriter] = [] @@ -62,6 +65,10 @@ async def serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter while line := await reader.readline(): request = json.loads(line) self.requests.append((connection, request)) + if request.get("name") in self.drop: + break + if request.get("name") in self.slow: + await asyncio.sleep(1) writer.write((json.dumps(self.reply(request)) + "\n").encode()) await writer.drain() writer.close() @@ -150,6 +157,8 @@ async def test_what_cua_driver_cant_do_or_refuses_fails_naming_it(module, daemon with pytest.raises(Unsupported, match="holds only modifier keys"): await driver.press_keys(["Control", "a", "b"]) + with pytest.raises(Unsupported, match="holds only modifier keys, not \\['a'\\]"): + await driver.click_at(1, 1, "left", 1, ["Shift", "a"]) with pytest.raises(Unsupported, match="no keys held"): await driver.scroll_at(1, 1, "up", 1, ["Shift"]) with pytest.raises(Exception, match="cua-driver click: click refused") as refused: @@ -230,12 +239,35 @@ async def test_a_read_is_sent_again_after_the_daemon_restarts_but_input_is_not(m assert (await driver.screen()).width == 1280 await daemon.stop() await daemon.start() - with pytest.raises(DeviceUnavailable): + with pytest.raises(DeliveryFailed, match="may have acted on"): await driver.click_at(1, 1, "left", 1, []) await driver.click_at(2, 2, "left", 1, []) assert [args["x"] for name, args in daemon.calls() if name == "click"] == [2] +@pytest.mark.asyncio +async def test_input_whose_reply_is_lost_fails_with_its_outcome_unknown(module, daemon): + daemon.drop.add("type_text") + with pytest.raises(DeliveryFailed, match="may have acted on") as lost: + await module.CuaDriver().type_in("hello") + assert lost.value.applied == "unknown" and [name for name, _ in daemon.calls()] == ["type_text"] + + +@pytest.mark.asyncio +async def test_a_request_cancelled_in_flight_leaves_no_reply_for_the_next_one(module, daemon): + driver = module.CuaDriver() + daemon.slow.add("get_screen_size") + screen = asyncio.ensure_future(driver.screen()) + await asyncio.sleep(0.2) + screen.cancel() + with contextlib.suppress(asyncio.CancelledError): + await screen + daemon.slow.clear() + + shot = await driver.grab() + assert (shot.width, shot.mode) == (1280, "png") and daemon.connections == 2 + + @pytest.mark.asyncio async def test_one_of_cua_drivers_read_only_tools_is_sent_again_after_a_restart_and_its_input_tools_are_not(module, daemon): @@ -247,7 +279,7 @@ async def test_one_of_cua_drivers_read_only_tools_is_sent_again_after_a_restart_ await daemon.stop() await daemon.start() clicked = await session.call_tool("click", {"element_token": "s1:1"}) - assert not read.isError and clicked.isError and "isn't answering" in clicked.content[0].text + assert not read.isError and clicked.isError and "may have acted on" in clicked.content[0].text assert [name for name, _ in daemon.calls()] == ["list_windows", "list_windows"] diff --git a/tst/unit/task_step/add_skills_env_test.py b/tst/unit/task_step/add_skills_env_test.py index 1d741dd02..8f059dd32 100644 --- a/tst/unit/task_step/add_skills_env_test.py +++ b/tst/unit/task_step/add_skills_env_test.py @@ -82,6 +82,18 @@ async def invoke(base_url, card, uri, params=None, timeout=30, verify=True, *, m "LINUX.md", "SKILL.md", "WORKFLOW.md"] +@pytest.mark.asyncio +async def test_a_skill_file_whose_path_leaves_its_folder_is_refused_before_anything_is_stored(agent, local_stores, + monkeypatch): + async def invoke(base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + return {"name": params["name"], "description": "d", "skill_md": "# Skill\n", "files": {"../../victim": "x"}} + + monkeypatch.setattr(env_module.protocol_v1, "invoke_extension", invoke) + with pytest.raises(ValueError, match="must stay inside its folder, not \\['../../victim'\\]"): + await _step("shop").execute(_context(_env())) + assert local_stores.get_object_store().list("") == [] and not agent.register.await_args_list + + @pytest.mark.asyncio async def test_an_env_behind_a_gateway_installs_each_child_envs_skill_under_the_childs_name(agent): gateway = _env("office", name="office", extensions=(), children=(("mail", (_SKILL,)), ("Chat Room", (_SKILL,)), From 0633b591a7685eee6857ee94e7b0b95cea505d4d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 15:46:52 -0700 Subject: [PATCH 09/13] fix(computer): give each cua-driver caller a session of its own, and simplify the driver - Each caller, told apart by MCP session, and the framework run in an unnamed cua-driver session of their own, stamped as the daemon request's session_id, as cua-driver's own MCP server does. All agents used to share one session, which owns screenshot frames, the agent cursor, config overrides and recordings. cua-driver ends a named session after five idle minutes and refuses every later call in it; the framework's calls ran in the named session "agentenv", so computer/v1 failed after five idle minutes. An unnamed session is started again on its next call. - Each call goes on a connection of its own, as cua-driver's MCP server sends them, which drops the kept connections' resend-after-restart and cancel handling. - driver.py starts the daemon when run as a script, exec'ing the executable computer-ready hashes, with the socket and no-telemetry settings defined once; daemon.sh is gone. - The computer skill builds its opening apart from its sections, and PRINTABLE_KEYS is shared from agentenv_protocol.computer. - The drivers README gains the version-bump steps and says element tokens belong to windows, not sessions. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/__init__.py | 2 + .../src/agentenv_protocol/computer/keys.py | 2 + .../src/agentenv_protocol/computer/skill.py | 26 ++-- src/agent_env/drivers/README.md | 36 +++-- .../drivers/envs/cua-driver/daemon.sh | 11 -- .../drivers/envs/cua-driver/driver.py | 141 ++++++++---------- .../drivers/envs/cua-driver/env.toml | 2 +- tst/integration/computer/cua_driver_test.py | 16 +- tst/unit/drivers/cua_driver_test.py | 72 ++++----- 9 files changed, 152 insertions(+), 156 deletions(-) delete mode 100644 src/agent_env/drivers/envs/cua-driver/daemon.sh diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 45af04bf8..6408b55e9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -60,6 +60,7 @@ type_text, write_files, ) +from .keys import PRINTABLE_KEYS from .manifest import ManifestSegment, RecordingManifest from .platform import Platform from .settings import SETTINGS_ENV, ComputerSettings @@ -100,6 +101,7 @@ def __getattr__(name: str) -> Any: "AppDeclaration", "APP_SETTINGS_ENV", "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", + "PRINTABLE_KEYS", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py index b8bab2aa6..54a731631 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/keys.py @@ -6,6 +6,8 @@ from .errors import InvalidArgument, Unsupported +#: The printable ASCII characters but the space, each the key value it types. +PRINTABLE_KEYS = tuple(chr(code) for code in range(0x21, 0x7F)) NAMED = ( "Alt", "AltGraph", "CapsLock", "Control", "Fn", "FnLock", "Hyper", "Meta", "NumLock", "ScrollLock", "Shift", "Super", "Symbol", "SymbolLock", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py index 829bffe56..0756845a9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from .keys import PRINTABLE_KEYS + if TYPE_CHECKING: from .driver import ComputerDriverEnvironment @@ -23,7 +25,6 @@ """, } _SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"} -_PRINTABLE = tuple(chr(code) for code in range(0x21, 0x7F)) async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, str]]: @@ -37,10 +38,18 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, session += f" ({desktop})" description = (f"How to use this computer through its computer_* tools: {system}, {display.width}x{display.height}, " "what its input takes, and its shortcuts.") - sections = [ + bundled = await driver.bundled_skill() + opening = ([ "# Using this computer", "You see this computer through `computer_screenshot` and drive it through the other `computer_*` tools, as a " "person at its screen would: look, then click, type and press keys on what the screen shows.", + ] if bundled is None else [ + "# This computer", + "Beside the tools above, this computer serves its `computer_*` tools: `computer_screenshot` shows the screen, " + "and the others click, type and press keys on what it shows.", + ]) + sections = [ + *opening, "## The computer", "\n".join([f"- {system}, {session}.", f"- You act as `{getpass.getuser()}`, whose home is `{Path.home()}`.", f"- The screen is {display.width}x{display.height}."]), @@ -51,13 +60,12 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, sections += ["## Shortcuts", shortcuts.rstrip()] if notes := driver.notes.strip(): sections += ["## Notes", notes] - if (bundled := await driver.bundled_skill()) is None: - return description, "\n\n".join(sections) + "\n", {} + body = "\n\n".join(sections) + "\n" + if bundled is None: + return description, body, {} own_description, own_body, files = bundled - sections[:2] = ["# This computer", "Beside the tools above, this computer serves its `computer_*` tools: " - "`computer_screenshot` shows the screen, and the others click, type and press keys on what it shows."] return (f"{own_description} This computer: {system}, {display.width}x{display.height}, and its computer_* tools.", - own_body.rstrip() + "\n\n" + "\n\n".join(sections) + "\n", files) + own_body.rstrip() + "\n\n" + body, files) def _input(driver: ComputerDriverEnvironment) -> str: @@ -78,6 +86,6 @@ def _input(driver: ComputerDriverEnvironment) -> str: def _keys(keys) -> str: """The keys a driver takes, every printable ASCII character said once when it takes them all.""" - every = set(_PRINTABLE) <= set(keys) - named = [key for key in keys if not (every and key in _PRINTABLE)] + every = set(PRINTABLE_KEYS) <= set(keys) + named = [key for key in keys if not (every and key in PRINTABLE_KEYS)] return "only " + ", ".join(f"`{key!r}`" for key in named) + (", and any printable ASCII character" if every else "") diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index ec7b58e00..c9bf86e91 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -14,18 +14,30 @@ there, and run `agentenv_protocol.computer.serve driver.py:ReferenceDriver` as a the shipped `desktop` example's driver. `python -m agentenv_protocol.computer.conformance ` checks it, or any driver, where it runs. -`envs/cua-driver/` is trycua's [cua-driver](https://github.com/trycua/cua/tree/main/libs/cua-driver) as a driver, -for the same X11 desktop on Linux (`amd64` and `arm64`). cua-driver 0.38.0 runs on the computer as a daemon, -`cua-driver serve`, started by `daemon.sh` once X answers, with approvals off and no telemetry. Agents get its own 66 -tools beside the `computer_*` tools, as `tools.json` lists them: element tokens from `get_window_state`, clicks and -typing by element or pixel, `run_script` and its browser tools, each call forwarded on its caller's own connection to the -daemon. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on its desktop-scope calls (XTest), -in a session of their own, so they never make an agent's element tokens stale. It has no pointer or keyboard hold group, -drags from a start to an end, and scrolls with no keys held. Its skill leads with cua-driver's own skill pack, as the -executable embeds it, its other files beside the SKILL.md. An env narrows which tools its agent sees with `[tools]`. -`requirements.txt` installs `cua-driver==0.38.0` from PyPI, whose wheel carries the executable, and `computer-ready` -checks that executable's sha256 and that the daemon lists the tools `tools.json` pins. Name it as `cua-driver`, the -alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/cua-driver`. +`envs/cua-driver/` is trycua's [cua-driver](https://github.com/trycua/cua/tree/main/libs/cua-driver) as a driver, for +the same X11 desktop on Linux (`amd64` and `arm64`). cua-driver 0.38.0 runs on the computer as a daemon, `cua-driver +serve`, which `driver.py` starts when run as a script, once X answers, with approvals off and no telemetry. Agents get +its own 66 tools beside the `computer_*` tools, as `tools.json` lists them: element tokens from `get_window_state`, +clicks and typing by element or pixel, `run_script` and its browser tools. Each call is forwarded to the daemon in a +cua-driver session of the caller's own unless it names one, as cua-driver's own MCP server gives each client. Element +tokens belong to a window, not a session: an agent reading a window another agent has read makes the other's tokens for +it stale. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on its desktop-scope calls (XTest) +in a session of their own; a desktop capture leaves window snapshots alone, so it never makes an agent's element tokens +stale. It has no pointer or keyboard hold group, drags from a start to an end, and scrolls with no keys held. Its skill +leads with cua-driver's own skill pack, as the executable embeds it, its other files beside the SKILL.md. An env narrows +which tools its agent sees with `[tools]`. `requirements.txt` installs `cua-driver==0.38.0` from PyPI, whose wheel +carries the executable, and `computer-ready` checks that executable's sha256 and that the daemon lists the tools +`tools.json` pins. Name it as `cua-driver`, the alias `aliases.toml` gives it, or by its id, +`@local/agentenv-framework/drivers/cua-driver`. + +To move it to another cua-driver version: + +1. Pin the version in `requirements.txt`. +2. Regenerate `tools.json` from that version's tools as its MCP server lists them: each tool's `name`, `description`, + `inputSchema` and `annotations`. Listing them takes a running daemon, so do it on a Linux test machine. +3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel. +4. Run `tst/unit/drivers/cua_driver_test.py`, then `tst/integration/computer/cua_driver_test.py` on Linux, where + `computer-ready` names any tool or executable that doesn't match its pin. This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/envs/cua-driver/daemon.sh b/src/agent_env/drivers/envs/cua-driver/daemon.sh deleted file mode 100644 index 6be1a0447..000000000 --- a/src/agent_env/drivers/envs/cua-driver/daemon.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# cua-driver's daemon, once X answers: from the session user's home, where the apps it launches start too, on the -# socket the driver reaches it at, with approvals off, since nobody is at the computer to give them, and nothing sent -# to trycua. -set -eu -export DO_NOT_TRACK=1 CUA_DRIVER_RS_TELEMETRY_ENABLED=false CUA_DRIVER_RS_UPDATE_CHECK=false -venv="$(cd "$(dirname "$0")" && pwd)/venv" -display="${DISPLAY#:}" -while [ ! -S "/tmp/.X11-unix/X${display%%.*}" ]; do sleep 0.2; done -cd "$HOME" -exec "$venv/bin/cua-driver" serve --socket /run/agentenv/cua-driver.sock --dangerously-bypass-approvals --no-overlay diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 1e28f3746..ae819cf72 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -1,10 +1,11 @@ -"""cua-driver as a computer driver, for an X11 desktop on Linux: trycua's cua-driver runs on the computer as a daemon -(``daemon.sh``), agents get its own tools as its pinned version lists them (``tools.json``), beside the ``computer_*`` -tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes line-delimited JSON on a Unix -socket; the framework's calls and each caller's run on connections of their own, the framework's in a session of its -own. When a connection drops, as when the daemon restarts, a read is sent once more on a new one; input is not, and -input whose reply is lost fails with its outcome unknown, since it may have arrived. It needs -``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" +"""cua-driver as a computer driver, for an X11 desktop on Linux: trycua's cua-driver runs on the computer as a daemon, +which this file starts when run as a script, agents get its own tools as its pinned version lists them (``tools.json``), +beside the ``computer_*`` tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes +line-delimited JSON on a Unix socket, and each call goes on a connection of its own, as cua-driver's own MCP server sends +them. The framework and each caller, told apart by MCP session, have an unnamed cua-driver session of their own, which +cua-driver starts again after ending it for idling, where a named one stays ended. A call the daemon can't take, as +while it restarts, fails as unavailable; input whose reply is lost fails with its outcome unknown, since it may have +arrived. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" from __future__ import annotations import asyncio @@ -16,6 +17,8 @@ import os import subprocess import tempfile +import time +import uuid from pathlib import Path from typing import Any @@ -23,6 +26,7 @@ from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext from agentenv_protocol.computer import ( + PRINTABLE_KEYS, ComputerDriverEnvironment, DeliveryFailed, DeviceUnavailable, @@ -40,7 +44,7 @@ type_text, ) -#: Where ``daemon.sh`` serves cua-driver's daemon. +#: Where cua-driver's daemon listens. SOCKET = Path("/run/agentenv/cua-driver.sock") #: cua-driver's tools as its pinned version lists them, which the running daemon must list the same. MANIFEST = Path(__file__).with_name("tools.json") @@ -49,8 +53,8 @@ "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", "arm64": "26e9eee8165e9ce7b614b5e3d68e5d963231385a82b65236406718b5ce1ff936", } -#: The session the framework's own calls run in, apart from every caller's. -FRAMEWORK_SESSION = "agentenv" +#: What keeps cua-driver from reaching trycua: no telemetry and no update checks. +QUIET = {"DO_NOT_TRACK": "1", "CUA_DRIVER_RS_TELEMETRY_ENABLED": "false", "CUA_DRIVER_RS_UPDATE_CHECK": "false"} #: W3C modifier keys and the names cua-driver holds them by. MODIFIERS = {"Control": "ctrl", "Shift": "shift", "Alt": "alt", "Meta": "super", "Super": "super"} #: W3C key values and the names cua-driver presses them by; any printable ASCII character is pressed as itself. @@ -59,7 +63,6 @@ "Insert": "insert", "Home": "home", "End": "end", "PageUp": "pageup", "PageDown": "pagedown", "ArrowUp": "up", "ArrowDown": "down", "ArrowLeft": "left", "ArrowRight": "right", **{f"F{n}": f"f{n}" for n in range(1, 13)}, } -PRINTABLE = tuple(chr(code) for code in range(0x21, 0x7F)) #: How long the skill pack's writing may take. SKILL_TIMEOUT_S = 60 #: The largest reply the daemon sends: a screenshot of a large display, base64. @@ -72,12 +75,12 @@ class CuaDriver(ComputerDriverEnvironment): buttons = ("left", "right", "middle") max_click_count = 3 max_scroll_amount = 50 - keys = (*MODIFIERS, *NAMED, *PRINTABLE) + keys = (*MODIFIERS, *NAMED, *PRINTABLE_KEYS) def __init__(self) -> None: super().__init__() - self._framework = _Connection() - self._callers: dict[str, _Connection] = {} + self._session = uuid.uuid4().hex + self._callers: dict[str | None, str] = {} self._pinned: list[ReadyCheck] | None = None self._skill: tuple[str, str, dict[str, str]] | None = None @@ -135,11 +138,10 @@ async def press_keys(self, keys) -> None: await self._send("press_key", modifiers=_modifiers(held), key=MODIFIERS.get(key) or NAMED.get(key, key)) async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: - """A call to one of cua-driver's own tools, on its caller's connection, answered as cua-driver answers it.""" - connection = self._callers.setdefault(self.caller_identity(ToolContext.current()), _Connection()) + """A call to one of cua-driver's own tools in its caller's session, answered as cua-driver answers it.""" + session = self._callers.setdefault(ToolContext.current().caller.session, uuid.uuid4().hex) try: - reply = await connection.request({"method": "call", "name": tool, "args": arguments}, - retry=tool in _read_only()) + reply = await _request({"method": "call", "name": tool, "args": arguments, "session_id": session}) except DeviceUnavailable as e: return _refused(e.message) if not reply.get("ok"): @@ -150,13 +152,10 @@ async def _send(self, name: str, **args: Any) -> None: await self._call(name, DeliveryFailed, scope="desktop", **args) async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: - """The result of one of cua-driver's tools called in the framework's session, a read (``DeviceUnavailable`` - when it fails) sent again on a new connection when this one dropped; ``failure`` names what went wrong when - it fails, refuses or loses its reply after it was sent.""" + """The result of one of cua-driver's tools called in the framework's session; ``failure`` names what went wrong + when it fails, refuses or loses its reply after it was sent.""" try: - reply = await self._framework.request({"method": "call", "name": name, - "args": {**args, "session": FRAMEWORK_SESSION}}, - retry=failure is DeviceUnavailable) + reply = await _request({"method": "call", "name": name, "args": args, "session_id": self._session}) except _ReplyLost as e: raise failure(e.message) from None if not reply.get("ok"): @@ -168,7 +167,7 @@ async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: async def _pinned_checks(self) -> list[ReadyCheck]: try: - listed = (await self._framework.request({"method": "list"}, retry=True))["result"]["tools"] + listed = (await _request({"method": "list"}))["result"]["tools"] except DeviceUnavailable: return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")] digest, expected = await asyncio.to_thread(_executable_sha256), EXECUTABLES.get(self.platform.arch or "") @@ -185,55 +184,25 @@ class _ReplyLost(DeviceUnavailable): """The connection dropped after a request was sent, so the daemon may have acted on it.""" -class _Connection: - """One connection to cua-driver's daemon, opened on first use and again after it drops, taking one request at a - time.""" - - def __init__(self) -> None: - self._stream: tuple[asyncio.StreamReader, asyncio.StreamWriter] | None = None - self._lock = asyncio.Lock() - - async def request(self, payload: dict, *, retry: bool = False) -> dict: - """The daemon's reply to ``payload``; with ``retry``, sent once more on a new connection when an open one has - dropped.""" - async with self._lock: - was_open = self._stream is not None - try: - return await self._exchange(payload) - except DeviceUnavailable: - if not (retry and was_open): - raise - return await self._exchange(payload) - - async def _exchange(self, payload: dict) -> dict: - """One request and its reply; a request cancelled in flight closes the connection, so its reply can't be read - as the next one's.""" - sent = False - try: - if self._stream is None: - self._stream = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) - reader, writer = self._stream - sent = True - writer.write((json.dumps(payload) + "\n").encode()) - await writer.drain() - line = await reader.readline() - if not line: - raise ConnectionError("the daemon closed the connection") - except (OSError, ValueError) as e: - self._close() - if sent: - raise _ReplyLost(f"cua-driver's daemon stopped answering after the request was sent, which it may " - f"have acted on: {e}") from None - raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None - except asyncio.CancelledError: - self._close() - raise +async def _request(payload: dict) -> dict: + """The daemon's reply to ``payload``, sent on a connection of its own; a cancelled request closes it, so nothing is + left to read its reply.""" + try: + reader, writer = await asyncio.open_unix_connection(str(SOCKET), limit=REPLY_LIMIT) + except OSError as e: + raise DeviceUnavailable(f"cua-driver's daemon isn't answering: {e}") from None + try: + writer.write((json.dumps(payload) + "\n").encode()) + await writer.drain() + line = await reader.readline() + if not line: + raise ConnectionError("the daemon closed the connection") return json.loads(line) - - def _close(self) -> None: - if self._stream is not None: - self._stream[1].close() - self._stream = None + except (OSError, ValueError) as e: + raise _ReplyLost(f"cua-driver's daemon stopped answering after the request was sent, which it may have acted " + f"on: {e}") from None + finally: + writer.close() @functools.cache @@ -241,11 +210,6 @@ def _manifest() -> list[dict]: return json.loads(MANIFEST.read_text()) -@functools.cache -def _read_only() -> frozenset[str]: - return frozenset(t["name"] for t in _manifest() if (t["annotations"] or {}).get("readOnlyHint")) - - def _modifiers(keys) -> list[str]: if held := [key for key in keys if key not in MODIFIERS]: raise Unsupported(f"cua-driver holds only modifier keys, not {held}") @@ -275,12 +239,27 @@ def _skill_pack() -> tuple[str, str, dict[str, str]]: with tempfile.TemporaryDirectory(prefix="cua-driver-skill-") as home: (Path(home) / ".claude").mkdir() subprocess.run([str(_executable()), "mcp-config", "--client", "claude"], capture_output=True, check=True, - timeout=SKILL_TIMEOUT_S, env={**os.environ, "HOME": home, "DO_NOT_TRACK": "1", - "CUA_DRIVER_RS_TELEMETRY_ENABLED": "false", - "CUA_DRIVER_RS_UPDATE_CHECK": "false"}) + timeout=SKILL_TIMEOUT_S, env={**os.environ, "HOME": home, **QUIET}) folder = Path(home) / ".cua-driver" / "skills" / "cua-driver" files = {path.name: path.read_text() for path in sorted(folder.iterdir()) if path.is_file()} _, frontmatter, body = files.pop("SKILL.md").split("---\n", 2) description = next(line.partition(":")[2].strip() for line in frontmatter.splitlines() if line.startswith("description:")) return description, body.lstrip("\n"), files + + +def _daemon() -> None: + """Run cua-driver's daemon once X answers: from the session user's home, where the apps it launches start too, on + the socket the driver reaches it at, with approvals off, since nobody is at the computer to give them, and nothing + sent to trycua.""" + display = os.environ["DISPLAY"].lstrip(":").partition(".")[0] + while not Path(f"/tmp/.X11-unix/X{display}").is_socket(): + time.sleep(0.2) + os.chdir(Path.home()) + executable = str(_executable()) + os.execve(executable, [executable, "serve", "--socket", str(SOCKET), "--dangerously-bypass-approvals", "--no-overlay"], + {**os.environ, **QUIET}) + + +if __name__ == "__main__": + _daemon() diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml index c5247c8b0..7e0bc60cc 100644 --- a/src/agent_env/drivers/envs/cua-driver/env.toml +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -5,6 +5,6 @@ port = 18765 [install.linux] python = "requirements.txt" services = [ - { name = "daemon", command = "./daemon.sh" }, + { name = "daemon", command = "venv/bin/python driver.py" }, { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, ] diff --git a/tst/integration/computer/cua_driver_test.py b/tst/integration/computer/cua_driver_test.py index fa20a1331..bd1f5a9dc 100644 --- a/tst/integration/computer/cua_driver_test.py +++ b/tst/integration/computer/cua_driver_test.py @@ -1,7 +1,7 @@ """The cua-driver driver on the ubuntu-24.04 base, as the cua-driver test bundle's desktop names it: its put composes an image for each arch this machine's Docker builds, gate 3 makes this machine's ready at full with cua-driver's own tools beside the computer_* tools; deployed, an agent's session drives Thunar by element token through cua-driver's tools, -computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local +in a cua-driver session another agent's doesn't see, computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local registry on localhost:5000.""" import asyncio @@ -45,7 +45,8 @@ def test_gate_3_makes_this_machines_arch_ready_with_cua_drivers_tools_beside_the @pytest.mark.asyncio -async def test_an_agent_drives_thunar_by_element_token_and_its_skill_leads_with_cua_drivers_pack(cua_desktop): +async def test_an_agent_drives_thunar_by_element_token_in_its_own_session_and_its_skill_leads_with_cua_drivers_pack( + cua_desktop): with namespace_routing(): deployed = await Env.get(cua_desktop.id).deploy() context = TaskStepContext(deployed_envs=[deployed]) @@ -53,6 +54,8 @@ async def test_an_agent_drives_thunar_by_element_token_and_its_skill_leads_with_ async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() + started = await session.call_tool("start_session", {"session": "agent"}) + mine = await session.call_tool("list_sessions", {}) launched = await session.call_tool("launch_app", {"name": "thunar"}) window = await _window(session, "user - Thunar") state = await session.call_tool("get_window_state", {"pid": window["pid"], @@ -61,12 +64,17 @@ async def test_an_agent_drives_thunar_by_element_token_and_its_skill_leads_with_ row = re.search(r'\[(\d+)\] push button "Open Parent"', tree).group(1) clicked = await session.call_tool("click", {"element_token": f"{snapshot}:{row}"}) parent = await _window(session, "home - Thunar") + async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as other: + await other.initialize() + others = await other.call_tool("list_sessions", {}) ready = await deployed.invoke(COMPUTER_READY, "status", None) skill = await deployed.invoke(SKILL_EXTENSION_URI, "get", {"name": "desktop"}) finally: report = await teardown_run(context) assert not launched.isError and not clicked.isError and parent["window_id"] == window["window_id"] + assert not started.isError and "agent" in _named(mine) and "agent" not in _named(others) checks = {check["name"]: check for check in ready["checks"]} assert ready["ready"] and checks["cua-driver"]["ok"] and checks["cua-driver tools"]["detail"] == "66 tools, as pinned" assert {"LINUX.md", "WORKFLOW.md", "RUNTIME.md"} <= set(skill["files"]) @@ -74,6 +82,10 @@ async def test_an_agent_drives_thunar_by_element_token_and_its_skill_leads_with_ assert not report.failed +def _named(listed) -> set[str]: + return {session["session"] for session in listed.structuredContent["sessions"]} + + async def _window(session: ClientSession, title: str) -> dict: """Thunar's window once its title is ``title``.""" for _ in range(40): diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index 37545ea7b..39eb035d6 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -1,6 +1,6 @@ """The cua-driver driver against a fake cua-driver daemon on a Unix socket: the computer/v1 operations become its -desktop-scope calls in the framework's session, its own tools are listed as pinned and forwarded on their caller's -connection as they came, readiness checks the pinned executable and tools, and the skill leads with cua-driver's pack.""" +desktop-scope calls in the framework's own session, its own tools are listed as pinned and forwarded in their caller's +own session as they came, readiness checks the pinned executable and tools, and the skill leads with cua-driver's pack.""" from __future__ import annotations import asyncio @@ -20,6 +20,7 @@ from PIL import Image as PILImage import agent_env +from agentenv_protocol import SESSION_META_KEY from agentenv_protocol.computer import DeliveryFailed, DeviceUnavailable, Unsupported DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" @@ -91,9 +92,9 @@ def reply(self, request: dict) -> dict: return {"ok": True, "result": {"content": [{"type": "text", "text": f"did {name}"}], "structuredContent": {"tool": name}}} - def calls(self, connection: int | None = None) -> list[tuple[str, dict]]: - return [(r["name"], r["args"]) for c, r in self.requests if r["method"] == "call" - and (connection is None or c == connection)] + def calls(self, session: str | None = None) -> list[tuple[str, dict]]: + return [(r["name"], r["args"]) for _, r in self.requests if r["method"] == "call" + and (session is None or r["session_id"] == session)] @pytest.fixture @@ -124,7 +125,7 @@ async def mcp_session(driver): @pytest.mark.asyncio -async def test_the_operations_become_desktop_scope_calls_in_the_frameworks_session(module, daemon): +async def test_the_operations_become_desktop_scope_calls_in_the_frameworks_own_session(module, daemon): driver = module.CuaDriver() size, shot = await driver.screen(), await driver.grab() @@ -136,18 +137,17 @@ async def test_the_operations_become_desktop_scope_calls_in_the_frameworks_sessi await driver.press_keys(["Enter"]) assert (size.width, size.height, shot.width, shot.height, shot.mode) == (1280, 800, 1280, 800, "png") - framework = {"session": "agentenv"} assert daemon.calls() == [ - ("get_screen_size", framework), ("get_desktop_state", framework), - ("click", {"scope": "desktop", "x": 100, "y": 200, "button": "right", "count": 2, "modifier": ["ctrl"], **framework}), + ("get_screen_size", {}), ("get_desktop_state", {}), + ("click", {"scope": "desktop", "x": 100, "y": 200, "button": "right", "count": 2, "modifier": ["ctrl"]}), ("drag", {"scope": "desktop", "from_x": 10, "from_y": 20, "to_x": 30, "to_y": 40, "button": "left", - "duration_ms": 300, "modifier": ["shift"], **framework}), - ("scroll", {"scope": "desktop", "x": 5, "y": 6, "direction": "down", "amount": 3, **framework}), - ("type_text", {"scope": "desktop", "text": "Hello, Wörld! 東京", **framework}), - ("press_key", {"scope": "desktop", "key": "/", "modifiers": ["ctrl"], **framework}), - ("press_key", {"scope": "desktop", "key": "enter", "modifiers": [], **framework}), + "duration_ms": 300, "modifier": ["shift"]}), + ("scroll", {"scope": "desktop", "x": 5, "y": 6, "direction": "down", "amount": 3}), + ("type_text", {"scope": "desktop", "text": "Hello, Wörld! 東京"}), + ("press_key", {"scope": "desktop", "key": "/", "modifiers": ["ctrl"]}), + ("press_key", {"scope": "desktop", "key": "enter", "modifiers": []}), ] - assert daemon.connections == 1 + assert len({r["session_id"] for _, r in daemon.requests}) == 1 @pytest.mark.asyncio @@ -167,7 +167,7 @@ async def test_what_cua_driver_cant_do_or_refuses_fails_naming_it(module, daemon @pytest.mark.asyncio -async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_on_their_callers_connection(module, daemon): +async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_in_their_callers_session(module, daemon): driver = module.CuaDriver() async with mcp_session(driver) as session: listed = {t.name: t for t in (await session.list_tools()).tools} @@ -179,13 +179,24 @@ async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_on_their_cal assert set(manifest) <= set(listed) and {"computer_click", "computer_screenshot"} <= set(listed) assert listed["get_window_state"].inputSchema == manifest["get_window_state"]["inputSchema"] assert listed["get_window_state"].annotations.readOnlyHint == manifest["get_window_state"]["annotations"]["readOnlyHint"] - framework, caller = 1, max(c for c, _ in daemon.requests) + framework, *_, caller = [r["session_id"] for _, r in daemon.requests if r["method"] == "call"] assert caller != framework and daemon.calls(caller) == [ ("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}), ("launch_app", {"name": "thunar"})] assert not launched.isError assert answered.structuredContent == {"tool": "get_window_state"} and answered.content[0].text == "did get_window_state" +@pytest.mark.asyncio +async def test_each_caller_gets_a_cua_driver_session_of_its_own(module, daemon): + async with mcp_session(module.CuaDriver()) as session: + for caller in ("a", "b", "a"): + await session.call_tool("list_windows", {}, meta={SESSION_META_KEY: caller}) + await session.call_tool("get_session", {"session": "mine"}, meta={SESSION_META_KEY: "b"}) + + a, b, again, named = [(r["session_id"], r["args"]) for _, r in daemon.requests if r["method"] == "call"] + assert a[0] != b[0] and again == a and named == (b[0], {"session": "mine"}) + + @pytest.mark.asyncio async def test_a_failed_call_to_one_of_cua_drivers_tools_reaches_its_caller_as_an_error(module, daemon): daemon.fail.add("list_apps") @@ -225,23 +236,19 @@ async def test_readiness_waits_for_a_daemon_that_isnt_answering(module, monkeypa @pytest.mark.asyncio -async def test_a_read_is_sent_again_after_the_daemon_restarts_but_input_is_not(module, daemon): +async def test_while_the_daemon_restarts_calls_fail_as_unavailable_and_reach_it_once_it_is_back(module, daemon): driver = module.CuaDriver() await driver.screen() await daemon.stop() with pytest.raises(DeviceUnavailable, match="cua-driver's daemon isn't answering"): await driver.screen() + with pytest.raises(DeviceUnavailable) as unsent: + await driver.click_at(1, 1, "left", 1, []) await daemon.start() - assert (await driver.screen()).width == 1280 - await daemon.stop() - await daemon.start() assert (await driver.screen()).width == 1280 - await daemon.stop() - await daemon.start() - with pytest.raises(DeliveryFailed, match="may have acted on"): - await driver.click_at(1, 1, "left", 1, []) await driver.click_at(2, 2, "left", 1, []) + assert type(unsent.value) is DeviceUnavailable and unsent.value.applied == "no" assert [args["x"] for name, args in daemon.calls() if name == "click"] == [2] @@ -268,21 +275,6 @@ async def test_a_request_cancelled_in_flight_leaves_no_reply_for_the_next_one(mo assert (shot.width, shot.mode) == (1280, "png") and daemon.connections == 2 -@pytest.mark.asyncio -async def test_one_of_cua_drivers_read_only_tools_is_sent_again_after_a_restart_and_its_input_tools_are_not(module, - daemon): - async with mcp_session(module.CuaDriver()) as session: - await session.call_tool("list_windows", {}) - await daemon.stop() - await daemon.start() - read = await session.call_tool("list_windows", {}) - await daemon.stop() - await daemon.start() - clicked = await session.call_tool("click", {"element_token": "s1:1"}) - assert not read.isError and clicked.isError and "may have acted on" in clicked.content[0].text - assert [name for name, _ in daemon.calls()] == ["list_windows", "list_windows"] - - def test_the_skill_pack_is_written_offline_from_the_executable(module, monkeypatch, tmp_path): executable = tmp_path / "cua-driver" executable.write_text("""#!/bin/sh From fdee99c59d034319136c3f1ffaa9f455281afa89 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 15:59:55 -0700 Subject: [PATCH 10/13] test(config): DISPLAY, which the cua-driver driver reads on the computer, isn't agent-env configuration Co-Authored-By: Claude Opus 5.5 --- tst/unit/config/test_explain_and_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tst/unit/config/test_explain_and_sources.py b/tst/unit/config/test_explain_and_sources.py index 676e016fb..07d55ad1f 100644 --- a/tst/unit/config/test_explain_and_sources.py +++ b/tst/unit/config/test_explain_and_sources.py @@ -163,7 +163,7 @@ def test_sources_says_how_the_file_was_found(config_file, tmp_path, monkeypatch) # Read for identity (USER), or set from [sandbox.providers.sail_vm.config] for the Sail SDK to read # (SAIL_*), never configuration agent-env takes, so none is a layer. # AGENT_ENV_INSTALLED_REQUIRES marks a command run again after installing what a bundle requires, so it never asks twice. -_NOT_CONFIGURATION = {"USER", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", "AGENT_ENV_INSTALLED_REQUIRES"} +_NOT_CONFIGURATION = {"USER", "DISPLAY", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", "AGENT_ENV_INSTALLED_REQUIRES"} def _env_vars_read_by(root): From f5260385996b93b7893e7ec4b290eec28e02c69b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 16:14:43 -0700 Subject: [PATCH 11/13] feat(computer): serve cua-driver's instructions, output schemas and skill resources as its own MCP server does - The SDK gains instructions(), what an environment's MCP server tells an agent when it connects; forwarded_resources() with ForwardedResource, MCP resources served on another server's behalf; and an output_schema on ForwardedTool, listed as the tool's outputSchema and not checked here. - The cua-driver driver serves what cua-driver mcp 0.38.0 serves: its instructions, its 66 tools with their output schemas (39 declare one), and its 11 skill resources at skill://cua-driver/, whose text is read from cua-driver mcp itself. mcp.json pins the instructions, tools and resource list and replaces tools.json. - A failed call is answered as cua-driver mcp answers one, structured content {"code": "tool_invocation_failed", "exit_code": n}, which every tool's output schema admits, so strict clients still pass the message on. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 17 +- .../src/agentenv_protocol/__init__.py | 2 + .../agent_env_environment.py | 59 +- .../tests/test_tool_annotations.py | 32 +- src/agent_env/drivers/README.md | 32 +- .../drivers/envs/cua-driver/driver.py | 57 +- .../drivers/envs/cua-driver/mcp.json | 10245 ++++++++++++++++ .../drivers/envs/cua-driver/tools.json | 3832 ------ tst/integration/computer/cua_driver_test.py | 14 +- tst/unit/drivers/cua_driver_test.py | 52 +- 11 files changed, 10456 insertions(+), 3888 deletions(-) create mode 100644 src/agent_env/drivers/envs/cua-driver/mcp.json delete mode 100644 src/agent_env/drivers/envs/cua-driver/tools.json diff --git a/AGENTS.md b/AGENTS.md index 57521db1e..9a7e53acb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,7 +124,7 @@ and deserialized through a registry. | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | | `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own tools forwarded to agents through `forwarded_tools` beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `instructions`, `forwarded_tools` and `forwarded_resources`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index d21dd5c67..ca764c9e5 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -243,12 +243,17 @@ Where the binding lives depends on the app. On mcp's own FastMCP it wraps the to ### Tools served on another server's behalf An environment can serve tools another server defines, each with that server's own schema: -`forwarded_tools()` returns `ForwardedTool(tool, call, annotations)` entries, where `tool` is an -`EnvironmentTool` carrying the other server's `inputSchema`, and `call` takes the call's arguments -as keywords, untouched, and returns that server's `CallToolResult`, which reaches the caller as it -is. They're listed on the card beside the `@tool` methods, chosen by `served_tools` and under tool -access like them; one named like another tool is refused. They need mcp's FastMCP. The cua-driver -driver serves cua-driver's own tools this way. +`forwarded_tools()` returns `ForwardedTool(tool, call, annotations, output_schema)` entries, where +`tool` is an `EnvironmentTool` carrying the other server's `inputSchema`, and `call` takes the +call's arguments as keywords, untouched, and returns that server's `CallToolResult`, which reaches +the caller as it is. An `output_schema` is listed as the tool's `outputSchema` and isn't checked +here, so `call` answers every result, its errors too, in a shape that schema admits. They're listed +on the card beside the `@tool` methods, chosen by `served_tools` and under tool access like them; +one named like another tool is refused. In the same way `forwarded_resources()` returns +`ForwardedResource(uri, name, description, mime_type, read)` entries, MCP resources whose `read` +returns their text, and `instructions()` returns what the MCP server tells an agent when it +connects, which clients may put in its context. All of them need mcp's FastMCP. The cua-driver +driver serves cua-driver's own instructions, tools and skill resources this way. ### Getting part of the data diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index 0b9f68f17..45d03549d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -11,6 +11,7 @@ AgentEnvEnvironment, AgentEnvFastMCPApplication, AgentEnvStarletteApplication, + ForwardedResource, ForwardedTool, add_data, agent_skill, @@ -89,6 +90,7 @@ "AgentEnvEnvironment", "AgentEnvFastMCPApplication", "AgentEnvStarletteApplication", + "ForwardedResource", "ForwardedTool", "create_fastmcp_app", "environment_card", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 119b93fae..bba93b75e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -14,7 +14,7 @@ import types import uuid from abc import ABC, abstractmethod -from typing import Annotated, Any, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints +from typing import Annotated, Any, Awaitable, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints from pydantic import TypeAdapter, ValidationError from pydantic.fields import FieldInfo @@ -208,12 +208,24 @@ def deco(fn: Callable) -> Callable: class ForwardedTool(NamedTuple): """A tool an environment serves on another server's behalf, as that server lists it: its descriptor, its own - schema included, and its MCP ``annotations``; ``call`` takes the call's arguments as keywords and returns the - other server's ``CallToolResult``, which reaches the caller as it is.""" + schema included, its MCP ``annotations`` and its ``output_schema``; ``call`` takes the call's arguments as keywords + and returns the other server's ``CallToolResult``, which reaches the caller as it is.""" tool: EnvironmentTool call: Callable[..., Any] annotations: dict | None = None + output_schema: dict | None = None + + +class ForwardedResource(NamedTuple): + """An MCP resource an environment serves on another server's behalf, as that server lists it; ``read`` returns its + text.""" + + uri: str + name: str + description: str | None + mime_type: str + read: Callable[[], Awaitable[str]] class _ToolOptions(NamedTuple): @@ -256,6 +268,11 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: tool_handlers = _served_tools(handler, _with_forwarded(_discover_tools(handler, environment_card.name), handler)) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") + hook = getattr(handler, "forwarded_resources", None) + self._resources: list[ForwardedResource] = hook() if hook is not None else [] + if self._resources and not self.supports_tools: + raise ValueError(f"forwarded resources require a FastMCP-backed application; {type(self).__name__} cannot " + "serve MCP resources") environment_card = _merge_tools(environment_card, tool_handlers) self._tool_access = ( ToolAccess(descriptor.name for descriptor, fn in tool_handlers if getattr(fn, _HIDDEN_BY_DEFAULT_ATTR, False)) @@ -290,6 +307,8 @@ def add_routes_to_app(self, app: Any, *, rpc_url: str = RPC_PATH, card_url: str self._add_route(app, path, [method], handler) for descriptor, fn in self._tools: self._register_tool(app, descriptor, fn) + for resource in self._resources: + self._register_resource(app, resource) # Last, so it is the outermost layer: a dispatch guard the handler installed before mounting # then runs under the bound ToolContext instead of outside it. self._bind_tool_calls(app) @@ -301,6 +320,9 @@ def _add_route(self, app: Any, path: str, methods: list, handler: Callable) -> N def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> None: raise NotImplementedError(f"{type(self).__name__} does not serve MCP tools") + def _register_resource(self, app: Any, resource: ForwardedResource) -> None: + raise NotImplementedError(f"{type(self).__name__} does not serve MCP resources") + def _check_tool_binding(self, app: Any) -> None: """Everything that can refuse the mount, before the app is touched; nothing to refuse without a tool dispatch.""" return None @@ -373,12 +395,23 @@ def _register_forwarded(self, app: Any, descriptor: EnvironmentTool, fn: Callabl if self._binding != "dispatch": raise RuntimeError(f"the forwarded tool {descriptor.name!r} needs mcp's FastMCP, whose tool manager takes a " "tool with its own schema") + schema = forwarded.output_schema app._tool_manager._tools[descriptor.name] = Tool( fn=fn, name=descriptor.name, description=descriptor.description or "", - parameters=descriptor.inputSchema or {"type": "object"}, fn_metadata=FuncMetadata(arg_model=_passed_through()), + parameters=descriptor.inputSchema or {"type": "object"}, + fn_metadata=FuncMetadata(arg_model=_passed_through(), output_schema=schema, + output_model=_as_it_came() if schema is not None else None), is_async=True, annotations=_mcp_tool_annotations(forwarded.annotations) if forwarded.annotations is not None else None) + def _register_resource(self, app: Any, resource: ForwardedResource) -> None: + from mcp.server.fastmcp.resources import FunctionResource + + if self._binding != "dispatch": + raise RuntimeError(f"the forwarded resource {resource.uri!r} needs mcp's FastMCP") + app.add_resource(FunctionResource(uri=resource.uri, name=resource.name, description=resource.description, + mime_type=resource.mime_type, fn=resource.read)) + def _check_tool_binding(self, app: Any) -> None: """Decide, before any route or tool is added, how every tools/call on ``app`` gets its ToolContext, and refuse what cannot be honoured. The standalone ``fastmcp`` package (a ``middleware`` list with @@ -542,7 +575,7 @@ def _build_card(self) -> EnvironmentCard: def create_app(self) -> Any: if self._mounted: raise RuntimeError(f"{type(self).__name__} is already mounted") - self.mcp = create_fastmcp_app(self, card=self._build_card()) + self.mcp = create_fastmcp_app(self, card=self._build_card(), instructions=self.instructions()) self._mounted = True return self.mcp @@ -600,6 +633,14 @@ def forwarded_tools(self) -> list[ForwardedTool]: the card, chosen by ``served_tools`` and under tool access like them. Needs mcp's FastMCP.""" return [] + def forwarded_resources(self) -> list[ForwardedResource]: + """MCP resources served on another server's behalf. Needs mcp's FastMCP.""" + return [] + + def instructions(self) -> str | None: + """What this environment's MCP server tells an agent when it connects, which clients may put in its context.""" + return None + def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -1135,6 +1176,14 @@ async def call(**arguments: Any) -> Any: return call +@functools.cache +def _as_it_came() -> type: + """mcp's FastMCP output model for a forwarded tool with an output schema: the other server's result, unchecked.""" + from pydantic import RootModel + + return RootModel[Any] + + @functools.cache def _passed_through() -> type: """mcp's FastMCP argument model for a forwarded tool: any arguments, each passed to the call as it came.""" diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index fbbdee94d..8b46a05d6 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -20,6 +20,7 @@ AgentEnvFastMCPApplication, EnvironmentCard, EnvironmentTool, + ForwardedResource, ForwardedTool, environment_card, tool, @@ -214,6 +215,7 @@ async def test_update_tool_needs_a_mounted_env_and_a_served_tool(): _CLICK = {"type": "object", "additionalProperties": False, "properties": { "element_token": {"type": "string"}, "x": {"type": "integer"}, "count": {"type": "integer", "minimum": 1}}} +_FOUND = {"type": "object", "required": ["rows"], "properties": {"rows": {"type": "array"}}} @environment_card(name="forwarding") @@ -232,8 +234,21 @@ async def click(**arguments) -> CallToolResult: ImageContent(type="image", data="aGk=", mimeType="image/png")], structuredContent={"hit": "button OK"}) + async def find(**arguments) -> CallToolResult: + return CallToolResult(isError=True, content=[TextContent(type="text", text="the daemon isn't answering")]) + return [ForwardedTool(EnvironmentTool(name="click", description="Click an element.", inputSchema=_CLICK), click, - {"readOnlyHint": False, "destructiveHint": False})] + {"readOnlyHint": False, "destructiveHint": False}), + ForwardedTool(EnvironmentTool(name="find", inputSchema={"type": "object"}), find, None, _FOUND)] + + def forwarded_resources(self) -> list[ForwardedResource]: + async def read() -> str: + return "# Guide" + + return [ForwardedResource("skill://other/SKILL.md", "other", "How to use it.", "text/markdown", read)] + + def instructions(self) -> str: + return "Read the guide first." @pytest.mark.asyncio @@ -244,14 +259,29 @@ async def test_a_forwarded_tool_is_listed_with_its_own_schema_and_called_with_it async with create_connected_server_and_client_session(env.create_app()) as session: listed = {t.name: t for t in (await session.list_tools()).tools} result = await session.call_tool("click", {"element_token": "s1:4", "count": 2}) + found = await session.call_tool("find", {}) assert listed["click"].inputSchema == _CLICK and listed["click"].description == "Click an element." assert listed["click"].annotations.readOnlyHint is False and listed["click"].outputSchema is None + assert listed["find"].outputSchema == _FOUND and found.isError and found.content[0].text == "the daemon isn't answering" assert calls == [{"element_token": "s1:4", "count": 2}] assert [c.type for c in result.content] == ["text", "image"] and result.structuredContent == {"hit": "button OK"} assert {t.name: t.inputSchema for t in env.mounted_card().capabilities.tools}["click"] == _CLICK +@pytest.mark.asyncio +async def test_forwarded_resources_and_the_instructions_are_served_as_the_other_server_serves_them(): + app = _Forwarding([]).create_app() + with anyio.fail_after(30): + async with create_connected_server_and_client_session(app) as session: + listed = (await session.list_resources()).resources + read = await session.read_resource("skill://other/SKILL.md") + + assert [(str(r.uri), r.name, r.description, r.mimeType) for r in listed] == [ + ("skill://other/SKILL.md", "other", "How to use it.", "text/markdown")] + assert read.contents[0].text == "# Guide" and app.instructions == "Read the guide first." + + def test_a_forwarded_tool_named_like_a_tool_is_refused(): class Clashing(_Forwarding): @tool() diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index c9bf86e91..4be6d4078 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -17,24 +17,28 @@ driver, where it runs. `envs/cua-driver/` is trycua's [cua-driver](https://github.com/trycua/cua/tree/main/libs/cua-driver) as a driver, for the same X11 desktop on Linux (`amd64` and `arm64`). cua-driver 0.38.0 runs on the computer as a daemon, `cua-driver serve`, which `driver.py` starts when run as a script, once X answers, with approvals off and no telemetry. Agents get -its own 66 tools beside the `computer_*` tools, as `tools.json` lists them: element tokens from `get_window_state`, -clicks and typing by element or pixel, `run_script` and its browser tools. Each call is forwarded to the daemon in a -cua-driver session of the caller's own unless it names one, as cua-driver's own MCP server gives each client. Element -tokens belong to a window, not a session: an agent reading a window another agent has read makes the other's tokens for -it stale. The `computer_*` tools, gate 3's probe, `computer-ready` and recording run on its desktop-scope calls (XTest) -in a session of their own; a desktop capture leaves window snapshots alone, so it never makes an agent's element tokens -stale. It has no pointer or keyboard hold group, drags from a start to an end, and scrolls with no keys held. Its skill -leads with cua-driver's own skill pack, as the executable embeds it, its other files beside the SKILL.md. An env narrows -which tools its agent sees with `[tools]`. `requirements.txt` installs `cua-driver==0.38.0` from PyPI, whose wheel -carries the executable, and `computer-ready` checks that executable's sha256 and that the daemon lists the tools -`tools.json` pins. Name it as `cua-driver`, the alias `aliases.toml` gives it, or by its id, -`@local/agentenv-framework/drivers/cua-driver`. +what cua-driver's own MCP server (`cua-driver mcp`) serves, as `mcp.json` pins it: its instructions when they connect, +its 66 tools beside the `computer_*` tools, with their output schemas (element tokens from `get_window_state`, clicks +and typing by element or pixel, `run_script` and its browser tools), and its skill pack as `skill://cua-driver/` +resources, whose text is read from that server. A call that fails is answered as that server answers one, +`tool_invocation_failed` with its exit code. Each call is forwarded to the daemon in a cua-driver session of the +caller's own unless it names one, as cua-driver's own MCP server gives each client. Element tokens belong to a window, +not a session: an agent reading a window another agent has read makes the other's tokens for it stale. The `computer_*` +tools, gate 3's probe, `computer-ready` and recording run on its desktop-scope calls (XTest) in a session of their own; +a desktop capture leaves window snapshots alone, so it never makes an agent's element tokens stale. It has no pointer or +keyboard hold group, drags from a start to an end, and scrolls with no keys held. Its skill leads with cua-driver's own +skill pack, as the executable embeds it, its other files beside the SKILL.md. An env narrows which tools its agent sees +with `[tools]`. `requirements.txt` installs `cua-driver==0.38.0` from PyPI, whose wheel carries the executable, and +`computer-ready` checks that executable's sha256 and that the daemon lists the tools `mcp.json` pins. Name it as +`cua-driver`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/cua-driver`. To move it to another cua-driver version: 1. Pin the version in `requirements.txt`. -2. Regenerate `tools.json` from that version's tools as its MCP server lists them: each tool's `name`, `description`, - `inputSchema` and `annotations`. Listing them takes a running daemon, so do it on a Linux test machine. +2. Regenerate `mcp.json` from what that version's MCP server, `cua-driver mcp`, says at connect and lists: its + `instructions`, each tool's `name`, `description`, `inputSchema`, `annotations` and `outputSchema`, and each + resource's `uri`, `name`, `description` and `mimeType`. That takes a running daemon, so do it on a Linux test + machine. 3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel. 4. Run `tst/unit/drivers/cua_driver_test.py`, then `tst/integration/computer/cua_driver_test.py` on Linux, where `computer-ready` names any tool or executable that doesn't match its pin. diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index ae819cf72..8ac7e1877 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -1,6 +1,7 @@ """cua-driver as a computer driver, for an X11 desktop on Linux: trycua's cua-driver runs on the computer as a daemon, -which this file starts when run as a script, agents get its own tools as its pinned version lists them (``tools.json``), -beside the ``computer_*`` tools, and the computer/v1 operations are its desktop-scope calls. The daemon takes +which this file starts when run as a script, agents get what its own MCP server serves at its pinned version +(``mcp.json``): its instructions, its tools beside the ``computer_*`` tools, and its skill resources, read from it. The +computer/v1 operations are its desktop-scope calls. The daemon takes line-delimited JSON on a Unix socket, and each call goes on a connection of its own, as cua-driver's own MCP server sends them. The framework and each caller, told apart by MCP session, have an unnamed cua-driver session of their own, which cua-driver starts again after ending it for idling, where a named one stays ended. A call the daemon can't take, as @@ -22,9 +23,12 @@ from pathlib import Path from typing import Any +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client from mcp.types import CallToolResult, TextContent +from pydantic import AnyUrl -from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext +from agentenv_protocol import EnvironmentTool, ForwardedResource, ForwardedTool, ToolContext from agentenv_protocol.computer import ( PRINTABLE_KEYS, ComputerDriverEnvironment, @@ -46,8 +50,9 @@ #: Where cua-driver's daemon listens. SOCKET = Path("/run/agentenv/cua-driver.sock") -#: cua-driver's tools as its pinned version lists them, which the running daemon must list the same. -MANIFEST = Path(__file__).with_name("tools.json") +#: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version: its +#: instructions, its tools, which the running daemon must list the same, and its skill resources. +MANIFEST = Path(__file__).with_name("mcp.json") #: The sha256 of the cua-driver executable each arch's pinned wheel installs. EXECUTABLES = { "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", @@ -83,11 +88,20 @@ def __init__(self) -> None: self._callers: dict[str | None, str] = {} self._pinned: list[ReadyCheck] | None = None self._skill: tuple[str, str, dict[str, str]] | None = None + self._texts: dict[str, str] | None = None def forwarded_tools(self) -> list[ForwardedTool]: return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]), - functools.partial(self._forward, t["name"]), t["annotations"] or None) - for t in _manifest()] + functools.partial(self._forward, t["name"]), t["annotations"] or None, t.get("outputSchema")) + for t in _manifest()["tools"]] + + def forwarded_resources(self) -> list[ForwardedResource]: + return [ForwardedResource(r["uri"], r["name"], r.get("description"), r["mimeType"], + functools.partial(self._resource, r["uri"])) + for r in _manifest()["resources"]] + + def instructions(self) -> str: + return _manifest()["instructions"] async def ready_checks(self) -> list[ReadyCheck]: """That the daemon answers as the pinned cua-driver does: its executable's sha256 and its tools, checked @@ -145,9 +159,15 @@ async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: except DeviceUnavailable as e: return _refused(e.message) if not reply.get("ok"): - return _refused(reply.get("error") or f"cua-driver refused {tool}") + return _refused(reply.get("error") or "daemon reported failure", reply.get("exit_code") or 1) return CallToolResult.model_validate(reply["result"]) + async def _resource(self, uri: str) -> str: + """One of cua-driver's skill resources as its own MCP server serves it, all of them read on first use.""" + if self._texts is None: + self._texts = await _read_resources([r["uri"] for r in _manifest()["resources"]]) + return self._texts[uri] + async def _send(self, name: str, **args: Any) -> None: await self._call(name, DeliveryFailed, scope="desktop", **args) @@ -172,12 +192,12 @@ async def _pinned_checks(self) -> list[ReadyCheck]: return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")] digest, expected = await asyncio.to_thread(_executable_sha256), EXECUTABLES.get(self.platform.arch or "") running = {t["name"]: t["input_schema"] for t in listed} - pinned = {t["name"]: t["inputSchema"] for t in _manifest()} + pinned = {t["name"]: t["inputSchema"] for t in _manifest()["tools"]} differ = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name)) return [ReadyCheck("cua-driver", digest == expected, "the pinned executable" if digest == expected else f"its executable's sha256 is {digest}, not the pinned {expected}"), ReadyCheck("cua-driver tools", not differ, f"{len(running)} tools, as pinned" if not differ else - f"these tools differ from tools.json: {', '.join(differ)}")] + f"these tools differ from mcp.json: {', '.join(differ)}")] class _ReplyLost(DeviceUnavailable): @@ -205,8 +225,17 @@ async def _request(payload: dict) -> dict: writer.close() +async def _read_resources(uris: list[str]) -> dict[str, str]: + """cua-driver's skill resources as its own MCP server serves them, on the running daemon.""" + server = StdioServerParameters(command=str(_executable()), args=["mcp", "--socket", str(SOCKET)], + env={**os.environ, **QUIET}) + async with stdio_client(server) as (read, write), ClientSession(read, write) as session: + await session.initialize() + return {uri: (await session.read_resource(AnyUrl(uri))).contents[0].text for uri in uris} + + @functools.cache -def _manifest() -> list[dict]: +def _manifest() -> dict: return json.loads(MANIFEST.read_text()) @@ -216,8 +245,10 @@ def _modifiers(keys) -> list[str]: return [MODIFIERS[key] for key in keys] -def _refused(message: str) -> CallToolResult: - return CallToolResult(isError=True, content=[TextContent(type="text", text=message)]) +def _refused(message: str, exit_code: int = 1) -> CallToolResult: + """A failed call as cua-driver's own MCP server answers one, which every tool's output schema admits.""" + return CallToolResult(isError=True, content=[TextContent(type="text", text=message)], + structuredContent={"code": "tool_invocation_failed", "exit_code": exit_code}) def _text(result: dict) -> str: diff --git a/src/agent_env/drivers/envs/cua-driver/mcp.json b/src/agent_env/drivers/envs/cua-driver/mcp.json new file mode 100644 index 000000000..d43790a74 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/mcp.json @@ -0,0 +1,10245 @@ +{ + "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AT-SPI): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused or unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`, LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support).", + "tools": [ + { + "name": "list_apps", + "description": "List Linux apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: reserved (Linux X11/Wayland focus model differs from frontmost-app); always false.\n- kind: `\"desktop\"` for XDG `.desktop` launcher entries.\n- launch_path: the launcher command from `Exec=` (field codes stripped). Pass to `launch_app(launch_path=...)`.\n- bundle_id: the XDG \"desktop file id\" — the `.desktop` file's path relative to its XDG `applications/` root with the `.desktop` suffix stripped and path separators replaced with `-` (e.g. `kde4/konqbrowser.desktop` → `kde4-konqbrowser`).\n- last_used: RFC3339 mtime of the `.desktop` file, when readable.\n- windows: the app's current top-level windows (same records as list_windows); empty for a process without one.\n\nRunning apps come from `/proc`. Installed apps come from XDG Desktop Entry files in $XDG_DATA_HOME/applications and each $XDG_DATA_DIRS entry's applications/ subdir. Entries with `NoDisplay=true` or `Hidden=true` are filtered. A `.desktop` file whose launcher matches a running process (by basename), or whose WM_CLASS a top-level window carries, is merged into a single entry with `running: true`.\n\nUse this for \"is X installed?\" as well as \"is X running?\". For per-window state — visibility, geometry, titles — call list_windows instead.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "apps": { + "items": { + "properties": { + "active": { + "type": "boolean" + }, + "bundle_id": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "type": [ + "string", + "null" + ] + }, + "last_used": { + "type": [ + "string", + "null" + ] + }, + "launch_path": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "running": { + "type": "boolean" + } + }, + "required": [ + "pid", + "name", + "running", + "active" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "apps" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "list_windows", + "description": "List top-level windows. Each record includes z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one). To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order. Filter with `app` or `pid`: an unfiltered call lists every window on the host.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", + "type": "string" + }, + "on_screen_only": { + "description": "When true, filter to visible windows only. Default false.", + "type": "boolean" + }, + "pid": { + "description": "Only list windows owned by this process ID.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "current_space_id": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "windows": { + "items": { + "properties": { + "app_name": { + "type": "string" + }, + "bounds": { + "properties": { + "height": { + "type": "number" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "current_space_id": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "is_on_screen": { + "type": "boolean" + }, + "layer": { + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "minimized": { + "type": [ + "boolean", + "null" + ] + }, + "on_current_space": { + "type": [ + "boolean", + "null" + ] + }, + "pid": { + "maximum": 4294967295, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "space_ids": { + "items": { + "minimum": 0, + "type": "integer" + }, + "type": [ + "array", + "null" + ] + }, + "title": { + "type": "string" + }, + "window_id": { + "minimum": 0, + "type": "integer" + }, + "z_index": { + "description": "Higher values are closer to the front. Null means the provider cannot observe stacking order; callers must not infer an order from array position or treat null as zero.", + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "window_id", + "pid", + "app_name", + "title", + "bounds", + "is_on_screen", + "z_index" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "windows" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_window_state", + "description": "Walk a running app's AT-SPI tree and return it ONCE, as compact Markdown by default (`tree_format:\"markdown\"`). Every actionable element is tagged `[N]` (an element_index); its element_token is `:N`, where `snapshot_id` is printed in the response header and returned in structuredContent. Pass `tree_format:\"elements\"` for the structured `elements` array instead (explicit `element_token`, `role`, `label`, `value`, `enabled`, `selected`, `actions`, `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`), or `\"both\"` for both (about twice the size). Set `query` to project the tree to matching rows plus their ancestor chain while preserving original indices; `total_element_count` reports the complete snapshot.\n\nSize: a read is bounded by default (`max_elements` 250 actionable nodes). When the walk stops at that budget the response says `Tree truncated at max_elements=…` and `truncated: true`; pass a larger `max_elements`, or `query` / `max_depth`, to reach the rest. `_note` is omitted unless `verbose:true`. `full_output:true` restores the previous full response: both representations, all metadata, and the walker's own limits (≤5 000 nodes).\n\nDIFF READS: pass `since: ` from an earlier read of the same window to get only what changed: `+` added rows, `~` changed rows, `-` removed rows (their ids are the old snapshot's), or `no change since …`. Rows not listed keep their previous `[N]` unless a `reindexed:` line says otherwise. The response carries a NEW snapshot_id: use it in element_tokens. An unknown, expired, other-window or differently-scoped (query/max_elements/max_depth) `since` falls back to a full read and `since_status` says why. The focused element is not reported on Linux yet.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error. Optional `max_dimension` caps the returned screenshot's long edge in pixels for a cheap thumbnail.\n\nOptional `max_elements` / `max_depth` bound the AT-SPI walk (Electron / large web apps produce 10k+ element trees). Markdown and structured elements are truncated identically. Default: `max_elements` 250.\n\nTIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. When omitted, a window's first snapshot gets a 2000 ms cold-start budget so Chromium/Electron can finish publishing its initial tree. Explicit values are always honored. Large apps (LibreOffice, GIMP, file managers) can exceed the budget; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.\n\nSCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.\n\nPOPUP MENUS: a context menu / popover / combo list is an override-redirect window that list_windows never shows. A click or right_click that opened one names it in its result (`popup: {window_id, bounds, title}`); pass that window_id here to walk the popup's own AT-SPI toplevel so its menu items get element indices (then click them by element_token). Omitting window_id while a popup of this pid is open walks that popup.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "capture_mode": { + "description": "DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.", + "enum": [ + "ax", + "vision" + ], + "type": "string" + }, + "display_only": { + "description": "Default false. With include_accessibility_tree:false, returns pixels for display without registering an action capture or replacing the agent snapshot. The image cannot ground input actions.", + "type": "boolean" + }, + "full_output": { + "description": "Default false. Set true to restore the previous full response: both `elements` and `tree_markdown`, all metadata, and the platform's own walk limits unless max_elements / max_depth are passed. Does not combine with `since`.", + "type": "boolean" + }, + "include_accessibility_tree": { + "description": "Default true — walk the AT-SPI tree and return it (per `tree_format`) alongside the screenshot. Set false to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return).", + "type": "boolean" + }, + "include_screenshot": { + "description": "Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action).", + "type": "boolean" + }, + "max_depth": { + "description": "Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps.", + "minimum": 1, + "type": "integer" + }, + "max_dimension": { + "description": "Legacy optional cap on the returned screenshot's long edge. Applied on top of the configured max_image_dimension ceiling when max_image_dimension is omitted.", + "minimum": 1, + "type": "integer" + }, + "max_elements": { + "description": "Cap on total AT-SPI nodes walked. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a large window.", + "minimum": 1, + "type": "integer" + }, + "max_image_dimension": { + "description": "Per-call long-edge override. This value wins over configured and legacy limits; 0 returns native-resolution PNG bytes. Omit to preserve configured behavior.", + "minimum": 0, + "type": "integer" + }, + "pid": { + "description": "Process ID that owns the window.", + "type": "integer" + }, + "query": { + "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", + "type": "string" + }, + "screenshot_out_file": { + "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output carries screenshot_file_path instead.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "since": { + "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that snapshot: `+` added rows, `~` changed rows, `-` removed rows (removed ids are the old snapshot's), or `no change`. Falls back to a full read when the snapshot is unknown, expired, belongs to another window, or was read with a different query/max_elements/max_depth; `since_status` says which. The response still carries a NEW snapshot_id and its element_tokens; rows not listed keep their previous [N] unless a `reindexed:` line says otherwise.", + "type": "string" + }, + "timeout_ms": { + "default": 1000, + "description": "Wall-clock budget in milliseconds for the accessibility-tree walk (default 1000, min 100, max 120000). Bounds the WHOLE walk. When the budget runs out the tool returns the PARTIAL tree it has, flagged with `truncated: true`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `elements_complete: false`; retry with a larger value (e.g. 5000) or narrow with `query` / `max_depth`.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + }, + "tree_format": { + "description": "Which tree representation to return. Default \"markdown\": the compact `tree_markdown` only, where a row `[N]` is addressed with element_token `:N`. \"elements\": the structured `elements` array only (per-element element_token, frame, parent_index, value). \"both\": the two together, as before; roughly doubles the size.", + "enum": [ + "markdown", + "elements", + "both" + ], + "type": "string" + }, + "verbose": { + "description": "Default false. Set true to include the `_note` string and the full `background_input` capability report (macOS), which are omitted by default to save context.", + "type": "boolean" + }, + "window_id": { + "description": "Native window identifier from list_windows, or the `popup.window_id` a click / right_click result named (an open context menu / popover; its menu items then get element indices). Omitted: the pid's open popup menu when one is mapped, else its focused / active / largest window.", + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "app_name": { + "type": [ + "string", + "null" + ] + }, + "degraded": { + "type": [ + "boolean", + "null" + ] + }, + "degraded_reason": { + "type": [ + "string", + "null" + ] + }, + "element_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "elements": { + "items": { + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "depth": { + "minimum": 0, + "type": "integer" + }, + "element_index": { + "minimum": 0, + "type": "integer" + }, + "element_token": { + "type": [ + "string", + "null" + ] + }, + "enabled": { + "type": [ + "boolean", + "null" + ] + }, + "frame": { + "properties": { + "h": { + "type": "number" + }, + "w": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "w", + "h" + ], + "type": [ + "object", + "null" + ] + }, + "in_web_content": { + "type": [ + "boolean", + "null" + ] + }, + "label": { + "type": [ + "string", + "null" + ] + }, + "max": { + "type": [ + "number", + "null" + ] + }, + "min": { + "type": [ + "number", + "null" + ] + }, + "parent_index": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "role": { + "type": "string" + }, + "selected": { + "type": [ + "boolean", + "null" + ] + }, + "value": { + "type": [ + "string", + "null" + ] + }, + "value_description": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "element_index", + "role", + "depth" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "elements_complete": { + "type": [ + "boolean", + "null" + ] + }, + "filtered_element_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "returned_element_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "screenshot_file_path": { + "type": [ + "string", + "null" + ] + }, + "screenshot_frame_valid": { + "type": [ + "boolean", + "null" + ] + }, + "screenshot_height": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "screenshot_mime_type": { + "type": [ + "string", + "null" + ] + }, + "screenshot_scale": { + "type": [ + "number", + "null" + ] + }, + "screenshot_width": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "since": { + "type": [ + "string", + "null" + ] + }, + "since_status": { + "type": [ + "string", + "null" + ] + }, + "snapshot_id": { + "type": [ + "string", + "null" + ] + }, + "total_element_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tree_diff": { + "type": [ + "string", + "null" + ] + }, + "tree_format": { + "type": [ + "string", + "null" + ] + }, + "tree_markdown": { + "type": [ + "string", + "null" + ] + }, + "truncated": { + "type": [ + "boolean", + "null" + ] + }, + "truncation_hint": { + "type": [ + "string", + "null" + ] + }, + "truncation_reason": { + "type": [ + "string", + "null" + ] + }, + "window_bounds": { + "properties": { + "height": { + "type": "number" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": [ + "object", + "null" + ] + }, + "window_id": { + "minimum": 0, + "type": "integer" + }, + "window_title": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "pid", + "window_id" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "verify_state", + "description": "Deterministically verify bounded predicates against one exact window. The driver evaluates structured window/accessibility state and may return the final screenshot as uninterpreted visual evidence for a multimodal caller. Predicate results are satisfied, unsatisfied, or unknown; unknown never implies success. Accessibility projections are conservative: absence remains unknown unless the observed search domain is proven exhaustive.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "expect": { + "description": "One to eight predicates, combined with logical AND.", + "items": { + "additionalProperties": false, + "properties": { + "element": { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "exists": { + "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", + "type": "boolean" + }, + "selected": { + "type": "boolean" + }, + "selector": { + "additionalProperties": false, + "properties": { + "label_contains": { + "minLength": 1, + "type": "string" + }, + "role": { + "minLength": 1, + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "value_equals": { + "type": "string" + } + }, + "required": [ + "selector" + ], + "type": "object" + }, + "window": { + "additionalProperties": false, + "properties": { + "bounds": { + "additionalProperties": false, + "properties": { + "height": { + "type": "number" + }, + "tolerance_px": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "exists": { + "type": "boolean" + } + }, + "required": [], + "type": "object" + } + }, + "required": [], + "type": "object" + }, + "maxItems": 8, + "minItems": 1, + "type": "array" + }, + "include_screenshot": { + "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", + "type": "boolean" + }, + "pid": { + "description": "Exact process whose window may be observed.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session. This\nfield never selects capture modality or authorization.", + "type": "string" + }, + "stable_samples": { + "default": 2, + "description": "Consecutive satisfied samples required before returning success.", + "maximum": 5, + "minimum": 1, + "type": "integer" + }, + "timeout_ms": { + "default": 5000, + "description": "Bounded wait. Zero performs one sample.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Exact native window identifier.", + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "expect" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "elapsed_ms": { + "minimum": 0, + "type": "integer" + }, + "predicates": { + "items": { + "properties": { + "index": { + "minimum": 0, + "type": "integer" + }, + "observed_json": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "status": { + "enum": [ + "satisfied", + "unsatisfied", + "unknown" + ], + "type": "string" + }, + "unknown_reason": { + "anyOf": [ + { + "enum": [ + "invalid_predicate", + "unsupported_predicate", + "untrusted_source", + "multi_match", + "target_missing", + "observation_unavailable", + "stability_unproven" + ], + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "index", + "status", + "unknown_reason", + "observed_json" + ], + "type": "object" + }, + "type": "array" + }, + "samples": { + "minimum": 0, + "type": "integer" + }, + "stable": { + "type": "boolean" + }, + "status": { + "enum": [ + "satisfied", + "unsatisfied", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "status", + "stable", + "elapsed_ms", + "samples", + "predicates" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "launch_app", + "description": "Launch a Linux app in the background. Provide launch_path (preferred — round-trip the value from list_apps), name (tried as a direct command, then matched against installed .desktop applications, then handed to xdg-open if it is a URL or existing file path), bundle_id (ignored on Linux), or urls (list of URLs to open). Resolution precedence: launch_path > name > bundle_id. Errors when the name resolves to nothing launchable.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "additional_arguments": { + "description": "Extra command-line arguments passed to the launched process.", + "items": { + "type": "string" + }, + "type": "array" + }, + "bundle_id": { + "description": "Ignored on Linux (macOS/Windows concept).", + "type": "string" + }, + "launch_path": { + "description": "Round-trip the `launch_path` returned by `list_apps` — the Exec= command from the .desktop file with XDG field codes already stripped. Highest precedence on Linux; spawned directly via the system shell.", + "type": "string" + }, + "name": { + "description": "App name or command to launch. Tried as a direct command first, then matched against installed .desktop applications (exact display name, desktop-file id, or Exec basename; else an unambiguous display-name substring).", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "urls": { + "description": "URLs to open via xdg-open.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "kill_app", + "description": "Force-terminate a process by pid (kill -9 equivalent on Linux). Use as escalation when the cooperative close path failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "PID of the process to terminate.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "bring_to_front", + "description": "Persistently activate a window so subsequent input lands on it. This deliberately breaks the no-foreground contract and is not part of the normal input ladder. For an ordinary `background_unavailable` response, retry only the refused action with `delivery_mode:\"foreground\"`; the input tool performs its own activate, act, and restore sequence. Use `bring_to_front` only for a focus-proxy surface that must remain foreground across multiple calls, such as a remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, with proper timestamp handling to beat focus-stealing prevention). Wayland: activates through a target-addressable compositor adapter (wlroots foreign-toplevel or the GNOME Shell helper) and refuses when the compositor offers no safe adapter. Matches the macOS / Windows bring_to_front rung.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "Process ID of the app to activate.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "X11 window id (xid) to activate. If omitted, the first window of `pid` is used.", + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "set_window_frame", + "description": "Set one exact top-level window's frame in the desktop-coordinate space reported by list_windows and verify the resulting geometry through an independent readback.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "height": { + "description": "New height, in the same units as list_windows bounds.", + "minimum": 1, + "type": "number" + }, + "pid": { + "description": "Process ID that owns the window.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "width": { + "description": "New width, in the same units as list_windows bounds.", + "minimum": 1, + "type": "number" + }, + "window_id": { + "description": "Window ID from list_windows.", + "minimum": 1, + "type": "integer" + }, + "x": { + "description": "New left edge in the desktop coordinate space reported by list_windows.", + "type": "number" + }, + "y": { + "description": "New top edge in the desktop coordinate space reported by list_windows.", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "invoke_menu", + "description": "Resolve an exact application-menu path one live native level at a time and invoke its final item through accessibility APIs. Missing, ambiguous, disabled, or structurally mismatched segments fail closed; this tool never falls back to pixels. Menu paths and refusals: `get_guide({topic:\"menus\"})`.", + "inputSchema": { + "additionalProperties": false, + "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).", + "properties": { + "path": { + "description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).", + "items": { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + "maxItems": 16, + "minItems": 1, + "type": "array" + }, + "pid": { + "description": "Process ID of the application that owns the menu.", + "minimum": 1, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window ID from list_windows whose menu is invoked.", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "path" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "click", + "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error.", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "capture_id": { + "description": "Optional one-shot binding to the exact PNG returned by get_window_state or get_desktop_state. When present, x/y are interpreted in that capture and admitted before native dispatch.", + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "count": { + "description": "Number of clicks in one press train: 1 (default), 2 = double-click, 3 = triple-click (line/paragraph selection). There is no separate triple_click tool — pass count: 3 here.", + "minimum": 1, + "type": "integer" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Coordinate frame (default \"window\"). Pass \"desktop\" with x,y and no pid/window_id for a screen-absolute click in get_desktop_state coordinates.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Exact window (kind \"window\", with pid and window_id) or desktop (kind \"desktop\", with display_id). \"primary\" is the portable desktop target.", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "double_click", + "description": "Double-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to). No focus steal. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "right_click", + "description": "Right-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to), one outside the window is refused (point_outside_window; x/y are window-local pixels, the result reports window_point / screen_point). No focus steal. When a context menu opens, the result names it (`popup: {window_id, bounds, title}`): call get_window_state(pid, window_id=) to index its menu items and click them by element_token. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "drag", + "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. Background delivery on X11 is one real held gesture on the session's virtual master pointer (path mpx_pointer: press, interpolated motion, release) — it lands on GTK/VCL/Qt/Chromium; a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it. Headless servers fall back to XSendEvent. duration_ms (default 500), steps (default 20). No focus steal. Limits and fallbacks: `get_guide({topic:\"drag\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "duration_ms": { + "description": "Total drag duration. Default: 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "from_x": { + "description": "Drag start X: window-local pixels of the target window's own get_window_state screenshot; pass scope:\"desktop\" for get_desktop_state pixels.", + "type": "number" + }, + "from_y": { + "description": "Drag start Y (same frame as from_x).", + "type": "number" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "modifier": { + "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\".", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "steps": { + "description": "Intermediate MotionNotify events. Default: 20.", + "maximum": 200, + "minimum": 1, + "type": "integer" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "to_x": { + "description": "Drag end X (same frame as from_x).", + "type": "number" + }, + "to_y": { + "description": "Drag end Y (same frame as from_x).", + "type": "number" + }, + "window_id": { + "description": "Target window XID. Required.", + "type": "integer" + } + }, + "required": [ + "from_x", + "from_y", + "to_x", + "to_y" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "mouse_button_down", + "description": "Press and hold a mouse button at (x,y) via background X11 delivery. Does not release the button; pair with mouse_drag / mouse_button_up. Returns the current held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [ + "pid", + "window_id", + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "mouse_drag", + "description": "Move a previously-held mouse button to a new point via background X11 delivery. Requires an active mouse_button_down state; does not release the button. Returns the updated held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "duration_ms": { + "description": "Total drag duration. Default: 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Optional process ID; must match the held button's target.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "steps": { + "description": "Intermediate MotionNotify events. Default: 20.", + "maximum": 200, + "minimum": 1, + "type": "integer" + }, + "window_id": { + "description": "Optional window id; must match the held button's target.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "mouse_button_up", + "description": "Release a previously-held mouse button via background X11 delivery. If x/y are omitted, releases at the last held position. Returns the current held-button state.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "from_zoom": { + "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", + "type": "boolean" + }, + "pid": { + "description": "Optional process ID; must match the held button's target.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", + "type": "string" + }, + "window_id": { + "description": "Optional window id; must match the held button's target.", + "type": "integer" + }, + "x": { + "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", + "type": "number" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "parallel_mouse_drag", + "description": "Run multiple mouse drag gestures concurrently via Linux MPX/XI2 virtual master pointers. Each drag item runs on its own session-scoped master pointer (true same-window concurrent draws on X11). Each item presses once, glides continuously through its whole path, and releases once — one smooth held drag, not a chain of clicks. A path is given either as a straight segment (from_x/from_y → to_x/to_y) or as a function `fn` = y(x) sampled over [x_from, x_to] in window-local pixels (e.g. fn:\"x\" is a diagonal, fn:\"300+120*sin(x/40)\" a sine wave). Functions support + - * / ^, sin/cos/tan, sqrt, abs, exp, ln, pi, e.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "drags": { + "description": "Two or more drag gestures to run concurrently, each on its own virtual master pointer.", + "items": { + "additionalProperties": false, + "properties": { + "button": { + "description": "Mouse button. Default \"left\".", + "enum": [ + "left", + "right", + "middle" + ], + "type": "string" + }, + "duration_ms": { + "description": "Default: 1500 for fn paths, 500 for straight.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "fn": { + "description": "Expression y(x) in window-local pixels; sampled over [x_from,x_to]. Mutually exclusive with from_x/to_x.", + "type": "string" + }, + "from_x": { + "description": "Straight-segment start X (window-local pixels).", + "type": "number" + }, + "from_y": { + "description": "Straight-segment start Y (window-local pixels).", + "type": "number" + }, + "path": { + "description": "Explicit window-local waypoints [[x,y],...] (>=2); pressed once, glided through, released once. Takes precedence over fn/from-to.", + "items": { + "items": { + "type": "number" + }, + "type": "array" + }, + "type": "array" + }, + "samples": { + "description": "Waypoints sampled along `fn`. Default: 80.", + "maximum": 400, + "minimum": 2, + "type": "integer" + }, + "session": { + "description": "Session/cursor id; also keys the virtual master pointer.", + "type": "string" + }, + "steps": { + "description": "Motion sub-steps along the whole path. Default: scaled to path length.", + "maximum": 300, + "minimum": 1, + "type": "integer" + }, + "to_x": { + "description": "Straight-segment end X (window-local pixels).", + "type": "number" + }, + "to_y": { + "description": "Straight-segment end Y (window-local pixels).", + "type": "number" + }, + "window_id": { + "description": "Window id from list_windows.", + "type": "integer" + }, + "x_from": { + "description": "Domain start (window-local x) when `fn` is used.", + "type": "number" + }, + "x_to": { + "description": "Domain end (window-local x) when `fn` is used.", + "type": "number" + } + }, + "required": [ + "session", + "window_id" + ], + "type": "object" + }, + "minItems": 2, + "type": "array" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "drags" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "type_text", + "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to type into the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to type into the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "text": { + "description": "Text to type.", + "type": "string" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Window-local pixels of the target window's own get_window_state screenshot by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\") — a bare pid+x/y is otherwise misinterpreted as window-local and can focus the wrong widget.", + "type": "number" + }, + "y": { + "description": "Pixel Y of the field (see x).", + "type": "number" + } + }, + "required": [ + "text" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "press_key", + "description": "Press a key via XSendEvent to a window. No focus steal.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "key": { + "description": "Key name: enter/return, tab, escape, space, backspace, delete, insert, home, end, pageup, pagedown, up, down, left, right, f1-f12, or any single ASCII character.", + "type": "string" + }, + "modifiers": { + "description": "Modifier keys held while the key is pressed: ctrl, shift, alt, super.", + "items": { + "type": "string" + }, + "type": "array" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to send the key to the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to send the key to the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_token. Window-local pixels by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", + "type": "number" + }, + "y": { + "description": "Pixel Y (see x).", + "type": "number" + } + }, + "required": [ + "key" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "hotkey", + "description": "Press a combination of keys simultaneously, e.g. [\"ctrl\",\"c\"] for Copy. Delivered to the target window without focus steal (real key events from a virtual master keyboard on X11); target does NOT need to be frontmost. Window-manager chords cannot reach the WM that way: alt+F4 on a window target closes that window through the WM (_NET_CLOSE_WINDOW, effect confirmed when it unmaps); super+*, alt+tab, ctrl+alt+* and other WM bindings are refused with code wm_chord_unavailable — use bring_to_front / list_windows / kill_app or delivery_mode:\"foreground\" instead.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "keys": { + "description": "Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"].", + "items": { + "type": "string" + }, + "minItems": 2, + "type": "array" + }, + "pid": { + "description": "Target process ID. Omit with scope \"desktop\" to send the chord to the focused application.", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with no pid to send the chord to the focused application. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.", + "type": "number" + }, + "y": { + "description": "Screenshot-pixel Y (see x).", + "type": "number" + } + }, + "required": [ + "keys" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "set_value", + "description": "Set value of an AT-SPI element via SetValue action.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Optional with element_token (the token carries the pid).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "value": { + "description": "New value. Written through AT-SPI EditableText/Value when the element exposes them; otherwise (GTK2 spin scales, VCL spin buttons) the field is clicked with the session's real pointer, its text selected and replaced through the virtual keyboard and committed with Tab, then read back.", + "type": "string" + }, + "window_id": { + "description": "Omit when element_token is supplied (the token carries it).", + "type": "integer" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "scroll", + "description": "Scroll the target pid's focused region via XSendEvent Button4/5. direction required; by defaults to line, amount defaults to 3.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "amount": { + "description": "Number of scroll steps. Default: 3.", + "maximum": 50, + "minimum": 1, + "type": "integer" + }, + "by": { + "description": "Scroll granularity. Default: line.", + "enum": [ + "line", + "page" + ], + "type": "string" + }, + "coordinate_frame": { + "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "cursor_id": { + "description": "Optional multi-cursor instance id. Default: 'default'.", + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "direction": { + "description": "Scroll direction.", + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + "element_token": { + "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", + "pattern": "^s[0-9a-f]{8}:[0-9]+$", + "type": "string" + }, + "pid": { + "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", + "type": "integer" + }, + "scope": { + "default": "window", + "description": "Use \"desktop\" with x,y and no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", + "type": "integer" + }, + "x": { + "description": "Window-local screenshot-pixel X of the scroll target. Pass with y and without element_token.", + "type": "number" + }, + "y": { + "description": "Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_token.", + "type": "number" + } + }, + "required": [ + "direction" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "clipboard_read", + "description": "List available system clipboard types and optionally return privacy-sensitive plain text. Clipboard content is never retained in telemetry.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include_text": { + "default": false, + "description": "Return plain-text clipboard content in addition to the available types.\nClipboard content is privacy-sensitive and is never retained in telemetry.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "content_redacted_from_telemetry": { + "type": "boolean" + }, + "privacy_sensitive": { + "type": "boolean" + }, + "supported": { + "type": "boolean" + }, + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "types": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "supported", + "types", + "text", + "privacy_sensitive", + "content_redacted_from_telemetry" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "clipboard_write", + "description": "Replace the system clipboard with exactly one value: plain text, an image from an absolute local path, or a file URL from an absolute local path. Returns the available types for read-back before paste.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "file_path": { + "description": "Absolute path to a local file to place on the clipboard as a file URL.", + "type": "string" + }, + "image_path": { + "description": "Absolute path to a local image to place on the clipboard.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "text": { + "description": "Plain text to place on the clipboard.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "content_redacted_from_telemetry": { + "type": "boolean" + }, + "privacy_sensitive": { + "type": "boolean" + }, + "supported": { + "type": "boolean" + }, + "types": { + "items": { + "type": "string" + }, + "type": "array" + }, + "written_type": { + "type": "string" + } + }, + "required": [ + "supported", + "written_type", + "types", + "privacy_sensitive", + "content_redacted_from_telemetry" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_screen_size", + "description": "Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "height": { + "type": "number" + }, + "scale_factor": { + "type": "number" + }, + "width": { + "type": "number" + } + }, + "required": [ + "width", + "height", + "scale_factor" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_desktop_state", + "description": "Capture the full display in the desktop action coordinate frame. Use the returned PNG directly as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. No AT-SPI walk.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "max_image_dimension": { + "description": "Optional long-edge cap for the returned PNG, in pixels (aspect ratio preserved). Omitted or 0 returns the full-size capture. When the cap downsizes the image, the response reports `screenshot_original_width/height`, and x/y read off it for this session's later scope:\"desktop\" actions (or passed with its `capture_id`) are mapped back to the full-size frame automatically.", + "minimum": 0, + "type": "integer" + }, + "screenshot_out_file": { + "description": "Write PNG here instead of base64.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "agent_overlay_capture": { + "additionalProperties": false, + "properties": { + "method": { + "type": [ + "string", + "null" + ] + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "status": { + "oneOf": [ + { + "const": "excluded", + "type": "string" + }, + { + "const": "not_present", + "type": "string" + }, + { + "const": "not_excluded", + "type": "string" + } + ] + } + }, + "required": [ + "status" + ], + "type": [ + "object", + "null" + ] + }, + "display": { + "type": "string" + }, + "platform": { + "enum": [ + "macos", + "linux", + "windows" + ], + "type": "string" + }, + "scale_factor": { + "type": "number" + }, + "screen_height": { + "type": "integer" + }, + "screen_width": { + "type": "integer" + }, + "screenshot_file_path": { + "type": "string" + }, + "screenshot_height": { + "type": "integer" + }, + "screenshot_mime_type": { + "const": "image/png" + }, + "screenshot_width": { + "type": "integer" + } + }, + "required": [ + "platform", + "display", + "screenshot_width", + "screenshot_height", + "screen_width", + "screen_height", + "scale_factor", + "screenshot_mime_type" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_cursor_position", + "description": "Return the current mouse cursor position in screen points (origin top-left).", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "available": { + "type": "boolean" + }, + "source": { + "type": "string" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "move_cursor", + "description": "Move the synthetic agent cursor without changing the user's pointer. Only an explicit scope=desktop request moves the real OS pointer in get_desktop_state coordinates.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "cursor_id": { + "description": "Cursor instance to move. Default: 'default'.", + "type": "string" + }, + "scope": { + "default": "window", + "description": "\"window\" (default) moves only the agent cursor overlay; \"desktop\" moves the real OS pointer.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target": { + "additionalProperties": true, + "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", + "properties": { + "display_id": { + "description": "Display id. Required when kind is desktop; use \"primary\".", + "type": "string" + }, + "kind": { + "description": "window requires pid and window_id; desktop requires display_id.", + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "description": "Process id. Required when kind is window.", + "minimum": 0, + "type": "integer" + }, + "window_id": { + "description": "Window id from list_windows. Required when kind is window.", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "x": { + "description": "Destination X. Window scope: screen coordinates of the agent cursor overlay. Desktop scope: get_desktop_state screenshot pixels.", + "type": "number" + }, + "y": { + "description": "Destination Y, in the same space as x.", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "set_agent_cursor_enabled", + "description": "Show or hide the agent cursor owned by a session.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "enabled": { + "description": "`true` shows the session's agent cursor overlay; `false` hides it.", + "type": "boolean" + }, + "session": { + "description": "Public label of the session that owns the cursor.", + "type": "string" + } + }, + "required": [ + "session", + "enabled" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "enabled" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "set_agent_cursor_motion", + "description": "Configure the movement style, timing, effects and visibility timing for a session cursor.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "arc_flow": { + "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", + "type": "number" + }, + "arc_size": { + "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", + "type": "number" + }, + "dwell_after_click_ms": { + "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", + "type": "number" + }, + "effects": { + "additionalProperties": false, + "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", + "properties": { + "glow": { + "description": "Soft glow around the cursor that grows with speed.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "magnet": { + "description": "Target glow when the `magnetic` style locks on.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "ripple": { + "description": "Ring that expands from the hotspot on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "squish": { + "description": "Brief scale-down of the cursor on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "trail": { + "description": "Short fading trail behind the cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "end_handle": { + "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "glide_duration_ms": { + "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", + "type": "number" + }, + "idle_hide_ms": { + "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", + "type": "number" + }, + "session": { + "description": "Public label of the session that owns the cursor. Omitted or null motion fields keep\ntheir current value.", + "type": "string" + }, + "spring": { + "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", + "type": "number" + }, + "start_handle": { + "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "style": { + "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", + "type": "number" + } + }, + "required": [ + "session" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "motion": { + "properties": { + "arc_flow": { + "type": "number" + }, + "arc_size": { + "type": "number" + }, + "dwell_after_click_ms": { + "type": "number" + }, + "effects": { + "properties": { + "glow": { + "type": "boolean" + }, + "magnet": { + "type": "boolean" + }, + "ripple": { + "type": "boolean" + }, + "squish": { + "type": "boolean" + }, + "trail": { + "type": "boolean" + } + }, + "required": [ + "trail", + "glow", + "magnet", + "ripple", + "squish" + ], + "type": [ + "object", + "null" + ] + }, + "end_handle": { + "type": "number" + }, + "glide_duration_ms": { + "type": "number" + }, + "idle_hide_ms": { + "type": "number" + }, + "spring": { + "type": "number" + }, + "start_handle": { + "type": "number" + }, + "style": { + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "type": "number" + } + }, + "required": [ + "start_handle", + "end_handle", + "arc_size", + "arc_flow", + "spring", + "glide_duration_ms", + "dwell_after_click_ms", + "idle_hide_ms", + "turn_radius" + ], + "type": "object" + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "motion" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_agent_cursor_state", + "description": "Return the session cursor's theme, semantic playback, position, visibility, and motion.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "Public label of the session whose cursor to inspect.", + "type": "string" + } + }, + "required": [ + "session" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "motion": { + "properties": { + "arc_flow": { + "type": "number" + }, + "arc_size": { + "type": "number" + }, + "dwell_after_click_ms": { + "type": "number" + }, + "effects": { + "properties": { + "glow": { + "type": "boolean" + }, + "magnet": { + "type": "boolean" + }, + "ripple": { + "type": "boolean" + }, + "squish": { + "type": "boolean" + }, + "trail": { + "type": "boolean" + } + }, + "required": [ + "trail", + "glow", + "magnet", + "ripple", + "squish" + ], + "type": [ + "object", + "null" + ] + }, + "end_handle": { + "type": "number" + }, + "glide_duration_ms": { + "type": "number" + }, + "idle_hide_ms": { + "type": "number" + }, + "spring": { + "type": "number" + }, + "start_handle": { + "type": "number" + }, + "style": { + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "type": "number" + } + }, + "required": [ + "start_handle", + "end_handle", + "arc_size", + "arc_flow", + "spring", + "glide_duration_ms", + "dwell_after_click_ms", + "idle_hide_ms", + "turn_radius" + ], + "type": "object" + }, + "position": { + "anyOf": [ + { + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "session": { + "type": "string" + }, + "theme": { + "properties": { + "fallback": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "string" + }, + "profile": { + "type": "string" + }, + "reduced_motion": { + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "id", + "version", + "profile", + "reduced_motion", + "fallback" + ], + "type": "object" + }, + "visual_state": { + "properties": { + "frame": { + "minimum": 0, + "type": "integer" + }, + "modifiers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "phase": { + "type": "string" + }, + "preempted_count": { + "minimum": 0, + "type": "integer" + }, + "requested_action": { + "enum": [ + "idle", + "observe", + "click", + "drag", + "scroll", + "text", + "key", + "navigate", + "app", + "transfer", + "record", + "system" + ], + "type": "string" + }, + "resolved_action": { + "enum": [ + "idle", + "observe", + "click", + "drag", + "scroll", + "text", + "key", + "navigate", + "app", + "transfer", + "record", + "system" + ], + "type": "string" + } + }, + "required": [ + "requested_action", + "resolved_action", + "modifiers", + "phase", + "frame", + "preempted_count" + ], + "type": "object" + } + }, + "required": [ + "session", + "enabled", + "position", + "theme", + "visual_state", + "motion" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "set_agent_cursor_theme", + "description": "Select an already-installed cursor theme for a session.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "reduced_motion": { + "default": "auto", + "description": "Theme animation policy: `on` uses the theme's reduced-motion frames, `off` always\nanimates, `auto` (default) leaves the choice to the host.", + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "session": { + "description": "Public label of the session that owns the cursor.", + "type": "string" + }, + "theme_id": { + "description": "Id of an installed cursor theme (see `cua-driver cursor-theme list`).", + "maxLength": 200, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "session", + "theme_id" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "session": { + "type": "string" + }, + "theme": { + "properties": { + "fallback": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "string" + }, + "profile": { + "type": "string" + }, + "reduced_motion": { + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "id", + "version", + "profile", + "reduced_motion", + "fallback" + ], + "type": "object" + } + }, + "required": [ + "session", + "theme" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "check_permissions", + "description": "Check required permissions for cua-driver-rs on Linux. How to get them granted: `get_guide({topic:\"permissions\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "prompt": { + "description": "Ignored: Linux has no permission prompts.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "health_report", + "description": "Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped. For standalone CuaDriver, use `cua-driver permissions grant` to verify it explicitly. For an embedded or linked driver, use the host's permission flow for the app or executable macOS attributes this process's permission request to; the standalone command does not verify the host's capture readiness. cua-driver owns the health model; consumers stay thin.\n\nInput — all optional:\n {\n \"include\": [\"\", ...], // run only these\n \"skip\": [\"\", ...] // skip these\n }\nIf both are given, `include` wins.\n\nCanonical check names:\n macOS : binary_version, platform_supported, session_active,\n bundle_identity, tcc_accessibility, tcc_screen_recording,\n ax_capability, screen_capture_capability, windowserver_connection\n Windows: binary_version, platform_supported, session_active,\n ax_capability (via UIA), screen_capture_capability (via DXGI)\n Linux : binary_version, platform_supported, session_active,\n ax_capability (via AT-SPI), screen_capture_capability (via X11)\n\nOutput — stable contract, schema_version=\"1\":\n {\n \"schema_version\": \"1\",\n \"platform\": \"darwin\" | \"win32\" | \"linux\",\n \"driver_version\": \"\",\n \"overall\": \"ok\" | \"degraded\" | \"failed\",\n \"checks\": [\n {\n \"name\": \"\",\n \"status\": \"pass\" | \"fail\" | \"skip\",\n \"message\": \"\",\n \"hint\": \"\",\n \"data\": { /* check-specific structured fields */ }\n },\n ...\n ]\n }\n\n`overall` rules:\n - `ok` — every non-skipped check passes\n - `degraded` — at least one non-core check fails (binary is still usable)\n - `failed` — any core check fails (binary_version, platform_supported, session_active)\n\nStability: schema_version=\"1\" is the contract. Future breaking changes will be `\"2\"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include": { + "description": "Only run these checks (canonical names). Wins over `skip`.", + "items": { + "type": "string" + }, + "type": "array" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "skip": { + "description": "Skip these checks (canonical names). Ignored when `include` is set.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "get_config", + "description": "Return current cua-driver-rs configuration.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "set_config", + "description": "Update cua-driver-rs configuration. capture_mode / max_image_dimension take effect immediately.\n\nPass each setting as its own field: `{\"capture_mode\": \"som\", \"max_image_dimension\": 0}`.\n\nThe experimental_pip keys persist to ~/.cua-driver/config.json and apply on next daemon restart (the PiP backend is initialised once at startup; Linux ships only the trait stub today — see issue #1729).\n\n`cursor.motion.style`, `cursor.motion.timing` and `cursor.motion.effects.` (`on`, `off`, or `default`) save the default cursor motion for sessions started afterwards. `cursor.motion: \"default\"` clears every saved cursor motion default.\n\n`cursor.badges` (`minimal` default, `verbose`, or `off`) sets the agent cursor badge for every cursor at once.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "capture_mode": { + "description": "Default capture mode for get_window_state. (\"som\"/\"screenshot\" still decode as deprecated aliases.)", + "enum": [ + "ax", + "vision" + ], + "type": "string" + }, + "cursor.badges": { + "description": "Agent cursor badge. minimal (default): no badge at rest; a brief chip for a foreground moment or a key combo, and the session name on hover or when two or more agent cursors are on screen. verbose: the earlier badge with the session name and delivery/target chips on every action. off: no badge. Applies immediately to every cursor.", + "enum": [ + "minimal", + "verbose", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion": { + "description": "Pass `default` to clear every saved cursor motion default.", + "enum": [ + "default" + ], + "type": "string" + }, + "cursor.motion.effects.glow": { + "description": "Saved default for the `glow` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.magnet": { + "description": "Saved default for the `magnet` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.ripple": { + "description": "Saved default for the `ripple` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.squish": { + "description": "Saved default for the `squish` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.effects.trail": { + "description": "Saved default for the `trail` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.motion.style": { + "description": "Saved default cursor motion style for new sessions. Overridden by start_session cursor_motion and set_agent_cursor_motion; reduced motion always wins. Built-in default: signature_arc.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "cursor.motion.timing": { + "description": "Saved default cursor move timing for new sessions. Built-in default: steady (a 600 ms floor on every move).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "experimental_pip": { + "description": "Enable the experimental PiP preview window (applies next restart; Linux backend stubbed).", + "type": "boolean" + }, + "experimental_pip_geometry": { + "description": "PiP window size + optional position in `WxH` or `WxH+X+Y` form.", + "type": "string" + }, + "max_image_dimension": { + "description": "Max dimension for screenshot resizing (0 = no limit).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "get_accessibility_tree", + "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "zoom", + "description": "Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixels, with 20% padding. Output is at most 500 px wide. Corners, not a size: x1,y1 is the top-left and x2,y2 the bottom-right (x,y,width,height is also accepted and converted).\n\nAfter a zoom, pass from_zoom=true to click/type_text to auto-translate coordinates back to full-window space. Coordinate actions return `screenshot_context_missing` when no current snapshot contains a screenshot owned by this session. `from_zoom` actions return `zoom_context_missing` when the zoom was never created or was replaced; call `get_window_state`, then `zoom`, again on the same connection.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "pid": { + "description": "Optional target pid. When omitted, the driver resolves the unique current snapshot for this session and window.", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "window_id": { + "description": "Window id of the window captured by get_window_state.", + "type": "integer" + }, + "x1": { + "description": "Left edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "x2": { + "description": "Right edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "y1": { + "description": "Top edge of the region in window-local screenshot pixels.", + "type": "number" + }, + "y2": { + "description": "Bottom edge of the region in window-local screenshot pixels.", + "type": "number" + } + }, + "required": [ + "window_id", + "x1", + "y1", + "x2", + "y2" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "page", + "description": "Legacy browser compatibility tool. Prefer get_browser_state and the typed browser_* tools for exact targeting, endpoint ownership, and consent. Read-only get_text and query_dom remain available by default. Mutating actions require the daemon operator to set CUA_DRIVER_ENABLE_LEGACY_PAGE_MUTATIONS=1 before daemon startup (restart the daemon after changing it); this escape hatch does not provide the typed browser surface's exact binding or existing-profile grant guarantees. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page.\n- query_dom: Find elements matching a CSS selector.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- insert_text: Insert `text` at whatever currently holds DOM focus in one native operation (CDP Input.insertText) — no synthesized key events, but more durable than a one-shot execute_javascript write since rich-text editors already have to treat it like an IME commit. Try this before type_keystrokes on a contenteditable that discarded an execute_javascript write. Click/focus the target field first.\n- type_keystrokes: Type `text` via real per-character keystroke events into whatever currently holds DOM focus. Slower than insert_text but the most durable rung — use it when insert_text also gets discarded, or the editor's own keydown/keyup handlers need to see real keys. Click/focus the target field first.\n- enable_javascript_apple_events: macOS-only — patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", + "inputSchema": { + "additionalProperties": false, + "properties": { + "action": { + "description": "Action to perform.", + "enum": [ + "execute_javascript", + "get_text", + "query_dom", + "click_element", + "insert_text", + "type_keystrokes", + "enable_javascript_apple_events" + ], + "type": "string" + }, + "attributes": { + "description": "Element attributes to include in query_dom results.", + "items": { + "type": "string" + }, + "type": "array" + }, + "bundle_id": { + "description": "Bundle ID of the browser. Required for enable_javascript_apple_events (macOS only).", + "type": "string" + }, + "cdp_port": { + "description": "Optional, for execute_javascript/insert_text/type_keystrokes: use this exact CDP port instead of auto-discovering one from pid. Needed when the port was opened via the browser's own remote-debugging toggle rather than a launch-time flag, since that path may not answer the auto-discovery probe.", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "css_selector": { + "description": "CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*').", + "type": "string" + }, + "javascript": { + "description": "JavaScript to execute. Required for execute_javascript.", + "type": "string" + }, + "pid": { + "description": "Target process ID.", + "type": "integer" + }, + "selector": { + "description": "CSS selector for click_element (e.g. 'button.submit', '#login a').", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "target_url_contains": { + "description": "Optional, for execute_javascript/insert_text/type_keystrokes: require exactly one browser tab whose URL contains this substring. Use this on a multi-tab browser — there's no built-in link between window_id and which tab a CDP call reaches.", + "type": "string" + }, + "text": { + "description": "Text to insert or type. Required for insert_text and type_keystrokes. The target field must already have DOM focus (click/focus it first).", + "type": "string" + }, + "user_has_confirmed_enabling": { + "description": "Must be true to proceed with enable_javascript_apple_events. This will quit and relaunch the browser.", + "type": "boolean" + }, + "window_id": { + "description": "Target window ID from list_windows.", + "type": "integer" + } + }, + "required": [ + "action" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "get_browser_state", + "description": "Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native browser window to classify it, correlate it to a CDP target (exact-or-refuse), and mint a session-scoped target id plus tab ids. Mode 2 (snapshot): pass target_id + tab_id. The dom_refs_v1 compatibility format returns composed DOM refs. semantic_v2 joins accessibility, DOM, layout, and viewport state; ranks visible content before retained/offscreen state; and returns a semantic outline, typed action refs, content refs, scoped reads, and opaque continuation. Never performs setup: a browser that is not attached yet refuses with browser_consent_required or browser_requires_setup, and detail.next_call names the browser_prepare call that attaches it. Use this route for any web page in Chrome, Edge or Brave; desktop pointer tools move the user's pointer. Setup and recovery: `get_guide({topic:\"browser\"})`.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "continuation": { + "description": "Opaque continuation minted by an earlier semantic_v2 response.", + "type": "string" + }, + "include_screenshot": { + "default": false, + "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.", + "type": "boolean" + }, + "pid": { + "description": "Native browser process id (bind mode).", + "type": "integer" + }, + "query": { + "description": "Read-only semantic match over role, accessible name, and visible text.", + "type": "string" + }, + "scope_ref": { + "description": "Current semantic/content ref whose subtree should be observed.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "snapshot_format": { + "description": "Versioned snapshot contract. dom_refs_v1 remains the compatibility default.", + "enum": [ + "dom_refs_v1", + "semantic_v2" + ], + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "window_id": { + "description": "Native window id owned by pid (bind mode).", + "type": "integer" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "browser_prepare", + "description": "Attach the browser tools to a browser. For a page in the user's running Chrome, Edge or Brave, call it once with pid, window_id, session and strategy {\"kind\":\"existing_profile\"} when get_browser_state refuses with browser_consent_required or browser_requires_setup; then call get_browser_state again and act with browser_click, browser_type or browser_pointer. It attaches, or refuses with nothing changed and says why. Explicitly prepare an owned DevTools endpoint for a browser. pid is required for an existing process or existing-profile attachment, and optional only for allow_launch=true with an isolated profile. Existing endpoints are detected without side effects. Acting setup for an isolated profile follows the runtime permission mode and optional capability manifest. It requires allow_launch=true, launches a separate browser, and never copies, modifies, or terminates the requested user profile. Without pid, only a platform-attested system Chrome/Edge installation (or a root-owned package payload on Linux) is eligible; redirects and user-controlled locations fail closed. Existing-profile attachment is explicit and follows the runtime's immutable permission mode: standard requires an explicit --grant existing-profile launch grant or an embedding authorization host, bounded requires a launch-approved exact resource manifest, and unrestricted requires explicit trusted startup risk acceptance. Ordinary MCP transport approval never proves profile authorization. On proven platforms, an authorized request also permits one bounded exact-window setup: open the recognized browser product's fixed remote-debugging page, toggle its uniquely matched per-instance checkbox, prove the PID-owned loopback endpoint, and close the temporary tab. Every visible effect is reported; ambiguity is refused. Where attaching would bring the browser in front of the user's app (macOS Chrome shows its own remote-debugging prompt in front whenever the driver opens a new browser connection; sessions for the same browser share one approved connection, kept for up to 30 minutes idle and 8 hours in total, so later sessions do not prompt again), existing-profile attachment refuses with background_unavailable unless delivery_mode is \"foreground\" (the app that was in front is brought back afterwards) or the browser window is on another Space than the user's, where it attaches in the background while the driver reverts the browser's own brief activations. A driver-owned isolated launch result carries an advisory lifecycle object: the browser process ends with the calling session after idle_expires_in_seconds without session activity, an isolated_named profile persists after the session and an isolated_new profile does not, and explicit end, disconnect, revocation or shutdown end it earlier. Setup and recovery: `get_guide({topic:\"browser\"})`.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "allow_launch": { + "description": "Allow a separate driver-owned isolated Chromium process to be launched (default false).", + "type": "boolean" + }, + "delivery_mode": { + "default": "background", + "description": "Existing-profile attachment only. background (default) attaches in the background when the browser window is on another Space than the user's, and otherwise refuses with background_unavailable, before any visible effect, where attaching would bring the browser in front of the user's app. foreground accepts that, for a screen nobody else is using or when the user agreed; the app that was in front is brought back afterwards.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "pid": { + "description": "Browser process id to prepare. Required except for a driver-owned isolated_new/isolated_named launch with allow_launch=true.", + "type": "integer" + }, + "profile": { + "additionalProperties": false, + "description": "Driver-owned isolated Chromium profile to launch with allow_launch=true. mode=isolated_new creates a fresh throwaway profile; mode=isolated_named reuses the named driver-owned profile. Never an existing user profile.", + "properties": { + "mode": { + "enum": [ + "isolated_new", + "isolated_named" + ], + "type": "string" + }, + "name": { + "description": "Required only for isolated_named; 1-64 path-safe ASCII characters.", + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": "object" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "strategy": { + "additionalProperties": false, + "description": "Attach to an already-running browser instead of launching one. Pass an object, {\"kind\":\"existing_profile\"}, not a string. kind=existing_profile attaches to the user's running profile at pid/window_id under the runtime's profile authorization.", + "properties": { + "kind": { + "enum": [ + "existing_profile" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "window_id": { + "description": "Exact native window approval anchor; required for strategy.kind=existing_profile.", + "type": "integer" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "browser_navigate", + "description": "Navigate one tab of an exactly-bound browser target to a new URL (http/https/about only). Refused for heuristic bindings. Navigation invalidates all p: refs for the tab.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "url": { + "description": "Destination URL (http:, https:, or about:).", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "url" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "browser_click", + "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "delivery_mode": { + "default": "background", + "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "input_route": { + "description": "\"trusted\" (default): Input.dispatchMouseEvent. It refuses rather than foregrounding a standalone browser. \"dom_event\": synthetic full-background DOM click, only when explicitly requested. Dispatch does not prove the control activated; refresh page state and verify the expected postcondition.", + "enum": [ + "trusted", + "dom_event" + ], + "type": "string" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "x": { + "description": "Viewport x (CSS px) — alternative to ref.", + "type": "number" + }, + "y": { + "description": "Viewport y (CSS px) — alternative to ref.", + "type": "number" + } + }, + "required": [ + "target_id", + "tab_id" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "browser_type", + "description": "Type text into an exactly-bound tab via the Input domain. mode=\"insert_text\" (default) uses Input.insertText; mode=\"keystrokes\" dispatches per-character key events. Both insert at the caret, so typing into a field that already holds text appends to it; pass replace=true to set the field instead, or to clear it by typing an empty string. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "mode": { + "description": "insert_text (default): bulk Input.insertText. keystrokes: per-character Input.dispatchKeyEvent.", + "enum": [ + "insert_text", + "keystrokes" + ], + "type": "string" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "replace": { + "description": "false (default): insert at the caret, appending to whatever the field already holds. true: select the element's whole content first so the text replaces it — with an empty text this clears the field. Replacement goes through the selection, so beforeinput/input still fire and framework state stays consistent.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + }, + "text": { + "description": "Text to type.", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "ref", + "text" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "browser_select", + "description": "Select one option of a native HTML ref through CDP. This bypasses native file pickers, rejects symlinks and non-regular files, and never returns local paths.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "files": { + "description": "Absolute paths of 1 to 32 local regular files to assign to the input.", + "items": { + "description": "Absolute path to one local regular file.", + "type": "string" + }, + "maxItems": 32, + "minItems": 1, + "type": "array" + }, + "ref": { + "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id from get_browser_state (session-scoped).", + "type": "string" + }, + "target_id": { + "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", + "type": "string" + } + }, + "required": [ + "target_id", + "tab_id", + "ref", + "files" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "browser_download", + "description": "Trigger one download through an exact live browser ref and save it inside an explicitly approved directory. Requires MCP-host destructive-tool approval, refuses ambiguous or stale capabilities, and never returns the source URL, filename, or destination path.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "destination_root": { + "description": "Absolute, existing, canonical directory approved to receive the download.", + "type": "string" + }, + "ref": { + "description": "Live page ref whose activation initiates the download.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", + "type": "string" + }, + "tab_id": { + "description": "Opaque exact tab id from get_browser_state.", + "type": "string" + }, + "target_id": { + "description": "Opaque exact browser target id from get_browser_state.", + "type": "string" + } + }, + "required": [ + "session", + "target_id", + "tab_id", + "ref", + "destination_root" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "browser_pointer", + "description": "Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. Semantic refs must declare pointer for hover, right-click, double-click, and drag; scroll accepts a scroll or pointer capability. The trusted route uses CDP Input events and refuses if standalone background posture cannot be preserved. The explicit dom_event route requires a page ref and synthesizes full-background DOM events. Never activates or brings a tab to the foreground.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "action": { + "description": "Pointer gesture. scroll needs delta_x or delta_y; drag needs destination_ref or to_x/to_y.", + "enum": [ + "hover", + "right_click", + "double_click", + "scroll", + "drag" + ], + "type": "string" + }, + "delivery_mode": { + "default": "background", + "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "delta_x": { + "description": "Horizontal scroll delta in CSS pixels.", + "type": "number" + }, + "delta_y": { + "description": "Vertical scroll delta in CSS pixels.", + "type": "number" + }, + "destination_ref": { + "description": "Drag destination page ref in the exact same frame.", + "type": "string" + }, + "input_route": { + "default": "trusted", + "description": "trusted sends CDP Input events; dom_event synthesizes DOM events in the page and requires ref.", + "enum": [ + "trusted", + "dom_event" + ], + "type": "string" + }, + "ref": { + "description": "Origin page ref. Alternative to x/y.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", + "type": "string" + }, + "tab_id": { + "description": "Opaque tab id minted by get_browser_state.", + "type": "string" + }, + "target_id": { + "description": "Opaque target id minted by get_browser_state.", + "type": "string" + }, + "to_x": { + "description": "Drag destination viewport x in CSS pixels.", + "type": "number" + }, + "to_y": { + "description": "Drag destination viewport y in CSS pixels.", + "type": "number" + }, + "x": { + "description": "Origin viewport x in CSS pixels.", + "type": "number" + }, + "y": { + "description": "Origin viewport y in CSS pixels.", + "type": "number" + } + }, + "required": [ + "target_id", + "tab_id", + "session", + "action" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "hint": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code" + ], + "type": [ + "object", + "null" + ] + }, + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "hint": { + "type": [ + "string", + "null" + ] + }, + "next_call": { + "additionalProperties": false, + "properties": { + "arguments": { + "type": "object" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "arguments" + ], + "type": [ + "object", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input" + ], + "type": "string" + }, + "summary": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "effect", + "route" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "start_recording", + "description": "Start trajectory recording for the calling session. Each action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) from that session writes a turn folder under `output_dir`. Without `session`, every call on the same connection is recorded, including named-session calls. Other connections and session lifecycle calls (`start_session` / `end_session`) are not recorded. CLI recordings started with `cua-driver recording start` are daemon-wide.\n\nEach turn folder holds:\n\n- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI state immediately before and after the action.\n- `before.png` / `after.png` — target-window screenshots immediately before and after the action.\n- `evidence.json` — capture status and a stable classification when an expected artifact could not be captured.\n- `app_state.json` — post-action AX/UIA snapshot for the target pid.\n- `screenshot.png` — compatibility alias of `after.png`.\n- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.\n- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.\n\nThe per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.\n\nTurn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.\n\n**Video is off by default.** Pass `record_video: true` to also capture the main display to `/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.\n\n**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) under the daemon's Screen Recording grant, with no ffmpeg subprocess. Requires macOS 15.0+. On macOS 26 (Tahoe), the first direct capture can also show a one-time consent asking to let Cua Driver bypass the system private window picker and directly access your screen and audio; choose Allow, or run `cua-driver permissions grant` beforehand to answer it up front. The recorder captures screen video only and does not enable system-audio capture.\n\n**Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.\n\nState persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "include_accessibility_tree": { + "default": true, + "description": "Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`.", + "type": "boolean" + }, + "output_dir": { + "description": "Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.", + "type": "string" + }, + "record_video": { + "description": "Capture the main display to /recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "state_timeout_ms": { + "default": 1000, + "description": "Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + } + }, + "required": [ + "output_dir" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "stop_recording", + "description": "Stop trajectory recording. Disables further per-turn capture and, when video was enabled, gracefully terminates the ffmpeg subprocess so the mp4's moov atom is finalized (the file is playable). Calling stop on an already-stopped session is a no-op. The response carries `last_video_path` pointing at the finalized mp4 (when video was on).\n\nA manual `stop_recording` is **unconditional** — it stops whatever recording is active regardless of which session started it. Ownership-scoped teardown (so one client disconnecting can't stop a recording a later client started) is handled by the registry's `session_end` lifecycle hook, not by this tool.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "get_recording_state", + "description": "Report the current trajectory recorder state: whether recording is enabled, the output directory (when enabled), and the 1-based counter for the next turn folder that will be written. Counter increments on every recorded action tool call and resets to 1 each time recording is (re-)enabled.\n\nPure read-only.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "replay_trajectory", + "description": "Replay a recorded trajectory by re-invoking every turn's tool call in lexical order. `dir` must point at a directory previously written by `start_recording`. Each `turn-NNNNN/` is parsed for `action.json`, and the recorded tool is called with its recorded `arguments` via the same dispatch path an MCP / CLI call uses.\n\nCaveats:\n- Element-token actions (`click({pid, element_token})` etc.) will fail because element tokens are per-snapshot and don't survive across sessions. Pixel clicks (`click({pid, x, y})`) and all keyboard tools replay cleanly. Failures are reported but don't stop replay unless `stop_on_error` is true.\n- `get_window_state` and other read-only tools are NOT currently recorded, so replays do not re-populate the per-(pid, window_id) element cache.\n- If recording is ENABLED while replay runs, the replay itself is recorded into the currently configured output directory. That's deliberate: recording a replay against a new build and diffing the two trajectories is the regression-test workflow.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "delay_ms": { + "description": "Milliseconds to sleep between turns, for human-observable pacing. Default 500.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "dir": { + "description": "Trajectory directory previously written by `start_recording`. Absolute or ~-rooted.", + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "stop_on_error": { + "description": "Stop replay on the first tool-call error. Default true — set false to best-effort through the full trajectory.", + "type": "boolean" + } + }, + "required": [ + "dir" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": false + } + }, + { + "name": "install_ffmpeg", + "description": "Install the ffmpeg binary used by start_recording's video capture (Linux/Windows; macOS records natively and needs no ffmpeg). Two-step and confirmed: called without `confirm` it only REPORTS the exact install command for this platform's package manager; pass `confirm: true` to actually run it. No-op if ffmpeg is already on PATH. ffmpeg is run as a separate process, never linked into the driver.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "confirm": { + "description": "Run the install command. Without it, only the planned command is reported.", + "type": "boolean" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "start_session", + "description": "Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "capture_scope": { + "description": "Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.", + "enum": [ + "auto", + "window", + "desktop" + ], + "type": "string" + }, + "cursor_motion": { + "additionalProperties": false, + "description": "Optional initial cursor motion (style, timing, effects and tuning). The host applies it\nbefore the cursor is first made visible. A later `set_agent_cursor_motion` call wins;\nthis wins over the saved default (`cursor.motion.*` in the driver config) and the\nbuilt-in `signature_arc`. Reduced motion always wins.", + "properties": { + "arc_flow": { + "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", + "type": "number" + }, + "arc_size": { + "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", + "type": "number" + }, + "dwell_after_click_ms": { + "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", + "type": "number" + }, + "effects": { + "additionalProperties": false, + "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", + "properties": { + "glow": { + "description": "Soft glow around the cursor that grows with speed.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "magnet": { + "description": "Target glow when the `magnetic` style locks on.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "ripple": { + "description": "Ring that expands from the hotspot on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "squish": { + "description": "Brief scale-down of the cursor on click.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "trail": { + "description": "Short fading trail behind the cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "end_handle": { + "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "glide_duration_ms": { + "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", + "type": "number" + }, + "idle_hide_ms": { + "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", + "type": "number" + }, + "spring": { + "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", + "type": "number" + }, + "start_handle": { + "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", + "type": "number" + }, + "style": { + "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "cursor_theme": { + "additionalProperties": true, + "description": "Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.", + "properties": { + "reduced_motion": { + "default": "auto", + "enum": [ + "auto", + "on", + "off" + ], + "type": "string" + }, + "theme_id": { + "type": "string" + } + }, + "required": [ + "theme_id" + ], + "type": "object" + }, + "session": { + "description": "Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "active": { + "type": "boolean" + }, + "capture_scope": { + "enum": [ + "auto", + "window", + "desktop" + ], + "type": "string" + }, + "cursor_motion": { + "additionalProperties": false, + "properties": { + "arc_flow": { + "type": [ + "number", + "null" + ] + }, + "arc_size": { + "type": [ + "number", + "null" + ] + }, + "dwell_after_click_ms": { + "type": [ + "number", + "null" + ] + }, + "effects": { + "additionalProperties": false, + "properties": { + "glow": { + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "magnet": { + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "ripple": { + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "squish": { + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "trail": { + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + } + }, + "type": [ + "object", + "null" + ] + }, + "end_handle": { + "type": [ + "number", + "null" + ] + }, + "glide_duration_ms": { + "type": [ + "number", + "null" + ] + }, + "idle_hide_ms": { + "type": [ + "number", + "null" + ] + }, + "spring": { + "type": [ + "number", + "null" + ] + }, + "start_handle": { + "type": [ + "number", + "null" + ] + }, + "style": { + "enum": [ + "signature_arc", + "spring_settle", + "magnetic", + "comet_swoop", + "adaptive", + "classic" + ], + "type": "string" + }, + "timing": { + "enum": [ + "native", + "fitts", + "fixed", + "relaxed", + "steady", + "calm", + "deliberate" + ], + "type": "string" + }, + "turn_radius": { + "type": [ + "number", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "desktop_capture_authorized": { + "type": "boolean" + }, + "desktop_unlocked": { + "type": "boolean" + }, + "effective_scope": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "escalation_detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "escalation_reason": { + "anyOf": [ + { + "enum": [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other" + ], + "type": "string" + }, + { + "type": "null" + } + ] + }, + "revived": { + "type": "boolean" + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "capture_scope", + "effective_scope", + "desktop_capture_authorized", + "desktop_unlocked", + "escalation_reason", + "escalation_detail", + "active", + "revived" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "escalate_session", + "description": "Deprecated compatibility tool for legacy capture-scope sessions. New callers select window or desktop modality on each action. No deescalate_session tool exists.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "detail": { + "description": "Optional bounded diagnostic detail. Never use secrets or page content.", + "maxLength": 200, + "type": "string" + }, + "reason": { + "description": "Why the window-scoped attempt failed and desktop capture is needed.", + "enum": [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other" + ], + "type": "string" + }, + "session": { + "description": "Public label of the legacy capture-scope session to escalate.", + "type": "string" + } + }, + "required": [ + "session", + "reason" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "capture_scope": { + "enum": [ + "auto", + "window", + "desktop" + ], + "type": "string" + }, + "desktop_capture_authorized": { + "type": "boolean" + }, + "desktop_unlocked": { + "type": "boolean" + }, + "effective_scope": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "escalation_detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "escalation_reason": { + "anyOf": [ + { + "enum": [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other" + ], + "type": "string" + }, + { + "type": "null" + } + ] + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "capture_scope", + "effective_scope", + "desktop_capture_authorized", + "desktop_unlocked", + "escalation_reason", + "escalation_detail" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_session", + "description": "Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "client_kind": { + "enum": [ + "cli", + "direct", + "mcp", + "python_sdk", + "typescript_sdk" + ], + "type": "string" + }, + "cursor_visible": { + "type": "boolean" + }, + "expires_in_seconds": { + "minimum": 0, + "type": "integer" + }, + "idle_seconds": { + "minimum": 0, + "type": "integer" + }, + "implicit": { + "type": "boolean" + }, + "recording_active": { + "type": "boolean" + }, + "session": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "state": { + "enum": [ + "active", + "ending" + ], + "type": "string" + }, + "transport": { + "enum": [ + "cli", + "daemon", + "mcp_stdio", + "mcp_http" + ], + "type": "string" + } + }, + "required": [ + "session", + "implicit", + "state", + "client_kind", + "transport", + "cursor_visible", + "recording_active", + "idle_seconds", + "expires_in_seconds" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "list_sessions", + "description": "List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "cursor": { + "description": "Opaque continuation cursor returned by a previous call.", + "type": "string" + }, + "limit": { + "description": "Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.", + "minimum": 0, + "type": "integer" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "next_cursor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "sessions": { + "items": { + "properties": { + "client_kind": { + "enum": [ + "cli", + "direct", + "mcp", + "python_sdk", + "typescript_sdk" + ], + "type": "string" + }, + "cursor_visible": { + "type": "boolean" + }, + "expires_in_seconds": { + "minimum": 0, + "type": "integer" + }, + "idle_seconds": { + "minimum": 0, + "type": "integer" + }, + "implicit": { + "type": "boolean" + }, + "recording_active": { + "type": "boolean" + }, + "session": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "state": { + "enum": [ + "active", + "ending" + ], + "type": "string" + }, + "transport": { + "enum": [ + "cli", + "daemon", + "mcp_stdio", + "mcp_http" + ], + "type": "string" + } + }, + "required": [ + "session", + "implicit", + "state", + "client_kind", + "transport", + "cursor_visible", + "recording_active", + "idle_seconds", + "expires_in_seconds" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "sessions", + "next_cursor" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_session_state", + "description": "Deprecated compatibility alias that reads a live legacy session's capture policy. Use get_session for lifecycle state.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "capture_scope": { + "enum": [ + "auto", + "window", + "desktop" + ], + "type": "string" + }, + "desktop_capture_authorized": { + "type": "boolean" + }, + "desktop_unlocked": { + "type": "boolean" + }, + "effective_scope": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "escalation_detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "escalation_reason": { + "anyOf": [ + { + "enum": [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other" + ], + "type": "string" + }, + { + "type": "null" + } + ] + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "capture_scope", + "effective_scope", + "desktop_capture_authorized", + "desktop_unlocked", + "escalation_reason", + "escalation_detail" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "end_session", + "description": "End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.", + "inputSchema": { + "additionalProperties": true, + "properties": { + "session": { + "description": "Optional public label to end. When omitted, end the caller's attached\nimplicit session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "active": { + "const": false + }, + "session": { + "type": "string" + } + }, + "required": [ + "session", + "active" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + }, + { + "name": "get_guide", + "description": "Read Cua Driver guidance on demand, taken from the guide bundled with this driver (no skill install needed). Topics: overview, drag, browser, permissions, batching, menus. Call with no topic to list them. Read-only.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "topic": { + "description": "One of: overview, drag, browser, permissions, batching, menus. Omit to list the topics.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + } + }, + { + "name": "run_steps", + "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Default app (name or bundle id) for steps that name no window.", + "type": "string" + }, + "delay_ms": { + "description": "Pause between steps in milliseconds (not after the last). Default 0.", + "maximum": 2000, + "minimum": 0, + "type": "integer" + }, + "delivery_mode": { + "description": "Default delivery_mode for steps whose tool takes one and that set none.", + "enum": [ + "background", + "foreground" + ], + "type": "string" + }, + "foreground_fallback": { + "description": "When a step is refused because its target ignores background input (Electron, LibreOffice and other toolkits), retry it once with delivery_mode:\"foreground\": the window is activated and the pointer may move. Default false.", + "type": "boolean" + }, + "observe": { + "description": "Optional end-of-batch observation: arguments for ONE get_window_state call. `pid` and `window_id` default to those of the last step that names both. Defaults to since=\"latest\" (only what changed since your last read of that window with the same query/max_elements/max_depth; a full read when there is none), include_screenshot=false and max_elements=250; pass include_screenshot=true to see the window, or since=null for a full read. Omit to read nothing. {} observes the last window.", + "type": "object" + }, + "pid": { + "description": "Default pid for steps that name no window (a step's own pid/window_id/app/window wins).", + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "steps": { + "description": "Ordered actions. Execution stops at the first failing step. Each step is {tool, args} or {: args}, plus optional wait_for / expect / timeout_ms; a step may also be only a wait_for or expect check.", + "items": { + "additionalProperties": true, + "properties": { + "args": { + "description": "Arguments for that tool, as in a direct call, optionally with role/name/nth/app/window to name the target instead of pid/window_id/element_token.", + "type": "object" + }, + "expect": { + "description": "Up to 4 checks that must hold after the action: {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}. Default timeout 2000 ms.", + "items": { + "type": "object" + }, + "maxItems": 4, + "type": "array" + }, + "timeout_ms": { + "description": "How long a named target may take to appear. Default 3000.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "tool": { + "description": "Action tool to run.", + "enum": [ + "click", + "double_click", + "right_click", + "set_value", + "type_text", + "press_key", + "hotkey", + "scroll", + "drag", + "move_cursor" + ], + "type": "string" + }, + "wait_for": { + "description": "Check that must hold before the action: {role?, name?, text?, gone?, app?, window?, timeout_ms?}. Default timeout 5000 ms.", + "type": "object" + } + }, + "type": "object" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" + }, + "window": { + "description": "Default window title substring for steps that name no window.", + "type": "string" + }, + "window_id": { + "description": "Default window_id for steps that name no window, with `pid`.", + "type": "integer" + } + }, + "required": [ + "steps" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "run_script", + "description": "Run a JavaScript program that drives apps through Cua Driver, in ONE call: loops, conditions, reads and checks between actions, and only the result comes back. Use it when the flow needs logic (a loop over rows, a branch on what the app shows, a retry); use run_steps for a fixed list of steps.\n\nThe script is the body of an async function: use `await`, and `return` a JSON value. It runs in a sandbox with no filesystem, network, process, timers or environment; the only API is `cua`:\n- `await cua.getApp(\"TextEdit\" | \"com.apple.TextEdit\" | {pid, windowId})` → app (follows the app's frontmost window unless a windowId is given)\n- `await cua.launch(nameOrBundleId)` → app; `cua.listApps()`, `cua.listWindows({pid?})`, `cua.sleep(ms)`, `cua.computer.target`\n- app.getState(opts?) → get_window_state result (`tree_markdown` or `tree_diff`, `snapshot_id`; pass get_window_state options such as {since:\"latest\"})\n- app.query({role?, name?, text?}) → matching elements across the app's windows {element_token, role, label, value, enabled, selected, frame}, then matching static-text rows {role, label, value, text, display_only} (read text such as a status line from their `value`)\n- app.waitFor({role?, name?, text?, gone?, value?, timeout_ms?}) and app.verify(predicate | predicates) (verify_state)\n- app.click(t), doubleClick(t), rightClick(t), typeText(text, t?), pressKey(\"return\" | \"cmd+s\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, direction, amount?), setValue(t, value), drag([x,y], [x,y]), zoom([x1,y1,x2,y2])\nwhere a target `t` is {role, name, nth?} (found fresh, waits up to 3 s), an element_token string, a row number from this app's last getState(), or [x, y] window pixels.\n- `cua.call(tool, args)` for any other allowed tool: click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, launch_app, list_apps, list_windows, get_window_state, verify_state, zoom.\nA failed driver call throws a CuaError (catch it to recover). Every call goes through the same session, permission and approval checks as a direct call.\n\nLimits: `timeout_ms` wall time (default 30000, max 120000), `max_calls` driver calls (default 100, max 500), 32 MiB heap. Returns the return value, console output, and a per-call log; an error names the script line, the call and the reason.\n\nExample:\nconst app = await cua.getApp(\"Calculator\");\nfor (const key of [\"7\", \"multiply\", \"6\", \"equals\"]) await app.click({role: \"button\", name: key});\nconst [display] = await app.query({role: \"statictext\"});\nreturn display.value; Batching rules: `get_guide({topic:\"batching\"})`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "max_calls": { + "description": "Most driver calls the script may make. Default 100.", + "maximum": 500, + "minimum": 1, + "type": "integer" + }, + "script": { + "description": "JavaScript: the body of an async function. `return` the result.", + "maxLength": 65536, + "minLength": 1, + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "timeout_ms": { + "description": "Wall-time limit. Default 30000.", + "maximum": 120000, + "minimum": 100, + "type": "integer" + } + }, + "required": [ + "script" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "check_for_update", + "description": "Check the saved stable/nightly Cua Driver channel for a release on GitHub. Returns current and selected channels, current and latest versions, an `update_available` boolean, the install one-liner, and the release notes URL. Read-only — never installs. Pacman-owned Linux executables return package-manager guidance without checking GitHub. Mirror of `cua-driver check-update --json`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": true, + "readOnlyHint": true + } + }, + { + "name": "install_extension", + "description": "Preview or install one Driver-managed optional extension. The first call without confirm returns the exact signed artifact, destination, license, source, and trust plan without mutation. Re-call with confirm=true to perform that exact verified installation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "confirm": { + "description": "Install the previewed extension. Omit or false for a read-only plan.", + "type": "boolean" + }, + "name": { + "description": "Extension to preview or install. Only perception (the local visual-region parser used by parse_visual_regions) is available.", + "enum": [ + "perception" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true, + "readOnlyHint": false + } + }, + { + "name": "parse_visual_regions", + "description": "Parse one immutable registered capture into model-neutral text and icon regions.", + "inputSchema": { + "additionalProperties": false, + "description": "Parse one immutable screenshot capture. Pass the `capture_id` returned by\n`get_window_state` or `get_desktop_state`; the Driver capture registry\nresolves it to the exact pixels and action-coordinate transform.", + "properties": { + "capture_id": { + "description": "`capture_id` of a screenshot returned by get_window_state or get_desktop_state.", + "minLength": 1, + "type": "string" + }, + "options": { + "additionalProperties": false, + "default": {}, + "description": "Optional, model-neutral controls for one bounded parse.", + "properties": { + "kinds": { + "description": "Region kinds to return (`text`, `icon`). Omit for all kinds.", + "items": { + "enum": [ + "text", + "icon" + ], + "type": "string" + }, + "maxItems": 2, + "minItems": 1, + "type": "array" + }, + "max_regions": { + "description": "Return at most this many regions.", + "minimum": 1, + "type": "integer" + }, + "min_confidence": { + "description": "Drop regions below this confidence (0 to 1).", + "maximum": 1, + "minimum": 0, + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "capture_id" + ], + "type": "object" + }, + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "outputSchema": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "capture": { + "properties": { + "action_coordinate_space": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "screenshot_pixels", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "affine", + "type": "string" + }, + "m11": { + "type": "number" + }, + "m12": { + "type": "number" + }, + "m21": { + "type": "number" + }, + "m22": { + "type": "number" + }, + "tx": { + "type": "number" + }, + "ty": { + "type": "number" + } + }, + "required": [ + "kind", + "m11", + "m12", + "m21", + "m22", + "tx", + "ty" + ], + "type": "object" + } + ] + }, + "capture_id": { + "type": "string" + }, + "captured_at": { + "type": [ + "string", + "null" + ] + }, + "screenshot": { + "properties": { + "height": { + "minimum": 1, + "type": "integer" + }, + "mime_type": { + "const": "image/png" + }, + "reference": { + "type": "string" + }, + "sha256": { + "type": [ + "string", + "null" + ] + }, + "width": { + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "reference", + "width", + "height", + "mime_type" + ], + "type": "object" + }, + "source": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "window", + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "pid", + "window_id" + ], + "type": "object" + }, + { + "properties": { + "display_id": { + "const": "primary" + }, + "kind": { + "const": "primary_desktop", + "type": "string" + } + }, + "required": [ + "kind", + "display_id" + ], + "type": "object" + } + ] + } + }, + "required": [ + "capture_id", + "source", + "screenshot", + "action_coordinate_space" + ], + "type": "object" + }, + "parser": { + "properties": { + "backend": { + "type": [ + "string", + "null" + ] + }, + "extension_id": { + "type": "string" + }, + "extension_version": { + "type": "string" + }, + "fixture_sha256": { + "type": [ + "string", + "null" + ] + }, + "model_id": { + "type": "string" + }, + "model_manifest_sha256": { + "type": [ + "string", + "null" + ] + }, + "model_source_revision": { + "type": [ + "string", + "null" + ] + }, + "model_version": { + "type": "string" + }, + "onnx_runtime_library_sha256": { + "type": [ + "string", + "null" + ] + }, + "onnx_runtime_version": { + "type": [ + "string", + "null" + ] + }, + "runtime": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "extension_id", + "extension_version", + "model_id", + "model_version" + ], + "type": "object" + }, + "regions": { + "items": { + "properties": { + "bounds": { + "properties": { + "height": { + "minimum": 1, + "type": "integer" + }, + "width": { + "minimum": 1, + "type": "integer" + }, + "x": { + "minimum": 0, + "type": "integer" + }, + "y": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "type": "object" + }, + "confidence": { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + "group_id": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": "string" + }, + "interactive": { + "type": "boolean" + }, + "kind": { + "enum": [ + "text", + "icon" + ], + "type": "string" + }, + "label": { + "type": [ + "string", + "null" + ] + }, + "parent_id": { + "type": [ + "string", + "null" + ] + }, + "reading_order": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "text": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id", + "kind", + "bounds", + "confidence", + "interactive" + ], + "type": "object" + }, + "type": "array" + }, + "request_id": { + "type": [ + "string", + "null" + ] + }, + "schema": { + "const": "cua.visual_regions_v1" + }, + "timing": { + "properties": { + "duration_ms": { + "minimum": 0, + "type": "integer" + }, + "inference_ms": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "preprocess_ms": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "duration_ms" + ], + "type": "object" + }, + "warnings": { + "items": { + "properties": { + "code": { + "type": "string" + }, + "detail": { + "type": [ + "string", + "null" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "code", + "message" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "schema", + "capture", + "parser", + "regions", + "timing" + ], + "type": "object" + }, + { + "additionalProperties": true, + "anyOf": [ + { + "required": [ + "refusal" + ] + }, + { + "required": [ + "status" + ] + }, + { + "required": [ + "code" + ] + } + ], + "type": "object" + } + ], + "type": "object" + } + } + ], + "resources": [ + { + "uri": "skill://cua-driver/BROWSER.md", + "name": "BROWSER.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/EMBEDDING.md", + "name": "EMBEDDING.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/LINUX.md", + "name": "LINUX.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/MACOS.md", + "name": "MACOS.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/README.md", + "name": "README.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/RECORDING.md", + "name": "RECORDING.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/RUNTIME.md", + "name": "RUNTIME.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/SKILL.md", + "name": "cua-driver", + "description": "Drive a native GUI app (macOS, Windows, Linux) via the cua-driver CLI (default) or MCP server; snapshot its accessibility tree, act through snapshot-bound element tokens, native menu paths, exact window geometry, or pixel coordinates, and verify from fresh state. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host, or to continue, resume, or recall recent Cua activity.", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/VISUAL.md", + "name": "VISUAL.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/WORKFLOW.md", + "name": "WORKFLOW.md", + "mimeType": "text/markdown" + }, + { + "uri": "skill://cua-driver/WINDOWS.md", + "name": "WINDOWS.md", + "mimeType": "text/markdown" + } + ] +} diff --git a/src/agent_env/drivers/envs/cua-driver/tools.json b/src/agent_env/drivers/envs/cua-driver/tools.json deleted file mode 100644 index c65229a17..000000000 --- a/src/agent_env/drivers/envs/cua-driver/tools.json +++ /dev/null @@ -1,3832 +0,0 @@ -[ - { - "name": "list_apps", - "description": "List Linux apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: reserved (Linux X11/Wayland focus model differs from frontmost-app); always false.\n- kind: `\"desktop\"` for XDG `.desktop` launcher entries.\n- launch_path: the launcher command from `Exec=` (field codes stripped). Pass to `launch_app(launch_path=...)`.\n- bundle_id: the XDG \"desktop file id\" — the `.desktop` file's path relative to its XDG `applications/` root with the `.desktop` suffix stripped and path separators replaced with `-` (e.g. `kde4/konqbrowser.desktop` → `kde4-konqbrowser`).\n- last_used: RFC3339 mtime of the `.desktop` file, when readable.\n- windows: the app's current top-level windows (same records as list_windows); empty for a process without one.\n\nRunning apps come from `/proc`. Installed apps come from XDG Desktop Entry files in $XDG_DATA_HOME/applications and each $XDG_DATA_DIRS entry's applications/ subdir. Entries with `NoDisplay=true` or `Hidden=true` are filtered. A `.desktop` file whose launcher matches a running process (by basename), or whose WM_CLASS a top-level window carries, is merged into a single entry with `running: true`.\n\nUse this for \"is X installed?\" as well as \"is X running?\". For per-window state — visibility, geometry, titles — call list_windows instead.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "list_windows", - "description": "List top-level windows. Each record includes z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one). To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order. Filter with `app` or `pid`: an unfiltered call lists every window on the host.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "app": { - "description": "Only windows of apps whose name contains this text (case-insensitive), or whose bundle id equals it. Prefer this or `pid` over an unfiltered call, which returns every window on the host.", - "type": "string" - }, - "on_screen_only": { - "description": "When true, filter to visible windows only. Default false.", - "type": "boolean" - }, - "pid": { - "description": "Only list windows owned by this process ID.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_window_state", - "description": "Walk a running app's AT-SPI tree and return it ONCE, as compact Markdown by default (`tree_format:\"markdown\"`). Every actionable element is tagged `[N]` (an element_index); its element_token is `:N`, where `snapshot_id` is printed in the response header and returned in structuredContent. Pass `tree_format:\"elements\"` for the structured `elements` array instead (explicit `element_token`, `role`, `label`, `value`, `enabled`, `selected`, `actions`, `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`), or `\"both\"` for both (about twice the size). Set `query` to project the tree to matching rows plus their ancestor chain while preserving original indices; `total_element_count` reports the complete snapshot.\n\nSize: a read is bounded by default (`max_elements` 250 actionable nodes). When the walk stops at that budget the response says `Tree truncated at max_elements=…` and `truncated: true`; pass a larger `max_elements`, or `query` / `max_depth`, to reach the rest. `_note` is omitted unless `verbose:true`. `full_output:true` restores the previous full response: both representations, all metadata, and the walker's own limits (≤5 000 nodes).\n\nDIFF READS: pass `since: ` from an earlier read of the same window to get only what changed: `+` added rows, `~` changed rows, `-` removed rows (their ids are the old snapshot's), or `no change since …`. Rows not listed keep their previous `[N]` unless a `reindexed:` line says otherwise. The response carries a NEW snapshot_id: use it in element_tokens. An unknown, expired, other-window or differently-scoped (query/max_elements/max_depth) `since` falls back to a full read and `since_status` says why. The focused element is not reported on Linux yet.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error. Optional `max_dimension` caps the returned screenshot's long edge in pixels for a cheap thumbnail.\n\nOptional `max_elements` / `max_depth` bound the AT-SPI walk (Electron / large web apps produce 10k+ element trees). Markdown and structured elements are truncated identically. Default: `max_elements` 250.\n\nTIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. When omitted, a window's first snapshot gets a 2000 ms cold-start budget so Chromium/Electron can finish publishing its initial tree. Explicit values are always honored. Large apps (LibreOffice, GIMP, file managers) can exceed the budget; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.\n\nSCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.\n\nPOPUP MENUS: a context menu / popover / combo list is an override-redirect window that list_windows never shows. A click or right_click that opened one names it in its result (`popup: {window_id, bounds, title}`); pass that window_id here to walk the popup's own AT-SPI toplevel so its menu items get element indices (then click them by element_token). Omitting window_id while a popup of this pid is open walks that popup.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "capture_mode": { - "description": "DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.", - "enum": [ - "ax", - "vision" - ], - "type": "string" - }, - "display_only": { - "description": "Default false. With include_accessibility_tree:false, returns pixels for display without registering an action capture or replacing the agent snapshot. The image cannot ground input actions.", - "type": "boolean" - }, - "full_output": { - "description": "Default false. Set true to restore the previous full response: both `elements` and `tree_markdown`, all metadata, and the platform's own walk limits unless max_elements / max_depth are passed. Does not combine with `since`.", - "type": "boolean" - }, - "include_accessibility_tree": { - "description": "Default true — walk the AT-SPI tree and return it (per `tree_format`) alongside the screenshot. Set false to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return).", - "type": "boolean" - }, - "include_screenshot": { - "description": "Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action).", - "type": "boolean" - }, - "max_depth": { - "description": "Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps.", - "minimum": 1, - "type": "integer" - }, - "max_dimension": { - "description": "Legacy optional cap on the returned screenshot's long edge. Applied on top of the configured max_image_dimension ceiling when max_image_dimension is omitted.", - "minimum": 1, - "type": "integer" - }, - "max_elements": { - "description": "Cap on total AT-SPI nodes walked. Default 250 (5 000 with full_output:true); the response states when the tree was cut. Raise it to read further into a large window.", - "minimum": 1, - "type": "integer" - }, - "max_image_dimension": { - "description": "Per-call long-edge override. This value wins over configured and legacy limits; 0 returns native-resolution PNG bytes. Omit to preserve configured behavior.", - "minimum": 0, - "type": "integer" - }, - "pid": { - "description": "Process ID that owns the window.", - "type": "integer" - }, - "query": { - "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.", - "type": "string" - }, - "screenshot_out_file": { - "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output carries screenshot_file_path instead.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "since": { - "description": "A `snapshot_id` from an earlier get_window_state of the same window, or \"latest\" for this window's most recent read with the same query/max_elements/max_depth. Use it on every re-read after acting. Returns only what changed against that snapshot: `+` added rows, `~` changed rows, `-` removed rows (removed ids are the old snapshot's), or `no change`. Falls back to a full read when the snapshot is unknown, expired, belongs to another window, or was read with a different query/max_elements/max_depth; `since_status` says which. The response still carries a NEW snapshot_id and its element_tokens; rows not listed keep their previous [N] unless a `reindexed:` line says otherwise.", - "type": "string" - }, - "timeout_ms": { - "default": 1000, - "description": "Wall-clock budget in milliseconds for the accessibility-tree walk (default 1000, min 100, max 120000). Bounds the WHOLE walk. When the budget runs out the tool returns the PARTIAL tree it has, flagged with `truncated: true`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `elements_complete: false`; retry with a larger value (e.g. 5000) or narrow with `query` / `max_depth`.", - "maximum": 120000, - "minimum": 100, - "type": "integer" - }, - "tree_format": { - "description": "Which tree representation to return. Default \"markdown\": the compact `tree_markdown` only, where a row `[N]` is addressed with element_token `:N`. \"elements\": the structured `elements` array only (per-element element_token, frame, parent_index, value). \"both\": the two together, as before; roughly doubles the size.", - "enum": [ - "markdown", - "elements", - "both" - ], - "type": "string" - }, - "verbose": { - "description": "Default false. Set true to include the `_note` string and the full `background_input` capability report (macOS), which are omitted by default to save context.", - "type": "boolean" - }, - "window_id": { - "description": "Native window identifier from list_windows, or the `popup.window_id` a click / right_click result named (an open context menu / popover; its menu items then get element indices). Omitted: the pid's open popup menu when one is mapped, else its focused / active / largest window.", - "type": "integer" - } - }, - "required": [ - "pid" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "verify_state", - "description": "Deterministically verify bounded predicates against one exact window. The driver evaluates structured window/accessibility state and may return the final screenshot as uninterpreted visual evidence for a multimodal caller. Predicate results are satisfied, unsatisfied, or unknown; unknown never implies success. Accessibility projections are conservative: absence remains unknown unless the observed search domain is proven exhaustive.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "expect": { - "description": "One to eight predicates, combined with logical AND.", - "items": { - "additionalProperties": false, - "properties": { - "element": { - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "exists": { - "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", - "type": "boolean" - }, - "selected": { - "type": "boolean" - }, - "selector": { - "additionalProperties": false, - "properties": { - "label_contains": { - "minLength": 1, - "type": "string" - }, - "role": { - "minLength": 1, - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "value_equals": { - "type": "string" - } - }, - "required": [ - "selector" - ], - "type": "object" - }, - "window": { - "additionalProperties": false, - "properties": { - "bounds": { - "additionalProperties": false, - "properties": { - "height": { - "type": "number" - }, - "tolerance_px": { - "maximum": 100, - "minimum": 0, - "type": "number" - }, - "width": { - "type": "number" - }, - "x": { - "type": "number" - }, - "y": { - "type": "number" - } - }, - "required": [ - "x", - "y", - "width", - "height" - ], - "type": "object" - }, - "exists": { - "type": "boolean" - } - }, - "required": [], - "type": "object" - } - }, - "required": [], - "type": "object" - }, - "maxItems": 8, - "minItems": 1, - "type": "array" - }, - "include_screenshot": { - "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", - "type": "boolean" - }, - "pid": { - "description": "Exact process whose window may be observed.", - "minimum": 1, - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session. This\nfield never selects capture modality or authorization.", - "type": "string" - }, - "stable_samples": { - "default": 2, - "description": "Consecutive satisfied samples required before returning success.", - "maximum": 5, - "minimum": 1, - "type": "integer" - }, - "timeout_ms": { - "default": 5000, - "description": "Bounded wait. Zero performs one sample.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Exact native window identifier.", - "type": "integer" - } - }, - "required": [ - "pid", - "window_id", - "expect" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "launch_app", - "description": "Launch a Linux app in the background. Provide launch_path (preferred — round-trip the value from list_apps), name (tried as a direct command, then matched against installed .desktop applications, then handed to xdg-open if it is a URL or existing file path), bundle_id (ignored on Linux), or urls (list of URLs to open). Resolution precedence: launch_path > name > bundle_id. Errors when the name resolves to nothing launchable.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "additional_arguments": { - "description": "Extra command-line arguments passed to the launched process.", - "items": { - "type": "string" - }, - "type": "array" - }, - "bundle_id": { - "description": "Ignored on Linux (macOS/Windows concept).", - "type": "string" - }, - "launch_path": { - "description": "Round-trip the `launch_path` returned by `list_apps` — the Exec= command from the .desktop file with XDG field codes already stripped. Highest precedence on Linux; spawned directly via the system shell.", - "type": "string" - }, - "name": { - "description": "App name or command to launch. Tried as a direct command first, then matched against installed .desktop applications (exact display name, desktop-file id, or Exec basename; else an unambiguous display-name substring).", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "urls": { - "description": "URLs to open via xdg-open.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "kill_app", - "description": "Force-terminate a process by pid (kill -9 equivalent on Linux). Use as escalation when the cooperative close path failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "pid": { - "description": "PID of the process to terminate.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "required": [ - "pid" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "bring_to_front", - "description": "Persistently activate a window so subsequent input lands on it. This deliberately breaks the no-foreground contract and is not part of the normal input ladder. For an ordinary `background_unavailable` response, retry only the refused action with `delivery_mode:\"foreground\"`; the input tool performs its own activate, act, and restore sequence. Use `bring_to_front` only for a focus-proxy surface that must remain foreground across multiple calls, such as a remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, with proper timestamp handling to beat focus-stealing prevention). Wayland: activates through a target-addressable compositor adapter (wlroots foreign-toplevel or the GNOME Shell helper) and refuses when the compositor offers no safe adapter. Matches the macOS / Windows bring_to_front rung.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "pid": { - "description": "Process ID of the app to activate.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "window_id": { - "description": "X11 window id (xid) to activate. If omitted, the first window of `pid` is used.", - "type": "integer" - } - }, - "required": [ - "pid" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "set_window_frame", - "description": "Set one exact top-level window's frame in the desktop-coordinate space reported by list_windows and verify the resulting geometry through an independent readback.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "height": { - "description": "New height, in the same units as list_windows bounds.", - "minimum": 1, - "type": "number" - }, - "pid": { - "description": "Process ID that owns the window.", - "minimum": 1, - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "width": { - "description": "New width, in the same units as list_windows bounds.", - "minimum": 1, - "type": "number" - }, - "window_id": { - "description": "Window ID from list_windows.", - "minimum": 1, - "type": "integer" - }, - "x": { - "description": "New left edge in the desktop coordinate space reported by list_windows.", - "type": "number" - }, - "y": { - "description": "New top edge in the desktop coordinate space reported by list_windows.", - "type": "number" - } - }, - "required": [ - "pid", - "window_id", - "x", - "y", - "width", - "height" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "invoke_menu", - "description": "Resolve an exact application-menu path one live native level at a time and invoke its final item through accessibility APIs. Missing, ambiguous, disabled, or structurally mismatched segments fail closed; this tool never falls back to pixels. Menu paths and refusals: `get_guide({topic:\"menus\"})`.", - "inputSchema": { - "additionalProperties": false, - "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).", - "properties": { - "path": { - "description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).", - "items": { - "maxLength": 200, - "minLength": 1, - "type": "string" - }, - "maxItems": 16, - "minItems": 1, - "type": "array" - }, - "pid": { - "description": "Process ID of the application that owns the menu.", - "minimum": 1, - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "window_id": { - "description": "Window ID from list_windows whose menu is invoked.", - "minimum": 1, - "type": "integer" - } - }, - "required": [ - "pid", - "window_id", - "path" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "click", - "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "button": { - "description": "Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error.", - "enum": [ - "left", - "right", - "middle" - ], - "type": "string" - }, - "capture_id": { - "description": "Optional one-shot binding to the exact PNG returned by get_window_state or get_desktop_state. When present, x/y are interpreted in that capture and admitted before native dispatch.", - "type": "string" - }, - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "count": { - "description": "Number of clicks in one press train: 1 (default), 2 = double-click, 3 = triple-click (line/paragraph selection). There is no separate triple_click tool — pass count: 3 here.", - "minimum": 1, - "type": "integer" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "modifier": { - "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", - "items": { - "type": "string" - }, - "type": "array" - }, - "pid": { - "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Coordinate frame (default \"window\"). Pass \"desktop\" with x,y and no pid/window_id for a screen-absolute click in get_desktop_state coordinates.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Exact window (kind \"window\", with pid and window_id) or desktop (kind \"desktop\", with display_id). \"primary\" is the portable desktop target.", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "double_click", - "description": "Double-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to). No focus steal. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "pid": { - "description": "Target process ID. Optional with element_token (the token carries the pid).", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "right_click", - "description": "Right-click at (x,y) or an element_token (AT-SPI bounds). Background delivery on X11 sends real button events from the session's virtual master pointer (path mpx_pointer); a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it (retargeted_to), one outside the window is refused (point_outside_window; x/y are window-local pixels, the result reports window_point / screen_point). No focus steal. When a context menu opens, the result names it (`popup: {window_id, bounds, title}`): call get_window_state(pid, window_id=) to index its menu items and click them by element_token. Provide either (window_id + x/y) or (pid + element_token). After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "modifier": { - "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", - "items": { - "type": "string" - }, - "type": "array" - }, - "pid": { - "description": "Target process ID. Optional with element_token (the token carries the pid).", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "drag", - "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. Background delivery on X11 is one real held gesture on the session's virtual master pointer (path mpx_pointer: press, interpolated motion, release) — it lands on GTK/VCL/Qt/Chromium; a point covered by another application's window is refused (target_occluded), one covered by this app's own dialog is retargeted to it. Headless servers fall back to XSendEvent. duration_ms (default 500), steps (default 20). No focus steal. Limits and fallbacks: `get_guide({topic:\"drag\"})`.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "button": { - "description": "Mouse button. Default \"left\".", - "enum": [ - "left", - "right", - "middle" - ], - "type": "string" - }, - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "duration_ms": { - "description": "Total drag duration. Default: 500.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "from_x": { - "description": "Drag start X: window-local pixels of the target window's own get_window_state screenshot; pass scope:\"desktop\" for get_desktop_state pixels.", - "type": "number" - }, - "from_y": { - "description": "Drag start Y (same frame as from_x).", - "type": "number" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "modifier": { - "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.", - "items": { - "type": "string" - }, - "type": "array" - }, - "pid": { - "description": "Target process ID. Required unless scope is \"desktop\".", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Use \"desktop\" with no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "steps": { - "description": "Intermediate MotionNotify events. Default: 20.", - "maximum": 200, - "minimum": 1, - "type": "integer" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "to_x": { - "description": "Drag end X (same frame as from_x).", - "type": "number" - }, - "to_y": { - "description": "Drag end Y (same frame as from_x).", - "type": "number" - }, - "window_id": { - "description": "Target window XID. Required.", - "type": "integer" - } - }, - "required": [ - "from_x", - "from_y", - "to_x", - "to_y" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "mouse_button_down", - "description": "Press and hold a mouse button at (x,y) via background X11 delivery. Does not release the button; pair with mouse_drag / mouse_button_up. Returns the current held-button state.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "button": { - "description": "Mouse button. Default \"left\".", - "enum": [ - "left", - "right", - "middle" - ], - "type": "string" - }, - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "pid": { - "description": "Target process ID.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", - "type": "string" - }, - "window_id": { - "description": "Window id from list_windows.", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "required": [ - "pid", - "window_id", - "x", - "y" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "mouse_drag", - "description": "Move a previously-held mouse button to a new point via background X11 delivery. Requires an active mouse_button_down state; does not release the button. Returns the updated held-button state.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "duration_ms": { - "description": "Total drag duration. Default: 500.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "pid": { - "description": "Optional process ID; must match the held button's target.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", - "type": "string" - }, - "steps": { - "description": "Intermediate MotionNotify events. Default: 20.", - "maximum": 200, - "minimum": 1, - "type": "integer" - }, - "window_id": { - "description": "Optional window id; must match the held button's target.", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "required": [ - "x", - "y" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "mouse_button_up", - "description": "Release a previously-held mouse button via background X11 delivery. If x/y are omitted, releases at the last held position. Returns the current held-button state.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "from_zoom": { - "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.", - "type": "boolean" - }, - "pid": { - "description": "Optional process ID; must match the held button's target.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.", - "type": "string" - }, - "window_id": { - "description": "Optional window id; must match the held button's target.", - "type": "integer" - }, - "x": { - "description": "Window-local pixel X of the target window's own get_window_state screenshot (0..screenshot_width). For get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Window-local pixel Y of the target window's own get_window_state screenshot (0..screenshot_height); see x.", - "type": "number" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "parallel_mouse_drag", - "description": "Run multiple mouse drag gestures concurrently via Linux MPX/XI2 virtual master pointers. Each drag item runs on its own session-scoped master pointer (true same-window concurrent draws on X11). Each item presses once, glides continuously through its whole path, and releases once — one smooth held drag, not a chain of clicks. A path is given either as a straight segment (from_x/from_y → to_x/to_y) or as a function `fn` = y(x) sampled over [x_from, x_to] in window-local pixels (e.g. fn:\"x\" is a diagonal, fn:\"300+120*sin(x/40)\" a sine wave). Functions support + - * / ^, sin/cos/tan, sqrt, abs, exp, ln, pi, e.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "drags": { - "description": "Two or more drag gestures to run concurrently, each on its own virtual master pointer.", - "items": { - "additionalProperties": false, - "properties": { - "button": { - "description": "Mouse button. Default \"left\".", - "enum": [ - "left", - "right", - "middle" - ], - "type": "string" - }, - "duration_ms": { - "description": "Default: 1500 for fn paths, 500 for straight.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "fn": { - "description": "Expression y(x) in window-local pixels; sampled over [x_from,x_to]. Mutually exclusive with from_x/to_x.", - "type": "string" - }, - "from_x": { - "description": "Straight-segment start X (window-local pixels).", - "type": "number" - }, - "from_y": { - "description": "Straight-segment start Y (window-local pixels).", - "type": "number" - }, - "path": { - "description": "Explicit window-local waypoints [[x,y],...] (>=2); pressed once, glided through, released once. Takes precedence over fn/from-to.", - "items": { - "items": { - "type": "number" - }, - "type": "array" - }, - "type": "array" - }, - "samples": { - "description": "Waypoints sampled along `fn`. Default: 80.", - "maximum": 400, - "minimum": 2, - "type": "integer" - }, - "session": { - "description": "Session/cursor id; also keys the virtual master pointer.", - "type": "string" - }, - "steps": { - "description": "Motion sub-steps along the whole path. Default: scaled to path length.", - "maximum": 300, - "minimum": 1, - "type": "integer" - }, - "to_x": { - "description": "Straight-segment end X (window-local pixels).", - "type": "number" - }, - "to_y": { - "description": "Straight-segment end Y (window-local pixels).", - "type": "number" - }, - "window_id": { - "description": "Window id from list_windows.", - "type": "integer" - }, - "x_from": { - "description": "Domain start (window-local x) when `fn` is used.", - "type": "number" - }, - "x_to": { - "description": "Domain end (window-local x) when `fn` is used.", - "type": "number" - } - }, - "required": [ - "session", - "window_id" - ], - "type": "object" - }, - "minItems": 2, - "type": "array" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "required": [ - "drags" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "type_text", - "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "pid": { - "description": "Target process ID. Omit with scope \"desktop\" to type into the focused application.", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Use \"desktop\" with no pid to type into the focused application. Default \"window\".", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "text": { - "description": "Text to type.", - "type": "string" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Window-local pixels of the target window's own get_window_state screenshot by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\") — a bare pid+x/y is otherwise misinterpreted as window-local and can focus the wrong widget.", - "type": "number" - }, - "y": { - "description": "Pixel Y of the field (see x).", - "type": "number" - } - }, - "required": [ - "text" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "press_key", - "description": "Press a key via XSendEvent to a window. No focus steal.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "key": { - "description": "Key name: enter/return, tab, escape, space, backspace, delete, insert, home, end, pageup, pagedown, up, down, left, right, f1-f12, or any single ASCII character.", - "type": "string" - }, - "modifiers": { - "description": "Modifier keys held while the key is pressed: ctrl, shift, alt, super.", - "items": { - "type": "string" - }, - "type": "array" - }, - "pid": { - "description": "Target process ID. Omit with scope \"desktop\" to send the key to the focused application.", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Use \"desktop\" with no pid to send the key to the focused application. Default \"window\".", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_token. Window-local pixels by default (same convention as click); for get_desktop_state pixels pass scope:\"desktop\" (or coordinate_frame:\"desktop\").", - "type": "number" - }, - "y": { - "description": "Pixel Y (see x).", - "type": "number" - } - }, - "required": [ - "key" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "hotkey", - "description": "Press a combination of keys simultaneously, e.g. [\"ctrl\",\"c\"] for Copy. Delivered to the target window without focus steal (real key events from a virtual master keyboard on X11); target does NOT need to be frontmost. Window-manager chords cannot reach the WM that way: alt+F4 on a window target closes that window through the WM (_NET_CLOSE_WINDOW, effect confirmed when it unmaps); super+*, alt+tab, ctrl+alt+* and other WM bindings are refused with code wm_chord_unavailable — use bring_to_front / list_windows / kill_app or delivery_mode:\"foreground\" instead.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "keys": { - "description": "Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"].", - "items": { - "type": "string" - }, - "minItems": 2, - "type": "array" - }, - "pid": { - "description": "Target process ID. Omit with scope \"desktop\" to send the chord to the focused application.", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Use \"desktop\" with no pid to send the chord to the focused application. Default \"window\".", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y or element_index; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.", - "type": "number" - }, - "y": { - "description": "Screenshot-pixel Y (see x).", - "type": "number" - } - }, - "required": [ - "keys" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "set_value", - "description": "Set value of an AT-SPI element via SetValue action.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "pid": { - "description": "Target process ID. Optional with element_token (the token carries the pid).", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "value": { - "description": "New value. Written through AT-SPI EditableText/Value when the element exposes them; otherwise (GTK2 spin scales, VCL spin buttons) the field is clicked with the session's real pointer, its text selected and replaced through the virtual keyboard and committed with Tab, then read back.", - "type": "string" - }, - "window_id": { - "description": "Omit when element_token is supplied (the token carries it).", - "type": "integer" - } - }, - "required": [ - "value" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "scroll", - "description": "Scroll the target pid's focused region via XSendEvent Button4/5. direction required; by defaults to line, amount defaults to 3.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "amount": { - "description": "Number of scroll steps. Default: 3.", - "maximum": 50, - "minimum": 1, - "type": "integer" - }, - "by": { - "description": "Scroll granularity. Default: line.", - "enum": [ - "line", - "page" - ], - "type": "string" - }, - "coordinate_frame": { - "description": "Frame of x/y (and from_x/from_y/to_x/to_y). Default \"window\": window-local screenshot pixels as returned by get_window_state. \"desktop\": full-screen pixels as returned by get_desktop_state, translated to the target window. Passing scope:\"desktop\" together with pid/window_id means the same thing.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "cursor_id": { - "description": "Optional multi-cursor instance id. Default: 'default'.", - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "direction": { - "description": "Scroll direction.", - "enum": [ - "up", - "down", - "left", - "right" - ], - "type": "string" - }, - "element_token": { - "description": "Element handle `:` from get_window_state: row `[11]` of a read whose snapshot_id is s0000002a is \"s0000002a:11\" (also in `structuredContent.elements[].element_token`). Returns an explicit stale error naming the current snapshots once a newer read supersedes it.", - "pattern": "^s[0-9a-f]{8}:[0-9]+$", - "type": "string" - }, - "pid": { - "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied (the token carries the pid).", - "type": "integer" - }, - "scope": { - "default": "window", - "description": "Use \"desktop\" with x,y and no pid/window_id for get_desktop_state screen coordinates. Default \"window\".", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "window_id": { - "description": "Window id from list_windows. Required with x/y; optional with element_token (the token carries it).", - "type": "integer" - }, - "x": { - "description": "Window-local screenshot-pixel X of the scroll target. Pass with y and without element_token.", - "type": "number" - }, - "y": { - "description": "Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_token.", - "type": "number" - } - }, - "required": [ - "direction" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "clipboard_read", - "description": "List available system clipboard types and optionally return privacy-sensitive plain text. Clipboard content is never retained in telemetry.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "include_text": { - "default": false, - "description": "Return plain-text clipboard content in addition to the available types.\nClipboard content is privacy-sensitive and is never retained in telemetry.", - "type": "boolean" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "clipboard_write", - "description": "Replace the system clipboard with exactly one value: plain text, an image from an absolute local path, or a file URL from an absolute local path. Returns the available types for read-back before paste.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "file_path": { - "description": "Absolute path to a local file to place on the clipboard as a file URL.", - "type": "string" - }, - "image_path": { - "description": "Absolute path to a local image to place on the clipboard.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "text": { - "description": "Plain text to place on the clipboard.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_screen_size", - "description": "Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_desktop_state", - "description": "Capture the full display in the desktop action coordinate frame. Use the returned PNG directly as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. No AT-SPI walk.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "max_image_dimension": { - "description": "Optional long-edge cap for the returned PNG, in pixels (aspect ratio preserved). Omitted or 0 returns the full-size capture. When the cap downsizes the image, the response reports `screenshot_original_width/height`, and x/y read off it for this session's later scope:\"desktop\" actions (or passed with its `capture_id`) are mapped back to the full-size frame automatically.", - "minimum": 0, - "type": "integer" - }, - "screenshot_out_file": { - "description": "Write PNG here instead of base64.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "get_cursor_position", - "description": "Return the current mouse cursor position in screen points (origin top-left).", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "move_cursor", - "description": "Move the synthetic agent cursor without changing the user's pointer. Only an explicit scope=desktop request moves the real OS pointer in get_desktop_state coordinates.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "cursor_id": { - "description": "Cursor instance to move. Default: 'default'.", - "type": "string" - }, - "scope": { - "default": "window", - "description": "\"window\" (default) moves only the agent cursor overlay; \"desktop\" moves the real OS pointer.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target": { - "additionalProperties": true, - "description": "Preferred per-call target: an exact window (`kind=\"window\"`, `pid`, `window_id`) or the\nprimary desktop (`kind=\"desktop\"`, `display_id=\"primary\"`).", - "properties": { - "display_id": { - "description": "Display id. Required when kind is desktop; use \"primary\".", - "type": "string" - }, - "kind": { - "description": "window requires pid and window_id; desktop requires display_id.", - "enum": [ - "window", - "desktop" - ], - "type": "string" - }, - "pid": { - "description": "Process id. Required when kind is window.", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "description": "Window id from list_windows. Required when kind is window.", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "x": { - "description": "Destination X. Window scope: screen coordinates of the agent cursor overlay. Desktop scope: get_desktop_state screenshot pixels.", - "type": "number" - }, - "y": { - "description": "Destination Y, in the same space as x.", - "type": "number" - } - }, - "required": [ - "x", - "y" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "set_agent_cursor_enabled", - "description": "Show or hide the agent cursor owned by a session.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "enabled": { - "description": "`true` shows the session's agent cursor overlay; `false` hides it.", - "type": "boolean" - }, - "session": { - "description": "Public label of the session that owns the cursor.", - "type": "string" - } - }, - "required": [ - "session", - "enabled" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "set_agent_cursor_motion", - "description": "Configure the movement style, timing, effects and visibility timing for a session cursor.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "arc_flow": { - "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", - "type": "number" - }, - "arc_size": { - "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", - "type": "number" - }, - "dwell_after_click_ms": { - "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", - "type": "number" - }, - "effects": { - "additionalProperties": false, - "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", - "properties": { - "glow": { - "description": "Soft glow around the cursor that grows with speed.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "magnet": { - "description": "Target glow when the `magnetic` style locks on.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "ripple": { - "description": "Ring that expands from the hotspot on click.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "squish": { - "description": "Brief scale-down of the cursor on click.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "trail": { - "description": "Short fading trail behind the cursor.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "end_handle": { - "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", - "type": "number" - }, - "glide_duration_ms": { - "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", - "type": "number" - }, - "idle_hide_ms": { - "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", - "type": "number" - }, - "session": { - "description": "Public label of the session that owns the cursor. Omitted or null motion fields keep\ntheir current value.", - "type": "string" - }, - "spring": { - "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", - "type": "number" - }, - "start_handle": { - "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", - "type": "number" - }, - "style": { - "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", - "enum": [ - "signature_arc", - "spring_settle", - "magnetic", - "comet_swoop", - "adaptive", - "classic" - ], - "type": "string" - }, - "timing": { - "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", - "enum": [ - "native", - "fitts", - "fixed", - "relaxed", - "steady", - "calm", - "deliberate" - ], - "type": "string" - }, - "turn_radius": { - "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", - "type": "number" - } - }, - "required": [ - "session" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_agent_cursor_state", - "description": "Return the session cursor's theme, semantic playback, position, visibility, and motion.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "Public label of the session whose cursor to inspect.", - "type": "string" - } - }, - "required": [ - "session" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "set_agent_cursor_theme", - "description": "Select an already-installed cursor theme for a session.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "reduced_motion": { - "default": "auto", - "description": "Theme animation policy: `on` uses the theme's reduced-motion frames, `off` always\nanimates, `auto` (default) leaves the choice to the host.", - "enum": [ - "auto", - "on", - "off" - ], - "type": "string" - }, - "session": { - "description": "Public label of the session that owns the cursor.", - "type": "string" - }, - "theme_id": { - "description": "Id of an installed cursor theme (see `cua-driver cursor-theme list`).", - "maxLength": 200, - "minLength": 1, - "type": "string" - } - }, - "required": [ - "session", - "theme_id" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "check_permissions", - "description": "Check required permissions for cua-driver-rs on Linux. How to get them granted: `get_guide({topic:\"permissions\"})`.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "prompt": { - "description": "Ignored: Linux has no permission prompts.", - "type": "boolean" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "health_report", - "description": "Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped. For standalone CuaDriver, use `cua-driver permissions grant` to verify it explicitly. For an embedded or linked driver, use the host's permission flow for the app or executable macOS attributes this process's permission request to; the standalone command does not verify the host's capture readiness. cua-driver owns the health model; consumers stay thin.\n\nInput — all optional:\n {\n \"include\": [\"\", ...], // run only these\n \"skip\": [\"\", ...] // skip these\n }\nIf both are given, `include` wins.\n\nCanonical check names:\n macOS : binary_version, platform_supported, session_active,\n bundle_identity, tcc_accessibility, tcc_screen_recording,\n ax_capability, screen_capture_capability, windowserver_connection\n Windows: binary_version, platform_supported, session_active,\n ax_capability (via UIA), screen_capture_capability (via DXGI)\n Linux : binary_version, platform_supported, session_active,\n ax_capability (via AT-SPI), screen_capture_capability (via X11)\n\nOutput — stable contract, schema_version=\"1\":\n {\n \"schema_version\": \"1\",\n \"platform\": \"darwin\" | \"win32\" | \"linux\",\n \"driver_version\": \"\",\n \"overall\": \"ok\" | \"degraded\" | \"failed\",\n \"checks\": [\n {\n \"name\": \"\",\n \"status\": \"pass\" | \"fail\" | \"skip\",\n \"message\": \"\",\n \"hint\": \"\",\n \"data\": { /* check-specific structured fields */ }\n },\n ...\n ]\n }\n\n`overall` rules:\n - `ok` — every non-skipped check passes\n - `degraded` — at least one non-core check fails (binary is still usable)\n - `failed` — any core check fails (binary_version, platform_supported, session_active)\n\nStability: schema_version=\"1\" is the contract. Future breaking changes will be `\"2\"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "include": { - "description": "Only run these checks (canonical names). Wins over `skip`.", - "items": { - "type": "string" - }, - "type": "array" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "skip": { - "description": "Skip these checks (canonical names). Ignored when `include` is set.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_config", - "description": "Return current cua-driver-rs configuration.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "set_config", - "description": "Update cua-driver-rs configuration. capture_mode / max_image_dimension take effect immediately.\n\nPass each setting as its own field: `{\"capture_mode\": \"som\", \"max_image_dimension\": 0}`.\n\nThe experimental_pip keys persist to ~/.cua-driver/config.json and apply on next daemon restart (the PiP backend is initialised once at startup; Linux ships only the trait stub today — see issue #1729).\n\n`cursor.motion.style`, `cursor.motion.timing` and `cursor.motion.effects.` (`on`, `off`, or `default`) save the default cursor motion for sessions started afterwards. `cursor.motion: \"default\"` clears every saved cursor motion default.\n\n`cursor.badges` (`minimal` default, `verbose`, or `off`) sets the agent cursor badge for every cursor at once.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "capture_mode": { - "description": "Default capture mode for get_window_state. (\"som\"/\"screenshot\" still decode as deprecated aliases.)", - "enum": [ - "ax", - "vision" - ], - "type": "string" - }, - "cursor.badges": { - "description": "Agent cursor badge. minimal (default): no badge at rest; a brief chip for a foreground moment or a key combo, and the session name on hover or when two or more agent cursors are on screen. verbose: the earlier badge with the session name and delivery/target chips on every action. off: no badge. Applies immediately to every cursor.", - "enum": [ - "minimal", - "verbose", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion": { - "description": "Pass `default` to clear every saved cursor motion default.", - "enum": [ - "default" - ], - "type": "string" - }, - "cursor.motion.effects.glow": { - "description": "Saved default for the `glow` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion.effects.magnet": { - "description": "Saved default for the `magnet` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion.effects.ripple": { - "description": "Saved default for the `ripple` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion.effects.squish": { - "description": "Saved default for the `squish` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion.effects.trail": { - "description": "Saved default for the `trail` cursor effect: on, off, or default (the style's own). Omit the key to leave it unchanged.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "cursor.motion.style": { - "description": "Saved default cursor motion style for new sessions. Overridden by start_session cursor_motion and set_agent_cursor_motion; reduced motion always wins. Built-in default: signature_arc.", - "enum": [ - "signature_arc", - "spring_settle", - "magnetic", - "comet_swoop", - "adaptive", - "classic" - ], - "type": "string" - }, - "cursor.motion.timing": { - "description": "Saved default cursor move timing for new sessions. Built-in default: steady (a 600 ms floor on every move).", - "enum": [ - "native", - "fitts", - "fixed", - "relaxed", - "steady", - "calm", - "deliberate" - ], - "type": "string" - }, - "experimental_pip": { - "description": "Enable the experimental PiP preview window (applies next restart; Linux backend stubbed).", - "type": "boolean" - }, - "experimental_pip_geometry": { - "description": "PiP window size + optional position in `WxH` or `WxH+X+Y` form.", - "type": "string" - }, - "max_image_dimension": { - "description": "Max dimension for screenshot resizing (0 = no limit).", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_accessibility_tree", - "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "zoom", - "description": "Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixels, with 20% padding. Output is at most 500 px wide. Corners, not a size: x1,y1 is the top-left and x2,y2 the bottom-right (x,y,width,height is also accepted and converted).\n\nAfter a zoom, pass from_zoom=true to click/type_text to auto-translate coordinates back to full-window space. Coordinate actions return `screenshot_context_missing` when no current snapshot contains a screenshot owned by this session. `from_zoom` actions return `zoom_context_missing` when the zoom was never created or was replaced; call `get_window_state`, then `zoom`, again on the same connection.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "pid": { - "description": "Optional target pid. When omitted, the driver resolves the unique current snapshot for this session and window.", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "window_id": { - "description": "Window id of the window captured by get_window_state.", - "type": "integer" - }, - "x1": { - "description": "Left edge of the region in window-local screenshot pixels.", - "type": "number" - }, - "x2": { - "description": "Right edge of the region in window-local screenshot pixels.", - "type": "number" - }, - "y1": { - "description": "Top edge of the region in window-local screenshot pixels.", - "type": "number" - }, - "y2": { - "description": "Bottom edge of the region in window-local screenshot pixels.", - "type": "number" - } - }, - "required": [ - "window_id", - "x1", - "y1", - "x2", - "y2" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "page", - "description": "Legacy browser compatibility tool. Prefer get_browser_state and the typed browser_* tools for exact targeting, endpoint ownership, and consent. Read-only get_text and query_dom remain available by default. Mutating actions require the daemon operator to set CUA_DRIVER_ENABLE_LEGACY_PAGE_MUTATIONS=1 before daemon startup (restart the daemon after changing it); this escape hatch does not provide the typed browser surface's exact binding or existing-profile grant guarantees. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page.\n- query_dom: Find elements matching a CSS selector.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- insert_text: Insert `text` at whatever currently holds DOM focus in one native operation (CDP Input.insertText) — no synthesized key events, but more durable than a one-shot execute_javascript write since rich-text editors already have to treat it like an IME commit. Try this before type_keystrokes on a contenteditable that discarded an execute_javascript write. Click/focus the target field first.\n- type_keystrokes: Type `text` via real per-character keystroke events into whatever currently holds DOM focus. Slower than insert_text but the most durable rung — use it when insert_text also gets discarded, or the editor's own keydown/keyup handlers need to see real keys. Click/focus the target field first.\n- enable_javascript_apple_events: macOS-only — patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", - "inputSchema": { - "additionalProperties": false, - "properties": { - "action": { - "description": "Action to perform.", - "enum": [ - "execute_javascript", - "get_text", - "query_dom", - "click_element", - "insert_text", - "type_keystrokes", - "enable_javascript_apple_events" - ], - "type": "string" - }, - "attributes": { - "description": "Element attributes to include in query_dom results.", - "items": { - "type": "string" - }, - "type": "array" - }, - "bundle_id": { - "description": "Bundle ID of the browser. Required for enable_javascript_apple_events (macOS only).", - "type": "string" - }, - "cdp_port": { - "description": "Optional, for execute_javascript/insert_text/type_keystrokes: use this exact CDP port instead of auto-discovering one from pid. Needed when the port was opened via the browser's own remote-debugging toggle rather than a launch-time flag, since that path may not answer the auto-discovery probe.", - "maximum": 65535, - "minimum": 1, - "type": "integer" - }, - "css_selector": { - "description": "CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*').", - "type": "string" - }, - "javascript": { - "description": "JavaScript to execute. Required for execute_javascript.", - "type": "string" - }, - "pid": { - "description": "Target process ID.", - "type": "integer" - }, - "selector": { - "description": "CSS selector for click_element (e.g. 'button.submit', '#login a').", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "target_url_contains": { - "description": "Optional, for execute_javascript/insert_text/type_keystrokes: require exactly one browser tab whose URL contains this substring. Use this on a multi-tab browser — there's no built-in link between window_id and which tab a CDP call reaches.", - "type": "string" - }, - "text": { - "description": "Text to insert or type. Required for insert_text and type_keystrokes. The target field must already have DOM focus (click/focus it first).", - "type": "string" - }, - "user_has_confirmed_enabling": { - "description": "Must be true to proceed with enable_javascript_apple_events. This will quit and relaunch the browser.", - "type": "boolean" - }, - "window_id": { - "description": "Target window ID from list_windows.", - "type": "integer" - } - }, - "required": [ - "action" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "get_browser_state", - "description": "Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native browser window to classify it, correlate it to a CDP target (exact-or-refuse), and mint a session-scoped target id plus tab ids. Mode 2 (snapshot): pass target_id + tab_id. The dom_refs_v1 compatibility format returns composed DOM refs. semantic_v2 joins accessibility, DOM, layout, and viewport state; ranks visible content before retained/offscreen state; and returns a semantic outline, typed action refs, content refs, scoped reads, and opaque continuation. Never performs setup: a browser that is not attached yet refuses with browser_consent_required or browser_requires_setup, and detail.next_call names the browser_prepare call that attaches it. Use this route for any web page in Chrome, Edge or Brave; desktop pointer tools move the user's pointer. Setup and recovery: `get_guide({topic:\"browser\"})`.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "continuation": { - "description": "Opaque continuation minted by an earlier semantic_v2 response.", - "type": "string" - }, - "include_screenshot": { - "default": false, - "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.", - "type": "boolean" - }, - "pid": { - "description": "Native browser process id (bind mode).", - "type": "integer" - }, - "query": { - "description": "Read-only semantic match over role, accessible name, and visible text.", - "type": "string" - }, - "scope_ref": { - "description": "Current semantic/content ref whose subtree should be observed.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "snapshot_format": { - "description": "Versioned snapshot contract. dom_refs_v1 remains the compatibility default.", - "enum": [ - "dom_refs_v1", - "semantic_v2" - ], - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id from get_browser_state (session-scoped).", - "type": "string" - }, - "target_id": { - "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", - "type": "string" - }, - "window_id": { - "description": "Native window id owned by pid (bind mode).", - "type": "integer" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "browser_prepare", - "description": "Attach the browser tools to a browser. For a page in the user's running Chrome, Edge or Brave, call it once with pid, window_id, session and strategy {\"kind\":\"existing_profile\"} when get_browser_state refuses with browser_consent_required or browser_requires_setup; then call get_browser_state again and act with browser_click, browser_type or browser_pointer. It attaches, or refuses with nothing changed and says why. Explicitly prepare an owned DevTools endpoint for a browser. pid is required for an existing process or existing-profile attachment, and optional only for allow_launch=true with an isolated profile. Existing endpoints are detected without side effects. Acting setup for an isolated profile follows the runtime permission mode and optional capability manifest. It requires allow_launch=true, launches a separate browser, and never copies, modifies, or terminates the requested user profile. Without pid, only a platform-attested system Chrome/Edge installation (or a root-owned package payload on Linux) is eligible; redirects and user-controlled locations fail closed. Existing-profile attachment is explicit and follows the runtime's immutable permission mode: standard requires an explicit --grant existing-profile launch grant or an embedding authorization host, bounded requires a launch-approved exact resource manifest, and unrestricted requires explicit trusted startup risk acceptance. Ordinary MCP transport approval never proves profile authorization. On proven platforms, an authorized request also permits one bounded exact-window setup: open the recognized browser product's fixed remote-debugging page, toggle its uniquely matched per-instance checkbox, prove the PID-owned loopback endpoint, and close the temporary tab. Every visible effect is reported; ambiguity is refused. Where attaching would bring the browser in front of the user's app (macOS Chrome shows its own remote-debugging prompt in front whenever the driver opens a new browser connection; sessions for the same browser share one approved connection, kept for up to 30 minutes idle and 8 hours in total, so later sessions do not prompt again), existing-profile attachment refuses with background_unavailable unless delivery_mode is \"foreground\" (the app that was in front is brought back afterwards) or the browser window is on another Space than the user's, where it attaches in the background while the driver reverts the browser's own brief activations. A driver-owned isolated launch result carries an advisory lifecycle object: the browser process ends with the calling session after idle_expires_in_seconds without session activity, an isolated_named profile persists after the session and an isolated_new profile does not, and explicit end, disconnect, revocation or shutdown end it earlier. Setup and recovery: `get_guide({topic:\"browser\"})`.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "allow_launch": { - "description": "Allow a separate driver-owned isolated Chromium process to be launched (default false).", - "type": "boolean" - }, - "delivery_mode": { - "default": "background", - "description": "Existing-profile attachment only. background (default) attaches in the background when the browser window is on another Space than the user's, and otherwise refuses with background_unavailable, before any visible effect, where attaching would bring the browser in front of the user's app. foreground accepts that, for a screen nobody else is using or when the user agreed; the app that was in front is brought back afterwards.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "pid": { - "description": "Browser process id to prepare. Required except for a driver-owned isolated_new/isolated_named launch with allow_launch=true.", - "type": "integer" - }, - "profile": { - "additionalProperties": false, - "description": "Driver-owned isolated Chromium profile to launch with allow_launch=true. mode=isolated_new creates a fresh throwaway profile; mode=isolated_named reuses the named driver-owned profile. Never an existing user profile.", - "properties": { - "mode": { - "enum": [ - "isolated_new", - "isolated_named" - ], - "type": "string" - }, - "name": { - "description": "Required only for isolated_named; 1-64 path-safe ASCII characters.", - "type": "string" - } - }, - "required": [ - "mode" - ], - "type": "object" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "strategy": { - "additionalProperties": false, - "description": "Attach to an already-running browser instead of launching one. Pass an object, {\"kind\":\"existing_profile\"}, not a string. kind=existing_profile attaches to the user's running profile at pid/window_id under the runtime's profile authorization.", - "properties": { - "kind": { - "enum": [ - "existing_profile" - ], - "type": "string" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "window_id": { - "description": "Exact native window approval anchor; required for strategy.kind=existing_profile.", - "type": "integer" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_navigate", - "description": "Navigate one tab of an exactly-bound browser target to a new URL (http/https/about only). Refused for heuristic bindings. Navigation invalidates all p: refs for the tab.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id from get_browser_state (session-scoped).", - "type": "string" - }, - "target_id": { - "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", - "type": "string" - }, - "url": { - "description": "Destination URL (http:, https:, or about:).", - "type": "string" - } - }, - "required": [ - "target_id", - "tab_id", - "url" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_click", - "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "delivery_mode": { - "default": "background", - "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "input_route": { - "description": "\"trusted\" (default): Input.dispatchMouseEvent. It refuses rather than foregrounding a standalone browser. \"dom_event\": synthetic full-background DOM click, only when explicitly requested. Dispatch does not prove the control activated; refresh page state and verify the expected postcondition.", - "enum": [ - "trusted", - "dom_event" - ], - "type": "string" - }, - "ref": { - "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id from get_browser_state (session-scoped).", - "type": "string" - }, - "target_id": { - "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", - "type": "string" - }, - "x": { - "description": "Viewport x (CSS px) — alternative to ref.", - "type": "number" - }, - "y": { - "description": "Viewport y (CSS px) — alternative to ref.", - "type": "number" - } - }, - "required": [ - "target_id", - "tab_id" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_type", - "description": "Type text into an exactly-bound tab via the Input domain. mode=\"insert_text\" (default) uses Input.insertText; mode=\"keystrokes\" dispatches per-character key events. Both insert at the caret, so typing into a field that already holds text appends to it; pass replace=true to set the field instead, or to clear it by typing an empty string. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "mode": { - "description": "insert_text (default): bulk Input.insertText. keystrokes: per-character Input.dispatchKeyEvent.", - "enum": [ - "insert_text", - "keystrokes" - ], - "type": "string" - }, - "ref": { - "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", - "type": "string" - }, - "replace": { - "description": "false (default): insert at the caret, appending to whatever the field already holds. true: select the element's whole content first so the text replaces it — with an empty text this clears the field. Replacement goes through the selection, so beforeinput/input still fire and framework state stays consistent.", - "type": "boolean" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id from get_browser_state (session-scoped).", - "type": "string" - }, - "target_id": { - "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", - "type": "string" - }, - "text": { - "description": "Text to type.", - "type": "string" - } - }, - "required": [ - "target_id", - "tab_id", - "ref", - "text" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_select", - "description": "Select one option of a native HTML ref through CDP. This bypasses native file pickers, rejects symlinks and non-regular files, and never returns local paths.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "files": { - "description": "Absolute paths of 1 to 32 local regular files to assign to the input.", - "items": { - "description": "Absolute path to one local regular file.", - "type": "string" - }, - "maxItems": 32, - "minItems": 1, - "type": "array" - }, - "ref": { - "description": "Page element ref in the p: namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.", - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id from get_browser_state (session-scoped).", - "type": "string" - }, - "target_id": { - "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).", - "type": "string" - } - }, - "required": [ - "target_id", - "tab_id", - "ref", - "files" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_download", - "description": "Trigger one download through an exact live browser ref and save it inside an explicitly approved directory. Requires MCP-host destructive-tool approval, refuses ambiguous or stale capabilities, and never returns the source URL, filename, or destination path.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "destination_root": { - "description": "Absolute, existing, canonical directory approved to receive the download.", - "type": "string" - }, - "ref": { - "description": "Live page ref whose activation initiates the download.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", - "type": "string" - }, - "tab_id": { - "description": "Opaque exact tab id from get_browser_state.", - "type": "string" - }, - "target_id": { - "description": "Opaque exact browser target id from get_browser_state.", - "type": "string" - } - }, - "required": [ - "session", - "target_id", - "tab_id", - "ref", - "destination_root" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "browser_pointer", - "description": "Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. Semantic refs must declare pointer for hover, right-click, double-click, and drag; scroll accepts a scroll or pointer capability. The trusted route uses CDP Input events and refuses if standalone background posture cannot be preserved. The explicit dom_event route requires a page ref and synthesizes full-background DOM events. Never activates or brings a tab to the foreground.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "action": { - "description": "Pointer gesture. scroll needs delta_x or delta_y; drag needs destination_ref or to_x/to_y.", - "enum": [ - "hover", - "right_click", - "double_click", - "scroll", - "drag" - ], - "type": "string" - }, - "delivery_mode": { - "default": "background", - "description": "background (default) refuses trusted input where it would activate the browser window, including Linux Chromium and macOS Chromium configurations whose trusted route cannot stay in the background. foreground accepts that activation, for a browser whose window nobody else is using (for example inside a sandbox).", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "delta_x": { - "description": "Horizontal scroll delta in CSS pixels.", - "type": "number" - }, - "delta_y": { - "description": "Vertical scroll delta in CSS pixels.", - "type": "number" - }, - "destination_ref": { - "description": "Drag destination page ref in the exact same frame.", - "type": "string" - }, - "input_route": { - "default": "trusted", - "description": "trusted sends CDP Input events; dom_event synthesizes DOM events in the page and requires ref.", - "enum": [ - "trusted", - "dom_event" - ], - "type": "string" - }, - "ref": { - "description": "Origin page ref. Alternative to x/y.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs.", - "type": "string" - }, - "tab_id": { - "description": "Opaque tab id minted by get_browser_state.", - "type": "string" - }, - "target_id": { - "description": "Opaque target id minted by get_browser_state.", - "type": "string" - }, - "to_x": { - "description": "Drag destination viewport x in CSS pixels.", - "type": "number" - }, - "to_y": { - "description": "Drag destination viewport y in CSS pixels.", - "type": "number" - }, - "x": { - "description": "Origin viewport x in CSS pixels.", - "type": "number" - }, - "y": { - "description": "Origin viewport y in CSS pixels.", - "type": "number" - } - }, - "required": [ - "target_id", - "tab_id", - "session", - "action" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "start_recording", - "description": "Start trajectory recording for the calling session. Each action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) from that session writes a turn folder under `output_dir`. Without `session`, every call on the same connection is recorded, including named-session calls. Other connections and session lifecycle calls (`start_session` / `end_session`) are not recorded. CLI recordings started with `cua-driver recording start` are daemon-wide.\n\nEach turn folder holds:\n\n- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI state immediately before and after the action.\n- `before.png` / `after.png` — target-window screenshots immediately before and after the action.\n- `evidence.json` — capture status and a stable classification when an expected artifact could not be captured.\n- `app_state.json` — post-action AX/UIA snapshot for the target pid.\n- `screenshot.png` — compatibility alias of `after.png`.\n- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.\n- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.\n\nThe per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.\n\nTurn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.\n\n**Video is off by default.** Pass `record_video: true` to also capture the main display to `/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.\n\n**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) under the daemon's Screen Recording grant, with no ffmpeg subprocess. Requires macOS 15.0+. On macOS 26 (Tahoe), the first direct capture can also show a one-time consent asking to let Cua Driver bypass the system private window picker and directly access your screen and audio; choose Allow, or run `cua-driver permissions grant` beforehand to answer it up front. The recorder captures screen video only and does not enable system-audio capture.\n\n**Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.\n\nState persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "include_accessibility_tree": { - "default": true, - "description": "Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`.", - "type": "boolean" - }, - "output_dir": { - "description": "Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.", - "type": "string" - }, - "record_video": { - "description": "Capture the main display to /recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.", - "type": "boolean" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "state_timeout_ms": { - "default": 1000, - "description": "Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json.", - "maximum": 120000, - "minimum": 100, - "type": "integer" - } - }, - "required": [ - "output_dir" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "stop_recording", - "description": "Stop trajectory recording. Disables further per-turn capture and, when video was enabled, gracefully terminates the ffmpeg subprocess so the mp4's moov atom is finalized (the file is playable). Calling stop on an already-stopped session is a no-op. The response carries `last_video_path` pointing at the finalized mp4 (when video was on).\n\nA manual `stop_recording` is **unconditional** — it stops whatever recording is active regardless of which session started it. Ownership-scoped teardown (so one client disconnecting can't stop a recording a later client started) is handled by the registry's `session_end` lifecycle hook, not by this tool.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_recording_state", - "description": "Report the current trajectory recorder state: whether recording is enabled, the output directory (when enabled), and the 1-based counter for the next turn folder that will be written. Counter increments on every recorded action tool call and resets to 1 each time recording is (re-)enabled.\n\nPure read-only.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "replay_trajectory", - "description": "Replay a recorded trajectory by re-invoking every turn's tool call in lexical order. `dir` must point at a directory previously written by `start_recording`. Each `turn-NNNNN/` is parsed for `action.json`, and the recorded tool is called with its recorded `arguments` via the same dispatch path an MCP / CLI call uses.\n\nCaveats:\n- Element-token actions (`click({pid, element_token})` etc.) will fail because element tokens are per-snapshot and don't survive across sessions. Pixel clicks (`click({pid, x, y})`) and all keyboard tools replay cleanly. Failures are reported but don't stop replay unless `stop_on_error` is true.\n- `get_window_state` and other read-only tools are NOT currently recorded, so replays do not re-populate the per-(pid, window_id) element cache.\n- If recording is ENABLED while replay runs, the replay itself is recorded into the currently configured output directory. That's deliberate: recording a replay against a new build and diffing the two trajectories is the regression-test workflow.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "delay_ms": { - "description": "Milliseconds to sleep between turns, for human-observable pacing. Default 500.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "dir": { - "description": "Trajectory directory previously written by `start_recording`. Absolute or ~-rooted.", - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "stop_on_error": { - "description": "Stop replay on the first tool-call error. Default true — set false to best-effort through the full trajectory.", - "type": "boolean" - } - }, - "required": [ - "dir" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "install_ffmpeg", - "description": "Install the ffmpeg binary used by start_recording's video capture (Linux/Windows; macOS records natively and needs no ffmpeg). Two-step and confirmed: called without `confirm` it only REPORTS the exact install command for this platform's package manager; pass `confirm: true` to actually run it. No-op if ffmpeg is already on PATH. ffmpeg is run as a separate process, never linked into the driver.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "confirm": { - "description": "Run the install command. Without it, only the planned command is reported.", - "type": "boolean" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "start_session", - "description": "Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "capture_scope": { - "description": "Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.", - "enum": [ - "auto", - "window", - "desktop" - ], - "type": "string" - }, - "cursor_motion": { - "additionalProperties": false, - "description": "Optional initial cursor motion (style, timing, effects and tuning). The host applies it\nbefore the cursor is first made visible. A later `set_agent_cursor_motion` call wins;\nthis wins over the saved default (`cursor.motion.*` in the driver config) and the\nbuilt-in `signature_arc`. Reduced motion always wins.", - "properties": { - "arc_flow": { - "description": "Added to the arc asymmetry of `signature_arc`, `spring_settle` and `comet_swoop`:\npositive moves the apex toward the destination. Clamped to -1..1 (default 0).", - "type": "number" - }, - "arc_size": { - "description": "Scales the arc of `signature_arc`, `spring_settle` and `comet_swoop`: 0.25 (default)\nkeeps the style's arc, 0 is a straight line, 0.5 doubles it. Clamped to 0..1.", - "type": "number" - }, - "dwell_after_click_ms": { - "description": "Pause after a click animation, in milliseconds. Clamped to 0..5000 (default 80).", - "type": "number" - }, - "effects": { - "additionalProperties": false, - "description": "Turn effects on or off. Omit a field to keep it; `\"default\"` restores the style default.", - "properties": { - "glow": { - "description": "Soft glow around the cursor that grows with speed.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "magnet": { - "description": "Target glow when the `magnetic` style locks on.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "ripple": { - "description": "Ring that expands from the hotspot on click.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "squish": { - "description": "Brief scale-down of the cursor on click.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - }, - "trail": { - "description": "Short fading trail behind the cursor.", - "enum": [ - "on", - "off", - "default" - ], - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "end_handle": { - "description": "Arc control-point offset from the end, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", - "type": "number" - }, - "glide_duration_ms": { - "description": "Move duration in milliseconds for `fixed` timing (1430 ms when 0, the default). A\nnonzero value with `native` or the default `steady` timing also fixes the duration.\nClamped to 0..5000.", - "type": "number" - }, - "idle_hide_ms": { - "description": "Hide the cursor after this many idle milliseconds; 0 never hides it. Clamped to\n0..60000 (default 15000).", - "type": "number" - }, - "spring": { - "description": "Arrival spring damping for `classic`: 1 is critically damped, 0.3 is bouncy. Clamped\nto 0.3..1 (default 0.72).", - "type": "number" - }, - "start_handle": { - "description": "Arc control-point offset from the start, as a fraction of the distance, for\n`signature_arc`, `spring_settle` and `comet_swoop`. Clamped to 0..1 (default 0.3).", - "type": "number" - }, - "style": { - "description": "Trajectory style. `signature_arc` (default) is one arc with a small follow-through;\n`spring_settle` lands with one soft bounce; `magnetic` is pulled into the target;\n`comet_swoop` is a wide arc with a short trail; `adaptive` picks a careful approach for\nsmall targets and a swoop for long moves; `classic` is the previous Dubins glide. When\nthe theme's reduced motion is on, every move is a short straight glide with no effects.", - "enum": [ - "signature_arc", - "spring_settle", - "magnetic", - "comet_swoop", - "adaptive", - "classic" - ], - "type": "string" - }, - "timing": { - "description": "`steady` (default) keeps the style's timing but makes every move take at least 600 ms,\nwith a gentler ease; `relaxed` and `calm` do the same with a 450 or 750 ms floor.\n`native` uses the style's own timing; `fitts` scales the move time with distance and\ntarget size; `deliberate` is a slower Fitts curve (450..1300 ms); `fixed` uses\nglide_duration_ms (1430 ms when 0).", - "enum": [ - "native", - "fitts", - "fixed", - "relaxed", - "steady", - "calm", - "deliberate" - ], - "type": "string" - }, - "turn_radius": { - "description": "Minimum turning radius of the `classic` glide path, in points; smaller turns tighter.\nClamped to 1..1000 (default 80).", - "type": "number" - } - }, - "required": [], - "type": "object" - }, - "cursor_theme": { - "additionalProperties": true, - "description": "Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.", - "properties": { - "reduced_motion": { - "default": "auto", - "enum": [ - "auto", - "on", - "off" - ], - "type": "string" - }, - "theme_id": { - "type": "string" - } - }, - "required": [ - "theme_id" - ], - "type": "object" - }, - "session": { - "description": "Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "escalate_session", - "description": "Deprecated compatibility tool for legacy capture-scope sessions. New callers select window or desktop modality on each action. No deescalate_session tool exists.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "detail": { - "description": "Optional bounded diagnostic detail. Never use secrets or page content.", - "maxLength": 200, - "type": "string" - }, - "reason": { - "description": "Why the window-scoped attempt failed and desktop capture is needed.", - "enum": [ - "ax_tree_pixel_mismatch", - "background_delivery_failed", - "foreground_ineffective", - "no_window_target", - "other" - ], - "type": "string" - }, - "session": { - "description": "Public label of the legacy capture-scope session to escalate.", - "type": "string" - } - }, - "required": [ - "session", - "reason" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": false, - "idempotentHint": false, - "openWorldHint": false - } - }, - { - "name": "get_session", - "description": "Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "session": { - "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "list_sessions", - "description": "List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "cursor": { - "description": "Opaque continuation cursor returned by a previous call.", - "type": "string" - }, - "limit": { - "description": "Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.", - "minimum": 0, - "type": "integer" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_session_state", - "description": "Deprecated compatibility alias that reads a live legacy session's capture policy. Use get_session for lifecycle state.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "session": { - "description": "Optional public label. When omitted, inspect the caller's attached\nimplicit session.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "end_session", - "description": "End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.", - "inputSchema": { - "additionalProperties": true, - "properties": { - "session": { - "description": "Optional public label to end. When omitted, end the caller's attached\nimplicit session.", - "type": "string" - } - }, - "required": [], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "get_guide", - "description": "Read Cua Driver guidance on demand, taken from the guide bundled with this driver (no skill install needed). Topics: overview, drag, browser, permissions, batching, menus. Call with no topic to list them. Read-only.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "topic": { - "description": "One of: overview, drag, browser, permissions, batching, menus. Omit to list the topics.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - }, - { - "name": "run_steps", - "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "app": { - "description": "Default app (name or bundle id) for steps that name no window.", - "type": "string" - }, - "delay_ms": { - "description": "Pause between steps in milliseconds (not after the last). Default 0.", - "maximum": 2000, - "minimum": 0, - "type": "integer" - }, - "delivery_mode": { - "description": "Default delivery_mode for steps whose tool takes one and that set none.", - "enum": [ - "background", - "foreground" - ], - "type": "string" - }, - "foreground_fallback": { - "description": "When a step is refused because its target ignores background input (Electron, LibreOffice and other toolkits), retry it once with delivery_mode:\"foreground\": the window is activated and the pointer may move. Default false.", - "type": "boolean" - }, - "observe": { - "description": "Optional end-of-batch observation: arguments for ONE get_window_state call. `pid` and `window_id` default to those of the last step that names both. Defaults to since=\"latest\" (only what changed since your last read of that window with the same query/max_elements/max_depth; a full read when there is none), include_screenshot=false and max_elements=250; pass include_screenshot=true to see the window, or since=null for a full read. Omit to read nothing. {} observes the last window.", - "type": "object" - }, - "pid": { - "description": "Default pid for steps that name no window (a step's own pid/window_id/app/window wins).", - "type": "integer" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "steps": { - "description": "Ordered actions. Execution stops at the first failing step. Each step is {tool, args} or {: args}, plus optional wait_for / expect / timeout_ms; a step may also be only a wait_for or expect check.", - "items": { - "additionalProperties": true, - "properties": { - "args": { - "description": "Arguments for that tool, as in a direct call, optionally with role/name/nth/app/window to name the target instead of pid/window_id/element_token.", - "type": "object" - }, - "expect": { - "description": "Up to 4 checks that must hold after the action: {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}. Default timeout 2000 ms.", - "items": { - "type": "object" - }, - "maxItems": 4, - "type": "array" - }, - "timeout_ms": { - "description": "How long a named target may take to appear. Default 3000.", - "maximum": 10000, - "minimum": 0, - "type": "integer" - }, - "tool": { - "description": "Action tool to run.", - "enum": [ - "click", - "double_click", - "right_click", - "set_value", - "type_text", - "press_key", - "hotkey", - "scroll", - "drag", - "move_cursor" - ], - "type": "string" - }, - "wait_for": { - "description": "Check that must hold before the action: {role?, name?, text?, gone?, app?, window?, timeout_ms?}. Default timeout 5000 ms.", - "type": "object" - } - }, - "type": "object" - }, - "maxItems": 64, - "minItems": 1, - "type": "array" - }, - "window": { - "description": "Default window title substring for steps that name no window.", - "type": "string" - }, - "window_id": { - "description": "Default window_id for steps that name no window, with `pid`.", - "type": "integer" - } - }, - "required": [ - "steps" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "run_script", - "description": "Run a JavaScript program that drives apps through Cua Driver, in ONE call: loops, conditions, reads and checks between actions, and only the result comes back. Use it when the flow needs logic (a loop over rows, a branch on what the app shows, a retry); use run_steps for a fixed list of steps.\n\nThe script is the body of an async function: use `await`, and `return` a JSON value. It runs in a sandbox with no filesystem, network, process, timers or environment; the only API is `cua`:\n- `await cua.getApp(\"TextEdit\" | \"com.apple.TextEdit\" | {pid, windowId})` → app (follows the app's frontmost window unless a windowId is given)\n- `await cua.launch(nameOrBundleId)` → app; `cua.listApps()`, `cua.listWindows({pid?})`, `cua.sleep(ms)`, `cua.computer.target`\n- app.getState(opts?) → get_window_state result (`tree_markdown` or `tree_diff`, `snapshot_id`; pass get_window_state options such as {since:\"latest\"})\n- app.query({role?, name?, text?}) → matching elements across the app's windows {element_token, role, label, value, enabled, selected, frame}, then matching static-text rows {role, label, value, text, display_only} (read text such as a status line from their `value`)\n- app.waitFor({role?, name?, text?, gone?, value?, timeout_ms?}) and app.verify(predicate | predicates) (verify_state)\n- app.click(t), doubleClick(t), rightClick(t), typeText(text, t?), pressKey(\"return\" | \"cmd+s\", t?), hotkey([\"cmd\",\"s\"]), scroll(t, direction, amount?), setValue(t, value), drag([x,y], [x,y]), zoom([x1,y1,x2,y2])\nwhere a target `t` is {role, name, nth?} (found fresh, waits up to 3 s), an element_token string, a row number from this app's last getState(), or [x, y] window pixels.\n- `cua.call(tool, args)` for any other allowed tool: click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, launch_app, list_apps, list_windows, get_window_state, verify_state, zoom.\nA failed driver call throws a CuaError (catch it to recover). Every call goes through the same session, permission and approval checks as a direct call.\n\nLimits: `timeout_ms` wall time (default 30000, max 120000), `max_calls` driver calls (default 100, max 500), 32 MiB heap. Returns the return value, console output, and a per-call log; an error names the script line, the call and the reason.\n\nExample:\nconst app = await cua.getApp(\"Calculator\");\nfor (const key of [\"7\", \"multiply\", \"6\", \"equals\"]) await app.click({role: \"button\", name: key});\nconst [display] = await app.query({role: \"statictext\"});\nreturn display.value; Batching rules: `get_guide({topic:\"batching\"})`.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "max_calls": { - "description": "Most driver calls the script may make. Default 100.", - "maximum": 500, - "minimum": 1, - "type": "integer" - }, - "script": { - "description": "JavaScript: the body of an async function. `return` the result.", - "maxLength": 65536, - "minLength": 1, - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - }, - "timeout_ms": { - "description": "Wall-time limit. Default 30000.", - "maximum": 120000, - "minimum": 100, - "type": "integer" - } - }, - "required": [ - "script" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "check_for_update", - "description": "Check the saved stable/nightly Cua Driver channel for a release on GitHub. Returns current and selected channels, current and latest versions, an `update_available` boolean, the install one-liner, and the release notes URL. Read-only — never installs. Pacman-owned Linux executables return package-manager guidance without checking GitHub. Mirror of `cua-driver check-update --json`.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": true - } - }, - { - "name": "install_extension", - "description": "Preview or install one Driver-managed optional extension. The first call without confirm returns the exact signed artifact, destination, license, source, and trust plan without mutation. Re-call with confirm=true to perform that exact verified installation.", - "inputSchema": { - "additionalProperties": false, - "properties": { - "confirm": { - "description": "Install the previewed extension. Omit or false for a read-only plan.", - "type": "boolean" - }, - "name": { - "description": "Extension to preview or install. Only perception (the local visual-region parser used by parse_visual_regions) is available.", - "enum": [ - "perception" - ], - "type": "string" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "required": [ - "name" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": false, - "destructiveHint": true, - "idempotentHint": false, - "openWorldHint": true - } - }, - { - "name": "parse_visual_regions", - "description": "Parse one immutable registered capture into model-neutral text and icon regions.", - "inputSchema": { - "additionalProperties": false, - "description": "Parse one immutable screenshot capture. Pass the `capture_id` returned by\n`get_window_state` or `get_desktop_state`; the Driver capture registry\nresolves it to the exact pixels and action-coordinate transform.", - "properties": { - "capture_id": { - "description": "`capture_id` of a screenshot returned by get_window_state or get_desktop_state.", - "minLength": 1, - "type": "string" - }, - "options": { - "additionalProperties": false, - "default": {}, - "description": "Optional, model-neutral controls for one bounded parse.", - "properties": { - "kinds": { - "description": "Region kinds to return (`text`, `icon`). Omit for all kinds.", - "items": { - "enum": [ - "text", - "icon" - ], - "type": "string" - }, - "maxItems": 2, - "minItems": 1, - "type": "array" - }, - "max_regions": { - "description": "Return at most this many regions.", - "minimum": 1, - "type": "integer" - }, - "min_confidence": { - "description": "Drop regions below this confidence (0 to 1).", - "maximum": 1, - "minimum": 0, - "type": "number" - } - }, - "required": [], - "type": "object" - }, - "session": { - "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", - "type": "string" - } - }, - "required": [ - "capture_id" - ], - "type": "object" - }, - "annotations": { - "readOnlyHint": true, - "destructiveHint": false, - "idempotentHint": true, - "openWorldHint": false - } - } -] diff --git a/tst/integration/computer/cua_driver_test.py b/tst/integration/computer/cua_driver_test.py index bd1f5a9dc..a30ded8df 100644 --- a/tst/integration/computer/cua_driver_test.py +++ b/tst/integration/computer/cua_driver_test.py @@ -1,7 +1,8 @@ """The cua-driver driver on the ubuntu-24.04 base, as the cua-driver test bundle's desktop names it: its put composes an image for each arch this machine's Docker builds, gate 3 makes this machine's ready at full with cua-driver's own tools -beside the computer_* tools; deployed, an agent's session drives Thunar by element token through cua-driver's tools, -in a cua-driver session another agent's doesn't see, computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local +beside the computer_* tools; deployed, an agent's session gets cua-driver's instructions at connect and reads its skill +resources, drives Thunar by element token through cua-driver's tools, in a cua-driver session another agent's doesn't +see, computer-ready reports the pinned cua-driver, and the skill leads with cua-driver's own pack. Needs Docker and the local registry on localhost:5000.""" import asyncio @@ -13,6 +14,7 @@ import pytest from mcp import ClientSession from mcp.client.streamable_http import streamablehttp_client +from pydantic import AnyUrl import agent_env from agentenv_protocol import SKILL_EXTENSION_URI @@ -26,14 +28,14 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] -MANIFEST = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "tools.json" +MANIFEST = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "mcp.json" COMPUTER_TOOLS = {"computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", "computer_key", "computer_wait"} def test_gate_3_makes_this_machines_arch_ready_with_cua_drivers_tools_beside_the_computers(cua_desktop): results = cua_desktop.metadata["composed"]["container"] - pinned = {tool["name"] for tool in json.loads(MANIFEST.read_text())} + pinned = {tool["name"] for tool in json.loads(MANIFEST.read_text())["tools"]} assert results[ARCH]["status"] == "ready" and results[ARCH]["emulated"] is False assert results[ARCH]["probe"]["passed"] and results[ARCH]["probe"]["level"] == "full" @@ -53,7 +55,8 @@ async def test_an_agent_drives_thunar_by_element_token_in_its_own_session_and_it try: async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): async with ClientSession(read, write) as session: - await session.initialize() + connected = await session.initialize() + linux = await session.read_resource(AnyUrl("skill://cua-driver/LINUX.md")) started = await session.call_tool("start_session", {"session": "agent"}) mine = await session.call_tool("list_sessions", {}) launched = await session.call_tool("launch_app", {"name": "thunar"}) @@ -75,6 +78,7 @@ async def test_an_agent_drives_thunar_by_element_token_in_its_own_session_and_it assert not launched.isError and not clicked.isError and parent["window_id"] == window["window_id"] assert not started.isError and "agent" in _named(mine) and "agent" not in _named(others) + assert connected.instructions == json.loads(MANIFEST.read_text())["instructions"] and "AT-SPI" in linux.contents[0].text checks = {check["name"]: check for check in ready["checks"]} assert ready["ready"] and checks["cua-driver"]["ok"] and checks["cua-driver tools"]["detail"] == "66 tools, as pinned" assert {"LINUX.md", "WORKFLOW.md", "RUNTIME.md"} <= set(skill["files"]) diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index 39eb035d6..113f57c8b 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -1,6 +1,6 @@ """The cua-driver driver against a fake cua-driver daemon on a Unix socket: the computer/v1 operations become its -desktop-scope calls in the framework's own session, its own tools are listed as pinned and forwarded in their caller's -own session as they came, readiness checks the pinned executable and tools, and the skill leads with cua-driver's pack.""" +desktop-scope calls in the framework's own session; its instructions, tools and skill resources are served as its own +MCP server serves them, its tools forwarded in their caller's own session as they came, readiness checks the pinned executable and tools, and the skill leads with cua-driver's pack.""" from __future__ import annotations import asyncio @@ -79,9 +79,12 @@ def reply(self, request: dict) -> dict: return {"ok": True, "result": {"tools": self.tools}} name = request["name"] if name in self.fail: - return {"ok": False, "error": f"unknown tool {name}", "exit_code": 1} + return {"ok": False, "error": f"unknown tool {name}", "exit_code": 64} if name in self.refuse: return {"ok": True, "result": {"isError": True, "content": [{"type": "text", "text": f"{name} refused"}]}} + if name == "list_windows": + return {"ok": True, "result": {"content": [{"type": "text", "text": "Found 0 windows"}], + "structuredContent": {"windows": []}}} if name == "get_screen_size": return {"ok": True, "result": {"content": [{"type": "text", "text": "1280x800"}], "structuredContent": {"width": 1280, "height": 800, "scale_factor": 1.0}}} @@ -111,7 +114,7 @@ def module(monkeypatch): async def daemon(module, monkeypatch): """A fake daemon on a socket the driver is pointed at, under a short /tmp path for macOS's socket path limit.""" with tempfile.TemporaryDirectory(dir="/tmp", prefix="cd-") as folder: - fake = FakeDaemon(module._manifest(), Path(folder) / "cua-driver.sock") + fake = FakeDaemon(module._manifest()["tools"], Path(folder) / "cua-driver.sock") monkeypatch.setattr(module, "SOCKET", fake.path) await fake.start() yield fake @@ -172,18 +175,22 @@ async def test_cua_drivers_tools_are_listed_as_pinned_and_forwarded_in_their_cal async with mcp_session(driver) as session: listed = {t.name: t for t in (await session.list_tools()).tools} await session.call_tool("computer_screenshot", {}) - answered = await session.call_tool("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}) + answered = await session.call_tool("get_accessibility_tree", {"pid": 42, "window_id": 7}) launched = await session.call_tool("launch_app", {"name": "thunar"}) - manifest = {t["name"]: t for t in module._manifest()} + manifest = {t["name"]: t for t in module._manifest()["tools"]} assert set(manifest) <= set(listed) and {"computer_click", "computer_screenshot"} <= set(listed) assert listed["get_window_state"].inputSchema == manifest["get_window_state"]["inputSchema"] assert listed["get_window_state"].annotations.readOnlyHint == manifest["get_window_state"]["annotations"]["readOnlyHint"] + assert {name: listed[name].outputSchema for name in manifest} == { + name: tool.get("outputSchema") for name, tool in manifest.items()} + assert driver.mcp.instructions == module._manifest()["instructions"] framework, *_, caller = [r["session_id"] for _, r in daemon.requests if r["method"] == "call"] assert caller != framework and daemon.calls(caller) == [ - ("get_window_state", {"pid": 42, "window_id": 7, "query": "OK"}), ("launch_app", {"name": "thunar"})] + ("get_accessibility_tree", {"pid": 42, "window_id": 7}), ("launch_app", {"name": "thunar"})] assert not launched.isError - assert answered.structuredContent == {"tool": "get_window_state"} and answered.content[0].text == "did get_window_state" + assert answered.structuredContent == {"tool": "get_accessibility_tree"} + assert answered.content[0].text == "did get_accessibility_tree" @pytest.mark.asyncio @@ -191,7 +198,7 @@ async def test_each_caller_gets_a_cua_driver_session_of_its_own(module, daemon): async with mcp_session(module.CuaDriver()) as session: for caller in ("a", "b", "a"): await session.call_tool("list_windows", {}, meta={SESSION_META_KEY: caller}) - await session.call_tool("get_session", {"session": "mine"}, meta={SESSION_META_KEY: "b"}) + await session.call_tool("get_accessibility_tree", {"session": "mine"}, meta={SESSION_META_KEY: "b"}) a, b, again, named = [(r["session_id"], r["args"]) for _, r in daemon.requests if r["method"] == "call"] assert a[0] != b[0] and again == a and named == (b[0], {"session": "mine"}) @@ -205,7 +212,9 @@ async def test_a_failed_call_to_one_of_cua_drivers_tools_reaches_its_caller_as_a await daemon.stop() down = await session.call_tool("list_windows", {}) assert failed.isError and failed.content[0].text == "unknown tool list_apps" + assert failed.structuredContent == {"code": "tool_invocation_failed", "exit_code": 64} assert down.isError and "cua-driver's daemon isn't answering" in down.content[0].text + assert down.structuredContent == {"code": "tool_invocation_failed", "exit_code": 1} @pytest.mark.asyncio @@ -217,13 +226,13 @@ async def test_readiness_checks_the_pinned_executable_and_tools_until_they_pass_ daemon.tools[0] = {**daemon.tools[0], "input_schema": {"type": "object"}} wrong = await driver.ready_checks() - daemon.tools[0] = {"name": daemon.tools[0]["name"], "input_schema": module._manifest()[0]["inputSchema"]} + daemon.tools[0] = {"name": daemon.tools[0]["name"], "input_schema": module._manifest()["tools"][0]["inputSchema"]} right = await driver.ready_checks() lists = sum(1 for _, r in daemon.requests if r["method"] == "list") again = await driver.ready_checks() assert [(c.name, c.ok) for c in wrong] == [("cua-driver", False), ("cua-driver tools", False)] - assert wrong[0].detail.startswith("its executable's sha256 is 0000") and module._manifest()[0]["name"] in wrong[1].detail + assert wrong[0].detail.startswith("its executable's sha256 is 0000") and module._manifest()["tools"][0]["name"] in wrong[1].detail assert [(c.name, c.ok) for c in right] == [("cua-driver", True), ("cua-driver tools", True)] assert again == right and sum(1 for _, r in daemon.requests if r["method"] == "list") == lists @@ -275,6 +284,27 @@ async def test_a_request_cancelled_in_flight_leaves_no_reply_for_the_next_one(mo assert (shot.width, shot.mode) == (1280, "png") and daemon.connections == 2 +@pytest.mark.asyncio +async def test_cua_drivers_skill_resources_are_listed_as_pinned_and_read_once_from_its_own_mcp_server(module, monkeypatch): + reads = [] + + async def read_resources(uris): + reads.append(uris) + return {uri: f"text of {uri}" for uri in uris} + + monkeypatch.setattr(module, "_read_resources", read_resources) + async with mcp_session(module.CuaDriver()) as session: + listed = (await session.list_resources()).resources + skill = await session.read_resource("skill://cua-driver/SKILL.md") + linux = await session.read_resource("skill://cua-driver/LINUX.md") + + pinned = module._manifest()["resources"] + assert [(str(r.uri), r.name, r.description, r.mimeType) for r in listed] == [ + (r["uri"], r["name"], r.get("description"), r["mimeType"]) for r in pinned] + assert skill.contents[0].text == "text of skill://cua-driver/SKILL.md" and linux.contents[0].text.endswith("LINUX.md") + assert reads == [[r["uri"] for r in pinned]] + + def test_the_skill_pack_is_written_offline_from_the_executable(module, monkeypatch, tmp_path): executable = tmp_path / "cua-driver" executable.write_text("""#!/bin/sh From 8d5092bbbd17add6e48f1328059c6f19bd643340 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 16:29:15 -0700 Subject: [PATCH 12/13] refactor(protocol): mark an env's MCP server instructions with @server_instructions An env marks the plain method that returns its MCP server's instructions with @server_instructions, as it marks its skill with @agent_skill, in place of an instructions() hook. They're read when the SDK builds the app, so a direct create_fastmcp_app(handler, ...) gets them too; an env has at most one such method, and a coroutine is refused at decoration. An app passed to mount keeps its own instructions, as the README says. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 23 +++++++++++++--- .../src/agentenv_protocol/__init__.py | 2 ++ .../agent_env_environment.py | 27 ++++++++++++++----- .../tests/test_tool_annotations.py | 20 +++++++++++++- .../drivers/envs/cua-driver/driver.py | 5 ++-- 6 files changed, 66 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9a7e53acb..bec2748dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,7 +124,7 @@ and deserialized through a registry. | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | | `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `instructions`, `forwarded_tools` and `forwarded_resources`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `forwarded_resources`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index ca764c9e5..fb56f6210 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -251,9 +251,26 @@ here, so `call` answers every result, its errors too, in a shape that schema adm on the card beside the `@tool` methods, chosen by `served_tools` and under tool access like them; one named like another tool is refused. In the same way `forwarded_resources()` returns `ForwardedResource(uri, name, description, mime_type, read)` entries, MCP resources whose `read` -returns their text, and `instructions()` returns what the MCP server tells an agent when it -connects, which clients may put in its context. All of them need mcp's FastMCP. The cua-driver -driver serves cua-driver's own instructions, tools and skill resources this way. +returns their text. Both need mcp's FastMCP. The cua-driver driver serves cua-driver's own tools +and skill resources this way, and its instructions with `@server_instructions`. + +### Instructions at connect + +An MCP server may answer `initialize` with `instructions`: how to use it, which a client may put in +the agent's context, as Claude Code does. Mark the plain method that returns them with +`@server_instructions`: + +```python +@environment_card(name="orders") +class Orders(AgentEnvEnvironment): + @server_instructions + def how_to(self) -> str: + return "Look an order up with get_order before refunding it; refunds can't be undone." +``` + +They're read once, when the SDK builds the app (`create_app`, `serve`, `create_fastmcp_app`), and +every session gets the same text. An env has at most one such method. An app you build yourself +and pass to `mount` keeps its own instructions: set them when you construct it. ### Getting part of the data diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index 45d03549d..f689cf701 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -15,6 +15,7 @@ ForwardedTool, add_data, agent_skill, + server_instructions, create_fastmcp_app, environment_card, extension, @@ -98,6 +99,7 @@ "add_data", "get_data", "agent_skill", + "server_instructions", "extension", "tool", "injecting", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index bba93b75e..13141d1f9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -39,6 +39,7 @@ _HIDDEN_BY_DEFAULT_ATTR = "_aee_hidden_by_default" _TOOL_OPTIONS_ATTR = "_aee_tool_options" _FORWARDED_ATTR = "_aee_forwarded" +_INSTRUCTIONS_ATTR = "_aee_instructions" _CARD_CONFIG_ATTR = "_aee_card_config" #: Set on the tool dispatch installed by ``_bind_tool_calls``; a guard downstream checks it at boot. _TOOL_CONTEXT_BOUND_ATTR = "__agentenv_tool_context__" @@ -129,6 +130,16 @@ def deco(fn: Callable) -> Callable: return deco +def server_instructions(fn: Callable) -> Callable: + """Mark the plain method that returns what this env's MCP server tells an agent when it connects, the + ``instructions`` of its initialize result, which clients may put in the agent's context. It's read once, when the + SDK builds the app (``create_app``, ``serve``, ``create_fastmcp_app``); an app passed to ``mount`` keeps its own.""" + if inspect.iscoroutinefunction(fn): + raise TypeError(f"@server_instructions marks a plain method, not the coroutine {fn.__name__!r}") + setattr(fn, _INSTRUCTIONS_ATTR, True) + return fn + + def agent_skill(fn: Callable) -> Callable: """Mark the method that writes this env's skill for agents, returning its description, its markdown body and, optionally, its other files by relative path; it is served under ``SKILL_EXTENSION_URI`` as ``get {name}``, which @@ -544,7 +555,7 @@ def create_fastmcp_app(handler: Any, *, card: EnvironmentCard, **fastmcp_kwargs: from mcp.server.fastmcp import FastMCP except ImportError as e: raise ImportError("create_fastmcp_app requires the 'mcp' package: pip install 'mcp>=1.25,<2'") from e - app = FastMCP(card.name, **fastmcp_kwargs) + app = FastMCP(card.name, **{"instructions": _instructions(handler), **fastmcp_kwargs}) app.settings.host = os.environ.get("MCP_HOST", "0.0.0.0") app.settings.port = int(os.environ.get("MCP_PORT", "18765")) app.settings.transport_security.enable_dns_rebinding_protection = False @@ -575,7 +586,7 @@ def _build_card(self) -> EnvironmentCard: def create_app(self) -> Any: if self._mounted: raise RuntimeError(f"{type(self).__name__} is already mounted") - self.mcp = create_fastmcp_app(self, card=self._build_card(), instructions=self.instructions()) + self.mcp = create_fastmcp_app(self, card=self._build_card()) self._mounted = True return self.mcp @@ -637,10 +648,6 @@ def forwarded_resources(self) -> list[ForwardedResource]: """MCP resources served on another server's behalf. Needs mcp's FastMCP.""" return [] - def instructions(self) -> str | None: - """What this environment's MCP server tells an agent when it connects, which clients may put in its context.""" - return None - def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -1009,6 +1016,14 @@ def _marked_members(handler: Any, marker_attr: str): yield attr, marker +def _instructions(handler: Any) -> str | None: + """What the handler's @server_instructions method returns; None without one.""" + marked = [method for method, _marker in _marked_members(handler, _INSTRUCTIONS_ATTR)] + if len(marked) > 1: + raise ValueError(f"an environment has one @server_instructions method, not {sorted(m.__name__ for m in marked)}") + return marked[0]() if marked else None + + def _discover_methods(handler: Any) -> dict: found: dict[str, Callable] = {} for attr, op in _marked_members(handler, _OP_ATTR): diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index 8b46a05d6..752d91c12 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -22,7 +22,9 @@ EnvironmentTool, ForwardedResource, ForwardedTool, + create_fastmcp_app, environment_card, + server_instructions, tool, ) @@ -247,7 +249,8 @@ async def read() -> str: return [ForwardedResource("skill://other/SKILL.md", "other", "How to use it.", "text/markdown", read)] - def instructions(self) -> str: + @server_instructions + def guide_first(self) -> str: return "Read the guide first." @@ -282,6 +285,21 @@ async def test_forwarded_resources_and_the_instructions_are_served_as_the_other_ assert read.contents[0].text == "# Guide" and app.instructions == "Read the guide first." +def test_server_instructions_are_one_plain_method_and_reach_an_app_built_from_the_handler(): + class Twice(_Forwarding): + @server_instructions + def again(self) -> str: + return "Or this." + + with pytest.raises(ValueError, match="one @server_instructions method, not \\['again', 'guide_first'\\]"): + Twice([]).create_app() + with pytest.raises(TypeError, match="a plain method, not the coroutine 'later'"): + @server_instructions + async def later(self) -> str: + return "Later." + assert create_fastmcp_app(_Forwarding([]), card=EnvironmentCard(name="forwarding")).instructions == "Read the guide first." + + def test_a_forwarded_tool_named_like_a_tool_is_refused(): class Clashing(_Forwarding): @tool() diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 8ac7e1877..93e2f6dc1 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -28,7 +28,7 @@ from mcp.types import CallToolResult, TextContent from pydantic import AnyUrl -from agentenv_protocol import EnvironmentTool, ForwardedResource, ForwardedTool, ToolContext +from agentenv_protocol import EnvironmentTool, ForwardedResource, ForwardedTool, ToolContext, server_instructions from agentenv_protocol.computer import ( PRINTABLE_KEYS, ComputerDriverEnvironment, @@ -100,7 +100,8 @@ def forwarded_resources(self) -> list[ForwardedResource]: functools.partial(self._resource, r["uri"])) for r in _manifest()["resources"]] - def instructions(self) -> str: + @server_instructions + def cua_driver_instructions(self) -> str: return _manifest()["instructions"] async def ready_checks(self) -> list[ReadyCheck]: From 4599261bac25108e4206e6ba9ac53d8d199534c4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 16:48:54 -0700 Subject: [PATCH 13/13] refactor(protocol): serve an env's MCP resources with @resource, in place of forwarded_resources() An env marks each method that serves MCP resources with @resource, as many as it likes, and the URI decides what's served: a fixed URI is one resource; a template such as orders://{order_id} is an MCP resource template whose parts the method takes; a template with listed= serves only the resources listed returns, as MCP lists them, each read through the method with its URI's parts. The cua-driver driver serves cua-driver's 11 skill files with @resource("skill://cua-driver/{file}", listed=...) over mcp.json's list. ForwardedResource and forwarded_resources() are gone. A URI served twice, a listed URI off its template and a method not taking exactly the template's parts are refused; listed resources keep an absent description absent, as cua-driver lists them. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 39 +++++++- .../src/agentenv_protocol/__init__.py | 4 +- .../agent_env_environment.py | 95 ++++++++++++++++--- .../tests/test_tool_annotations.py | 76 ++++++++++++--- .../drivers/envs/cua-driver/driver.py | 18 ++-- 6 files changed, 188 insertions(+), 46 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bec2748dc..77248e0be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,7 +124,7 @@ and deserialized through a registry. | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | | `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `forwarded_resources`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index fb56f6210..79290eeb8 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -249,10 +249,41 @@ call's arguments as keywords, untouched, and returns that server's `CallToolResu the caller as it is. An `output_schema` is listed as the tool's `outputSchema` and isn't checked here, so `call` answers every result, its errors too, in a shape that schema admits. They're listed on the card beside the `@tool` methods, chosen by `served_tools` and under tool access like them; -one named like another tool is refused. In the same way `forwarded_resources()` returns -`ForwardedResource(uri, name, description, mime_type, read)` entries, MCP resources whose `read` -returns their text. Both need mcp's FastMCP. The cua-driver driver serves cua-driver's own tools -and skill resources this way, and its instructions with `@server_instructions`. +one named like another tool is refused. They need mcp's FastMCP. The cua-driver driver serves +cua-driver's own tools this way. + +### Resources + +Mark a method that serves MCP resources with `@resource`; an env may mark several, and each read +calls the method, which returns text or bytes. The URI decides what's served: + +```python +@environment_card(name="orders") +class Orders(AgentEnvEnvironment): + @resource("docs://orders/refund-policy", mime_type="text/markdown") + def refund_policy(self) -> str: + """When a refund is allowed and who approves it.""" + return POLICY.read_text() + + @resource("orders://{order_id}") + async def order(self, order_id: str) -> str: + return json.dumps(await self.db.order(order_id)) + + @resource("skill://upstream/{file}", listed=lambda env: UPSTREAM["resources"]) + async def upstream_file(self, file: str) -> str: + return await read_upstream(f"skill://upstream/{file}") +``` + +- A fixed URI is one resource. Its name defaults to the method's name and its description to the + docstring. +- A template such as `orders://{order_id}` is an MCP resource template; the method takes its parts. +- A template with `listed` serves the resources `listed` returns, each as MCP lists one (`uri`, + `name`, `description`, `mimeType`), and only those. Each is read through the method with its + URI's parts. That's how an env serves another server's resources from a list: the cua-driver + driver serves cua-driver's skill files this way. + +A URI served twice, a listed URI that doesn't match its template, and a method that doesn't take +exactly the template's parts are refused. Resources need mcp's FastMCP. ### Instructions at connect diff --git a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py index f689cf701..b081ab73c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/__init__.py @@ -11,10 +11,10 @@ AgentEnvEnvironment, AgentEnvFastMCPApplication, AgentEnvStarletteApplication, - ForwardedResource, ForwardedTool, add_data, agent_skill, + resource, server_instructions, create_fastmcp_app, environment_card, @@ -91,7 +91,6 @@ "AgentEnvEnvironment", "AgentEnvFastMCPApplication", "AgentEnvStarletteApplication", - "ForwardedResource", "ForwardedTool", "create_fastmcp_app", "environment_card", @@ -99,6 +98,7 @@ "add_data", "get_data", "agent_skill", + "resource", "server_instructions", "extension", "tool", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 13141d1f9..f2b3b4676 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -14,7 +14,7 @@ import types import uuid from abc import ABC, abstractmethod -from typing import Annotated, Any, Awaitable, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints +from typing import Annotated, Any, Callable, ClassVar, Literal, NamedTuple, Union, get_args, get_origin, get_type_hints from pydantic import TypeAdapter, ValidationError from pydantic.fields import FieldInfo @@ -40,6 +40,7 @@ _TOOL_OPTIONS_ATTR = "_aee_tool_options" _FORWARDED_ATTR = "_aee_forwarded" _INSTRUCTIONS_ATTR = "_aee_instructions" +_RESOURCE_ATTR = "_aee_resource" _CARD_CONFIG_ATTR = "_aee_card_config" #: Set on the tool dispatch installed by ``_bind_tool_calls``; a guard downstream checks it at boot. _TOOL_CONTEXT_BOUND_ATTR = "__agentenv_tool_context__" @@ -130,6 +131,28 @@ def deco(fn: Callable) -> Callable: return deco +def resource(uri: str, *, name: str | None = None, description: str | None = None, mime_type: str = "text/plain", + listed: Callable[[Any], list[dict]] | None = None) -> Callable: + """Mark a method that serves MCP resources, returning their text or bytes; an env may mark several, and each read + calls the method. A fixed ``uri`` is one resource. A template such as ``orders://{order_id}`` is an MCP resource + template, the method taking its parts, unless ``listed`` is given: it takes the env and returns the resources to + list, each as MCP lists one (``uri``, ``name``, ``description``, ``mimeType``), each read through the method with + its URI's parts. ``name`` defaults to the method's name and ``description`` to its docstring.""" + parts = set(re.findall(r"{(\w+)}", uri)) + + def mark(fn: Callable) -> Callable: + params = set(list(inspect.signature(fn).parameters)[1:]) + if params != parts: + raise ValueError(f"@resource {uri!r} needs a method taking {sorted(parts) or 'nothing'} besides self, " + f"not {sorted(params)}") + if listed is not None and not parts: + raise ValueError(f"@resource {uri!r} lists resources only from a template URI") + setattr(fn, _RESOURCE_ATTR, _ResourceMarker(uri, name, description, mime_type, listed)) + return fn + + return mark + + def server_instructions(fn: Callable) -> Callable: """Mark the plain method that returns what this env's MCP server tells an agent when it connects, the ``instructions`` of its initialize result, which clients may put in the agent's context. It's read once, when the @@ -228,15 +251,23 @@ class ForwardedTool(NamedTuple): output_schema: dict | None = None -class ForwardedResource(NamedTuple): - """An MCP resource an environment serves on another server's behalf, as that server lists it; ``read`` returns its - text.""" +class _ResourceMarker(NamedTuple): + uri: str + name: str | None + description: str | None + mime_type: str + listed: Callable[[Any], list[dict]] | None + + +class _ServedResource(NamedTuple): + """One resource, or a resource template when ``template``, as it is registered: ``read`` returns its content.""" uri: str name: str description: str | None mime_type: str - read: Callable[[], Awaitable[str]] + read: Callable + template: bool class _ToolOptions(NamedTuple): @@ -279,10 +310,9 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None: tool_handlers = _served_tools(handler, _with_forwarded(_discover_tools(handler, environment_card.name), handler)) if tool_handlers and not self.supports_tools: raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools") - hook = getattr(handler, "forwarded_resources", None) - self._resources: list[ForwardedResource] = hook() if hook is not None else [] + self._resources = _discover_resources(handler) if self._resources and not self.supports_tools: - raise ValueError(f"forwarded resources require a FastMCP-backed application; {type(self).__name__} cannot " + raise ValueError(f"@resource methods require a FastMCP-backed application; {type(self).__name__} cannot " "serve MCP resources") environment_card = _merge_tools(environment_card, tool_handlers) self._tool_access = ( @@ -331,7 +361,7 @@ def _add_route(self, app: Any, path: str, methods: list, handler: Callable) -> N def _register_tool(self, app: Any, descriptor: EnvironmentTool, fn: Callable) -> None: raise NotImplementedError(f"{type(self).__name__} does not serve MCP tools") - def _register_resource(self, app: Any, resource: ForwardedResource) -> None: + def _register_resource(self, app: Any, resource: _ServedResource) -> None: raise NotImplementedError(f"{type(self).__name__} does not serve MCP resources") def _check_tool_binding(self, app: Any) -> None: @@ -415,11 +445,17 @@ def _register_forwarded(self, app: Any, descriptor: EnvironmentTool, fn: Callabl is_async=True, annotations=_mcp_tool_annotations(forwarded.annotations) if forwarded.annotations is not None else None) - def _register_resource(self, app: Any, resource: ForwardedResource) -> None: + def _register_resource(self, app: Any, resource: _ServedResource) -> None: + """A resource as it is, its description absent when it has none, or a template through FastMCP's own + decorator.""" from mcp.server.fastmcp.resources import FunctionResource if self._binding != "dispatch": - raise RuntimeError(f"the forwarded resource {resource.uri!r} needs mcp's FastMCP") + raise RuntimeError(f"the resource {resource.uri!r} needs mcp's FastMCP") + if resource.template: + app.resource(resource.uri, name=resource.name, description=resource.description, + mime_type=resource.mime_type)(resource.read) + return app.add_resource(FunctionResource(uri=resource.uri, name=resource.name, description=resource.description, mime_type=resource.mime_type, fn=resource.read)) @@ -644,10 +680,6 @@ def forwarded_tools(self) -> list[ForwardedTool]: the card, chosen by ``served_tools`` and under tool access like them. Needs mcp's FastMCP.""" return [] - def forwarded_resources(self) -> list[ForwardedResource]: - """MCP resources served on another server's behalf. Needs mcp's FastMCP.""" - return [] - def on_tool_call(self, context: ToolContext) -> Any: """Runs before every tools/call on this environment's app, under the bound ``context``. @@ -1016,6 +1048,39 @@ def _marked_members(handler: Any, marker_attr: str): yield attr, marker +def _discover_resources(handler: Any) -> list[_ServedResource]: + """The resources and resource templates the handler's @resource methods serve; a URI served twice is refused.""" + served: list[_ServedResource] = [] + for method, marker in _marked_members(handler, _RESOURCE_ATTR): + name, description = marker.name or method.__name__, marker.description or inspect.getdoc(method) + if marker.listed is None: + served.append(_ServedResource(marker.uri, name, description, marker.mime_type, method, "{" in marker.uri)) + continue + for entry in marker.listed(handler): + parts = _template_parts(marker.uri, entry["uri"]) + if parts is None: + raise ValueError(f"the listed resource {entry['uri']!r} doesn't match {marker.uri!r}") + served.append(_ServedResource(entry["uri"], entry["name"], entry.get("description"), + entry.get("mimeType") or marker.mime_type, _with_parts(method, parts), False)) + uris = [resource.uri for resource in served] + if duplicated := sorted({uri for uri in uris if uris.count(uri) > 1}): + raise ValueError(f"resources served twice: {duplicated}") + return served + + +def _template_parts(template: str, uri: str) -> dict[str, str] | None: + """The parts of ``uri`` that ``template``'s ``{name}`` slots match, each within one path segment.""" + match = re.fullmatch(re.sub(r"\\\{(\w+)\\\}", r"(?P<\1>[^/]+)", re.escape(template)), uri) + return match.groupdict() if match else None + + +def _with_parts(method: Callable, parts: dict[str, str]) -> Callable: + def read() -> Any: + return method(**parts) + + return read + + def _instructions(handler: Any) -> str | None: """What the handler's @server_instructions method returns; None without one.""" marked = [method for method, _marker in _marked_members(handler, _INSTRUCTIONS_ATTR)] diff --git a/packages/agentenv-protocol/tests/test_tool_annotations.py b/packages/agentenv-protocol/tests/test_tool_annotations.py index 752d91c12..6e2248d35 100644 --- a/packages/agentenv-protocol/tests/test_tool_annotations.py +++ b/packages/agentenv-protocol/tests/test_tool_annotations.py @@ -1,6 +1,8 @@ """@tool annotations, structured output and _meta on both FastMCPs, served_tools choosing what a handler serves,\nupdate_tool changing a served description, and forwarded tools served with their own schemas.""" from __future__ import annotations +import base64 +import re from typing import Annotated import anyio @@ -20,10 +22,10 @@ AgentEnvFastMCPApplication, EnvironmentCard, EnvironmentTool, - ForwardedResource, ForwardedTool, create_fastmcp_app, environment_card, + resource, server_instructions, tool, ) @@ -243,12 +245,6 @@ async def find(**arguments) -> CallToolResult: {"readOnlyHint": False, "destructiveHint": False}), ForwardedTool(EnvironmentTool(name="find", inputSchema={"type": "object"}), find, None, _FOUND)] - def forwarded_resources(self) -> list[ForwardedResource]: - async def read() -> str: - return "# Guide" - - return [ForwardedResource("skill://other/SKILL.md", "other", "How to use it.", "text/markdown", read)] - @server_instructions def guide_first(self) -> str: return "Read the guide first." @@ -272,17 +268,71 @@ async def test_a_forwarded_tool_is_listed_with_its_own_schema_and_called_with_it assert {t.name: t.inputSchema for t in env.mounted_card().capabilities.tools}["click"] == _CLICK +_OTHER_FILES = [ + {"uri": "skill://other/SKILL.md", "name": "other", "description": "The skill.", "mimeType": "text/markdown"}, + {"uri": "skill://other/LINUX.md", "name": "LINUX.md", "mimeType": "text/markdown"}, +] + + +@environment_card(name="library") +class _Library(AgentEnvEnvironment): + @resource("docs://library/guide", mime_type="text/markdown") + def guide(self) -> str: + """How to use it.""" + return "# Guide" + + @resource("skill://other/{file}", listed=lambda env: _OTHER_FILES) + async def other_file(self, file: str) -> str: + return f"# {file}" + + @resource("orders://{order_id}", mime_type="application/octet-stream") + def order(self, order_id: str) -> bytes: + return order_id.encode() + + @pytest.mark.asyncio -async def test_forwarded_resources_and_the_instructions_are_served_as_the_other_server_serves_them(): - app = _Forwarding([]).create_app() +async def test_fixed_listed_and_templated_resources_are_served_by_their_methods(): with anyio.fail_after(30): - async with create_connected_server_and_client_session(app) as session: + async with create_connected_server_and_client_session(_Library().create_app()) as session: listed = (await session.list_resources()).resources - read = await session.read_resource("skill://other/SKILL.md") + templates = (await session.list_resource_templates()).resourceTemplates + guide = await session.read_resource("docs://library/guide") + linux = await session.read_resource("skill://other/LINUX.md") + order = await session.read_resource("orders://42") assert [(str(r.uri), r.name, r.description, r.mimeType) for r in listed] == [ - ("skill://other/SKILL.md", "other", "How to use it.", "text/markdown")] - assert read.contents[0].text == "# Guide" and app.instructions == "Read the guide first." + ("docs://library/guide", "guide", "How to use it.", "text/markdown"), + ("skill://other/SKILL.md", "other", "The skill.", "text/markdown"), + ("skill://other/LINUX.md", "LINUX.md", None, "text/markdown")] + assert [(t.uriTemplate, t.name) for t in templates] == [("orders://{order_id}", "order")] + assert guide.contents[0].text == "# Guide" and linux.contents[0].text == "# LINUX.md" + assert base64.b64decode(order.contents[0].blob) == b"42" + + +def test_a_resource_served_twice_listed_off_its_template_or_taking_other_parts_is_refused(): + class Twice(_Library): + @resource("docs://library/guide") + def again(self) -> str: + return "again" + + class Elsewhere(_Library): + @resource("skill://more/{file}", listed=lambda env: [{"uri": "skill://elsewhere/A.md", "name": "A.md"}]) + def more(self, file: str) -> str: + return file + + def nothing(self) -> str: + return "" + + with pytest.raises(ValueError, match=re.escape("resources served twice: ['docs://library/guide']")): + Twice().create_app() + with pytest.raises(ValueError, match=re.escape("'skill://elsewhere/A.md' doesn't match 'skill://more/{file}'")): + Elsewhere().create_app() + with pytest.raises(ValueError, match=re.escape("needs a method taking ['order_id'] besides self, not []")): + resource("orders://{order_id}")(nothing) + with pytest.raises(ValueError, match="lists resources only from a template URI"): + resource("docs://library/all", listed=lambda env: [])(nothing) + with pytest.raises(RuntimeError, match="the resource 'docs://library/guide' needs mcp's FastMCP"): + _Library().mount(fastmcp.FastMCP("library")) def test_server_instructions_are_one_plain_method_and_reach_an_app_built_from_the_handler(): diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 93e2f6dc1..c4e84f7c6 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -28,7 +28,7 @@ from mcp.types import CallToolResult, TextContent from pydantic import AnyUrl -from agentenv_protocol import EnvironmentTool, ForwardedResource, ForwardedTool, ToolContext, server_instructions +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, resource, server_instructions from agentenv_protocol.computer import ( PRINTABLE_KEYS, ComputerDriverEnvironment, @@ -95,10 +95,12 @@ def forwarded_tools(self) -> list[ForwardedTool]: functools.partial(self._forward, t["name"]), t["annotations"] or None, t.get("outputSchema")) for t in _manifest()["tools"]] - def forwarded_resources(self) -> list[ForwardedResource]: - return [ForwardedResource(r["uri"], r["name"], r.get("description"), r["mimeType"], - functools.partial(self._resource, r["uri"])) - for r in _manifest()["resources"]] + @resource("skill://cua-driver/{file}", listed=lambda env: _manifest()["resources"]) + async def skill_file(self, file: str) -> str: + """One of cua-driver's skill files as its own MCP server serves it, all of them read on first use.""" + if self._texts is None: + self._texts = await _read_resources([r["uri"] for r in _manifest()["resources"]]) + return self._texts[f"skill://cua-driver/{file}"] @server_instructions def cua_driver_instructions(self) -> str: @@ -163,12 +165,6 @@ async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: return _refused(reply.get("error") or "daemon reported failure", reply.get("exit_code") or 1) return CallToolResult.model_validate(reply["result"]) - async def _resource(self, uri: str) -> str: - """One of cua-driver's skill resources as its own MCP server serves it, all of them read on first use.""" - if self._texts is None: - self._texts = await _read_resources([r["uri"] for r in _manifest()["resources"]]) - return self._texts[uri] - async def _send(self, name: str, **args: Any) -> None: await self._call(name, DeliveryFailed, scope="desktop", **args)