diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index ab74b5948..a8f486707 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -47,7 +47,7 @@ with `[tools]`. `requirements.txt` installs `cua-driver==0.38.0` from PyPI, whos `cua-driver`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/cua-driver`. On macOS (`arm64`, composed onto a VM bundle such as `macos-26`), cua-driver is CuaDriver.app, from the cua-driver-rs -0.38.0 release, its tarball checked by sha256 and its app put in `/Applications` by `install-app.sh`. Its daemon is a +0.47.10 release, its tarball checked by sha256 and its app put in `/Applications` by `install-app.sh`. Its daemon is a direct LaunchAgent, CuaDriver.app's own executable, since macOS gives the app's TCC permissions, Accessibility, Post Event and Screen Recording, which `[install.macos]` grants it, only to the executable launchd starts itself; its socket is `/tmp/agentenv-cua-driver.sock`. `requirements-macos.txt` installs the protocol alone. Agents get the 62 tools @@ -55,9 +55,10 @@ is `/tmp/agentenv-cua-driver.sock`. `requirements-macos.txt` installs the protoc sha256, those tools, and that cua-driver reports Accessibility and Screen Recording granted. Its pointer's actions go in the foreground, where cua-driver otherwise keeps from moving a person's pointer over a background app, and a click or a drag first rests the pointer where it presses, since an app such as Tk reads where the pointer is when it -handles the press, and a button the pointer hasn't yet entered loses the click (P4, a button and a checkbox take a click). cua-driver 0.38.0 fails two of the probe's checks on macOS for defects of its own: it posts each typed -character's key-up with the A key's code (P6, typed text arrives as key events), and a drag's mouse-down after the -pointer has moved along its path (P7). `env.toml`'s `[[known_failures.macos]]` names both with that reason, so gate 3 +handles the press, and a button the pointer hasn't yet entered loses the click (P4, a button and a checkbox take a click). cua-driver fails two of the probe's checks on macOS for defects of its own: it types each character +as a key-down and key-up of the A key's code carrying the character as Unicode, so the text arrives but an app reading +key events sees the A key (P6, typed text arrives as key events), and 0.38.0 posted a drag's mouse-down after the +pointer had moved along its path (P7), which 0.47.10, posting a drag's events the same way, hasn't yet shown. `env.toml`'s `[[known_failures.macos]]` names both with that reason, so gate 3 records them in the build's probe (`known_failures`) and doesn't fail the build for them; any other failure still does. A driver's `known_failures`, by OS type, names a check by its id and name, with why. One that a probe didn't see fail is recorded in its probe (`unused_known_failures`, with what it saw) and warned of, so a waiver doesn't outlive its diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 137dad4e4..e949004b0 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -98,7 +98,7 @@ def _desktop() -> str | None: EXECUTABLES = { "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", "arm64": "26e9eee8165e9ce7b614b5e3d68e5d963231385a82b65236406718b5ce1ff936", - "macos/arm64": "b4d974a9c1c92a700cd7be7ede987aea4a1e6835e37039ce3d628b71bd5f0ca8", + "macos/arm64": "f026e4dea3d058c7c702dfebca93853e6f9410ce76b73c06f35eae1a81bbc873", "windows/amd64": "ffa58d88779ee219304b0c307f4cfa970fe822f830c6ecf9254eda5c29e22bf2", "hyprland/amd64": "8dd0f74928cff484343ed65e3ac4d1dacdccc3950b8665e1102ac463af0a967b", } diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml index ff8e7d8aa..2f2ef7824 100644 --- a/src/agent_env/drivers/envs/cua-driver/env.toml +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -23,7 +23,7 @@ services = [ [install.macos] python = "requirements-macos.txt" downloads = [ - { url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.38.0/cua-driver-rs-0.38.0-darwin-arm64.tar.gz", sha256 = "5f71191c082ca07c9dfc13b7c9a6eec60a82818c245a93534cc0eb7911471d72", to = "CuaDriver.tar.gz", arch = "arm64" }, + { url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.47.10/cua-driver-rs-0.47.10-darwin-arm64.tar.gz", sha256 = "40954056eeb011660abe1ff31693676a5cce6d53b2c0e6afbafb30c9b805c141", to = "CuaDriver.tar.gz", arch = "arm64" }, ] run = [{ script = "install-app.sh" }] grants = [{ app = "/Applications/CuaDriver.app", services = ["accessibility", "post_event", "screen_capture"] }] @@ -43,16 +43,16 @@ services = [ { name = "driver", command = "venv/Scripts/python.exe -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, ] -# cua-driver 0.38.0's own defects on macOS, which gate 3 records without failing the build. +# cua-driver's own defects on macOS, which gate 3 records without failing the build. [[known_failures.macos]] check = "P6" name = "typed text arrives as key events" -reason = "cua-driver 0.38.0 posts each typed character's key-up with keycode 0, the A key, so an app sees 'a' released; its pressed keys release as they should" +reason = "cua-driver 0.47.10 types each character as a key-down and key-up of keycode 0, the A key, carrying the character as Unicode, so the text arrives whole but an app reading key events sees the A key; Return, Tab and the keys computer_key presses are real keys" [[known_failures.macos]] check = "P7" name = "a drag starts and ends where it says, through its waypoints" -reason = "cua-driver 0.38.0 posts a drag's mouse-down after the pointer has moved along its path, on some first probes after a boot 3 to 10 px from the start" +reason = "cua-driver 0.38.0 posted a drag's mouse-down after the pointer had moved along its path, on some first probes after a boot 3 to 10 px from the start; 0.47.10 posts a drag's events as 0.38.0 did and passed its first 13 probes, too few to call an intermittent failure fixed" # cua-driver 0.38.0's own defect on Windows, which gate 3 records without failing the build. [[known_failures.windows]] diff --git a/src/agent_env/drivers/envs/cua-driver/install-app.sh b/src/agent_env/drivers/envs/cua-driver/install-app.sh index d255befa3..467f10bfe 100755 --- a/src/agent_env/drivers/envs/cua-driver/install-app.sh +++ b/src/agent_env/drivers/envs/cua-driver/install-app.sh @@ -5,7 +5,7 @@ set -eu work=$(mktemp -d) tar -xzf CuaDriver.tar.gz -C "$work" rm -rf /Applications/CuaDriver.app -ditto "$work/cua-driver-rs-0.38.0-darwin-arm64/CuaDriver.app" /Applications/CuaDriver.app +ditto "$work/cua-driver-rs-0.47.10-darwin-arm64/CuaDriver.app" /Applications/CuaDriver.app rm -rf "$work" xattr -dr com.apple.quarantine /Applications/CuaDriver.app || true rm -f CuaDriver.tar.gz diff --git a/src/agent_env/drivers/envs/cua-driver/mcp-macos.json b/src/agent_env/drivers/envs/cua-driver/mcp-macos.json index b230f1595..b682dc259 100644 --- a/src/agent_env/drivers/envs/cua-driver/mcp-macos.json +++ b/src/agent_env/drivers/envs/cua-driver/mcp-macos.json @@ -1,5 +1,5 @@ { - "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AX): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused or unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nmacOS: if `check_permissions` shows Accessibility or Screen Recording off, ask the user; approve Cua Driver's own prompts only if the owner explicitly authorized it (MACOS.md).\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`.", + "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AX): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step, batching actions: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused/unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nmacOS: if `check_permissions` shows Accessibility or Screen Recording off, ask the user; approve Cua Driver's own prompts only if the owner explicitly authorized it (MACOS.md).\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`.", "tools": [ { "name": "list_apps", @@ -344,7 +344,7 @@ "type": "string" }, "screenshot_out_file": { - "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead.", + "description": "When set, write the screenshot (PNG, or JPEG with screenshot_format:\"jpeg\") to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead.", "type": "string" }, "session": { @@ -378,6 +378,20 @@ "window_id": { "description": "Target window ID from list_windows.", "type": "integer" + }, + "screenshot_format": { + "description": "Encoding of the returned screenshot. Default \"png\". \"jpeg\" is smaller, drops alpha, and mints no capture_id (capture_id-bound tools need a PNG read); pixel coordinates are unchanged.", + "enum": [ + "png", + "jpeg" + ], + "type": "string" + }, + "screenshot_quality": { + "description": "JPEG quality, 1-95, with screenshot_format:\"jpeg\" only. Default 85.", + "maximum": 95, + "minimum": 1, + "type": "integer" } }, "required": [ @@ -1321,6 +1335,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -1565,6 +1582,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -1600,7 +1620,7 @@ }, { "name": "click", - "description": "Click a target pid. Prefer `element_token` (from get_window_state): it identifies one snapshot element, works on backgrounded, minimized or off-Space windows, and moves no cursor or focus. Use `x, y` only for canvas / video / WebGL / custom-drawn surfaces missing from the AX tree.\n\n- element_token: AX action path. Tokens are scoped to (pid, window_id) and stale once the window is snapshotted again.\n- x, y: window-local pixels of the get_window_state screenshot, sent as CGEvent mouse events. Needs a visible window. `modifier` takes cmd/shift/option/ctrl.\n\n`button` defaults to left; on the AX path \"right\" maps to AXShowMenu and \"middle\" falls back to a pixel click at the element center. `action`: press (default), show_menu, pick, confirm, cancel, open, show_alternate_ui, show_default_ui; an element listing showalternateui reveals its hover-only controls with show_alternate_ui, in the background and without the pointer. `from_zoom:true` translates zoom-image pixels after a `zoom` call. Pressing a popup button opens its menu and leaves it open until an item is chosen or it is dismissed; in the background the user's front window keeps key focus meanwhile. To choose an option in one call use `set_value`. In the background a web `` (Chrome, Electron) is not opened, because its menu would raise the browser window: the click is refused with `select_menu_would_raise_window`; use `set_value`. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -1712,6 +1732,14 @@ "y": { "description": "Y in screenshot pixels from the image selected by target.", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [], @@ -1888,6 +1916,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -1959,6 +1990,14 @@ "y": { "description": "Screen Y coordinate (pixel path).", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [], @@ -2135,6 +2174,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -2213,6 +2255,14 @@ "y": { "description": "Y in window-local screenshot pixels. Must be provided together with x.", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [], @@ -2389,6 +2439,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -2536,6 +2589,14 @@ "window_id": { "description": "CGWindowID for the window the pixel coordinates were measured against. Optional only when pid owns exactly one eligible top-level window; otherwise the action refuses with ambiguous_window_target.", "type": "integer" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -2717,6 +2778,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -2752,12 +2816,12 @@ }, { "name": "type_text", - "description": "Insert text via `AXSetAttribute(kAXSelectedText)` into the element given by `element_token`, or the pid's focused element. Does not press keys: use `press_key` / `hotkey` for Return, Escape, arrows. If AX insertion is unsupported (Chromium/Electron), it falls back to CGEvent typing when the route fits the transport budget; longer routes are refused with a safe chunk size.\n\nWeb content (browser tabs, Slack, VS Code): AXValue is not proof the DOM saw the text, so the result is `effect:\"unverifiable\"` and Electron web content refuses background delivery. For a browser tab use the `page` tool. For an embedded web view pass `x, y` (no element_token) to pixel-click the field then type in one call; a pixel click will not reliably open a closed control, so AX-press it open first. Confirm from the screenshot, and use `delivery_mode:\"foreground\"` only if background still drops the text.", + "description": "Insert text via `AXSetAttribute(kAXSelectedText)` into the element given by `element_token`, or the pid's focused element. Does not press keys: use `press_key` / `hotkey` for Return, Escape, arrows. If AX insertion is unsupported (Chromium/Electron), it falls back to CGEvent typing when the route fits the transport budget; longer routes are refused with a safe chunk size.\n\nWeb content (browser tabs, Slack, VS Code): AXValue is not proof the DOM saw the text, so the result is `effect:\"unverifiable\"` and Electron web content refuses background delivery. For a browser tab use the `page` tool. For an embedded web view pass `x, y` (no element_token) to pixel-click the field then type in one call; a pixel click will not reliably open a closed control, so AX-press it open first. Confirm from the screenshot, and use `delivery_mode:\"foreground\"` only if background still drops the text. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { "delay_ms": { - "description": "Ms between characters in the CGEvent fallback. Default 30.", + "description": "Ms between characters in the CGEvent fallback. Default 0: characters go in batched key events, fast. Set a positive value for one paced key per character (debounced search boxes).", "maximum": 200, "minimum": 0, "type": "integer" @@ -2839,6 +2903,14 @@ "y": { "description": "Screenshot-pixel Y (see x).", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -3017,6 +3089,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -3052,7 +3127,7 @@ }, { "name": "press_key", - "description": "Press and release one key. `delivery_mode` as in click/type_text.\n- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.\n- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.\n\nAn element target whose focus cannot be confirmed is refused with `target_focus_unconfirmed`, effect \"refused\", and no key dispatched.\nConfirmed only when a native AX value/selection read-back changes; otherwise `effect:\"unverifiable\"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit.", + "description": "Press and release one key. `delivery_mode` as in click/type_text.\n- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.\n- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.\n\nAn element target whose focus cannot be confirmed is refused with `target_focus_unconfirmed`, effect \"refused\", and no key dispatched.\nConfirmed only when a native AX value/selection read-back changes; otherwise `effect:\"unverifiable\"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -3140,6 +3215,14 @@ "y": { "description": "Screenshot-pixel Y (see x).", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -3318,6 +3401,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -3353,7 +3439,7 @@ }, { "name": "hotkey", - "description": "Press a key combination \u2014 e.g. `[\"cmd\", \"c\"]` for Copy, `[\"cmd\", \"shift\", \"4\"]` for screenshot selection. Follows the same `delivery_mode` ladder as click/type_text \u2014 it does NOT raise the window by default:\n\u2022 `background` (default): post the combo to the target pid WITHOUT fronting or raising it \u2014 uses the macOS 14+ auth-message envelope so Chromium/Electron accept it as trusted live input. With an AX target, focus that exact element first. No top-level focus steal. `window_id` here only targets the combo; it does not raise. Background key events never reach a native app's menus, so when the combo is a menu item's key equivalent (cmd+s, cmd+w, ...) and the app is not in front, that menu item runs through accessibility instead (a disabled Save is saved through the app's scripting interface and verified on disk); a menu item macOS keeps disabled in the background is refused with `background_unavailable`, and no keys are sent.\n\u2022 `foreground`: briefly front the window (NSMenu path, < 1 ms via SLPSSetFrontProcessWithOptions) so native menu key-equivalents (Cmd+Z, Cmd+W) dispatch, then restore the prior frontmost \u2014 the explicit escalation for menu-bar shortcuts on non-Chromium apps that ignore a background combo. With an AX target or x,y, the focused field receives the chord through the foreground HID queue (needed by native Chromium fields such as the omnibox). Requires window_id.\n\nA combo is never driver-verifiable (no read-back) \u2192 effect:\"unverifiable\"; confirm via screenshot. NOTE: a keyboard combo does NOT focus a text field \u2014 to type into a backgrounded Electron input, establish real renderer focus with a PIXEL click first, then `type_text`. If an app only accepts paste, call `clipboard_write`, then `clipboard_read` and verify its types (and text when applicable) before selecting or replacing editor content; only then send Cmd+V.\n\nRecognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. Non-modifier keys use the same vocabulary as `press_key`. Order: modifiers first, one non-modifier last.", + "description": "Press a key combination \u2014 e.g. `[\"cmd\", \"c\"]` for Copy, `[\"cmd\", \"shift\", \"4\"]` for screenshot selection. Follows the same `delivery_mode` ladder as click/type_text \u2014 it does NOT raise the window by default:\n\u2022 `background` (default): post the combo to the target pid WITHOUT fronting or raising it \u2014 uses the macOS 14+ auth-message envelope so Chromium/Electron accept it as trusted live input. With an AX target, focus that exact element first. No top-level focus steal. `window_id` here only targets the combo; it does not raise. Background key events never reach a native app's menus, so when the combo is a menu item's key equivalent (cmd+s, cmd+w, ...) and the app is not in front, that menu item runs through accessibility instead (a disabled Save is saved through the app's scripting interface and verified on disk); a menu item macOS keeps disabled in the background is refused with `background_unavailable`. With serve --no-automation-prompts, scripting without granted consent refuses with `automation_consent_required`, without keyboard fallback.\n\u2022 `foreground`: briefly front the window (NSMenu path, < 1 ms via SLPSSetFrontProcessWithOptions) so native menu key-equivalents (Cmd+Z, Cmd+W) dispatch, then restore the prior frontmost \u2014 the explicit escalation for menu-bar shortcuts on non-Chromium apps that ignore a background combo. With an AX target or x,y, the focused field receives the chord through the foreground HID queue (needed by native Chromium fields such as the omnibox). Requires window_id.\n\nA combo is never driver-verifiable (no read-back) \u2192 effect:\"unverifiable\"; confirm via screenshot. NOTE: a keyboard combo does NOT focus a text field \u2014 to type into a backgrounded Electron input, establish real renderer focus with a PIXEL click first, then `type_text`. If an app only accepts paste, call `clipboard_write`, then `clipboard_read` and verify its types (and text when applicable) before selecting or replacing editor content; only then send Cmd+V.\n\nRecognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. Non-modifier keys use the same vocabulary as `press_key`. Order: modifiers first, one non-modifier last.", "inputSchema": { "additionalProperties": false, "properties": { @@ -3438,6 +3524,14 @@ "y": { "description": "Screenshot-pixel Y (see x).", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -3616,6 +3710,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -3651,7 +3748,7 @@ }, { "name": "set_value", - "description": "Set an element's value by `element_token`. A popup button / select dropdown: selects the menu item whose title or value matches `value` (case-insensitive). Safari `` is set through the DOM; AppKit and Chromium popups list items only while open, so the menu is opened, the item pressed and the menu closed again (in the background the user's front window keeps key focus). An unknown value lists the options. Any other element: writes AXValue (sliders, steppers, date pickers, native text fields).\n\nA Finder file name cell (list or icon view, not being edited) is renamed through Finder's scripting interface with Finder left in the background, and the new name is read back from disk; the value is the full new name with its extension. macOS may prompt for Automation consent; with serve --no-automation-prompts, ungranted consent refuses with automation_consent_required. Finder's Get Info `Name` field is refused with `file_name_needs_rename`. For free-form text in web inputs use `type_text_chars`: WebKit ignores AXValue writes.", "inputSchema": { "additionalProperties": false, "properties": { @@ -3675,6 +3772,14 @@ "window_id": { "description": "Window ID. Omit with element_token.", "type": "integer" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -3853,6 +3958,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -3989,6 +4097,14 @@ "y": { "description": "Window-local screenshot Y.", "type": "number" + }, + "post_action_wait": { + "description": "Whether the result waits for the post-action window-change report (default \"default\": up to 300 ms for a new window, menu, dialog or foreground change). \"none\" returns as soon as the input is delivered and says window changes were not observed; the watch keeps running in the background and a change it sees is reported on the next action's result. Only paths that already observe window changes are affected; for example a right_click that does not watch is unchanged.", + "enum": [ + "default", + "none" + ], + "type": "string" } }, "required": [ @@ -4167,6 +4283,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -4902,6 +5021,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -5773,7 +5895,7 @@ }, { "name": "set_config", - "description": "Update cua-driver-rs configuration. Changes to max_image_dimension take effect immediately. The experimental_pip keys are persisted to ~/.cua-driver/config.json and take effect on the next daemon restart (the PiP backend is initialised once at startup).\n\nNote: capture_mode is a per-call param (on get_window_state / click), not a stored setting. Capture modality is selected by each action's target; the old capture_scope config key is retired.\n\nCursor motion defaults: cursor.motion.style, cursor.motion.timing and cursor.motion.effects. are saved to ~/.cua-driver/config.json and seed sessions started afterwards. start_session cursor_motion and set_agent_cursor_motion override them; reduced motion always wins.\n\nAgent cursor badge: cursor.badges is minimal (default), verbose or off, saved to ~/.cua-driver/config.json and applied to every cursor at once.", + "description": "Update cua-driver-rs configuration. Changes to max_image_dimension take effect immediately. The experimental_pip keys are persisted to ~/.cua-driver/config.json and take effect on the next daemon restart (the PiP backend is initialised once at startup).\n\nNote: capture_mode is a per-call param (on get_window_state / click), not a stored setting. Capture modality is selected by each action's target; the old capture_scope config key is retired.\n\nCursor motion defaults: cursor.motion.style, cursor.motion.timing and cursor.motion.effects. are saved to ~/.cua-driver/config.json and seed sessions started afterwards. start_session cursor_motion and set_agent_cursor_motion override them; reduced motion always wins.\n\nAgent cursor badge: cursor.badges is minimal (default), verbose or off, saved to ~/.cua-driver/config.json and applied to every cursor at once.\n\nAgent cursor extras: cursor.float (float at rest and while moving) and cursor.halo (soft glow) are on (default), off or default; cursor.marks (action marks) is on, off (default) or default; cursor.rotate is heading (default), lean, off or default. All are saved to ~/.cua-driver/config.json and applied to every cursor at once.", "inputSchema": { "additionalProperties": false, "properties": { @@ -5879,6 +6001,54 @@ "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" + }, + "cursor.float": { + "description": "Agent cursor float: a slow four-second sway, bob and slight tilt, at rest and while moving, that eases to zero on every arrival and holds there for the click, drag, drop, scroll or typing, so they land exactly upright on the hotspot. on (default). off: the cursor never floats and a resting overlay stops redrawing. Applies immediately to every cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.halo": { + "description": "Agent cursor halo: a soft glow in the cursor's colour behind the arrow, so it is easier to follow on any background. on (default). off: no halo. Applies immediately to every cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.marks": { + "description": "Agent cursor action marks: the small cue next to the arrow for observe, click, type, keys, scroll, drag, apps, files, recording and system calls, plus the focus box around a clicked element on macOS. off (default): a plain arrow. Applies immediately to every cursor.", + "enum": [ + "on", + "off", + "default" + ], + "type": "string" + }, + "cursor.rotate": { + "description": "Agent cursor rotation while it moves: heading (default) points the tip along the path; lean tilts the arrow at most 30 degrees toward where it is going (on is the same as heading). Both return upright before arrival, so clicks, typing and scrolling stay upright. off: the arrow always stays upright. Applies immediately to every cursor.", + "enum": [ + "off", + "lean", + "heading", + "on", + "default" + ], + "type": "string" + }, + "fast_path": { + "description": "Experimental fast path (default off). shadow: check cached targets at act time and report would-refuse verdicts; guard: refuse an action whose target changed since it was read (target_changed); on: guard, plus a small delta in each action result and tree-only re-reads. Runtime only, not saved; env CUA_DRIVER_FAST_PATH seeds it at start.", + "enum": [ + "off", + "shadow", + "guard", + "on" + ], + "type": "string" } }, "type": "object" @@ -5966,7 +6136,7 @@ }, { "name": "page", - "description": "Legacy browser compatibility tool. Prefer get_browser_state and the typed browser_* tools for exact targeting, endpoint ownership, and consent. Read-only get_text and query_dom remain available by default. Mutating actions require the daemon operator to set CUA_DRIVER_ENABLE_LEGACY_PAGE_MUTATIONS=1 before daemon startup (restart the daemon after changing it); this escape hatch does not provide the typed browser surface's exact binding or existing-profile grant guarantees. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page.\n- query_dom: Find elements matching a CSS selector.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- insert_text: Insert `text` at whatever currently holds DOM focus in one native operation (CDP Input.insertText) \u2014 no synthesized key events, but more durable than a one-shot execute_javascript write since rich-text editors already have to treat it like an IME commit. Try this before type_keystrokes on a contenteditable that discarded an execute_javascript write. Click/focus the target field first.\n- type_keystrokes: Type `text` via real per-character keystroke events into whatever currently holds DOM focus. Slower than insert_text but the most durable rung \u2014 use it when insert_text also gets discarded, or the editor's own keydown/keyup handlers need to see real keys. Click/focus the target field first.\n- enable_javascript_apple_events: macOS-only \u2014 patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", + "description": "Legacy browser compatibility tool. Prefer get_browser_state and the typed browser_* tools for exact targeting, endpoint ownership, and consent. Read-only get_text and query_dom remain available by default. Mutating actions require the daemon operator to set CUA_DRIVER_ENABLE_LEGACY_PAGE_MUTATIONS=1 before daemon startup (restart the daemon after changing it); this escape hatch does not provide the typed browser surface's exact binding or existing-profile grant guarantees. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page. On macOS, structuredContent.source identifies javascript, cdp, or ax_fallback; a successful AX fallback preserves the triggering error and truncation state in structuredContent.\n- query_dom: Find elements matching a CSS selector, with the same macOS fallback metadata contract as get_text.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- insert_text: Insert `text` at whatever currently holds DOM focus in one native operation (CDP Input.insertText) \u2014 no synthesized key events, but more durable than a one-shot execute_javascript write since rich-text editors already have to treat it like an IME commit. Try this before type_keystrokes on a contenteditable that discarded an execute_javascript write. Click/focus the target field first.\n- type_keystrokes: Type `text` via real per-character keystroke events into whatever currently holds DOM focus. Slower than insert_text but the most durable rung \u2014 use it when insert_text also gets discarded, or the editor's own keydown/keyup handlers need to see real keys. Click/focus the target field first.\n- enable_javascript_apple_events: macOS-only \u2014 patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", "inputSchema": { "additionalProperties": false, "properties": { @@ -6232,7 +6402,7 @@ }, { "name": "browser_click", - "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.", + "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If a click's handler opens a page-owned JavaScript dialog, on either route, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.", "inputSchema": { "additionalProperties": true, "properties": { @@ -6455,6 +6625,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -6706,6 +6879,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -7197,6 +7373,9 @@ "string", "null" ] + }, + "fast_path": { + "type": "object" } }, "required": [ @@ -7232,7 +7411,7 @@ }, { "name": "start_recording", - "description": "Start trajectory recording for the calling session. Each action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) from that session writes a turn folder under `output_dir`. Without `session`, every call on the same connection is recorded, including named-session calls. Other connections and session lifecycle calls (`start_session` / `end_session`) are not recorded. CLI recordings started with `cua-driver recording start` are daemon-wide.\n\nEach turn folder holds:\n\n- `before_state.json` / `after_state.json` \u2014 application AX/UIA/AT-SPI state immediately before and after the action.\n- `before.png` / `after.png` \u2014 target-window screenshots immediately before and after the action.\n- `evidence.json` \u2014 capture status and a stable classification when an expected artifact could not be captured.\n- `app_state.json` \u2014 post-action AX/UIA snapshot for the target pid.\n- `screenshot.png` \u2014 compatibility alias of `after.png`.\n- `action.json` \u2014 tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.\n- `click.png` \u2014 for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.\n\nThe per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.\n\nTurn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.\n\n**Video is off by default.** Pass `record_video: true` to also capture the main display to `/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.\n\n**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) under the daemon's Screen Recording grant, with no ffmpeg subprocess. Requires macOS 15.0+. On macOS 26 (Tahoe), the first direct capture can also show a one-time consent asking to let Cua Driver bypass the system private window picker and directly access your screen and audio; choose Allow, or run `cua-driver permissions grant` beforehand to answer it up front. The recorder captures screen video only and does not enable system-audio capture.\n\n**Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.\n\nState persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.", + "description": "Start trajectory recording for the calling session. Each action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) from that session writes a turn folder under `output_dir`. Without `session`, every call on the same connection is recorded, including named-session calls. Other connections and session lifecycle calls (`start_session` / `end_session`) are not recorded. CLI recordings started with `cua-driver recording start` are daemon-wide.\n\nEach turn folder holds:\n\n- `before_state.json` / `after_state.json` \u2014 application AX/UIA/AT-SPI state immediately before and after the action.\n- `before.png` / `after.png` \u2014 target-window screenshots immediately before and after the action.\n- `evidence.json` \u2014 capture status and a stable classification when an expected artifact could not be captured.\n- `app_state.json` \u2014 post-action AX/UIA snapshot for the target pid.\n- `screenshot.png` \u2014 compatibility alias of `after.png`.\n- `action.json` \u2014 tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.\n- `click.png` \u2014 for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.\n\nThe per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.\n\nPass `redact_values` with any secret the session will type (for example a password): each occurrence in a turn's text files is replaced with `[redacted]`, and such turns are not replayed.\n\nTurn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.\n\n**Video is off by default.** Pass `record_video: true` to also capture the main display to `/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.\n\n**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) under the daemon's Screen Recording grant, with no ffmpeg subprocess. Requires macOS 15.0+. On macOS 26 (Tahoe), the first direct capture can also show a one-time consent asking to let Cua Driver bypass the system private window picker and directly access your screen and audio; choose Allow, or run `cua-driver permissions grant` beforehand to answer it up front. The recorder captures screen video only and does not enable system-audio capture.\n\n**Windows + Linux X11 use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.\n\n**Linux Wayland routes by compositor**: GNOME uses the trusted Shell helper's frames, wlroots compositors use `wf-recorder`, and compositors without wlr-screencopy (KDE/KWin) record the monitor the user grants through the xdg-desktop-portal ScreenCast (one consent dialog per install; requires a build with the `portal-capture` feature and ffmpeg on PATH). Unsupported combinations report the exact limitation in `last_error`.\n\nState persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.", "inputSchema": { "additionalProperties": false, "properties": { @@ -7259,6 +7438,16 @@ "maximum": 120000, "minimum": 100, "type": "integer" + }, + "redact_values": { + "description": "Secret values, such as passwords or tokens, that must never be written to the recording. Every occurrence in a turn's action.json and state files is replaced with `[redacted]`, and the turn is marked `redacted: true` so replay_trajectory skips it. The values are held in memory only, never written, and cleared when recording stops. Screenshots and video are not altered.", + "items": { + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "minItems": 1, + "type": "array" } }, "required": [ @@ -8384,7 +8573,7 @@ }, { "name": "run_steps", - "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", + "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Steps pass the same checks as direct calls. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", "inputSchema": { "additionalProperties": false, "properties": {