From 23d75ddd47c71132577e63b397e6bbfc5a7801c3 Mon Sep 17 00:00:00 2001 From: Adam Crocker <77861255+captcrock@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:32:39 -0500 Subject: [PATCH 1/2] TD-5227: each agent has its own cursor on the real display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent doing computer use had no cursor of its own on the real display: in full control it moved the user's pointer, and with several agents at once nothing showed which was where. TD-3402 kept the agent cursor on the Screen pane only; DECISIONS.md replaces that rule. Two renderers, one identity per agent: - tst-cu-mcp's macOS helper draws a click-through, non-shareable cursor panel per agent (arrow and label in its color) at its last move, click or acted-on element, fading 10 s after its last action. Reduced motion: no glide, no click ring. Colors come from a machine-wide flock slot registry; labels from the MCP client's name. - The element tools run in one agent-desktop session with its own multi-agent cursor, each TST Desk session enabled with the same label and color. Each TST Desk session is an agent ("TST Desk · "): the session loop sets a context variable, and the MCP driver names the acting session with the internal overlay_agent tool before each pointer action, under one lock. The display switches are mirrored into the shared policy file (overlay.enabled, overlay.cursor) so every client follows them live. Stop hides every drawn cursor and disables agent-desktop's overlay. The helper's line protocol moves to overlay/darwin_wire.py so the helper stays near its size limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HUthoCdmzg2hy89ssk2cSF --- DECISIONS.md | 87 +++++ core/tests/test_cu_agent_cursors.py | 324 ++++++++++++++++++ core/tstd/cu_agents.py | 80 +++++ core/tstd/cu_policy.py | 40 ++- core/tstd/daemon.py | 41 ++- core/tstd/desktop/ax_cli.py | 17 +- core/tstd/desktop/ax_cursor.py | 145 ++++++++ core/tstd/desktop/ax_driver.py | 18 +- core/tstd/desktop/inprocess.py | 9 + core/tstd/desktop/mcp_driver.py | 68 +++- core/tstd/desktop/mock.py | 5 + core/tstd/desktop/protocol.py | 8 +- core/tstd/local_worker.py | 2 +- core/tstd/session.py | 4 + docs/configuration.md | 13 +- docs/tst-desk-backlog.md | 27 ++ mcp/tst-cu-mcp/pyproject.toml | 2 +- mcp/tst-cu-mcp/src/tst_cu_mcp/config.py | 6 +- .../src/tst_cu_mcp/input_control.py | 8 +- .../src/tst_cu_mcp/overlay/__init__.py | 49 ++- .../src/tst_cu_mcp/overlay/agents.py | 190 ++++++++++ .../src/tst_cu_mcp/overlay/control.py | 104 ++++-- .../src/tst_cu_mcp/overlay/darwin.py | 13 + .../src/tst_cu_mcp/overlay/darwin_cursor.py | 321 +++++++++++++++++ .../src/tst_cu_mcp/overlay/darwin_helper.py | 98 ++---- .../src/tst_cu_mcp/overlay/darwin_wire.py | 80 +++++ .../src/tst_cu_mcp/overlay/linux.py | 8 + .../src/tst_cu_mcp/overlay/win32.py | 8 + mcp/tst-cu-mcp/src/tst_cu_mcp/server.py | 14 + .../src/tst_cu_mcp/stdio_transport.py | 17 + mcp/tst-cu-mcp/src/tst_cu_mcp/ui.py | 18 + mcp/tst-cu-mcp/tests/test_agent_cursors.py | 279 +++++++++++++++ mcp/tst-cu-mcp/tests/test_overlay_gating.py | 17 +- mcp/tst-cu-mcp/tests/test_overlay_rules.py | 103 +++++- mcp/tst-cu-mcp/tests/test_stdio_server.py | 3 +- 35 files changed, 2090 insertions(+), 136 deletions(-) create mode 100644 core/tests/test_cu_agent_cursors.py create mode 100644 core/tstd/cu_agents.py create mode 100644 core/tstd/desktop/ax_cursor.py create mode 100644 mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/agents.py create mode 100644 mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_cursor.py create mode 100644 mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_wire.py create mode 100644 mcp/tst-cu-mcp/tests/test_agent_cursors.py diff --git a/DECISIONS.md b/DECISIONS.md index 7a415319..abaf9a5d 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -19717,3 +19717,90 @@ signing them in `build_sidecar.py` before bundling (the bundler re-signs every `externalBin` anyway, and the files in `shell/binaries/` would stop matching their pins); bumping to agent-desktop 1.1.0, released the same day (the tools were built and tested against 1.0.2). + +## 2026-11-03 — TD-5227: each agent has its own cursor on the real display (Class B; replaces the TD-3402 cursor rule) + +**Decision:** TD-3402 decided the agent cursor is a CSS overlay on the Screen +pane only, never on the real display. That rule is replaced: every +computer-use agent now also has its own drawn cursor on the real display, +with its own color and label. The Screen-pane cursor stays as it was. + +1. **Two renderers, one identity.** + - tst-cu-mcp's overlay helper (macOS) draws one small panel per agent: + an arrow in the agent's color and a pill with its label. It is the same + kind of window as the ring: borderless, click-through, never takes + focus, joins every space, one level above the ring, and + `NSWindowSharingNone`, so no screenshot contains it (it is also ordered + out during a grab). It goes to the agent's last move, click (with a + short ring), or background element action (the acted element's centre, + from the frame the host's `ui_act` now returns). This covers every + tst-cu-mcp client: TST Desk's sessions, Grok's own copy, any other MCP + client. + - agent-desktop's own presentation-only multi-agent cursor draws for the + element tools (`desktop_act` and the rest), which never go through + tst-cu-mcp. TST Desk runs one agent-desktop session per daemon run + (`session start --cursor --multi-agent --no-trace`) and enables each + TST Desk session as an agent with the same label and color. +2. **Who is an agent.** One tst-cu-mcp process, labelled from its MCP + client's `clientInfo.name` ("Claude", "Grok", ...), or one TST Desk + session, labelled "TST Desk · ". The daemon's sessions share + one sidecar, so the driver names the acting session with the internal + `overlay_agent` tool before each pointer action, under one lock; the + session is known from a context variable the session loop sets. +3. **Colors.** A machine-wide slot registry: slot N is a lock file under the + policy folder's `agents/`, held with a non-blocking `flock` by the process + that claimed it, released by the OS when that process exits. Eight + colors, none close to the rust ring; past eight live agents a color + repeats and the label still tells them apart. TST Desk takes each + session's color from the sidecar and hands it to agent-desktop. +4. **What "separate cursor" means in each mode.** The drawn cursor is a + window. No cursor API is called and the hardware pointer is never moved + or replaced by it. + - **background**: agents act through accessibility (tst-cu-mcp's + `ui_action`, agent-desktop's headless actions). Your pointer does not + move at all. Each agent's drawn cursor is the only on-screen sign of + where it is working. No ring (TD-5226). + - **full_control**: the coordinate tools still move the real pointer, + because a synthesized click lands under it; there is one system + pointer and agents share it with you. Here the separate cursor marks + where each agent last acted, in its own color and label, so several + agents can be told apart even though the real pointer only shows + whoever moved it last. agent-desktop hides its own cursor during + physical (`--headed`) input. The ring marks the display being acted on. +5. **Switches.** `cu-indicators.yaml` stays the source of truth. The daemon + mirrors "Show on the real display" and "Agent cursor" into the shared + policy file as `overlay.enabled` and `overlay.cursor`; every tst-cu-mcp + client re-reads them when the file changes, so the switches apply to + clients TST Desk did not start. agent-desktop's cursor follows the same + two switches. +6. **Idle and Stop.** A drawn cursor fades out 10 s after its agent's last + action; agent-desktop uses its own fade. Stop hides every drawn cursor + within a quarter second (the TD-5226 watcher) and disables + agent-desktop's overlay for the whole session, which removes every one of + its cursors. They return on the next action after Resume. +7. **Reduced motion.** The drawn cursor jumps instead of gliding and does + not ring on a click. agent-desktop reads the same system setting itself + (its 1.0.2 source checks `accessibilityDisplayShouldReduceMotion`). +8. **Screenshots.** agent-desktop's cursor windows are ordinary capturable + windows (1.0.2, and upstream head at the time of writing). TST Desk's + capture therefore builds its image from the on-screen windows minus any + owned by `agent-desktop` (`CGWindowListCreateImageFromArray`); with none + on screen it uses the plain capture as before. Screenshots taken by + other apps can still show agent-desktop's cursor until upstream marks + those windows non-shareable. +9. **Platforms.** The drawn cursor is macOS-only for now; the Windows and + Linux painters ignore it. agent-desktop's cursor works wherever its + overlay does. + +**Rejected:** agent-desktop's cursor alone (it follows only its own tools, +which are off by default and are not what Grok, other clients or the +coordinate tools use, and its windows are capturable); the drawn cursor +alone (the element tools act outside tst-cu-mcp, and agent-desktop already +draws there with travel and an element highlight); moving the hardware +pointer to show an agent (it would fight the user and the other agents); +one sidecar per session to tell sessions apart (a process per chat for an +identity the lock-guarded `overlay_agent` call already gives). + +**Known limits:** an agent that alternates between a coordinate tool and an +element tool can show two cursors with its color and label for up to 10 s; +on Windows and Linux only agent-desktop's cursor exists. diff --git a/core/tests/test_cu_agent_cursors.py b/core/tests/test_cu_agent_cursors.py new file mode 100644 index 00000000..1060b68c --- /dev/null +++ b/core/tests/test_cu_agent_cursors.py @@ -0,0 +1,324 @@ +"""Each session is its own computer-use agent with its own cursor (TD-5227). + +* A session's loop runs as that session's agent (a context variable). +* The sidecar is told which agent acts before each pointer action, under + one lock, and its color comes back for agent-desktop. +* agent-desktop runs in one cursor session per daemon run, each agent + enabled once with the same label and color; Stop and the switches turn + every cursor off; shutdown ends the session. +* The display switches are mirrored into the shared policy file. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from tstd.cu_agents import CuAgent, current_cu_agent, fallback_color, set_title_lookup +from tstd.cu_policy import CuPolicy, save_cu_policy, save_overlay_prefs +from tstd.daemon import Daemon +from tstd.desktop.ax_cli import AgentDesktopCli +from tstd.desktop.ax_cursor import AxCursor +from tstd.desktop.mcp_driver import McpDesktopDriver +from tstd.session import Session, SessionRunner + + +@pytest.fixture(autouse=True) +def _no_titles() -> Any: + yield + set_title_lookup(lambda _sid: None) + + +class TestAgentIdentity: + def test_id_is_short_and_safe_for_agent_desktop(self) -> None: + agent = CuAgent("3f2a9c1e-77b0-4d1a-9e0f-123456789abc") + assert agent.agent_id == "tst-3f2a9c1e77b0" + + def test_label_names_the_chat(self) -> None: + set_title_lookup(lambda sid: "Fix the login form" if sid == "s1" else None) + assert CuAgent("s1").label == "TST Desk · Fix the login form" + assert CuAgent("s2").label == "TST Desk" + + def test_a_long_title_is_cut_to_forty(self) -> None: + set_title_lookup(lambda _sid: "word " * 30) + label = CuAgent("s1").label + assert len(label) == 40 + assert label.endswith("…") + + def test_fallback_color_is_stable(self) -> None: + assert fallback_color("tst-abc") == fallback_color("tst-abc") + + +async def test_a_sessions_loop_acts_as_that_session() -> None: + session = Session("/tmp/ws") + seen: list[CuAgent | None] = [] + + async def loop(sess: Session) -> None: + async def tool() -> None: + seen.append(current_cu_agent.get()) + + await asyncio.create_task(tool()) + + runner = SessionRunner(session, loop_factory=loop) + await runner.start() + for _ in range(100): + if seen: + break + await asyncio.sleep(0.01) + assert seen == [CuAgent(session.id)] + assert current_cu_agent.get() is None + + +class FakeClient: + """Records sidecar calls; ``overlay_agent`` answers with a color.""" + + def __init__(self, *, fail: bool = False) -> None: + self.calls: list[tuple[str, dict[str, Any]]] = [] + self.fail = fail + + async def call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> Any: + self.calls.append((name, dict(arguments or {}))) + if name == "overlay_agent": + if self.fail: + raise RuntimeError("sidecar gone") + body = {"agent_id": arguments["agent_id"], "color": "#16A34A"} # type: ignore[index] + return {"content": [{"type": "text", "text": json.dumps(body)}]} + return {"content": [{"type": "text", "text": "{}"}]} + + async def aclose(self) -> None: + return None + + +def _driver(client: FakeClient) -> McpDesktopDriver: + return McpDesktopDriver(["tst-cu-mcp"], platform="darwin", client=client) # type: ignore[arg-type] + + +class TestSidecarAnnounce: + async def test_pointer_actions_name_the_agent_once_per_change(self) -> None: + client = FakeClient() + driver = _driver(client) + a, b = CuAgent("aaaa"), CuAgent("bbbb") + token = current_cu_agent.set(a) + await driver.click(1, 1) + await driver.move(2, 2) + current_cu_agent.set(b) + await driver.scroll(0, -1, x=3, y=3) + current_cu_agent.reset(token) + names = [(name, args.get("agent_id")) for name, args in client.calls] + assert names == [ + ("overlay_agent", "tst-aaaa"), + ("click", None), + ("move_mouse", None), + ("overlay_agent", "tst-bbbb"), + ("scroll", None), + ] + + async def test_no_session_means_no_announcement(self) -> None: + client = FakeClient() + await _driver(client).click(1, 1) + assert [name for name, _ in client.calls] == ["click"] + + async def test_the_color_comes_back_for_agent_desktop(self) -> None: + driver = _driver(FakeClient()) + assert await driver.agent_color(CuAgent("aaaa")) == "#16A34A" + + async def test_a_failed_announcement_still_acts(self) -> None: + client = FakeClient(fail=True) + driver = _driver(client) + token = current_cu_agent.set(CuAgent("aaaa")) + try: + await driver.click(1, 1) + finally: + current_cu_agent.reset(token) + assert [name for name, _ in client.calls] == ["overlay_agent", "click"] + assert await driver.agent_color(CuAgent("aaaa")) is None + + +class FakeCli: + """A ScopedRunner that records argv and answers like agent-desktop.""" + + def __init__(self, *, start_fails: bool = False) -> None: + self.argv: list[list[str]] = [] + self.start_fails = start_fails + + async def run_with(self, prefix: list[str], command: str, *args: str) -> Any: + self.argv.append([*prefix, command, *args]) + if command == "session" and args[:1] == ("start",): + if self.start_fails: + raise RuntimeError("unknown subcommand") + return {"session_id": "run-1"} + return {} + + +class TestAgentDesktopCursor: + def _cursor(self, wanted: list[bool]) -> AxCursor: + async def color_for(agent: CuAgent) -> str | None: + return "#8B5CF6" if agent.agent_id == "tst-aaaa" else None + + return AxCursor(wanted=lambda: wanted[0], color_for=color_for) + + async def test_first_command_starts_a_cursor_session_and_enables_the_agent(self) -> None: + cli, cursor = FakeCli(), self._cursor([True]) + token = current_cu_agent.set(CuAgent("aaaa")) + try: + first = await cursor.scope(cli) + again = await cursor.scope(cli) + finally: + current_cu_agent.reset(token) + assert first == again == ["--session", "run-1", "--agent-id", "tst-aaaa"] + assert cli.argv == [ + ["session", "start", "--cursor", "--multi-agent", "--no-trace", "--name", "tst-desk"], + [ + "--session", + "run-1", + "--agent-id", + "tst-aaaa", + "cursor-overlay", + "enable", + "--label", + "TST Desk", + "--fill", + "#8B5CF6", + ], + ] + + async def test_a_second_agent_gets_its_own_cursor(self) -> None: + cli, cursor = FakeCli(), self._cursor([True]) + for sid in ("aaaa", "bbbb"): + token = current_cu_agent.set(CuAgent(sid)) + await cursor.scope(cli) + current_cu_agent.reset(token) + enables = [argv for argv in cli.argv if "enable" in argv] + assert [argv[3] for argv in enables] == ["tst-aaaa", "tst-bbbb"] + # No color from the sidecar: the stable fallback. + assert enables[1][-1] == fallback_color("tst-bbbb") + + async def test_cursor_off_until_wanted_starts_nothing(self) -> None: + cli, cursor = FakeCli(), self._cursor([False]) + assert await cursor.scope(cli) == [] + assert cli.argv == [] + + async def test_turning_off_removes_every_cursor_and_keeps_the_namespace(self) -> None: + wanted = [True] + cli, cursor = FakeCli(), self._cursor(wanted) + await cursor.scope(cli) + wanted[0] = False + prefix = await cursor.scope(cli) + await cursor.scope(cli) + assert prefix[:2] == ["--session", "run-1"] + disables = [argv for argv in cli.argv if "disable" in argv] + assert disables == [["--session", "run-1", "cursor-overlay", "disable"]] + wanted[0] = True + await cursor.scope(cli) + assert cli.argv[-2] == [ + "--session", + "run-1", + "cursor-overlay", + "enable", + "--multi-agent", + ] + + async def test_stop_turns_cursors_off_now_and_shutdown_ends_the_session(self) -> None: + cli, cursor = FakeCli(), self._cursor([True]) + await cursor.scope(cli) + await cursor.off(cli) + await cursor.off(cli) + await cursor.end(cli) + assert cli.argv[-2:] == [ + ["--session", "run-1", "cursor-overlay", "disable"], + ["session", "end", "run-1"], + ] + + async def test_a_failed_start_runs_without_a_cursor_and_does_not_retry(self) -> None: + cli, cursor = FakeCli(start_fails=True), self._cursor([True]) + assert await cursor.scope(cli) == [] + assert await cursor.scope(cli) == [] + assert len(cli.argv) == 1 + + async def test_the_cli_puts_the_scope_before_the_command(self, tmp_path: Path) -> None: + binary = tmp_path / "agent-desktop" + binary.write_bytes(b"x") + seen: list[list[str]] = [] + + async def runner(argv: list[str], env: Any, limit: float) -> tuple[int, bytes, bytes]: + seen.append(argv) + if argv[1:3] == ["session", "start"]: + return 0, b'{"ok": true, "data": {"session_id": "run-9"}}', b"" + return 0, b'{"ok": true, "data": {}}', b"" + + cli = AgentDesktopCli(binary, sha256="", runner=runner, packaged=True) + cli.cursor = self._cursor([True]) + await cli.run("list-windows", "--app", "Finder") + assert seen[-1][1:] == [ + "--session", + "run-9", + "--agent-id", + "tst-desk", + "list-windows", + "--app", + "Finder", + ] + + +class TestSharedSwitches: + def test_overlay_prefs_keep_every_other_key(self, tmp_path: Path) -> None: + path = tmp_path / "config.yaml" + save_cu_policy(CuPolicy(mode="full_control", denied_apps=("Mail",)), path) + assert save_overlay_prefs(enabled=True, cursor=False, path=path) is True + body = yaml.safe_load(path.read_text(encoding="utf-8")) + assert body["overlay"] == {"enabled": True, "cursor": False} + assert body["mode"] == "full_control" + assert body["scoping"]["denied_apps"] == ["Mail"] + # Unchanged switches do not rewrite the file. + assert save_overlay_prefs(enabled=True, cursor=False, path=path) is False + # Settings → Computer use keeps the cursor switch. + save_cu_policy(CuPolicy(mode="background"), path) + assert yaml.safe_load(path.read_text(encoding="utf-8"))["overlay"]["cursor"] is False + + def test_an_unreadable_file_is_left_alone(self, tmp_path: Path) -> None: + path = tmp_path / "config.yaml" + path.write_text("- not a mapping\n", encoding="utf-8") + assert save_overlay_prefs(enabled=False, cursor=False, path=path) is False + assert path.read_text(encoding="utf-8") == "- not a mapping\n" + + async def test_settings_mirror_the_switches_and_stop_clears_agent_desktop( + self, tmp_path: Path + ) -> None: + daemon = Daemon(data_dir=tmp_path) + + class Ax: + offs = 0 + + async def cursor_off(self) -> None: + Ax.offs += 1 + + def set_killed(self, killed: bool) -> None: + return None + + async def aclose(self) -> None: + return None + + daemon.ax_driver = Ax() # type: ignore[assignment] + raw = await daemon._handle_message( + json.dumps( + { + "type": "set_cu_indicators", + "glow": True, + "agent_cursor": False, + "show_on_real_display": True, + } + ), + None, + ) + assert raw is not None + shared = yaml.safe_load((Path.home() / ".tst-cu-mcp" / "config.yaml").read_text()) + assert shared["overlay"] == {"enabled": True, "cursor": False} + assert Ax.offs == 1 + await daemon._handle_message(json.dumps({"type": "set_cu_kill", "killed": True}), None) + assert Ax.offs == 2 + await daemon._shutdown() diff --git a/core/tstd/cu_agents.py b/core/tstd/cu_agents.py new file mode 100644 index 00000000..c6d46b01 --- /dev/null +++ b/core/tstd/cu_agents.py @@ -0,0 +1,80 @@ +"""Which TST Desk session is acting, for each agent's own cursor (TD-5227). + +Every session's computer use draws its own cursor on the real display, with +its own color and a label naming the chat. The session loop sets +:data:`current_cu_agent` once when it starts; asyncio copies context into +every task the loop spawns, so the drivers read the acting session without +any change to the tool signatures. + +Colors are assigned by tst-cu-mcp's machine-wide registry (the sidecar +answers ``overlay_agent`` with one), so a TST Desk session never shares a +color with another live client. :func:`fallback_color` covers drivers with +no sidecar. +""" + +from __future__ import annotations + +from collections.abc import Callable +from contextvars import ContextVar +from dataclasses import dataclass + +#: Mirrors tst_cu_mcp.overlay.agents.PALETTE without a cross-package import. +PALETTE = ( + "#2F6FDE", + "#16A34A", + "#8B5CF6", + "#DB2777", + "#0891B2", + "#4F46E5", + "#65A30D", + "#475569", +) +_LABEL_MAX = 40 +_PREFIX = "TST Desk" + +_title_lookup: Callable[[str], str | None] = lambda _sid: None # noqa: E731 + + +@dataclass(frozen=True) +class CuAgent: + """One session as a computer-use agent.""" + + session_id: str + + @property + def agent_id(self) -> str: + """Stable, short, and safe for agent-desktop's ``--agent-id``.""" + return "tst-" + "".join(ch for ch in self.session_id if ch.isalnum())[:12] + + @property + def label(self) -> str: + """``TST Desk · ``, at most 40 characters.""" + title = " ".join((_title_lookup(self.session_id) or "").split()) + if not title: + return _PREFIX + text = f"{_PREFIX} · {title}" + return text if len(text) <= _LABEL_MAX else text[: _LABEL_MAX - 1].rstrip() + "…" + + +#: The session whose loop is running in this context, or None (no session). +current_cu_agent: ContextVar[CuAgent | None] = ContextVar("current_cu_agent", default=None) + + +def set_title_lookup(lookup: Callable[[str], str | None]) -> None: + """The daemon's session store answers a session id with its chat title.""" + global _title_lookup + _title_lookup = lookup + + +def fallback_color(agent_id: str) -> str: + """A stable palette color when no sidecar assigns one.""" + return PALETTE[sum(agent_id.encode("utf-8")) % len(PALETTE)] + + +__all__ = [ + "PALETTE", + "CuAgent", + "current_cu_agent", + "fallback_color", + "set_title_lookup", +] diff --git a/core/tstd/cu_policy.py b/core/tstd/cu_policy.py index 46c72bca..2f513461 100644 --- a/core/tstd/cu_policy.py +++ b/core/tstd/cu_policy.py @@ -79,7 +79,10 @@ def save_cu_policy(policy: CuPolicy, path: Path | None = None) -> None: killswitch = _as_map(existing.get("killswitch")) body: dict[str, Any] = { "actuation": {"enabled": policy.enabled}, - "overlay": {"enabled": overlay.get("enabled", True)}, + "overlay": { + "enabled": overlay.get("enabled", True), + "cursor": overlay.get("cursor", True), + }, "mode": policy.mode, "scoping": { "allowed_apps": list(policy.allowed_apps), @@ -100,6 +103,41 @@ def save_cu_policy(policy: CuPolicy, path: Path | None = None) -> None: raise +def save_overlay_prefs(*, enabled: bool, cursor: bool, path: Path | None = None) -> bool: + """Mirror TST Desk's display switches into the shared policy file (TD-5227). + + ``overlay.enabled`` is "Show on the real display" and ``overlay.cursor`` + is "Agent cursor". Every tst-cu-mcp client re-reads them when the file + changes, so the switches apply to all of them, not only TST Desk's own. + Every other key is kept as it is. Returns True when the file changed. + """ + target = path if path is not None else cu_policy_path() + existing: dict[str, Any] = {} + if target.exists(): + try: + loaded = yaml.safe_load(target.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return False + if loaded is not None and not isinstance(loaded, dict): + return False + existing = loaded or {} + overlay = _as_map(existing.get("overlay")) + if overlay.get("enabled", True) is enabled and overlay.get("cursor", True) is cursor: + return False + existing["overlay"] = {**overlay, "enabled": enabled, "cursor": cursor} + target.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=target.parent, prefix=".config.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + yaml.safe_dump(existing, handle, sort_keys=False) + Path(tmp).replace(target) + except Exception: + with suppress(OSError): + Path(tmp).unlink() + raise + return True + + def _as_map(value: object) -> dict[str, Any]: return value if isinstance(value, dict) else {} diff --git a/core/tstd/daemon.py b/core/tstd/daemon.py index 28ce27d2..9b802320 100644 --- a/core/tstd/daemon.py +++ b/core/tstd/daemon.py @@ -111,13 +111,14 @@ from .context.stack import build_instruction_stack from .context_pins import PinOutsideError, add_pin, list_pin_cards, project_capacity, remove_pin from .coworker import load_coworker, save_coworker +from .cu_agents import set_title_lookup from .cu_indicators import ( CuIndicatorPrefs, load_cu_indicators, save_cu_indicators, set_current_prefs, ) -from .cu_policy import CuPolicy, load_cu_policy, save_cu_policy +from .cu_policy import CuPolicy, load_cu_policy, save_cu_policy, save_overlay_prefs from .data_dir_lock import acquire_data_dir_lock from .desktop import ( AccessibilityDriver, @@ -125,6 +126,7 @@ accessibility_driver_from_config, desktop_driver_from_config, ) +from .desktop.ax_cursor import AxCursor from .desktop.grounding_client import GroundingClient from .desktop.host_probe import host_permissions from .desktop.permissions import ( @@ -1138,6 +1140,7 @@ def __init__( self._tasks: list[asyncio.Task[Any]] = [] self.session_registry = SessionRegistry() self._session_store = SessionStore(self.data_dir) + set_title_lookup(self._session_title) # The path is this daemon's file, never the no-arg library cache. # A zero-arg test double still injects; a real loader is not caught # and retried against the default path (TD-4843). @@ -1191,6 +1194,7 @@ def __init__( self.remote_attach_enabled, self.remote_attach_last_bind = load_remote_attach(self.data_dir) self.cu_indicators = load_cu_indicators(self.data_dir) set_current_prefs(self.cu_indicators) + self._mirror_overlay_prefs() self.cu_policy = load_cu_policy() self.workspace_pins = load_workspace_pins(self.data_dir) self.session_stars = load_session_stars(self.data_dir) @@ -1245,6 +1249,14 @@ def __init__( self.ax_driver: AccessibilityDriver | None = accessibility_driver_from_config( self.config, self.data_dir ) + if self.ax_driver is not None: + # agent-desktop draws each session's cursor for these tools, in + # the color the sidecar gave that session (TD-5227). + self.ax_driver.attach_cursor( + AxCursor( + wanted=self._agent_cursor_wanted, color_for=self.desktop_driver.agent_color + ) + ) # Browser CU (TD-1710): mock unless computer_use.browser is playwright # and Playwright is importable. Profile lives under the data dir. self.browser_driver: BrowserDriver = browser_driver_from_config(self.config, self.data_dir) @@ -1295,6 +1307,26 @@ def set_computer_use_killed(self, killed: bool, *, write_file: bool = True) -> N return log.info("computer-use stop-file %s: %s", "created" if killed else "removed", path) + def _agent_cursor_wanted(self) -> bool: + """Each agent's real-display cursor: both switches on and not stopped.""" + prefs = self.cu_indicators + return prefs.agent_cursor and prefs.show_on_real_display and not self._cu_killed + + def _mirror_overlay_prefs(self) -> None: + """Hand the display switches to every tst-cu-mcp client (TD-5227).""" + prefs = self.cu_indicators + try: + save_overlay_prefs(enabled=prefs.show_on_real_display, cursor=prefs.agent_cursor) + except OSError as exc: + log.warning("computer-use display switches not shared: %s", exc) + + def _session_title(self, session_id: str) -> str | None: + """A chat's title for its agent cursor label.""" + for record in self._session_store.records(): + if record.session_id == session_id: + return record.title or record.auto_title + return None + async def _apply_cu_kill(self, killed: bool, *, write_file: bool) -> None: """Set the switch and close what an engaged switch closes.""" self.set_computer_use_killed(killed, write_file=write_file) @@ -1302,6 +1334,8 @@ async def _apply_cu_kill(self, killed: bool, *, write_file: bool) -> None: for sess in await self.session_registry.list_sessions(): await sess.close_cu_session() await self.desktop_driver.set_overlay_session(False) + if self.ax_driver is not None: + await self.ax_driver.cursor_off() async def _cu_stop_watch(self, interval: float = CU_STOP_POLL_SECONDS) -> None: """Follow a stop-file created or removed outside TST Desk (TD-5223). @@ -2634,6 +2668,8 @@ async def _shutdown_subsystems(self, phases: ShutdownPhases) -> None: await phases.run("session_index", asyncio.to_thread(self._close_session_index)) await phases.run("desktop", self.desktop_driver.aclose()) + if self.ax_driver is not None: + await phases.run("accessibility", self.ax_driver.aclose()) await phases.run("browser", self.browser_driver.aclose()) await phases.run("mcp", self._mcp.aclose()) # TD-5139: language servers are the daemon's children. @@ -3909,6 +3945,9 @@ async def send(reply: str) -> None: ) save_cu_indicators(self.data_dir, self.cu_indicators) set_current_prefs(self.cu_indicators) + await asyncio.to_thread(self._mirror_overlay_prefs) + if self.ax_driver is not None and not self._agent_cursor_wanted(): + await self.ax_driver.cursor_off() return (await self._setup_state_event()).model_dump_json() if isinstance(msg, SetCuPolicy): diff --git a/core/tstd/desktop/ax_cli.py b/core/tstd/desktop/ax_cli.py index 157d850f..c807701d 100644 --- a/core/tstd/desktop/ax_cli.py +++ b/core/tstd/desktop/ax_cli.py @@ -18,7 +18,7 @@ from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from .protocol import DesktopError @@ -154,6 +154,10 @@ async def _run_process( return proc.returncode if proc.returncode is not None else -1, out, err +if TYPE_CHECKING: + from .ax_cursor import AxCursor + + @dataclass(frozen=True) class CliFailure: """A parsed ``ok: false`` envelope.""" @@ -247,6 +251,8 @@ def __init__( self._env = child_env(os.environ if base_env is None else base_env, state_dir) # (inode, size, mtime) of the file last hashed: a changed file is hashed again. self._verified: tuple[int, int, int] | None = None + # agent-desktop's per-agent cursor (TD-5227); None runs without it. + self.cursor: AxCursor | None = None @property def env(self) -> Mapping[str, str]: @@ -286,9 +292,14 @@ async def _verify(self) -> Path: return binary async def run(self, command: str, *args: str) -> Any: - """Run one command and return its ``data``. Refusals raise ``DesktopError``.""" + """Run one command as the acting session's agent. Refusals raise ``DesktopError``.""" + prefix = await self.cursor.scope(self) if self.cursor is not None else [] + return await self.run_with(prefix, command, *args) + + async def run_with(self, prefix: list[str], command: str, *args: str) -> Any: + """Run one command after global flags (``--session``, ``--agent-id``).""" binary = await self._verify() - argv = [str(binary), command, *args] + argv = [str(binary), *prefix, command, *args] result = await self._once(argv) if isinstance(result, CliFailure): if not result.may_retry(command): diff --git a/core/tstd/desktop/ax_cursor.py b/core/tstd/desktop/ax_cursor.py new file mode 100644 index 00000000..c0aa4927 --- /dev/null +++ b/core/tstd/desktop/ax_cursor.py @@ -0,0 +1,145 @@ +"""agent-desktop's own per-agent cursor for the accessibility tools (TD-5227). + +The ``desktop_*`` element tools act through agent-desktop without moving the +pointer. agent-desktop draws a presentation-only cursor for its actions, one +per agent when its session runs with ``--multi-agent``. TST Desk uses that +renderer for these tools and gives each session the same label and color as +its tst-cu-mcp cursor, so an agent looks the same whichever path it acts on. + +How it is wired: + +* One agent-desktop session per daemon run, started on the first command + while the cursor is wanted (``session start --cursor --multi-agent + --no-trace``). From then on every command runs in that session's + namespace with ``--session`` and ``--agent-id``, cursor on or off, so a + ref from a snapshot always resolves in the same namespace. +* Each session's agent is enabled once with its label and fill color. +* The cursor is wanted while "Agent cursor" and "Show on the real display" + are both on and computer use is not stopped. Turning it off (Stop + included) disables the overlay for the whole session, which removes every + cursor; it comes back on the next command once wanted again. +* No session is ever started while the cursor is off: the tools then run + exactly as before. + +A failure here never fails a tool: the command runs without the cursor. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any, Protocol + +from ..cu_agents import CuAgent, current_cu_agent, fallback_color +from ..logging import get_logger + +log = get_logger("tstd.desktop.ax_cursor") + +#: The agent id used when no session is acting (a tool called outside a loop). +DEFAULT_AGENT = CuAgent("desk") + + +class ScopedRunner(Protocol): + async def run_with(self, prefix: list[str], command: str, *args: str) -> Any: ... + + +class AxCursor: + """Starts, scopes and switches agent-desktop's multi-agent cursor.""" + + def __init__( + self, + *, + wanted: Callable[[], bool], + color_for: Callable[[CuAgent], Awaitable[str | None]], + ) -> None: + self._wanted = wanted + self._color_for = color_for + self._session: str | None = None + self._enabled: set[str] = set() + self._overlay_on = False + self._failed = False + + @property + def session_id(self) -> str | None: + return self._session + + async def scope(self, cli: ScopedRunner) -> list[str]: + """Global flags for the next command; turns the cursor on or off first.""" + agent = current_cu_agent.get() or DEFAULT_AGENT + wanted = self._wanted() + if self._session is None: + if not wanted or self._failed: + return [] + await self._start(cli) + if self._session is None: + return [] + prefix = ["--session", self._session, "--agent-id", agent.agent_id] + if not wanted: + await self.off(cli) + return prefix + try: + if not self._overlay_on: + await cli.run_with( + ["--session", self._session], "cursor-overlay", "enable", "--multi-agent" + ) + self._overlay_on = True + self._enabled.clear() + if agent.agent_id not in self._enabled: + color = await self._color_for(agent) or fallback_color(agent.agent_id) + await cli.run_with( + prefix, "cursor-overlay", "enable", "--label", agent.label, "--fill", color + ) + self._enabled.add(agent.agent_id) + except Exception as exc: + log.warning("agent-desktop cursor not enabled: %s", exc) + return prefix + + async def off(self, cli: ScopedRunner) -> None: + """Remove every agent's cursor (Stop, or a switch turned off).""" + if self._session is None or not self._overlay_on: + return + try: + await cli.run_with(["--session", self._session], "cursor-overlay", "disable") + except Exception as exc: + log.warning("agent-desktop cursor not disabled: %s", exc) + self._overlay_on = False + self._enabled.clear() + + async def end(self, cli: ScopedRunner) -> None: + """End the session at shutdown; its cursors go with it.""" + session, self._session = self._session, None + self._overlay_on = False + self._enabled.clear() + if session is None: + return + try: + await cli.run_with([], "session", "end", session) + except Exception as exc: + log.warning("agent-desktop session not ended: %s", exc) + + async def _start(self, cli: ScopedRunner) -> None: + try: + data = await cli.run_with( + [], + "session", + "start", + "--cursor", + "--multi-agent", + "--no-trace", + "--name", + "tst-desk", + ) + except Exception as exc: + # An older or odd binary: run the tools without a cursor rather + # than retrying a failing start on every command. + self._failed = True + log.warning("agent-desktop cursor session not started: %s", exc) + return + session = data.get("session_id") if isinstance(data, dict) else None + if isinstance(session, str) and session: + self._session = session + self._overlay_on = True + else: + self._failed = True + + +__all__ = ["DEFAULT_AGENT", "AxCursor"] diff --git a/core/tstd/desktop/ax_driver.py b/core/tstd/desktop/ax_driver.py index 1c67eed8..6601506d 100644 --- a/core/tstd/desktop/ax_driver.py +++ b/core/tstd/desktop/ax_driver.py @@ -18,13 +18,16 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from ..cu_policy import CuPolicy, load_cu_policy from . import ax_tree from .ax_cli import AgentDesktopCli from .protocol import DesktopError +if TYPE_CHECKING: + from .ax_cursor import AxCursor + #: desktop_act action → CLI command, and whether it takes a value. ACTIONS: dict[str, tuple[str, bool]] = { "click": ("click", False), @@ -97,8 +100,19 @@ def __init__( def set_killed(self, killed: bool) -> None: self.killed = bool(killed) + def attach_cursor(self, cursor: AxCursor) -> None: + """Draw agent-desktop's per-agent cursor for these tools (TD-5227).""" + self._cli.cursor = cursor + + async def cursor_off(self) -> None: + """Remove every agent's cursor now (Stop, or a switch turned off).""" + if self._cli.cursor is not None: + await self._cli.cursor.off(self._cli) + async def aclose(self) -> None: - """Nothing to reap: each command is its own short-lived process.""" + """End the cursor session, if one was started; commands leave nothing else.""" + if self._cli.cursor is not None: + await self._cli.cursor.end(self._cli) # -- policy ----------------------------------------------------------- diff --git a/core/tstd/desktop/inprocess.py b/core/tstd/desktop/inprocess.py index 8e7e07ac..fdab2925 100644 --- a/core/tstd/desktop/inprocess.py +++ b/core/tstd/desktop/inprocess.py @@ -14,6 +14,7 @@ import os from typing import Any +from ..cu_agents import CuAgent from .protocol import DesktopError _STOP_ENV = "TST_CU_MCP_STOP" @@ -50,6 +51,14 @@ def set_killed(self, killed: bool) -> None: else: os.environ.pop(_STOP_ENV, None) + async def agent_color(self, agent: CuAgent) -> str | None: + try: + from tst_cu_mcp.overlay import get_overlay + + return str(get_overlay().use_agent(agent.agent_id, agent.label).color) + except Exception: + return None + async def set_overlay_session(self, active: bool, session: str = "") -> None: try: from tst_cu_mcp.overlay import get_overlay diff --git a/core/tstd/desktop/mcp_driver.py b/core/tstd/desktop/mcp_driver.py index f6e1b6e0..ecd3afc5 100644 --- a/core/tstd/desktop/mcp_driver.py +++ b/core/tstd/desktop/mcp_driver.py @@ -9,13 +9,16 @@ import asyncio import json -from typing import Any +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar +from ..cu_agents import CuAgent, current_cu_agent from .permissions import parse_mcp_permissions_result from .protocol import DesktopError from .stdio_mcp import StdioMcpClient, map_mcp_error LIVE_PLATFORMS = frozenset({"darwin", "win32", "linux"}) +_T = TypeVar("_T") # Product tools → existing MCP tool names. Points, not last-screenshot # image space: tstd does not yet thread capture metadata between calls. @@ -46,10 +49,48 @@ def __init__( self.platform = self._platform self._client = client if client is not None else StdioMcpClient(command, env=env) self.killed = False + # TD-5227: which session's cursor the sidecar draws next. + self._agent_lock = asyncio.Lock() + self._announced: tuple[str, str] | None = None + self._colors: dict[str, str] = {} def set_killed(self, killed: bool) -> None: self.killed = bool(killed) + async def agent_color(self, agent: CuAgent) -> str | None: + """Name ``agent`` to the sidecar; its cursor color, or None if unknown.""" + async with self._agent_lock: + return await self._announce(agent) + + async def _announce(self, agent: CuAgent) -> str | None: + label = agent.label + if self._announced == (agent.agent_id, label): + return self._colors.get(agent.agent_id) + try: + result = await self._client.call_tool( + "overlay_agent", {"agent_id": agent.agent_id, "label": label} + ) + color = _agent_color(result) + except Exception: + return None + self._announced = (agent.agent_id, label) + if color is not None: + self._colors[agent.agent_id] = color + return color + + async def _as_agent(self, act: Callable[[], Awaitable[_T]]) -> _T: + """Run one pointer action as the acting session (TD-5227). + + The sidecar draws the cursor of whichever agent it was last told + about, so naming the agent and acting happen under one lock: two + sessions acting at once cannot swap each other's cursors. + """ + agent = current_cu_agent.get() + async with self._agent_lock: + if agent is not None: + await self._announce(agent) + return await act() + async def set_overlay_session(self, active: bool, session: str = "") -> None: """Tell the sidecar one session's computer-use episode opened or closed.""" try: @@ -95,9 +136,8 @@ async def screenshot(self, display: int | None = None) -> str: async def move(self, x: float, y: float, expect_window: str | None = None) -> str: self._refuse_if_no_live_path() self._refuse_if_killed() - result = await self._client.call_tool( - MCP_TOOLS["move"], - _point_args(x, y, expect_window), + result = await self._as_agent( + lambda: self._client.call_tool(MCP_TOOLS["move"], _point_args(x, y, expect_window)) ) return json.dumps({"moved_to": {"x": x, "y": y}, "sidecar": result}, default=str) @@ -114,7 +154,7 @@ async def click( args = _point_args(x, y, expect_window) args["button"] = button args["count"] = count - result = await self._client.call_tool(MCP_TOOLS["click"], args) + result = await self._as_agent(lambda: self._client.call_tool(MCP_TOOLS["click"], args)) return json.dumps( {"clicked": {"x": x, "y": y}, "button": button, "count": count, "sidecar": result}, default=str, @@ -149,7 +189,7 @@ async def scroll( args["y"] = y if expect_window is not None: args["expect_window"] = expect_window - await self._client.call_tool(MCP_TOOLS["scroll"], args) + await self._as_agent(lambda: self._client.call_tool(MCP_TOOLS["scroll"], args)) return json.dumps({"scrolled": {"dx": dx, "dy": dy}}) async def foreground_window(self) -> dict[str, Any]: @@ -175,6 +215,22 @@ async def aclose(self) -> None: await self._client.aclose() +def _agent_color(result: Any) -> str | None: + """The ``color`` from an ``overlay_agent`` reply (a JSON text block).""" + for block in _content_blocks(result): + text = block.get("text") if isinstance(block, dict) else getattr(block, "text", None) + if not isinstance(text, str): + continue + try: + body = json.loads(text) + except json.JSONDecodeError: + continue + color = body.get("color") if isinstance(body, dict) else None + if isinstance(color, str) and len(color) == 7 and color.startswith("#"): + return color + return None + + def _point_args(x: float, y: float, expect_window: str | None) -> dict[str, Any]: args: dict[str, Any] = {"x": x, "y": y, "coordinate_space": "points"} if expect_window is not None: diff --git a/core/tstd/desktop/mock.py b/core/tstd/desktop/mock.py index 8f6406f3..9ea0cff6 100644 --- a/core/tstd/desktop/mock.py +++ b/core/tstd/desktop/mock.py @@ -11,6 +11,7 @@ import json from typing import Any +from ..cu_agents import CuAgent, fallback_color from .permissions import normalize_cu_platform, windows_report from .permissions_linux import linux_report from .protocol import ( @@ -60,6 +61,10 @@ def __init__( def set_killed(self, killed: bool) -> None: self.killed = bool(killed) + async def agent_color(self, agent: CuAgent) -> str | None: + self.calls.append(("overlay_agent", {"agent_id": agent.agent_id, "label": agent.label})) + return fallback_color(agent.agent_id) + async def set_overlay_session(self, active: bool, session: str = "") -> None: self.calls.append(("overlay_session", {"active": bool(active), "session": session})) diff --git a/core/tstd/desktop/protocol.py b/core/tstd/desktop/protocol.py index 996117a4..5bf9d656 100644 --- a/core/tstd/desktop/protocol.py +++ b/core/tstd/desktop/protocol.py @@ -8,7 +8,10 @@ from __future__ import annotations import base64 -from typing import Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from ..cu_agents import CuAgent # 1x1 PNG so CI can exercise screenshot without a display. TINY_PNG_B64 = ( @@ -82,6 +85,9 @@ def set_killed(self, killed: bool) -> None: async def set_overlay_session(self, active: bool, session: str = "") -> None: """Open or close one session's real-display ring episode (TD-3407, TD-5226).""" + async def agent_color(self, agent: CuAgent) -> str | None: + """Name a session's agent; its cursor color, or None (TD-5227).""" + async def screenshot(self, display: int | None = None) -> str: """Return a JSON object with ``png_base64`` (and size if known).""" diff --git a/core/tstd/local_worker.py b/core/tstd/local_worker.py index 113758f2..c46c5d64 100644 --- a/core/tstd/local_worker.py +++ b/core/tstd/local_worker.py @@ -22,7 +22,7 @@ #: is registered as ``computer-use`` (``grok_home.computer_use_mcp``). GROK_CU_PREFIX = "computer-use__" #: Diagnostics that neither look at nor touch the desktop. -_GROK_CU_META = frozenset({"check_permissions", "health", "overlay_session"}) +_GROK_CU_META = frozenset({"check_permissions", "health", "overlay_session", "overlay_agent"}) CuSurface = Literal["desktop", "browser"] diff --git a/core/tstd/session.py b/core/tstd/session.py index c287a48f..1f720fac 100644 --- a/core/tstd/session.py +++ b/core/tstd/session.py @@ -1698,6 +1698,10 @@ async def start(self) -> None: async def _run(self) -> None: """Run the session loop with lifecycle management.""" + from .cu_agents import CuAgent, current_cu_agent + + # Every tool this loop runs acts as this session's agent (TD-5227). + current_cu_agent.set(CuAgent(self.session.id)) try: await self.session.set_state("running") await self._loop_factory(self.session) diff --git a/docs/configuration.md b/docs/configuration.md index 173a5a9e..64b385da 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -3313,7 +3313,18 @@ proposals for the run summary and never accepts them either. The classifier stil **Computer-use glow**, **Agent cursor**, and **Show indicators on the real display** are also not keys in this file. Settings → Appearance persists them as `cu-indicators.yaml` in the user data dir (glow and cursor default on; the real-display ring defaults on). Glow and -the agent cursor are drawn on the Screen pane. The rust ring on the real display +the agent cursor are drawn on the Screen pane. With both **Agent cursor** and **Show +indicators on the real display** on, each computer-use agent also gets its own cursor on +the real display (TD-5227): an arrow and a label in that agent's color ("TST Desk · " for a TST Desk session, the client's name for Grok or another assistant). It goes +to the agent's last move, click or acted-on control and fades 10 seconds after its last +action. It is a drawn window, left out of screenshots; it never moves your pointer. In +`background` mode your pointer stays still and this cursor shows where each agent works; +in `full_control` the real pointer still moves for each click, and the drawn cursors keep +the agents apart. The element tools (`desktop_act` and the rest) show agent-desktop's own +cursor in the same color and label. On macOS only, apart from agent-desktop's. Stop hides +every one. TST Desk copies the two switches into `~/.tst-cu-mcp/config.yaml` as +`overlay.enabled` and `overlay.cursor`, so other tst-cu-mcp clients follow them too. The rust ring on the real display means "an agent has the real screen", so it shows only in `full_control` mode (the `mode` in `~/.tst-cu-mcp/config.yaml`, re-read when it changes); in `background` mode it stays off on every display (TD-5226). On macOS it lights only diff --git a/docs/tst-desk-backlog.md b/docs/tst-desk-backlog.md index 5fabc8a9..8a282f62 100644 --- a/docs/tst-desk-backlog.md +++ b/docs/tst-desk-backlog.md @@ -13042,6 +13042,33 @@ step 1 and the Mac app since TD-5228; on Windows the binary comes from --- +### TD-5227 — Each agent has its own cursor on the real display +**Size:** 5 · **Depends on:** TD-3402, TD-5220, TD-5226 +**Status:** In review 2026-11-03. + +An agent doing computer use had no cursor of its own on the real display. In +full control it moved the user's pointer, and with several agents at once +nothing showed which agent was where. TD-3402 had kept the agent cursor on the +Screen pane only. agent-desktop (TD-5220) already draws a presentation-only +cursor per agent, but only for its own tools, which are off by default and not +what the coordinate tools, Grok or other assistants use, and its windows show +up in screenshots. This story combines the two with one identity per agent. + +**Acceptance criteria:** +- [x] DECISIONS.md replaces the TD-3402 cursor rule and says what "separate cursor" means in background and in full control +- [x] Each tst-cu-mcp client draws its own cursor on macOS (arrow and label in its color) at its last move, click or acted-on element, in a click-through, non-shareable window; no cursor API is called and the hardware pointer is never moved for it +- [x] Each TST Desk session is its own agent ("TST Desk · "), named to the shared sidecar before each pointer action under one lock +- [x] Colors come from a machine-wide registry, so live clients do not share one while colors remain; a client that exits frees its color +- [x] The element tools run in one agent-desktop session with its multi-agent cursor, each session enabled with the same label and color +- [x] Both switches in `cu-indicators.yaml` apply, live, to every tst-cu-mcp client through the shared policy file, and to agent-desktop's cursor +- [x] Reduced motion: no glide and no click ring; agent-desktop reads the system setting itself +- [x] Stop hides every drawn cursor within a quarter second and disables agent-desktop's overlay; a cursor fades 10 s after its agent's last action +- [x] TST Desk's own screenshots leave agent-desktop's cursor windows out +- [x] Tests: identity, colors across processes, the cursor model, the wire, the daemon's announce, the agent-desktop session and the shared switches, each shown to fail with its piece removed +- [ ] Live on a Mac: two agents at once show two distinct cursors; the pointer stays still in background mode; screenshots contain neither the ring nor any cursor; Stop clears everything + +--- + ### TD-5228 — The Mac app ships agent-desktop **Size:** 3 · **Depends on:** TD-5221 **Status:** In review 2026-11-03. The live check runs separately against a signed build. diff --git a/mcp/tst-cu-mcp/pyproject.toml b/mcp/tst-cu-mcp/pyproject.toml index 342a57e5..80a4602f 100644 --- a/mcp/tst-cu-mcp/pyproject.toml +++ b/mcp/tst-cu-mcp/pyproject.toml @@ -81,7 +81,7 @@ ignore_missing_imports = true # bases are Any, which strict mode refuses. It only ever runs inside the helper # child process, so the loosening stays contained there. [[tool.mypy.overrides]] -module = ["tst_cu_mcp.overlay.darwin_helper"] +module = ["tst_cu_mcp.overlay.darwin_helper", "tst_cu_mcp.overlay.darwin_cursor"] disallow_subclassing_any = false # The Windows backend names ctypes attributes (WinDLL, WINFUNCTYPE, ...) that diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/config.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/config.py index 2dd5de85..0f2bcb16 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/config.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/config.py @@ -26,6 +26,9 @@ class Config: stop_file: str | None = None # Real-display glow; the TST_CU_MCP_OVERLAY env var overrides this. overlay_enabled: bool = True + # Each agent's drawn cursor on the real display (TD-5227). Needs the + # overlay too; TST Desk writes both from its Settings switches. + cursor_enabled: bool = True # Empty allowlist means every app except denied. Non-empty is an allowlist. allowed_apps: tuple[str, ...] = () denied_apps: tuple[str, ...] = () @@ -82,7 +85,7 @@ def save_config(cfg: Config, path: Path | None = None) -> None: target.parent.mkdir(parents=True, exist_ok=True) body: dict[str, Any] = { "actuation": {"enabled": cfg.actuation_enabled}, - "overlay": {"enabled": cfg.overlay_enabled}, + "overlay": {"enabled": cfg.overlay_enabled, "cursor": cfg.cursor_enabled}, "mode": cfg.mode, "scoping": { "allowed_apps": list(cfg.allowed_apps), @@ -123,6 +126,7 @@ def load_config(path: Path | None = None) -> Config: actuation_enabled=_actuation_flag(actuation.get("enabled", True), target), stop_file=killswitch.get("stop_file"), overlay_enabled=bool(overlay.get("enabled", True)), + cursor_enabled=overlay.get("cursor", True) is not False, allowed_apps=_string_tuple(scoping.get("allowed_apps")), denied_apps=_string_tuple(scoping.get("denied_apps")), mode=_mode_flag(raw.get("mode") or scoping.get("mode") or "background", target), diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py index 54ea5fa5..3a57d85f 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py @@ -16,7 +16,11 @@ :func:`tst_cu_mcp.overlay.get_overlay` so the real-display glow tracks the attempt — including attempts that then fail validation, which are still the agent driving. A pointer action first aims the glow at the display under its -point. A refusal from the backend itself — the OS said no, as with a +point, and once it is valid and in front, moves the acting agent's own drawn +cursor there (TD-5227) before the backend acts. That cursor is a window; the +hardware pointer moves only because the backend's act moves it. + +A refusal from the backend itself — the OS said no, as with a missing Accessibility grant — puts the glow out again (:func:`tst_cu_mcp.safety.notify_blocked`): nothing was driven. @@ -132,6 +136,7 @@ def move_mouse(x: float, y: float, expect_window: str | None = None) -> None: get_overlay().activity() assert_on_screen(x, y) assert_foreground(expect_window) + get_overlay().pointer(x, y) _actuate(get_backend().move_mouse, float(x), float(y)) @@ -154,6 +159,7 @@ def click( raise ValueError(f"click count too large ({count} > {MAX_CLICK_COUNT})") assert_on_screen(x, y) assert_foreground(expect_window) + get_overlay().pointer(x, y, click=True) _actuate(get_backend().click, float(x), float(y), button, count) diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py index 24a381ea..79c39f38 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py @@ -33,7 +33,10 @@ import sys from collections.abc import Iterator from contextlib import contextmanager -from typing import Protocol +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from tst_cu_mcp.overlay.agents import Agent, AgentBook #: Env override resolved before the config file. The daemon sets this from #: the user's ``show_on_real_display`` pref when it spawns the sidecar, so @@ -46,6 +49,8 @@ CMD_SHOW = "show" CMD_HIDE = "hide" CMD_TARGET = "target" +CMD_CURSOR = "cursor" +CMD_CURSOR_HIDE = "cursor_hide" CMD_GRAB_BEGIN = "grab_begin" CMD_GRAB_END = "grab_end" CMD_QUIT = "quit" @@ -79,6 +84,14 @@ def set_target(self, display_id: int | None) -> None: """The display being acted on (a display id from :mod:`displays`).""" ... + def use_agent(self, key: str, label: str = "") -> Agent: + """Name the agent whose actions follow; returns its cursor identity.""" + ... + + def pointer(self, x: float, y: float, *, click: bool = False) -> None: + """The acting agent's cursor goes to this global point (TD-5227).""" + ... + @contextmanager def grab_hidden(self) -> Iterator[None]: """Hide the glow for the duration of one screenshot grab.""" @@ -110,6 +123,14 @@ def set_target(self, display_id: int | None) -> None: """Light this display only, where the platform can; else a no-op.""" ... + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + """Draw ``agent``'s cursor at a global point; a no-op where unsupported.""" + ... + + def cursor_hide(self, key: str | None) -> None: + """Hide one agent's cursor, or every cursor when ``key`` is None.""" + ... + @contextmanager def grab_hidden(self) -> Iterator[None]: yield @@ -137,6 +158,20 @@ def notify_blocked(self) -> None: def set_target(self, display_id: int | None) -> None: return None + def use_agent(self, key: str, label: str = "") -> Agent: + # Colors still matter with no painter: TST Desk hands them to + # agent-desktop's own cursor. + return _null_book().use(key, label) + + def pointer(self, x: float, y: float, *, click: bool = False) -> None: + return None + + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + return None + + def cursor_hide(self, key: str | None) -> None: + return None + @contextmanager def grab_hidden(self) -> Iterator[None]: yield @@ -145,6 +180,18 @@ def shutdown(self) -> None: return None +_book: AgentBook | None = None + + +def _null_book() -> AgentBook: + global _book + if _book is None: + from tst_cu_mcp.overlay.agents import AgentBook + + _book = AgentBook() + return _book + + def overlay_enabled(platform: str | None = None) -> bool: """Whether the real-display glow should run at all. diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/agents.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/agents.py new file mode 100644 index 00000000..6aad8bc8 --- /dev/null +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/agents.py @@ -0,0 +1,190 @@ +"""Who is acting: one cursor identity per agent (TD-5227). + +Every computer-use client gets its own cursor color and label on the real +display. A client is one tst-cu-mcp process (another assistant's copy, +Grok's copy), or one TST Desk session inside the daemon's shared sidecar, +which names itself with the internal ``overlay_agent`` tool before acting. + +Colors come from a machine-wide slot registry, so two live clients never +share a color while there are colors left: slot N is a lock file held with +a non-blocking ``flock`` by whichever process claimed it. The OS drops the +lock when that process exits, so a crashed client frees its color without +any cleanup. Past the last slot a color repeats; the label still tells the +agents apart. +""" + +from __future__ import annotations + +import contextlib +import os +import re +from dataclasses import dataclass +from pathlib import Path + +#: Cursor fills, distinct from each other and from the rust ring. +PALETTE = ( + "#2F6FDE", # blue + "#16A34A", # green + "#8B5CF6", # violet + "#DB2777", # pink + "#0891B2", # teal + "#4F46E5", # indigo + "#65A30D", # olive + "#475569", # slate +) + +#: The key a client uses until it names an agent (outside clients always do). +CLIENT_KEY = "client" + +_KEY_OK = re.compile(r"[^A-Za-z0-9_-]") +_LABEL_MAX = 40 +#: The connected MCP client's label: every agent's until it is named. +_client_label = "Agent" + + +def set_client_name(name: str) -> None: + """Record ``clientInfo.name`` from the handshake (the stdio relay does).""" + global _client_label + if name.strip(): + _client_label = label_for_client(name) + + +def client_label() -> str: + return _client_label + + +@dataclass(frozen=True) +class Agent: + """One cursor: a stable key, the text beside it, and its fill.""" + + key: str + label: str + color: str + + +def clean_key(raw: str) -> str: + """Letters, digits, ``-`` and ``_`` only (it travels on a line protocol).""" + key = _KEY_OK.sub("", raw)[:64] + return key or CLIENT_KEY + + +def clean_label(raw: str) -> str: + """One line, trimmed, at most 40 characters.""" + label = " ".join(raw.split()) + return label[:_LABEL_MAX].rstrip() or "Agent" + + +def label_for_client(name: str) -> str: + """A readable label from an MCP client's ``clientInfo.name``.""" + low = name.casefold() + for needle, label in ( + ("claude", "Claude"), + ("grok", "Grok"), + ("tst-desk", "TST Desk"), + ("tstd", "TST Desk"), + ("codex", "Codex"), + ("cursor", "Cursor"), + ): + if needle in low: + return label + words = re.sub(r"[-_.]+", " ", name).strip() + return clean_label(words.title()) if words else "Agent" + + +class SlotRegistry: + """Machine-wide color slots: one held lock file per live agent.""" + + def __init__(self, directory: Path | None = None, size: int = len(PALETTE)) -> None: + self._directory = directory + self._size = size + self._held: dict[str, tuple[int, int]] = {} + + def _dir(self) -> Path: + if self._directory is not None: + return self._directory + from tst_cu_mcp.config import policy_path + + return policy_path().parent / "agents" + + def claim(self, key: str) -> int: + """This key's slot, claiming the lowest free one the first time.""" + if key in self._held: + return self._held[key][0] + try: + import fcntl + except ImportError: + return _fallback_slot(key, self._size) + directory = self._dir() + try: + directory.mkdir(parents=True, exist_ok=True) + except OSError: + return _fallback_slot(key, self._size) + for slot in range(self._size): + try: + fd = os.open(directory / f"slot-{slot}.lock", os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + continue + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + continue + self._held[key] = (slot, fd) + return slot + return _fallback_slot(key, self._size) + + def release(self, key: str) -> None: + """Give the key's slot back (closing the file drops the lock).""" + held = self._held.pop(key, None) + if held is not None: + with contextlib.suppress(OSError): + os.close(held[1]) + + def release_all(self) -> None: + for key in list(self._held): + self.release(key) + + +def _fallback_slot(key: str, size: int) -> int: + """A stable slot when no lock can be taken (no fcntl, unwritable dir).""" + return sum(key.encode("utf-8")) % max(1, size) + + +class AgentBook: + """This process's agents and which one is acting now.""" + + def __init__(self, registry: SlotRegistry | None = None) -> None: + self._registry = registry or SlotRegistry() + self._labels: dict[str, str] = {} + self._current = CLIENT_KEY + + def use(self, key: str, label: str = "") -> Agent: + """Make ``key`` the acting agent (labelled ``label`` if given).""" + clean = clean_key(key) + if label.strip(): + self._labels[clean] = clean_label(label) + self._current = clean + return self.current() + + def current(self) -> Agent: + key = self._current + label = self._labels.get(key, client_label()) + color = PALETTE[self._registry.claim(key) % len(PALETTE)] + return Agent(key=key, label=label, color=color) + + def close(self) -> None: + self._registry.release_all() + + +__all__ = [ + "CLIENT_KEY", + "PALETTE", + "Agent", + "AgentBook", + "SlotRegistry", + "clean_key", + "clean_label", + "client_label", + "label_for_client", + "set_client_name", +] diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py index d3d3255c..4035e7ab 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py @@ -1,32 +1,37 @@ -"""When the ring shows and where, for every painter (TD-5226). +"""When the ring and the agent cursors show, and where (TD-5226, TD-5227). -Painters (darwin, win32, linux) only know how to light and darken a ring. -This controller owns the rules: +Painters (darwin, win32, linux) only light, darken and aim. This controller +owns the rules: * **Mode.** The ring means "the agent has the real screen". In background mode (``mode: background`` in the policy file) agents work through accessibility calls and never take the screen, so the ring stays off on every display, even when such an agent falls back to a coordinate tool. - Only ``full_control`` lights it. The file is re-read when it changes, so - Settings applies without restarting any client. + Only ``full_control`` lights it. The agent cursor shows in both modes. +* **Switches.** ``overlay.enabled`` (TST Desk's "show on the real display") + gates both; ``overlay.cursor`` (its "agent cursor") gates the cursor. The + file is re-read when it changes, so Settings applies to every running + client without a restart. * **Display.** Only the display being acted on is lit: the one the last capture targeted, or the one under the last pointer action. Before either, the painter's default (the primary display). Painters that cannot light a single display (Windows and Linux today) ring every display. * **Episodes.** TST Desk brackets each session's computer use with ``overlay_session``; the ring stays up while any session's episode is - open, so one session finishing does not darken another's ring. A client - that never brackets (any other MCP client) lights the ring on activity, - and it fades after :data:`UNBRACKETED_LINGER_SECONDS` without one. -* **Stop.** While the kill-switch is engaged the ring is dark. A watcher - checks every :data:`WATCH_SECONDS`, so Stop clears the ring without - waiting for the agent's next action. + open. A client that never brackets lights the ring per action, and it + fades :data:`UNBRACKETED_LINGER_SECONDS` after the last one. +* **Cursors.** Each agent's cursor goes to its last move, click or acted-on + element; the painter fades it when that agent goes idle. It never touches + the hardware pointer. +* **Stop.** While the kill-switch is engaged the ring and every cursor are + dark. A watcher re-checks every :data:`WATCH_SECONDS`, so Stop clears them + without waiting for the agent's next action. The state is level-triggered: each entry point and each watcher tick works -out whether the ring is wanted and talks to the painter only when that, or -the target display, changes. Painter calls happen under one lock because -the watcher thread and the tool thread both reach the painter. The failure -rule of the package holds: nothing here may raise into a tool. +out what is wanted and talks to the painter only on a change. Painter calls +happen under one lock because the watcher thread and the tool thread both +reach the painter. The package's failure rule holds: nothing here raises +into a tool. """ from __future__ import annotations @@ -39,10 +44,13 @@ from pathlib import Path from typing import TYPE_CHECKING +from tst_cu_mcp.config import Config +from tst_cu_mcp.overlay.agents import Agent, AgentBook + if TYPE_CHECKING: from tst_cu_mcp.overlay import Painter -#: How often the watcher re-checks Stop, the mode and the linger. +#: How often the watcher re-checks Stop, the policy file and the linger. WATCH_SECONDS = 0.25 #: A non-bracketing client's ring fades this long after its last action. UNBRACKETED_LINGER_SECONDS = 30.0 @@ -50,12 +58,13 @@ FULL_CONTROL = "full_control" -class PolicyMode: - """The policy file's ``mode``, re-parsed only when the file changes. +class PolicyFile: + """The policy file as a :class:`Config`, re-parsed only when it changes. - No file means the installed config (permissive defaults: background). - A file that fails to parse keeps the last good answer, or background, - so a broken edit darkens the ring rather than lighting it. + No file means the installed config (permissive defaults: background + mode, overlay and cursor on). A file that fails to parse reads as + background with the overlay off, so a broken edit darkens rather than + lights. """ def __init__(self, path: Callable[[], Path] | None = None) -> None: @@ -63,9 +72,9 @@ def __init__(self, path: Callable[[], Path] | None = None) -> None: self._path = path or policy_path self._stamp: tuple[float, int] | None = None - self._mode = "background" + self._config = Config(overlay_enabled=False) - def __call__(self) -> str: + def __call__(self) -> Config: from tst_cu_mcp import safety from tst_cu_mcp.config import load_config @@ -74,15 +83,15 @@ def __call__(self) -> str: stat = target.stat() except OSError: self._stamp = None - return safety.active_config().mode + return safety.active_config() stamp = (stat.st_mtime, stat.st_size) if stamp != self._stamp: self._stamp = stamp try: - self._mode = load_config(target).mode + self._config = load_config(target) except Exception: - self._mode = "background" - return self._mode + self._config = Config(overlay_enabled=False) + return self._config def _killswitch() -> bool: @@ -95,21 +104,23 @@ def _killswitch() -> bool: class GatedOverlay: - """An :class:`~tst_cu_mcp.overlay.Overlay` that applies the ring's rules.""" + """An :class:`~tst_cu_mcp.overlay.Overlay` that applies the rules.""" def __init__( self, painter: Painter, *, - mode: Callable[[], str] | None = None, + policy: Callable[[], Config] | None = None, stopped: Callable[[], bool] = _killswitch, clock: Callable[[], float] = time.monotonic, linger: float = UNBRACKETED_LINGER_SECONDS, + agents: AgentBook | None = None, watch: bool = True, ) -> None: self.painter = painter self.name = painter.name - self._mode = mode or PolicyMode() + self.agents = agents or AgentBook() + self._policy = policy or PolicyFile() self._stopped = stopped self._clock = clock self._linger = linger @@ -122,6 +133,7 @@ def __init__( self._target: int | None = None self._sent_target: int | None = None self._shown = False + self._cursors_out = False self._watcher: threading.Thread | None = None self._halt = threading.Event() @@ -164,6 +176,23 @@ def set_target(self, display_id: int | None) -> None: self._target = display_id self._sync() + def use_agent(self, key: str, label: str = "") -> Agent: + """Name the agent whose actions follow (TST Desk's shared sidecar).""" + with self._lock: + return self.agents.use(key, label) + + def pointer(self, x: float, y: float, *, click: bool = False) -> None: + """The acting agent's cursor goes to this global point (points).""" + with self._lock: + try: + if not self._cursor_wanted(): + return + self.painter.cursor(self.agents.current(), float(x), float(y), click) + self._cursors_out = True + self._start_watcher() + except Exception: + return + @contextmanager def grab_hidden(self) -> Iterator[None]: """Hide the painter for one capture, holding the lock only at the edges.""" @@ -183,6 +212,7 @@ def shutdown(self) -> None: watcher.join(timeout=1.0) with self._lock, contextlib.suppress(Exception): self.painter.shutdown() + self.agents.close() # -- rules ------------------------------------------------------------ @@ -190,7 +220,8 @@ def wanted(self) -> bool: """Whether the ring should be lit right now.""" if self._blocked or self._stopped(): return False - if self._mode() != FULL_CONTROL: + policy = self._policy() + if not policy.overlay_enabled or policy.mode != FULL_CONTROL: return False if self._episodes: return True @@ -198,6 +229,12 @@ def wanted(self) -> bool: return False return self._clock() - self._last_activity < self._linger + def _cursor_wanted(self) -> bool: + if self._stopped(): + return False + policy = self._policy() + return policy.overlay_enabled and policy.cursor_enabled + def tick(self) -> None: """One watcher pass. Public so tests drive it without a thread.""" with self._lock: @@ -209,6 +246,9 @@ def _sync(self) -> None: # Stop ends an unbracketed client's claim on the ring: after # Resume it takes a new action to light it again. self._last_activity = None + if self._cursors_out and not self._cursor_wanted(): + self.painter.cursor_hide(None) + self._cursors_out = False wanted = self.wanted() if wanted and self._target is not None and self._target != self._sent_target: self.painter.set_target(self._target) @@ -239,5 +279,5 @@ def _watch_loop(self) -> None: "UNBRACKETED_LINGER_SECONDS", "WATCH_SECONDS", "GatedOverlay", - "PolicyMode", + "PolicyFile", ] diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py index 72dfa7be..e4a07579 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py @@ -30,6 +30,8 @@ from typing import Protocol from tst_cu_mcp.overlay import ( + CMD_CURSOR, + CMD_CURSOR_HIDE, CMD_GRAB_BEGIN, CMD_GRAB_END, CMD_HIDE, @@ -37,6 +39,7 @@ CMD_SHOW, CMD_TARGET, ) +from tst_cu_mcp.overlay.agents import Agent, clean_key, clean_label #: ``show``/``hide`` are acked by the helper's reader thread on arrival. FAST_ACK_TIMEOUT = 1.0 @@ -171,6 +174,16 @@ def set_target(self, display_id: int | None) -> None: if display_id is not None and display_id >= 0: self._send(f"{CMD_TARGET} {int(display_id)}") + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + # One line: key, point (global, top-left origin), fill, click, label. + self._send( + f"{CMD_CURSOR} {clean_key(agent.key)} {x:.1f} {y:.1f} {agent.color} " + f"{1 if click else 0} {clean_label(agent.label)}" + ) + + def cursor_hide(self, key: str | None) -> None: + self._send(f"{CMD_CURSOR_HIDE} {'*' if key is None else clean_key(key)}") + @contextmanager def grab_hidden(self) -> Iterator[None]: hidden = self._send(CMD_GRAB_BEGIN, timeout=GRAB_ACK_TIMEOUT) diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_cursor.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_cursor.py new file mode 100644 index 00000000..a1ee500d --- /dev/null +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_cursor.py @@ -0,0 +1,321 @@ +"""Agent cursors for the macOS helper (TD-5227). + +Each agent gets a small borderless panel: an arrow in its color with a +white rim, and a pill with its label. Like the ring's panels these are +click-through, never take focus, join every space, and are left out of +every screen capture (``NSWindowSharingNone``), so no agent ever sees a +cursor in a screenshot. They are windows, not a cursor API: the hardware +pointer is never moved or replaced. + +The motion lives in :class:`CursorModel`, plain Python the tests drive with +a fake clock: a cursor glides to each new point, a click plays a short +ring, and an agent idle for :data:`CURSOR_IDLE_SECONDS` fades out. With +Reduce Motion on, a cursor jumps and nothing ripples. Only +:func:`build_cursor_layer` touches AppKit. + +Line protocol (handled by the helper's main thread): + +* ``cursor <#rrggbb> `` — global + point in points, top-left origin (the CGEvent space). +* ``cursor_hide `` or ``cursor_hide *`` — hide now, no fade. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any + +#: An agent with no new action for this long fades out. +CURSOR_IDLE_SECONDS = 10.0 +#: Glide speed: the share of the remaining distance covered per second. +GLIDE_RATE = 14.0 +FADE_IN_RATE = 8.0 +FADE_OUT_RATE = 3.0 +RIPPLE_SECONDS = 0.45 + +#: Panel size and where the arrow's tip sits in it (points, top-left). +PANEL_WIDTH = 300.0 +PANEL_HEIGHT = 72.0 +TIP_X = 16.0 +TIP_Y = 16.0 +#: Above the ring (1000), still below the system cursor. +CURSOR_WINDOW_LEVEL = 1001 + +#: The arrow, as offsets from its tip with y growing downward. +ARROW = ( + (0.0, 0.0), + (0.0, 19.0), + (4.5, 15.0), + (7.6, 22.0), + (10.6, 20.7), + (7.6, 13.9), + (13.4, 13.9), +) + + +@dataclass +class CursorState: + """One agent's cursor: where it is drawn, where it is going, how lit.""" + + key: str + color: str + label: str + x: float + y: float + tx: float + ty: float + last_seen: float + alpha: float = 0.0 + ripple: float = 0.0 + shown: bool = True + + +@dataclass +class CursorModel: + """Every agent's cursor, advanced once per helper tick.""" + + cursors: dict[str, CursorState] = field(default_factory=dict) + + def place( + self, key: str, x: float, y: float, color: str, label: str, click: bool, now: float + ) -> None: + state = self.cursors.get(key) + if state is None or state.alpha <= 0.01 or not state.shown: + # A cursor coming back from hidden starts where it is going, + # not at a stale spot from minutes ago. + state = CursorState(key, color, label, x, y, x, y, now) + self.cursors[key] = state + state.tx, state.ty = x, y + state.color, state.label = color, label + state.last_seen = now + state.shown = True + if click: + state.ripple = 1.0 + + def hide(self, key: str | None) -> None: + for state in self.cursors.values(): + if key is None or state.key == key: + state.shown = False + state.alpha = 0.0 + state.ripple = 0.0 + + def step(self, now: float, dt: float, reduce_motion: bool) -> None: + for state in self.cursors.values(): + if reduce_motion: + state.x, state.y = state.tx, state.ty + state.ripple = 0.0 + else: + share = min(1.0, max(0.0, dt) * GLIDE_RATE) + state.x += (state.tx - state.x) * share + state.y += (state.ty - state.y) * share + state.ripple = max(0.0, state.ripple - dt / RIPPLE_SECONDS) + live = state.shown and now - state.last_seen < CURSOR_IDLE_SECONDS + goal = 1.0 if live else 0.0 + rate = FADE_IN_RATE if goal > state.alpha else FADE_OUT_RATE + state.alpha += (goal - state.alpha) * min(1.0, max(0.0, dt) * rate) + if goal == 0.0 and state.alpha < 0.02: + state.alpha = 0.0 + + +def parse_cursor_args(arg: str) -> tuple[str, float, float, str, bool, str] | None: + """``cursor``'s argument, or ``None`` when it is malformed.""" + parts = arg.split(" ", 5) + if len(parts) < 5: + return None + key, raw_x, raw_y, color, raw_click = parts[:5] + label = parts[5].strip() if len(parts) == 6 else "" + try: + x, y = float(raw_x), float(raw_y) + except ValueError: + return None + if not (math.isfinite(x) and math.isfinite(y)) or parse_hex(color) is None or not key: + return None + return key, x, y, color, raw_click == "1", label or "Agent" + + +def parse_hex(color: str) -> tuple[float, float, float] | None: + """``#rrggbb`` as 0..1 floats.""" + if len(color) != 7 or not color.startswith("#"): + return None + try: + return tuple(int(color[i : i + 2], 16) / 255.0 for i in (1, 3, 5)) # type: ignore[return-value] + except ValueError: + return None + + +def panel_origin(x: float, y: float, primary_height: float) -> tuple[float, float]: + """Cocoa origin (bottom-left, y up) for a panel whose tip is at (x, y).""" + return x - TIP_X, primary_height - y + TIP_Y - PANEL_HEIGHT + + +def build_cursor_layer() -> Any: + """Define the AppKit view and return a layer the helper ticks.""" + from AppKit import NSView + + class CursorView(NSView): + """Draws one agent's arrow, click ring and label pill.""" + + def isFlipped(self) -> bool: + return True + + def drawRect_(self, _rect: Any) -> None: + from AppKit import NSBezierPath, NSColor, NSFont, NSString + from Foundation import NSMakePoint, NSMakeRect + + red, green, blue = getattr(self, "fill", (0.18, 0.43, 0.87)) + fill = NSColor.colorWithCalibratedRed_green_blue_alpha_(red, green, blue, 1.0) + white = NSColor.colorWithCalibratedWhite_alpha_(1.0, 0.95) + ripple = float(getattr(self, "ripple", 0.0)) + if ripple > 0.0: + radius = 4.0 + 14.0 * (1.0 - ripple) + ring = NSBezierPath.bezierPathWithOvalInRect_( + NSMakeRect(TIP_X - radius, TIP_Y - radius, 2 * radius, 2 * radius) + ) + ring.setLineWidth_(2.0) + NSColor.colorWithCalibratedRed_green_blue_alpha_( + red, green, blue, 0.7 * ripple + ).setStroke() + ring.stroke() + arrow = NSBezierPath.bezierPath() + first, *rest = ARROW + arrow.moveToPoint_(NSMakePoint(TIP_X + first[0], TIP_Y + first[1])) + for dx, dy in rest: + arrow.lineToPoint_(NSMakePoint(TIP_X + dx, TIP_Y + dy)) + arrow.closePath() + arrow.setLineJoinStyle_(1) + fill.setFill() + arrow.fill() + arrow.setLineWidth_(1.5) + white.setStroke() + arrow.stroke() + text = NSString.stringWithString_(str(getattr(self, "text", ""))) + attrs = { + "NSFont": NSFont.systemFontOfSize_weight_(11.0, 0.3), + "NSColor": NSColor.whiteColor(), + } + size = text.sizeWithAttributes_(attrs) + left, top = TIP_X + 14.0, TIP_Y + 18.0 + width = min(size.width + 14.0, PANEL_WIDTH - left - 2.0) + height = size.height + 6.0 + pill = NSBezierPath.bezierPathWithRoundedRect_xRadius_yRadius_( + NSMakeRect(left, top, width, height), height / 2.0, height / 2.0 + ) + fill.setFill() + pill.fill() + pill.setLineWidth_(1.0) + white.setStroke() + pill.stroke() + text.drawInRect_withAttributes_( + NSMakeRect(left + 7.0, top + 3.0, width - 14.0, size.height), attrs + ) + + return CursorLayer(CursorView) + + +class CursorLayer: + """The helper's cursors: the model plus one panel per agent.""" + + def __init__(self, view_class: Any) -> None: + self.model = CursorModel() + self.panels: dict[str, Any] = {} + self._view_class = view_class + self._drawn: dict[str, tuple[str, str, float]] = {} + self._primary_height = 0.0 + self.screens_changed() + + def screens_changed(self) -> None: + from AppKit import NSScreen + + screens = NSScreen.screens() + self._primary_height = float(screens[0].frame().size.height) if screens else 0.0 + + def command(self, name: str, arg: str, now: float) -> None: + if name == "cursor_hide": + key = arg.strip() + self.model.hide(None if key in ("", "*") else key) + self._apply_hidden() + return + parsed = parse_cursor_args(arg) + if parsed is not None: + key, x, y, color, click, label = parsed + self.model.place(key, x, y, color, label, click, now) + + def hide_now(self) -> None: + """Out of the way for a capture; the next tick puts them back.""" + for panel in self.panels.values(): + panel.orderOut_(None) + + def tick(self, now: float, dt: float, reduce_motion: bool, grabbing: bool) -> None: + self.model.step(now, dt, reduce_motion) + for key, state in self.model.cursors.items(): + if state.alpha <= 0.0 or grabbing: + if key in self.panels: + self.panels[key].orderOut_(None) + continue + panel = self.panels.get(key) or self._make_panel(key) + self._redraw(key, state) + panel.setFrameOrigin_(panel_origin(state.x, state.y, self._primary_height)) + panel.setAlphaValue_(state.alpha) + panel.orderFrontRegardless() + + def _apply_hidden(self) -> None: + for key, state in self.model.cursors.items(): + if state.alpha <= 0.0 and key in self.panels: + self.panels[key].orderOut_(None) + + def _redraw(self, key: str, state: CursorState) -> None: + look = (state.color, state.label, round(state.ripple, 2)) + if self._drawn.get(key) == look: + return + self._drawn[key] = look + view = self.panels[key].contentView() + view.fill = parse_hex(state.color) or (0.18, 0.43, 0.87) + view.text = state.label + view.ripple = state.ripple + view.setNeedsDisplay_(True) + + def _make_panel(self, key: str) -> Any: + from AppKit import ( + NSColor, + NSPanel, + NSWindowCollectionBehaviorCanJoinAllSpaces, + NSWindowCollectionBehaviorFullScreenAuxiliary, + NSWindowSharingNone, + NSWindowStyleMaskBorderless, + NSWindowStyleMaskNonactivatingPanel, + ) + from Foundation import NSMakeRect + + frame = NSMakeRect(0.0, 0.0, PANEL_WIDTH, PANEL_HEIGHT) + panel = NSPanel.alloc().initWithContentRect_styleMask_backing_defer_( + frame, + NSWindowStyleMaskBorderless | NSWindowStyleMaskNonactivatingPanel, + 2, + False, + ) + panel.setLevel_(CURSOR_WINDOW_LEVEL) + panel.setIgnoresMouseEvents_(True) + panel.setSharingType_(NSWindowSharingNone) + panel.setOpaque_(False) + panel.setBackgroundColor_(NSColor.clearColor()) + panel.setHasShadow_(False) + panel.setCollectionBehavior_( + NSWindowCollectionBehaviorCanJoinAllSpaces + | NSWindowCollectionBehaviorFullScreenAuxiliary + ) + panel.setContentView_(self._view_class.alloc().initWithFrame_(frame)) + panel.setAlphaValue_(0.0) + self.panels[key] = panel + return panel + + +__all__ = [ + "CURSOR_IDLE_SECONDS", + "CursorModel", + "CursorState", + "build_cursor_layer", + "panel_origin", + "parse_cursor_args", + "parse_hex", +] diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py index a665ed9a..70e8cd64 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py @@ -25,6 +25,8 @@ actually ordered the panels out/in. That ordering is the guarantee a screenshot capture relies on: between the parent receiving the ``grab_begin`` ack and sending ``grab_end``, no ring pixel exists. +* ``cursor ...`` / ``cursor_hide ...`` — agent cursors, drawn by + :mod:`tst_cu_mcp.overlay.darwin_cursor` (TD-5227). Acked on arrival. * ``quit`` — clean exit; stdin EOF triggers it too, so a dead parent never leaves a halo behind. @@ -39,11 +41,25 @@ import math import queue -import sys import threading import time from typing import Any +from tst_cu_mcp.overlay.darwin_wire import ( + _CURSOR_COMMANDS, + _GRAB_BEGIN, + _GRAB_END, + _HIDE, + _QUIT, + _SHOW, + _TARGET, + Command, + PanelKey, + _parse_display_id, + _read_stdin, + lit_keys, +) + # --- geometry / motion ------------------------------------------------------ #: Edge gap, stroke widths and corner radius, in points. @@ -70,23 +86,6 @@ #: NSEventTypeApplicationDefined — the wake-up that lets stop_ unwind. _APPLICATION_DEFINED_EVENT_TYPE = 15 -_SHOW, _HIDE, _TARGET, _GRAB_BEGIN, _GRAB_END, _QUIT = ( - "show", - "hide", - "target", - "grab_begin", - "grab_end", - "quit", -) -#: Commands whose ack must wait for the main thread to have acted. -_MAIN_THREAD_ACKS = frozenset({_GRAB_BEGIN, _GRAB_END}) -#: Reader-thread patience with the main thread before reporting failure. -_MAIN_ACK_TIMEOUT = 2.5 - -#: (name, argument, ack). The argument is the rest of the line, or "". -Command = tuple[str, str, threading.Event | None] -PanelKey = tuple[int, str, int, int, int, int] - def main() -> None: """Build the app, wire stdin to the command queue, and run forever.""" @@ -102,6 +101,9 @@ def main() -> None: commands: queue.Queue[Command] = queue.Queue() controller = _build_controller(commands) + from tst_cu_mcp.overlay.darwin_cursor import build_cursor_layer + + controller.cursors = build_cursor_layer() reader = threading.Thread(target=_read_stdin, args=(commands,), daemon=True) reader.start() @@ -160,6 +162,7 @@ def initWithQueue_(self, q: queue.Queue[Command]) -> Any: # its panel. The display id is what ``target`` names. self.panels: dict[PanelKey, Any] = {} self.target: int | None = None + self.cursors: Any = None self.visible = False self.grab_depth = 0 self.last_activity = 0.0 @@ -201,9 +204,13 @@ def tick_(self, _timer: Any) -> None: self._drain_commands() self._advance_fade(elapsed) self._apply_alpha(elapsed) + if self.cursors is not None: + self.cursors.tick(now, elapsed, _reduce_motion(), self.grab_depth > 0) def screensChanged_(self, _note: Any) -> None: self.rebuild_panels() + if self.cursors is not None: + self.cursors.screens_changed() # -- state transitions ------------------------------------------------- @@ -213,7 +220,10 @@ def _drain_commands(self) -> None: command, arg, ack = self.commands.get_nowait() except queue.Empty: return - if command == _TARGET: + if command in _CURSOR_COMMANDS: + if self.cursors is not None: + self.cursors.command(command, arg, time.monotonic()) + elif command == _TARGET: _retarget(self, _parse_display_id(arg)) elif command == _SHOW: self.visible = True @@ -226,6 +236,8 @@ def _drain_commands(self) -> None: # Instant, not faded: a capture cannot wait out an ease. self.grab_depth += 1 self._hide_panels_now() + if self.cursors is not None: + self.cursors.hide_now() elif command == _GRAB_END: self.grab_depth = max(0, self.grab_depth - 1) if self.grab_depth == 0 and self.visible: @@ -379,16 +391,6 @@ def _retarget(controller: Any, display_id: int | None) -> None: controller._ensure_front() -def lit_keys(keys: list[PanelKey], target: int | None) -> list[PanelKey]: - """The target display's panel, else the primary display's. - - NSScreen.screens() lists the primary (menu-bar) display first and the - panel dict keeps that order, so the first key is the fallback. - """ - chosen = [key for key in keys if key[0] == target] - return chosen or keys[:1] - - def _screen_number(screen: Any) -> int: """The CGDirectDisplayID AppKit stores as ``NSScreenNumber``; -1 if absent.""" try: @@ -397,15 +399,6 @@ def _screen_number(screen: Any) -> int: return -1 -def _parse_display_id(arg: str) -> int | None: - """``target``'s argument: a non-negative display id, else ``None``.""" - try: - value = int(arg.strip()) - except ValueError: - return None - return value if value >= 0 else None - - def _rust() -> tuple[float, float, float]: from AppKit import NSApplication @@ -425,34 +418,5 @@ def _reduce_motion() -> bool: return False -def _read_stdin(commands: queue.Queue[Command]) -> None: - """Reader thread: enqueue lines, hand back acks. - - ``show``/``hide`` ack immediately — they are queued and the tick will - act within ~33ms. The grab pair waits for the main thread's event, so - the parent's capture ordering holds. EOF means the parent is gone: - quit rather than haunt the screen. - """ - try: - for line in sys.stdin: - command, _, arg = line.strip().partition(" ") - if not command: - continue - if command in _MAIN_THREAD_ACKS: - ack: threading.Event = threading.Event() - commands.put((command, arg, ack)) - print("ok" if ack.wait(_MAIN_ACK_TIMEOUT) else "err", flush=True) - elif command == _QUIT: - commands.put((_QUIT, "", None)) - return - else: - commands.put((command, arg, None)) - print("ok", flush=True) - except Exception: - pass - finally: - commands.put((_QUIT, "", None)) - - if __name__ == "__main__": main() diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_wire.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_wire.py new file mode 100644 index 00000000..fee450e2 --- /dev/null +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_wire.py @@ -0,0 +1,80 @@ +"""The macOS overlay helper's line protocol, free of AppKit (TD-5226, TD-5227). + +One command per stdin line, ``name`` then an optional argument (the rest of +the line). The reader thread queues each one and prints ``ok`` or ``err``: +at once for most commands, and only after the main thread has acted for +the grab pair. Kept apart from :mod:`darwin_helper` so it is testable on any +platform and the helper stays small. +""" + +from __future__ import annotations + +import queue +import sys +import threading + +_SHOW, _HIDE, _TARGET, _GRAB_BEGIN, _GRAB_END, _QUIT = ( + "show", + "hide", + "target", + "grab_begin", + "grab_end", + "quit", +) +_CURSOR_COMMANDS = frozenset({"cursor", "cursor_hide"}) +#: Commands whose ack must wait for the main thread to have acted. +_MAIN_THREAD_ACKS = frozenset({_GRAB_BEGIN, _GRAB_END}) +#: Reader-thread patience with the main thread before reporting failure. +_MAIN_ACK_TIMEOUT = 2.5 + +#: (name, argument, ack). The argument is the rest of the line, or "". +Command = tuple[str, str, threading.Event | None] +PanelKey = tuple[int, str, int, int, int, int] + + +def lit_keys(keys: list[PanelKey], target: int | None) -> list[PanelKey]: + """The target display's panel, else the primary display's. + + NSScreen.screens() lists the primary (menu-bar) display first and the + panel dict keeps that order, so the first key is the fallback. + """ + chosen = [key for key in keys if key[0] == target] + return chosen or keys[:1] + + +def _parse_display_id(arg: str) -> int | None: + """``target``'s argument: a non-negative display id, else ``None``.""" + try: + value = int(arg.strip()) + except ValueError: + return None + return value if value >= 0 else None + + +def _read_stdin(commands: queue.Queue[Command]) -> None: + """Reader thread: enqueue lines, hand back acks. + + ``show``/``hide`` ack immediately — they are queued and the tick will + act within ~33ms. The grab pair waits for the main thread's event, so + the parent's capture ordering holds. EOF means the parent is gone: + quit rather than haunt the screen. + """ + try: + for line in sys.stdin: + command, _, arg = line.strip().partition(" ") + if not command: + continue + if command in _MAIN_THREAD_ACKS: + ack: threading.Event = threading.Event() + commands.put((command, arg, ack)) + print("ok" if ack.wait(_MAIN_ACK_TIMEOUT) else "err", flush=True) + elif command == _QUIT: + commands.put((_QUIT, "", None)) + return + else: + commands.put((command, arg, None)) + print("ok", flush=True) + except Exception: + pass + finally: + commands.put((_QUIT, "", None)) diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py index 857596b0..c78e1860 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py @@ -13,6 +13,7 @@ from contextlib import contextmanager, suppress from typing import Any +from tst_cu_mcp.overlay.agents import Agent from tst_cu_mcp.overlay.style import ( CORE_WIDTH_PX, INSET_PX, @@ -81,6 +82,13 @@ def set_target(self, display_id: int | None) -> None: """Every display keeps its bars; lighting one display is darwin-only so far.""" del display_id + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + """The drawn agent cursor is darwin-only so far (TD-5227).""" + del agent, x, y, click + + def cursor_hide(self, key: str | None) -> None: + del key + @contextmanager def grab_hidden(self) -> Iterator[None]: with self._lock: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py index 0cd1a4a1..d8680136 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py @@ -13,6 +13,7 @@ from contextlib import contextmanager, suppress from typing import Any +from tst_cu_mcp.overlay.agents import Agent from tst_cu_mcp.overlay.style import ( CORE_WIDTH_PX, INSET_PX, @@ -88,6 +89,13 @@ def set_target(self, display_id: int | None) -> None: """Every display keeps its bars; lighting one display is darwin-only so far.""" del display_id + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + """The drawn agent cursor is darwin-only so far (TD-5227).""" + del agent, x, y, click + + def cursor_hide(self, key: str | None) -> None: + del key + @contextmanager def grab_hidden(self) -> Iterator[None]: with self._lock: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py index 92a5b82e..d635f37a 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py @@ -449,6 +449,20 @@ def scroll( if internal_tools_enabled(): + @server.tool( + name="overlay_agent", + description=( + "Internal TST Desk signal: the session whose actions follow, so " + "its drawn cursor has its own color and label. Not a model tool." + ), + structured_output=False, + ) + def overlay_agent(agent_id: str, label: str = "") -> dict[str, Any]: + from tst_cu_mcp.overlay import get_overlay + + agent = get_overlay().use_agent(agent_id, label) + return {"agent_id": agent.key, "label": agent.label, "color": agent.color} + @server.tool( name="overlay_session", description=( diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/stdio_transport.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/stdio_transport.py index 3ddd85a2..240c88d7 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/stdio_transport.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/stdio_transport.py @@ -63,6 +63,7 @@ async def read_relay() -> None: """Forward frames toward the server, then hold EOF until drained.""" async with to_server_send: async for item in wire_read: + _note_client(item) if ( isinstance(item, SessionMessage) and isinstance(item.message, JSONRPCRequest) @@ -105,6 +106,22 @@ async def write_relay() -> None: ) +def _note_client(item: object) -> None: + """Remember the client's name from ``initialize`` for its cursor label.""" + if not ( + isinstance(item, SessionMessage) + and isinstance(item.message, JSONRPCRequest) + and item.message.method == "initialize" + and isinstance(item.message.params, dict) + ): + return + info = item.message.params.get("clientInfo") + if isinstance(info, dict) and isinstance(info.get("name"), str): + from tst_cu_mcp.overlay.agents import set_client_name + + set_client_name(info["name"]) + + def _stamp_unanswered_hook( item: SessionMessage, hook: Callable[[], Awaitable[None]], diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/ui.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/ui.py index 830cfb6d..963890f7 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/ui.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/ui.py @@ -96,10 +96,28 @@ def ui_action( except Exception: safety.notify_blocked() raise + _point_at(result.get("frame")) result["app"] = target.to_dict() return result +def _point_at(frame: object) -> None: + """The acting agent's cursor goes to the element it just acted on. + + Background actions never move the pointer, so this drawn cursor is the + only on-screen sign of where the agent works (TD-5227). Best effort: no + frame (the host could not read one) means no cursor move. + """ + if not isinstance(frame, dict): + return + try: + x = float(frame["x"]) + float(frame["width"]) / 2.0 + y = float(frame["y"]) + float(frame["height"]) / 2.0 + except (KeyError, TypeError, ValueError): + return + get_overlay().pointer(x, y, click=True) + + def launch_app(app: str) -> dict[str, Any]: """Open *app* by name or bundle id. Does not take the pointer.""" safety.ensure_actuation_allowed() diff --git a/mcp/tst-cu-mcp/tests/test_agent_cursors.py b/mcp/tst-cu-mcp/tests/test_agent_cursors.py new file mode 100644 index 00000000..afefe761 --- /dev/null +++ b/mcp/tst-cu-mcp/tests/test_agent_cursors.py @@ -0,0 +1,279 @@ +"""Agent cursors (TD-5227): identity, colors, the helper's motion, the wire. + +No AppKit here: the helper's motion is a plain model, and the painter's +line protocol is checked against a recording transport. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +from tst_cu_mcp import ui +from tst_cu_mcp.overlay import set_overlay +from tst_cu_mcp.overlay.agents import ( + PALETTE, + Agent, + AgentBook, + SlotRegistry, + clean_key, + clean_label, + label_for_client, + set_client_name, +) +from tst_cu_mcp.overlay.darwin import DarwinOverlay +from tst_cu_mcp.overlay.darwin_cursor import ( + CURSOR_IDLE_SECONDS, + TIP_X, + TIP_Y, + CursorModel, + panel_origin, + parse_cursor_args, + parse_hex, +) + +posix_only = pytest.mark.skipif(sys.platform == "win32", reason="flock slots are POSIX") + + +class TestIdentity: + @pytest.mark.parametrize( + "name, label", + [ + ("claude-ai", "Claude"), + ("Grok Build", "Grok"), + ("tstd", "TST Desk"), + ("my_custom-client", "My Custom Client"), + ("", "Agent"), + ], + ) + def test_client_names_read_as_labels(self, name: str, label: str) -> None: + assert label_for_client(name) == label + + def test_keys_and_labels_survive_the_line_protocol(self) -> None: + assert clean_key("a b\nc;d-1_2") == "abcd-1_2" + assert clean_key(" ") == "client" + assert clean_label(" two\n lines ") == "two lines" + assert len(clean_label("x" * 100)) == 40 + + def test_unnamed_agents_carry_the_clients_label(self, tmp_path: Path) -> None: + book = AgentBook(SlotRegistry(tmp_path)) + set_client_name("claude-ai") + try: + assert book.current().label == "Claude" + assert book.use("s1", "Fix the login").label == "Fix the login" + assert book.use("s1").label == "Fix the login" + finally: + set_client_name("Agent") + + +@posix_only +class TestSlots: + def test_live_agents_never_share_a_color(self, tmp_path: Path) -> None: + registry = SlotRegistry(tmp_path) + slots = [registry.claim(f"agent-{n}") for n in range(len(PALETTE))] + assert sorted(slots) == list(range(len(PALETTE))) + assert registry.claim("agent-3") == slots[3] + + def test_a_released_slot_is_reused(self, tmp_path: Path) -> None: + registry = SlotRegistry(tmp_path) + first = registry.claim("a") + registry.claim("b") + registry.release("a") + assert registry.claim("c") == first + + def test_another_process_holding_a_slot_moves_us_on(self, tmp_path: Path) -> None: + """The registry is machine-wide: a second client gets the next color.""" + holder = subprocess.Popen( + [ + sys.executable, + "-c", + textwrap.dedent( + f""" + import sys + from pathlib import Path + from tst_cu_mcp.overlay.agents import SlotRegistry + r = SlotRegistry(Path({str(tmp_path)!r})) + print(r.claim("other"), flush=True) + sys.stdin.read() + """ + ), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + env={**os.environ}, + ) + try: + assert holder.stdout is not None + held = int(holder.stdout.readline()) + assert SlotRegistry(tmp_path).claim("mine") != held + finally: + assert holder.stdin is not None + holder.stdin.close() + holder.wait(timeout=10) + # Its lock went with it: the slot is free again. + assert SlotRegistry(tmp_path).claim("later") == held + + def test_past_the_palette_colors_repeat_without_failing(self, tmp_path: Path) -> None: + registry = SlotRegistry(tmp_path, size=2) + registry.claim("a") + registry.claim("b") + assert registry.claim("c") in (0, 1) + + +class TestCursorModel: + def test_new_cursor_starts_at_its_point_and_glides_to_the_next(self) -> None: + model = CursorModel() + model.place("a", 100, 100, "#2F6FDE", "A", False, now=0.0) + model.step(now=0.0, dt=0.1, reduce_motion=False) + state = model.cursors["a"] + assert (state.x, state.y) == (100, 100) + model.place("a", 200, 100, "#2F6FDE", "A", False, now=0.1) + model.step(now=0.13, dt=0.033, reduce_motion=False) + assert 100 < state.x < 200 + for tick in range(30): + model.step(now=0.2 + tick * 0.033, dt=0.033, reduce_motion=False) + assert state.x == pytest.approx(200, abs=0.5) + + def test_reduce_motion_jumps_and_never_ripples(self) -> None: + model = CursorModel() + model.place("a", 100, 100, "#2F6FDE", "A", False, now=0.0) + model.step(now=0.0, dt=0.1, reduce_motion=True) + model.place("a", 300, 50, "#2F6FDE", "A", True, now=0.1) + model.step(now=0.13, dt=0.033, reduce_motion=True) + state = model.cursors["a"] + assert (state.x, state.y) == (300, 50) + assert state.ripple == 0.0 + + def test_a_click_ripples_then_settles(self) -> None: + model = CursorModel() + model.place("a", 10, 10, "#2F6FDE", "A", True, now=0.0) + model.step(now=0.03, dt=0.03, reduce_motion=False) + assert 0 < model.cursors["a"].ripple < 1 + model.step(now=1.0, dt=1.0, reduce_motion=False) + assert model.cursors["a"].ripple == 0.0 + + def test_an_idle_agent_fades_out(self) -> None: + model = CursorModel() + model.place("a", 10, 10, "#2F6FDE", "A", False, now=0.0) + model.step(now=0.5, dt=0.5, reduce_motion=False) + assert model.cursors["a"].alpha > 0.9 + now = 0.5 + while now < CURSOR_IDLE_SECONDS + 3.0: + now += 0.1 + model.step(now=now, dt=0.1, reduce_motion=False) + assert model.cursors["a"].alpha == 0.0 + + def test_hide_is_instant_and_per_agent(self) -> None: + model = CursorModel() + model.place("a", 10, 10, "#2F6FDE", "A", False, now=0.0) + model.place("b", 20, 20, "#16A34A", "B", False, now=0.0) + model.step(now=0.5, dt=0.5, reduce_motion=False) + model.hide("a") + assert model.cursors["a"].alpha == 0.0 + assert model.cursors["b"].alpha > 0.9 + model.hide(None) + assert model.cursors["b"].alpha == 0.0 + + +class TestWire: + def test_cursor_arguments_parse_and_reject_garbage(self) -> None: + assert parse_cursor_args("s1 10.5 -20 #2F6FDE 1 Fix the login") == ( + "s1", + 10.5, + -20.0, + "#2F6FDE", + True, + "Fix the login", + ) + assert parse_cursor_args("s1 10 20 #2F6FDE 0")[-1] == "Agent" # type: ignore[index] + assert parse_cursor_args("s1 ten 20 #2F6FDE 0 x") is None + assert parse_cursor_args("s1 nan 20 #2F6FDE 0 x") is None + assert parse_cursor_args("s1 10 20 blue 0 x") is None + assert parse_cursor_args("s1 10 20") is None + + def test_hex_colors(self) -> None: + assert parse_hex("#FF0080") == (1.0, 0.0, 128 / 255) + assert parse_hex("FF0080") is None + assert parse_hex("#GG0000") is None + + def test_panel_tip_lands_on_the_point(self) -> None: + # Cocoa y grows up from the primary display's bottom edge. + x, y = panel_origin(500, 200, primary_height=1080) + assert x + TIP_X == 500 + assert 1080 - (y + 72 - TIP_Y) == 200 + + def test_darwin_painter_sends_one_line_per_cursor(self) -> None: + sent: list[str] = [] + + class Transport: + def send(self, command: str, *, timeout: float = 1.0) -> bool: + sent.append(command) + return True + + def close(self) -> None: + return None + + painter = DarwinOverlay(spawn=Transport) + painter.cursor(Agent("s 1", "Fix\nlogin", "#2F6FDE"), 10, 20.25, True) + painter.cursor_hide("s1") + painter.cursor_hide(None) + assert sent == [ + "cursor s1 10.0 20.2 #2F6FDE 1 Fix login", + "cursor_hide s1", + "cursor_hide *", + ] + + +class TestElementPoint: + """Background actions put the cursor on the element they acted on.""" + + class Recorder: + def __init__(self) -> None: + self.points: list[tuple[float, float, bool]] = [] + + def pointer(self, x: float, y: float, *, click: bool = False) -> None: + self.points.append((x, y, click)) + + def test_frame_center_with_a_click(self) -> None: + recorder = self.Recorder() + set_overlay(recorder) # type: ignore[arg-type] + ui._point_at({"x": 100, "y": 50, "width": 40, "height": 20}) + assert recorder.points == [(120.0, 60.0, True)] + + @pytest.mark.parametrize("frame", [None, {}, {"x": 1}, {"x": "a", "y": 1, "width": 1}]) + def test_no_usable_frame_moves_nothing(self, frame: Any) -> None: + recorder = self.Recorder() + set_overlay(recorder) # type: ignore[arg-type] + ui._point_at(frame) + assert recorder.points == [] + + +def test_the_handshake_names_the_client() -> None: + from mcp.shared.message import SessionMessage + from mcp_types import JSONRPCRequest + + from tst_cu_mcp.overlay.agents import client_label + from tst_cu_mcp.stdio_transport import _note_client + + try: + _note_client( + SessionMessage( + JSONRPCRequest( + jsonrpc="2.0", + id=1, + method="initialize", + params={"clientInfo": {"name": "grok", "version": "1"}}, + ) + ) + ) + assert client_label() == "Grok" + _note_client("not a message") + finally: + set_client_name("Agent") diff --git a/mcp/tst-cu-mcp/tests/test_overlay_gating.py b/mcp/tst-cu-mcp/tests/test_overlay_gating.py index e8d14fbd..8795ad0e 100644 --- a/mcp/tst-cu-mcp/tests/test_overlay_gating.py +++ b/mcp/tst-cu-mcp/tests/test_overlay_gating.py @@ -34,6 +34,7 @@ reset_overlay, set_overlay, ) +from tst_cu_mcp.overlay.agents import Agent from tst_cu_mcp.overlay.darwin import DarwinOverlay ONE_DISPLAY = [ @@ -90,6 +91,12 @@ def end_session(self, key: str = "") -> None: def set_target(self, display_id: int | None) -> None: self.calls.append(f"target:{display_id}") + def use_agent(self, key: str, label: str = "") -> Agent: + return Agent(key=key, label=label or "Agent", color="#2F6FDE") + + def pointer(self, x: float, y: float, *, click: bool = False) -> None: + self.calls.append("cursor_click" if click else "cursor_move") + def activity(self) -> None: self.calls.append("activity") @@ -289,8 +296,14 @@ def fake_capture_png(_self: Any, _rect: Any) -> bytes: @pytest.mark.parametrize( "action, expected", [ - (lambda: input_control.move_mouse(10, 10), ["target:1", "activity", "move"]), - (lambda: input_control.click(10, 10), ["target:1", "activity", "click"]), + ( + lambda: input_control.move_mouse(10, 10), + ["target:1", "activity", "cursor_move", "move"], + ), + ( + lambda: input_control.click(10, 10), + ["target:1", "activity", "cursor_click", "click"], + ), (lambda: input_control.type_text("hello"), ["activity", "type"]), (lambda: input_control.press_keys("ctrl+c"), ["activity", "press"]), (lambda: input_control.scroll(0, 3), ["activity", "scroll"]), diff --git a/mcp/tst-cu-mcp/tests/test_overlay_rules.py b/mcp/tst-cu-mcp/tests/test_overlay_rules.py index 271c8103..4f366476 100644 --- a/mcp/tst-cu-mcp/tests/test_overlay_rules.py +++ b/mcp/tst-cu-mcp/tests/test_overlay_rules.py @@ -7,6 +7,8 @@ * Non-bracketing clients: the ring fades after the linger, and a client that brackets never falls back to the linger. * Stop and refusals darken it until the next action. +* Cursors (TD-5227): the acting agent's cursor follows its points in both + modes, needs both switches, and every cursor goes on Stop. """ from __future__ import annotations @@ -23,8 +25,9 @@ from tst_cu_mcp import safety from tst_cu_mcp.config import Config, save_config from tst_cu_mcp.overlay import OVERLAY_ENV, get_overlay, reset_overlay -from tst_cu_mcp.overlay.control import GatedOverlay, PolicyMode -from tst_cu_mcp.overlay.darwin_helper import _parse_display_id, _read_stdin, lit_keys +from tst_cu_mcp.overlay.agents import Agent, AgentBook, SlotRegistry +from tst_cu_mcp.overlay.control import GatedOverlay, PolicyFile +from tst_cu_mcp.overlay.darwin_wire import _parse_display_id, _read_stdin, lit_keys class Painter: @@ -48,6 +51,13 @@ def notify_blocked(self) -> None: def set_target(self, display_id: int | None) -> None: self.calls.append(f"target:{display_id}") + def cursor(self, agent: Agent, x: float, y: float, click: bool) -> None: + verb = "click" if click else "move" + self.calls.append(f"cursor:{agent.key}:{agent.label}:{verb}:{x:.0f},{y:.0f}") + + def cursor_hide(self, key: str | None) -> None: + self.calls.append(f"cursor_hide:{key}") + @contextmanager def grab_hidden(self) -> Iterator[None]: self.calls.append("grab_begin") @@ -61,13 +71,22 @@ def shutdown(self) -> None: class World: - """Mode, Stop and time the controller reads, all set by the test.""" + """Policy, Stop and time the controller reads, all set by the test.""" def __init__(self) -> None: self.mode = "full_control" + self.overlay = True + self.cursor = True self.stopped = False self.now = 1000.0 + def policy(self) -> Config: + return Config( + mode=self.mode, # type: ignore[arg-type] + overlay_enabled=self.overlay, + cursor_enabled=self.cursor, + ) + @pytest.fixture def world() -> World: @@ -80,13 +99,14 @@ def painter() -> Painter: @pytest.fixture -def ring(world: World, painter: Painter) -> GatedOverlay: +def ring(world: World, painter: Painter, tmp_path: Path) -> GatedOverlay: return GatedOverlay( painter, - mode=lambda: world.mode, + policy=world.policy, stopped=lambda: world.stopped, clock=lambda: world.now, linger=30.0, + agents=AgentBook(SlotRegistry(tmp_path / "agents")), watch=False, ) @@ -226,7 +246,7 @@ class Broken(Painter): def activity(self) -> None: raise RuntimeError("window server gone") - ring = GatedOverlay(Broken(), mode=lambda: world.mode, stopped=lambda: False, watch=False) + ring = GatedOverlay(Broken(), policy=world.policy, stopped=lambda: False, watch=False) ring.activity() ring.tick() @@ -241,26 +261,81 @@ def test_get_overlay_wraps_the_painter(self, monkeypatch: pytest.MonkeyPatch) -> assert isinstance(resolved.painter, Painter) -class TestPolicyMode: +class TestPolicyFile: def test_missing_file_uses_the_installed_config(self, tmp_path: Path) -> None: safety.set_config(Config(mode="full_control")) try: - assert PolicyMode(lambda: tmp_path / "absent.yaml")() == "full_control" + assert PolicyFile(lambda: tmp_path / "absent.yaml")().mode == "full_control" finally: safety.set_config(None) def test_reads_the_file_and_follows_edits(self, tmp_path: Path) -> None: path = tmp_path / "config.yaml" save_config(Config(mode="full_control"), path) - mode = PolicyMode(lambda: path) - assert mode() == "full_control" - save_config(Config(mode="background", allowed_apps=("Finder",)), path) - assert mode() == "background" + policy = PolicyFile(lambda: path) + assert policy().mode == "full_control" + save_config(Config(mode="background", cursor_enabled=False, allowed_apps=("A",)), path) + assert policy().mode == "background" + assert policy().cursor_enabled is False - def test_a_broken_file_reads_as_background(self, tmp_path: Path) -> None: + def test_a_broken_file_reads_as_background_with_the_overlay_off(self, tmp_path: Path) -> None: path = tmp_path / "config.yaml" path.write_text("mode: sideways\n", encoding="utf-8") - assert PolicyMode(lambda: path)() == "background" + broken = PolicyFile(lambda: path)() + assert broken.mode == "background" + assert broken.overlay_enabled is False + + +class TestCursors: + def test_the_acting_agents_cursor_follows_in_background_too( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + world.mode = "background" + ring.pointer(10, 20) + ring.pointer(30, 40, click=True) + assert painter.calls == [ + "cursor:client:Agent:move:10,20", + "cursor:client:Agent:click:30,40", + ] + + def test_named_agents_get_their_own_cursor_and_color( + self, ring: GatedOverlay, painter: Painter + ) -> None: + first = ring.use_agent("session-a", "Fix the login") + ring.pointer(1, 1) + second = ring.use_agent("session-b", "Write the report") + ring.pointer(2, 2) + assert first.color != second.color + assert painter.calls == [ + "cursor:session-a:Fix the login:move:1,1", + "cursor:session-b:Write the report:move:2,2", + ] + + @pytest.mark.parametrize("switch", ["overlay", "cursor"]) + def test_either_switch_off_draws_nothing( + self, ring: GatedOverlay, painter: Painter, world: World, switch: str + ) -> None: + setattr(world, switch, False) + ring.pointer(10, 20) + assert painter.calls == [] + + def test_switching_off_hides_cursors_already_out( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.pointer(10, 20) + world.cursor = False + ring.tick() + ring.tick() + assert painter.calls == ["cursor:client:Agent:move:10,20", "cursor_hide:None"] + + def test_stop_hides_every_cursor_and_draws_no_more( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.pointer(10, 20) + world.stopped = True + ring.tick() + ring.pointer(30, 40) + assert painter.calls == ["cursor:client:Agent:move:10,20", "cursor_hide:None"] class TestHelperParsing: diff --git a/mcp/tst-cu-mcp/tests/test_stdio_server.py b/mcp/tst-cu-mcp/tests/test_stdio_server.py index ac21d312..28a7499f 100644 --- a/mcp/tst-cu-mcp/tests/test_stdio_server.py +++ b/mcp/tst-cu-mcp/tests/test_stdio_server.py @@ -201,6 +201,7 @@ def test_the_episode_bracket_is_not_on_the_model_facing_surface( computer-use episode is running when none is.""" listed = {tool["name"] for tool in session["responses"][2]["result"]["tools"]} assert "overlay_session" not in listed + assert "overlay_agent" not in listed def test_the_daemons_own_child_still_gets_the_episode_bracket(self) -> None: """The other half: gating it off the model must not take it away from @@ -208,7 +209,7 @@ def test_the_daemons_own_child_still_gets_the_episode_bracket(self) -> None: result = run_session(handshake_requests(), internal=True) listed = {tool["name"] for tool in result["responses"][2]["result"]["tools"]} assert "overlay_session" in listed - assert listed == TOOL_NAMES | {"overlay_session"} + assert listed == TOOL_NAMES | {"overlay_session", "overlay_agent"} def test_every_tool_has_a_description(self, session: dict[str, Any]) -> None: for tool in session["responses"][2]["result"]["tools"]: From 35b7e189cc9a9bc448d93abf490b8c5ba9037a7a Mon Sep 17 00:00:00 2001 From: Adam Crocker <77861255+captcrock@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:32:23 -0500 Subject: [PATCH 2/2] TD-5227: ui_act reports the element frame; captures leave agent-desktop's windows out Two host pieces for per-agent cursors on the real display. ui_act frame. A successful ui_act (press, set_value, focus, raise, show_menu) now replies with "frame": {"x", "y", "width", "height"} in global top-left points, the same space as CGEvent, CGWindow bounds and the display bounds, so the caller can show the action where it happened. The frame comes from AXPosition and AXSize and is read before the action, because a press can close or move the element. It is best effort: when either attribute is missing, is not an AXValue of the right type, or holds a non-finite number, the key is left out and the action still runs. Following TD-5225, the host checks CFGetTypeID against AXValueGetTypeID before AXValueGetValue (cu_cf::ax_point / ax_size). The Python fallback (darwin_ax._local_act) adds the same key through pyobjc. Captures without agent-desktop's windows. agent-desktop draws its agent cursor in ordinary, capturable windows, so a plain on-screen capture showed that cursor to the model, and one agent's cursor in another agent's screenshot. The host capture (new cu_capture module) and the Python local capture (new darwin_capture module) now list on-screen windows (CGWindowListCreate plus CGWindowListCreateDescriptionFromArray) and, when any is owned by a process named exactly "agent-desktop", build the image with CGWindowListCreateImageFromArray from the rest, front to back. With no such window the existing CGWindowListCreateImage call runs unchanged, and a null from-array image falls back to it. The filter is a pure function with unit tests on both sides. The new code lives in its own module so cu_agent.rs gets smaller; capture_png now holds the image as an Owned reference. Class A notes: a window whose description is missing (it closed between the two calls) is kept; owner matching is exact and case-sensitive. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HUthoCdmzg2hy89ssk2cSF --- .../src/tst_cu_mcp/backends/darwin.py | 18 +- .../src/tst_cu_mcp/backends/darwin_ax.py | 51 +++- .../src/tst_cu_mcp/backends/darwin_capture.py | 94 +++++++ .../tests/test_darwin_capture_exclude.py | 149 ++++++++++ mcp/tst-cu-mcp/tests/test_ui_act_frame.py | 159 +++++++++++ shell/src/cu_agent.rs | 48 +--- shell/src/cu_ax.rs | 80 +++++- shell/src/cu_capture.rs | 260 ++++++++++++++++++ shell/src/cu_cf.rs | 109 ++++++++ shell/src/lib.rs | 1 + 10 files changed, 912 insertions(+), 57 deletions(-) create mode 100644 mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_capture.py create mode 100644 mcp/tst-cu-mcp/tests/test_darwin_capture_exclude.py create mode 100644 mcp/tst-cu-mcp/tests/test_ui_act_frame.py create mode 100644 shell/src/cu_capture.rs diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin.py index 5e975332..02c791df 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin.py @@ -24,6 +24,7 @@ from typing import Any from tst_cu_mcp.backends.base import Rect +from tst_cu_mcp.backends.darwin_capture import screen_image from tst_cu_mcp.displays import DisplayInfo from tst_cu_mcp.focus import WindowInfo from tst_cu_mcp.permissions import ACCESSIBILITY, NO_HOST_FIX, SCREEN_RECORDING, build_report @@ -677,24 +678,11 @@ def _cg_capture_png(rect: Rect) -> bytes | None: if w <= 0 or h <= 0: return None try: - from Quartz import ( - CGImageGetHeight, - CGImageGetWidth, - CGRectMake, - CGWindowListCreateImage, - kCGNullWindowID, - kCGWindowImageDefault, - kCGWindowListOptionOnScreenOnly, - ) + from Quartz import CGImageGetHeight, CGImageGetWidth, CGRectMake except (ImportError, AttributeError): return None try: - image = CGWindowListCreateImage( - CGRectMake(float(x), float(y), float(w), float(h)), - kCGWindowListOptionOnScreenOnly, - kCGNullWindowID, - kCGWindowImageDefault, - ) + image = screen_image(CGRectMake(float(x), float(y), float(w), float(h))) except Exception: return None if image is None: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_ax.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_ax.py index 9caf094c..5cbd8c88 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_ax.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_ax.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +import math import sys from typing import Any @@ -126,6 +127,8 @@ def _read_element(element: Any) -> dict[str, Any]: def _local_act(pid: int, element_id: str, action: str, value: str) -> dict[str, Any]: target = _element_at(pid, element_id) + # Read first: a press can close the element or move it. + frame = _frame(target) if action == "press": _perform(target, "AXPress") elif action == "raise": @@ -140,7 +143,53 @@ def _local_act(pid: int, element_id: str, action: str, value: str) -> dict[str, raise ValueError( f"unknown ui action {action!r}; use one of press, set_value, focus, raise, show_menu" ) - return {"ok": True, "pid": pid, "id": element_id, "action": action, "path": "local"} + result: dict[str, Any] = { + "ok": True, + "pid": pid, + "id": element_id, + "action": action, + "path": "local", + } + if frame is not None: + result["frame"] = frame + return result + + +def _frame(element: Any) -> dict[str, float] | None: + """Where *element* is, in global top-left points (the CGEvent and CGWindow + space), so the caller can show the action there. Best effort: ``None`` + when either attribute is missing or not the AXValue it should be.""" + try: + from ApplicationServices import kAXValueCGPointType, kAXValueCGSizeType + except (ImportError, AttributeError): + return None + position = _ax_pair(_attr(element, "AXPosition"), kAXValueCGPointType, ("x", "y")) + size = _ax_pair(_attr(element, "AXSize"), kAXValueCGSizeType, ("width", "height")) + if position is None or size is None: + return None + return {"x": position[0], "y": position[1], "width": size[0], "height": size[1]} + + +def _ax_pair(value: Any, kind: int, fields: tuple[str, str]) -> tuple[float, float] | None: + """Two finite numbers out of an AXValue of *kind*. Only an AXValueRef is + handed to AXValueGetValue: reading a value as the wrong CF type is what + crashed the host (TD-5225).""" + if value is None: + return None + try: + from ApplicationServices import AXValueGetTypeID, AXValueGetValue, CFGetTypeID + except (ImportError, AttributeError): + return None + try: + if CFGetTypeID(value) != AXValueGetTypeID(): + return None + result = AXValueGetValue(value, kind, None) + if not isinstance(result, tuple) or len(result) < 2 or not result[0]: + return None + pair = (float(getattr(result[1], fields[0])), float(getattr(result[1], fields[1]))) + except Exception: + return None + return pair if all(math.isfinite(n) for n in pair) else None def _application(pid: int) -> Any: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_capture.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_capture.py new file mode 100644 index 00000000..ad7b677f --- /dev/null +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/backends/darwin_capture.py @@ -0,0 +1,94 @@ +"""Screen capture that leaves out agent-desktop's cursor windows. + +agent-desktop draws each agent's cursor in ordinary, capturable windows. A +plain on-screen capture would show those cursors to the model, and one +agent's cursor in another agent's screenshot. When such a window is on +screen, the image is composed from the on-screen window list without them; +otherwise the plain capture runs unchanged. Mirrors the host's +``cu_capture.rs``. Function-local pyobjc so the module imports anywhere. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +EXCLUDED_OWNER = "agent-desktop" + + +def kept_window_ids(windows: Iterable[tuple[int, str]]) -> list[int]: + """Window ids a capture draws: every ``(id, owner name)`` pair, front to + back, except agent-desktop's, in the same order.""" + return [window_id for window_id, owner in windows if owner != EXCLUDED_OWNER] + + +def screen_image(rect: Any) -> Any: + """The on-screen ``CGImage`` of *rect* (a ``CGRect`` in global points) + without agent-desktop's windows. Raises what the plain call raises.""" + from Quartz import ( + CGWindowListCreateImage, + kCGNullWindowID, + kCGWindowImageDefault, + kCGWindowListOptionOnScreenOnly, + ) + + kept = _kept_on_screen() + if kept is not None: + image = _image_of(rect, kept) + if image is not None: + return image + return CGWindowListCreateImage( + rect, kCGWindowListOptionOnScreenOnly, kCGNullWindowID, kCGWindowImageDefault + ) + + +def _image_of(rect: Any, window_ids: list[int]) -> Any: + try: + from Quartz import CGWindowListCreateImageFromArray, kCGWindowImageDefault + + return CGWindowListCreateImageFromArray(rect, window_ids, kCGWindowImageDefault) + except Exception: + return None + + +def _kept_on_screen() -> list[int] | None: + """Window ids to draw, front to back, or ``None`` when no agent-desktop + window is on screen (the plain capture is then the same picture, and + cheaper) or the window list cannot be read.""" + try: + from Quartz import ( + CGWindowListCreate, + CGWindowListCreateDescriptionFromArray, + kCGNullWindowID, + kCGWindowListOptionOnScreenOnly, + ) + + ids = CGWindowListCreate(kCGWindowListOptionOnScreenOnly, kCGNullWindowID) + described = CGWindowListCreateDescriptionFromArray(ids) if ids else None + except Exception: + return None + windows = _owners(ids or (), described or ()) + kept = kept_window_ids(windows) + return kept if len(kept) < len(windows) else None + + +def _owners(ids: Iterable[Any], described: Iterable[Any]) -> list[tuple[int, str]]: + """``(id, owner name)`` for each id, in the ids' order. A window with no + description (it closed meanwhile) gets an empty owner, so it is kept.""" + names: dict[int, str] = {} + for entry in described: + getter = getattr(entry, "get", None) + if getter is None: + continue + try: + names[int(getter("kCGWindowNumber"))] = str(getter("kCGWindowOwnerName") or "") + except (TypeError, ValueError): + continue + windows: list[tuple[int, str]] = [] + for raw in ids: + try: + window_id = int(raw) + except (TypeError, ValueError): + continue + windows.append((window_id, names.get(window_id, ""))) + return windows diff --git a/mcp/tst-cu-mcp/tests/test_darwin_capture_exclude.py b/mcp/tst-cu-mcp/tests/test_darwin_capture_exclude.py new file mode 100644 index 00000000..d2e8f603 --- /dev/null +++ b/mcp/tst-cu-mcp/tests/test_darwin_capture_exclude.py @@ -0,0 +1,149 @@ +"""Local captures leave agent-desktop's cursor windows out (TD-5227). + +agent-desktop draws each agent's cursor in ordinary on-screen windows. A +screenshot must not show them: the model would see its own cursor, and one +agent would see another's. The fake stands in for pyobjc's Quartz, so these +run without macOS. +""" + +from __future__ import annotations + +import sys +from types import ModuleType +from typing import Any, cast + +import pytest + +from tst_cu_mcp.backends import darwin +from tst_cu_mcp.backends.darwin_capture import kept_window_ids, screen_image + +PLAIN = "plain image" +FROM_ARRAY = "from-array image" +RECT = (0.0, 0.0, 800.0, 600.0) + + +class FakeQuartz: + kCGWindowListOptionOnScreenOnly = 1 + kCGNullWindowID = 0 + kCGWindowImageDefault = 0 + + def __init__(self, windows: list[tuple[int, str]], *, described: bool = True) -> None: + self.windows = windows + self.described = described + self.from_array_result: Any = FROM_ARRAY + self.list_error: Exception | None = None + self.calls: list[tuple[str, Any]] = [] + + def CGWindowListCreate(self, option: int, relative_to: int) -> tuple[int, ...]: + if self.list_error is not None: + raise self.list_error + assert (option, relative_to) == (1, 0) + return tuple(window_id for window_id, _ in self.windows) + + def CGWindowListCreateDescriptionFromArray(self, ids: tuple[int, ...]) -> list[dict[str, Any]]: + listed = [ + {"kCGWindowNumber": window_id, "kCGWindowOwnerName": owner} + for window_id, owner in self.windows + if window_id in ids + ] + # Real descriptions skip windows that closed after the id list was read. + return listed if self.described else listed[1:] + + def CGWindowListCreateImageFromArray(self, rect: Any, ids: list[int], option: int) -> Any: + self.calls.append(("from_array", list(ids))) + return self.from_array_result + + def CGWindowListCreateImage(self, rect: Any, option: int, window: int, image: int) -> Any: + self.calls.append(("plain", (option, window, image))) + return PLAIN + + def CGRectMake(self, x: float, y: float, w: float, h: float) -> tuple[float, ...]: + return (x, y, w, h) + + def CGImageGetWidth(self, _image: Any) -> int: + return 800 + + def CGImageGetHeight(self, _image: Any) -> int: + return 600 + + +def install(monkeypatch: pytest.MonkeyPatch, fake: FakeQuartz) -> FakeQuartz: + monkeypatch.setitem(sys.modules, "Quartz", cast(ModuleType, fake)) + return fake + + +DESKTOP = [(3, "Window Server"), (40, "Finder"), (12, "TST Desk"), (2, "Dock")] + + +def test_kept_ids_drop_agent_desktop_and_keep_the_order() -> None: + windows = [ + (91, "agent-desktop"), + (40, "Finder"), + (92, "agent-desktop"), + (12, "TST Desk"), + (3, "Window Server"), + ] + assert kept_window_ids(windows) == [40, 12, 3] + + +def test_only_the_exact_owner_name_is_dropped() -> None: + windows = [(1, "agent-desktop-helper"), (2, "Agent-Desktop"), (3, " agent-desktop"), (4, "")] + assert kept_window_ids(windows) == [1, 2, 3, 4] + + +def test_no_agent_desktop_window_keeps_the_plain_capture(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install(monkeypatch, FakeQuartz(DESKTOP)) + + assert screen_image(RECT) == PLAIN + assert fake.calls == [("plain", (1, 0, 0))] + + +def test_agent_desktop_windows_are_left_out(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install(monkeypatch, FakeQuartz([(91, "agent-desktop"), *DESKTOP])) + + assert screen_image(RECT) == FROM_ARRAY + assert fake.calls == [("from_array", [3, 40, 12, 2])] + + +def test_a_null_from_array_image_falls_back_to_the_plain_capture( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake = install(monkeypatch, FakeQuartz([(91, "agent-desktop"), *DESKTOP])) + fake.from_array_result = None + + assert screen_image(RECT) == PLAIN + assert [name for name, _ in fake.calls] == ["from_array", "plain"] + + +def test_an_unreadable_window_list_keeps_the_plain_capture( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake = install(monkeypatch, FakeQuartz([(91, "agent-desktop"), *DESKTOP])) + fake.list_error = RuntimeError("window server busy") + + assert screen_image(RECT) == PLAIN + assert [name for name, _ in fake.calls] == ["plain"] + + +def test_a_window_without_a_description_is_kept(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install( + monkeypatch, FakeQuartz([(7, "Safari"), (91, "agent-desktop"), *DESKTOP], described=False) + ) + + assert screen_image(RECT) == FROM_ARRAY + assert fake.calls == [("from_array", [7, 3, 40, 12, 2])] + + +def test_the_local_capture_path_uses_it(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install(monkeypatch, FakeQuartz([(91, "agent-desktop"), *DESKTOP])) + encoded: list[Any] = [] + + def png(image: Any) -> bytes: + encoded.append(image) + return b"\x89PNG fake" + + monkeypatch.setattr(darwin, "_png_from_cgimage", png) + + assert darwin._cg_capture_png((0, 0, 800, 600)) == b"\x89PNG fake" + assert encoded == [FROM_ARRAY] + assert fake.calls == [("from_array", [3, 40, 12, 2])] diff --git a/mcp/tst-cu-mcp/tests/test_ui_act_frame.py b/mcp/tst-cu-mcp/tests/test_ui_act_frame.py new file mode 100644 index 00000000..d8aef808 --- /dev/null +++ b/mcp/tst-cu-mcp/tests/test_ui_act_frame.py @@ -0,0 +1,159 @@ +"""A local ``ui_action`` reports where the element is (TD-5227). + +The frame is read before the action (a press can close or move the element) +and is best effort: anything other than a point and a size of the right +AXValue type leaves it out, and reading it never fails the action. The fake +stands in for pyobjc's ApplicationServices, so these run without macOS. +""" + +from __future__ import annotations + +import sys +from types import ModuleType, SimpleNamespace +from typing import Any, cast + +import pytest + +from tst_cu_mcp.backends import darwin_ax + +POINT = 1 +SIZE = 2 +AX_VALUE_TYPE_ID = 121 +AX_ERROR_NO_VALUE = -25212 + + +class FakeAxValue: + def __init__(self, kind: int, **fields: float) -> None: + self.kind = kind + self.fields = fields + + +class FakeAppServices: + """The slice of pyobjc's ApplicationServices that ``_local_act`` calls.""" + + kAXValueCGPointType = POINT + kAXValueCGSizeType = SIZE + + def __init__(self, attrs: dict[str, Any]) -> None: + self.attrs = attrs + self.events: list[str] = [] + self.type_id_error: Exception | None = None + + def AXValueGetTypeID(self) -> int: + return AX_VALUE_TYPE_ID + + def CFGetTypeID(self, value: Any) -> int: + if self.type_id_error is not None: + raise self.type_id_error + return AX_VALUE_TYPE_ID if isinstance(value, FakeAxValue) else 7 + + def AXValueGetValue(self, value: Any, kind: int, _out: None) -> tuple[bool, Any]: + if not isinstance(value, FakeAxValue): + # Undefined for a non-AXValue: model it as plausible garbage, so + # only the CF type check keeps it out of the reply. + return (True, SimpleNamespace(x=0.0, y=0.0, width=1.0, height=1.0)) + if value.kind != kind: + return (False, None) + return (True, SimpleNamespace(**value.fields)) + + def AXUIElementCopyAttributeValue(self, _el: Any, name: str, _out: None) -> tuple[int, Any]: + self.events.append(f"read {name}") + if name in self.attrs: + return (0, self.attrs[name]) + return (AX_ERROR_NO_VALUE, None) + + def AXUIElementPerformAction(self, _el: Any, action: str) -> int: + self.events.append(f"perform {action}") + self.attrs.clear() # the press closed the element + return 0 + + def AXUIElementSetAttributeValue(self, _el: Any, name: str, _value: Any) -> int: + self.events.append(f"set {name}") + return 0 + + +def install(monkeypatch: pytest.MonkeyPatch, attrs: dict[str, Any]) -> FakeAppServices: + fake = FakeAppServices(attrs) + monkeypatch.setitem(sys.modules, "ApplicationServices", cast(ModuleType, fake)) + monkeypatch.setattr(darwin_ax, "_element_at", lambda _pid, _id: object()) + return fake + + +def button_at(x: float, y: float, width: float, height: float) -> dict[str, Any]: + return { + "AXPosition": FakeAxValue(POINT, x=x, y=y), + "AXSize": FakeAxValue(SIZE, width=width, height=height), + } + + +def test_a_press_reports_the_frame_read_before_it(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install(monkeypatch, button_at(120.0, 48.5, 64.0, 22.0)) + + result = darwin_ax._local_act(4242, "0.3.1", "press", "") + + assert result["ok"] is True + assert result["action"] == "press" + assert result["frame"] == {"x": 120.0, "y": 48.5, "width": 64.0, "height": 22.0} + assert fake.events.index("perform AXPress") > fake.events.index("read AXSize") + + +@pytest.mark.parametrize("action", ["set_value", "focus", "raise", "show_menu"]) +def test_every_action_reports_the_frame(monkeypatch: pytest.MonkeyPatch, action: str) -> None: + install(monkeypatch, button_at(-1440.0, 10.0, 300.0, 24.0)) + + result = darwin_ax._local_act(4242, "0.1", action, "hello") + + assert result["frame"] == {"x": -1440.0, "y": 10.0, "width": 300.0, "height": 24.0} + + +@pytest.mark.parametrize( + "attrs", + [ + pytest.param({"AXPosition": FakeAxValue(POINT, x=1.0, y=2.0)}, id="no-size"), + pytest.param({"AXSize": FakeAxValue(SIZE, width=3.0, height=4.0)}, id="no-position"), + pytest.param( + {"AXPosition": "12,40", "AXSize": FakeAxValue(SIZE, width=3.0, height=4.0)}, + id="position-is-a-string", + ), + pytest.param( + { + "AXPosition": FakeAxValue(POINT, x=1.0, y=2.0), + "AXSize": FakeAxValue(POINT, x=3.0, y=4.0), + }, + id="size-is-a-point", + ), + pytest.param( + { + "AXPosition": FakeAxValue(POINT, x=float("nan"), y=2.0), + "AXSize": FakeAxValue(SIZE, width=3.0, height=4.0), + }, + id="not-finite", + ), + ], +) +def test_an_unreadable_frame_is_left_out( + monkeypatch: pytest.MonkeyPatch, attrs: dict[str, Any] +) -> None: + install(monkeypatch, attrs) + + result = darwin_ax._local_act(4242, "0.1", "press", "") + + assert result["ok"] is True + assert "frame" not in result + + +def test_a_failing_frame_read_never_fails_the_action(monkeypatch: pytest.MonkeyPatch) -> None: + fake = install(monkeypatch, button_at(1.0, 2.0, 3.0, 4.0)) + fake.type_id_error = RuntimeError("bridge refused the value") + + result = darwin_ax._local_act(4242, "0.1", "press", "") + + assert result["ok"] is True + assert "frame" not in result + assert "perform AXPress" in fake.events + + +def test_no_frame_without_pyobjc(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "ApplicationServices", None) + + assert darwin_ax._frame(object()) is None diff --git a/shell/src/cu_agent.rs b/shell/src/cu_agent.rs index fbd29db1..b533bcfc 100644 --- a/shell/src/cu_agent.rs +++ b/shell/src/cu_agent.rs @@ -1,7 +1,7 @@ //! Computer-use actuator in the TST Desk host process. //! //! Checkout Python and the `tstd` sidecar are the wrong TCC identity. -//! Capture (`CGWindowListCreateImage`) and input (`CGEventPost`) run here so +//! Capture (`cu_capture`) and input (`CGEventPost`) run here so //! System Settings attributes both to **TST Desk**, not a helper named //! `tst-desk` / `tstd`. The protocol matches `core/tstd/cu_host.py` so //! Darwin `tst-cu-mcp` stays a Unix-socket client. @@ -274,7 +274,6 @@ mod macos { const K_CG_MOUSE_BUTTON_LEFT: u32 = 0; const K_CG_MOUSE_BUTTON_RIGHT: u32 = 1; const K_CG_MOUSE_EVENT_CLICK_STATE: u32 = 1; - const K_CG_WINDOW_LIST_OPTION_ON_SCREEN_ONLY: u32 = 1; const K_CF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; const K_CG_SCROLL_EVENT_UNIT_LINE: u32 = 0; @@ -362,18 +361,6 @@ mod macos { y: f64, } - #[repr(C)] - struct CGSize { - width: f64, - height: f64, - } - - #[repr(C)] - struct CGRect { - origin: CGPoint, - size: CGSize, - } - #[repr(C)] struct CFDictionaryKeyCallBacks { version: isize, @@ -410,12 +397,6 @@ mod macos { fn CGEventSetFlags(event: *mut c_void, flags: u64); fn CGEventKeyboardSetUnicodeString(event: *mut c_void, length: u32, string: *const u16); fn CGEventPost(tap: u32, event: *mut c_void); - fn CGWindowListCreateImage( - rect: CGRect, - list_option: u32, - window_id: u32, - image_option: u32, - ) -> *mut c_void; fn CGImageGetWidth(image: *mut c_void) -> usize; fn CGImageGetHeight(image: *mut c_void) -> usize; fn CGPreflightScreenCaptureAccess() -> u8; @@ -912,27 +893,15 @@ mod macos { if w <= 0 || h <= 0 { return None; } + let owned = crate::cu_capture::screen_image( + f64::from(x), + f64::from(y), + f64::from(w), + f64::from(h), + )?; + let image = owned.as_ptr(); unsafe { - let image = CGWindowListCreateImage( - CGRect { - origin: CGPoint { - x: f64::from(x), - y: f64::from(y), - }, - size: CGSize { - width: f64::from(w), - height: f64::from(h), - }, - }, - K_CG_WINDOW_LIST_OPTION_ON_SCREEN_ONLY, - 0, - 0, - ); - if image.is_null() { - return None; - } if CGImageGetWidth(image) < 1 || CGImageGetHeight(image) < 1 { - CFRelease(image); return None; } let uti = CFStringCreateWithCString( @@ -976,7 +945,6 @@ mod macos { if !uti.is_null() { CFRelease(uti); } - CFRelease(image); png } } diff --git a/shell/src/cu_ax.rs b/shell/src/cu_ax.rs index 058e2a1d..2a19306a 100644 --- a/shell/src/cu_ax.rs +++ b/shell/src/cu_ax.rs @@ -146,6 +146,8 @@ unsafe fn element_at(app: *mut c_void, element_id: &str) -> Option { } unsafe fn perform(element: *mut c_void, action: &str, value: &str) -> Value { + // Read first: a press can close the element or move it. + let frame = frame(element); let err = match action { "press" => perform_named(element, "AXPress"), "raise" => perform_named(element, "AXRaise"), @@ -162,7 +164,31 @@ unsafe fn perform(element: *mut c_void, action: &str, value: &str) -> Value { if err != AX_OK { return json!({"ok": false, "error": format!("AX {action} refused (error {err})")}); } - json!({"ok": true, "action": action, "path": "host"}) + done(action, frame) +} + +/// The success reply, with the element's frame when it could be read. +fn done(action: &str, frame: Option) -> Value { + let mut reply = json!({"ok": true, "action": action, "path": "host"}); + if let Some(frame) = frame { + reply["frame"] = frame; + } + reply +} + +/// Where the element is, in global top-left points (the CGEvent and +/// CGWindow space), so the caller can show the action there. Best effort: +/// `None` when either attribute is missing or not the AXValue it should be. +unsafe fn frame(element: *mut c_void) -> Option { + let position = attr_copy(element, "AXPosition")?; + let size = attr_copy(element, "AXSize")?; + frame_of(position.as_ptr(), size.as_ptr()) +} + +unsafe fn frame_of(position: *const c_void, size: *const c_void) -> Option { + let (x, y) = cu_cf::ax_point(position)?; + let (width, height) = cu_cf::ax_size(size)?; + Some(json!({"x": x, "y": y, "width": width, "height": height})) } unsafe fn perform_named(element: *mut c_void, name: &str) -> i32 { @@ -226,3 +252,55 @@ unsafe fn action_names(element: *mut c_void) -> Vec { names.push("focus".into()); names } + +#[cfg(test)] +mod tests { + use super::*; + + #[link(name = "ApplicationServices", kind = "framework")] + extern "C" { + fn AXValueCreate(the_type: u32, value_ptr: *const c_void) -> *const c_void; + } + + const POINT: u32 = 1; + const SIZE: u32 = 2; + + fn ax_value(kind: u32, fields: [f64; 2]) -> Owned { + unsafe { Owned::adopt(AXValueCreate(kind, fields.as_ptr().cast())).unwrap() } + } + + #[test] + fn frame_is_position_and_size_in_points() { + let position = ax_value(POINT, [120.0, 48.5]); + let size = ax_value(SIZE, [64.0, 22.0]); + let got = unsafe { frame_of(position.as_ptr(), size.as_ptr()) }; + assert_eq!( + got, + Some(json!({"x": 120.0, "y": 48.5, "width": 64.0, "height": 22.0})) + ); + } + + #[test] + fn frame_is_none_when_an_attribute_is_the_wrong_type() { + let position = ax_value(POINT, [1.0, 2.0]); + let size = ax_value(SIZE, [3.0, 4.0]); + let text = cu_cf::cfstring("AXSize").unwrap(); + unsafe { + assert_eq!(frame_of(size.as_ptr(), position.as_ptr()), None); + assert_eq!(frame_of(position.as_ptr(), text.as_ptr()), None); + assert_eq!(frame_of(position.as_ptr(), std::ptr::null()), None); + } + } + + #[test] + fn success_reply_carries_the_frame_only_when_read() { + let frame = json!({"x": 1.0, "y": 2.0, "width": 3.0, "height": 4.0}); + let with = done("press", Some(frame.clone())); + assert_eq!(with["frame"], frame); + assert_eq!(with["ok"], true); + assert_eq!(with["action"], "press"); + let without = done("focus", None); + assert!(without.get("frame").is_none()); + assert_eq!(without["ok"], true); + } +} diff --git a/shell/src/cu_capture.rs b/shell/src/cu_capture.rs new file mode 100644 index 00000000..8d517c90 --- /dev/null +++ b/shell/src/cu_capture.rs @@ -0,0 +1,260 @@ +//! Screen capture that leaves out agent-desktop's cursor windows (TD-5227). +//! +//! agent-desktop draws each agent's cursor in ordinary, capturable windows. +//! A plain on-screen capture would show those cursors to the model, and one +//! agent's cursor in another agent's screenshot. When such a window is on +//! screen, the image is composed from the on-screen window list without +//! them; otherwise the plain capture runs unchanged. + +/// The window ids a capture draws: every window in `windows` (id and owner +/// name, front to back) except agent-desktop's, in the same order. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +pub fn kept_windows(windows: &[(u32, String)]) -> Vec { + const EXCLUDED_OWNER: &str = "agent-desktop"; + windows + .iter() + .filter(|(_, owner)| owner != EXCLUDED_OWNER) + .map(|(id, _)| *id) + .collect() +} + +#[cfg(target_os = "macos")] +pub use macos::screen_image; + +#[cfg(target_os = "macos")] +mod macos { + use std::collections::HashMap; + use std::ffi::c_void; + use std::ptr; + + use super::kept_windows; + use crate::cu_cf::{self, Owned}; + + const K_CG_WINDOW_LIST_OPTION_ON_SCREEN_ONLY: u32 = 1; + const K_CG_NULL_WINDOW_ID: u32 = 0; + const K_CG_WINDOW_IMAGE_DEFAULT: u32 = 0; + + #[repr(C)] + #[derive(Clone, Copy)] + struct CGPoint { + x: f64, + y: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct CGSize { + width: f64, + height: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct CGRect { + origin: CGPoint, + size: CGSize, + } + + #[link(name = "CoreGraphics", kind = "framework")] + extern "C" { + fn CGWindowListCreate(option: u32, relative_to_window: u32) -> *const c_void; + fn CGWindowListCreateDescriptionFromArray(window_array: *const c_void) -> *const c_void; + fn CGWindowListCreateImage( + rect: CGRect, + list_option: u32, + window_id: u32, + image_option: u32, + ) -> *mut c_void; + fn CGWindowListCreateImageFromArray( + rect: CGRect, + window_array: *const c_void, + image_option: u32, + ) -> *mut c_void; + static kCGWindowNumber: *const c_void; + static kCGWindowOwnerName: *const c_void; + } + + #[link(name = "CoreFoundation", kind = "framework")] + extern "C" { + fn CFGetTypeID(cf: *const c_void) -> usize; + fn CFArrayGetTypeID() -> usize; + fn CFDictionaryGetTypeID() -> usize; + fn CFArrayGetCount(the_array: *const c_void) -> isize; + fn CFArrayGetValueAtIndex(the_array: *const c_void, idx: isize) -> *const c_void; + fn CFArrayCreate( + alloc: *const c_void, + values: *const *const c_void, + num_values: isize, + callbacks: *const c_void, + ) -> *const c_void; + fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void; + } + + /// The on-screen pixels of a rect in global top-left points, as a + /// CGImage, with agent-desktop's windows left out. + pub fn screen_image(x: f64, y: f64, width: f64, height: f64) -> Option { + let rect = CGRect { + origin: CGPoint { x, y }, + size: CGSize { width, height }, + }; + // SAFETY: each call gets live CF objects or null, and every Create + // result is adopted exactly once. + unsafe { + if let Some(kept) = kept_on_screen() { + let image = id_array(&kept).and_then(|ids| { + Owned::adopt(CGWindowListCreateImageFromArray( + rect, + ids.as_ptr(), + K_CG_WINDOW_IMAGE_DEFAULT, + )) + }); + if image.is_some() { + return image; + } + } + Owned::adopt(CGWindowListCreateImage( + rect, + K_CG_WINDOW_LIST_OPTION_ON_SCREEN_ONLY, + K_CG_NULL_WINDOW_ID, + K_CG_WINDOW_IMAGE_DEFAULT, + )) + } + } + + /// The window ids to draw, front to back, or `None` when no + /// agent-desktop window is on screen: the plain capture is then the + /// same picture and cheaper. + unsafe fn kept_on_screen() -> Option> { + let ids = Owned::adopt(CGWindowListCreate( + K_CG_WINDOW_LIST_OPTION_ON_SCREEN_ONLY, + K_CG_NULL_WINDOW_ID, + ))?; + let windows = owners(ids.as_ptr()); + let kept = kept_windows(&windows); + (kept.len() < windows.len()).then_some(kept) + } + + /// (id, owner name) for each window of a CGWindowList id array, in the + /// array's order. A window with no description (it closed meanwhile) + /// gets an empty owner, so it is kept. + unsafe fn owners(ids: *const c_void) -> Vec<(u32, String)> { + let mut names: HashMap = HashMap::new(); + if let Some(described) = Owned::adopt(CGWindowListCreateDescriptionFromArray(ids)) { + for entry in cu_cf::items(described.as_ptr()) { + let entry = entry.as_ptr(); + if CFGetTypeID(entry) != CFDictionaryGetTypeID() { + continue; + } + let number = cu_cf::number_value(CFDictionaryGetValue(entry, kCGWindowNumber)); + if let Some(id) = number.and_then(window_id) { + let owner = CFDictionaryGetValue(entry, kCGWindowOwnerName); + names.insert(id, cu_cf::text(owner)); + } + } + } + window_ids(ids) + .into_iter() + .map(|id| (id, names.remove(&id).unwrap_or_default())) + .collect() + } + + /// The ids in a CGWindowList array. Its values are CGWindowIDs stored as + /// pointer-sized integers, not CF objects, so nothing is retained. + unsafe fn window_ids(ids: *const c_void) -> Vec { + if ids.is_null() || CFGetTypeID(ids) != CFArrayGetTypeID() { + return Vec::new(); + } + (0..CFArrayGetCount(ids).max(0)) + .filter_map(|idx| u32::try_from(CFArrayGetValueAtIndex(ids, idx) as usize).ok()) + .collect() + } + + /// The same array shape CGWindowListCreate returns. NULL callbacks: the + /// values are integers, not objects to retain or release. + unsafe fn id_array(ids: &[u32]) -> Option { + let values: Vec<*const c_void> = + ids.iter().map(|&id| id as usize as *const c_void).collect(); + Owned::adopt(CFArrayCreate( + ptr::null(), + values.as_ptr(), + values.len() as isize, + ptr::null(), + )) + } + + fn window_id(number: f64) -> Option { + let whole = number.fract() == 0.0 && (0.0..=f64::from(u32::MAX)).contains(&number); + whole.then_some(number as u32) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn window_ids_survive_the_pointer_sized_array() { + let ids = [3, 10_238, u32::MAX]; + let array = unsafe { id_array(&ids) }.unwrap(); + assert_eq!(unsafe { window_ids(array.as_ptr()) }, ids); + } + + #[test] + fn window_ids_of_a_non_array_is_empty() { + let text = cu_cf::cfstring("not a list").unwrap(); + unsafe { + assert!(window_ids(text.as_ptr()).is_empty()); + assert!(window_ids(ptr::null()).is_empty()); + } + } + + #[test] + fn window_numbers_must_be_whole_u32s() { + assert_eq!(window_id(42.0), Some(42)); + assert_eq!(window_id(f64::from(u32::MAX)), Some(u32::MAX)); + assert_eq!(window_id(1.5), None); + assert_eq!(window_id(-1.0), None); + assert_eq!(window_id(f64::from(u32::MAX) + 1.0), None); + } + } +} + +#[cfg(test)] +mod tests { + use super::kept_windows; + + fn listing(rows: &[(u32, &str)]) -> Vec<(u32, String)> { + rows.iter() + .map(|(id, owner)| (*id, owner.to_string())) + .collect() + } + + #[test] + fn agent_desktop_windows_are_left_out_in_order() { + let windows = listing(&[ + (91, "agent-desktop"), + (40, "Finder"), + (92, "agent-desktop"), + (12, "TST Desk"), + (3, "Window Server"), + ]); + assert_eq!(kept_windows(&windows), vec![40, 12, 3]); + } + + #[test] + fn only_the_exact_owner_name_is_left_out() { + let windows = listing(&[ + (1, "agent-desktop-helper"), + (2, "Agent-Desktop"), + (3, " agent-desktop"), + (4, ""), + ]); + assert_eq!(kept_windows(&windows), vec![1, 2, 3, 4]); + } + + #[test] + fn nothing_to_leave_out_keeps_every_window() { + let windows = listing(&[(5, "Safari"), (6, "Dock")]); + assert_eq!(kept_windows(&windows), vec![5, 6]); + assert!(kept_windows(&[]).is_empty()); + } +} diff --git a/shell/src/cu_cf.rs b/shell/src/cu_cf.rs index e9d10eaa..7ab99c35 100644 --- a/shell/src/cu_cf.rs +++ b/shell/src/cu_cf.rs @@ -16,6 +16,8 @@ use std::ptr; const K_CF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; // i32, matching the declaration in cu_identity.rs (the extern must agree). const K_CF_NUMBER_DOUBLE_TYPE: i32 = 13; +const K_AX_VALUE_CG_POINT_TYPE: u32 = 1; +const K_AX_VALUE_CG_SIZE_TYPE: u32 = 2; #[link(name = "CoreFoundation", kind = "framework")] extern "C" { @@ -45,6 +47,12 @@ extern "C" { static kCFBooleanTrue: *const c_void; } +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn AXValueGetTypeID() -> usize; + fn AXValueGetValue(value: *const c_void, the_type: u32, value_ptr: *mut c_void) -> u8; +} + /// One reference to a CF object, released on drop. pub struct Owned(*const c_void); @@ -152,6 +160,48 @@ pub unsafe fn items(cf: *const c_void) -> Vec { .collect() } +/// A CFNumber as a finite double. `None` for any other type. +/// +/// # Safety +/// `cf` must be null or a live CF object. +pub unsafe fn number_value(cf: *const c_void) -> Option { + if cf.is_null() || CFGetTypeID(cf) != CFNumberGetTypeID() { + return None; + } + number_of(cf) +} + +/// `(x, y)` of an AXValue holding a CGPoint, such as `AXPosition`. +/// +/// # Safety +/// `cf` must be null or a live CF object. +pub unsafe fn ax_point(cf: *const c_void) -> Option<(f64, f64)> { + ax_pair(cf, K_AX_VALUE_CG_POINT_TYPE) +} + +/// `(width, height)` of an AXValue holding a CGSize, such as `AXSize`. +/// +/// # Safety +/// `cf` must be null or a live CF object. +pub unsafe fn ax_size(cf: *const c_void) -> Option<(f64, f64)> { + ax_pair(cf, K_AX_VALUE_CG_SIZE_TYPE) +} + +/// CGPoint and CGSize are both two doubles, so one buffer serves both. +/// AXValueGetValue refuses a value stored as another type (a CGRect +/// would not fit), but only an AXValueRef may be passed to it at all. +unsafe fn ax_pair(cf: *const c_void, kind: u32) -> Option<(f64, f64)> { + if cf.is_null() || CFGetTypeID(cf) != AXValueGetTypeID() { + return None; + } + let mut pair = [0.0_f64; 2]; + if AXValueGetValue(cf, kind, pair.as_mut_ptr().cast()) == 0 { + return None; + } + let [a, b] = pair; + (a.is_finite() && b.is_finite()).then_some((a, b)) +} + unsafe fn string_of(cf: *const c_void) -> String { let len = CFStringGetLength(cf); if len < 0 { @@ -213,6 +263,17 @@ mod tests { static kCFBooleanFalse: *const c_void; } + #[link(name = "ApplicationServices", kind = "framework")] + extern "C" { + fn AXValueCreate(the_type: u32, value_ptr: *const c_void) -> *const c_void; + } + + const K_AX_VALUE_CG_RECT_TYPE: u32 = 3; + + fn ax_value(kind: u32, fields: [f64; N]) -> Owned { + unsafe { Owned::adopt(AXValueCreate(kind, fields.as_ptr().cast())).unwrap() } + } + fn number(n: f64) -> Owned { unsafe { Owned::adopt(CFNumberCreate( @@ -309,6 +370,54 @@ mod tests { } } + #[test] + fn number_value_reads_numbers_only() { + let s = cfstring("42").unwrap(); + unsafe { + assert_eq!(number_value(number(42.0).0), Some(42.0)); + assert_eq!(number_value(s.0), None); + assert_eq!(number_value(boolean_true()), None); + assert_eq!(number_value(ptr::null()), None); + } + } + + #[test] + fn ax_values_read_back_as_a_point_and_a_size() { + let position = ax_value(K_AX_VALUE_CG_POINT_TYPE, [12.5, -40.0]); + let size = ax_value(K_AX_VALUE_CG_SIZE_TYPE, [300.0, 22.0]); + unsafe { + assert_eq!(ax_point(position.0), Some((12.5, -40.0))); + assert_eq!(ax_size(size.0), Some((300.0, 22.0))); + } + } + + #[test] + fn only_the_matching_ax_value_type_is_read() { + let position = ax_value(K_AX_VALUE_CG_POINT_TYPE, [1.0, 2.0]); + let size = ax_value(K_AX_VALUE_CG_SIZE_TYPE, [3.0, 4.0]); + // Four doubles: copying it into a point would overrun the buffer. + let rect = ax_value(K_AX_VALUE_CG_RECT_TYPE, [1.0, 2.0, 3.0, 4.0]); + let s = cfstring("AXPosition").unwrap(); + unsafe { + assert_eq!(ax_size(position.0), None); + assert_eq!(ax_point(size.0), None); + assert_eq!(ax_point(rect.0), None); + assert_eq!(ax_point(s.0), None); + assert_eq!(ax_size(number(5.0).0), None); + assert_eq!(ax_point(ptr::null()), None); + } + } + + #[test] + fn non_finite_ax_values_are_refused() { + let position = ax_value(K_AX_VALUE_CG_POINT_TYPE, [f64::NAN, 2.0]); + let size = ax_value(K_AX_VALUE_CG_SIZE_TYPE, [10.0, f64::INFINITY]); + unsafe { + assert_eq!(ax_point(position.0), None); + assert_eq!(ax_size(size.0), None); + } + } + #[test] fn whole_numbers_print_without_a_decimal_point() { assert_eq!(format_number(3.0), "3"); diff --git a/shell/src/lib.rs b/shell/src/lib.rs index 2b322960..97e9825f 100644 --- a/shell/src/lib.rs +++ b/shell/src/lib.rs @@ -3,6 +3,7 @@ pub(crate) mod cu_agent; pub(crate) mod cu_stop; #[cfg(target_os = "macos")] mod cu_ax; +mod cu_capture; #[cfg(target_os = "macos")] mod cu_cf; pub mod cu_identity;