From e9411551142913d493053431c3751643e7fc5026 Mon Sep 17 00:00:00 2001 From: Adam Crocker <77861255+captcrock@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:11:22 -0500 Subject: [PATCH] TD-5226: the ring marks the display the agent is acting on, in full control only The ring lit every display at once, also in background mode, and a client that never sent the episode bracket (Grok's own copy, another MCP client) left it on until its process exited. Grok sessions in the packaged app also lit the daemon's sidecar, a second ring on a display it was not acting on, and one session closing its episode darkened another session's ring. tst-cu-mcp now puts a platform-neutral controller (overlay/control.py) in front of the painters: - the ring shows only while the policy file's mode is full_control; the file is re-parsed when it changes, and a broken file reads as background - it is aimed at the display being acted on: the last screenshot's, or the one under the last move or click; the macOS helper lights only that display's panel (target ); Windows and Linux still ring all - overlay_session takes the session id and the ring holds while any session's episode is open - a client that never brackets lights it per action; it fades 30 s later - a watcher re-checks Stop, the mode and the fade every quarter second Grok's child always paints, with the user's show_on_real_display switch, and Grok sessions tag their episodes for the window only. The helper's retarget is a module function: PyObjC reads a method name without trailing underscores as a zero-argument selector and refused the class, which only the live run caught. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HUthoCdmzg2hy89ssk2cSF --- DECISIONS.md | 54 ++++ core/tests/test_cu_kill.py | 2 +- core/tests/test_cu_session.py | 20 +- core/tests/test_grok_home.py | 28 +- core/tests/test_grok_loop.py | 8 +- core/tstd/desktop/inprocess.py | 6 +- core/tstd/desktop/mcp_driver.py | 8 +- core/tstd/desktop/mock.py | 4 +- core/tstd/desktop/protocol.py | 4 +- core/tstd/grok_home.py | 17 +- core/tstd/grok_loop.py | 8 +- core/tstd/session.py | 25 +- docs/configuration.md | 20 +- docs/tst-desk-backlog.md | 26 ++ mcp/tst-cu-mcp/src/tst_cu_mcp/capture.py | 2 + .../src/tst_cu_mcp/input_control.py | 24 +- .../src/tst_cu_mcp/overlay/__init__.py | 68 +++- .../src/tst_cu_mcp/overlay/control.py | 243 +++++++++++++++ .../src/tst_cu_mcp/overlay/darwin.py | 5 + .../src/tst_cu_mcp/overlay/darwin_helper.py | 99 +++++- .../src/tst_cu_mcp/overlay/linux.py | 4 + .../src/tst_cu_mcp/overlay/win32.py | 4 + mcp/tst-cu-mcp/src/tst_cu_mcp/server.py | 7 +- mcp/tst-cu-mcp/tests/test_overlay_gating.py | 44 ++- mcp/tst-cu-mcp/tests/test_overlay_rules.py | 290 ++++++++++++++++++ 25 files changed, 923 insertions(+), 97 deletions(-) create mode 100644 mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py create mode 100644 mcp/tst-cu-mcp/tests/test_overlay_rules.py diff --git a/DECISIONS.md b/DECISIONS.md index bbbcf3d3..e943a9d2 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -19609,3 +19609,57 @@ process. It never removes the file. own handler (`on_shortcut`). Quick entry used to build the plugin with a single handler, which would have left no room for a second shortcut. +## 2026-11-03 — TD-5226: the ring marks the acted-on display, in full control only (Class B; revises TD-3402 and TD-3407) + +**Decision:** The real-display ring means "an agent has the real screen". +tst-cu-mcp now applies its rules in one platform-neutral controller +(`overlay/control.py`) in front of the painters, which only light, darken +and aim. + +1. **Full control only.** The ring shows only while the policy file's + `mode` is `full_control`. In `background` mode it is dark on every + display, even if a background-mode agent falls back to a coordinate + tool. The file is re-parsed when its size or mtime changes, so Settings + applies to every running client. A file that fails to parse reads as + background: a broken edit darkens the ring rather than lighting it. With + no file at all a client uses its installed config, whose default is also + background. +2. **One display.** The ring follows the display being acted on: the last + screenshot's display, or the display under the last pointer action + (move, click, and the scroll that moves first). The macOS helper keeps a + panel per screen and lights only the target's, named by its + CGDirectDisplayID (`target `); an unknown id or no target yet means + the primary display. Windows and Linux X11 still ring every display; + aiming them is a follow-up. +3. **Episodes per session.** `overlay_session` takes the session id. The + ring holds while any session's episode is open, so one TST Desk session + ending no longer darkens another's. +4. **Non-bracketing clients fade.** A client that never sends + `overlay_session` lights the ring on each action, and it fades 30 + seconds after the last one. A client that has bracketed once never falls + back to this rule, so TST Desk's own ring still lasts exactly as long as + the episode. +5. **Stop is watched, not waited for.** A watcher thread re-checks Stop, + the mode and the fade every quarter second. Stop fades the ring out (the + helper's normal hide, under a second) and drops a non-bracketing + client's claim, so after Resume it takes a new action to light it. TST + Desk's own episodes are closed by Stop already (TD-3407). +6. **The copy that acts paints.** Grok's own tst-cu-mcp child always paints + (with the user's `show_on_real_display` switch passed through), and Grok + sessions tag their episodes for the window only, so the daemon's sidecar + no longer lights a second ring on a display it is not acting on. Before + this the child was told not to paint when a command was configured, + because nothing ended its ring; rule 4 now does. + +**Rationale:** The user works across several displays and could not tell +which one an agent was on. In background mode nothing takes the screen, so +a ring there signals something that is not happening. Level-triggered state +(work out "wanted" on every call and tick, talk to the painter only on a +change) keeps the watcher and the tool thread from fighting, under one lock. + +**Rejected:** ringing every display with the acted-on one brighter (still +says "everything is taken" in background mode, and two weights of ring read +as two agents); hiding the ring after a fixed time inside an episode (the +TD-3407 reason stands: a long think between actions is still the agent +driving); a stop-file watch inside the AppKit helper (it would need the +config path and the mode as well; the sidecar already has both). diff --git a/core/tests/test_cu_kill.py b/core/tests/test_cu_kill.py index 0e81c881..bbe1a4d7 100644 --- a/core/tests/test_cu_kill.py +++ b/core/tests/test_cu_kill.py @@ -72,7 +72,7 @@ async def test_set_cu_kill_emits_cu_session_false(tmp_path: Path) -> None: assert session.cu_session_active is False tags = [e for e in session.event_log.all_events if isinstance(e, CuSession)] assert tags[-1].active is False - assert ("overlay_session", {"active": False}) in mock.calls + assert ("overlay_session", {"active": False, "session": sid}) in mock.calls runner = daemon.session_registry.get_runner(sid) if runner is not None: await runner.cancel() diff --git a/core/tests/test_cu_session.py b/core/tests/test_cu_session.py index 36e1f238..2dd72e22 100644 --- a/core/tests/test_cu_session.py +++ b/core/tests/test_cu_session.py @@ -21,7 +21,7 @@ async def test_first_cu_tool_opens_once_and_overlay_is_told() -> None: tags = [e for e in session.event_log.all_events if isinstance(e, CuSession)] assert len(tags) == 1 assert tags[0].active is True - assert driver.calls == [("overlay_session", {"active": True})] + assert driver.calls == [("overlay_session", {"active": True, "session": session.id})] @pytest.mark.asyncio @@ -44,11 +44,25 @@ async def test_close_emits_false_and_is_idempotent() -> None: tags = [e for e in session.event_log.all_events if isinstance(e, CuSession)] assert [t.active for t in tags] == [True, False] assert driver.calls == [ - ("overlay_session", {"active": True}), - ("overlay_session", {"active": False}), + ("overlay_session", {"active": True, "session": session.id}), + ("overlay_session", {"active": False, "session": session.id}), ] +@pytest.mark.asyncio +async def test_window_only_episode_leaves_the_ring_to_the_acting_copy() -> None: + """Grok's own tst-cu-mcp paints its ring; the sidecar must not light one too.""" + session = Session("/tmp/ws") + driver = MockDesktopDriver() + session._overlay_session = driver.set_overlay_session + await session.open_cu_session("computer-use__click", ring=False) + await session.close_cu_session() + + tags = [e for e in session.event_log.all_events if isinstance(e, CuSession)] + assert [t.active for t in tags] == [True, False] + assert driver.calls == [] + + @pytest.mark.asyncio async def test_cancel_closes_the_episode() -> None: session = Session("/tmp/ws") diff --git a/core/tests/test_grok_home.py b/core/tests/test_grok_home.py index ee893d15..8ddce998 100644 --- a/core/tests/test_grok_home.py +++ b/core/tests/test_grok_home.py @@ -125,9 +125,27 @@ def test_computer_use_mcp_from_command(tmp_path: Path, monkeypatch: pytest.Monke assert spec is not None assert spec["name"] == "computer-use" assert spec["command"] == str(binary.resolve()) - # A configured command is the daemon's driver too: it paints the ring - # from cu_session tags, so this child is told not to. - assert spec["env"] == [{"name": "TST_CU_MCP_OVERLAY", "value": "0"}] + # The child acts, so it paints its own ring (TD-5226), following the + # user's show_on_real_display switch. + assert spec["env"] == [{"name": "TST_CU_MCP_OVERLAY", "value": "1"}] + + +def test_computer_use_mcp_follows_the_real_display_switch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tstd.cu_indicators import CuIndicatorPrefs, reset_cu_indicators, set_current_prefs + + monkeypatch.delenv("TST_CU_AGENT_SOCK", raising=False) + binary = tmp_path / "tst-cu-mcp" + binary.write_text("#!/bin/sh\n", encoding="utf-8") + binary.chmod(0o755) + set_current_prefs(CuIndicatorPrefs(show_on_real_display=False)) + try: + spec = computer_use_mcp(str(binary)) + finally: + reset_cu_indicators() + assert spec is not None + assert {"name": "TST_CU_MCP_OVERLAY", "value": "0"} in spec["env"] def test_acp_mcp_servers_merges_computer_use( @@ -165,8 +183,8 @@ def test_computer_use_mcp_finds_checkout_from_workspace( spec = computer_use_mcp("", search_from=str(tmp_path / "Documents")) assert spec is not None assert spec["command"] == str(binary.resolve()) - # No configured driver means nobody else paints; this child keeps the ring. - assert {"name": "TST_CU_MCP_OVERLAY", "value": "0"} not in spec["env"] + # Found on disk: same rule, the child paints its own ring. + assert {"name": "TST_CU_MCP_OVERLAY", "value": "1"} in spec["env"] def test_open_in_terminal_rejects_control_chars() -> None: diff --git a/core/tests/test_grok_loop.py b/core/tests/test_grok_loop.py index ab450abd..3ff52fbb 100644 --- a/core/tests/test_grok_loop.py +++ b/core/tests/test_grok_loop.py @@ -195,10 +195,8 @@ async def prompt_with_cu( assert types.index("cu_session") == types.index("tool_call", types.index("tool_call") + 1) + 1 # Closed at turn end, not at the last tool. assert types[types.index("turn_complete") + 1] == "cu_session" - assert driver.calls == [ - ("overlay_session", {"active": True}), - ("overlay_session", {"active": False}), - ] + # The ring is Grok's own child's (TD-5226): the daemon's sidecar stays dark. + assert driver.calls == [] await session.cancel() await asyncio.wait_for(task, timeout=2) @@ -237,7 +235,7 @@ async def prompt_then_die( assert done and done[0].failed is True tags = [e for e in session.event_log.events_from(1) if isinstance(e, CuSession)] assert [t.active for t in tags] == [True, False] - assert driver.calls[-1] == ("overlay_session", {"active": False}) + assert driver.calls == [] await session.cancel() await asyncio.wait_for(task, timeout=2) diff --git a/core/tstd/desktop/inprocess.py b/core/tstd/desktop/inprocess.py index 33c212ee..8e7e07ac 100644 --- a/core/tstd/desktop/inprocess.py +++ b/core/tstd/desktop/inprocess.py @@ -50,15 +50,15 @@ def set_killed(self, killed: bool) -> None: else: os.environ.pop(_STOP_ENV, None) - async def set_overlay_session(self, active: bool) -> None: + async def set_overlay_session(self, active: bool, session: str = "") -> None: try: from tst_cu_mcp.overlay import get_overlay overlay = get_overlay() if active: - overlay.begin_session() + overlay.begin_session(session) else: - overlay.end_session() + overlay.end_session(session) except Exception: return diff --git a/core/tstd/desktop/mcp_driver.py b/core/tstd/desktop/mcp_driver.py index f1497411..f6e1b6e0 100644 --- a/core/tstd/desktop/mcp_driver.py +++ b/core/tstd/desktop/mcp_driver.py @@ -50,10 +50,12 @@ def __init__( def set_killed(self, killed: bool) -> None: self.killed = bool(killed) - async def set_overlay_session(self, active: bool) -> None: - """Tell the sidecar the computer-use episode opened or closed.""" + async def set_overlay_session(self, active: bool, session: str = "") -> None: + """Tell the sidecar one session's computer-use episode opened or closed.""" try: - await self._client.call_tool("overlay_session", {"active": bool(active)}) + await self._client.call_tool( + "overlay_session", {"active": bool(active), "session": session} + ) except Exception: return diff --git a/core/tstd/desktop/mock.py b/core/tstd/desktop/mock.py index bd034ab5..8f6406f3 100644 --- a/core/tstd/desktop/mock.py +++ b/core/tstd/desktop/mock.py @@ -60,8 +60,8 @@ def __init__( def set_killed(self, killed: bool) -> None: self.killed = bool(killed) - async def set_overlay_session(self, active: bool) -> None: - self.calls.append(("overlay_session", {"active": bool(active)})) + async def set_overlay_session(self, active: bool, session: str = "") -> None: + self.calls.append(("overlay_session", {"active": bool(active), "session": session})) def _refuse_if_wayland(self) -> None: if normalize_cu_platform(self.platform) != "linux": diff --git a/core/tstd/desktop/protocol.py b/core/tstd/desktop/protocol.py index abe27702..996117a4 100644 --- a/core/tstd/desktop/protocol.py +++ b/core/tstd/desktop/protocol.py @@ -79,8 +79,8 @@ class DesktopDriver(Protocol): def set_killed(self, killed: bool) -> None: """Engage or clear the kill-switch. Screenshot still runs.""" - async def set_overlay_session(self, active: bool) -> None: - """Open or close the real-display computer-use ring (TD-3407).""" + async def set_overlay_session(self, active: bool, session: str = "") -> None: + """Open or close one session's real-display ring episode (TD-3407, TD-5226).""" async def screenshot(self, display: int | None = None) -> str: """Return a JSON object with ``png_base64`` (and size if known).""" diff --git a/core/tstd/grok_home.py b/core/tstd/grok_home.py index db788cf2..eef74543 100644 --- a/core/tstd/grok_home.py +++ b/core/tstd/grok_home.py @@ -235,9 +235,6 @@ def computer_use_mcp( import shlex argv = shlex.split(cu_command) - # A configured command is also the daemon's own driver, which paints - # the real-display ring from cu_session tags; this child must not. - configured = bool(argv) if not argv: from .desktop.factory import packaged_cu_argv @@ -260,11 +257,11 @@ def computer_use_mcp( "name": "computer-use", "command": str(binary.resolve()), "args": argv[1:], - "env": _host_env(argv, paint=not configured), + "env": _host_env(argv), } -def _host_env(argv: list[str], *, paint: bool = True) -> list[dict[str, str]]: +def _host_env(argv: list[str]) -> list[dict[str, str]]: env: list[dict[str, str]] = [] from .cu_host import SOCK_ENV @@ -273,10 +270,12 @@ def _host_env(argv: list[str], *, paint: bool = True) -> list[dict[str, str]]: env.append({"name": SOCK_ENV, "value": sock}) if argv and Path(argv[0]).name.startswith("tstd") and "--cu-mcp" in argv[1:]: env.append({"name": "TST_CU_MCP_HOST", "value": "1"}) - if not paint: - # Nothing tells this child when a turn ends, so a ring it lit would - # linger; the daemon's driver paints and closes it instead. - env.append({"name": OVERLAY_ENV, "value": "0"}) + # This child acts, so it paints the ring on the display it acts on; Grok + # sessions do not light the daemon's sidecar (TD-5226). With no episode + # bracket its ring fades on idle. The user's switch still applies. + from .cu_indicators import current_prefs + + env.append({"name": OVERLAY_ENV, "value": "1" if current_prefs().show_on_real_display else "0"}) return env diff --git a/core/tstd/grok_loop.py b/core/tstd/grok_loop.py index 0dce86c5..62ca47a8 100644 --- a/core/tstd/grok_loop.py +++ b/core/tstd/grok_loop.py @@ -393,10 +393,10 @@ async def on_update(params: dict[str, Any]) -> None: seq=1, ) ) - # Same episode tags as the native loop (TD-3407): the pane glow - # and the daemon's real-display ring follow cu_session, which - # is the only thing that can close them on this engine. - await session.open_cu_session(name) + # Same episode tags as the native loop (TD-3407) for the pane + # glow. The real-display ring is Grok's own tst-cu-mcp child's: + # it acts, so it knows the display (TD-5226). + await session.open_cu_session(name, ring=False) after_tool = True return if kind == "tool_call_update": diff --git a/core/tstd/session.py b/core/tstd/session.py index 32b9ced1..c287a48f 100644 --- a/core/tstd/session.py +++ b/core/tstd/session.py @@ -546,7 +546,10 @@ def __init__(self, workspace_path: str) -> None: # TD-3407: computer-use episode is open this turn. The overlay # callback is the sidecar ring; None on a tombstone or mock-only. self.cu_session_active = False - self._overlay_session: Callable[[bool], Awaitable[None]] | None = None + # (active, session id): the sidecar holds the ring while any + # session's episode is open (TD-5226). + self._overlay_session: Callable[[bool, str], Awaitable[None]] | None = None + self._cu_ring = False # TD-4101: unattended runs iterate without a user message. # Interactive sessions leave this false and wait on the queue. self.autonomy = False @@ -609,8 +612,14 @@ def mark_class_c(self, reason: str) -> None: if self.autonomy_stop_reason is None: self.autonomy_stop_reason = reason - async def open_cu_session(self, tool_name: str) -> None: - """Open the computer-use episode on the first CU tool of a turn.""" + async def open_cu_session(self, tool_name: str, *, ring: bool = True) -> None: + """Open the computer-use episode on the first CU tool of a turn. + + ``ring=False`` tags the episode for the window only: the copy of + tst-cu-mcp that acts paints the real-display ring itself (Grok's own + child does, TD-5226), so the daemon's sidecar must not light a second + one on a display it is not acting on. + """ from .local_worker import is_cu_tool from .protocol import CuSession @@ -618,8 +627,9 @@ async def open_cu_session(self, tool_name: str) -> None: return self.cu_session_active = True await self.event_log.add(CuSession(session_id=self.id, active=True, seq=1)) - if self._overlay_session is not None: - await self._overlay_session(True) + self._cu_ring = ring and self._overlay_session is not None + if self._cu_ring and self._overlay_session is not None: + await self._overlay_session(True, self.id) async def close_cu_session(self) -> None: """Close the computer-use episode. Idempotent.""" @@ -629,8 +639,9 @@ async def close_cu_session(self) -> None: return self.cu_session_active = False await self.event_log.add(CuSession(session_id=self.id, active=False, seq=1)) - if self._overlay_session is not None: - await self._overlay_session(False) + if self._cu_ring and self._overlay_session is not None: + self._cu_ring = False + await self._overlay_session(False, self.id) async def _observe_turn_end(self, event: DaemonEvent, _log: SessionEventLog) -> None: """Lower the open-turn count when the loop reports a turn complete.""" diff --git a/docs/configuration.md b/docs/configuration.md index 4428a334..57a8c316 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -294,7 +294,8 @@ second (TD-5223). Settings → Computer use (TD-4830) writes `~/.tst-cu-mcp/config.yaml` (or `$TST_CU_MCP_CONFIG`): `actuation.enabled`, `mode` (`background` or `full_control`), `scoping.denied_apps`, `scoping.unhide_on_finish`. The -MCP server re-reads that file on each background tool call. +MCP server re-reads that file on each background tool call, and the real-display +ring follows `mode` whenever the file changes (TD-5226). Browser computer-use (TD-1710): `browser` selects the in-process mock (CI, never launches Chrome) or Playwright with a persistent profile @@ -3307,11 +3308,18 @@ proposals for the run summary and never accepts them either. The classifier stil not keys in this file. Settings → Appearance persists them as `cu-indicators.yaml` in the user data dir (glow and cursor default on; the real-display ring defaults on). Glow and the agent cursor are drawn on the Screen pane. The rust ring on the real display -follows the same `cu_session` open/close tags (TD-3407) on macOS, Windows, and -Linux X11 — it stays up until the turn ends, you cancel, or the kill-switch -fires, not for eight seconds after the last click. Wayland has no real-display -ring. Screenshots hide the ring for the grab so it cannot appear in the frame. -This is not a second hardware pointer. +means "an agent has the real screen", so it shows only in `full_control` mode +(the `mode` in `~/.tst-cu-mcp/config.yaml`, re-read when it changes); in +`background` mode it stays off on every display (TD-5226). On macOS it lights only +the display being acted on: the one the last screenshot targeted, or the one +under the last pointer action, else the primary display. Windows and Linux X11 +still ring every display. For a TST Desk session it follows the `cu_session` +open/close tags (TD-3407) and stays up while any session's episode is open; a +client that never sends them (Grok's own copy, another MCP client) lights it on +each action and it fades 30 seconds after the last one. Stop fades it out within +about a second, without waiting for the agent's next action. Wayland has no real-display ring. The ring's windows are left out +of screenshots, and a grab also hides it, so it cannot appear in the frame. This +is not a second hardware pointer. **Background**, **Strength**, **Motion** and **Click ripples** (TD-5202) are not keys in this file either. Settings → Appearance keeps them per device, like the theme, in the window's local storage: diff --git a/docs/tst-desk-backlog.md b/docs/tst-desk-backlog.md index 77b93d52..360c04aa 100644 --- a/docs/tst-desk-backlog.md +++ b/docs/tst-desk-backlog.md @@ -13041,6 +13041,32 @@ step 1; elsewhere the binary comes from `binary` or `PATH`. --- +### TD-5226 — The ring marks the display the agent is acting on, and only in full control +**Size:** 3 · **Depends on:** TD-3402, TD-3407, TD-5223 +**Status:** In review 2026-11-03. + +The ring lit every display at once, so with several monitors it did not say +which one the agent was working on. It also lit in background mode, where the +agent works through accessibility calls and never takes the screen. A client +that never sends the episode bracket (Grok's own copy of tst-cu-mcp, another +desktop assistant) lit it on its first action and nothing ever put it out: on +one Mac a ring stayed on all four displays for hours. Grok sessions in the +packaged app also lit the daemon's sidecar from their episode tags, a second +ring on a display neither copy was acting on. And one TST Desk session closing +its episode darkened the ring of another session that was still working. + +**Acceptance criteria:** +- [x] The ring shows only when the policy file's `mode` is `full_control`; in `background` it is dark on every display, and a mode change applies within a quarter second without restarting any client +- [x] macOS lights only the display being acted on: the last screenshot's display, or the display under the last pointer action; before either, the primary display. Windows and Linux X11 keep ringing every display +- [x] Episodes are per session: the ring holds while any TST Desk session's episode is open +- [x] A client that never brackets lights the ring per action, and it fades 30 seconds after the last one +- [x] Stop fades the ring out within about a second (checked every quarter second), without waiting for the agent's next action; a client that never brackets needs a new action after Resume +- [x] Grok sessions leave the ring to Grok's own copy, which paints on the display it acts on and follows the user's `show_on_real_display` switch +- [x] Tests for every rule against a recording painter, each shown to fail with its rule removed (`test_overlay_rules.py`), plus the helper's argument parsing and panel choice +- [ ] Live on a Mac with several displays: acting on a secondary display rings that display only; background mode shows no ring; screenshots do not contain it; Stop darkens it + +--- + ### TD-5225 — Reading a window's accessibility tree crashes TST Desk **Size:** 2 · **Depends on:** TD-4829 **Status:** In review 2026-11-02. diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/capture.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/capture.py index 7dabeb95..7a925de3 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/capture.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/capture.py @@ -123,6 +123,8 @@ def capture( displays = list_displays() display = select_display(displays, display_index) rect = resolve_region(display, region) + # The ring follows the display the agent is looking at (TD-5226). + get_overlay().set_target(display.display_id) # The real-display glow must never paint into the frame it signals for. # grab_hidden orders the panels out and only returns once they are gone diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py index a35cd992..54ea5fa5 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/input_control.py @@ -15,7 +15,8 @@ Immediately after the kill-switch passes, every action pings :func:`tst_cu_mcp.overlay.get_overlay` so the real-display glow tracks the attempt — including attempts that then fail validation, which are still the -agent driving. A refusal from the backend itself — the OS said no, as with a +agent driving. A pointer action first aims the glow at the display under its +point. A refusal from the backend itself — the OS said no, as with a missing Accessibility grant — puts the glow out again (:func:`tst_cu_mcp.safety.notify_blocked`): nothing was driven. @@ -60,6 +61,25 @@ def _actuate(act: Callable[..., None], *args: Any) -> None: raise +def display_at(x: float, y: float) -> int | None: + """The id of the display containing the global point, or ``None``.""" + for display in list_displays(): + if display.x <= x < display.x + display.width and ( + display.y <= y < display.y + display.height + ): + return display.display_id + return None + + +def _aim_overlay(x: float, y: float) -> None: + """Point the ring at the display this pointer action lands on.""" + try: + target = display_at(x, y) + except Exception: + return + get_overlay().set_target(target) + + def _display_bounds_union() -> tuple[int, int, int, int] | None: """Return (min_x, min_y, max_x, max_y) across all displays, or None if none.""" displays = list_displays() @@ -108,6 +128,7 @@ def cursor_position() -> tuple[int, int]: def move_mouse(x: float, y: float, expect_window: str | None = None) -> None: """Move the cursor to a global coordinate.""" safety.ensure_actuation_allowed() + _aim_overlay(x, y) get_overlay().activity() assert_on_screen(x, y) assert_foreground(expect_window) @@ -123,6 +144,7 @@ def click( ) -> None: """Click at a global coordinate. ``count`` >= 2 produces a multi-click.""" safety.ensure_actuation_allowed() + _aim_overlay(x, y) get_overlay().activity() if button not in MOUSE_BUTTONS: raise ValueError(f"unknown button {button!r}; use one of {MOUSE_BUTTONS}") diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py index d1e4b25b..24a381ea 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/__init__.py @@ -1,16 +1,18 @@ """The real-display glow: signaling, never input. -While computer use is active, a rust ring glows around the edges of every -display — the machine itself saying "the agent is driving", the way Claude -Desktop lights up the screen it controls. The overlay is owned by this +While an agent drives the real screen (full-control mode), a rust ring glows +around the edges of the display it is acting on — the machine itself saying +"the agent is driving". :mod:`tst_cu_mcp.overlay.control` holds the rules +(mode, target display, episodes, idle fade, Stop); the platform painters +below only light and darken. The overlay is owned by this sidecar because the sidecar is what actuates: keeping it here makes the hide-for-capture ordering two function calls in one process instead of a cross-process race, so a screenshot can never contain the glow it just hid. Platform resolution mirrors :mod:`tst_cu_mcp.backends`: a real -painter on darwin, win32, and Linux X11. Wayland stays a no-op -(TD-2002). Unlike backends, the resolved instance is cached — some +painter on darwin, win32, and Linux X11, wrapped in the rules controller. +Wayland stays a no-op (TD-2002). Unlike backends, the resolved instance is cached — some painters own a helper or a pulse thread that must not be spawned per call. Tests install fakes with :func:`set_overlay` and clear with :func:`reset_overlay`. @@ -43,6 +45,7 @@ # hidden/shown, which is what gives a capture its ordering guarantee. CMD_SHOW = "show" CMD_HIDE = "hide" +CMD_TARGET = "target" CMD_GRAB_BEGIN = "grab_begin" CMD_GRAB_END = "grab_end" CMD_QUIT = "quit" @@ -56,22 +59,26 @@ class Overlay(Protocol): name: str - def begin_session(self) -> None: - """Computer-use episode opened: show the ring and keep it up.""" + def begin_session(self, key: str = "") -> None: + """A client session's computer-use episode opened.""" ... - def end_session(self) -> None: - """Computer-use episode closed: hide the ring.""" + def end_session(self, key: str = "") -> None: + """That episode closed.""" ... def activity(self) -> None: - """An actuation happened. Same as ``begin_session`` (idempotent).""" + """An actuation is about to run.""" ... def notify_blocked(self) -> None: """Actuation was refused: the glow must not be lit.""" ... + def set_target(self, display_id: int | None) -> None: + """The display being acted on (a display id from :mod:`displays`).""" + ... + @contextmanager def grab_hidden(self) -> Iterator[None]: """Hide the glow for the duration of one screenshot grab.""" @@ -82,15 +89,43 @@ def shutdown(self) -> None: ... +class Painter(Protocol): + """A platform ring: light it, darken it, aim it. No rules of its own.""" + + name: str + + def begin_session(self) -> None: ... + + def end_session(self) -> None: + """Darken the ring.""" + ... + + def activity(self) -> None: + """Light the ring (idempotent).""" + ... + + def notify_blocked(self) -> None: ... + + def set_target(self, display_id: int | None) -> None: + """Light this display only, where the platform can; else a no-op.""" + ... + + @contextmanager + def grab_hidden(self) -> Iterator[None]: + yield + + def shutdown(self) -> None: ... + + class NullOverlay: """The honest no-op: unsupported platforms and degraded states.""" name = "null" - def begin_session(self) -> None: + def begin_session(self, key: str = "") -> None: return None - def end_session(self) -> None: + def end_session(self, key: str = "") -> None: return None def activity(self) -> None: @@ -99,6 +134,9 @@ def activity(self) -> None: def notify_blocked(self) -> None: return None + def set_target(self, display_id: int | None) -> None: + return None + @contextmanager def grab_hidden(self) -> Iterator[None]: yield @@ -159,13 +197,15 @@ def get_overlay() -> Overlay: _overlay = NullOverlay() return _overlay try: - _overlay = _resolve_painter() + from tst_cu_mcp.overlay.control import GatedOverlay + + _overlay = GatedOverlay(_resolve_painter()) except Exception: _overlay = NullOverlay() return _overlay -def _resolve_painter() -> Overlay: +def _resolve_painter() -> Painter: """The platform painter. Import-time OS work stays inside each module.""" target = sys.platform if target == "darwin": diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py new file mode 100644 index 00000000..d3d3255c --- /dev/null +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/control.py @@ -0,0 +1,243 @@ +"""When the ring shows and where, for every painter (TD-5226). + +Painters (darwin, win32, linux) only know how to light and darken a ring. +This controller owns the rules: + +* **Mode.** The ring means "the agent has the real screen". In background + mode (``mode: background`` in the policy file) agents work through + accessibility calls and never take the screen, so the ring stays off on + every display, even when such an agent falls back to a coordinate tool. + Only ``full_control`` lights it. The file is re-read when it changes, so + Settings applies without restarting any client. +* **Display.** Only the display being acted on is lit: the one the last + capture targeted, or the one under the last pointer action. Before + either, the painter's default (the primary display). Painters that cannot + light a single display (Windows and Linux today) ring every display. +* **Episodes.** TST Desk brackets each session's computer use with + ``overlay_session``; the ring stays up while any session's episode is + open, so one session finishing does not darken another's ring. A client + that never brackets (any other MCP client) lights the ring on activity, + and it fades after :data:`UNBRACKETED_LINGER_SECONDS` without one. +* **Stop.** While the kill-switch is engaged the ring is dark. A watcher + checks every :data:`WATCH_SECONDS`, so Stop clears the ring without + waiting for the agent's next action. + +The state is level-triggered: each entry point and each watcher tick works +out whether the ring is wanted and talks to the painter only when that, or +the target display, changes. Painter calls happen under one lock because +the watcher thread and the tool thread both reach the painter. The failure +rule of the package holds: nothing here may raise into a tool. +""" + +from __future__ import annotations + +import contextlib +import threading +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tst_cu_mcp.overlay import Painter + +#: How often the watcher re-checks Stop, the mode and the linger. +WATCH_SECONDS = 0.25 +#: A non-bracketing client's ring fades this long after its last action. +UNBRACKETED_LINGER_SECONDS = 30.0 +#: The mode that lights the ring. +FULL_CONTROL = "full_control" + + +class PolicyMode: + """The policy file's ``mode``, re-parsed only when the file changes. + + No file means the installed config (permissive defaults: background). + A file that fails to parse keeps the last good answer, or background, + so a broken edit darkens the ring rather than lighting it. + """ + + def __init__(self, path: Callable[[], Path] | None = None) -> None: + from tst_cu_mcp.config import policy_path + + self._path = path or policy_path + self._stamp: tuple[float, int] | None = None + self._mode = "background" + + def __call__(self) -> str: + from tst_cu_mcp import safety + from tst_cu_mcp.config import load_config + + target = self._path() + try: + stat = target.stat() + except OSError: + self._stamp = None + return safety.active_config().mode + stamp = (stat.st_mtime, stat.st_size) + if stamp != self._stamp: + self._stamp = stamp + try: + self._mode = load_config(target).mode + except Exception: + self._mode = "background" + return self._mode + + +def _killswitch() -> bool: + from tst_cu_mcp import safety + + try: + return safety.killswitch_engaged() + except Exception: + return True + + +class GatedOverlay: + """An :class:`~tst_cu_mcp.overlay.Overlay` that applies the ring's rules.""" + + def __init__( + self, + painter: Painter, + *, + mode: Callable[[], str] | None = None, + stopped: Callable[[], bool] = _killswitch, + clock: Callable[[], float] = time.monotonic, + linger: float = UNBRACKETED_LINGER_SECONDS, + watch: bool = True, + ) -> None: + self.painter = painter + self.name = painter.name + self._mode = mode or PolicyMode() + self._stopped = stopped + self._clock = clock + self._linger = linger + self._watch = watch + self._lock = threading.RLock() + self._episodes: set[str] = set() + self._bracketing = False + self._blocked = False + self._last_activity: float | None = None + self._target: int | None = None + self._sent_target: int | None = None + self._shown = False + self._watcher: threading.Thread | None = None + self._halt = threading.Event() + + # -- the Overlay protocol --------------------------------------------- + + def begin_session(self, key: str = "") -> None: + """Open one client session's episode; the ring holds while any is open.""" + with self._lock: + self._bracketing = True + self._episodes.add(key) + self._blocked = False + self._sync() + self._start_watcher() + + def end_session(self, key: str = "") -> None: + """Close one episode. The ring goes dark when the last one closes.""" + with self._lock: + self._episodes.discard(key) + self._sync() + + def activity(self) -> None: + """An actuation is about to run.""" + with self._lock: + if not self._stopped(): + self._blocked = False + self._last_activity = self._clock() + self._sync() + self._start_watcher() + + def notify_blocked(self) -> None: + """An actuation was refused: dark until the next one goes through.""" + with self._lock: + self._blocked = True + self._sync() + + def set_target(self, display_id: int | None) -> None: + """The display being acted on (the last capture's, or under the pointer).""" + with self._lock: + if display_id is not None: + self._target = display_id + self._sync() + + @contextmanager + def grab_hidden(self) -> Iterator[None]: + """Hide the painter for one capture, holding the lock only at the edges.""" + with self._lock: + grab = self.painter.grab_hidden() + grab.__enter__() + try: + yield + finally: + with self._lock: + grab.__exit__(None, None, None) + + def shutdown(self) -> None: + self._halt.set() + watcher = self._watcher + if watcher is not None and watcher is not threading.current_thread(): + watcher.join(timeout=1.0) + with self._lock, contextlib.suppress(Exception): + self.painter.shutdown() + + # -- rules ------------------------------------------------------------ + + def wanted(self) -> bool: + """Whether the ring should be lit right now.""" + if self._blocked or self._stopped(): + return False + if self._mode() != FULL_CONTROL: + return False + if self._episodes: + return True + if self._bracketing or self._last_activity is None: + return False + return self._clock() - self._last_activity < self._linger + + def tick(self) -> None: + """One watcher pass. Public so tests drive it without a thread.""" + with self._lock: + self._sync() + + def _sync(self) -> None: + try: + if self._stopped(): + # Stop ends an unbracketed client's claim on the ring: after + # Resume it takes a new action to light it again. + self._last_activity = None + wanted = self.wanted() + if wanted and self._target is not None and self._target != self._sent_target: + self.painter.set_target(self._target) + self._sent_target = self._target + if wanted and not self._shown: + self.painter.activity() + self._shown = True + elif not wanted and self._shown: + self.painter.end_session() + self._shown = False + except Exception: + # The painter degrades itself; the rules never raise into a tool. + self._shown = False + + def _start_watcher(self) -> None: + if not self._watch or self._watcher is not None: + return + self._watcher = threading.Thread(target=self._watch_loop, name="overlay-watch", daemon=True) + self._watcher.start() + + def _watch_loop(self) -> None: + while not self._halt.wait(WATCH_SECONDS): + self.tick() + + +__all__ = [ + "FULL_CONTROL", + "UNBRACKETED_LINGER_SECONDS", + "WATCH_SECONDS", + "GatedOverlay", + "PolicyMode", +] diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py index d1b0821d..72dfa7be 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin.py @@ -35,6 +35,7 @@ CMD_HIDE, CMD_QUIT, CMD_SHOW, + CMD_TARGET, ) #: ``show``/``hide`` are acked by the helper's reader thread on arrival. @@ -166,6 +167,10 @@ def activity(self) -> None: def notify_blocked(self) -> None: self._send(CMD_HIDE) + def set_target(self, display_id: int | None) -> None: + if display_id is not None and display_id >= 0: + self._send(f"{CMD_TARGET} {int(display_id)}") + @contextmanager def grab_hidden(self) -> Iterator[None]: hidden = self._send(CMD_GRAB_BEGIN, timeout=GRAB_ACK_TIMEOUT) diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py index 6e03e25c..a665ed9a 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/darwin_helper.py @@ -3,7 +3,9 @@ One borderless, click-through, topmost panel per display strokes a rust ring around the screen edges while computer use is active — same accent as tst-desk's Screen-pane glow, so "the agent is driving" reads the same -in both places. The ring pulses like ``GlowLayer``'s CSS animation, sits +in both places. Only the panel of the display being acted on is lit +(``target``); before any target arrives that is the primary display. +The ring pulses like ``GlowLayer``'s CSS animation, sits at the screen-saver window level so it floats above everything, ignores mouse events so it can never eat input, and stays up until an explicit hide (computer-use session close), not a linger timeout. @@ -16,6 +18,9 @@ * ``show`` / ``hide`` — acked by the reader thread the moment they are queued; the next tick (~33ms) acts on them. +* ``target `` — light that display's panel instead (a + CGDirectDisplayID, the same number as ``NSScreenNumber``). Acked on + arrival like ``show``. An unknown id falls back to the primary display. * ``grab_begin`` / ``grab_end`` — acked only after the main thread has actually ordered the panels out/in. That ordering is the guarantee a screenshot capture relies on: between the parent receiving the @@ -65,9 +70,10 @@ #: NSEventTypeApplicationDefined — the wake-up that lets stop_ unwind. _APPLICATION_DEFINED_EVENT_TYPE = 15 -_SHOW, _HIDE, _GRAB_BEGIN, _GRAB_END, _QUIT = ( +_SHOW, _HIDE, _TARGET, _GRAB_BEGIN, _GRAB_END, _QUIT = ( "show", "hide", + "target", "grab_begin", "grab_end", "quit", @@ -77,7 +83,9 @@ #: Reader-thread patience with the main thread before reporting failure. _MAIN_ACK_TIMEOUT = 2.5 -Command = tuple[str, threading.Event | None] +#: (name, argument, ack). The argument is the rest of the line, or "". +Command = tuple[str, str, threading.Event | None] +PanelKey = tuple[int, str, int, int, int, int] def main() -> None: @@ -148,8 +156,10 @@ def initWithQueue_(self, q: queue.Queue[Command]) -> Any: if self is None: return None self.commands = q - # Keyed by (name, frame) so a display change rebuilds its panel. - self.panels: dict[tuple[str, int, int, int, int], Any] = {} + # Keyed by (display id, name, frame) so a display change rebuilds + # its panel. The display id is what ``target`` names. + self.panels: dict[PanelKey, Any] = {} + self.target: int | None = None self.visible = False self.grab_depth = 0 self.last_activity = 0.0 @@ -200,10 +210,12 @@ def screensChanged_(self, _note: Any) -> None: def _drain_commands(self) -> None: while True: try: - command, ack = self.commands.get_nowait() + command, arg, ack = self.commands.get_nowait() except queue.Empty: return - if command == _SHOW: + if command == _TARGET: + _retarget(self, _parse_display_id(arg)) + elif command == _SHOW: self.visible = True self.last_activity = time.monotonic() if self.grab_depth == 0: @@ -226,9 +238,10 @@ def _drain_commands(self) -> None: def rebuild_panels(self) -> None: from AppKit import NSScreen - def _key(screen: Any) -> tuple[str, int, int, int, int]: + def _key(screen: Any) -> PanelKey: frame = screen.frame() return ( + _screen_number(screen), str(screen.localizedName()), round(frame.origin.x), round(frame.origin.y), @@ -243,6 +256,12 @@ def _key(screen: Any) -> tuple[str, int, int, int, int]: for key, screen in wanted.items(): if key not in self.panels: self.panels[key] = self._make_panel(screen) + # A display that came or went may change which panel is lit. + if self.visible and self.grab_depth == 0: + self._ensure_front() + + def _lit_keys(self) -> list[PanelKey]: + return lit_keys(list(self.panels), self.target) def _make_panel(self, screen: Any) -> Any: from AppKit import ( @@ -287,8 +306,13 @@ def _make_panel(self, screen: Any) -> Any: def _ensure_front(self) -> None: # orderFrontRegardless, not makeKeyAndOrderFront: the glow may # never take focus or interrupt whatever the user is doing. - for panel in self.panels.values(): - panel.orderFrontRegardless() + lit = set(self._lit_keys()) + for key, panel in self.panels.items(): + if key in lit: + panel.orderFrontRegardless() + else: + panel.setAlphaValue_(0.0) + panel.orderOut_(None) def _hide_panels_now(self) -> None: self.fade_alpha = 0.0 @@ -315,8 +339,8 @@ def _apply_alpha(self, elapsed: float) -> None: else PULSE_MIN + (1.0 - PULSE_MIN) * (0.5 + 0.5 * math.sin(self.phase)) ) alpha = self.fade_alpha * pulse - for panel in self.panels.values(): - panel.setAlphaValue_(alpha) + for key in self._lit_keys(): + self.panels[key].setAlphaValue_(alpha) def _stop_app(self) -> None: from AppKit import NSApplication, NSEvent @@ -341,6 +365,47 @@ def _stop_app(self) -> None: return GlowController.alloc().initWithQueue_(commands) +def _retarget(controller: Any, display_id: int | None) -> None: + """Light ``display_id``'s panel instead. + + A module function, not a controller method: PyObjC reads a method name + without trailing underscores as a zero-argument selector and refuses a + method that takes one. + """ + if display_id is None or display_id == controller.target: + return + controller.target = display_id + if controller.visible and controller.grab_depth == 0: + controller._ensure_front() + + +def lit_keys(keys: list[PanelKey], target: int | None) -> list[PanelKey]: + """The target display's panel, else the primary display's. + + NSScreen.screens() lists the primary (menu-bar) display first and the + panel dict keeps that order, so the first key is the fallback. + """ + chosen = [key for key in keys if key[0] == target] + return chosen or keys[:1] + + +def _screen_number(screen: Any) -> int: + """The CGDirectDisplayID AppKit stores as ``NSScreenNumber``; -1 if absent.""" + try: + return int(screen.deviceDescription()["NSScreenNumber"]) + except Exception: + return -1 + + +def _parse_display_id(arg: str) -> int | None: + """``target``'s argument: a non-negative display id, else ``None``.""" + try: + value = int(arg.strip()) + except ValueError: + return None + return value if value >= 0 else None + + def _rust() -> tuple[float, float, float]: from AppKit import NSApplication @@ -370,23 +435,23 @@ def _read_stdin(commands: queue.Queue[Command]) -> None: """ try: for line in sys.stdin: - command = line.strip() + command, _, arg = line.strip().partition(" ") if not command: continue if command in _MAIN_THREAD_ACKS: ack: threading.Event = threading.Event() - commands.put((command, ack)) + commands.put((command, arg, ack)) print("ok" if ack.wait(_MAIN_ACK_TIMEOUT) else "err", flush=True) elif command == _QUIT: - commands.put((_QUIT, None)) + commands.put((_QUIT, "", None)) return else: - commands.put((command, None)) + commands.put((command, arg, None)) print("ok", flush=True) except Exception: pass finally: - commands.put((_QUIT, None)) + commands.put((_QUIT, "", None)) if __name__ == "__main__": diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py index 4006041e..857596b0 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/linux.py @@ -77,6 +77,10 @@ def activity(self) -> None: def notify_blocked(self) -> None: self.end_session() + def set_target(self, display_id: int | None) -> None: + """Every display keeps its bars; lighting one display is darwin-only so far.""" + del display_id + @contextmanager def grab_hidden(self) -> Iterator[None]: with self._lock: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py index 9bac134f..0cd1a4a1 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/overlay/win32.py @@ -84,6 +84,10 @@ def activity(self) -> None: def notify_blocked(self) -> None: self.end_session() + def set_target(self, display_id: int | None) -> None: + """Every display keeps its bars; lighting one display is darwin-only so far.""" + del display_id + @contextmanager def grab_hidden(self) -> Iterator[None]: with self._lock: diff --git a/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py b/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py index f86efef7..92a5b82e 100644 --- a/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py +++ b/mcp/tst-cu-mcp/src/tst_cu_mcp/server.py @@ -457,13 +457,14 @@ def scroll( ), structured_output=False, ) - def overlay_session(active: bool) -> dict[str, Any]: + def overlay_session(active: bool, session: str = "") -> dict[str, Any]: from tst_cu_mcp.overlay import get_overlay + # One episode per client session: the ring holds while any is open. if active: - get_overlay().begin_session() + get_overlay().begin_session(session) else: - get_overlay().end_session() + get_overlay().end_session(session) return {"active": bool(active)} @server.tool( diff --git a/mcp/tst-cu-mcp/tests/test_overlay_gating.py b/mcp/tst-cu-mcp/tests/test_overlay_gating.py index 0ee391a2..e8d14fbd 100644 --- a/mcp/tst-cu-mcp/tests/test_overlay_gating.py +++ b/mcp/tst-cu-mcp/tests/test_overlay_gating.py @@ -81,12 +81,15 @@ class RecordingOverlay: def __init__(self) -> None: self.calls: list[str] = [] - def begin_session(self) -> None: + def begin_session(self, key: str = "") -> None: self.calls.append("begin") - def end_session(self) -> None: + def end_session(self, key: str = "") -> None: self.calls.append("end") + def set_target(self, display_id: int | None) -> None: + self.calls.append(f"target:{display_id}") + def activity(self) -> None: self.calls.append("activity") @@ -219,6 +222,14 @@ def test_grab_pair_brackets_and_survives_failed_begin(self) -> None: assert dead.commands == ["grab_begin"] assert overlay2.degraded is True + def test_set_target_names_the_display(self) -> None: + transport = RecordingTransport() + overlay = DarwinOverlay(spawn=lambda: transport) + overlay.set_target(51) + overlay.set_target(None) + overlay.set_target(-1) + assert transport.commands == ["target 51"] + def test_shutdown_closes_transport(self) -> None: transport = RecordingTransport() overlay = DarwinOverlay(spawn=lambda: transport) @@ -276,22 +287,30 @@ def fake_capture_png(_self: Any, _rect: Any) -> bytes: monkeypatch.setattr(backends, "get_backend", lambda: backend) @pytest.mark.parametrize( - "action, act", + "action, expected", [ - (lambda: input_control.move_mouse(10, 10), "move"), - (lambda: input_control.click(10, 10), "click"), - (lambda: input_control.type_text("hello"), "type"), - (lambda: input_control.press_keys("ctrl+c"), "press"), - (lambda: input_control.scroll(0, 3), "scroll"), + (lambda: input_control.move_mouse(10, 10), ["target:1", "activity", "move"]), + (lambda: input_control.click(10, 10), ["target:1", "activity", "click"]), + (lambda: input_control.type_text("hello"), ["activity", "type"]), + (lambda: input_control.press_keys("ctrl+c"), ["activity", "press"]), + (lambda: input_control.scroll(0, 3), ["activity", "scroll"]), ], ids=["move", "click", "type", "press", "scroll"], ) def test_actuation_pokes_activity_first( - self, recorder: RecordingOverlay, fake_desktop: None, action: Any, act: str + self, recorder: RecordingOverlay, fake_desktop: None, action: Any, expected: list[str] ) -> None: action() - # The glow lights on intent; the backend act follows it. - assert recorder.calls == ["activity", act] + # A pointer act aims the ring at its display, the glow lights on + # intent, and the backend act follows. + assert recorder.calls == expected + + def test_pointer_off_every_display_aims_nowhere( + self, recorder: RecordingOverlay, fake_desktop: None + ) -> None: + with pytest.raises(ValueError, match="off-screen"): + input_control.move_mouse(5000, 10) + assert recorder.calls == ["target:None", "activity"] def test_refusal_notifies_blocked( self, recorder: RecordingOverlay, fake_desktop: None, tmp_path: Path @@ -324,7 +343,8 @@ def test_capture_brackets_backend_grab( ) -> None: result = capture.capture(max_long_edge=64) assert result.png_bytes.startswith(b"\x89PNG") - assert recorder.calls == ["grab_begin", "grab", "grab_end"] + # The ring moves to the captured display before the grab hides it. + assert recorder.calls == ["target:1", "grab_begin", "grab", "grab_end"] def test_capture_without_overlay_still_works(self, fake_desktop: None) -> None: set_overlay(NullOverlay()) diff --git a/mcp/tst-cu-mcp/tests/test_overlay_rules.py b/mcp/tst-cu-mcp/tests/test_overlay_rules.py new file mode 100644 index 00000000..271c8103 --- /dev/null +++ b/mcp/tst-cu-mcp/tests/test_overlay_rules.py @@ -0,0 +1,290 @@ +"""When the ring shows and where (TD-5226), pinned against a recording painter. + +* Mode: only full control lights it; background never does, and a mode + flip while lit darkens or relights it on the next watcher tick. +* Display: the painter is aimed before it is lit, and re-aimed while lit. +* Episodes: one per client session; the ring holds while any is open. +* Non-bracketing clients: the ring fades after the linger, and a client + that brackets never falls back to the linger. +* Stop and refusals darken it until the next action. +""" + +from __future__ import annotations + +import io +import queue +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +from tst_cu_mcp import safety +from tst_cu_mcp.config import Config, save_config +from tst_cu_mcp.overlay import OVERLAY_ENV, get_overlay, reset_overlay +from tst_cu_mcp.overlay.control import GatedOverlay, PolicyMode +from tst_cu_mcp.overlay.darwin_helper import _parse_display_id, _read_stdin, lit_keys + + +class Painter: + name = "painter" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def begin_session(self) -> None: + self.calls.append("begin") + + def end_session(self) -> None: + self.calls.append("hide") + + def activity(self) -> None: + self.calls.append("show") + + def notify_blocked(self) -> None: + self.calls.append("blocked") + + def set_target(self, display_id: int | None) -> None: + self.calls.append(f"target:{display_id}") + + @contextmanager + def grab_hidden(self) -> Iterator[None]: + self.calls.append("grab_begin") + try: + yield + finally: + self.calls.append("grab_end") + + def shutdown(self) -> None: + self.calls.append("shutdown") + + +class World: + """Mode, Stop and time the controller reads, all set by the test.""" + + def __init__(self) -> None: + self.mode = "full_control" + self.stopped = False + self.now = 1000.0 + + +@pytest.fixture +def world() -> World: + return World() + + +@pytest.fixture +def painter() -> Painter: + return Painter() + + +@pytest.fixture +def ring(world: World, painter: Painter) -> GatedOverlay: + return GatedOverlay( + painter, + mode=lambda: world.mode, + stopped=lambda: world.stopped, + clock=lambda: world.now, + linger=30.0, + watch=False, + ) + + +class TestMode: + def test_background_never_lights( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + world.mode = "background" + ring.set_target(51) + ring.begin_session("s1") + ring.activity() + ring.tick() + assert painter.calls == [] + + def test_full_control_lights_the_aimed_display( + self, ring: GatedOverlay, painter: Painter + ) -> None: + ring.set_target(51) + ring.begin_session("s1") + assert painter.calls == ["target:51", "show"] + + def test_flip_to_background_darkens_and_back_relights( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.begin_session("s1") + world.mode = "background" + ring.tick() + world.mode = "full_control" + ring.tick() + assert painter.calls == ["show", "hide", "show"] + + +class TestDisplay: + def test_retarget_while_lit_moves_the_ring(self, ring: GatedOverlay, painter: Painter) -> None: + ring.begin_session("s1") + ring.set_target(51) + ring.set_target(51) + ring.set_target(53) + assert painter.calls == ["show", "target:51", "target:53"] + + def test_target_while_dark_waits_for_the_ring( + self, ring: GatedOverlay, painter: Painter + ) -> None: + ring.set_target(51) + ring.set_target(53) + assert painter.calls == [] + ring.activity() + assert painter.calls == ["target:53", "show"] + + +class TestEpisodes: + def test_ring_holds_while_any_session_is_open( + self, ring: GatedOverlay, painter: Painter + ) -> None: + ring.begin_session("a") + ring.begin_session("b") + ring.end_session("a") + assert painter.calls == ["show"] + ring.end_session("b") + assert painter.calls == ["show", "hide"] + + def test_a_bracketing_client_never_lingers( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.begin_session("a") + ring.activity() + ring.end_session("a") + world.now += 1.0 + ring.tick() + assert painter.calls == ["show", "hide"] + + +class TestUnbracketed: + def test_activity_lights_then_fades_after_the_linger( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.activity() + world.now += 29.0 + ring.tick() + assert painter.calls == ["show"] + world.now += 2.0 + ring.tick() + assert painter.calls == ["show", "hide"] + + def test_new_activity_restarts_the_linger( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.activity() + world.now += 20.0 + ring.activity() + world.now += 20.0 + ring.tick() + assert painter.calls == ["show"] + + +class TestStop: + def test_stop_darkens_on_the_next_tick( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.activity() + world.stopped = True + ring.tick() + assert painter.calls == ["show", "hide"] + + def test_activity_while_stopped_stays_dark_and_resume_needs_an_action( + self, ring: GatedOverlay, painter: Painter, world: World + ) -> None: + ring.activity() + world.stopped = True + ring.activity() + world.stopped = False + ring.tick() + assert painter.calls == ["show", "hide"] + ring.activity() + assert painter.calls == ["show", "hide", "show"] + + def test_refusal_stays_dark_across_ticks_until_the_next_action( + self, ring: GatedOverlay, painter: Painter + ) -> None: + ring.begin_session("a") + ring.notify_blocked() + ring.tick() + assert painter.calls == ["show", "hide"] + ring.activity() + assert painter.calls == ["show", "hide", "show"] + + +class TestSafety: + def test_grab_brackets_the_painter(self, ring: GatedOverlay, painter: Painter) -> None: + with ring.grab_hidden(): + painter.calls.append("") + assert painter.calls == ["grab_begin", "", "grab_end"] + + def test_a_failing_painter_never_raises(self, world: World) -> None: + class Broken(Painter): + def activity(self) -> None: + raise RuntimeError("window server gone") + + ring = GatedOverlay(Broken(), mode=lambda: world.mode, stopped=lambda: False, watch=False) + ring.activity() + ring.tick() + + def test_get_overlay_wraps_the_painter(self, monkeypatch: pytest.MonkeyPatch) -> None: + import tst_cu_mcp.overlay as overlay_pkg + + monkeypatch.setenv(OVERLAY_ENV, "1") + monkeypatch.setattr(overlay_pkg, "_resolve_painter", Painter) + reset_overlay() + resolved = get_overlay() + assert isinstance(resolved, GatedOverlay) + assert isinstance(resolved.painter, Painter) + + +class TestPolicyMode: + def test_missing_file_uses_the_installed_config(self, tmp_path: Path) -> None: + safety.set_config(Config(mode="full_control")) + try: + assert PolicyMode(lambda: tmp_path / "absent.yaml")() == "full_control" + finally: + safety.set_config(None) + + def test_reads_the_file_and_follows_edits(self, tmp_path: Path) -> None: + path = tmp_path / "config.yaml" + save_config(Config(mode="full_control"), path) + mode = PolicyMode(lambda: path) + assert mode() == "full_control" + save_config(Config(mode="background", allowed_apps=("Finder",)), path) + assert mode() == "background" + + def test_a_broken_file_reads_as_background(self, tmp_path: Path) -> None: + path = tmp_path / "config.yaml" + path.write_text("mode: sideways\n", encoding="utf-8") + assert PolicyMode(lambda: path)() == "background" + + +class TestHelperParsing: + """The AppKit helper's pure parts run anywhere.""" + + def test_display_id_argument(self) -> None: + assert _parse_display_id("51") == 51 + assert _parse_display_id(" 0 ") == 0 + assert _parse_display_id("-1") is None + assert _parse_display_id("main") is None + + def test_lit_keys_prefers_the_target_then_the_primary(self) -> None: + keys = [(52, "A", 0, 0, 1, 1), (51, "B", 1, 0, 1, 1)] + assert lit_keys(keys, 51) == [keys[1]] + assert lit_keys(keys, 99) == [keys[0]] + assert lit_keys(keys, None) == [keys[0]] + assert lit_keys([], 51) == [] + + def test_reader_splits_arguments_and_acks(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "stdin", io.StringIO("target 51\nshow\n")) + out = io.StringIO() + monkeypatch.setattr(sys, "stdout", out) + commands: queue.Queue[tuple[str, str, object]] = queue.Queue() + _read_stdin(commands) # type: ignore[arg-type] + got = [commands.get_nowait()[:2] for _ in range(commands.qsize())] + assert got == [("target", "51"), ("show", ""), ("quit", "")] + assert out.getvalue().split() == ["ok", "ok"]