From 3ba2dfba3c8688250beac2dafc429a8a1d20ab50 Mon Sep 17 00:00:00 2001 From: ATG Date: Fri, 23 Oct 2026 02:08:14 +0530 Subject: [PATCH] feat(macos): add list_windows and a window_id for get_app_state get_app_state always reads the app's focused window, and nothing lets a caller see or pick another one (#83, finding 1). list_windows lists a running app's windows as accessibility reports them on the current Space, with the CGWindowID that query already takes as window_id, plus title, subrole, focused, minimized and bounds. It never launches, activates or raises the app. get_app_state takes the same window_id and roots the snapshot at that window through the existing _AXUIElementGetWindow binding: no activation, no raise, no title fallback to another window. Follow-up reads (action read-back, sky_key read-back) stay on that window until get_app_state is called without window_id, or fall back to the key window once it closes. Focus held in another window is not reported for a bound window, so type_text refuses instead of writing into the other window. Linux and Windows do not expose list_windows (unsupportedTool, like query and select_text) and reject window_id on get_app_state. --- README.md | 2 +- apps/OpenComputerUseLinux/main.go | 3 + apps/OpenComputerUseLinux/main_test.go | 4 ++ apps/OpenComputerUseWindows/main.go | 3 + apps/OpenComputerUseWindows/main_test.go | 4 ++ docs/ARCHITECTURE.md | 2 +- docs/QUALITY_SCORE.md | 2 +- .../2026-10/20261023-0206-list-windows.md | 51 ++++++++++++++++ docs/references/js-repl.md | 5 ++ docs/releases/feature-release-notes.md | 1 + .../AccessibilitySnapshot.swift | 58 +++++++++++++++++-- .../ComputerUseService.swift | 41 ++++++++++--- .../ComputerUseToolDispatcher.swift | 7 ++- .../OpenComputerUseKit/MCPServer.swift | 2 +- .../OpenComputerUseKit/ToolDefinitions.swift | 12 ++++ .../LockedUseDispatchTests.swift | 3 +- .../OpenComputerUseKitTests.swift | 8 ++- .../TargetedQuerySafetyTests.swift | 17 ++++++ scripts/node-repl/open-computer-use-repl.mjs | 9 ++- .../node-repl/open-computer-use-repl.test.mjs | 16 +++++ skills/open-computer-use/SKILL.md | 4 ++ skills/open-computer-use/references/usage.md | 29 +++++++++- 22 files changed, 258 insertions(+), 25 deletions(-) create mode 100644 docs/histories/2026-10/20261023-0206-list-windows.md diff --git a/README.md b/README.md index 596d5a4a..cb9e2bb4 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ evaluation. `ocu repl` keeps them for the current terminal session until `.exit`, Ctrl-D, or termination; `.reset` clears its bindings. On macOS, the separate hidden `Open Computer Use.app` permission agent may remain resident so later commands reuse the same permission identity. `ocu mcp` continues to expose -the nine core native tools, plus `query` on macOS. +the nine core native tools, plus `query` and `list_windows` on macOS. The npm launcher itself currently requires Node.js 18 or newer. Its help always lists `js` / `repl`, while `ocu capabilities --json` reports whether Node, the REPL diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go index 30e98d2d..e09405c6 100644 --- a/apps/OpenComputerUseLinux/main.go +++ b/apps/OpenComputerUseLinux/main.go @@ -222,6 +222,9 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult { if windowPlacement != "keep" { return textResult("window_placement '"+windowPlacement+"' is not supported on Linux", true) } + if _, ok := args["window_id"]; ok { + return textResult("window_id is not supported on Linux", true) + } includeAX, err := optionalBool(args, "include_ax") if err != nil { return textResult(err.Error(), true) diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go index 17fd6ae8..6b99504a 100644 --- a/apps/OpenComputerUseLinux/main_test.go +++ b/apps/OpenComputerUseLinux/main_test.go @@ -674,6 +674,10 @@ func TestLinuxWindowPlacementSchemaAndUnsupportedAgentDisplay(t *testing.T) { if !result.IsError || result.Content[0].Text != "window_placement 'agent_display' is not supported on Linux" { t.Fatalf("agent_display result = %#v", result) } + result = newService().callTool("get_app_state", map[string]any{"app": "Text Editor", "window_id": 41}) + if !result.IsError || result.Content[0].Text != "window_id is not supported on Linux" { + t.Fatalf("window_id result = %#v", result) + } } func TestDoctorJSONRejectsUnsupportedPermissionDiagnostics(t *testing.T) { diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go index ec4a7368..a4a6d800 100644 --- a/apps/OpenComputerUseWindows/main.go +++ b/apps/OpenComputerUseWindows/main.go @@ -285,6 +285,9 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult { if windowPlacement != "keep" { return textResult("window_placement '"+windowPlacement+"' is not supported on Windows", true) } + if _, ok := args["window_id"]; ok { + return textResult("window_id is not supported on Windows", true) + } includeAX, err := optionalBool(args, "include_ax") if err != nil { return textResult(err.Error(), true) diff --git a/apps/OpenComputerUseWindows/main_test.go b/apps/OpenComputerUseWindows/main_test.go index b30a7bcd..1bc43c86 100644 --- a/apps/OpenComputerUseWindows/main_test.go +++ b/apps/OpenComputerUseWindows/main_test.go @@ -494,6 +494,10 @@ func TestWindowsWindowPlacementSchemaAndUnsupportedAgentDisplay(t *testing.T) { if !result.IsError || result.Content[0].Text != "window_placement 'agent_display' is not supported on Windows" { t.Fatalf("agent_display result = %#v", result) } + result = newService().callTool("get_app_state", map[string]any{"app": "Notepad", "window_id": 41}) + if !result.IsError || result.Content[0].Text != "window_id is not supported on Windows" { + t.Fatalf("window_id result = %#v", result) + } } func TestDoctorJSONRejectsUnsupportedPermissionDiagnostics(t *testing.T) { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e956c721..6628becb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -75,7 +75,7 @@ - 对真实 app 的 `get_app_state` / action tool 入口，当前只保留一层密码管理器 bundle denylist：bundle-id 直传时直接返回 safety denial；名称匹配时默认不解析到这些 app。终端、Chrome / Atlas 和系统组件不再属于内置阻止目标。 - 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出，便于后续把 `element_index` 和截图坐标统一到同一套参考系。 - 三个平台的显式 `get_app_state` 均支持 `include_ax` / `include_screenshot`，默认 true，至少保留一项。JS 对应 `includeAX` / `includeScreenshot`，CLI snapshot 提供同名连字符参数。关闭 AX 跳过树、焦点和选区，并使旧快照 / 查询索引及别名失效；关闭截图跳过捕获与编码，元素操作仍可用，截图坐标操作须重新获取图片。参数经 macOS app agent 转发；不改变动作回读，也不做自动图片去重。见 [按需观察](selective-observation.md)。 -- macOS 额外提供 `query` 和 `select_text`，当前 native surface 为 11 个工具；Linux / Windows 仍为 9 个核心工具。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引，支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义；写入前及返回前检查会话，验证文本未变且实际选区等于请求，即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出；结果刷新使用 read-only recovery，不通过激活、键盘或真实指针补救。 +- macOS 额外提供 `query`、`select_text` 和 `list_windows`，当前 native surface 为 12 个工具；Linux / Windows 仍为 9 个核心工具，并对 `get_app_state` 的 `window_id` 返回不支持错误。`list_windows` 列出 app 在当前 Space 的 AX 窗口及其 CGWindowID；`window_id` 把快照、截图、焦点和后续动作回读绑定到该窗口（不激活、不抬起），直到不带 `window_id` 再次读取或该窗口关闭。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引，支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义；写入前及返回前检查会话，验证文本未变且实际选区等于请求，即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出；结果刷新使用 read-only recovery，不通过激活、键盘或真实指针补救。 - `click` / `set_value` 在执行真实动作前后，会额外驱动一层透明 `SoftwareCursorOverlay` window：两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核，显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器，优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹；首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点，后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample，而是经过一层独立的 visual dynamics 状态，把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble，`set_value` 则只做 settle / idle，不给 pulse；两者收尾后会在目标点继续保持 idle 状态，等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动；只有连续 30 秒没有新动作时才做 cleanup，这样连续 tool call 不会反复从 fresh `(0,0)` 起步；如果宿主在任务 / turn 结束时发出 `turn-ended`，cursor 会立即消失并清掉本轮位置状态。 - overlay 用目标点相对窗口的锚点跟随窗口移动；行进、settle、点击反馈和 idle 都读取最新缓存帧，发现位置变化时直接重定位可见 tip 并清掉旧速度。窗口帧在后台串行读取，最多 30 Hz、同时最多一个请求，避免 AX 阻塞渲染线程；优先读取快照持有的 AX 窗口，仅在没有 AX 元素时使用窗口列表，读取失败保留最后有效帧。只保留当前窗口的追踪状态，reset / idle 超时后释放；这层跟随只调整显示位置，输入投递仍走既有路径。 - overlay 的 visual style 不再自己从官方 app bundle 裁 `SoftwareCursor` 小图；主 MCP runtime 现在和 `CursorMotion` 一样优先渲染仓库里沉淀的 `official-software-cursor-window-252.png` baseline，只有资源缺失时才退回 `OpenComputerUseKit` 内部的程序化 pointer/fog fallback。命中点 anchor 仍固定在 `126x126` 画布里的同一组 tip-offset 上；glyph 自身的 neutral heading 继续沿用 `CursorMotion` / 官方 baseline 的 `-3π/4`。主 runtime overlay window 按 AppKit 全局坐标移动，因此在把 AX / `CGWindowList` 产出的 y-down screen-space 点击目标喂给 overlay 之前，会先转换成对应屏幕的 AppKit 全局坐标；路径选路用屏幕上实际可见的 AppKit forward heading，进入 visual dynamics / render state 前则把 velocity 的 y 轴翻回 CursorMotion 的 y-down screen state，再交给 AppKit 绘制层做角度和 `dy` 翻转。程序化 fallback 保留 neutral artwork correction，把它的天然轮廓轴对齐到 `CursorMotion` / 官方 baseline 的 `-3π/4` forward 方向，但不让实验线依赖 runtime 代码。 diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md index 56dc1241..65b1caa4 100644 --- a/docs/QUALITY_SCORE.md +++ b/docs/QUALITY_SCORE.md @@ -11,7 +11,7 @@ | 区域 | 评分 | 原因 | 下一步 | | --- | --- | --- | --- | -| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导，以及共享的 9 个核心 tools；macOS 另有 query、select_text，JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 | +| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导，以及共享的 9 个核心 tools；macOS 另有 query、select_text、list_windows（以及 get_app_state 的 window_id），JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 | | Windows runtime | C | 已新增独立 Go `.exe`，通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`；默认不再自动启动 app、执行 `SetFocus`，或让 `type_text` 走可能抢前台的 UIA text fallback；截图会过滤隐藏、最小化、cloaked、离屏和退化窗口，并在 Go 层复核 PNG，但整体仍是功能性第一版。 | 维护 WPF 观察 / 输入 / 截图边界 fixture 和 Windows PR smoke，扩展更多工具与 toolkit；补 installer/signing，以及更原生的 Go UIA 实现或更稳定的 bridge。 | | Linux runtime | C | 已新增独立 Go binary，通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`；Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence，并已接入 npm bundled artifact 分发，但截图在 GNOME Wayland 下仍只能 best-effort，coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke，补完整工具覆盖、portal/compositor screenshot 路径，以及更原生的 Go D-Bus/libatspi bridge。 | | macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口，离线和遮罩 rehearsal 有回归；安装恢复和无许可双保护释放已实测，独立无遮罩 Return 解锁 / 重锁已实测，受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过，完整会话认证状态 / 故障链路尚未通过实测；正常启用改为管理员 enable 与签名校验，普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认，继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 | diff --git a/docs/histories/2026-10/20261023-0206-list-windows.md b/docs/histories/2026-10/20261023-0206-list-windows.md new file mode 100644 index 00000000..2e898962 --- /dev/null +++ b/docs/histories/2026-10/20261023-0206-list-windows.md @@ -0,0 +1,51 @@ +## [2026-10-23 02:06] | Task: Add list_windows and get_app_state window_id + +### Execution Context + +- Agent ID: Claude Code 子 Agent（运行时未暴露 ID） +- Base Model: Claude Opus 5.5（`claude-opus-5-5`） +- Runtime: Claude Code（Claude Agent SDK），macOS 本机 + +### User Query + +把 fork 里的 `list_windows` 移植到上游，回应 #83 的第 1 条（多窗口 app 中读取和操作落到错误窗口）；再判断 `get_app_state` 能否小而安全地按窗口绑定，能就一起加上，只能列出窗口却不能选中其一只算半个功能。 + +### Changes Overview + +**Scope:** macOS `OpenComputerUseKit`、Linux / Windows Go runtime 的参数校验、Node REPL、文档。 + +- 新增 native `list_windows`：只解析已运行的 app，不启动、不激活、不抬起；按 AX `AXWindows` 返回当前 Space 的窗口，每项包含 `window_id`（与 `query` 相同的 CGWindowID，经现有 `_AXUIElementGetWindow` 绑定得到）、`title`、`subrole`、`focused`、`minimized` 和屏幕坐标 `bounds`。与 `query` 一样在读取前检查会话租约，输出复用 `query` 的 JSON 编码。 +- `get_app_state` 新增 `window_id`：用已有的 `windowElement(for:)` 找到该 AX 窗口作为快照根；不走激活 / 抬起的恢复路径，不按标题回退到别的窗口；找不到时报错并提示调用 `list_windows`。 +- 绑定会延续到后续回读：service 按进程记住 `window_id`，动作回读与 `sky_key` 的回读继续读该窗口；再次不带 `window_id` 调用 `get_app_state` 即解除；窗口已关闭时回读回到 key window。 +- 绑定窗口不是 key window 时，app 的键盘焦点不算这个窗口的焦点：快照不报告它，`type_text` 因此拒绝写入另一个窗口的输入框，而不是静默写进去。`sky_key` 会把按键送到绑定窗口。 +- Linux / Windows：不实现 `list_windows`（与 `query`、`select_text` 一样返回 `unsupportedTool`），`get_app_state` 收到 `window_id` 时明确报不支持，不静默忽略。 +- Node REPL：`app.listWindows()`，`getAXState({ windowId })`。 +- 更新工具清单与描述测试、会话租约测试、参数校验测试，以及 SKILL、usage、js-repl、README、ARCHITECTURE、QUALITY_SCORE 和功能发布记录。 + +### Design Intent + +#83 的第 1 条是缺功能：快照永远绑定 app 的 focused window，调用方既看不到其他窗口，也选不了。`query` 已经接受 `window_id`，这里补上取得 id 的途径，并让 `get_app_state` 用同一个 id 绑定快照。绑定必须覆盖动作回读，否则点击之后的回读会切回 key window 并替换缓存，后续索引又落到另一个窗口，问题原样回来。键盘输入天然跟随 app 的焦点，因此只在焦点不属于绑定窗口时让快照不报告焦点，避免 `type_text` 写错窗口；更大的“窗口作用域索引”不在本次范围。 + +### Validation + +- `swift test`（在共享 GUI 锁下运行完整套件）：383 tests，7 skipped，0 failures。新增 / 更新：工具数量与名单、`list_windows` 只读注解与 schema、`window_id` schema、非法 `window_id`（0、-1、1.5、true、"7"）、缺少 `app`、不存在的 app 返回 `appNotFound` 而不启动、会话租约撤销时 `list_windows` 在解析目标前失败。 +- Node REPL：45 tests passed（新增 `listWindows` / `windowId` 转发与异常响应）。 +- Go：Windows `go test` 通过；Linux 新增用例通过，Linux 包里 6 个需要 Linux python3 bridge 的既有用例在 macOS 上失败，与未改动的 main 相同；两平台 `go vet`（GOOS=linux / windows）通过。 +- 真机：一个不激活、不抢焦点的探针 app 打开两个窗口 A（key）/ B。`list_windows` 列出两者且 A 为 focused；不带 `window_id` 读到 A；`window_id=B` 读到 B 的树；按 B 的索引点击后回读仍是 B 且按钮状态已变；A 的输入框持有焦点时，`type_text` auto 报错而不是写入 A；`sky_key` 把文字打进 B；再次不带 `window_id` 读回 A；不存在的 id 明确报错。前台 app 前后不变，探针已退出。 +- `make check-docs`、`git diff --check` 通过。 + +### Files Modified + +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift` +- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift` +- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TargetedQuerySafetyTests.swift` +- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/LockedUseDispatchTests.swift` +- `apps/OpenComputerUseLinux/main.go`、`apps/OpenComputerUseLinux/main_test.go` +- `apps/OpenComputerUseWindows/main.go`、`apps/OpenComputerUseWindows/main_test.go` +- `scripts/node-repl/open-computer-use-repl.mjs`、`scripts/node-repl/open-computer-use-repl.test.mjs` +- `skills/open-computer-use/SKILL.md`、`skills/open-computer-use/references/usage.md` +- `docs/references/js-repl.md`、`docs/ARCHITECTURE.md`、`docs/QUALITY_SCORE.md`、`docs/releases/feature-release-notes.md`、`README.md` diff --git a/docs/references/js-repl.md b/docs/references/js-repl.md index 02bea02d..1dd823ae 100644 --- a/docs/references/js-repl.md +++ b/docs/references/js-repl.md @@ -88,6 +88,11 @@ verification throw through `js`. Verification still reads the applied range when action read-back is disabled. The tool does not use keyboard/pointer fallback or activate the application. Linux/Windows report the native unsupported-tool error. +On macOS, `await app.listWindows()` returns the app's windows as +`[{ window_id, title, subrole, focused, minimized, bounds }]`, and +`app.getAXState({ windowId })` reads one of them instead of the key window; later +actions stay on it until a state read without `windowId`. + Discovery is available through `await cua.getState()` and `await cua.listApps()`. `listApps()` normalizes the native human-readable app catalog into an array of `{ id, displayName, lastUsedDate?, useCount?, diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md index c000ea0c..dcdcae2e 100644 --- a/docs/releases/feature-release-notes.md +++ b/docs/releases/feature-release-notes.md @@ -4,6 +4,7 @@ | 日期 | 功能域 | 用户价值 | 变更摘要 | | --- | --- | --- | --- | +| 2026-10-23 | macOS 多窗口选择 | 多窗口 app 中 Agent 能看到全部窗口并指定要读取和操作的那一个，不再只能落在 key window。 | 新增 native `list_windows` 与 JS `app.listWindows()`，返回当前 Space 的 AX 窗口、`window_id`、标题、subrole、焦点、最小化状态和屏幕坐标；`get_app_state` 新增 `window_id`，按 CGWindowID 绑定快照、截图、焦点与后续动作回读，不激活、不抬起窗口，窗口关闭后回到 key window；Linux / Windows 对 `window_id` 明确报不支持。 | | 2026-10-13 | Windows 滚动与坐标 | Windows 上 `scroll` 不再在默认配置下直接失败；150% 缩放且窗口位于屏幕右侧时，`get_app_state` 也能返回截图和窗口 bounds。 | 修复 `scroll` 请求漏传 `windowBounds` 导致坐标回退必然报 `No usable screenshot coordinates`；滚轮消息保留屏幕坐标并修正横向方向，PowerShell bridge 启动时对齐 DPI 物理坐标；`ScrollPattern` 在当前状态不可滚动时回退到坐标路径而不是让整个 tool call 失败。 | | 2026-10-13 | Windows 全局输入兜底 | 需要真实指针或键盘的 Windows 动作（`click_method=global`、WinUI 键盘输入、子控件点击）在显式授权后可用，默认路径完全不变。 | Windows 实现 `click_method=global` 真实指针点击；`drag` / `scroll` 的坐标回退在 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` 时使用真实指针，按键另需显式 `key_method=global`，`app_post` 仍保持 HWND 消息语义；未授权时在任何物理输入前拒绝，核对目标前台 / 命中 / 几何并在异常时释放输入。 | | 2026-10-17 | 1.1.2 修复发布 | 软件光标持续跟随移动窗口，Linux 批量动作可减少回读开销，安装链接统一到新仓库。 | 同步所有版本源至 1.1.2；发布窗口锚点跟踪、Linux 动作回读配置与仓库改名后的 npm / 插件元信息，沿用既有命令和 App 签名身份。 | diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift index 2fbfc30a..fde3c13c 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift @@ -189,7 +189,8 @@ enum SnapshotBuilder { textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation, - observation: SnapshotObservationOptions = .defaults + observation: SnapshotObservationOptions = .defaults, + windowID: CGWindowID? = nil ) throws -> AppSnapshot { try observation.validate() if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() { @@ -208,7 +209,17 @@ enum SnapshotBuilder { || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL)) let systemWide = AXUIElementCreateSystemWide() var focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute) - var focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) + // A window asked for by id is read as it is: never swapped for another, activated or raised. + let recoveryPolicy = windowID == nil ? recoveryPolicy : .readOnly + var focusedWindow: AXUIElement? + if let windowID { + guard let named = windowElement(for: windowID, appElement: appElement) else { + throw ComputerUseError.stateUnavailable("window_id \(windowID) is not a current window of \(app.bundleIdentifier ?? app.name). Call list_windows for its current windows.") + } + focusedWindow = named + } else { + focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) + } if focusedWindow == nil, recoveryPolicy == .allowActivation, recoverVisibleWindow(for: app, appElement: appElement, preferredWindow: nil) { @@ -228,7 +239,7 @@ enum SnapshotBuilder { // the title/area heuristics remain the fallback. var windowCapture = TimingLog.measure("snapshot.window_capture") { WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow), capture: observation.includeScreenshot) - ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot) + ?? (windowID != nil ? nil : WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot)) } if windowCapture == nil, recoveryPolicy == .allowActivation, @@ -257,7 +268,8 @@ enum SnapshotBuilder { textLimit: textLimit, treeLimits: treeLimits, lazyWebAccessibility: lazyWebAccessibility, - observation: observation + observation: observation, + windowBound: windowID != nil ) } @@ -272,7 +284,8 @@ enum SnapshotBuilder { textLimit: SnapshotTextLimit, treeLimits: AccessibilityTreeLimits, lazyWebAccessibility: Bool, - observation: SnapshotObservationOptions + observation: SnapshotObservationOptions, + windowBound: Bool = false ) -> AppSnapshot { let windowBounds = windowCapture.bounds let screenshot = observation.includeScreenshot ? windowCapture.screenshotResult() @@ -286,7 +299,12 @@ enum SnapshotBuilder { windowElement: rootElement, elements: [:], screenshotNote: screenshot.note, observation: observation) } - let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) + var focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) + // A window read by id need not be the key window; focus held in another window is not its + // focus, so type_text cannot write into that other window. + if windowBound, let element = focusedElement, !belongsToWindow(element, window: rootElement) { + focusedElement = nil + } let selectedText = focusedElement.flatMap { copySelectedText($0, textLimit: textLimit) } let context = RenderContext( windowBounds: windowBounds, @@ -2384,6 +2402,34 @@ extension SnapshotBuilder { ) } + /// The app's windows as accessibility lists them on this Space, read-only: the window_id that + /// get_app_state and query take, title, subrole, whether it is the app's focused window, + /// whether it is minimized, and its frame in screen points. + static func listWindows(for app: RunningAppDescriptor) throws -> [[String: Any]] { + guard PermissionDiagnostics.current().accessibilityTrusted else { + throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `open-computer-use doctor` and grant access to Open Computer Use.") + } + + let appElement = AXUIElementCreateApplication(app.pid) + AXUIElementSetMessagingTimeout(appElement, 0.1) + let focusedID = copyElement(appElement, attribute: kAXFocusedWindowAttribute).flatMap { SkyLightSPI.shared.windowID(for: $0) } + return (copyArray(appElement, attribute: kAXWindowsAttribute) ?? []).compactMap { window in + guard let id = SkyLightSPI.shared.windowID(for: window) else { return nil } + var entry: [String: Any] = [ + "window_id": Int(id), + "title": stringValue(of: window, attribute: kAXTitleAttribute) ?? "", + "subrole": stringValue(of: window, attribute: kAXSubroleAttribute) ?? "", + "focused": id == focusedID, + "minimized": boolValue(of: window, attribute: kAXMinimizedAttribute) == true, + ] + if let frame = axFrame(position: attributeValue(of: window, attribute: kAXPositionAttribute), + size: attributeValue(of: window, attribute: kAXSizeAttribute)) { + entry["bounds"] = ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height] + } + return entry + } + } + /// Current window bounds and element frame for a queried control, so an action /// lands correctly even if the window moved after the query. static func currentGeometry(of record: ElementRecord, in context: TargetedAX.WindowContext, diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift index a74e8230..0893374d 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift @@ -542,6 +542,9 @@ func shouldPreferContainingWebRowAXClickCandidate( public final class ComputerUseService { private var snapshotsByApp: [String: AppSnapshot] = [:] private var invalidatedSnapshotAliases: Set = [] + // The window get_app_state was given as window_id, per app process. Follow-up reads (an + // action's read-back) stay on it until get_app_state is called without one or it closes. + private var boundWindows: [pid_t: CGWindowID] = [:] // Read per call: the app agent applies the caller's OPEN_COMPUTER_USE_* variables // for the duration of each request. private var actionReadBack: Bool { actionReadBackEnabled(environment: ProcessInfo.processInfo.environment) } @@ -579,7 +582,7 @@ public final class ComputerUseService { return .text("ok") } - return snapshotResult(for: try refreshSnapshot(for: query, recoveryPolicy: recoveryPolicy), style: .actionResult) + return snapshotResult(for: try refreshBoundSnapshot(for: query, recoveryPolicy: recoveryPolicy), style: .actionResult) } public func listApps() -> ToolCallResult { @@ -590,20 +593,27 @@ public final class ComputerUseService { ) } + /// The windows of an already-running app, without launching, activating or raising it. + public func listWindows(app query: String) throws -> [[String: Any]] { + try requireUsableComputerUseSession() + return try SnapshotBuilder.listWindows(for: AppDiscovery.resolveRunning(query)) + } + public func getAppState( app query: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, windowPlacement: WindowPlacement = .keep, - observation: SnapshotObservationOptions = .defaults + observation: SnapshotObservationOptions = .defaults, + windowID: CGWindowID? = nil ) throws -> ToolCallResult { try observation.validate() guard windowPlacement != .keep else { - return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, observation: observation), style: .fullState) + return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, observation: observation, windowID: windowID), style: .fullState) } // Explicit placements never activate or raise; they only move the frame. - let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation) + let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation, windowID: windowID) guard snapshot.mode != .fixture else { throw ComputerUseError.message("window_placement '\(windowPlacement.rawValue)' is not supported for fixture apps") } @@ -619,7 +629,7 @@ public final class ComputerUseService { break } - return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation), style: .fullState) + return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation, windowID: windowID), style: .fullState) } public func click( @@ -920,7 +930,7 @@ public final class ComputerUseService { let target = try skyKeyTarget(for: snapshot) try InputSimulation.typeTextWithSkyLight(text, windowID: target.windowID, pid: target.pid) return snapshotResult( - for: try refreshSnapshot(for: query, recoveryPolicy: keyActionSnapshotRecoveryPolicy(for: keyMethod)), + for: try refreshBoundSnapshot(for: query, recoveryPolicy: keyActionSnapshotRecoveryPolicy(for: keyMethod)), style: .actionResult ) } @@ -951,7 +961,7 @@ public final class ComputerUseService { let target = try skyKeyTarget(for: snapshot) try InputSimulation.pressKeyWithSkyLight(key, windowID: target.windowID, pid: target.pid) return snapshotResult( - for: try refreshSnapshot(for: query, recoveryPolicy: keyActionSnapshotRecoveryPolicy(for: keyMethod)), + for: try refreshBoundSnapshot(for: query, recoveryPolicy: keyActionSnapshotRecoveryPolicy(for: keyMethod)), style: .actionResult ) } @@ -1219,7 +1229,8 @@ public final class ComputerUseService { textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation, - observation: SnapshotObservationOptions = .defaults + observation: SnapshotObservationOptions = .defaults, + windowID: CGWindowID? = nil ) throws -> AppSnapshot { let app = try AppDiscovery.resolve(query) let snapshot = try SnapshotBuilder.build( @@ -1227,13 +1238,25 @@ public final class ComputerUseService { textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: recoveryPolicy, - observation: observation + observation: observation, + windowID: windowID ) + boundWindows[app.pid] = windowID rememberSnapshot(snapshot, for: query) return snapshot } + /// A follow-up read: the window get_app_state bound with window_id, or the app's key window + /// when none is bound or the bound one has closed (an action can close it). + private func refreshBoundSnapshot(for query: String, recoveryPolicy: SnapshotRecoveryPolicy) throws -> AppSnapshot { + if let pid = snapshotsByApp[query.lowercased()]?.app.pid, let windowID = boundWindows[pid], + let snapshot = try? refreshSnapshot(for: query, recoveryPolicy: recoveryPolicy, windowID: windowID) { + return snapshot + } + return try refreshSnapshot(for: query, recoveryPolicy: recoveryPolicy) + } + func rememberSnapshot(_ snapshot: AppSnapshot, for query: String) { let app = snapshot.app if !snapshot.observation.includeAX { diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift index cfa984c0..e915ddc9 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift @@ -48,6 +48,8 @@ public final class ComputerUseToolDispatcher { switch name { case "list_apps": return service.listApps() + case "list_windows": + return .text(try renderedQueryMatches(["windows": try service.listWindows(app: requireString("app", in: arguments))])) case "query": let matches = try service.query( app: requireString("app", in: arguments), @@ -71,7 +73,8 @@ public final class ComputerUseToolDispatcher { observation: SnapshotObservationOptions( includeAX: try strictObservationBool("include_ax", in: arguments), includeScreenshot: try strictObservationBool("include_screenshot", in: arguments) - ) + ), + windowID: try validatedQueryWindowID(arguments["window_id"]) ) case "click": return try service.click( @@ -175,7 +178,7 @@ public final class ComputerUseToolDispatcher { } } - /// Machine-readable matches with explicit completeness metadata. + /// Machine-readable matches with explicit completeness metadata (also list_windows' windows). private func renderedQueryMatches(_ matches: [String: Any]) throws -> String { let data = try JSONSerialization.data(withJSONObject: matches, options: [.sortedKeys]) guard let text = String(data: data, encoding: .utf8) else { diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift index d1ab4359..f65db0ee 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift @@ -7,7 +7,7 @@ Some apps might have a separate dedicated plugin or skill. You may want to use t Begin by calling `get_app_state` every turn you want to use Computer Use to get the latest state before acting. Codex will automatically stop the session after each assistant turn, so this step is required before interacting with apps in a new assistant turn. -The available tools are list_apps, get_app_state, click, perform_secondary_action, scroll, drag, type_text, press_key, set_value, select_text, and query. If any of these are not available in your environment, use tool_search to surface one before calling any Computer Use action tools. +The available tools are list_apps, get_app_state, click, perform_secondary_action, scroll, drag, type_text, press_key, set_value, select_text, query, and list_windows. If any of these are not available in your environment, use tool_search to surface one before calling any Computer Use action tools. Computer Use tools allow you to use the user's apps in the background, so while you're using an app, the user can continue to use other apps on their computer. Avoid doing anything that would disrupt the user's active session, such as overwriting the contents of their clipboard, unless they asked you to! diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift index d6c0fd3b..645cc599 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift @@ -104,6 +104,7 @@ public enum ToolDefinitions { description: "keep (default) leaves the window where it is. agent_display (macOS) parks the app's window on an invisible display owned by the agent so a covered or other-Space window renders, exposes its full tree and accepts input without touching the user's Space, focus or pointer; the window disappears from the user's desktop until restore or process exit. restore puts it back.", enumValues: WindowPlacement.allCases.map(\.rawValue) ), + "window_id": positiveIntegerProperty(description: "Read this window (a window_id from list_windows) instead of the app's key window, without activating or raising it. Later actions and their results stay on this window until get_app_state is called without window_id or the window closes. Keyboard input still goes to the app's focused window unless key_method is sky_key."), ], required: ["app"] ) @@ -114,6 +115,17 @@ public enum ToolDefinitions { annotations: readOnlyAnnotations(), inputSchema: objectSchema(properties: [:], required: []) ), + ToolDefinition( + name: "list_windows", + description: "List a running app's windows as accessibility reports them on the current Space: window_id (what get_app_state and query take), title, subrole, focused, minimized, and bounds in screen points. Does not launch, activate or raise the app. This tool is part of plugin `Computer Use`.", + annotations: readOnlyAnnotations(), + inputSchema: objectSchema( + properties: [ + "app": stringProperty(description: "App name or bundle identifier"), + ], + required: ["app"] + ) + ), ToolDefinition( name: "perform_secondary_action", description: "Invoke a secondary accessibility action exposed by an element. This tool is part of plugin `Computer Use`.", diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/LockedUseDispatchTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/LockedUseDispatchTests.swift index 57421c5e..581b5c61 100644 --- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/LockedUseDispatchTests.swift +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/LockedUseDispatchTests.swift @@ -40,7 +40,8 @@ final class LockedUseDispatchTests: XCTestCase { { try SkyClickDispatcher.click(target: click, clickCount: 1) }, { try SkyKeyboardDispatcher.typeText(target: keyboard, text: "probe") }, { try SkyKeyboardDispatcher.pressKey(target: keyboard, key: "cmd+a") }, - { _ = try ComputerUseService().query(app: "NoSuchRevokedLeaseApp", text: "Send") } + { _ = try ComputerUseService().query(app: "NoSuchRevokedLeaseApp", text: "Send") }, + { _ = try ComputerUseService().listWindows(app: "NoSuchRevokedLeaseApp") } ] for action in actions { try LockedUseActionScope.withValidator({ throw Revoked.lease }, body: { diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift index 1bf4c58d..11ef0e66 100644 --- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift @@ -279,7 +279,7 @@ final class OpenComputerUseKitTests: XCTestCase { } func testToolDefinitionCount() { - XCTAssertEqual(ToolDefinitions.all.count, 11) + XCTAssertEqual(ToolDefinitions.all.count, 12) XCTAssertEqual( ToolDefinitions.all.map(\.name).sorted(), [ @@ -287,6 +287,7 @@ final class OpenComputerUseKitTests: XCTestCase { "drag", "get_app_state", "list_apps", + "list_windows", "perform_secondary_action", "press_key", "query", @@ -732,6 +733,11 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertEqual(getAppStateProperties?["max_tree_depth"]?["type"] as? String, "integer") XCTAssertEqual(getAppStateProperties?["max_tree_depth"]?["minimum"] as? Int, 1) XCTAssertEqual(getAppStateSchema?["required"] as? [String], ["app"]) + XCTAssertEqual(getAppStateProperties?["window_id"]?["type"] as? String, "integer") + XCTAssertEqual(getAppStateProperties?["window_id"]?["minimum"] as? Int, 1) + XCTAssertEqual(tools["list_windows"]?.annotations["readOnlyHint"] as? Bool, true) + XCTAssertEqual(tools["list_windows"]?.inputSchema["required"] as? [String], ["app"]) + XCTAssertTrue(tools["list_windows"]?.description.hasSuffix("This tool is part of plugin `Computer Use`.") == true) let scrollPages = (tools["scroll"]?.inputSchema["properties"] as? [String: [String: Any]])?["pages"] XCTAssertEqual(scrollPages?["type"] as? String, "number") XCTAssertEqual( diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TargetedQuerySafetyTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TargetedQuerySafetyTests.swift index 63dddb30..97db3fc3 100644 --- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TargetedQuerySafetyTests.swift +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TargetedQuerySafetyTests.swift @@ -16,6 +16,23 @@ final class TargetedQuerySafetyTests: XCTestCase { XCTAssertTrue(result.primaryText?.contains("window_id") == true) } + func testListWindowsAndWindowBindingRejectBadInputsWithoutLaunching() throws { + let dispatcher = ComputerUseToolDispatcher() + XCTAssertEqual(dispatcher.callToolAsResult(name: "list_windows", arguments: [:]).primaryText, "Missing required argument: app") + for value: Any in [0, -1, 1.5, true, "7"] { + let result = dispatcher.callToolAsResult(name: "get_app_state", arguments: ["app": "NoSuchReviewApp", "window_id": value]) + XCTAssertTrue(result.isError) + XCTAssertTrue(result.primaryText?.contains("window_id") == true, "\(value)") + } + guard LockedUseSession.current().state == .unlocked else { + throw XCTSkip("Requires a completed unlocked console session") + } + // list_windows resolves running apps only: an absent app is reported, never launched. + let absent = dispatcher.callToolAsResult(name: "list_windows", arguments: ["app": "NoSuchReviewApp"]) + XCTAssertTrue(absent.isError) + XCTAssertEqual(absent.primaryText, "appNotFound(\"NoSuchReviewApp\")") + } + func testQueryStringsAndExactRejectSilentCoercion() throws { XCTAssertEqual(try validatedQueryString("Send", key: "text"), "Send") XCTAssertThrowsError(try validatedQueryString(3, key: "text")) diff --git a/scripts/node-repl/open-computer-use-repl.mjs b/scripts/node-repl/open-computer-use-repl.mjs index 5e5c780e..ebce60a4 100755 --- a/scripts/node-repl/open-computer-use-repl.mjs +++ b/scripts/node-repl/open-computer-use-repl.mjs @@ -38,7 +38,8 @@ The runtime exposes an asynchronous app-bound API: - \`await cua.listApps({ emit? })\`: list current apps. - \`await cua.getApp(nameOrBundleID, { initialState? })\`: bind an app; initialState defaults to true. Use false for a lightweight binding before query. - \`await app.query({ text?, role?, exact?, limit?, maxNodes?, windowId? })\`: macOS-only AX lookup; returns matches and truncation metadata, with indexes usable in this native session. -- \`await app.getAXState({ emit?, textLimit?, maxTreeNodes?, maxTreeDepth?, windowPlacement? })\` +- \`await app.listWindows()\`: macOS-only; the app's windows on this Space as \`[{ window_id, title, subrole, focused, minimized, bounds }]\`. +- \`await app.getAXState({ emit?, textLimit?, maxTreeNodes?, maxTreeDepth?, windowPlacement?, windowId? })\`: windowId (macOS) reads that window instead of the key window; later actions stay on it until a read without windowId. - \`await app.getScreenshot({ emit? })\` - \`await app.getAXStateAndScreenshot(options?)\` - Optional observation on macOS, Windows and Linux: \`app.getScreenshot({ includeAX: false })\` skips AX/focus/selection and invalidates prior element indexes; refresh AX before element actions. \`app.getAXState({ includeScreenshot: false })\` skips capture; screenshot-coordinate actions require a fresh screenshot. Omitted options retain full native observation. @@ -259,6 +260,7 @@ function optionsToSnapshotArgs(options = {}) { if (options.maxTreeNodes !== undefined) args.max_tree_nodes = options.maxTreeNodes; if (options.maxTreeDepth !== undefined) args.max_tree_depth = options.maxTreeDepth; if (options.windowPlacement !== undefined) args.window_placement = options.windowPlacement; + if (options.windowId !== undefined) args.window_id = options.windowId; for (const [option, argument] of [["includeAX", "include_ax"], ["includeScreenshot", "include_screenshot"]]) { if (options[option] !== undefined) { if (typeof options[option] !== "boolean") throw new Error(`${option} must be a boolean`); @@ -326,6 +328,11 @@ export function createCuaApi(native, activeOutput) { if (!isPlainObject(response) || !Array.isArray(response.matches) || typeof response.truncated !== "boolean" || typeof response.stop_reason !== "string") throw new Error("Invalid query response"); return response; }, + async listWindows() { + const response = JSON.parse(toolResultText(await call("list_windows", { app }))); + if (!isPlainObject(response) || !Array.isArray(response.windows)) throw new Error("Invalid list_windows response"); + return response.windows; + }, async getAXState(options = {}) { const result = await call("get_app_state", { app, ...optionsToSnapshotArgs(options) }); const text = toolResultText(result); diff --git a/scripts/node-repl/open-computer-use-repl.test.mjs b/scripts/node-repl/open-computer-use-repl.test.mjs index ed1c83b4..f39b97be 100644 --- a/scripts/node-repl/open-computer-use-repl.test.mjs +++ b/scripts/node-repl/open-computer-use-repl.test.mjs @@ -394,6 +394,22 @@ test("query returns truncation metadata and rejects malformed native responses", assert.match(bad.content.at(-1).text, /Invalid query response/); }); +test("listWindows returns the native windows and getAXState binds one by windowId", async () => { + const native = mockNative(), session = new PersistentJavaScriptSession({ native }); + const request = native.request; + native.request = async (method, params) => params.name === "list_windows" + ? (native.calls.push(params), textResult(JSON.stringify({ windows: [{ window_id: 41, title: "Notes", focused: false }] }))) + : request(method, params); + const result = await session.run(`var windowedApp = await cua.getApp("Text", { initialState: false }); var windows = await windowedApp.listWindows(); await windowedApp.getAXState({ windowId: windows[0].window_id, emit: false }); nodeRepl.write(windows[0].title);`); + assert.equal(result.isError, false, result.content.at(-1).text); + assert.deepEqual(native.calls.map(c => c.arguments), [{ app: "Text" }, { app: "Text", window_id: 41 }]); + assert.equal(result.content.at(-1).text, "Notes"); + native.request = async () => textResult("[]"); + const bad = await session.run(`await windowedApp.listWindows();`); + assert.equal(bad.isError, true); + assert.match(bad.content.at(-1).text, /Invalid list_windows response/); +}); + for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) { test(`${Session.name}: selectText forwards all selection modes without a snapshot`, async t => { diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.md index c2004401..a7ddc6f3 100644 --- a/skills/open-computer-use/SKILL.md +++ b/skills/open-computer-use/SKILL.md @@ -24,6 +24,10 @@ first match. JavaScript exposes `app.selectText(elementIndex, text, options?)`. macOS additionally exposes `query`: a targeted accessibility lookup that finds controls by text and/or role without rendering a tree or taking a screenshot. +macOS also exposes `list_windows`, which lists a running app's windows with their `window_id`. +Pass that id to `get_app_state` as `window_id` to read and act on a window other than the app's +key window; see [references/usage.md](references/usage.md#choosing-a-window-macos). + ## Core Workflow 1. On macOS, run `sw_vers -productVersion` before invoking the CLI and require macOS 14.0 or later. On older versions, explain that the runtime cannot launch; do not recommend `doctor` or permission changes as a fix for binary incompatibility. diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.md index ae1621a6..74d7b1e4 100644 --- a/skills/open-computer-use/references/usage.md +++ b/skills/open-computer-use/references/usage.md @@ -40,6 +40,7 @@ type_text press_key set_value select_text (macOS runtime only) +list_windows (macOS runtime only) ``` macOS also exposes `query` for targeted AX lookup. These nine core tools remain the cross-platform compatibility surface. The bundled Codex plugin uses a @@ -106,7 +107,7 @@ ocu repl session and top-level bindings for the current terminal until `.exit`, Ctrl-D, or termination; `.editor` / `.end` accepts multiline input and `.reset` discards JavaScript bindings. Use `nodeRepl.write(value)` for explicit output. -The native `ocu mcp` surface exposes the nine core tools above, plus `query` on macOS. Run +The native `ocu mcp` surface exposes the nine core tools above, plus `query`, `select_text` and `list_windows` on macOS. Run `ocu capabilities --json` to inspect component availability instead of inferring support from a command being present in help. @@ -183,6 +184,32 @@ open-computer-use snapshot --max-tree-nodes 3000 --max-tree-depth 96 "Google Chr Actions revalidate the process/window/control and current geometry. On failure, query again. See [targeted query](../../../docs/targeted-query.md) for options and examples. +## Choosing a Window (macOS) + +`get_app_state` reads the app's key window. When the app has several windows, list them and +pass the one you want as `window_id`: + +```sh +ocu call --calls '[ + {"tool":"list_windows","args":{"app":"Google Chrome"}}, + {"tool":"get_app_state","args":{"app":"Google Chrome","window_id":4127}} +]' +``` + +`list_windows` returns `{ "windows": [{ window_id, title, subrole, focused, minimized, bounds }] }` +for the windows accessibility reports on the current Space; `bounds` is in screen points. It only +reads running apps and never launches, activates or raises them. `query` takes the same `window_id`. + +With `window_id`, the tree, screenshot, focus and selection come from that window, which is not +activated or raised. Element and coordinate actions then target it, and their read-back stays on it, +until `get_app_state` is called without `window_id` or the window closes (the read-back then returns +the key window). An id that is not a current window of the app returns an error. Keyboard input +follows the app's keyboard focus: `type_text` refuses when the focused field is in another window, +and `key_method: "sky_key"` delivers keys to the bound window. A coordinate click on a part of the +window covered by another window of the same app needs `click_method: "sky_click"`. In JavaScript: +`await app.listWindows()` and `await app.getAXState({ windowId })`. Linux and Windows do not +implement `list_windows`, and reject `window_id`. + ## Choosing a Click Method `click_method` is optional. Omitting it uses `auto`, which preserves the platform's existing semantic-first behavior. Explicit methods never fall back to a different implementation: