diff --git a/.github/workflows/windows-observation.yml b/.github/workflows/windows-observation.yml new file mode 100644 index 00000000..ef48f36a --- /dev/null +++ b/.github/workflows/windows-observation.yml @@ -0,0 +1,47 @@ +name: Windows observation + +on: + pull_request: + paths: + - 'apps/OpenComputerUseWindows/**' + - 'scripts/tests/windows-observation*' + - 'scripts/tests/fixtures/WindowsObservationFixture.ps1' + - '.github/workflows/windows-observation.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: windows-observation-${{ github.ref }} + cancel-in-progress: true + +jobs: + windows-observation: + runs-on: windows-2025 + timeout-minutes: 15 + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: '24' + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c + with: + go-version: '1.26.1' + cache: false + - name: Windows Go tests and build + working-directory: apps/OpenComputerUseWindows + run: | + go test ./... + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + go build -o "$env:RUNNER_TEMP/ocu-observation.exe" . + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: PowerShell collection branches + run: | + powershell.exe -NoProfile -NonInteractive -File ./scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Real WPF and native MCP + run: ./scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:RUNNER_TEMP/ocu-observation.exe" diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift index 9c7deb72..94b8c5b2 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift @@ -523,9 +523,9 @@ private final class AppAgentConnection: @unchecked Sendable { let service = ComputerUseService() return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS) - case let .snapshot(app, textLimit, treeLimits): + case let .snapshot(app, textLimit, treeLimits, observation): let service = ComputerUseService() - let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? "" + let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContent return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS) case let .call(invocation): diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift index fef1c196..7c4ce10e 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift @@ -65,9 +65,9 @@ enum OpenComputerUseMain { case .listApps: let service = ComputerUseService() print(service.listApps().primaryText ?? "") - case let .snapshot(app, textLimit, treeLimits): + case let .snapshot(app, textLimit, treeLimits, observation): let service = ComputerUseService() - print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? "") + print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContent) case let .call(invocation): if VisualCursorSupport.isEnabled { _ = NSApplication.shared.setActivationPolicy(.accessory) diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go index 28b68d83..30e98d2d 100644 --- a/apps/OpenComputerUseLinux/main.go +++ b/apps/OpenComputerUseLinux/main.go @@ -88,6 +88,7 @@ type elementRecord struct { } type appSnapshot struct { + ScreenshotOmitted bool `json:"-"` App appDescriptor `json:"app"` WindowTitle string `json:"windowTitle,omitempty"` WindowBounds *frame `json:"windowBounds,omitempty"` @@ -116,6 +117,9 @@ func (s *appSnapshot) renderedText() string { fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name), } lines = append(lines, s.TreeLines...) + if s.ScreenshotOmitted { + lines = append(lines, "Screenshot omitted by include_screenshot=false") + } if strings.TrimSpace(s.SelectedText) != "" { lines = append(lines, "", fmt.Sprintf("Selected text: [%s]", s.SelectedText)) } else if strings.TrimSpace(s.FocusedSummary) != "" { @@ -139,6 +143,9 @@ func (s *appSnapshot) result() toolCallResult { } type linuxRequest struct { + IncludeAX *bool `json:"include_ax,omitempty"` + IncludeScreenshot *bool `json:"include_screenshot,omitempty"` + Tool string `json:"tool"` App string `json:"app,omitempty"` Element *elementRecord `json:"element,omitempty"` @@ -215,7 +222,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult { if windowPlacement != "keep" { return textResult("window_placement '"+windowPlacement+"' is not supported on Linux", true) } - return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth) + includeAX, err := optionalBool(args, "include_ax") + if err != nil { + return textResult(err.Error(), true) + } + includeScreenshot, err := optionalBool(args, "include_screenshot") + if err != nil { + return textResult(err.Error(), true) + } + if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot { + return textResult("At least one of include_ax or include_screenshot must be true", true) + } + return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot) case "click": clickMethod, err := parseClickMethod(optionalString(args, "click_method")) if err != nil { @@ -284,11 +302,11 @@ func (s *service) listApps() toolCallResult { return textResult(response.Text, false) } -func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult { +func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult { if app == "" { return textResult("Missing required argument: app", true) } - request := linuxRequest{Tool: "get_app_state", App: app} + request := linuxRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot} if textLimit != nil { request.TextLimit = textLimit.runtimeValue() } @@ -302,7 +320,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, ma if result.IsError { return result } - return snapshot.result() + result = snapshot.result() + if includeAX != nil && !*includeAX { + result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions." + } + return result } func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult { @@ -328,6 +350,9 @@ func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, if snapshot == nil { return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true) } + if elementIndex == "" && snapshot.ScreenshotOmitted { + return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true) + } request := linuxRequest{ Tool: "click", App: app, @@ -414,6 +439,9 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResu if snapshot == nil { return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true) } + if snapshot.ScreenshotOmitted { + return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true) + } return s.actionResult(app, linuxRequest{Tool: "drag", App: app, FromX: fromX, FromY: fromY, ToX: toX, ToY: toY, WindowBounds: snapshot.WindowBounds}) } @@ -512,6 +540,25 @@ func (s *service) refreshSnapshot(app string, request linuxRequest) (*appSnapsho if response.Snapshot == nil { return nil, textResult("Linux runtime did not return an app snapshot.", true) } + if request.IncludeScreenshot != nil && !*request.IncludeScreenshot { + response.Snapshot.ScreenshotPNGBase64 = "" + response.Snapshot.ScreenshotOmitted = true + } + if request.IncludeAX != nil && !*request.IncludeAX { + // A screenshot cannot validate AX indexes, even if the window title matches. + // Invalidate every alias of the prior observation, including PID queries. + previous := s.currentSnapshot(app) + for alias, cached := range s.snapshots { + if cached == previous || cached.App.PID == response.Snapshot.App.PID || + (previous != nil && cached.App.PID == previous.App.PID) { + delete(s.snapshots, alias) + } + } + response.Snapshot.Elements = nil + response.Snapshot.TreeLines = nil + response.Snapshot.FocusedSummary = "" + response.Snapshot.SelectedText = "" + } s.rememberSnapshot(app, response.Snapshot) return response.Snapshot, toolCallResult{} } @@ -1011,6 +1058,18 @@ func optionalString(args map[string]any, key string) string { return value } +func optionalBool(args map[string]any, key string) (*bool, error) { + value, exists := args[key] + if !exists { + return nil, nil + } + boolean, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("%s must be a boolean", key) + } + return &boolean, nil +} + func requiredElementIndex(args map[string]any) string { return strings.TrimSpace(optionalElementIndex(args)) } @@ -1240,7 +1299,7 @@ func toolDefinitions() []toolDefinition { }, { Name: "get_app_state", - Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.", + Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.", Annotations: readOnlyAnnotations(), InputSchema: objectSchema(map[string]any{ "app": stringProperty("App name or bundle identifier"), @@ -1248,6 +1307,9 @@ func toolDefinitions() []toolDefinition { "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."), "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."), "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Linux.", windowPlacementValues), + + "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."}, + "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."}, }, []string{"app"}), }, { @@ -1402,13 +1464,16 @@ func runCLI(args []string, stdout io.Writer) error { fmt.Fprintln(stdout, result.Content[0].Text) return nil case "snapshot": - app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:]) + snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:]) if err != nil { return err } - toolArgs := map[string]any{ - "app": app, + app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs) + if err != nil { + return err } + toolArgs := observation + toolArgs["app"] = app if textLimit != nil { toolArgs["text_limit"] = textLimit.runtimeValue() } @@ -1443,6 +1508,37 @@ func runCLI(args []string, stdout io.Writer) error { } } +func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) { + remaining := []string{} + options := map[string]any{} + for index := 0; index < len(args); index++ { + option := args[index] + if option != "--include-ax" && option != "--include-screenshot" { + remaining = append(remaining, option) + continue + } + index++ + if index >= len(args) { + return nil, nil, fmt.Errorf("%s requires true or false", option) + } + var value bool + switch strings.ToLower(args[index]) { + case "true": + value = true + case "false": + value = false + default: + return nil, nil, fmt.Errorf("%s requires true or false", option) + } + key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_") + options[key] = value + } + if options["include_ax"] == false && options["include_screenshot"] == false { + return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true") + } + return remaining, options, nil +} + func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) { var app string var textLimit *textLimit @@ -1746,7 +1842,7 @@ func helpText(command string) string { case "call": return "Usage:\n open-computer-use call [--args '']\n open-computer-use call --calls ''\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n" case "snapshot": - return "Usage:\n open-computer-use snapshot [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] \n\nPrint the current Linux AT-SPI snapshot for the target app.\n" + return "Usage:\n open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] \n\nPrint the current Linux AT-SPI snapshot for the target app.\n" default: return `Open Computer Use for Linux diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go index 13c4f7c4..17fd6ae8 100644 --- a/apps/OpenComputerUseLinux/main_test.go +++ b/apps/OpenComputerUseLinux/main_test.go @@ -3,9 +3,11 @@ package main import ( "bytes" "encoding/json" + "fmt" "net" "os" "path/filepath" + "strconv" "strings" "testing" ) @@ -118,6 +120,143 @@ func TestActionWithoutReadBackPropagatesBridgeErrors(t *testing.T) { } } +func observationPythonResponse(t *testing.T, response string) string { + t.Helper() + dir := t.TempDir() + requestPath := filepath.Join(dir, "request.json") + script := "#!/bin/sh\ncat \"$2\" > \"$OCU_TEST_OBSERVATION_REQUEST\"\nprintf '%s\\n' \"$OCU_TEST_OBSERVATION_RESPONSE\"\n" + if err := os.WriteFile(filepath.Join(dir, "python3"), []byte(script), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("OCU_TEST_OBSERVATION_REQUEST", requestPath) + t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", response) + return requestPath +} + +func TestSelectiveObservationRequestAndDefaultCompatibility(t *testing.T) { + for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} { + t.Run(fmt.Sprint(flags), func(t *testing.T) { + path := observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":42}}}`) + args := map[string]any{"app": "Editor"} + for key, value := range flags { + args[key] = value + } + if result := newService().callTool("get_app_state", args); result.IsError { + t.Fatalf("observation failed: %#v", result) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var request map[string]any + if err := json.Unmarshal(data, &request); err != nil { + t.Fatal(err) + } + for _, key := range []string{"include_ax", "include_screenshot"} { + want, present := flags[key] + got, sent := request[key] + if present != sent || got != want { + t.Fatalf("%s serialized as %#v (%v), want %#v (%v)", key, got, sent, want, present) + } + } + }) + } +} + +func TestSelectiveObservationRejectsInvalidOptions(t *testing.T) { + for _, args := range []map[string]any{ + {"app": "Editor", "include_ax": "false"}, + {"app": "Editor", "include_screenshot": nil}, + {"app": "Editor", "include_ax": false, "include_screenshot": false}, + } { + if result := newService().callTool("get_app_state", args); !result.IsError { + t.Fatalf("invalid options accepted: %#v", args) + } + } +} + +func TestSelectiveObservationSchemaKeepsFullDefaults(t *testing.T) { + tool := findToolDefinition(t, "get_app_state") + properties := tool.InputSchema["properties"].(map[string]any) + for _, name := range []string{"include_ax", "include_screenshot"} { + property := properties[name].(map[string]any) + if property["type"] != "boolean" || property["default"] != true { + t.Fatalf("%s schema = %#v", name, property) + } + } + if required := tool.InputSchema["required"].([]string); len(required) != 1 || required[0] != "app" { + t.Fatalf("observation options should be optional: %#v", required) + } +} + +func TestScreenshotOnlyInvalidatesIndexesAcrossWindowAndProcessChanges(t *testing.T) { + for _, pid := range []int{42, 43} { + t.Run(strconv.Itoa(pid), func(t *testing.T) { + path := observationPythonResponse(t, fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"windowTitle":"New Window","windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`, pid)) + s := newService() + old := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, WindowTitle: "Old Window", Elements: []elementRecord{{Index: 7}}} + s.rememberSnapshot("old-window-query", old) + s.snapshots["editor"] = old + other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}} + s.rememberSnapshot("Other", other) + result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false}) + if result.IsError || len(result.Content) != 2 || !strings.Contains(result.Content[0].Text, "indexes invalidated") { + t.Fatalf("screenshot-only result = %#v", result) + } + if s.currentSnapshot("old-window-query") != nil || s.currentSnapshot("Other") != other { + t.Fatal("old alias retained or unrelated app modified") + } + if pid != 42 && s.currentSnapshot("42") != nil { + t.Fatal("old PID alias retained across process change") + } + if _, err := lookupElement(s.currentSnapshot("Editor"), "7"); err == nil { + t.Fatal("stale index remains usable") + } + if result := s.setValue("Editor", "7", "bad"); !result.IsError { + t.Fatal("stale element action accepted") + } + x, y := 10.0, 20.0 + if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError { + t.Fatalf("coordinate action unavailable: %#v", result) + } + data, err := os.ReadFile(path) + if err != nil || !strings.Contains(string(data), `"tool":"click"`) { + t.Fatalf("coordinate click not sent: %s, %v", data, err) + } + t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"elements":[{"index":8}]}}`, pid)) + if result := s.callTool("get_app_state", map[string]any{"app": "Editor"}); result.IsError { + t.Fatalf("AX refresh failed: %#v", result) + } + if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil { + t.Fatal("explicit AX did not restore indexes") + } + }) + } +} + +func TestScreenshotOnlyInvalidatesOlderAliasesWhenProcessChanges(t *testing.T) { + observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":43},"windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`) + s := newService() + older := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}} + s.rememberSnapshot("older-query", older) + s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}}) + unrelated := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}} + s.rememberSnapshot("Other", unrelated) + result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false}) + if result.IsError { + t.Fatalf("screenshot-only observation failed: %#v", result) + } + for _, alias := range []string{"older-query", "42"} { + if s.currentSnapshot(alias) != nil { + t.Fatalf("stale AX alias %q survived process replacement", alias) + } + } + if s.currentSnapshot("Other") != unrelated { + t.Fatal("unrelated app observation invalidated") + } +} + func TestClickMethodSchemaAndParser(t *testing.T) { tool := findToolDefinition(t, "click") properties := tool.InputSchema["properties"].(map[string]any) diff --git a/apps/OpenComputerUseLinux/observation_cli_test.go b/apps/OpenComputerUseLinux/observation_cli_test.go new file mode 100644 index 00000000..546f6b75 --- /dev/null +++ b/apps/OpenComputerUseLinux/observation_cli_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "strings" + "testing" +) + +func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) { + args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"}) + if err != nil { + t.Fatal(err) + } + app, _, nodes, _, err := parseSnapshotArgs(args) + if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false { + t.Fatalf("%v %v %v", args, flags, err) + } + for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} { + if _, _, err := parseSnapshotObservationArgs(bad); err == nil { + t.Fatalf("invalid flags accepted: %v", bad) + } + } + _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"}) + if err != nil || len(defaults) != 0 { + t.Fatalf("legacy call changed: %v %v", defaults, err) + } + if !strings.Contains(helpText("snapshot"), "--include-screenshot") { + t.Fatal("snapshot help omits observation flags") + } +} diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py index 3fc4c41a..d5f72e0c 100644 --- a/apps/OpenComputerUseLinux/runtime.py +++ b/apps/OpenComputerUseLinux/runtime.py @@ -442,21 +442,24 @@ def visit(node, depth): return summary, selection, False -def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH): +def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH, include_ax=True, include_screenshot=True): app = resolve_app(query) window_index, window = main_window(app) bounds = extents(window) - records, lines = render_tree( - window, - bounds, - [window_index], - text_limit=text_limit, - max_tree_nodes=max_tree_nodes, - max_tree_depth=max_tree_depth, - ) - focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth) - if focus_truncated: - lines.append("[Focus search incomplete: node/depth budget reached.]") + records, lines = [], [] + focus, selection = None, None + if include_ax: + records, lines = render_tree( + window, + bounds, + [window_index], + text_limit=text_limit, + max_tree_nodes=max_tree_nodes, + max_tree_depth=max_tree_depth, + ) + focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth) + if focus_truncated: + lines.append("[Focus search incomplete: node/depth budget reached.]") pid = node_pid(app) return { "app": { @@ -466,7 +469,7 @@ def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEM }, "windowTitle": limit_text(node_name(window), text_limit=text_limit), "windowBounds": bounds, - "screenshotPngBase64": capture_window_png(bounds), + "screenshotPngBase64": capture_window_png(bounds) if include_screenshot else None, "treeLines": lines, "focusedSummary": focus, "selectedText": selection, @@ -803,6 +806,11 @@ def perform_operation(operation): if tool == "list_apps": return {"ok": True, "text": list_apps_text()} if tool == "get_app_state": + for key in ("include_ax", "include_screenshot"): + if key in operation and not isinstance(operation[key], bool): + raise RuntimeError(key + " must be a boolean") + if not operation.get("include_ax", True) and not operation.get("include_screenshot", True): + raise RuntimeError("At least one of include_ax or include_screenshot must be true") return { "ok": True, "snapshot": build_snapshot( @@ -810,6 +818,8 @@ def perform_operation(operation): text_limit=parse_text_limit(operation.get("text_limit"), DEFAULT_TEXT_LIMIT), max_tree_nodes=positive_int(operation.get("max_tree_nodes"), MAX_ELEMENTS), max_tree_depth=positive_int(operation.get("max_tree_depth"), MAX_DEPTH), + include_ax=operation.get("include_ax", True), + include_screenshot=operation.get("include_screenshot", True), ), } diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.py index 05280b06..25cc814d 100644 --- a/apps/OpenComputerUseLinux/runtime_test.py +++ b/apps/OpenComputerUseLinux/runtime_test.py @@ -172,6 +172,71 @@ def test_unknown_modifier_is_rejected(self): self.assertEqual(self.events, []) +class RuntimeSelectiveObservationTests(unittest.TestCase): + def setUp(self): + self.runtime = load_runtime() + self.app = object() + self.window = object() + self.patches = [] + for name, value in { + "resolve_app": self.app, + "main_window": (0, self.window), + "extents": {"x": 1, "y": 2, "width": 300, "height": 200}, + "node_pid": 42, + "node_name": "Editor", + "render_tree": ([{"index": 7}], ["tree"]), + "focus_details": ("focus", "selection", False), + "capture_window_png": "png", + }.items(): + patch = mock.patch.object(self.runtime, name, return_value=value) + self.patches.append(patch) + patch.start() + self.addCleanup(patch.stop) + + def test_defaults_still_collect_full_state(self): + state = self.runtime.build_snapshot("Editor") + self.assertEqual(state["elements"], [{"index": 7}]) + self.assertEqual(state["screenshotPngBase64"], "png") + self.runtime.focus_details.assert_called_once() + + def test_screenshot_only_skips_tree_focus_and_selection(self): + state = self.runtime.build_snapshot("Editor", include_ax=False) + self.assertEqual(state["screenshotPngBase64"], "png") + self.assertEqual(state["elements"], []) + self.assertEqual(state["treeLines"], []) + self.assertIsNone(state["focusedSummary"]) + self.assertIsNone(state["selectedText"]) + for name in ("render_tree", "focus_details"): + getattr(self.runtime, name).assert_not_called() + + def test_ax_only_skips_capture(self): + state = self.runtime.build_snapshot("Editor", include_screenshot=False) + self.assertEqual(state["treeLines"], ["tree"]) + self.assertIsNone(state["screenshotPngBase64"]) + self.runtime.capture_window_png.assert_not_called() + + def test_ax_only_preserves_bounded_focus_reporting(self): + self.runtime.focus_details.return_value = (None, None, True) + state = self.runtime.build_snapshot("Editor", text_limit=20, max_tree_nodes=2, + max_tree_depth=1, include_screenshot=False) + self.runtime.focus_details.assert_called_once_with(self.window, 20, 2, 1) + self.assertIn("[Focus search incomplete: node/depth budget reached.]", state["treeLines"]) + self.runtime.capture_window_png.assert_not_called() + + def test_tool_forwards_explicit_observation_flags(self): + state = self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", "include_ax": False}) + self.assertTrue(state["ok"]) + self.runtime.render_tree.assert_not_called() + + def test_invalid_or_empty_observations_are_rejected(self): + for flags in ({"include_ax": "false"}, {"include_screenshot": None}, + {"include_ax": 0}, {"include_ax": False, "include_screenshot": False}): + with self.subTest(flags=flags): + with self.assertRaises(RuntimeError): + self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", **flags}) + self.runtime.resolve_app.assert_not_called() + + class HiddenDetailsTests(unittest.TestCase): def setUp(self): self.runtime = load_runtime() diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go index 47bef737..8b99ffb5 100644 --- a/apps/OpenComputerUseWindows/main.go +++ b/apps/OpenComputerUseWindows/main.go @@ -93,6 +93,7 @@ type elementRecord struct { } type appSnapshot struct { + ScreenshotOmitted bool `json:"-"` App appDescriptor `json:"app"` WindowTitle string `json:"windowTitle,omitempty"` WindowBounds *frame `json:"windowBounds,omitempty"` @@ -121,7 +122,9 @@ func (s *appSnapshot) renderedText() string { fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name), } lines = append(lines, s.TreeLines...) - if !s.hasUsableScreenshot() { + if s.ScreenshotOmitted { + lines = append(lines, "", "Screenshot omitted by include_screenshot=false") + } else if !s.hasUsableScreenshot() { lines = append(lines, "", "Screenshot unavailable. Restore a visible, non-minimized, on-screen window before using screenshot-coordinate actions; element_index actions may still work.") } if strings.TrimSpace(s.SelectedText) != "" { @@ -198,28 +201,30 @@ func validateScreenshotPNGBase64(encoded string) string { } type psRequest struct { - Tool string `json:"tool"` - App string `json:"app,omitempty"` - Element *elementRecord `json:"element,omitempty"` - X *float64 `json:"x,omitempty"` - Y *float64 `json:"y,omitempty"` - FromX *float64 `json:"from_x,omitempty"` - FromY *float64 `json:"from_y,omitempty"` - ToX *float64 `json:"to_x,omitempty"` - ToY *float64 `json:"to_y,omitempty"` - ClickCount int `json:"click_count,omitempty"` - MouseButton string `json:"mouse_button,omitempty"` - ClickMethod string `json:"click_method,omitempty"` - Action string `json:"action,omitempty"` - Direction string `json:"direction,omitempty"` - Pages float64 `json:"pages,omitempty"` - Text string `json:"text,omitempty"` - Key string `json:"key,omitempty"` - Value string `json:"value,omitempty"` - WindowBounds *frame `json:"windowBounds,omitempty"` - TextLimit any `json:"text_limit,omitempty"` - MaxTreeNodes int `json:"max_tree_nodes,omitempty"` - MaxTreeDepth int `json:"max_tree_depth,omitempty"` + IncludeAX *bool `json:"include_ax,omitempty"` + IncludeScreenshot *bool `json:"include_screenshot,omitempty"` + Tool string `json:"tool"` + App string `json:"app,omitempty"` + Element *elementRecord `json:"element,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` + FromX *float64 `json:"from_x,omitempty"` + FromY *float64 `json:"from_y,omitempty"` + ToX *float64 `json:"to_x,omitempty"` + ToY *float64 `json:"to_y,omitempty"` + ClickCount int `json:"click_count,omitempty"` + MouseButton string `json:"mouse_button,omitempty"` + ClickMethod string `json:"click_method,omitempty"` + Action string `json:"action,omitempty"` + Direction string `json:"direction,omitempty"` + Pages float64 `json:"pages,omitempty"` + Text string `json:"text,omitempty"` + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` + WindowBounds *frame `json:"windowBounds,omitempty"` + TextLimit any `json:"text_limit,omitempty"` + MaxTreeNodes int `json:"max_tree_nodes,omitempty"` + MaxTreeDepth int `json:"max_tree_depth,omitempty"` } type textLimit struct { @@ -243,10 +248,11 @@ type psResponse struct { type service struct { snapshots map[string]*appSnapshot + runBridge func(psRequest) (*psResponse, error) } func newService() *service { - return &service{snapshots: map[string]*appSnapshot{}} + return &service{snapshots: map[string]*appSnapshot{}, runBridge: runPowerShell} } func (s *service) callTool(name string, args map[string]any) toolCallResult { @@ -273,7 +279,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult { if windowPlacement != "keep" { return textResult("window_placement '"+windowPlacement+"' is not supported on Windows", true) } - return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth) + includeAX, err := optionalBool(args, "include_ax") + if err != nil { + return textResult(err.Error(), true) + } + includeScreenshot, err := optionalBool(args, "include_screenshot") + if err != nil { + return textResult(err.Error(), true) + } + if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot { + return textResult("At least one of include_ax or include_screenshot must be true", true) + } + return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot) case "click": clickMethod, err := parseClickMethod(optionalString(args, "click_method")) if err != nil { @@ -329,7 +346,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult { } func (s *service) listApps() toolCallResult { - response, err := runPowerShell(psRequest{Tool: "list_apps"}) + response, err := s.runBridge(psRequest{Tool: "list_apps"}) if err != nil { return textResult(err.Error(), true) } @@ -342,11 +359,11 @@ func (s *service) listApps() toolCallResult { return textResult(response.Text, false) } -func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult { +func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult { if app == "" { return textResult("Missing required argument: app", true) } - request := psRequest{Tool: "get_app_state", App: app} + request := psRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot} if textLimit != nil { request.TextLimit = textLimit.runtimeValue() } @@ -360,7 +377,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, ma if result.IsError { return result } - return snapshot.result() + result = snapshot.result() + if includeAX != nil && !*includeAX { + result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions." + } + return result } func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult { @@ -479,7 +500,7 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResu } func screenshotCoordinatesUnavailableResult(app string) toolCallResult { - return textResult("No usable screenshot is available for "+app+". Restore a visible, non-minimized, on-screen window and run get_app_state again before using screenshot coordinates.", true) + return textResult("No usable screenshot is available for "+app+". Run get_app_state with include_screenshot=true on a visible, non-minimized, on-screen window before using screenshot coordinates.", true) } func (s *service) typeText(app, text, keyMethod string) toolCallResult { @@ -545,7 +566,7 @@ func (s *service) currentSnapshot(app string) *appSnapshot { } func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot, toolCallResult) { - response, err := runPowerShell(request) + response, err := s.runBridge(request) if err != nil { return nil, textResult(err.Error(), true) } @@ -555,6 +576,25 @@ func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot, if response.Snapshot == nil { return nil, textResult("Windows runtime did not return an app snapshot.", true) } + if request.IncludeScreenshot != nil && !*request.IncludeScreenshot { + response.Snapshot.ScreenshotPNGBase64 = "" + response.Snapshot.ScreenshotOmitted = true + } + if request.IncludeAX != nil && !*request.IncludeAX { + // A screenshot cannot validate AX indexes, even if the window title matches. + // Invalidate every alias of the prior observation, including PID queries. + previous := s.currentSnapshot(app) + for alias, cached := range s.snapshots { + if cached == previous || cached.App.PID == response.Snapshot.App.PID || + (previous != nil && cached.App.PID == previous.App.PID) { + delete(s.snapshots, alias) + } + } + response.Snapshot.Elements = nil + response.Snapshot.TreeLines = nil + response.Snapshot.FocusedSummary = "" + response.Snapshot.SelectedText = "" + } response.Snapshot.sanitizeScreenshot() s.rememberSnapshot(app, response.Snapshot) return response.Snapshot, toolCallResult{} @@ -641,6 +681,18 @@ func optionalString(args map[string]any, key string) string { return value } +func optionalBool(args map[string]any, key string) (*bool, error) { + value, exists := args[key] + if !exists { + return nil, nil + } + boolean, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("%s must be a boolean", key) + } + return &boolean, nil +} + func requiredElementIndex(args map[string]any) string { return strings.TrimSpace(optionalElementIndex(args)) } @@ -861,14 +913,16 @@ func toolDefinitions() []toolDefinition { }, { Name: "get_app_state", - Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.", + Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.", Annotations: readOnlyAnnotations(), InputSchema: objectSchema(map[string]any{ - "app": stringProperty("App name or bundle identifier"), - "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."), - "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."), - "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."), - "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues), + "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."}, + "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."}, + "app": stringProperty("App name or bundle identifier"), + "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."), + "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."), + "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."), + "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues), }, []string{"app"}), }, { @@ -1023,13 +1077,16 @@ func runCLI(args []string, stdout io.Writer) error { fmt.Fprintln(stdout, result.Content[0].Text) return nil case "snapshot": - app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:]) + snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:]) if err != nil { return err } - toolArgs := map[string]any{ - "app": app, + app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs) + if err != nil { + return err } + toolArgs := observation + toolArgs["app"] = app if textLimit != nil { toolArgs["text_limit"] = textLimit.runtimeValue() } @@ -1064,6 +1121,37 @@ func runCLI(args []string, stdout io.Writer) error { } } +func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) { + remaining := []string{} + options := map[string]any{} + for index := 0; index < len(args); index++ { + option := args[index] + if option != "--include-ax" && option != "--include-screenshot" { + remaining = append(remaining, option) + continue + } + index++ + if index >= len(args) { + return nil, nil, fmt.Errorf("%s requires true or false", option) + } + var value bool + switch strings.ToLower(args[index]) { + case "true": + value = true + case "false": + value = false + default: + return nil, nil, fmt.Errorf("%s requires true or false", option) + } + key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_") + options[key] = value + } + if options["include_ax"] == false && options["include_screenshot"] == false { + return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true") + } + return remaining, options, nil +} + func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) { var app string var textLimit *textLimit @@ -1367,7 +1455,7 @@ func helpText(command string) string { case "call": return "Usage:\n open-computer-use.exe call [--args '']\n open-computer-use.exe call --calls ''\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n" case "snapshot": - return "Usage:\n open-computer-use.exe snapshot [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] \n\nPrint the current Windows UI Automation snapshot for the target app.\n" + return "Usage:\n open-computer-use.exe snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] \n\nPrint the current Windows UI Automation snapshot for the target app.\n" default: return `Open Computer Use for Windows diff --git a/apps/OpenComputerUseWindows/observation_cli_test.go b/apps/OpenComputerUseWindows/observation_cli_test.go new file mode 100644 index 00000000..546f6b75 --- /dev/null +++ b/apps/OpenComputerUseWindows/observation_cli_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "strings" + "testing" +) + +func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) { + args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"}) + if err != nil { + t.Fatal(err) + } + app, _, nodes, _, err := parseSnapshotArgs(args) + if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false { + t.Fatalf("%v %v %v", args, flags, err) + } + for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} { + if _, _, err := parseSnapshotObservationArgs(bad); err == nil { + t.Fatalf("invalid flags accepted: %v", bad) + } + } + _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"}) + if err != nil || len(defaults) != 0 { + t.Fatalf("legacy call changed: %v %v", defaults, err) + } + if !strings.Contains(helpText("snapshot"), "--include-screenshot") { + t.Fatal("snapshot help omits observation flags") + } +} diff --git a/apps/OpenComputerUseWindows/observation_test.go b/apps/OpenComputerUseWindows/observation_test.go new file mode 100644 index 00000000..e3cb9462 --- /dev/null +++ b/apps/OpenComputerUseWindows/observation_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestObservationRequestSerializationAndChannels(t *testing.T) { + for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} { + s := newService() + s.runBridge = func(request psRequest) (*psResponse, error) { + data, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + var sent map[string]any + if err := json.Unmarshal(data, &sent); err != nil { + t.Fatal(err) + } + for _, key := range []string{"include_ax", "include_screenshot"} { + want, present := flags[key] + got, exists := sent[key] + if present != exists || want != got { + t.Fatalf("%s serialized as %#v, want %#v", key, sent, flags) + } + } + return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, + WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200), + TreeLines: []string{"entry"}, FocusedSummary: "focus", SelectedText: "中文🙂", Elements: []elementRecord{{Index: 7}}}}, nil + } + args := map[string]any{"app": "Editor"} + for key, value := range flags { + args[key] = value + } + result := s.callTool("get_app_state", args) + if result.IsError { + t.Fatal(result) + } + if flags["include_screenshot"] == false { + if len(result.Content) != 1 || !strings.Contains(result.Content[0].Text, "include_screenshot=false") { + t.Fatal(result) + } + } else if len(result.Content) != 2 { + t.Fatal(result) + } + if flags["include_ax"] == false { + cached := s.currentSnapshot("Editor") + if len(cached.Elements) != 0 || len(cached.TreeLines) != 0 || cached.FocusedSummary != "" || cached.SelectedText != "" { + t.Fatal(cached) + } + if !strings.Contains(result.Content[0].Text, "indexes invalidated") { + t.Fatal(result) + } + } + } +} + +func TestObservationRejectsInvalidOptionsBeforeBridge(t *testing.T) { + s := newService() + s.runBridge = func(psRequest) (*psResponse, error) { t.Fatal("invalid observation reached bridge"); return nil, nil } + for _, args := range []map[string]any{{"include_ax": "false"}, {"include_screenshot": nil}, {"include_ax": 0}, {"include_ax": false, "include_screenshot": false}} { + args["app"] = "Editor" + if result := s.callTool("get_app_state", args); !result.IsError { + t.Fatal(args) + } + } +} + +func TestObservationInvalidatesOlderAliasesAcrossProcessReplacement(t *testing.T) { + s := newService() + s.rememberSnapshot("older-query", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}}) + s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}}) + other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}} + s.rememberSnapshot("Other", other) + s.runBridge = func(request psRequest) (*psResponse, error) { + return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 43}, + WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200), Elements: []elementRecord{{Index: 8}}}}, nil + } + if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false}); result.IsError { + t.Fatal(result) + } + if s.currentSnapshot("older-query") != nil || s.currentSnapshot("42") != nil || s.currentSnapshot("Other") != other { + t.Fatal("stale aliases retained or unrelated app invalidated") + } + if result := s.setValue("Editor", "7", "bad"); !result.IsError { + t.Fatal("stale element action accepted") + } + x, y := 10.0, 20.0 + if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError { + t.Fatal("screenshot-only coordinate action failed", result) + } + if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_screenshot": false}); result.IsError { + t.Fatal(result) + } + if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil { + t.Fatal("AX refresh did not restore indexes", err) + } + if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); !result.IsError { + t.Fatal("AX-only observation accepted screenshot coordinates") + } + if result := s.setValue("Editor", "8", "works"); result.IsError { + t.Fatal("AX-only element action failed", result) + } +} + +func TestObservationSchemaDefaults(t *testing.T) { + props := findToolDefinition(t, "get_app_state").InputSchema["properties"].(map[string]any) + for _, key := range []string{"include_ax", "include_screenshot"} { + value := props[key].(map[string]any) + if value["type"] != "boolean" || value["default"] != true { + t.Fatal(value) + } + } +} diff --git a/apps/OpenComputerUseWindows/runtime.ps1 b/apps/OpenComputerUseWindows/runtime.ps1 index 3cf4df04..6cbe7ad9 100644 --- a/apps/OpenComputerUseWindows/runtime.ps1 +++ b/apps/OpenComputerUseWindows/runtime.ps1 @@ -692,11 +692,38 @@ function Get-SelectedText($processId, $TextLimit = $script:DefaultTextLimit) { return $null } -function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth) { +function Resolve-ObservationOptions($operation) { + $values = @{ include_ax = $true; include_screenshot = $true } + foreach ($key in @("include_ax", "include_screenshot")) { + $property = $operation.PSObject.Properties[$key] + if ($null -ne $property) { + if ($property.Value -isnot [bool]) { throw "$key must be a boolean" } + $values[$key] = $property.Value + } + } + if (-not $values.include_ax -and -not $values.include_screenshot) { + throw "At least one of include_ax or include_screenshot must be true" + } + return [pscustomobject]$values +} + +function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth, [bool]$IncludeAX = $true, [bool]$IncludeScreenshot = $true) { + if (-not $IncludeAX -and -not $IncludeScreenshot) { + throw "At least one of include_ax or include_screenshot must be true" + } $process = Resolve-App $query $element = Get-MainElement $process $bounds = Get-WindowBounds $process $element - $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth + $rendered = @{ lines = @(); records = @() } + $focus = $null + $selection = $null + $image = $null + if ($IncludeAX) { + $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth + $focus = Get-FocusedSummary $process.Id $TextLimit + $selection = Get-SelectedText $process.Id $TextLimit + } + if ($IncludeScreenshot) { $image = Capture-WindowPngBase64 $bounds } [pscustomobject]@{ app = [pscustomobject]@{ name = $process.ProcessName @@ -705,10 +732,10 @@ function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [ } windowTitle = Limit-Text $process.MainWindowTitle $TextLimit windowBounds = $bounds - screenshotPngBase64 = Capture-WindowPngBase64 $bounds + screenshotPngBase64 = $image treeLines = @($rendered.lines) - focusedSummary = Get-FocusedSummary $process.Id $TextLimit - selectedText = Get-SelectedText $process.Id $TextLimit + focusedSummary = $focus + selectedText = $selection elements = @($rendered.records) } } @@ -982,7 +1009,8 @@ try { if ($operation.tool -eq "list_apps") { $response = [pscustomobject]@{ ok = $true; text = (List-Apps) } } elseif ($operation.tool -eq "get_app_state") { - $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth)) } + $observation = Resolve-ObservationOptions $operation + $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth) $observation.include_ax $observation.include_screenshot) } } else { $process = Resolve-App $operation.app $hwnd = [IntPtr]$process.MainWindowHandle diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8fc5d799..012785f4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -74,6 +74,7 @@ - `open-computer-use call --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` / `--calls-file ` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行默认会在成功的相邻操作之间 sleep 1 秒,也可以用 `--sleep ` 覆盖;遇到 `isError=true` 的 tool result 后停止。 - 对真实 app 的 `get_app_state` / action tool 入口,当前只保留一层密码管理器 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app。终端、Chrome / Atlas 和系统组件不再属于内置阻止目标。 - 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。 +- 三个平台的显式 `get_app_state` 均支持 `include_ax` / `include_screenshot`,默认 true,至少保留一项。JS 对应 `includeAX` / `includeScreenshot`,CLI snapshot 提供同名连字符参数。关闭 AX 跳过树、焦点和选区,并使旧快照 / 查询索引及别名失效;关闭截图跳过捕获与编码,元素操作仍可用,截图坐标操作须重新获取图片。参数经 macOS app agent 转发;不改变动作回读,也不做自动图片去重。见 [按需观察](selective-observation.md)。 - macOS 额外提供 `query` 和 `select_text`,当前 native surface 为 11 个工具;Linux / Windows 仍为 9 个核心工具。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引,支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义;写入前及返回前检查会话,验证文本未变且实际选区等于请求,即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出;结果刷新使用 read-only recovery,不通过激活、键盘或真实指针补救。 - `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动;只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。 - overlay 用目标点相对窗口的锚点跟随窗口移动;行进、settle、点击反馈和 idle 都读取最新缓存帧,发现位置变化时直接重定位可见 tip 并清掉旧速度。窗口帧在后台串行读取,最多 30 Hz、同时最多一个请求,避免 AX 阻塞渲染线程;优先读取快照持有的 AX 窗口,仅在没有 AX 元素时使用窗口列表,读取失败保留最后有效帧。只保留当前窗口的追踪状态,reset / idle 超时后释放;这层跟随只调整显示位置,输入投递仍走既有路径。 @@ -127,7 +128,7 @@ - 这 9 个 tool 的协议面与 macOS 主线保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime id / automation metadata,coordinate action 使用 screenshot/window-relative 坐标。 - Windows `click_method=accessibility` 映射到 UI Automation pattern,`app_post` 映射到 HWND `PostMessage`;macOS-only 的 `sky_click` 和没有实现的 `global` 都会在 snapshot lookup 前明确返回 unsupported。`type_text` / `press_key` 的 `key_method=sky_key` 同样在 snapshot lookup 前返回 unsupported。`auto` 仍保持 UIA 优先、window message fallback 的现有行为。 - Windows UI Automation 需要运行在已登录用户的桌面 session 里。通过 SSH 作为脱离桌面的后台进程运行时,PowerShell 可以启动并返回 JSON,但系统可能不给它暴露顶层窗口;这种情况下 `list_apps` 会是空,`get_app_state` 可能返回 `appNotFound(...)`。 -- 当前 Windows 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/onboarding、没有 code signing,也没有独立的 Windows smoke fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。 +- Windows 已有独立 WPF 观察 fixture 和 Windows PR smoke,覆盖观察通道及截图边界。其余工具 / toolkit 覆盖,以及 visual cursor overlay、installer/onboarding、code signing 仍待补齐。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。 ### 7. Linux Runtime diff --git a/docs/CICD.md b/docs/CICD.md index 589239cc..36383177 100644 --- a/docs/CICD.md +++ b/docs/CICD.md @@ -19,6 +19,10 @@ 所有 GitHub Actions 都已经 pin 到 commit SHA。后续升级 action 时,也要继续保持这个约束。 +## Windows 观察回归 + +`.github/workflows/windows-observation.yml` 在相关 PR 上运行 Windows Go 测试、PowerShell 生产采集函数测试,以及独立 WPF / native MCP smoke,也支持手动触发。工作流仅需 `contents: read`,沿用固定 SHA 的 checkout / setup-node / setup-go,不发布制品。桌面 smoke 使用自己的窗口,验证采集开关、索引与 UIA 设值,以及正常 / 1x1 / 最小化 / cloaked 窗口的截图边界。复现命令见 [按需观察](selective-observation.md)。 + ## 推荐接入顺序 1. 保留 `ci.yml`,作为仓库的基础门禁。 diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md index 407622c4..70026603 100644 --- a/docs/QUALITY_SCORE.md +++ b/docs/QUALITY_SCORE.md @@ -12,7 +12,7 @@ | 区域 | 评分 | 原因 | 下一步 | | --- | --- | --- | --- | | 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及共享的 9 个核心 tools;macOS 另有 query、select_text,JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 | -| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 把当前实机 screenshot 边界用例收敛成仓库内 Windows fixture / 交互式 smoke,再补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 | +| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 维护 WPF 观察 / 截图边界 fixture 和 Windows PR smoke,扩展更多工具与 toolkit;补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 | | Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke,补完整工具覆盖、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 | | macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口,离线和遮罩 rehearsal 有回归;安装恢复和无许可双保护释放已实测,独立无遮罩 Return 解锁 / 重锁已实测,受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过,完整会话认证状态 / 故障链路尚未通过实测;正常启用改为管理员 enable 与签名校验,普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认,继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 | | 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 | diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md index 3bf18441..06cc2adc 100644 --- a/docs/RELIABILITY.md +++ b/docs/RELIABILITY.md @@ -19,7 +19,9 @@ - macOS 其他 Desktop 实机回归(需要第二个 Desktop,测试会短暂切换 Space 启动目标):`OPEN_COMPUTER_USE_RUN_CROSS_SPACE_LIVE_TEST=1 swift test --filter CrossSpaceLiveTests` - Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64` - Linux 隐藏控件实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --hidden-checks`,在私有 D-Bus / Xvfb 中验证真实 GTK 隐藏状态、属性读取、索引及页面切换;依赖与范围见 [测试说明](linux-hidden-details.md)。 +- Linux 按需观察实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --observation-checks`,验证真实 GTK 采集通道、native MCP / Worker JS 输出、旧索引拒绝与恢复;另外需要 Node.js,见 [测试说明](selective-observation.md)。 - Windows runtime:`(cd apps/OpenComputerUseWindows && go test ./...)`、`./scripts/build-open-computer-use-windows.sh --arch amd64`;截图相关变更还要在 Windows 交互桌面验证正常窗口有 image、1x1/minimized/cloaked 窗口无 image。 +- Windows 观察实测:`pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath /path/to/open-computer-use.exe`,使用独立 WPF 窗口走真实 UIA / native MCP,验证采集开关、索引与截图边界;专用 PR 工作流见 [CI 说明](CICD.md)。 - 本地诊断: - `open-computer-use doctor` - `open-computer-use snapshot ` diff --git a/docs/exec-plans/completed/20261017-pr101-selective-observation.md b/docs/exec-plans/completed/20261017-pr101-selective-observation.md new file mode 100644 index 00000000..cc440bba --- /dev/null +++ b/docs/exec-plans/completed/20261017-pr101-selective-observation.md @@ -0,0 +1,42 @@ +# PR #101: cross-platform selective observation + +## Goal + +Integrate current main, review and fix selective AX/screenshot collection, extend it to macOS and Windows at the user's request, verify through CLI/native MCP/current JS, and push follow-up commits to the original PR. Do not merge. + +## Scope and decisions + +- Both channels default to true; at least one is required. Existing calls and action read-back behavior stay compatible. +- AX-only skips capture/encoding; screenshot-only skips tree/focus/selection and invalidates old snapshot aliases, including PID replacement and macOS query indexes. Fresh AX restores element actions. +- AX-only cannot validate screenshot coordinates: require a fresh image observation before coordinate click/drag. +- Use disposable AppKit, GTK and WPF fixtures. macOS follows normal signing/session/permission admission; Linux uses private Xvfb/D-Bus on the existing container engine; Windows uses the hosted Actions desktop. No personal apps or local desktop VM. +- Automatic unchanged-image suppression remains separate. It needs exact image comparison, first/reset/forced output and session/app/window scope; unchanged AX does not establish unchanged pixels. + +## Progress + +- [x] Merge main and retain existing bounded-focus, hidden-node and read-back behavior. +- [x] Add macOS collection gates, schema/dispatcher/app-agent/CLI support and scoped cache/query invalidation. +- [x] Add Windows Go/PowerShell collection gates, strict options, cache behavior and protocol tests. +- [x] Expose matching snapshot flags on all platforms and reject AX-only coordinate input. +- [x] Verify normal signed macOS release and real GTK through CLI/native MCP/Worker JS; run relevant suites/builds. +- [x] Complete hosted real Windows WPF/UIA/MCP/CLI/Worker JS and screenshot-boundary regression. +- [x] Synchronize final docs/history/PR, remotes and original PR branch; leave PR open. + +## Validation and findings + +- Main through `7cb8bb7` integrated, preserving the original contribution. +- Older aliases after PID replacement initially retained stale indexes; tests now cover invalidation for both previous/new PIDs and preserve unrelated applications. +- macOS CLI originally printed only its first text block; it now retains omission notes too. +- Swift: 382 tests, 8 skipped, zero failures; selective-observation 8/8. Existing native tool and cursor smoke passed. +- Normally signed macOS release + real AppKit AX/native MCP/CLI/Worker JS passed valid PNG, trace-confirmed collection gates, snapshot/query invalidation, coordinate refusal, real value update and recovery. Fixture cannot become key/main. +- Linux: Go 31, Python 35 and real GTK/AT-SPI/Xvfb passed collection helper gates, PNG, Unicode selection, old name/PID index refusal, coordinate actions, AX recovery and CLI/Worker JS. Existing focus and hidden-detail checks passed. +- Node: 44 passed. Windows Go: 29 passed. Production PowerShell function tests passed under PowerShell 7 and actual Windows PowerShell 5.1 with instrumented dependencies. +- Linux/Windows ARM64/AMD64 builds and targeted vet passed; syntax/docs/whitespace/action-pinning checks passed. +- WinForms fixture did not expose a settable ValuePattern; the positive UIA-write fixture was changed to WPF instead of broadening production input fallbacks. The real WPF/UIA/CLI/native MCP/Worker JS desktop smoke passed on Windows 2025. +- Temporary Linux containers and build artifacts removed. No Wayland/other-toolkit performance claim or version bump. + +## Outcome + +All three platforms now provide the same explicit observation choices through CLI/native MCP/JS. Real desktop tests passed for AppKit, GTK and WPF; the Windows job additionally verified exact 1x1 Win32, minimized and DWM-cloaked screenshot suppression and image recovery. WPF DPI layout is avoided only for the exact one-pixel fixture; production screenshot thresholds and input fallbacks were unchanged. + +Windows desktop evidence: [Windows observation run](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) at `ae8ec2f`. Final follow-up commits are pushed to #101; the PR remains open for the maintainer to merge. Automatic image deduplication remains in the debt tracker. diff --git a/docs/exec-plans/tech-debt-tracker.md b/docs/exec-plans/tech-debt-tracker.md index a1055be2..c3e424ca 100644 --- a/docs/exec-plans/tech-debt-tracker.md +++ b/docs/exec-plans/tech-debt-tracker.md @@ -4,4 +4,5 @@ | 日期 | 区域 | 债务描述 | 为什么会存在 | 计划中的后续动作 | | --- | --- | --- | --- | --- | +| 2026-10-17 | 重复截图 | 请求截图时仍会返回未变化的画面。 | #101 已统一三个平台的显式采集开关;相同图像省略是独立能力。 | 评估按会话 / 应用 / 窗口隔离的精确图像去重;首帧、重置与强制返回必须明确,不能用 AX 未变推断画面未变,也不把去重当成跳过截图采集。 | | 2026-04-17 | 普通 app AX snapshot | Finder 路径已经能拿到前台窗口子树并输出 window-relative frame,但当前还缺更多真实 app 回归样本,无法证明这套 rooting / traversal 对复杂 app 都稳定。 | 这一轮先把 Finder 这类真实 app 的坐标换算和窗口子树收敛好,再把 deterministic 回归继续留给 fixture。 | 增加 Safari / System Settings / Activity Monitor 等真实 app 样本验证,并继续收敛 `kAXMainWindowAttribute`、focused element parent chain 和多窗口回退策略。 | diff --git a/docs/histories/2026-10/20261015-2023-linux-selective-observation.md b/docs/histories/2026-10/20261015-2023-linux-selective-observation.md new file mode 100644 index 00000000..967ef8e9 --- /dev/null +++ b/docs/histories/2026-10/20261015-2023-linux-selective-observation.md @@ -0,0 +1,66 @@ +## [2026-10-15 20:23] | Task: Add opt-in selective Linux observation + +### Execution Context +- Agent: Codex +- Runtime: Linux development environment + +### User Request +Offer cheaper explicit observation as a compatibility-preserving option, following the existing lightweight app binding approach. + +### Changes +- Add optional Linux `get_app_state` booleans `include_ax` and `include_screenshot`, both defaulting to true. +- Skip AX tree, focus and selection collection for screenshot-only reads; skip capture/PNG encoding for AX-only reads. +- Forward optional REPL `includeAX` and `includeScreenshot` without changing legacy calls. +- Explicitly invalidate prior AX indexes and their aliases after screenshot-only reads, retain current window geometry for coordinate actions, and report the invalidation. +- Add Python collection-path tests, Go protocol/cache/schema tests, Node compatibility tests, usage documentation and release notes. + +### Design Intent +Changing `getScreenshot()` defaults would also change its implicit index refresh. Keep existing behavior and make selective observation opt-in. A screenshot cannot validate old indexes, including after a window/process switch, so clear them rather than guessing from matching titles. A fresh AX observation restores indexes. + +### Validation +- Linux Go suite and Python bridge unit tests. +- Node REPL regression tests, including unchanged legacy request arguments. +- REPL tests: 23 passed. Full Node suite: 36/37 passed; the staged npm packaging test requires macOS `lipo` and fails identically on the unmodified base checkout on Linux. +- Documentation and whitespace checks. +- No live desktop timing, macOS runtime or Windows runtime validation is claimed. This change is independent of the action read-back fix and does not change input delivery or bridge process lifetime. + +### Files Modified +- `apps/OpenComputerUseLinux/main.go` +- `apps/OpenComputerUseLinux/main_test.go` +- `apps/OpenComputerUseLinux/runtime.py` +- `apps/OpenComputerUseLinux/runtime_test.py` +- `scripts/node-repl/open-computer-use-repl.mjs` +- `scripts/node-repl/open-computer-use-repl.test.mjs` +- `skills/open-computer-use/references/usage.md` +- `docs/ARCHITECTURE.md` +- `docs/releases/feature-release-notes.md` + +## 2026-10-17:同步主线、修复别名遗漏并补实测 + +用户要求处理 #101,基于最新 main 解决冲突、检查测试并推送,补真实 GTK;同时评估截图可选与未变化图片重复返回的关系。 + +- 合入主线 `7cb8bb7`(含 #103),保留原贡献提交及两边的 Go / Python 测试、架构和发布记录。将条件 AX 采集接入现有一次有界 focus_details,保留焦点预算提示及隐藏节点选项。 +- 发现只取截图时,应用 PID 改变后较早观察的查询别名可能残留。新增独立测试先复现失败,再清理新旧 PID 对应的全部观察别名,同时保留其他应用缓存。 +- JS 在两种观察通道同时关闭时提前拒绝,不产生 native 请求;Persistent / Worker 两种会话均测试参数、错误及旧调用兼容性。 +- 扩展 `--observation-checks`:在真实 GTK 上包装生产采集函数,确认 AX-only 没有截图、image-only 没有树或焦点读取,完整模式同时采集;检查真实 PNG、中文 / emoji 选区、名称 / PID 别名下旧索引拒绝、坐标点击及 AX 刷新后的设值。独立 GTK 状态确认结果。 +- 新增 Worker JS → native MCP → GTK smoke,验证选择通道、图片输出、旧索引异常、AX 恢复及默认完整观察。环境为现有引擎上的临时 Debian 12 ARM64 容器、私有 Xvfb / D-Bus、GTK 3 / AT-SPI 2 和 Node 24,不启动桌面 VM。 + +验证:Python 35 项、Linux Go 30 项、Node 44 项通过;Linux ARM64 / AMD64 构建、targeted vet、shell / Python / Node 语法检查通过;按需观察真实链路、原有默认动作回读、焦点 / Unicode 选区和隐藏详情 smoke 均通过。文档与 whitespace 检查通过。未运行 Wayland 或其他 toolkit,不宣称全任务提速,无版本 bump。 + +需求评估:#101 提供 Linux 显式通道选择,不做截图去重,也未实现其他平台的通道开关。现有 macOS / Linux JS 动作已禁用自动回读,getAXState 不输出图片,但仍可能浪费 native 采集。未变化图片省略通常仍需截图后比较,主要节省传输和模型输入;后续的跨平台支持、会话 / 应用 / 窗口隔离、首帧 / 重置 / 强制返回和精确图像比较已记录到技术债追踪,不以 AX 未变作为画面未变的证据。 + +验证结束后已删除临时容器和独立构建制品,未影响其他容器服务。 + +## 2026-10-17:按用户要求统一三个平台 + +用户明确要求扩展 macOS / Windows,而不只实现 Linux。本阶段取代前文的 Linux-only 范围;自动图片去重仍单独跟踪。 + +- macOS 新增观察选项、严格布尔校验、schema / dispatcher / app-agent CLI 转发,截图关闭时跳过捕获和编码,AX 关闭时跳过树、选区、菜单及懒加载重试。fixture 同步行为。纯截图清理旧进程快照别名和 query 索引,旧别名不能触发隐式刷新继续使用旧编号。 +- Windows Go / PowerShell 支持同样的参数、默认和空观察拒绝;按通道跳过生产采集函数,防御性清理结果中的禁用通道及旧缓存。新增纯传输测试接口,不改变生产 Windows / PowerShell 宿主限制。 +- 三个平台的 snapshot CLI 新增 --include-ax / --include-screenshot true|false。只取 AX 后拒绝截图坐标输入,保留元素操作。macOS CLI 改为打印所有文本说明但不输出图片字节,修复实测发现的省略原因丢失。 +- macOS 正常签名 release + 自有 AppKit 窗口实测通过:CLI / MCP / Worker JS、PNG、timing 分支证据、普通 / query 索引失效、坐标拒绝与 AX 恢复;窗口不能成为 key/main,未修改系统准入或权限。 +- 本地检查:Swift 382 项(8 跳过、0 失败),观察专项 8 项;Linux Go 31 项、Python 35 项、Node 44 项;Windows Go 29 项与生产 PowerShell 函数分支测试通过;Linux / Windows 双架构构建及 targeted vet 通过。真实 GTK 观察、焦点、隐藏详情及 macOS 原生工具 / cursor smoke 通过。 +- 增加专门的 Windows Actions 检查与临时 WPF 夹具,真实 Windows 2025 桌面上的 Go / Windows PowerShell 5.1 / native MCP / CLI / Worker JS 全部通过。覆盖三个通道模式、旧索引拒绝、AX 恢复、真实 Unicode 设值和坐标拒绝;正常窗口返回 PNG,物理 1×1 Win32 小窗口、最小化及 DWM-cloaked 窗口不返回图片,恢复后重新返回 PNG。工作流使用仓库既有固定 SHA actions 和 contents:read,无发布步骤。 +- 初版 WinForms 控件不提供可写 ValuePattern,正向设值测试改用标准 WPF TextBox,没有扩大生产输入 fallback。夹具状态文件用无 BOM UTF-8 原子写入,避免 Windows PowerShell 空字符串与 null 的文件替换差异。精确 1×1 用同进程 Win32 窗口,避开 WPF 高 DPI 最小尺寸取整。 +- Windows [完整实测记录](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) 通过于 `ae8ec2f`。覆盖 AppKit / GTK / WPF,不宣称所有 toolkit、Wayland 或完整任务性能均已验证。 +- 文档同步为三平台使用方式、可复现测试命令、Windows CI 和当前覆盖范围;原 execution plan 完成归档,PR 保持开放,未执行合并。 diff --git a/docs/linux-selective-observation.md b/docs/linux-selective-observation.md new file mode 100644 index 00000000..931b6be6 --- /dev/null +++ b/docs/linux-selective-observation.md @@ -0,0 +1,3 @@ +# Linux 按需观察 + +现已扩展到 macOS、Windows 和 Linux,完整参数、CLI / MCP 用法及测试见[按需观察](selective-observation.md)。 diff --git a/docs/references/js-repl.md b/docs/references/js-repl.md index a4259c9e..46c3aedb 100644 --- a/docs/references/js-repl.md +++ b/docs/references/js-repl.md @@ -54,6 +54,15 @@ typeText(text, options?) performSecondaryAction(elementIndex, action) ``` +On macOS, Windows and Linux, observation methods also accept optional `includeAX` and +`includeScreenshot` booleans, both defaulting to true. Use +`app.getAXState({includeScreenshot:false})` to skip native capture, or +`app.getScreenshot({includeAX:false})` to skip the AX/focus walk. The latter +invalidates previous element indexes; read AX again before element actions. +Disabling both channels is rejected before a native call. These options do not +deduplicate unchanged images. See [selective observation](../selective-observation.md) +for collection semantics and real GTK/Worker validation. + On macOS, `selectText` selects exact text inside a text element. `prefix` and `suffix` match the immediately adjacent text to disambiguate repeated occurrences. `selection` is `text` (default), `cursor_before` or `cursor_after`. Snapshot and `query` indexes diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md index c63ff06d..cfa1d004 100644 --- a/docs/releases/feature-release-notes.md +++ b/docs/releases/feature-release-notes.md @@ -13,6 +13,7 @@ | 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 | | 2026-10-15 | Linux 焦点与选区 | 大型界面减少重复查找,预算不足时明确提示,选中文字能正确返回。 | 复用快照已选窗口,一次查找生成焦点摘要与选区;按节点和深度预算停止读取,使用标准 AT-SPI 选区接口,并补隔离 GTK 验证。 | | 2026-10-15 | Linux 隐藏控件详情 | 隐藏节点较多时,可减少不需要的属性读取。 | 新增默认关闭的 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN`;保留节点、索引和可见子节点,状态未知时完整读取,补充真实 GTK 标签页及 native MCP 回归。详见 [使用与验证](../linux-hidden-details.md)。 | +| 2026-10-15 | 按需观察 | 只需控件信息时可省去截图,只需画面时可省去 AX 遍历。 | 统一 macOS / Windows / Linux 的 include_ax / include_screenshot、JS 和 CLI snapshot 参数,默认完整采集;纯截图清理旧索引,只取 AX 拒绝截图坐标输入,补真实 AppKit / GTK / WPF、Windows 截图边界与 PowerShell 分支验证。详见 [使用与验证](../selective-observation.md)。 | | 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 | | 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 | | 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 | diff --git a/docs/selective-observation.md b/docs/selective-observation.md new file mode 100644 index 00000000..4b834a66 --- /dev/null +++ b/docs/selective-observation.md @@ -0,0 +1,82 @@ +# 按需观察 + +macOS、Windows、Linux 的 `get_app_state` 都可以选择读取 AX 或截图。原生参数 `include_ax` / `include_screenshot` 对应 JS 的 `includeAX` / `includeScreenshot`,均默认 `true`,不能同时设为 `false`。 + +| 需要什么 | JS 调用 | 实际采集 | +| --- | --- | --- | +| 只读控件和选区 | `await app.getAXState({includeScreenshot: false})` | AX、焦点、选区;不截图、不编码 | +| 只看画面 | `await app.getScreenshot({includeAX: false})` | 截图;不遍历 AX、不查焦点和选区 | +| 两者都要 | `await app.getAXStateAndScreenshot()` | 完整 AX 和截图 | + +```js +var app = await cua.getApp("gedit", {initialState: false}); +await app.getAXState({includeScreenshot: false}); +``` + +只取截图后,旧元素索引及查询别名会失效,包括应用重启前的索引。坐标操作仍可使用本次观察的窗口位置;元素操作要先重新读取 AX。窗口标题相同不代表旧索引仍然有效。 + +不传参数的调用保持原行为。只取 AX 后可继续操作元素,但截图坐标点击 / 拖拽要先重新获取截图;只取截图后则要重新读取 AX 才能使用元素索引。macOS 的 query 索引也会随纯截图观察失效。 + +## CLI 和 MCP + +JS 的 `ocu js` / `ocu repl` 和 MCP `js` 使用上面的调用。直接调用原生 CLI / MCP 时使用 snake_case: + +```sh +ocu call get_app_state --args '{"app":"TextEdit","include_screenshot":false}' +ocu snapshot --include-screenshot false TextEdit +ocu snapshot --include-ax false TextEdit +``` + +原生 MCP `get_app_state` 接收同样的 `include_ax` / `include_screenshot` 布尔值。`snapshot` 两个参数都接受 `true` / `false`,不能同时关闭。`snapshot` 一直只打印文本;需要图片内容时使用 JS 或 `call` / MCP。 + +这两个选项只控制本次显式观察,不改变动作回读。 + +## 和重复截图的关系 + +macOS / Linux 下的 JS 动作已默认关闭自动回读;`getAXState()` 本身也不向模型输出图片,但默认仍会在 native 层采集图片后丢弃。`includeScreenshot: false` 可以在三个平台连这部分采集和编码一起省掉。直接使用 native MCP 时,它也会省去返回的 image block。 + +这里不做截图比较:请求了截图,即使画面相同也照常返回。“画面未变则省略图片”需要单独实现,通常仍要先截图再比较,主要减少传输和模型输入。后续可按会话、应用和窗口保存精确图像指纹,首次观察及状态重置时返回图片,并提供强制返回方式;不能仅凭 AX 没变化就认定画面没变化。 + +## 测试 + +```sh +swift test +python3 apps/OpenComputerUseLinux/runtime_test.py +(cd apps/OpenComputerUseLinux && go test ./...) +(cd apps/OpenComputerUseWindows && go test ./...) +node --test scripts/node-repl/*.test.mjs +``` + +真实 GTK 测试复用 [私有 D-Bus / Xvfb 环境](linux-action-read-back.md),另外需要 Node.js: + +```sh +(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-observation-test .) +bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-observation-test --observation-checks +``` + +脚本包装真实生产采集函数,确认关闭的通道没有被调用;再通过 native MCP 和 Worker JS 检查 PNG、选区、旧索引拒绝、坐标操作及 AX 恢复。GTK 独立导出状态确认动作结果。这是 X11 / GTK / AT-SPI 测试,不代表 Wayland 或完整任务性能测试。 + +macOS 使用正常签名且已授权的 OCU 可执行文件验证真实 AppKit 窗口,无需修改准入配置: + +```sh +node scripts/tests/macos-observation-smoke.mjs "/path/to/Open Computer Use.app/Contents/MacOS/OpenComputerUse" +``` + +它通过真实 AX / native MCP / CLI / Worker JS 验证三个采集模式、PNG、快照和 query 索引失效、坐标拒绝和元素恢复,并用现有 timing 日志确认跳过的采集分支未执行。窗口禁止成为 key/main window;脚本不向个人应用输入。 + +Windows 的 PowerShell 采集分支可用以下命令单独验证: + +```powershell +pwsh -NoProfile -File scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1 +``` + +这个测试解析生产脚本并执行其中的观察函数,使用带计数的 UIA / 截图依赖验证分支。它不等于 Windows 桌面实测。 + +Windows 桌面测试需要已登录的交互会话、Windows PowerShell 和 Node.js: + +```powershell +go -C apps/OpenComputerUseWindows build -o "$env:TEMP/ocu-observation.exe" . +pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:TEMP/ocu-observation.exe" +``` + +脚本创建自己的 WPF 窗口,通过真实 UIA / native MCP / CLI / Worker JS 检查通道选择、旧索引拒绝、AX 恢复、Unicode 设值和截图坐标拒绝;再用同进程的 Win32 小窗口验证物理 1×1 边界,并检查 WPF 最小化、cloaked 时不返回图片,恢复后重新返回 PNG。`.github/workflows/windows-observation.yml` 在 Windows runner 执行这组检查。WPF 的通过结果不代表所有 WinForms / WinUI / Electron 控件都支持相同 UIA pattern。 diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift index 3a3db1e3..2fbfc30a 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift @@ -148,6 +148,7 @@ public struct AppSnapshot { let elements: [Int: ElementRecord] var screenshotNote: String? = nil + var observation: SnapshotObservationOptions = .defaults public var renderedText: String { renderedText(style: .fullState) @@ -161,6 +162,9 @@ public struct AppSnapshot { lines.append("App=\(appReference) (pid \(app.pid))") lines.append("Window: \(quoted(displayTitle)), App: \(app.name).") lines.append(contentsOf: treeLines) + if !observation.includeAX { + lines.append(SnapshotObservationOptions.accessibilityOmittedNote) + } if let selectedText, !selectedText.isEmpty { lines.append("") @@ -184,10 +188,12 @@ enum SnapshotBuilder { for app: RunningAppDescriptor, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, - recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation + recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation, + observation: SnapshotObservationOptions = .defaults ) throws -> AppSnapshot { + try observation.validate() if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() { - return buildFixtureSnapshot(app: app, state: fixtureState) + return buildFixtureSnapshot(app: app, state: fixtureState, observation: observation) } try requireUsableComputerUseSession() @@ -198,8 +204,8 @@ enum SnapshotBuilder { } let appElement = AXUIElementCreateApplication(app.pid) - let lazyWebAccessibility = enableBestEffortAccessibilityModes(appElement) - || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL) + let lazyWebAccessibility = observation.includeAX && (enableBestEffortAccessibilityModes(appElement) + || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL)) let systemWide = AXUIElementCreateSystemWide() var focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute) var focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) @@ -221,8 +227,8 @@ enum SnapshotBuilder { // the tree, the capture and every element frame refer to the same window; // the title/area heuristics remain the fallback. var windowCapture = TimingLog.measure("snapshot.window_capture") { - WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow)) - ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle) + WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow), capture: observation.includeScreenshot) + ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot) } if windowCapture == nil, recoveryPolicy == .allowActivation, @@ -231,8 +237,8 @@ enum SnapshotBuilder { if let recoveredWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) { rootWindow = recoveredWindow windowTitle = stringValue(of: recoveredWindow, attribute: kAXTitleAttribute) - windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow)) - ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle) + windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow), capture: observation.includeScreenshot) + ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot) } } @@ -250,7 +256,8 @@ enum SnapshotBuilder { systemWide: systemWide, textLimit: textLimit, treeLimits: treeLimits, - lazyWebAccessibility: lazyWebAccessibility + lazyWebAccessibility: lazyWebAccessibility, + observation: observation ) } @@ -264,11 +271,21 @@ enum SnapshotBuilder { systemWide: AXUIElement, textLimit: SnapshotTextLimit, treeLimits: AccessibilityTreeLimits, - lazyWebAccessibility: Bool + lazyWebAccessibility: Bool, + observation: SnapshotObservationOptions ) -> AppSnapshot { let windowBounds = windowCapture.bounds - let screenshot = windowCapture.screenshotResult() + let screenshot = observation.includeScreenshot ? windowCapture.screenshotResult() + : ScreenshotEncodingResult(data: nil, note: SnapshotObservationOptions.screenshotOmittedNote) let screenshotData = screenshot.data + guard observation.includeAX else { + return AppSnapshot(app: app, windowTitle: windowTitle, windowBounds: windowBounds, + targetWindowID: windowCapture.windowID, targetWindowLayer: windowCapture.layer, + screenshotData: screenshotData, mode: .accessibility, treeLines: [], + focusedSummary: nil, focusedElement: nil, selectedText: nil, + windowElement: rootElement, elements: [:], screenshotNote: screenshot.note, + observation: observation) + } let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) let selectedText = focusedElement.flatMap { copySelectedText($0, textLimit: textLimit) } let context = RenderContext( @@ -330,7 +347,8 @@ enum SnapshotBuilder { selectedText: selectedText, windowElement: rootElement, elements: renderer.records, - screenshotNote: screenshot.note + screenshotNote: screenshot.note, + observation: observation ) } @@ -444,14 +462,15 @@ enum SnapshotBuilder { return copyElement(appElement, attribute: kAXFocusedUIElementAttribute) } - private static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState) -> AppSnapshot { + static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState, + observation: SnapshotObservationOptions = .defaults) -> AppSnapshot { var lines: [String] = [] var records: [Int: ElementRecord] = [:] let focusedIdentifier = state.focusedIdentifier var focusedSummary: String? - for element in state.elements.sorted(by: { $0.index < $1.index }) { + for element in (observation.includeAX ? state.elements.sorted(by: { $0.index < $1.index }) : []) { let titleSegment = element.title.map { " \($0)" } ?? "" let valueSegment = element.value.map { " Value: \($0)" } ?? "" let actionsSegment = element.actions.isEmpty ? "" : " Secondary Actions: \(element.actions.joined(separator: ", "))" @@ -485,9 +504,11 @@ enum SnapshotBuilder { treeLines: lines, focusedSummary: focusedSummary, focusedElement: nil, - selectedText: state.selectedText, + selectedText: observation.includeAX ? state.selectedText : nil, windowElement: nil, - elements: records + elements: records, + screenshotNote: observation.includeScreenshot ? nil : SnapshotObservationOptions.screenshotOmittedNote, + observation: observation ) } } diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift index 76485aa5..a74e8230 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift @@ -541,6 +541,7 @@ func shouldPreferContainingWebRowAXClickCandidate( public final class ComputerUseService { private var snapshotsByApp: [String: AppSnapshot] = [:] + private var invalidatedSnapshotAliases: Set = [] // Read per call: the app agent applies the caller's OPEN_COMPUTER_USE_* variables // for the duration of each request. private var actionReadBack: Bool { actionReadBackEnabled(environment: ProcessInfo.processInfo.environment) } @@ -593,14 +594,16 @@ public final class ComputerUseService { app query: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, - windowPlacement: WindowPlacement = .keep + windowPlacement: WindowPlacement = .keep, + observation: SnapshotObservationOptions = .defaults ) throws -> ToolCallResult { + try observation.validate() guard windowPlacement != .keep else { - return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits), style: .fullState) + return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, observation: observation), style: .fullState) } // Explicit placements never activate or raise; they only move the frame. - let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly) + let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation) guard snapshot.mode != .fixture else { throw ComputerUseError.message("window_placement '\(windowPlacement.rawValue)' is not supported for fixture apps") } @@ -616,7 +619,7 @@ public final class ComputerUseService { break } - return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly), style: .fullState) + return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation), style: .fullState) } public func click( @@ -641,6 +644,9 @@ public final class ComputerUseService { let snapshot = try snapshotForAction(app: query, elementIndex: elementIndex) let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left + if elementIndex == nil, !snapshot.observation.includeScreenshot { + throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions") + } if snapshot.mode == .fixture { guard clickMethod == .auto else { throw ComputerUseError.message( @@ -878,6 +884,9 @@ public final class ComputerUseService { public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> ToolCallResult { let snapshot = try currentSnapshot(for: query) + guard snapshot.observation.includeScreenshot else { + throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions") + } if snapshot.mode == .fixture { try FixtureBridge.post(FixtureCommand(kind: "drag", identifier: "fixture-drag-pad", x: fromX, y: fromY, toX: toX, toY: toY)) Thread.sleep(forTimeInterval: 0.15) @@ -1185,7 +1194,7 @@ public final class ComputerUseService { return dict } - private func currentSnapshot(for query: String) throws -> AppSnapshot { + func currentSnapshot(for query: String) throws -> AppSnapshot { if let snapshot = snapshotsByApp[query.lowercased()] { if snapshot.mode != .fixture { do { @@ -1198,6 +1207,9 @@ public final class ComputerUseService { return snapshot } + if invalidatedSnapshotAliases.contains(query.lowercased()) { + throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.accessibilityOmittedNote) + } return try refreshSnapshot(for: query) } @@ -1206,27 +1218,47 @@ public final class ComputerUseService { for query: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, - recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation + recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation, + observation: SnapshotObservationOptions = .defaults ) throws -> AppSnapshot { let app = try AppDiscovery.resolve(query) let snapshot = try SnapshotBuilder.build( for: app, textLimit: textLimit, treeLimits: treeLimits, - recoveryPolicy: recoveryPolicy + recoveryPolicy: recoveryPolicy, + observation: observation ) + rememberSnapshot(snapshot, for: query) + return snapshot + } + + func rememberSnapshot(_ snapshot: AppSnapshot, for query: String) { + let app = snapshot.app + if !snapshot.observation.includeAX { + var pids: Set = [app.pid] + if let previous = snapshotsByApp[query.lowercased()] { pids.insert(previous.app.pid) } + let aliases = snapshotsByApp.compactMap { pids.contains($0.value.app.pid) ? $0.key : nil } + for alias in aliases { + snapshotsByApp.removeValue(forKey: alias) + invalidatedSnapshotAliases.insert(alias) + } + targetedElements.removeAll(forPIDs: pids) + } + let keys = Set([ query.lowercased(), + String(app.pid), app.name.lowercased(), (app.bundleIdentifier ?? "").lowercased(), ].filter { !$0.isEmpty }) for key in keys { snapshotsByApp[key] = snapshot + invalidatedSnapshotAliases.remove(key) } - return snapshot } private func lookupElement(snapshot: AppSnapshot, index: String) throws -> ElementRecord { diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift index e104de92..cfa984c0 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift @@ -67,7 +67,11 @@ public final class ComputerUseToolDispatcher { maxNodeCount: try optionalPositiveInt("max_tree_nodes", in: arguments), maxDepth: try optionalPositiveInt("max_tree_depth", in: arguments) ), - windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments)) + windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments)), + observation: SnapshotObservationOptions( + includeAX: try strictObservationBool("include_ax", in: arguments), + includeScreenshot: try strictObservationBool("include_screenshot", in: arguments) + ) ) case "click": return try service.click( @@ -154,6 +158,14 @@ public final class ComputerUseToolDispatcher { return value } + private func strictObservationBool(_ key: String, in arguments: [String: Any]) throws -> Bool { + guard let value = arguments[key] else { return true } + guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else { + throw ComputerUseError.invalidArguments("\(key) must be a boolean") + } + return number.boolValue + } + private func optionalBool(_ key: String, in arguments: [String: Any]) -> Bool? { switch arguments[key] { case let value as Bool: return value diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift index 0eeb6fa4..60dd85b5 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift @@ -7,7 +7,7 @@ public enum OpenComputerUseCLICommand: Equatable { case lockedUseStatus(json: Bool) case lockedUseManagement(action: String, validation: Bool) case listApps - case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults) + case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, observation: SnapshotObservationOptions = .defaults) case call(OpenComputerUseCallInvocation) case turnEnded(payload: String?) case help(command: String?) @@ -175,12 +175,14 @@ public func openComputerUseHelpText(command: String? = nil) -> String { case "snapshot": return """ Usage: - open-computer-use snapshot [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] + open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit ] [--max-tree-nodes ] [--max-tree-depth ] Arguments: App name or bundle identifier to inspect. Options: + --include-ax Include AX/focus/selection (true by default). False invalidates prior indexes. + --include-screenshot Include screenshot capture (true by default). At least one channel must be true. --text-limit Override the default 500 character text limit. Use `max` for full text. --max-tree-nodes Override the default 1200 node accessibility tree budget. --max-tree-depth Override the default 64 level accessibility tree depth. @@ -301,6 +303,7 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm return .help(command: "snapshot") } + var observation = SnapshotObservationOptions.defaults var app: String? var textLimit = SnapshotTextLimit.defaults var maxTreeNodes: Int? @@ -310,6 +313,13 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm while index < arguments.count { let argument = arguments[index] switch argument { + case "--include-ax", "--include-screenshot": + index += 1 + guard index < arguments.count, let value = Bool(arguments[index].lowercased()) else { + throw OpenComputerUseCLIError(message: "\(argument) requires true or false", helpCommand: "snapshot") + } + if argument == "--include-ax" { observation.includeAX = value } + else { observation.includeScreenshot = value } case "--text-limit": index += 1 guard index < arguments.count else { @@ -348,13 +358,17 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm throw OpenComputerUseCLIError(message: "snapshot requires an app name or bundle identifier", helpCommand: "snapshot") } + guard observation.includeAX || observation.includeScreenshot else { + throw OpenComputerUseCLIError(message: "At least one of include_ax or include_screenshot must be true", helpCommand: "snapshot") + } return .snapshot( app: app, textLimit: textLimit, treeLimits: AccessibilityTreeLimits.defaults.replacing( maxNodeCount: maxTreeNodes, maxDepth: maxTreeDepth - ) + ), + observation: observation ) } diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift new file mode 100644 index 00000000..942d826b --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift @@ -0,0 +1,21 @@ +import Foundation + +public struct SnapshotObservationOptions: Equatable, Sendable { + public var includeAX: Bool + public var includeScreenshot: Bool + + public static let defaults = SnapshotObservationOptions() + public static let accessibilityOmittedNote = "Accessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions." + public static let screenshotOmittedNote = "Screenshot omitted by include_screenshot=false" + + public init(includeAX: Bool = true, includeScreenshot: Bool = true) { + self.includeAX = includeAX + self.includeScreenshot = includeScreenshot + } + + public func validate() throws { + guard includeAX || includeScreenshot else { + throw ComputerUseError.invalidArguments("At least one of include_ax or include_screenshot must be true") + } + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift index b58381da..c8a6cff6 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift @@ -60,6 +60,10 @@ final class TargetedElementRegistry { } func remove(_ index: Int) { entries.removeValue(forKey: index) } + func removeAll(forPIDs pids: Set) { + entries = entries.filter { !pids.contains($0.value.app.pid) } + order.removeAll { entries[$0] == nil } + } func clear() { entries.removeAll(); order.removeAll() } } diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift index e53da124..d6c0fd3b 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift @@ -94,6 +94,8 @@ public enum ToolDefinitions { annotations: defaultAnnotations(), inputSchema: objectSchema( properties: [ + "include_ax": ["type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips collection and invalidates previous element indexes; read AX again before element actions."], + "include_screenshot": ["type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and encoding; screenshot-coordinate actions require a fresh screenshot."], "app": stringProperty(description: "App name or bundle identifier"), "text_limit": textLimitProperty(description: "Maximum text characters to return. Use \"max\" for full text. Defaults to 500."), "max_tree_nodes": positiveIntegerProperty(description: "Maximum accessibility tree nodes to render. Defaults to 1200."), diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift index d1f4737e..37276e6e 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift @@ -43,6 +43,13 @@ public struct ToolCallResult: @unchecked Sendable { content.first(where: { $0.dictionary["type"] as? String == "text" })?.dictionary["text"] as? String } + /// Plain-text CLI output includes observation notes but never image bytes. + public var textContent: String { + content.compactMap { item in + item.dictionary["type"] as? String == "text" ? item.dictionary["text"] as? String : nil + }.joined(separator: "\n") + } + public var asDictionary: [String: Any] { [ "content": content.map(\.dictionary), diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift new file mode 100644 index 00000000..e196be72 --- /dev/null +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift @@ -0,0 +1,114 @@ +import AppKit +import XCTest +@testable import OpenComputerUseKit + +final class SelectiveObservationTests: XCTestCase { + private func app(pid: Int32 = 42, name: String = "ObservationFixture") -> RunningAppDescriptor { + .init(name: name, bundleIdentifier: "test.\(name)", pid: pid, runningApplication: .current) + } + + private func snapshot(pid: Int32 = 42, name: String = "ObservationFixture", + options: SnapshotObservationOptions = .defaults) -> AppSnapshot { + let state = FixtureAppState(windowTitle: "test", windowBounds: .init(rect: CGRect(x: 0, y: 0, width: 300, height: 200)), + focusedIdentifier: "input", elements: [ + .init(identifier: "input", index: 7, role: "text", title: "Input", value: "中文🙂", actions: [], + frame: .init(rect: CGRect(x: 10, y: 10, width: 100, height: 30))) + ], selectedText: "中文🙂") + return SnapshotBuilder.buildFixtureSnapshot(app: app(pid: pid, name: name), state: state, observation: options) + } + + func testEmptyObservationFailsBeforeResolvingApp() { + XCTAssertThrowsError(try ComputerUseService().getAppState(app: "missing-test-app", + observation: .init(includeAX: false, includeScreenshot: false))) { + XCTAssertTrue(String(describing: $0).contains("At least one")) + } + } + + func testDispatcherRequiresActualJSONBooleans() { + let dispatcher = ComputerUseToolDispatcher() + for key in ["include_ax", "include_screenshot"] { + for value: Any in ["false", 0, 1, NSNull(), [], [:]] { + XCTAssertThrowsError(try dispatcher.callTool(name: "get_app_state", arguments: ["app": "missing-test-app", key: value])) { + XCTAssertTrue(String(describing: $0).contains("\(key) must be a boolean")) + } + } + } + } + + func testSchemaHasMatchingFullDefaults() throws { + let tool = try XCTUnwrap(ToolDefinitions.all.first { $0.name == "get_app_state" }) + let properties = try XCTUnwrap(tool.inputSchema["properties"] as? [String: Any]) + for key in ["include_ax", "include_screenshot"] { + let value = try XCTUnwrap(properties[key] as? [String: Any]) + XCTAssertEqual(value["type"] as? String, "boolean") + XCTAssertEqual(value["default"] as? Bool, true) + } + } + + func testCLIObservationOptionsAndDefaultCompatibility() throws { + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"]), .snapshot(app: "TextEdit")) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "--include-screenshot", "false", "TextEdit"]), + .snapshot(app: "TextEdit", observation: .init(includeScreenshot: false))) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit", "--include-ax", "FALSE"]), + .snapshot(app: "TextEdit", observation: .init(includeAX: false))) + for args in [["--include-ax"], ["--include-screenshot", "0"], ["--include-ax", "no"], + ["--include-ax", "false", "--include-screenshot", "false"]] { + XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"] + args)) + } + } + + func testFixtureChannelsPreserveOrOmitAXAndCoordinateGuidance() { + let full = snapshot() + XCTAssertEqual(full.selectedText, "中文🙂") + XCTAssertEqual(full.elements.count, 1) + let image = snapshot(options: .init(includeAX: false)) + XCTAssertTrue(image.elements.isEmpty) + XCTAssertTrue(image.treeLines.isEmpty) + XCTAssertNil(image.focusedSummary) + XCTAssertNil(image.selectedText) + XCTAssertTrue(image.renderedText.contains("indexes invalidated")) + let ax = snapshot(options: .init(includeScreenshot: false)) + XCTAssertEqual(ax.elements.count, 1) + XCTAssertEqual(ax.selectedText, "中文🙂") + XCTAssertNil(ax.screenshotData) + XCTAssertEqual(ax.screenshotNote, SnapshotObservationOptions.screenshotOmittedNote) + } + + func testScreenshotOnlyInvalidatesOlderAliasesAcrossPIDChangeAndAXRestoresIndexes() throws { + let service = ComputerUseService() + service.rememberSnapshot(snapshot(), for: "older-query") + service.rememberSnapshot(snapshot(), for: "ObservationFixture") + service.rememberSnapshot(snapshot(pid: 99, name: "Other"), for: "Other") + service.rememberSnapshot(snapshot(pid: 43, options: .init(includeAX: false)), for: "ObservationFixture") + XCTAssertThrowsError(try service.currentSnapshot(for: "older-query")) + XCTAssertThrowsError(try service.currentSnapshot(for: "42")) + XCTAssertTrue(try service.currentSnapshot(for: "ObservationFixture").elements.isEmpty) + XCTAssertEqual(try service.currentSnapshot(for: "Other").elements.count, 1) + service.rememberSnapshot(snapshot(pid: 43, options: .init(includeScreenshot: false)), for: "ObservationFixture") + XCTAssertEqual(try service.currentSnapshot(for: "ObservationFixture").elements.count, 1) + } + + func testQueryIndexInvalidationIsScopedAndNeverReissuesIDs() throws { + let registry = TargetedElementRegistry() + let first = TargetedAppIdentity(pid: 42, bundleIdentifier: "a", launchedAt: nil) + let other = TargetedAppIdentity(pid: 99, bundleIdentifier: "b", launchedAt: nil) + let a = registry.insert(app: first) { _ in "a" } + let b = registry.insert(app: other) { _ in "b" } + registry.removeAll(forPIDs: [42]) + XCTAssertThrowsError(try registry.resolve(a, app: first)) + XCTAssertEqual(try registry.resolve(b, app: other), "b") + XCTAssertGreaterThan(registry.insert(app: first) { _ in "new" }, b) + } + + func testAXOnlyRejectsCoordinateInputBeforeAnyFixtureCommand() { + let service = ComputerUseService() + service.rememberSnapshot(snapshot(options: .init(includeScreenshot: false)), for: "fixture") + XCTAssertThrowsError(try service.click(app: "fixture", elementIndex: nil, x: 10, y: 10, + clickCount: 1, mouseButton: "left")) { + XCTAssertTrue(String(describing: $0).contains("include_screenshot=false")) + } + XCTAssertThrowsError(try service.drag(app: "fixture", fromX: 1, fromY: 1, toX: 2, toY: 2)) { + XCTAssertTrue(String(describing: $0).contains("include_screenshot=false")) + } + } +} diff --git a/scripts/node-repl/open-computer-use-repl.mjs b/scripts/node-repl/open-computer-use-repl.mjs index 89b798c5..5e5c780e 100755 --- a/scripts/node-repl/open-computer-use-repl.mjs +++ b/scripts/node-repl/open-computer-use-repl.mjs @@ -41,6 +41,7 @@ The runtime exposes an asynchronous app-bound API: - \`await app.getAXState({ emit?, textLimit?, maxTreeNodes?, maxTreeDepth?, windowPlacement? })\` - \`await app.getScreenshot({ emit? })\` - \`await app.getAXStateAndScreenshot(options?)\` +- Optional observation on macOS, Windows and Linux: \`app.getScreenshot({ includeAX: false })\` skips AX/focus/selection and invalidates prior element indexes; refresh AX before element actions. \`app.getAXState({ includeScreenshot: false })\` skips capture; screenshot-coordinate actions require a fresh screenshot. Omitted options retain full native observation. - \`await app.click(elementIndexOrPoint, { mouseButton?, clickCount?, clickMethod? })\` - \`await app.scroll(elementIndex, direction, pages?)\` - \`await app.drag([fromX, fromY], [toX, toY])\` @@ -258,6 +259,15 @@ function optionsToSnapshotArgs(options = {}) { if (options.maxTreeNodes !== undefined) args.max_tree_nodes = options.maxTreeNodes; if (options.maxTreeDepth !== undefined) args.max_tree_depth = options.maxTreeDepth; if (options.windowPlacement !== undefined) args.window_placement = options.windowPlacement; + for (const [option, argument] of [["includeAX", "include_ax"], ["includeScreenshot", "include_screenshot"]]) { + if (options[option] !== undefined) { + if (typeof options[option] !== "boolean") throw new Error(`${option} must be a boolean`); + args[argument] = options[option]; + } + } + if (args.include_ax === false && args.include_screenshot === false) { + throw new Error("At least one of includeAX or includeScreenshot must be true"); + } return args; } diff --git a/scripts/node-repl/open-computer-use-repl.test.mjs b/scripts/node-repl/open-computer-use-repl.test.mjs index f343bdcf..5c7aaad0 100644 --- a/scripts/node-repl/open-computer-use-repl.test.mjs +++ b/scripts/node-repl/open-computer-use-repl.test.mjs @@ -102,6 +102,42 @@ test("app-bound API composes actions and state in one js call", async () => { assert.ok(result.content.some(item => item.type === "text" && item.text === "state:Text")); }); +for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) { + test(`${Session.name}: selective observation is explicit and legacy calls keep their arguments`, async t => { + const native = mockNative(); + const session = new Session({ native }); + t.after(() => session.close?.()); + const result = await session.run(` + var selectiveApp = await cua.getApp("Text", { initialState: false }); + await selectiveApp.getScreenshot({ includeAX: false, emit: false }); + await selectiveApp.getAXState({ includeScreenshot: false, emit: false }); + await selectiveApp.getScreenshot({ emit: false }); + await selectiveApp.getAXState({ emit: false }); + `); + assert.equal(result.isError, false); + assert.deepEqual(native.calls.map(call => call.arguments), [ + { app: "Text", include_ax: false, text_limit: 1 }, + { app: "Text", include_screenshot: false }, + { app: "Text", text_limit: 1 }, + { app: "Text" }, + ]); + const count = native.calls.length; + for (const code of [ + 'await selectiveApp.getScreenshot({ includeAX: "false" })', + 'await selectiveApp.getAXState({ includeScreenshot: null })', + ]) { + const invalid = await session.run(code); + assert.equal(invalid.isError, true); + assert.match(invalid.content.at(-1).text, /must be a boolean/); + } + assert.equal(native.calls.length, count); + const empty = await session.run('await selectiveApp.getAXStateAndScreenshot({ includeAX: false, includeScreenshot: false })'); + assert.equal(empty.isError, true); + assert.match(empty.content.at(-1).text, /At least one/); + assert.equal(native.calls.length, count); + }); +} + test("app-bound API forwards background-operation options", async () => { const native = mockNative(); const session = new PersistentJavaScriptSession({ native }); diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.sh index 8a9507fc..608c637a 100755 --- a/scripts/run-linux-read-back-smoke.sh +++ b/scripts/run-linux-read-back-smoke.sh @@ -3,7 +3,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" if [[ "$(uname -s)" != Linux || $# -lt 1 ]]; then - echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks]" >&2 + echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks | --observation-checks]" >&2 exit 2 fi diff --git a/scripts/tests/fixtures/MacOSObservationFixture.swift b/scripts/tests/fixtures/MacOSObservationFixture.swift new file mode 100644 index 00000000..d5b5d2c6 --- /dev/null +++ b/scripts/tests/fixtures/MacOSObservationFixture.swift @@ -0,0 +1,25 @@ +import AppKit + +final class ObservationWindow: NSWindow { + override var canBecomeKey: Bool { false } + override var canBecomeMain: Bool { false } +} + +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +let window = ObservationWindow(contentRect: NSRect(x: 200, y: 200, width: 420, height: 180), + styleMask: [.titled, .closable], backing: .buffered, defer: false) +window.title = "OCU Observation Fixture" +window.isReleasedWhenClosed = false +let input = NSTextField(frame: NSRect(x: 20, y: 75, width: 380, height: 30)) +input.stringValue = "initial-中文🙂" +input.setAccessibilityLabel("Read-back input") +window.contentView?.addSubview(input) +window.orderFront(nil) +let statePath = CommandLine.arguments[1] +let timer = Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { _ in + let data = try! JSONSerialization.data(withJSONObject: ["text": input.stringValue]) + try? data.write(to: URL(fileURLWithPath: statePath), options: .atomic) +} +Timer.scheduledTimer(withTimeInterval: 120, repeats: false) { _ in app.terminate(nil) } +app.run() diff --git a/scripts/tests/fixtures/WindowsObservationFixture.ps1 b/scripts/tests/fixtures/WindowsObservationFixture.ps1 new file mode 100644 index 00000000..9d30f81f --- /dev/null +++ b/scripts/tests/fixtures/WindowsObservationFixture.ps1 @@ -0,0 +1,105 @@ +param([Parameter(Mandatory = $true)][string]$StatePath) +$ErrorActionPreference = "Stop" +Add-Type -AssemblyName PresentationFramework +Add-Type -TypeDefinition @" +using System; +using System.Runtime.InteropServices; +public static class ObservationDwm { + [StructLayout(LayoutKind.Sequential)] + public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } + [DllImport("user32.dll")] + public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern IntPtr CreateWindowEx(uint extendedStyle, string windowClass, string title, uint style, + int x, int y, int width, int height, IntPtr parent, IntPtr menu, IntPtr instance, IntPtr parameter); + [DllImport("user32.dll")] + public static extern bool DestroyWindow(IntPtr hwnd); + public static IntPtr CreateTinyWindow() { + return CreateWindowEx(0, "STATIC", "OCU Windows Observation Fixture", 0x90000000u, + 80, 80, 1, 1, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero); + } + [DllImport("dwmapi.dll")] + public static extern int DwmSetWindowAttribute(IntPtr hwnd, int attr, ref int value, int size); + [DllImport("dwmapi.dll")] + public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out int value, int size); +} +"@ +$window = New-Object System.Windows.Window +$window.Title = "OCU Windows Observation Fixture" +$window.ShowInTaskbar = $true +$window.Left = 80 +$window.Top = 80 +$window.Width = 600 +$window.Height = 280 +$inputBox = New-Object System.Windows.Controls.TextBox +[System.Windows.Automation.AutomationProperties]::SetName($inputBox, "Observation input") +$inputBox.Margin = New-Object System.Windows.Thickness(20) +$inputBox.Height = 30 +$inputBox.VerticalAlignment = "Center" +$inputBox.Text = "initial-value" +$window.Content = $inputBox +$state = @{ sequence = 0; text = ""; width = 0; height = 0; minimized = $false; cloakHResult = 0; cloaked = 0 } +$stateEncoding = New-Object System.Text.UTF8Encoding($false) +$tinyWindow = @{ handle = [IntPtr]::Zero } +$commandPath = $StatePath + ".command" +$timer = New-Object System.Windows.Threading.DispatcherTimer +$timer.Interval = [TimeSpan]::FromMilliseconds(50) +$timer.Add_Tick({ + $hwnd = (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle + if (Test-Path -LiteralPath $commandPath) { + try { + $command = [System.IO.File]::ReadAllText($commandPath) | ConvertFrom-Json + if ($command.sequence -gt $state.sequence) { + switch ($command.kind) { + "tiny" { + # WPF rounds a one-DIP window to multiple physical + # pixels on high-DPI desktops. Use an owned Win32 + # fixture window for the exact physical 1x1 case. + $window.Hide() + $tinyWindow.handle = [ObservationDwm]::CreateTinyWindow() + } + "minimize" { $window.WindowState = "Minimized" } + "cloak" { + $value = 1 + $state.cloakHResult = [ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4) + } + "restore" { + $value = 0 + [void][ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4) + if ($tinyWindow.handle -ne [IntPtr]::Zero) { + [void][ObservationDwm]::DestroyWindow($tinyWindow.handle) + $tinyWindow.handle = [IntPtr]::Zero + } + $window.WindowState = "Normal" + $window.Width = 600 + $window.Height = 280 + $window.Show() + } + } + $state.sequence = $command.sequence + } + } catch { } + } + $hwnd = if ($tinyWindow.handle -ne [IntPtr]::Zero) { $tinyWindow.handle } + else { (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle } + $cloaked = 0 + [void][ObservationDwm]::DwmGetWindowAttribute($hwnd, 14, [ref]$cloaked, 4) + $state.cloaked = $cloaked + $state.text = $inputBox.Text + $rect = New-Object ObservationDwm+RECT + [void][ObservationDwm]::GetWindowRect($hwnd, [ref]$rect) + $state.width = $rect.Right - $rect.Left + $state.height = $rect.Bottom - $rect.Top + $state.minimized = $window.WindowState -eq "Minimized" + $temporary = $StatePath + ".tmp" + [System.IO.File]::WriteAllText($temporary, ($state | ConvertTo-Json -Compress), $stateEncoding) + try { + if (Test-Path -LiteralPath $StatePath) { [System.IO.File]::Replace($temporary, $StatePath, ($StatePath + ".bak")) } + else { [System.IO.File]::Move($temporary, $StatePath) } + } catch [System.IO.IOException] { } +}) +$timer.Start() +$window.Add_Closed({ $timer.Stop() }) +$application = New-Object System.Windows.Application +$application.ShutdownMode = "OnMainWindowClose" +[void]$application.Run($window) diff --git a/scripts/tests/linux-observation-js-smoke.mjs b/scripts/tests/linux-observation-js-smoke.mjs new file mode 100644 index 00000000..d106c331 --- /dev/null +++ b/scripts/tests/linux-observation-js-smoke.mjs @@ -0,0 +1,56 @@ +#!/usr/bin/env node +// Runs inside the private GTK smoke session; Python owns fixture lifecycle. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs"; + +const [nativePath, appName, stateFile] = process.argv.slice(2); +assert.ok(nativePath && appName && stateFile, "Run through --observation-checks"); +const native = new JsonLinePeer({ command: nativePath, args: ["mcp"], + env: { ...process.env, OPEN_COMPUTER_USE_ACTION_READ_BACK: "0" } }); +let session; +async function run(code, expectedError) { + const result = await session.run(code); + const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n"); + if (expectedError) { + assert.equal(result.isError, true, text); + assert.match(text, expectedError); + } else assert.equal(result.isError, false, text); + return result; +} +try { + await native.initialize(); + const request = native.request.bind(native); + native.request = async (method, params, timeout) => { + const result = await request(method, params, timeout); + if (method === "tools/call" && params.name === "get_app_state" && !result.isError) { + const images = result.content.filter(item => item.type === "image"); + assert.equal(images.length, params.arguments.include_screenshot === false ? 0 : 1); + } + return result; + }; + session = new WorkerJavaScriptSession({ native }); + const ax = await run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false}); + var ax = await app.getAXState({includeScreenshot:false}); + var oldIndex = ax.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1];`); + assert.equal(ax.content.some(item => item.type === "image"), false); + const shot = await run(`var screenshot = await app.getScreenshot({includeAX:false}); + if(screenshot.subarray(0,8).toString('hex') !== '89504e470d0a1a0a') throw Error('Invalid PNG');`); + assert.equal(shot.content.filter(item => item.type === "image").length, 1); + await run(`await app.setValue(oldIndex, "must-not-write");`, /unknown element_index/); + await run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false}); + var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1]; + await app.setValue(index, "worker-live-中文🙂");`); + const deadline = Date.now() + 5000; + while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文🙂") { + assert.ok(Date.now() < deadline, "GTK value did not update"); + await new Promise(resolve => setTimeout(resolve, 30)); + } + const full = await run(`await app.getAXStateAndScreenshot();`); + assert.equal(full.content.filter(item => item.type === "image").length, 1); + assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文🙂"))); + console.log("Worker JS → native MCP → GTK: selective collection, PNG output, stale-index error and AX recovery passed."); +} finally { + await session?.close(); + native.close(); +} diff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.py index cebad77b..b6181e8d 100644 --- a/scripts/tests/linux-read-back-smoke.py +++ b/scripts/tests/linux-read-back-smoke.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -"""Exercise native Linux MCP actions against an isolated real GTK fixture.""" +"""Exercise native Linux MCP observation and actions against an isolated GTK fixture.""" import argparse +import base64 from contextlib import ExitStack import importlib.util import json @@ -10,6 +11,7 @@ import re import select import subprocess +import struct import sys import tempfile import time @@ -285,14 +287,98 @@ def page_is(number): print("Native MCP: default/opt-in output, stable indexes, page changes and full detail restoration passed.") -def smoke(native, focus_checks=False, hidden_checks=False): +def check_observation(client, native, name, state_file, stderr): + wait_for(lambda: json.loads(state_file.read_text())["selected"] == "中文🙂", "fixture selection") + runtime_path = Path(__file__).resolve().parents[2] / "apps/OpenComputerUseLinux/runtime.py" + spec = importlib.util.spec_from_file_location("observation_smoke_runtime", runtime_path) + runtime = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runtime) + + def png(data): + raw = base64.b64decode(data, validate=True) + assert raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR", raw[:24] + width, height = struct.unpack(">II", raw[16:24]) + assert width > 0 and height > 0, (width, height) + + def snapshot(**flags): + with ExitStack() as stack: + calls = {key: stack.enter_context(mock.patch.object(runtime, key, wraps=getattr(runtime, key))) + for key in ("render_tree", "focus_details", "capture_window_png")} + result = runtime.build_snapshot(name, **flags) + return result, {key: query.call_count for key, query in calls.items()} + + full, counts = snapshot() + assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 1}, counts + png(full["screenshotPngBase64"]) + assert full["selectedText"] == "中文🙂", full + ax, counts = snapshot(include_screenshot=False) + assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 0}, counts + assert ax["screenshotPngBase64"] is None and ax["elements"] == full["elements"], ax + assert ax["selectedText"] == "中文🙂", ax + image, counts = snapshot(include_ax=False) + assert counts == {"render_tree": 0, "focus_details": 0, "capture_window_png": 1}, counts + png(image["screenshotPngBase64"]) + assert image["elements"] == [] and image["treeLines"] == [], image + assert image["focusedSummary"] is None and image["selectedText"] is None, image + assert image["windowBounds"] == full["windowBounds"], image + print("Real GTK collection: full, AX-only (no capture), image-only (no tree/focus) passed.") + + ax = client.call("get_app_state", name, include_screenshot=False) + expect_snapshot(ax) + assert all(item["type"] != "image" for item in ax["content"]), ax + assert "Selected text: [中文🙂]" in text(ax), text(ax) + refused = client.call("click", name, x=10, y=10) + assert refused.get("isError") and "include_screenshot=false" in text(refused), refused + # The standalone snapshot and JSON call entry points must preserve flags. + cli = subprocess.run([native, "snapshot", "--include-screenshot", "false", name], + check=True, capture_output=True, text=True, timeout=40) + assert "Read-back input" in cli.stdout and "include_screenshot=false" in cli.stdout, cli.stdout + cli = subprocess.run([native, "call", "get_app_state", "--args", + json.dumps({"app": name, "include_screenshot": False})], + check=True, capture_output=True, text=True, timeout=40) + assert all(item["type"] != "image" for item in json.loads(cli.stdout)["content"]), cli.stdout + old_index = element_index(ax, "Read-back input") + invalid = client.call("get_app_state", name, include_ax=False, include_screenshot=False) + assert invalid.get("isError"), invalid + image = client.call("get_app_state", name, include_ax=False) + assert not image.get("isError"), image + images = [item for item in image["content"] if item["type"] == "image"] + assert len(images) == 1, image + png(images[0]["data"]) + assert "indexes invalidated" in text(image), text(image) + for omitted in ("Read-back input", "The focused UI element", "Selected text:"): + assert omitted not in text(image), text(image) + for alias in (name, str(full["app"]["pid"])): + result = client.call("set_value", alias, element_index=old_index, value="must-not-write") + assert result.get("isError"), result + assert json.loads(state_file.read_text())["text"] == "prefix-中文🙂-suffix" + + button = next(row for row in full["elements"] if row["name"] == "Increment") + bounds = button["frame"] + result = client.call("click", name, x=bounds["x"] + bounds["width"] / 2, + y=bounds["y"] + bounds["height"] / 2) + assert not result.get("isError"), result + wait_for(lambda: json.loads(state_file.read_text())["count"] == 1, "coordinate click after image-only read") + refreshed = client.call("get_app_state", name, include_screenshot=False) + expect_snapshot(refreshed) + result = client.call("set_value", name, element_index=element_index(refreshed, "Read-back input"), + value="selective-中文🙂") + assert not result.get("isError"), result + wait_for(lambda: json.loads(state_file.read_text())["text"] == "selective-中文🙂", "AX index recovery") + print("Native MCP: valid PNG, omitted channels, stale name/PID indexes refused, coordinate action and AX recovery passed.") + + subprocess.run(["node", str(Path(__file__).with_name("linux-observation-js-smoke.mjs")), + native, name, str(state_file)], check=True, timeout=30) + + +def smoke(native, focus_checks=False, hidden_checks=False, observation_checks=False): with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory: directory = Path(directory) state_file = directory / "state.json" name = "OCUReadBackFixture" + str(os.getpid()) with (directory / "stderr.log").open("w+") as stderr: arguments = [sys.executable, __file__, "--fixture", name, "--state-file", str(state_file)] - if focus_checks: + if focus_checks or observation_checks: arguments.append("--focus-checks") if hidden_checks: arguments.append("--hidden-checks") @@ -308,6 +394,9 @@ def smoke(native, focus_checks=False, hidden_checks=False): if focus_checks: check_focus(client, name, state_file) return + if observation_checks: + check_observation(client, native, name, state_file, stderr) + return if hidden_checks: check_hidden(client, native, name, state_file, stderr) observed = client.call("get_app_state", name) @@ -381,6 +470,7 @@ def acknowledged(tool, **arguments): checks = parser.add_mutually_exclusive_group() checks.add_argument("--focus-checks", action="store_true") checks.add_argument("--hidden-checks", action="store_true") + checks.add_argument("--observation-checks", action="store_true") parser.add_argument("--state-file", type=Path) args = parser.parse_args() if args.fixture: @@ -388,4 +478,4 @@ def acknowledged(tool, **arguments): else: if not args.native: parser.error("--native is required") - smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks) + smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks, args.observation_checks) diff --git a/scripts/tests/macos-observation-smoke.mjs b/scripts/tests/macos-observation-smoke.mjs new file mode 100644 index 00000000..73df8fef --- /dev/null +++ b/scripts/tests/macos-observation-smoke.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs"; + +assert.equal(process.platform, "darwin"); +assert.ok(process.argv[2], "Pass the normally signed OCU app executable with existing permissions/admission"); +const nativePath = path.resolve(process.argv[2]); +const directory = mkdtempSync(path.join(tmpdir(), "ocu-observation-")); +const fixturePath = path.join(directory, "OCUObservationFixture"); +const stateFile = path.join(directory, "state.json"); +const appName = "OCUObservationFixture"; +const env = { ...process.env, OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY: "1", + OPEN_COMPUTER_USE_VISUAL_CURSOR: "0", OPEN_COMPUTER_USE_ACTION_READ_BACK: "0", + OPEN_COMPUTER_USE_DEBUG_TIMING: "1", OPEN_COMPUTER_USE_CONFIG_FILE: path.join(directory, "config.json") }; +for (const key of Object.keys(env)) if (key.startsWith("OPEN_COMPUTER_USE_IMAGE_")) delete env[key]; +writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "{}"); +let fixture, native, session; +const text = result => result.content.filter(item => item.type === "text").map(item => item.text).join("\n"); +const images = result => result.content.filter(item => item.type === "image"); +function indexOf(state) { + const match = state.match(/^\s*(\d+)\s+[^\n]*Read-back input(?:[\s,]|$)/m); + assert.ok(match, state); + return match[1]; +} +async function waitFor(test) { + const deadline = Date.now() + 8000; + while (!test()) { assert.ok(Date.now() < deadline, "Fixture timeout"); await new Promise(r => setTimeout(r, 30)); } +} +async function call(name, args, expectedError) { + const result = await native.request("tools/call", { name, arguments: { app: appName, ...args } }); + assert.equal(!!result.isError, !!expectedError, text(result)); + if (expectedError) assert.match(text(result), expectedError); + return result; +} +async function observe(args) { + native.stderr = ""; + const result = await call("get_app_state", args); + await new Promise(r => setTimeout(r, 50)); + if (args.include_screenshot === false) { + assert.equal(images(result).length, 0); + assert.doesNotMatch(native.stderr, /timing snapshot\.capture_(?:hw|sck) /); + assert.match(native.stderr, /timing snapshot\.tree_walk /); + } else { + assert.equal(images(result).length, 1, text(result)); + assert.equal(Buffer.from(images(result)[0].data, "base64").subarray(0, 8).toString("hex"), "89504e470d0a1a0a"); + assert.match(native.stderr, /timing snapshot\.capture_(?:hw|sck) /); + } + if (args.include_ax === false) { + assert.doesNotMatch(native.stderr, /timing snapshot\.tree_walk /); + assert.ok(!text(result).includes("Read-back input")); + assert.match(text(result), /indexes invalidated/); + } + return result; +} +try { + const source = fileURLToPath(new URL("./fixtures/MacOSObservationFixture.swift", import.meta.url)); + const build = spawnSync("swiftc", [source, "-o", fixturePath], { encoding: "utf8" }); + assert.equal(build.status, 0, build.stderr); + fixture = spawn(fixturePath, [stateFile], { stdio: ["ignore", "ignore", "ignore"] }); + await waitFor(() => existsSync(stateFile)); + native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env }); + await native.initialize(); + const full = await observe({}); + const oldIndex = indexOf(text(full)); + const queried = JSON.parse(text(await call("query", { role: "AXTextField" }))); + assert.equal(queried.matches.length, 1); + await observe({ include_ax: false }); + await call("set_value", { element_index: oldIndex, value: "bad" }, /unknown element_index/); + await call("set_value", { element_index: queried.matches[0].index, value: "bad" }, /expired|no longer available/); + const ax = await observe({ include_screenshot: false }); + await call("click", { x: 10, y: 10 }, /include_screenshot=false/); + await call("set_value", { element_index: indexOf(text(ax)), value: "native-live-中文🙂" }); + await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "native-live-中文🙂"); + for (const command of [["snapshot", "--include-screenshot", "false", appName], + ["call", "get_app_state", "--args", JSON.stringify({ app: appName, include_screenshot: false })]]) { + const cli = spawnSync(nativePath, command, { env, encoding: "utf8", timeout: 15000 }); + assert.equal(cli.status, 0, cli.stderr); + assert.match(cli.stdout, /include_screenshot=false/); + if (command[0] === "call") assert.equal(images(JSON.parse(cli.stdout)).length, 0); + } + session = new WorkerJavaScriptSession({ native }); + let result = await session.run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false}); + var state = await app.getAXState({includeScreenshot:false});`); + assert.equal(result.isError, false, text(result)); + assert.equal(images(result).length, 0); + result = await session.run("await app.getScreenshot({includeAX:false});"); + assert.equal(result.isError, false, text(result)); + assert.equal(images(result).length, 1); + result = await session.run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false}); + var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:[\\s,]|$)/m)[1]; + await app.setValue(index, "worker-live-中文🙂");`); + assert.equal(result.isError, false, text(result)); + await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "worker-live-中文🙂"); + console.log("macOS real AX/native MCP/CLI/Worker JS: collection gates, PNG, snapshot/query invalidation, coordinate refusal and element recovery passed."); +} finally { + await session?.close(); + native?.close(); + if (fixture) { + fixture.kill("SIGTERM"); + if (fixture.exitCode === null && fixture.signalCode === null) await new Promise(resolve => { + const timer = setTimeout(() => { fixture.kill("SIGKILL"); resolve(); }, 3000); + fixture.once("exit", () => { clearTimeout(timer); resolve(); }); + }); + } + rmSync(directory, { recursive: true, force: true }); +} diff --git a/scripts/tests/windows-observation-js-smoke.mjs b/scripts/tests/windows-observation-js-smoke.mjs new file mode 100644 index 00000000..38bb8d74 --- /dev/null +++ b/scripts/tests/windows-observation-js-smoke.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +// The PowerShell desktop smoke owns this fixture and its lifecycle. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs"; + +const [nativePath, app, stateFile] = process.argv.slice(2); +const native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env: process.env }); +let session; +async function run(code, expectedError) { + const result = await session.run(code, 20000); + const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n"); + assert.equal(!!result.isError, !!expectedError, text); + if (expectedError) assert.match(text, expectedError); + return result; +} +try { + await native.initialize(); + session = new WorkerJavaScriptSession({ native }); + const ax = await run(`var app = await cua.getApp(${JSON.stringify(app)}, {initialState:false}); + var state = await app.getAXState({includeScreenshot:false}); + function inputIndex(state) { + var line = state.split("\\n").find(line => /^\\s*\\d+\\s+edit Observation input(?:\\s|$)/.test(line)); + if(!line) throw Error("Fixture input not found in AX"); + return line.match(/^\\s*(\\d+)/)[1]; + } + var oldIndex = inputIndex(state);`); + assert.equal(ax.content.some(item => item.type === "image"), false); + const screenshot = await run("await app.getScreenshot({includeAX:false});"); + assert.equal(screenshot.content.filter(item => item.type === "image").length, 1); + await run('await app.setValue(oldIndex, "must-not-write");', /unknown element_index/); + await run(`var state = await app.getAXState({includeScreenshot:false,emit:false}); + await app.setValue(inputIndex(state), "worker-live-中文");`); + const deadline = Date.now() + 8000; + while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文") { + assert.ok(Date.now() < deadline, "Real Windows text did not update"); + await new Promise(resolve => setTimeout(resolve, 50)); + } + const full = await run("await app.getAXStateAndScreenshot();"); + assert.equal(full.content.filter(item => item.type === "image").length, 1); + assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文"))); + console.log("Windows Worker JS → native MCP → WPF: channels, stale-index errors, Unicode update and default full observation passed."); +} finally { + await session?.close(); + native.close(); +} diff --git a/scripts/tests/windows-observation-smoke.ps1 b/scripts/tests/windows-observation-smoke.ps1 new file mode 100644 index 00000000..7568382d --- /dev/null +++ b/scripts/tests/windows-observation-smoke.ps1 @@ -0,0 +1,117 @@ +param([Parameter(Mandatory = $true)][string]$NativePath) +$ErrorActionPreference = "Stop" +if ($env:OS -ne "Windows_NT") { throw "Windows desktop required" } +$NativePath = (Resolve-Path -LiteralPath $NativePath).Path +$directory = Join-Path ([System.IO.Path]::GetTempPath()) ("ocu-observation-" + [Guid]::NewGuid()) +[void](New-Item -ItemType Directory -Path $directory) +$stateFile = Join-Path $directory "state.json" +$fixture = $null +$native = $null +$requestId = 0 +$sequence = 0 + +function Assert($condition, $message) { if (-not $condition) { throw $message } } +function Wait-For($test, $description) { + $deadline = [DateTime]::UtcNow.AddSeconds(15) + do { + try { if (& $test) { return } } catch { } + Start-Sleep -Milliseconds 50 + } while ([DateTime]::UtcNow -lt $deadline) + throw "Timeout: $description" +} +function Read-State { [System.IO.File]::ReadAllText($stateFile) | ConvertFrom-Json } +function Send-Request($method, $parameters) { + $script:requestId += 1 + $message = @{ jsonrpc = "2.0"; id = $script:requestId; method = $method; params = $parameters } + $native.StandardInput.WriteLine(($message | ConvertTo-Json -Depth 20 -Compress)) + $native.StandardInput.Flush() + $read = $native.StandardOutput.ReadLineAsync() + Assert ($read.Wait(40000)) "MCP timed out" + Assert (-not [string]::IsNullOrWhiteSpace($read.Result)) "MCP exited" + $response = $read.Result | ConvertFrom-Json + Assert ($response.id -eq $script:requestId -and $null -eq $response.error) "Unexpected RPC response" + return $response.result +} +function Call-Tool($tool, $arguments) { + if (-not $arguments.ContainsKey("app")) { $arguments.app = [string]$fixture.Id } + Send-Request "tools/call" @{ name = $tool; arguments = $arguments } +} +function Text($result) { ($result.content | Where-Object type -eq "text" | ForEach-Object text) -join "`n" } +function Images($result) { @($result.content | Where-Object type -eq "image") } +function Input-Index($result) { + $matches = [regex]::Matches((Text $result), '(?m)^\s*(\d+)\s+edit Observation input(?:\s|$)') + Assert ($matches.Count -eq 1) (Text $result) + $matches[0].Groups[1].Value +} +function Change-Window($kind) { + $script:sequence += 1 + $message = @{ sequence = $script:sequence; kind = $kind } | ConvertTo-Json -Compress + [System.IO.File]::WriteAllText(($stateFile + ".command"), $message, [System.Text.Encoding]::UTF8) + Wait-For { (Read-State).sequence -eq $script:sequence } "fixture window command" + Start-Sleep -Milliseconds 200 +} +try { + $fixtureSource = Join-Path $PSScriptRoot "fixtures/WindowsObservationFixture.ps1" + $fixture = Start-Process powershell.exe -ArgumentList @("-NoProfile", "-STA", "-File", "`"$fixtureSource`"", "-StatePath", "`"$stateFile`"") -PassThru -RedirectStandardOutput (Join-Path $directory "fixture.out") -RedirectStandardError (Join-Path $directory "fixture.err") + Wait-For { Test-Path -LiteralPath $stateFile } "WPF startup" + Wait-For { (Get-Process -Id $fixture.Id).MainWindowHandle -ne [IntPtr]::Zero } "visible WPF main window" + $start = New-Object System.Diagnostics.ProcessStartInfo + $start.FileName = $NativePath + $start.Arguments = "mcp" + $start.UseShellExecute = $false + $start.RedirectStandardInput = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + $start.StandardOutputEncoding = [System.Text.Encoding]::UTF8 + $native = New-Object System.Diagnostics.Process + $native.StartInfo = $start + [void]$native.Start() + $null = Send-Request "initialize" @{ protocolVersion = "2025-03-26"; capabilities = @{}; clientInfo = @{ name = "windows-observation-smoke"; version = "1" } } + $full = Call-Tool "get_app_state" @{} + Assert (-not $full.isError -and (Images $full).Count -eq 1) (Text $full) + Write-Output "Normal WPF window returned a validated PNG." + $oldIndex = Input-Index $full + $image = Call-Tool "get_app_state" @{ include_ax = $false } + Assert (-not $image.isError -and (Images $image).Count -eq 1) (Text $image) + Assert ((Text $image) -match "indexes invalidated" -and (Text $image) -notmatch "Observation input|initial-value") "AX not omitted" + $stale = Call-Tool "set_value" @{ element_index = $oldIndex; value = "must-not-write" } + Assert $stale.isError "Stale index accepted" + $ax = Call-Tool "get_app_state" @{ include_screenshot = $false } + Assert (-not $ax.isError -and (Images $ax).Count -eq 0) (Text $ax) + $coordinates = Call-Tool "click" @{ x = 10; y = 10 } + Assert $coordinates.isError "AX-only coordinate action accepted" + $value = "live-" + [char]0x4E2D + [char]0x6587 + $write = Call-Tool "set_value" @{ element_index = (Input-Index $ax); value = $value } + Assert (-not $write.isError) (Text $write) + Wait-For { (Read-State).text -eq $value } "real UIA value update" + Write-Output "Selective channels, index refusal/recovery and real Unicode text update passed." + foreach ($kind in @("tiny", "minimize", "cloak")) { + Change-Window $kind + if ($kind -eq "tiny") { + $state = Read-State + Assert ($state.width -eq 1 -and $state.height -eq 1) "Window did not become 1x1: $($state.width)x$($state.height)" + } + if ($kind -eq "minimize") { Assert (Read-State).minimized "Window did not minimize" } + if ($kind -eq "cloak") { Assert ((Read-State).cloakHResult -eq 0 -and (Read-State).cloaked -ne 0) "Window did not cloak" } + $unusable = Call-Tool "get_app_state" @{} + Assert ((Images $unusable).Count -eq 0) "$kind window returned an image" + Change-Window "restore" + $restored = Call-Tool "get_app_state" @{} + Assert (-not $restored.isError -and (Images $restored).Count -eq 1) (Text $restored) + Write-Output "$kind window omitted its image; restored window returned a PNG." + } + $cli = & $NativePath snapshot --include-screenshot false ([string]$fixture.Id) + Assert ($LASTEXITCODE -eq 0 -and ($cli -join "`n") -match "include_screenshot=false") "CLI flag not forwarded" + & node (Join-Path $PSScriptRoot "windows-observation-js-smoke.mjs") $NativePath ([string]$fixture.Id) $stateFile + Assert ($LASTEXITCODE -eq 0) "Worker JS desktop smoke failed" + Write-Output "Windows real WPF/native MCP/CLI: channels, stale indexes, AX recovery, coordinate refusal, Unicode text update, 1x1/minimized/cloaked screenshot filtering passed." +} finally { + if ($null -ne $native -and -not $native.HasExited) { + $native.StandardInput.Close() + if (-not $native.WaitForExit(3000)) { $native.Kill(); $native.WaitForExit() } + } + if ($null -ne $fixture -and -not $fixture.HasExited) { $fixture.Kill(); $fixture.WaitForExit() } + $fixtureError = Join-Path $directory "fixture.err" + if (Test-Path -LiteralPath $fixtureError) { Get-Content -LiteralPath $fixtureError | Write-Output } + Remove-Item -LiteralPath $directory -Recurse -Force +} diff --git a/scripts/tests/windows-observation.test.ps1 b/scripts/tests/windows-observation.test.ps1 new file mode 100644 index 00000000..b9e8e3eb --- /dev/null +++ b/scripts/tests/windows-observation.test.ps1 @@ -0,0 +1,67 @@ +param([Parameter(Mandatory = $true)][string]$RuntimePath) +$ErrorActionPreference = "Stop" + +# Execute the production observation functions with instrumented UIA/capture +# dependencies. This can run under pwsh on Linux; it is not a Windows UI test. +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors) +if ($errors.Count -gt 0) { throw ($errors | Out-String) } +$functions = $ast.FindAll({ param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -in @("Resolve-ObservationOptions", "Build-Snapshot") +}, $true) +if ($functions.Count -ne 2) { throw "Production observation functions missing" } +foreach ($function in $functions) { Invoke-Expression $function.Extent.Text } + +$DefaultTextLimit = 500 +$AccessibilityTreeMaxNodeCount = 1200 +$AccessibilityTreeMaxDepth = 64 +function Resolve-App($query) { [pscustomobject]@{ ProcessName = $query; Id = 42; MainWindowTitle = "Fixture" } } +function Get-MainElement($process) { "window" } +function Get-WindowBounds($process, $element) { [pscustomobject]@{ x = 0; y = 0; width = 300; height = 200 } } +function Limit-Text($text, $limit) { $text } +function Render-Tree($element, $bounds, $limit, $nodes, $depth) { + $script:calls.tree += 1 + [pscustomobject]@{ lines = @("Input"); records = @([pscustomobject]@{ index = 7 }) } +} +function Get-FocusedSummary($processId, $limit) { $script:calls.focus += 1; "focus" } +function Get-SelectedText($processId, $limit) { $script:calls.selection += 1; "selection" } +function Capture-WindowPngBase64($bounds) { $script:calls.capture += 1; "png" } +function Assert($condition, $message) { if (-not $condition) { throw $message } } + +foreach ($mode in @( + @{ ax = $true; image = $true }, + @{ ax = $true; image = $false }, + @{ ax = $false; image = $true } +)) { + $script:calls = @{ tree = 0; focus = 0; selection = 0; capture = 0 } + $snapshot = Build-Snapshot "Fixture" -IncludeAX $mode.ax -IncludeScreenshot $mode.image + Assert ($snapshot.app.pid -eq 42 -and $snapshot.windowBounds.width -eq 300) "Window metadata missing" + Assert ($script:calls.tree -eq [int]$mode.ax -and $script:calls.focus -eq [int]$mode.ax -and + $script:calls.selection -eq [int]$mode.ax -and $script:calls.capture -eq [int]$mode.image) "Disabled collection still executed" + if ($mode.ax) { + Assert ($snapshot.elements.Count -eq 1 -and $snapshot.selectedText -eq "selection") "AX missing" + } else { + Assert ($snapshot.elements.Count -eq 0 -and $snapshot.treeLines.Count -eq 0 -and + $null -eq $snapshot.focusedSummary -and $null -eq $snapshot.selectedText) "AX was not omitted" + } + Assert (($snapshot.screenshotPngBase64 -eq "png") -eq $mode.image) "Screenshot mode mismatch" +} + +$defaults = Resolve-ObservationOptions ('{}' | ConvertFrom-Json) +Assert ($defaults.include_ax -and $defaults.include_screenshot) "Defaults changed" +foreach ($payload in @('{"include_ax":false}', '{"include_screenshot":false}')) { + $flags = Resolve-ObservationOptions ($payload | ConvertFrom-Json) + Assert (-not ($flags.include_ax -and $flags.include_screenshot)) "False flag lost" +} +foreach ($payload in @('{"include_ax":"false"}', '{"include_ax":0}', '{"include_screenshot":null}', + '{"include_ax":false,"include_screenshot":false}')) { + $failed = $false + try { $null = Resolve-ObservationOptions ($payload | ConvertFrom-Json) } catch { $failed = $true } + Assert $failed "Invalid observation accepted" +} +$failed = $false +try { $null = Build-Snapshot "Fixture" -IncludeAX $false -IncludeScreenshot $false } catch { $failed = $true } +Assert $failed "Empty snapshot accepted" +Write-Output "PowerShell production functions: parse, defaults, strict booleans and all collection modes passed (instrumented dependencies)." diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.md index 84048c7f..51d2899b 100644 --- a/skills/open-computer-use/references/usage.md +++ b/skills/open-computer-use/references/usage.md @@ -52,6 +52,33 @@ await app.typeText("Hello"); await app.getAXState(); ``` +### Selective observation (opt-in) + +The native `get_app_state` tool on macOS, Windows and Linux accepts `include_ax` and +`include_screenshot`. Both default to `true`; at least one must remain true. +The REPL forwards the corresponding optional `includeAX` / `includeScreenshot` +booleans: + +```js +var editor = await cua.getApp("gedit", { initialState: false }); +await editor.getScreenshot({ includeAX: false }); // no AX, focus or selection walk +await editor.click([100, 100]); // window-relative coordinates +await editor.getAXState({ includeScreenshot: false }); // no capture or PNG encoding +// Use newly observed element indexes only after inspecting the AX result. +``` + +Screenshot-only observation invalidates prior element indexes and cached query +aliases for that app, even if the window title is unchanged. Its result explicitly +reports this. Coordinate actions use the newly observed window bounds; element +actions require a fresh AX observation. Existing calls without these options keep +their full native observation behavior. These flags are implemented on all three native platforms; action read-back is unchanged. + +These options select collection, not image deduplication. An unchanged screenshot +is still returned when requested. macOS/Linux JS actions already disable automatic +read-back, and getAXState does not emit images; includeScreenshot:false additionally skips +native capture on all three platforms. See `docs/selective-observation.md` for real GTK +and Worker JS validation. + The code-first surface also forwards macOS background-operation options: ```js