From 21461a2583c6d9e289e190c8054cc785fb326e9c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:42:19 -0800 Subject: [PATCH 01/12] feat(computer): the website browser's control connection and its record of tabs, downloads, dialogs and launches agentenv's pipe to a website browser now carries events. At each launch its BrowserControl attaches to every page with flattened sessions and records the open tabs in opened order and which is active, the tabs closed and who closed them, downloads with their size and sha256, and JavaScript dialogs, which the new dialogs setting has agentenv accept or dismiss. The app runtime logs each launch with its cause, each stop it made and each exit it didn't, per boot, and the browser serves all of it as browser/v1 record. hidden_page() gives agentenv a page of its own for reading state. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 18 +- .../agentenv_protocol/computer/app_data.py | 2 +- .../agentenv_protocol/computer/app_runtime.py | 50 +++- .../src/agentenv_protocol/website_browser.py | 267 ++++++++++++++++-- .../tests/test_website_browser.py | 170 ++++++++++- src/agent_env/browsers/README.md | 2 +- src/agent_env/browsers/envs/chromium/env.toml | 1 + tst/integration/computer/chromium_app_test.py | 95 ++++++- .../computer/website_browser_test.py | 3 +- 9 files changed, 555 insertions(+), 53 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 828fe8299..af171fe90 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -757,8 +757,10 @@ are the profile's template. The base class adds: - **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, - 1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; and `geolocation` and - `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`. + 1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; `geolocation` and + `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who + answers a page's JavaScript dialogs, `beforeunload` included: `agent` (the default), or agentenv, + accepting (`accept`, a `prompt()` with its default text) or dismissing (`dismiss`) each. - **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, its HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check`, `--lang` and, when `/dev/shm` holds under 1 GiB (Docker gives 64 MiB), @@ -786,8 +788,16 @@ are the profile's template. The base class adds: launches it again, and a launch while it runs, from its menu launcher say, opens another window over the pipe rather than a second copy, which would hand its window to the first and exit. The ready check `devtools_answers` asks `Browser.getVersion` over the pipe. -- **`browser/v1`** on its card, params only: `chromium` (its `version`), `clock: "real"` and, alone, - `window`. +- **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches + to every page with flattened sessions and keeps a record: open tabs in opened order, the one whose page + has focus `active`; `closed_tabs`, each with `closed_at` and `closed_by` (`agentenv`, over this + connection; `browser`, anything else; `relaunch`); `downloads`, into `~/Downloads`, each with its state + and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it. + `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record. +- **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and + `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each + launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made + and each exit it didn't, since the computer booted. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in `~/Downloads`, folders it makes. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 3505055ef..475562fd2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -80,7 +80,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. raise finally: if running: - await self.runtime.launch_app() + await self.runtime.launch_app(op) except Exception as e: if not answer.done(): answer.set_exception(e) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index fc30ec1bf..c0feaad7b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -5,10 +5,12 @@ import asyncio import inspect +import json import os import shlex import time from collections.abc import Awaitable, Callable +from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -80,14 +82,14 @@ async def start(self) -> None: try: if relaunch: await self.stop_app() - await self.launch_app() + await self.launch_app("server_restart" if relaunch else "boot") except Exception as e: self.launch_failed = f"{type(e).__name__}: {e}" - async def launch_app(self) -> int: + async def launch_app(self, cause: str = "launch") -> int: """Start the app from its folder, marked as launched, with the fds the app gives it, whose copies here are - closed once it has them; its pid. With ``launch_running``, an app this server launched that still runs does - that instead of starting a second copy, as a browser opens another window.""" + closed once it has them, and logged with ``cause``; its pid. With ``launch_running``, an app this server + launched that still runs does that instead of starting a second copy, as a browser opens another window.""" if self.launch_running is not None and (running := _running()) is not None: await self.launch_running() return running @@ -104,6 +106,7 @@ async def launch_app(self) -> int: PID.write_text(str(pid)) _mark().parent.mkdir(parents=True, exist_ok=True) _mark().touch() + self.log("launch", pid, cause=cause) return pid async def stop_app(self) -> bool: @@ -115,9 +118,30 @@ async def stop_app(self) -> bool: PID.unlink() if not session.alive(pid): return False + self.log("stop", pid) await session.stop(pid, STOP_TIMEOUT_S) return True + def exited(self) -> None: + """Log that the app this server launched last ended on its own; nothing once the server has stopped it.""" + if (pid := _recorded_pid()) is not None: + self.log("exit", pid) + + def log(self, event: str, pid: int, **fields: str) -> None: + """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server until the computer + reboots.""" + entry = {"at": datetime.now(timezone.utc).isoformat(), "event": event, "pid": pid, **fields} + _log().parent.mkdir(parents=True, exist_ok=True) + with _log().open("a", encoding="utf-8") as log: + log.write(json.dumps(entry) + "\n") + + def launches(self) -> list[dict]: + """Every launch, stop and exit of the app since the computer booted, in order.""" + try: + return [json.loads(line) for line in _log().read_text(encoding="utf-8").splitlines() if line] + except FileNotFoundError: + return [] + async def readiness(self) -> AppReadiness: ready = self.launch.ready if self.launch is not None else None checks: list[tuple[str, Awaitable[Optional[str]]]] = [] @@ -154,6 +178,17 @@ def _mark() -> Path: return LAUNCHED / Path.cwd().name +def _log() -> Path: + return LAUNCHED / f"{Path.cwd().name}.launches" + + +def _recorded_pid() -> Optional[int]: + try: + return int(PID.read_text()) + except (FileNotFoundError, ValueError): + return None + + def _launched_at() -> Optional[float]: """When the app was last launched since the computer booted, by its mark.""" try: @@ -164,11 +199,8 @@ def _launched_at() -> Optional[float]: def _running() -> Optional[int]: """The pid of the app this server launched last, while it runs.""" - try: - pid = int(PID.read_text()) - except (FileNotFoundError, ValueError): - return None - return pid if session.alive(pid) else None + pid = _recorded_pid() + return pid if pid is not None and session.alive(pid) else None async def _bounded(name: str, check: Awaitable[Optional[str]]) -> Optional[str]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 8d7c354ee..c541a0291 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -7,19 +7,22 @@ from __future__ import annotations import asyncio +import contextlib +import hashlib import itertools import json import logging import os import shutil +from collections.abc import AsyncIterator +from datetime import datetime, timezone from pathlib import Path from typing import Any, Awaitable, Callable, ClassVar, Literal, Optional from pydantic import BaseModel, Field, field_validator -from .agent_env_environment import agent_skill +from .agent_env_environment import agent_skill, extension from .computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check -from .types import EnvironmentExtension logger = logging.getLogger(__name__) @@ -36,6 +39,12 @@ SHM_MIN_BYTES = 1 << 30 #: Chromium's content setting that denies a permission without a prompt. BLOCK = 2 +#: Who closed a tab: agentenv over its own connection, anything else in the browser, or the browser's relaunch. +AGENTENV, BROWSER_CLOSED, RELAUNCH = "agentenv", "browser", "relaunch" +#: Who answered a dialog when agentenv didn't. +AGENT = "agent" +#: The targets agentenv's connection follows. +PAGES = [{"type": "page"}] #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -77,7 +86,8 @@ class DevToolsError(RuntimeError): class DevToolsPipe: """A CDP connection over a Chromium's debugging pipe, each message JSON ended by a NUL. It connects anew at every - launch; a call with no browser connected, or whose browser exits before answering, raises ConnectionError.""" + launch; a call with no browser connected, or whose browser exits before answering, raises ConnectionError. Each + event, a message with no ``id``, goes to every listener, and each closer hears when the browser closes the pipe.""" #: How long a call waits for the browser's answer when it names no wait of its own. call_timeout_s: ClassVar[float] = 30.0 @@ -87,6 +97,16 @@ def __init__(self) -> None: self._waiting: dict[int, asyncio.Future] = {} self._writer: Optional[asyncio.WriteTransport] = None self._reading: Optional[asyncio.Task] = None + self._listeners: list[Callable[[dict], None]] = [] + self._closers: list[Callable[[], None]] = [] + + def listen(self, listener: Callable[[dict], None]) -> None: + """Hand ``listener`` each event: its ``method``, ``params`` and, on a page's session, ``sessionId``.""" + self._listeners.append(listener) + + def on_close(self, closer: Callable[[], None]) -> None: + """Call ``closer`` when the browser closes its end of the pipe, not when this end closes it.""" + self._closers.append(closer) async def connect(self, to_browser: int, from_browser: int) -> None: """Take over this end of a new pipe, ending any connection to a browser launched before.""" @@ -128,6 +148,10 @@ async def _read(self, reader: asyncio.StreamReader) -> None: try: while True: message = json.loads((await reader.readuntil(b"\0"))[:-1]) + if "id" not in message: + for listener in self._listeners: + _heard(listener, message) + continue answer = self._waiting.get(message.get("id")) if answer is not None and not answer.done(): answer.set_result(message) @@ -135,6 +159,8 @@ async def _read(self, reader: asyncio.StreamReader) -> None: if self._writer is not None: self._writer.close() self._fail_waiting() + for closer in self._closers: + _heard(closer) def _fail_waiting(self) -> None: for answer in self._waiting.values(): @@ -142,6 +168,181 @@ def _fail_waiting(self) -> None: answer.set_exception(ConnectionError("the browser closed its debugging pipe")) +def _heard(callback: Callable, *args: Any) -> None: + try: + callback(*args) + except Exception: + logger.exception("a DevToolsPipe listener failed") + +class BrowserControl: + """agentenv's control of a browser over its pipe: at each launch it attaches to every page with flattened sessions + and keeps a record of the browser, its open tabs in opened order, those closed and who closed them, its downloads + with their size and sha256, and its JavaScript dialogs, which it answers when ``dialogs`` says to.""" + + def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> None: + self.devtools, self.downloads_dir, self.dialogs = devtools, downloads, dialogs + self.tabs: dict[str, dict] = {} + self.closed_tabs: list[dict] = [] + self.downloads: dict[str, dict] = {} + self.dialog_log: list[dict] = [] + self._sessions: dict[str, str] = {} + self._hidden: set[str] = set() + self._closing: set[str] = set() + self._open_dialogs: dict[str, dict] = {} + self._answering: set[str] = set() + self._tasks: set[asyncio.Future] = set() + devtools.listen(self._event) + + async def attach(self) -> None: + """Follow a browser just launched: the tabs of the one before closed by its relaunch, and every page this one + has or opens attached, its downloads going to ``downloads_dir`` with their events.""" + now = _now() + self.closed_tabs += [{**tab, "closed_at": now, "closed_by": RELAUNCH} for tab in self.tabs.values()] + self.tabs.clear() + self._sessions.clear() + self._open_dialogs.clear() + await self.devtools.call("Target.setDiscoverTargets", {"discover": True, "filter": PAGES}) + await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, + "flatten": True, "filter": PAGES}) + await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "allow", "eventsEnabled": True, + "downloadPath": str(self.downloads_dir)}) + + async def close_tab(self, target: str) -> None: + """Close the tab ``target``, which the record says agentenv closed.""" + self._closing.add(target) + await self.devtools.call("Target.closeTarget", {"targetId": target}) + + @contextlib.asynccontextmanager + async def hidden_page(self, url: str = "about:blank") -> AsyncIterator[str]: + """A page of agentenv's own for reading state, in no window and never in the record; its session's id. It's + closed after.""" + target = (await self.devtools.call("Target.createTarget", + {"url": url, "hidden": True, "background": True}))["targetId"] + self._hidden.add(target) + self.tabs.pop(target, None) + try: + yield (await self.devtools.call("Target.attachToTarget", {"targetId": target, "flatten": True}))["sessionId"] + finally: + with contextlib.suppress(ConnectionError, DevToolsError): + await self.devtools.call("Target.closeTarget", {"targetId": target}) + + async def active_tab(self) -> Optional[str]: + """The tab whose page has focus, else the first visible one; CDP has no event for a tab made active.""" + visible = None + for session, target in list(self._sessions.items()): + if target not in self.tabs: + continue + try: + state = (await self.devtools.call( + "Runtime.evaluate", {"expression": "[document.visibilityState, document.hasFocus()]", + "returnByValue": True}, session_id=session, timeout_s=2))["result"]["value"] + except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError): + continue + if state[1]: + return target + if state[0] == "visible" and visible is None: + visible = target + return visible + + async def record(self) -> dict: + """The record: open ``tabs``, each with whether it's ``active``, ``closed_tabs``, ``downloads`` and + ``dialogs``.""" + active = await self.active_tab() + return {"tabs": [{**tab, "active": tab["id"] == active} for tab in self.tabs.values()], + "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()), + "dialogs": list(self.dialog_log)} + + def _event(self, message: dict) -> None: + handler = getattr(self, "_on_" + message.get("method", "").replace(".", "_"), None) + if handler is not None: + handler(message.get("params", {}), message.get("sessionId")) + + def _on_Target_targetCreated(self, params: dict, session: Optional[str]) -> None: + info = params["targetInfo"] + if info.get("type") == "page" and info["targetId"] not in self._hidden and info["targetId"] not in self.tabs: + self.tabs[info["targetId"]] = {"id": info["targetId"], "url": info.get("url", ""), + "title": info.get("title", ""), "opened_at": _now()} + + def _on_Target_targetInfoChanged(self, params: dict, session: Optional[str]) -> None: + info = params["targetInfo"] + if (tab := self.tabs.get(info["targetId"])) is not None: + tab.update(url=info.get("url", tab["url"]), title=info.get("title", tab["title"])) + + def _on_Target_targetDestroyed(self, params: dict, session: Optional[str]) -> None: + target = params["targetId"] + if (tab := self.tabs.pop(target, None)) is not None: + self.closed_tabs.append({**tab, "closed_at": _now(), + "closed_by": AGENTENV if target in self._closing else BROWSER_CLOSED}) + self._closing.discard(target) + self._hidden.discard(target) + + def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> None: + if params["targetInfo"].get("type") == "page": + self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"] + self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"])) + + def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: + self._sessions.pop(params["sessionId"], None) + + def _on_Page_javascriptDialogOpening(self, params: dict, session: Optional[str]) -> None: + dialog = {"tab": self._sessions.get(session), "type": params.get("type"), "message": params.get("message", ""), + "url": params.get("url", ""), "opened_at": _now(), "answer": None, "by": None} + self.dialog_log.append(dialog) + self._open_dialogs[session] = dialog + if self.dialogs != AGENT: + self._answering.add(session) + self._spawn(self.devtools.call("Page.handleJavaScriptDialog", { + "accept": self.dialogs == "accept", "promptText": params.get("defaultPrompt", "")}, session_id=session)) + + def _on_Page_javascriptDialogClosed(self, params: dict, session: Optional[str]) -> None: + if (dialog := self._open_dialogs.pop(session, None)) is not None: + dialog.update(answer="accepted" if params.get("result") else "dismissed", answered_at=_now(), + by=AGENTENV if session in self._answering else AGENT, + **({"input": params["userInput"]} if params.get("userInput") else {})) + self._answering.discard(session) + + def _on_Browser_downloadWillBegin(self, params: dict, session: Optional[str]) -> None: + self.downloads[params["guid"]] = {"guid": params["guid"], "url": params.get("url", ""), + "filename": params.get("suggestedFilename", ""), "state": "inProgress", + "started_at": _now()} + + def _on_Browser_downloadProgress(self, params: dict, session: Optional[str]) -> None: + if (download := self.downloads.get(params["guid"])) is None: + return + download["state"] = params["state"] + if params["state"] == "completed": + path = params.get("filePath") or str(self.downloads_dir / download["filename"]) + self._spawn(self._hashed(download, Path(path))) + elif params["state"] == "canceled": + download["ended_at"] = _now() + + async def _hashed(self, download: dict, path: Path) -> None: + size, sha256 = await asyncio.to_thread(_size_and_sha256, path) + download.update(path=str(path), size=size, sha256=sha256, ended_at=_now()) + + def _spawn(self, work: Awaitable) -> None: + task = asyncio.ensure_future(work) + self._tasks.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task: asyncio.Future) -> None: + self._tasks.discard(task) + if not task.cancelled() and (error := task.exception()) is not None: + logger.warning(f"the browser's control connection: {type(error).__name__}: {error}") + + +def _size_and_sha256(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + with path.open("rb") as file: + while chunk := file.read(1 << 20): + digest.update(chunk) + return path.stat().st_size, digest.hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + class Window(BaseModel): width: int = Field(1280, gt=0) height: int = Field(800, gt=0) @@ -160,6 +361,8 @@ class BrowserSettings(BaseModel): #: Whether a page asking for the location, or to show notifications, is denied without a prompt or asks. geolocation: Literal["deny", "ask"] = "deny" notifications: Literal["deny", "ask"] = "deny" + #: Who answers a page's JavaScript dialogs: the agent, or agentenv accepting or dismissing each. + dialogs: Literal["agent", "accept", "dismiss"] = "agent" @field_validator("start_urls") @classmethod @@ -186,10 +389,13 @@ class WebsiteBrowserEnvironment(ComputerAppEnvironment): def __init__(self) -> None: #: Its CDP connection, over the pipe its server launches it with. self.devtools = DevToolsPipe() - self._start_tabs: Optional[asyncio.Future] = None + self._following: Optional[asyncio.Future] = None super().__init__() for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) + #: agentenv's control of it over that connection, and the record it keeps. + self.control = BrowserControl(self.devtools, Path.home() / DOWNLOADS, self.settings.dialogs) + self.devtools.on_close(self._computer_app.exited) @classmethod def declaration(cls) -> AppDeclaration: @@ -250,19 +456,23 @@ def _command_line_tabs(self) -> list[str]: async def _launch_fds(self) -> dict[int, int]: """Before each launch, an empty cache and the deploy's prompt answers in its profile; and a new debugging pipe: - the browser's ends as its fds 3 and 4, this server's ends its CDP connection, over which the start tabs its - command line can't open are opened.""" + the browser's ends as its fds 3 and 4, this server's ends its CDP connection, which follows the browser and + then opens the start tabs its command line can't.""" shutil.rmtree(self.cache_dir(), ignore_errors=True) self._write_prompt_answers() to_browser_read, to_browser_write = os.pipe() from_browser_read, from_browser_write = os.pipe() await self.devtools.connect(to_browser_write, from_browser_read) - rest = self.settings.start_urls[len(self._command_line_tabs()):] - self._start_tabs = asyncio.ensure_future(self._open_tabs(rest)) if rest else None - if self._start_tabs is not None: - self._start_tabs.add_done_callback(lambda done: done.cancelled() or done.exception()) + self._following = asyncio.ensure_future(self._follow(self.settings.start_urls[len(self._command_line_tabs()):])) + self._following.add_done_callback(lambda done: done.cancelled() or done.exception()) return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} + async def _follow(self, urls: list[str]) -> None: + """Its control attached, then each of ``urls`` in a tab behind the active one, in order.""" + await self.control.attach() + for url in urls: + await self.devtools.call("Target.createTarget", {"url": url, "background": True}) + def _write_prompt_answers(self) -> None: """Its profile's preferences, which it reads as it starts, denying location and notification prompts or leaving them to ask, as the deploy's settings say.""" @@ -281,11 +491,6 @@ def _write_prompt_answers(self) -> None: preferences.parent.mkdir(parents=True, exist_ok=True) preferences.write_text(json.dumps(current), encoding="utf-8") - async def _open_tabs(self, urls: list[str]) -> None: - """Each of ``urls`` in a tab behind the active one, in order.""" - for url in urls: - await self.devtools.call("Target.createTarget", {"url": url, "background": True}) - def _relaunch_on_start(self) -> bool: return True @@ -306,22 +511,30 @@ def write_policies(self) -> None: @ready_check async def devtools_answers(self) -> bool: - """It answers over its pipe and has opened its start tabs.""" + """It answers over its pipe, its control follows it and it has opened its start tabs.""" try: await self.devtools.call("Browser.getVersion", timeout_s=self.devtools_timeout_s) except (ConnectionError, DevToolsError, asyncio.TimeoutError): return False - tabs = self._start_tabs - return tabs is None or (tabs.done() and not tabs.cancelled() and tabs.exception() is None) - - def declared_extensions(self) -> list[EnvironmentExtension]: - params: dict[str, Any] = {"chromium": self.version, "clock": "real"} - if _alone(): - params["window"] = self.settings.window.model_dump() - browser = EnvironmentExtension( - uri=BROWSER, params=params, - description="A website browser agentenv runs: its build, the clock its pages see and, alone, its window.") - return [*super().declared_extensions(), browser] + following = self._following + return following is not None and following.done() and not following.cancelled() and \ + following.exception() is None + + @extension(BROWSER, op="record", method="GET", description=( + "A website browser agentenv runs: its build, the clock its pages see and, alone, its window. record returns " + "what agentenv's own connection to it saw: its open tabs, each with whether it's active, the tabs closed and " + "who closed them, its downloads with their size and sha256, its JavaScript dialogs and who answered each, and " + "each launch, stop and exit of it since the computer booted, with its cause.")) + async def browser_record(self) -> dict: + return {**await self.control.record(), "launches": self._computer_app.launches()} + + def extension_params(self, uri: str) -> dict: + if uri == BROWSER: + params: dict[str, Any] = {"chromium": self.version, "clock": "real"} + if _alone(): + params["window"] = self.settings.window.model_dump() + return params + return super().extension_params(uri) @agent_skill def browser_skill(self) -> tuple[str, str]: diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index a4470a2ad..1b27bf8c0 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio +import hashlib import json import os import sys @@ -18,7 +19,7 @@ from agentenv_protocol.computer import app_runtime from agentenv_protocol.computer.platform import Platform from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, - POLICIES, SITE_PROXY_ENV, SKILL, DevToolsError, + POLICIES, SITE_PROXY_ENV, SKILL, BrowserControl, DevToolsError, WebsiteBrowserEnvironment) PROXY = "http://127.0.0.1:3128" @@ -36,6 +37,11 @@ message = json.loads(raw) if message["method"] == "Browser.getVersion": answer = {"result": {"product": "Fake/1"}} + elif message["method"] in ("Target.setDiscoverTargets", "Target.setAutoAttach", "Browser.setDownloadBehavior"): + answer = {"result": {}} + if message["method"] == "Target.setDiscoverTargets": + created = {"targetInfo": {"targetId": "start", "type": "page", "url": "about:blank", "title": ""}} + os.write(4, json.dumps({"method": "Target.targetCreated", "params": created}).encode() + b"\\0") elif message["method"] == "Target.createTarget": with open("tabs.txt", "a") as tabs: tabs.write(("window " if message["params"].get("newWindow") else "") + message["params"]["url"] + "\\n") @@ -269,11 +275,11 @@ async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_rea version = await browser.devtools.call("Browser.getVersion") with pytest.raises(DevToolsError, match="unknown method"): await browser.devtools.call("Page.navigate", {"url": "about:blank"}) - launched = await browser.devtools_answers() + launched = await _answering(browser) await runtime.stop_app() stopped = await browser.devtools_answers() await runtime.launch_app() - relaunched = await browser.devtools_answers() + relaunched = await _answering(browser) await runtime.stop_app() browser.devtools.close() @@ -294,7 +300,7 @@ async def test_a_launch_while_it_runs_opens_a_window_over_its_pipe_and_starts_no (browser_home / ".cache/chromium").mkdir(parents=True, exist_ok=True) (browser_home / ".cache/chromium/kept").touch() second = await runtime.launch_app() - answers = await browser.devtools_answers() + answers = await _answering(browser) await runtime.stop_app() browser.devtools.close() @@ -391,13 +397,13 @@ def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): - in_computer = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + in_computer = Browser().extension_params(BROWSER) monkeypatch.setenv(ALONE_ENV, "1") monkeypatch.setenv(SITE_PROXY_ENV, PROXY) - alone = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + alone = Browser().extension_params(BROWSER) - assert in_computer.params == {"chromium": "156.0.8078.4", "clock": "real"} - assert alone.params == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}} + assert in_computer == {"chromium": "156.0.8078.4", "clock": "real"} + assert alone == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}} @pytest.mark.asyncio @@ -420,3 +426,151 @@ def test_an_app_that_isnt_a_browser_launches_and_declares_as_before(): assert stored["os"]["linux"]["launch"] == {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True, "timeout_s": 60.0}} + +async def _answering(browser) -> bool: + """Whether ``browser``'s ready check says it answers, within five seconds.""" + for _ in range(100): + if await browser.devtools_answers(): + return True + await asyncio.sleep(0.05) + return False + + +class StubDevTools: + """A browser connection that answers each call from ``answers`` by method, records it, and hands the control the + events a test sends.""" + + def __init__(self, answers=None): + self.calls, self.answers, self.listener = [], answers or {}, None + + def listen(self, listener): + self.listener = listener + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + self.calls.append((method, params or {}, session_id)) + answer = self.answers.get(method, {}) + return answer(params, session_id) if callable(answer) else answer + + def send(self, method, params, session=None): + self.listener({"method": method, "params": params, **({"sessionId": session} if session else {})}) + + +def _page(target, url="about:blank", title=""): + return {"targetInfo": {"targetId": target, "type": "page", "url": url, "title": title}} + + +@pytest.mark.asyncio +async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_leaves_out_its_hidden_pages(tmp_path): + devtools = StubDevTools({"Target.createTarget": {"targetId": "hidden"}, + "Target.attachToTarget": {"sessionId": "s-hidden"}}) + control = BrowserControl(devtools, tmp_path, "agent") + await control.attach() + + for target in ("a", "b", "c"): + devtools.send("Target.targetCreated", _page(target)) + devtools.send("Target.targetInfoChanged", _page("a", "https://slack.com/", "Slack")) + devtools.send("Target.targetDestroyed", {"targetId": "b"}) + await control.close_tab("c") + devtools.send("Target.targetDestroyed", {"targetId": "c"}) + async with control.hidden_page() as session: + devtools.send("Target.targetCreated", _page("hidden")) + record = await control.record() + await control.attach() + + assert [tab["id"] for tab in record["tabs"]] == ["a"] and record["tabs"][0]["title"] == "Slack" + assert [(tab["id"], tab["closed_by"]) for tab in record["closed_tabs"]] == [("b", "browser"), ("c", "agentenv")] + assert session == "s-hidden" and ("Target.closeTarget", {"targetId": "hidden"}, None) in devtools.calls + assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs][-1] == ("a", "relaunch") + assert [call[0] for call in devtools.calls[:3]] == ["Target.setDiscoverTargets", "Target.setAutoAttach", + "Browser.setDownloadBehavior"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dialogs, sent, answer, by", [ + ("accept", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"), + ("dismiss", {"accept": False, "promptText": "yes"}, "dismissed", "agentenv"), + ("agent", None, "dismissed", "agent"), +]) +async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_says_and_records_who_did( + tmp_path, dialogs, sent, answer, by): + devtools = StubDevTools() + control = BrowserControl(devtools, tmp_path, dialogs) + devtools.send("Target.targetCreated", _page("a")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a")}) + + devtools.send("Page.javascriptDialogOpening", {"type": "prompt", "message": "Sure?", "url": "https://a.test/", + "defaultPrompt": "yes"}, "s-a") + await asyncio.sleep(0) + devtools.send("Page.javascriptDialogClosed", {"result": answer == "accepted"}, "s-a") + + assert ("Page.enable", {}, "s-a") in devtools.calls + handled = [(params, session) for method, params, session in devtools.calls + if method == "Page.handleJavaScriptDialog"] + assert handled == ([(sent, "s-a")] if sent else []) + (dialog,) = control.dialog_log + assert (dialog["tab"], dialog["type"], dialog["message"], dialog["answer"], dialog["by"]) == ( + "a", "prompt", "Sure?", answer, by) + + +@pytest.mark.asyncio +async def test_its_control_records_a_completed_download_with_its_size_and_sha256(tmp_path): + devtools = StubDevTools() + control = BrowserControl(devtools, tmp_path, "agent") + (tmp_path / "report.pdf").write_bytes(b"%PDF-1.7 report") + + devtools.send("Browser.downloadWillBegin", {"guid": "g", "url": "https://a.test/report.pdf", + "suggestedFilename": "report.pdf"}) + devtools.send("Browser.downloadProgress", {"guid": "g", "state": "inProgress", "receivedBytes": 3}) + in_progress = dict(control.downloads["g"]) + devtools.send("Browser.downloadProgress", {"guid": "g", "state": "completed"}) + for _ in range(50): + if "sha256" in control.downloads["g"]: + break + await asyncio.sleep(0.01) + + done = control.downloads["g"] + assert in_progress["state"] == "inProgress" and "sha256" not in in_progress + assert (done["state"], done["size"], done["path"]) == ("completed", 15, str(tmp_path / "report.pdf")) + assert done["sha256"] == hashlib.sha256(b"%PDF-1.7 report").hexdigest() + + +@pytest.mark.asyncio +async def test_its_control_says_the_tab_whose_page_has_focus_is_active_else_the_first_visible_one(tmp_path): + states = {"s-a": ["hidden", False], "s-b": ["visible", False], "s-c": ["visible", True]} + devtools = StubDevTools({"Runtime.evaluate": lambda params, session: {"result": {"value": states[session]}}}) + control = BrowserControl(devtools, tmp_path, "agent") + for target in ("a", "b", "c"): + devtools.send("Target.targetCreated", _page(target)) + devtools.send("Target.attachedToTarget", {"sessionId": f"s-{target}", **_page(target)}) + + focused = await control.active_tab() + states["s-c"] = ["hidden", False] + visible = await control.active_tab() + + assert (focused, visible) == ("c", "b") + + +@pytest.mark.asyncio +async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_exit_it_didnt_ask_for(monkeypatch, + tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + browser = FakeBrowser() + runtime = browser._computer_app + + first = await runtime.launch_app("boot") + await _answering(browser) + record = await browser.browser_record() + await runtime.stop_app() + second = await runtime.launch_app("data/reset") + await _answering(browser) + os.killpg(second, 9) + for _ in range(50): + if any(entry["event"] == "exit" for entry in runtime.launches()): + break + await asyncio.sleep(0.05) + browser.devtools.close() + + assert [tab["id"] for tab in record["tabs"]] == ["start"] + assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in runtime.launches()] == [ + ("launch", "boot", first), ("stop", None, first), ("launch", "data/reset", second), ("exit", None, second)] diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 18e3e0901..43bb3783d 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -8,7 +8,7 @@ build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists i flags, and writes managed policies to `/etc/opt/chrome_for_testing/policies/managed/` for unattended use. It runs without Chromium's sandbox, which a container can't give it. Its env.toml spells out its settings and their defaults under `[settings]`: `start_urls`, opened at boot, the first tab active; `window`, alone only; `locale`; `time_zone`; and -`geolocation` and `notifications`, `deny` or `ask`. A computer env sets its own beside it +`geolocation` and `notifications`, `deny` or `ask`; and `dialogs`, `agent`, `accept` or `dismiss`. A computer env sets its own beside it (`apps = [{ env = "chromium", settings = { start_urls = ["https://example.test/"] } }]`), and a deploy overrides any of them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which the image has none of, so each deploy and each reset starts with a clean one. Name it from another bundle as `chromium`, diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 520321950..8a3a20ae1 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -10,3 +10,4 @@ locale = "en-US" time_zone = "UTC" geolocation = "deny" notifications = "deny" +dialogs = "agent" diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index f0e121f60..9464c17ab 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -5,14 +5,18 @@ website browser it runs Chrome itself with agentenv's flags on its debugging pipe, keeps its shared memory in files on Docker's 64 MiB /dev/shm, answers location prompts and leaves notification ones to ask as the deploy says, and its menu launcher, run while it runs, opens another window of it with no second copy, its cache kept and it still ready; and the -desktop that gives it French defaults deploys with them, a deploy's time zone over them, each recorded. Needs Docker and -the local registry on localhost:5000.""" +desktop that gives it French defaults deploys with them, a deploy's time zone over them, each recorded; and the record +its control connection keeps lists a tab an agent opened and closed, a download with its size and sha256, a confirm() +accepted for it, and each launch, exit and stop with its cause, across a kill and a restart of its server. Needs Docker +and the local registry on localhost:5000.""" import asyncio +import hashlib import json import platform import re +import httpx import pytest from mcp import ClientSession from mcp.client.streamable_http import streamablehttp_client @@ -21,6 +25,7 @@ from agent_env.env import Env from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready from agent_env.env.envs.computer.utils import inside from agent_env.store.routing import namespace_routing @@ -54,6 +59,32 @@ def log_message(self, *args): threading.Thread(target=server.serve_forever, daemon=True).start() time.sleep(float(sys.argv[1])) """ +RECORDED = "http://127.0.0.1:8767/" +# Serves, on loopback for the seconds it's given, a page that asks to confirm and then downloads report.bin. +SERVE_RECORDED = """import http.server, sys, threading, time +REPORT = b"agentenv report\\n" * 64 +class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/report.bin": + body, kind = REPORT, "application/octet-stream" + else: + body = (b"asking") + kind = "text/html" + self.send_response(200) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + if self.path == "/report.bin": + self.send_header("Content-Disposition", "attachment; filename=report.bin") + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = http.server.ThreadingHTTPServer(("127.0.0.1", 8767), Page) +threading.Thread(target=server.serve_forever, daemon=True).start() +time.sleep(float(sys.argv[1])) +""" +REPORT_SHA256 = hashlib.sha256(b"agentenv report\n" * 64).hexdigest() # Each window the window manager manages, its class and its title. WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') @@ -149,6 +180,66 @@ async def test_a_desktop_that_gives_it_defaults_deploys_with_them_and_a_deploys_ assert not report.failed +@pytest.mark.asyncio +async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with_its_cause(chromium_desktop): + settings = {"start_urls": [RECORDED], "dialogs": "accept"} + with namespace_routing(): + deployed = await Env.get(chromium_desktop.id).deploy(env_params={"app_settings": {"chromium": settings}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + card = await _app(deployed, WELL_KNOWN_PATH, "chromium") + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "45"], + timeout_s=75)) + await asyncio.sleep(3) + await _keys(deployed, ["F5"]) + accepted = await _chromium_windows(deployed, "accepted") + await _keys(deployed, ["Control", "t"]) + await asyncio.sleep(2) + await _keys(deployed, ["Control", "w"]) + await serving + recorded = await _record(deployed, card, lambda r: r["downloads"] and "sha256" in r["downloads"][0] + and r["closed_tabs"]) + browser = await _browser_process(deployed) + await inside.run(deployed, ["kill", "-9", str(browser["pid"])], timeout_s=30) + exited = await _record(deployed, card, lambda r: r["launches"][-1]["event"] == "exit") + await inside.run(deployed, ["sh", "-c", LAUNCHER], timeout_s=60) + relaunched = await _ready(deployed, card, "chromium") + await inside.run(deployed, ["pkill", "-f", "computer.serve_app chromium.py"], timeout_s=30) + restarted = await _record(deployed, card, lambda r: r["launches"][-1].get("cause") == "server_restart") + finally: + report = await teardown_run(context) + + assert accepted[0].startswith("accepted") + (dialog,) = recorded["dialogs"] + assert (dialog["type"], dialog["message"], dialog["answer"], dialog["by"]) == ("confirm", "Proceed?", "accepted", + "agentenv") + (download,) = recorded["downloads"] + assert (download["filename"], download["state"], download["size"], download["sha256"]) == ( + "report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256) + assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["browser"] + assert [tab["url"] for tab in recorded["tabs"]] == [RECORDED] and recorded["tabs"][0]["active"] is True + assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"] + assert exited["launches"][0]["cause"] == "boot" and relaunched["ready"] is True, relaunched + events = [(entry["event"], entry.get("cause")) for entry in restarted["launches"]] + assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "launch")] + assert events[3:] in ([("stop", None), ("launch", "server_restart")], [("launch", "server_restart")]), events + assert not report.failed + + +async def _record(deployed, card: dict, until) -> dict: + """The browser's record, once ``until`` says it has what's waited for, or as it last was within a minute.""" + path = protocol_v1.find_extension_method(card, BROWSER, "record")["endpoint"] + for _ in range(60): + try: + record = await _app(deployed, path, "chromium") + if until(record): + return record + except httpx.HTTPError: + pass + await asyncio.sleep(1) + return record + + async def _tab_to(deployed, title: str): """Control+Tab pressed as an agent presses it, again while the window still shows the tab before, which a press can reach before the next tab opens; the last press's result, and the window titles once one starts with diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index 9f22a7c58..020421a40 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -127,7 +127,8 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun assert zone.startswith("Europe/Paris") and loopback.startswith("Loopback page") assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( policies["stdout"]).items() - assert protocol_v1.extension_params(card, BROWSER) == {"chromium": "156.0.8078.4", "clock": "real"} + params = protocol_v1.extension_params(card, BROWSER) + assert (params["chromium"], params["clock"], list(params["methods"])) == ("156.0.8078.4", "real", ["record"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness From d8837bcb295f9061bc1a8f7f32b9f19a1fe22f38 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:01:26 -0800 Subject: [PATCH 02/12] fix(computer): close a gone browser's tabs when its pipe closes, log data causes as data/add and data/reset The record kept a killed browser's tabs open until the next launch; they now close as soon as its pipe does, by exit or by its server's stop. A launch after a data call logs the public data/add or data/reset, the launch log moves into the app's .agentenv folder beside its pid, and the active-tab check's wait and the hash's chunk size are named. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 3 +- .../agentenv_protocol/computer/app_data.py | 6 ++- .../agentenv_protocol/computer/app_runtime.py | 19 +++++---- .../src/agentenv_protocol/website_browser.py | 39 +++++++++++++------ .../tests/test_website_browser.py | 5 ++- tst/integration/computer/chromium_app_test.py | 16 ++++---- 6 files changed, 56 insertions(+), 32 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index af171fe90..39a791ef1 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -791,7 +791,8 @@ are the profile's template. The base class adds: - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches to every page with flattened sessions and keeps a record: open tabs in opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and `closed_by` (`agentenv`, over this - connection; `browser`, anything else; `relaunch`); `downloads`, into `~/Downloads`, each with its state + connection; `browser`, anything else; `exit` or `stop`, when the browser ended on its own or its server + stopped it); `downloads`, into `~/Downloads`, each with its state and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it. `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record. - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 475562fd2..c1ab84f2d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -15,7 +15,7 @@ from ..agent_env_environment import OP_ADD_DATA, OP_GET_DATA, OP_RESET_DATA from ..transfers import WriteNamespaceGrant -from ..types import FilePart, uploaded_object_path +from ..types import METHOD_ADD, METHOD_RESET, FilePart, uploaded_object_path from .dataplane import file_parts, part_bytes if TYPE_CHECKING: @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) +#: The cause a launch after each data call logs. +_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_RESET_DATA: METHOD_RESET} #: Where an app's server keeps its paths as installed, in the app's folder. INITIAL = Path(".agentenv/initial") @@ -80,7 +82,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. raise finally: if running: - await self.runtime.launch_app(op) + await self.runtime.launch_app(_CAUSES[op]) except Exception as e: if not answer.done(): answer.set_exception(e) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index c0feaad7b..e380b2934 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -28,6 +28,9 @@ #: Where an app's server keeps the pid of the app it launched last, in the app's folder, so a server that restarts can #: still stop it. PID = Path(".agentenv/app.pid") +#: Where an app's server logs each launch, stop and exit of the app, beside its pid; a boot starts the app's folder as +#: its image has it, so the log is the boot's. +LAUNCHES = Path(".agentenv/launches") #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 @@ -122,14 +125,16 @@ async def stop_app(self) -> bool: await session.stop(pid, STOP_TIMEOUT_S) return True - def exited(self) -> None: - """Log that the app this server launched last ended on its own; nothing once the server has stopped it.""" - if (pid := _recorded_pid()) is not None: - self.log("exit", pid) + def exited(self) -> bool: + """Log that the app this server launched last ended on its own, unless the server stopped it; whether it + did.""" + if (pid := _recorded_pid()) is None: + return False + self.log("exit", pid) + return True def log(self, event: str, pid: int, **fields: str) -> None: - """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server until the computer - reboots.""" + """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" entry = {"at": datetime.now(timezone.utc).isoformat(), "event": event, "pid": pid, **fields} _log().parent.mkdir(parents=True, exist_ok=True) with _log().open("a", encoding="utf-8") as log: @@ -179,7 +184,7 @@ def _mark() -> Path: def _log() -> Path: - return LAUNCHED / f"{Path.cwd().name}.launches" + return LAUNCHES def _recorded_pid() -> Optional[int]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index c541a0291..c75186240 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -39,8 +39,13 @@ SHM_MIN_BYTES = 1 << 30 #: Chromium's content setting that denies a permission without a prompt. BLOCK = 2 -#: Who closed a tab: agentenv over its own connection, anything else in the browser, or the browser's relaunch. -AGENTENV, BROWSER_CLOSED, RELAUNCH = "agentenv", "browser", "relaunch" +#: Who closed a tab: agentenv over its own connection, or anything else in the browser; or how its browser ended, on +#: its own or stopped by its server. +AGENTENV, BROWSER_CLOSED, EXIT, STOP = "agentenv", "browser", "exit", "stop" +#: How long the active tab's check waits on each page. +ACTIVE_CHECK_TIMEOUT_S = 2.0 +#: How much of a download each read hashes. +HASH_CHUNK_BYTES = 1 << 20 #: Who answered a dialog when agentenv didn't. AGENT = "agent" #: The targets agentenv's connection follows. @@ -194,19 +199,23 @@ def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> Non devtools.listen(self._event) async def attach(self) -> None: - """Follow a browser just launched: the tabs of the one before closed by its relaunch, and every page this one - has or opens attached, its downloads going to ``downloads_dir`` with their events.""" - now = _now() - self.closed_tabs += [{**tab, "closed_at": now, "closed_by": RELAUNCH} for tab in self.tabs.values()] - self.tabs.clear() - self._sessions.clear() - self._open_dialogs.clear() + """Follow a browser just launched: any tabs the one before left open closed by its server's stop, and every + page this one has or opens attached, its downloads going to ``downloads_dir`` with their events.""" + self.ended(STOP) await self.devtools.call("Target.setDiscoverTargets", {"discover": True, "filter": PAGES}) await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True, "filter": PAGES}) await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "allow", "eventsEnabled": True, "downloadPath": str(self.downloads_dir)}) + def ended(self, how: str) -> None: + """The browser is gone, ``how`` it ended: its open tabs are closed by that, and its sessions with them.""" + now = _now() + self.closed_tabs += [{**tab, "closed_at": now, "closed_by": how} for tab in self.tabs.values()] + self.tabs.clear() + self._sessions.clear() + self._open_dialogs.clear() + async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" self._closing.add(target) @@ -235,7 +244,8 @@ async def active_tab(self) -> Optional[str]: try: state = (await self.devtools.call( "Runtime.evaluate", {"expression": "[document.visibilityState, document.hasFocus()]", - "returnByValue": True}, session_id=session, timeout_s=2))["result"]["value"] + "returnByValue": True}, session_id=session, + timeout_s=ACTIVE_CHECK_TIMEOUT_S))["result"]["value"] except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError): continue if state[1]: @@ -334,7 +344,7 @@ def _finished(self, task: asyncio.Future) -> None: def _size_and_sha256(path: Path) -> tuple[int, str]: digest = hashlib.sha256() with path.open("rb") as file: - while chunk := file.read(1 << 20): + while chunk := file.read(HASH_CHUNK_BYTES): digest.update(chunk) return path.stat().st_size, digest.hexdigest() @@ -395,7 +405,7 @@ def __init__(self) -> None: (Path.home() / folder).mkdir(exist_ok=True) #: agentenv's control of it over that connection, and the record it keeps. self.control = BrowserControl(self.devtools, Path.home() / DOWNLOADS, self.settings.dialogs) - self.devtools.on_close(self._computer_app.exited) + self.devtools.on_close(self._pipe_closed) @classmethod def declaration(cls) -> AppDeclaration: @@ -467,6 +477,11 @@ async def _launch_fds(self) -> dict[int, int]: self._following.add_done_callback(lambda done: done.cancelled() or done.exception()) return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} + def _pipe_closed(self) -> None: + """The browser closed its pipe: its tabs closed by its exit when it ended on its own, else by its server's + stop.""" + self.control.ended(EXIT if self._computer_app.exited() else STOP) + async def _follow(self, urls: list[str]) -> None: """Its control attached, then each of ``urls`` in a tab behind the active one, in order.""" await self.control.attach() diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 1b27bf8c0..aa164dcf9 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -480,7 +480,7 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ assert [tab["id"] for tab in record["tabs"]] == ["a"] and record["tabs"][0]["title"] == "Slack" assert [(tab["id"], tab["closed_by"]) for tab in record["closed_tabs"]] == [("b", "browser"), ("c", "agentenv")] assert session == "s-hidden" and ("Target.closeTarget", {"targetId": "hidden"}, None) in devtools.calls - assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs][-1] == ("a", "relaunch") + assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs][-1] == ("a", "stop") assert [call[0] for call in devtools.calls[:3]] == ["Target.setDiscoverTargets", "Target.setAutoAttach", "Browser.setDownloadBehavior"] @@ -569,8 +569,11 @@ async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_ex if any(entry["event"] == "exit" for entry in runtime.launches()): break await asyncio.sleep(0.05) + after_exit = await browser.browser_record() browser.devtools.close() assert [tab["id"] for tab in record["tabs"]] == ["start"] + assert after_exit["tabs"] == [] and [(tab["id"], tab["closed_by"]) for tab in after_exit["closed_tabs"]] == [ + ("start", "stop"), ("start", "exit")] assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in runtime.launches()] == [ ("launch", "boot", first), ("stop", None, first), ("launch", "data/reset", second), ("exit", None, second)] diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 9464c17ab..389d2e9ed 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -210,15 +210,13 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with report = await teardown_run(context) assert accepted[0].startswith("accepted") - (dialog,) = recorded["dialogs"] - assert (dialog["type"], dialog["message"], dialog["answer"], dialog["by"]) == ("confirm", "Proceed?", "accepted", - "agentenv") - (download,) = recorded["downloads"] - assert (download["filename"], download["state"], download["size"], download["sha256"]) == ( - "report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256) - assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["browser"] - assert [tab["url"] for tab in recorded["tabs"]] == [RECORDED] and recorded["tabs"][0]["active"] is True - assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"] + assert [(d["type"], d["message"], d["answer"], d["by"]) for d in recorded["dialogs"]] == [ + ("confirm", "Proceed?", "accepted", "agentenv")], recorded + assert [(d["filename"], d["state"], d["size"], d["sha256"]) for d in recorded["downloads"]] == [ + ("report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256)], recorded + assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["browser"], recorded + assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded + assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited assert exited["launches"][0]["cause"] == "boot" and relaunched["ready"] is True, relaunched events = [(entry["event"], entry.get("cause")) for entry in restarted["launches"]] assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "launch")] From 18ba863c43fb9ea7732d3b0b57a0a8b727f88d5a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:03:59 -0800 Subject: [PATCH 03/12] test(computer): every run of the recorded page's dialog and download is recorded alike Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/chromium_app_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 389d2e9ed..45ffb05e3 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -210,10 +210,13 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with report = await teardown_run(context) assert accepted[0].startswith("accepted") - assert [(d["type"], d["message"], d["answer"], d["by"]) for d in recorded["dialogs"]] == [ - ("confirm", "Proceed?", "accepted", "agentenv")], recorded - assert [(d["filename"], d["state"], d["size"], d["sha256"]) for d in recorded["downloads"]] == [ - ("report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256)], recorded + # Chrome reloads its error page for the start tab once the page's server is up, and F5 reloads it too, so the page + # may run more than once; each run's dialog and download must be recorded alike. + assert recorded["dialogs"] and {(d["type"], d["message"], d["answer"], d["by"]) for d in recorded["dialogs"]} == { + ("confirm", "Proceed?", "accepted", "agentenv")}, recorded + assert recorded["downloads"] and {(d["filename"], d["state"], d["size"], d["sha256"]) + for d in recorded["downloads"]} == { + ("report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256)}, recorded assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["browser"], recorded assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited From 1be6c524602f52a650755028070e98155dea4c1b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:56:26 -0800 Subject: [PATCH 04/12] feat(computer): unknown closers, exit codes, beforeunload always left, and when the record starts From the review of the design: a tab agentenv didn't close is closed_by unknown; an exit is logged with its code or signal, and nothing relaunches the browser; agentenv's dialog answers always leave a page asking before it's left, so dismiss never blocks a navigation; and the record says since when it covers. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 17 +++++++----- .../agentenv_protocol/computer/app_runtime.py | 19 +++++++++++--- .../src/agentenv_protocol/computer/session.py | 17 ++++++++++++ .../src/agentenv_protocol/website_browser.py | 26 ++++++++++++------- .../tests/test_website_browser.py | 21 ++++++++------- tst/integration/computer/chromium_app_test.py | 3 ++- 6 files changed, 72 insertions(+), 31 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 39a791ef1..93779fc3a 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -759,8 +759,9 @@ are the profile's template. The base class adds: the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, 1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; `geolocation` and `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who - answers a page's JavaScript dialogs, `beforeunload` included: `agent` (the default), or agentenv, - accepting (`accept`, a `prompt()` with its default text) or dismissing (`dismiss`) each. + answers a page's JavaScript dialogs: `agent` (the default), or agentenv, accepting (`accept`, a `prompt()` + with its default text) or dismissing (`dismiss`) each. Either way agentenv leaves a page that asks before + it's left (`beforeunload`), so none of its answers blocks a navigation. - **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, its HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check`, `--lang` and, when `/dev/shm` holds under 1 GiB (Docker gives 64 MiB), @@ -789,16 +790,18 @@ are the profile's template. The base class adds: the pipe rather than a second copy, which would hand its window to the first and exit. The ready check `devtools_answers` asks `Browser.getVersion` over the pipe. - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches - to every page with flattened sessions and keeps a record: open tabs in opened order, the one whose page - has focus `active`; `closed_tabs`, each with `closed_at` and `closed_by` (`agentenv`, over this - connection; `browser`, anything else; `exit` or `stop`, when the browser ended on its own or its server - stopped it); `downloads`, into `~/Downloads`, each with its state + to every page with flattened sessions and keeps a record, `since` its server started: open tabs in + opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and + `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser + ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it. `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record. - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made - and each exit it didn't, since the computer booted. + and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted. + Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra + failure, its ready check false. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in `~/Downloads`, folders it makes. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index e380b2934..a5f0c88d2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -8,6 +8,7 @@ import json import os import shlex +import signal import time from collections.abc import Awaitable, Callable from datetime import datetime, timezone @@ -31,6 +32,8 @@ #: Where an app's server logs each launch, stop and exit of the app, beside its pid; a boot starts the app's folder as #: its image has it, so the log is the boot's. LAUNCHES = Path(".agentenv/launches") +#: How long an app that ended on its own may take to be reaped, for its exit code. +EXIT_WAIT_S = 5.0 #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 @@ -64,6 +67,7 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[] self.launch_fds = launch_fds self.relaunch_on_start = relaunch_on_start self.launch_running = launch_running + self.exits: set[asyncio.Future] = set() self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} @@ -126,14 +130,21 @@ async def stop_app(self) -> bool: return True def exited(self) -> bool: - """Log that the app this server launched last ended on its own, unless the server stopped it; whether it - did.""" + """Log that the app this server launched last ended on its own, with its exit code or the signal that ended + it, unless the server stopped it; whether it did.""" if (pid := _recorded_pid()) is None: return False - self.log("exit", pid) + noting = asyncio.ensure_future(self._log_exit(pid)) + self.exits.add(noting) + noting.add_done_callback(self.exits.discard) return True - def log(self, event: str, pid: int, **fields: str) -> None: + async def _log_exit(self, pid: int) -> None: + code = await session.returncode(pid, EXIT_WAIT_S) + self.log("exit", pid, **({} if code is None else + {"signal": signal.Signals(-code).name} if code < 0 else {"code": code})) + + def log(self, event: str, pid: int, **fields: Any) -> None: """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" entry = {"at": datetime.now(timezone.utc).isoformat(), "event": event, "pid": pid, **fields} _log().parent.mkdir(parents=True, exist_ok=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index f9ca0d261..7ecc6d2d8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -17,6 +17,8 @@ WITH_FDS = Path(__file__).with_name("_with_fds.py") #: The processes launched into the session, kept until they exit so each is reaped. _running: set[asyncio.Task] = set() +#: Each launched process's reaping, by pid, until its exit is asked for: its return code, negative for a signal. +_exits: dict[int, asyncio.Future] = {} async def wait_until_up() -> None: @@ -38,9 +40,24 @@ async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[ reaped = asyncio.ensure_future(process.wait()) _running.add(reaped) reaped.add_done_callback(_running.discard) + _exits[process.pid] = reaped return process.pid +async def returncode(pid: int, timeout_s: float) -> Optional[int]: + """The return code the process ``launch`` started as ``pid`` exited with, negative for the signal that ended it, + once it's reaped within ``timeout_s``; None for one ``launch`` didn't start or that hasn't exited.""" + reaped = _exits.get(pid) + if reaped is None: + return None + try: + code = await asyncio.wait_for(asyncio.shield(reaped), timeout_s) + except TimeoutError: + return None + _exits.pop(pid, None) + return code + + async def stop(pid: int, timeout_s: float) -> None: """End the session ``launch`` started as ``pid``, every process in it: asked to exit, then killed after ``timeout_s``.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index c75186240..2e35b740a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -39,9 +39,9 @@ SHM_MIN_BYTES = 1 << 30 #: Chromium's content setting that denies a permission without a prompt. BLOCK = 2 -#: Who closed a tab: agentenv over its own connection, or anything else in the browser; or how its browser ended, on -#: its own or stopped by its server. -AGENTENV, BROWSER_CLOSED, EXIT, STOP = "agentenv", "browser", "exit", "stop" +#: Who closed a tab: agentenv over its own connection, or someone it can't tell; or how its browser ended, on its own +#: or stopped by its server. +AGENTENV, UNKNOWN, EXIT, STOP = "agentenv", "unknown", "exit", "stop" #: How long the active tab's check waits on each page. ACTIVE_CHECK_TIMEOUT_S = 2.0 #: How much of a download each read hashes. @@ -181,11 +181,15 @@ def _heard(callback: Callable, *args: Any) -> None: class BrowserControl: """agentenv's control of a browser over its pipe: at each launch it attaches to every page with flattened sessions - and keeps a record of the browser, its open tabs in opened order, those closed and who closed them, its downloads - with their size and sha256, and its JavaScript dialogs, which it answers when ``dialogs`` says to.""" + and keeps a record of the browser since this server started, its open tabs in opened order, those closed and who + closed them, its downloads with their size and sha256, and its JavaScript dialogs, which it answers when + ``dialogs`` says to: always leaving a page that asks before it's left, so no answer of agentenv's blocks a + navigation.""" def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> None: self.devtools, self.downloads_dir, self.dialogs = devtools, downloads, dialogs + #: When this record starts: a server that restarts starts another. + self.since = _now() self.tabs: dict[str, dict] = {} self.closed_tabs: list[dict] = [] self.downloads: dict[str, dict] = {} @@ -255,10 +259,10 @@ async def active_tab(self) -> Optional[str]: return visible async def record(self) -> dict: - """The record: open ``tabs``, each with whether it's ``active``, ``closed_tabs``, ``downloads`` and - ``dialogs``.""" + """The record ``since`` it started: open ``tabs``, each with whether it's ``active``, ``closed_tabs``, + ``downloads`` and ``dialogs``.""" active = await self.active_tab() - return {"tabs": [{**tab, "active": tab["id"] == active} for tab in self.tabs.values()], + return {"since": self.since, "tabs": [{**tab, "active": tab["id"] == active} for tab in self.tabs.values()], "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()), "dialogs": list(self.dialog_log)} @@ -282,7 +286,7 @@ def _on_Target_targetDestroyed(self, params: dict, session: Optional[str]) -> No target = params["targetId"] if (tab := self.tabs.pop(target, None)) is not None: self.closed_tabs.append({**tab, "closed_at": _now(), - "closed_by": AGENTENV if target in self._closing else BROWSER_CLOSED}) + "closed_by": AGENTENV if target in self._closing else UNKNOWN}) self._closing.discard(target) self._hidden.discard(target) @@ -301,8 +305,10 @@ def _on_Page_javascriptDialogOpening(self, params: dict, session: Optional[str]) self._open_dialogs[session] = dialog if self.dialogs != AGENT: self._answering.add(session) + leave = params.get("type") == "beforeunload" self._spawn(self.devtools.call("Page.handleJavaScriptDialog", { - "accept": self.dialogs == "accept", "promptText": params.get("defaultPrompt", "")}, session_id=session)) + "accept": leave or self.dialogs == "accept", "promptText": params.get("defaultPrompt", "")}, + session_id=session)) def _on_Page_javascriptDialogClosed(self, params: dict, session: Optional[str]) -> None: if (dialog := self._open_dialogs.pop(session, None)) is not None: diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index aa164dcf9..3d5c43526 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -478,7 +478,8 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ await control.attach() assert [tab["id"] for tab in record["tabs"]] == ["a"] and record["tabs"][0]["title"] == "Slack" - assert [(tab["id"], tab["closed_by"]) for tab in record["closed_tabs"]] == [("b", "browser"), ("c", "agentenv")] + assert [(tab["id"], tab["closed_by"]) for tab in record["closed_tabs"]] == [("b", "unknown"), ("c", "agentenv")] + assert record["since"] == control.since assert session == "s-hidden" and ("Target.closeTarget", {"targetId": "hidden"}, None) in devtools.calls assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs][-1] == ("a", "stop") assert [call[0] for call in devtools.calls[:3]] == ["Target.setDiscoverTargets", "Target.setAutoAttach", @@ -486,19 +487,20 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ @pytest.mark.asyncio -@pytest.mark.parametrize("dialogs, sent, answer, by", [ - ("accept", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"), - ("dismiss", {"accept": False, "promptText": "yes"}, "dismissed", "agentenv"), - ("agent", None, "dismissed", "agent"), -]) +@pytest.mark.parametrize("dialogs, kind, sent, answer, by", [ + ("accept", "prompt", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"), + ("dismiss", "prompt", {"accept": False, "promptText": "yes"}, "dismissed", "agentenv"), + ("dismiss", "beforeunload", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"), + ("agent", "prompt", None, "dismissed", "agent"), +], ids=["accept", "dismiss", "dismiss-leaves", "agent"]) async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_says_and_records_who_did( - tmp_path, dialogs, sent, answer, by): + tmp_path, dialogs, kind, sent, answer, by): devtools = StubDevTools() control = BrowserControl(devtools, tmp_path, dialogs) devtools.send("Target.targetCreated", _page("a")) devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a")}) - devtools.send("Page.javascriptDialogOpening", {"type": "prompt", "message": "Sure?", "url": "https://a.test/", + devtools.send("Page.javascriptDialogOpening", {"type": kind, "message": "Sure?", "url": "https://a.test/", "defaultPrompt": "yes"}, "s-a") await asyncio.sleep(0) devtools.send("Page.javascriptDialogClosed", {"result": answer == "accepted"}, "s-a") @@ -509,7 +511,7 @@ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_ assert handled == ([(sent, "s-a")] if sent else []) (dialog,) = control.dialog_log assert (dialog["tab"], dialog["type"], dialog["message"], dialog["answer"], dialog["by"]) == ( - "a", "prompt", "Sure?", answer, by) + "a", kind, "Sure?", answer, by) @pytest.mark.asyncio @@ -577,3 +579,4 @@ async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_ex ("start", "stop"), ("start", "exit")] assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in runtime.launches()] == [ ("launch", "boot", first), ("stop", None, first), ("launch", "data/reset", second), ("exit", None, second)] + assert runtime.launches()[-1]["signal"] == "SIGKILL" diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 45ffb05e3..011fa6c2e 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -217,9 +217,10 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with assert recorded["downloads"] and {(d["filename"], d["state"], d["size"], d["sha256"]) for d in recorded["downloads"]} == { ("report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256)}, recorded - assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["browser"], recorded + assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["unknown"], recorded assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited + assert exited["launches"][1]["signal"] == "SIGKILL" and recorded["since"], exited assert exited["launches"][0]["cause"] == "boot" and relaunched["ready"] is True, relaunched events = [(entry["event"], entry.get("cause")) for entry in restarted["launches"]] assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "launch")] From 8fd4b81f81602282165772ea94ce6ca88515a26c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 19:15:53 -0800 Subject: [PATCH 05/12] fix(computer): keep only the latest unasked exits, and click into chromium before its tab test's Control+Tab session kept every launched process's reaping until its exit was asked for; it now drops all but the latest 16 no one asked about. The tab test clicks into Chromium's window before pressing Control+Tab, since on a CI runner the window opened at boot didn't have the keyboard and no press reached it; the test's waits are named. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/session.py | 9 ++++++++ .../tests/test_website_browser.py | 21 ++++++++++++------- tst/integration/computer/chromium_app_test.py | 17 ++++++++++++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 7ecc6d2d8..39f00cd3d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -19,6 +19,8 @@ _running: set[asyncio.Task] = set() #: Each launched process's reaping, by pid, until its exit is asked for: its return code, negative for a signal. _exits: dict[int, asyncio.Future] = {} +#: How many exits no one has asked for are kept, the latest. +KEPT_EXITS = 16 async def wait_until_up() -> None: @@ -41,9 +43,16 @@ async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[ _running.add(reaped) reaped.add_done_callback(_running.discard) _exits[process.pid] = reaped + reaped.add_done_callback(lambda _: _forget_old_exits()) return process.pid +def _forget_old_exits() -> None: + """Drop all but the latest ``KEPT_EXITS`` exits no one has asked for.""" + for pid in [pid for pid, reaped in _exits.items() if reaped.done()][:-KEPT_EXITS]: + del _exits[pid] + + async def returncode(pid: int, timeout_s: float) -> Optional[int]: """The return code the process ``launch`` started as ``pid`` exited with, negative for the signal that ended it, once it's reaped within ``timeout_s``; None for one ``launch`` didn't start or that hasn't exited.""" diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 3d5c43526..1f99a80af 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -10,6 +10,7 @@ import hashlib import json import os +import signal import sys import pytest @@ -427,12 +428,16 @@ def test_an_app_that_isnt_a_browser_launches_and_declares_as_before(): "ready": {"window": True, "timeout_s": 60.0}} +#: How long a test waits on the fake browser, and how often it looks. +WAIT_S, POLL_S = 5.0, 0.05 + + async def _answering(browser) -> bool: - """Whether ``browser``'s ready check says it answers, within five seconds.""" - for _ in range(100): + """Whether ``browser``'s ready check says it answers, within ``WAIT_S``.""" + for _ in range(int(WAIT_S / POLL_S)): if await browser.devtools_answers(): return True - await asyncio.sleep(0.05) + await asyncio.sleep(POLL_S) return False @@ -525,10 +530,10 @@ async def test_its_control_records_a_completed_download_with_its_size_and_sha256 devtools.send("Browser.downloadProgress", {"guid": "g", "state": "inProgress", "receivedBytes": 3}) in_progress = dict(control.downloads["g"]) devtools.send("Browser.downloadProgress", {"guid": "g", "state": "completed"}) - for _ in range(50): + for _ in range(int(WAIT_S / POLL_S)): if "sha256" in control.downloads["g"]: break - await asyncio.sleep(0.01) + await asyncio.sleep(POLL_S) done = control.downloads["g"] assert in_progress["state"] == "inProgress" and "sha256" not in in_progress @@ -566,11 +571,11 @@ async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_ex await runtime.stop_app() second = await runtime.launch_app("data/reset") await _answering(browser) - os.killpg(second, 9) - for _ in range(50): + os.killpg(second, signal.SIGKILL) + for _ in range(int(WAIT_S / POLL_S)): if any(entry["event"] == "exit" for entry in runtime.launches()): break - await asyncio.sleep(0.05) + await asyncio.sleep(POLL_S) after_exit = await browser.browser_record() browser.devtools.close() diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 011fa6c2e..ded44ac9a 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -243,12 +243,17 @@ async def _record(deployed, card: dict, until) -> dict: async def _tab_to(deployed, title: str): - """Control+Tab pressed as an agent presses it, again while the window still shows the tab before, which a press + """Control+Tab pressed as an agent presses it, after a click into Chromium's window, which a window opened with no + input may need before it has the keyboard, and again while the window still shows the tab before, which a press can reach before the next tab opens; the last press's result, and the window titles once one starts with ``title``.""" + x, y = await _window_center(deployed) async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() + await session.call_tool("computer_screenshot", {}) + clicked = await session.call_tool("computer_click", {"x": x, "y": y}) + assert not clicked.isError, clicked for attempt in range(3): keyed = await session.call_tool("computer_key", {"keys": ["Control", "Tab"]}) try: @@ -258,6 +263,16 @@ async def _tab_to(deployed, title: str): raise +async def _window_center(deployed) -> tuple[int, int]: + """The middle of Chromium's largest window, in screen pixels.""" + tree = (await inside.run(deployed, ["xwininfo", "-root", "-tree"], timeout_s=30))["stdout"] + windows = [(int(w) * int(h), int(x) + int(w) // 2, int(y) + int(h) // 2) for w, h, x, y in re.findall( + r'\("[^"]*" "Chromium"\)\s+(\d+)x(\d+)\+-?\d+\+-?\d+\s+\+(-?\d+)\+(-?\d+)', tree)] + assert windows, tree + _, x, y = max(windows) + return x, y + + async def _chromium_windows(deployed, title: str, seconds: int = 60, count: int = 1) -> list[str]: """The titles of the computer's Chromium windows once there are ``count`` and one starts with ``title``.""" for _ in range(seconds): From 55958dafc5f2711e086455632821b3d4072d4189 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 19:30:38 -0800 Subject: [PATCH 06/12] fix(computer): a stopped process's exit is dropped, and the chromium tests click into its window before every key press Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/session.py | 3 +- tst/integration/computer/chromium_app_test.py | 46 +++++++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 39f00cd3d..b56b8eb0b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -69,7 +69,8 @@ async def returncode(pid: int, timeout_s: float) -> Optional[int]: async def stop(pid: int, timeout_s: float) -> None: """End the session ``launch`` started as ``pid``, every process in it: asked to exit, then killed after - ``timeout_s``.""" + ``timeout_s``. No one asks how it exited once it's stopped.""" + _exits.pop(pid, None) try: os.killpg(pid, signal.SIGTERM) except ProcessLookupError: diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index ded44ac9a..fadfd92cc 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -26,7 +26,7 @@ from agent_env.env import Env from agentenv_protocol.types import WELL_KNOWN_PATH from agentenv_protocol.website_browser import BROWSER -from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready +from tst.integration.computer.website_browser_test import _app, _browser_process, _ready from agent_env.env.envs.computer.utils import inside from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run @@ -85,6 +85,8 @@ def log_message(self, *args): time.sleep(float(sys.argv[1])) """ REPORT_SHA256 = hashlib.sha256(b"agentenv report\n" * 64).hexdigest() +#: How long apart the key chords a test presses into Chromium are. +KEY_GAP_S = 2.0 # Each window the window manager manages, its class and its title. WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') @@ -135,7 +137,7 @@ async def test_it_runs_chrome_itself_on_its_pipe_answers_prompts_as_the_deploy_s serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_PERMISSIONS, "60"], timeout_s=90)) await asyncio.sleep(3) - await _keys(deployed, ["F5"]) + await _keys_in_chromium(deployed, ["F5"]) permissions = await _chromium_windows(deployed, "denied") await serving await inside.run(deployed, ["sh", "-c", "mkdir -p ~/.cache/chromium && touch ~/.cache/chromium/kept"], @@ -191,11 +193,9 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "45"], timeout_s=75)) await asyncio.sleep(3) - await _keys(deployed, ["F5"]) + await _keys_in_chromium(deployed, ["F5"]) accepted = await _chromium_windows(deployed, "accepted") - await _keys(deployed, ["Control", "t"]) - await asyncio.sleep(2) - await _keys(deployed, ["Control", "w"]) + await _keys_in_chromium(deployed, ["Control", "t"], ["Control", "w"]) await serving recorded = await _record(deployed, card, lambda r: r["downloads"] and "sha256" in r["downloads"][0] and r["closed_tabs"]) @@ -242,18 +242,34 @@ async def _record(deployed, card: dict, until) -> dict: return record -async def _tab_to(deployed, title: str): - """Control+Tab pressed as an agent presses it, after a click into Chromium's window, which a window opened with no - input may need before it has the keyboard, and again while the window still shows the tab before, which a press - can reach before the next tab opens; the last press's result, and the window titles once one starts with - ``title``.""" +async def _keys_in_chromium(deployed, *chords: list[str]) -> None: + """Each chord pressed as an agent presses it, ``KEY_GAP_S`` apart, after a click into Chromium's window.""" + async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await _click_into_chromium(deployed, session) + for keys in chords: + pressed = await session.call_tool("computer_key", {"keys": keys}) + assert not pressed.isError, pressed + await asyncio.sleep(KEY_GAP_S) + + +async def _click_into_chromium(deployed, session: ClientSession) -> None: + """A click into the middle of Chromium's window, which a window opened with no input may need before it has the + keyboard.""" x, y = await _window_center(deployed) + await session.initialize() + await session.call_tool("computer_screenshot", {}) + clicked = await session.call_tool("computer_click", {"x": x, "y": y}) + assert not clicked.isError, clicked + + +async def _tab_to(deployed, title: str): + """Control+Tab pressed as an agent presses it, after a click into Chromium's window, and again while the window + still shows the tab before, which a press can reach before the next tab opens; the last press's result, and the + window titles once one starts with ``title``.""" async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): async with ClientSession(read, write) as session: - await session.initialize() - await session.call_tool("computer_screenshot", {}) - clicked = await session.call_tool("computer_click", {"x": x, "y": y}) - assert not clicked.isError, clicked + await _click_into_chromium(deployed, session) for attempt in range(3): keyed = await session.call_tool("computer_key", {"keys": ["Control", "Tab"]}) try: From c30e4219624d333b28fe82f6eca7851181be21a1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 19:46:09 -0800 Subject: [PATCH 07/12] test(computer): the browser tests' key helper clicks into Chromium's window first, for website_browser_test too Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/chromium_app_test.py | 71 +++++-------------- .../computer/website_browser_test.py | 20 +++++- 2 files changed, 35 insertions(+), 56 deletions(-) diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index fadfd92cc..9fdfa5b9a 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -18,15 +18,13 @@ import httpx import pytest -from mcp import ClientSession -from mcp.client.streamable_http import streamablehttp_client from agentenv_protocol import client as protocol_v1 from agent_env.env import Env from agentenv_protocol.types import WELL_KNOWN_PATH from agentenv_protocol.website_browser import BROWSER -from tst.integration.computer.website_browser_test import _app, _browser_process, _ready +from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready from agent_env.env.envs.computer.utils import inside from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run @@ -85,8 +83,6 @@ def log_message(self, *args): time.sleep(float(sys.argv[1])) """ REPORT_SHA256 = hashlib.sha256(b"agentenv report\n" * 64).hexdigest() -#: How long apart the key chords a test presses into Chromium are. -KEY_GAP_S = 2.0 # Each window the window manager manages, its class and its title. WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') @@ -107,7 +103,7 @@ async def test_a_deploy_opens_its_start_urls_unprompted_and_a_reset_clears_its_p context = TaskStepContext(deployed_envs=[deployed]) try: opened = await _chromium_windows(deployed, "Alpha page") - keyed, tabbed = await _tab_to(deployed, "Bravo page") + tabbed = await _tab_to(deployed, "Bravo page") policies = await inside.run( deployed, ["cat", "/etc/opt/chrome_for_testing/policies/managed/agentenv.json"], timeout_s=30) await inside.run(deployed, ["sh", "-c", "touch ~/.config/chromium/marker"], timeout_s=30) @@ -118,7 +114,7 @@ async def test_a_deploy_opens_its_start_urls_unprompted_and_a_reset_clears_its_p report = await teardown_run(context) assert opened == ["Alpha page - Google Chrome for Testing"] - assert not keyed.isError and tabbed == ["Bravo page - Google Chrome for Testing"] + assert tabbed == ["Bravo page - Google Chrome for Testing"] assert {"BrowserSignin": 0, "CommandLineFlagSecurityWarningsEnabled": False}.items() <= json.loads( policies["stdout"]).items() assert marker["exit_code"] != 0 and reopened == ["Alpha page - Google Chrome for Testing"] @@ -137,7 +133,7 @@ async def test_it_runs_chrome_itself_on_its_pipe_answers_prompts_as_the_deploy_s serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_PERMISSIONS, "60"], timeout_s=90)) await asyncio.sleep(3) - await _keys_in_chromium(deployed, ["F5"]) + await _keys(deployed, ["F5"]) permissions = await _chromium_windows(deployed, "denied") await serving await inside.run(deployed, ["sh", "-c", "mkdir -p ~/.cache/chromium && touch ~/.cache/chromium/kept"], @@ -193,9 +189,9 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "45"], timeout_s=75)) await asyncio.sleep(3) - await _keys_in_chromium(deployed, ["F5"]) + await _keys(deployed, ["F5"]) accepted = await _chromium_windows(deployed, "accepted") - await _keys_in_chromium(deployed, ["Control", "t"], ["Control", "w"]) + await _keys(deployed, ["Control", "t"], ["Control", "w"]) await serving recorded = await _record(deployed, card, lambda r: r["downloads"] and "sha256" in r["downloads"][0] and r["closed_tabs"]) @@ -242,51 +238,16 @@ async def _record(deployed, card: dict, until) -> dict: return record -async def _keys_in_chromium(deployed, *chords: list[str]) -> None: - """Each chord pressed as an agent presses it, ``KEY_GAP_S`` apart, after a click into Chromium's window.""" - async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): - async with ClientSession(read, write) as session: - await _click_into_chromium(deployed, session) - for keys in chords: - pressed = await session.call_tool("computer_key", {"keys": keys}) - assert not pressed.isError, pressed - await asyncio.sleep(KEY_GAP_S) - - -async def _click_into_chromium(deployed, session: ClientSession) -> None: - """A click into the middle of Chromium's window, which a window opened with no input may need before it has the - keyboard.""" - x, y = await _window_center(deployed) - await session.initialize() - await session.call_tool("computer_screenshot", {}) - clicked = await session.call_tool("computer_click", {"x": x, "y": y}) - assert not clicked.isError, clicked - - -async def _tab_to(deployed, title: str): - """Control+Tab pressed as an agent presses it, after a click into Chromium's window, and again while the window - still shows the tab before, which a press can reach before the next tab opens; the last press's result, and the - window titles once one starts with ``title``.""" - async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): - async with ClientSession(read, write) as session: - await _click_into_chromium(deployed, session) - for attempt in range(3): - keyed = await session.call_tool("computer_key", {"keys": ["Control", "Tab"]}) - try: - return keyed, await _chromium_windows(deployed, title, seconds=20) - except TimeoutError: - if attempt == 2: - raise - - -async def _window_center(deployed) -> tuple[int, int]: - """The middle of Chromium's largest window, in screen pixels.""" - tree = (await inside.run(deployed, ["xwininfo", "-root", "-tree"], timeout_s=30))["stdout"] - windows = [(int(w) * int(h), int(x) + int(w) // 2, int(y) + int(h) // 2) for w, h, x, y in re.findall( - r'\("[^"]*" "Chromium"\)\s+(\d+)x(\d+)\+-?\d+\+-?\d+\s+\+(-?\d+)\+(-?\d+)', tree)] - assert windows, tree - _, x, y = max(windows) - return x, y +async def _tab_to(deployed, title: str) -> list[str]: + """Control+Tab pressed as an agent presses it, and again while the window still shows the tab before, which a + press can reach before the next tab opens; the window titles once one starts with ``title``.""" + for attempt in range(3): + await _keys(deployed, ["Control", "Tab"]) + try: + return await _chromium_windows(deployed, title, seconds=20) + except TimeoutError: + if attempt == 2: + raise async def _chromium_windows(deployed, title: str, seconds: int = 60, count: int = 1) -> list[str]: diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index 020421a40..2155b86d7 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -74,6 +74,8 @@ def log_message(self, *args): argv = open(f"/proc/{pid}/cmdline", "rb").read().replace(b"\\0", b" ").decode().split() print(json.dumps({"pid": pid, "argv": argv, "listening": len(listening & sockets)})) """ +#: How long apart the key chords a test presses into Chromium are. +KEY_GAP_S = 2.0 # Each window the window manager manages, its class and its title. WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') @@ -148,13 +150,29 @@ async def _window_titled(deployed, title: str) -> str: async def _keys(deployed, *chords: list[str]) -> None: - """Each chord pressed through the computer's own tools, as an agent presses it.""" + """Each chord pressed through the computer's own tools, as an agent presses it, ``KEY_GAP_S`` apart, after a click + into the middle of Chromium's window, which a window opened with no input may need before it has the keyboard.""" + x, y = await _window_center(deployed) async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() + await session.call_tool("computer_screenshot", {}) + clicked = await session.call_tool("computer_click", {"x": x, "y": y}) + assert not clicked.isError, clicked for keys in chords: pressed = await session.call_tool("computer_key", {"keys": keys}) assert not pressed.isError, pressed + await asyncio.sleep(KEY_GAP_S) + + +async def _window_center(deployed) -> tuple[int, int]: + """The middle of Chromium's largest window, in screen pixels.""" + tree = (await inside.run(deployed, ["xwininfo", "-root", "-tree"], timeout_s=30))["stdout"] + windows = [(int(w) * int(h), int(x) + int(w) // 2, int(y) + int(h) // 2) for w, h, x, y in re.findall( + r'\("[^"]*" "Chromium"\)\s+(\d+)x(\d+)\+-?\d+\+-?\d+\s+\+(-?\d+)\+(-?\d+)', tree)] + assert windows, tree + _, x, y = max(windows) + return x, y async def _browser_process(deployed, other_than: int | None = None) -> dict: From 659d6133696e3df3e9cadce887968a88231872e1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 20:51:05 -0800 Subject: [PATCH 08/12] test(computer): an agent answers a dialog chromium shows it, and same-name downloads are each hashed at their own path Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/chromium_app_test.py | 62 +++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 9fdfa5b9a..c4628ab98 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -58,13 +58,15 @@ def log_message(self, *args): time.sleep(float(sys.argv[1])) """ RECORDED = "http://127.0.0.1:8767/" -# Serves, on loopback for the seconds it's given, a page that asks to confirm and then downloads report.bin. +# Serves, on loopback for the seconds it's given, a page that asks to confirm and then downloads report.bin, whose +# content differs at each request: the nth is _report(n). SERVE_RECORDED = """import http.server, sys, threading, time -REPORT = b"agentenv report\\n" * 64 +served = [] class Page(http.server.BaseHTTPRequestHandler): def do_GET(self): if self.path == "/report.bin": - body, kind = REPORT, "application/octet-stream" + served.append(1) + body, kind = b"agentenv report %d\\n" % len(served) * 64, "application/octet-stream" else: body = (b"asking") @@ -82,7 +84,9 @@ def log_message(self, *args): threading.Thread(target=server.serve_forever, daemon=True).start() time.sleep(float(sys.argv[1])) """ -REPORT_SHA256 = hashlib.sha256(b"agentenv report\n" * 64).hexdigest() +# A page that asks to confirm as soon as it loads, and says in its title what it was told. +AGENT_DIALOG = ("data:text/html,asking") # Each window the window manager manages, its class and its title. WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') @@ -186,15 +190,16 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with context = TaskStepContext(deployed_envs=[deployed]) try: card = await _app(deployed, WELL_KNOWN_PATH, "chromium") - serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "45"], - timeout_s=75)) + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "90"], + timeout_s=120)) await asyncio.sleep(3) await _keys(deployed, ["F5"]) accepted = await _chromium_windows(deployed, "accepted") await _keys(deployed, ["Control", "t"], ["Control", "w"]) + first = await _record(deployed, card, lambda r: _hashed(r) and r["closed_tabs"]) + await _keys(deployed, ["F5"]) + recorded = await _record(deployed, card, lambda r: len(_hashed(r)) > len(_hashed(first))) await serving - recorded = await _record(deployed, card, lambda r: r["downloads"] and "sha256" in r["downloads"][0] - and r["closed_tabs"]) browser = await _browser_process(deployed) await inside.run(deployed, ["kill", "-9", str(browser["pid"])], timeout_s=30) exited = await _record(deployed, card, lambda r: r["launches"][-1]["event"] == "exit") @@ -210,9 +215,13 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with # may run more than once; each run's dialog and download must be recorded alike. assert recorded["dialogs"] and {(d["type"], d["message"], d["answer"], d["by"]) for d in recorded["dialogs"]} == { ("confirm", "Proceed?", "accepted", "agentenv")}, recorded - assert recorded["downloads"] and {(d["filename"], d["state"], d["size"], d["sha256"]) - for d in recorded["downloads"]} == { - ("report.bin", "completed", len(b"agentenv report\n" * 64), REPORT_SHA256)}, recorded + # Every download is named report.bin, so Chrome saves the later ones as report (1).bin and on: each must be hashed + # at its own path, as the content served for it. + downloads = recorded["downloads"] + assert len(downloads) >= 2 and {(d["filename"], d["state"]) for d in downloads} == {("report.bin", "completed")} + assert len({d["path"] for d in downloads}) == len(downloads), recorded + assert sorted(d["sha256"] for d in downloads) == sorted( + hashlib.sha256(_report(n)).hexdigest() for n in range(1, len(downloads) + 1)), recorded assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["unknown"], recorded assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited @@ -224,6 +233,37 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with assert not report.failed +@pytest.mark.asyncio +async def test_with_dialogs_left_to_the_agent_chromium_shows_one_and_records_the_agents_answer(chromium_desktop): + with namespace_routing(): + deployed = await Env.get(chromium_desktop.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [AGENT_DIALOG]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + card = await _app(deployed, WELL_KNOWN_PATH, "chromium") + pending = await _record(deployed, card, lambda r: r["dialogs"]) + await _keys(deployed, ["Enter"]) + answered_title = await _chromium_windows(deployed, "accepted") + answered = await _record(deployed, card, lambda r: r["dialogs"] and r["dialogs"][0]["answer"]) + finally: + report = await teardown_run(context) + + assert [(d["type"], d["answer"], d["by"]) for d in pending["dialogs"]] == [("confirm", None, None)], pending + assert answered_title[0].startswith("accepted") + assert [(d["type"], d["message"], d["answer"], d["by"]) for d in answered["dialogs"]] == [ + ("confirm", "Proceed?", "accepted", "agent")], answered + assert not report.failed + + +def _report(n: int) -> bytes: + """The content SERVE_RECORDED serves at its nth request for report.bin.""" + return b"agentenv report %d\n" % n * 64 + + +def _hashed(record: dict) -> list[dict]: + return [download for download in record["downloads"] if "sha256" in download] + + async def _record(deployed, card: dict, until) -> dict: """The browser's record, once ``until`` says it has what's waited for, or as it last was within a minute.""" path = protocol_v1.find_extension_method(card, BROWSER, "record")["endpoint"] From 2e183beba35de12cddece4e786e26c636ba11813 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:07:52 -0800 Subject: [PATCH 09/12] fix(computer): hash each download at the file Chrome saved it as, which Chrome 156 doesn't report Chrome 156 sends no filePath with a completed download, so the record fell back to ~/Downloads/ and gave every later download of the same name the first file's path and hash. A completed download with no path now takes the newest file named as Chrome names repeats (report.bin, report (1).bin, ...) that no other download holds and that was written since it began, of the size it received; it's chosen as the download completes, so two can't claim one file. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/website_browser.py | 36 +++++++++++++++++-- .../tests/test_website_browser.py | 24 +++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 2e35b740a..49a9d484c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -13,7 +13,9 @@ import json import logging import os +import re import shutil +import time from collections.abc import AsyncIterator from datetime import datetime, timezone from pathlib import Path @@ -46,6 +48,8 @@ ACTIVE_CHECK_TIMEOUT_S = 2.0 #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 +#: How much earlier than a download's start a file it was saved as may say it was written, for clocks' rounding. +CLOCK_SLACK_S = 1.0 #: Who answered a dialog when agentenv didn't. AGENT = "agent" #: The targets agentenv's connection follows. @@ -200,6 +204,7 @@ def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> Non self._open_dialogs: dict[str, dict] = {} self._answering: set[str] = set() self._tasks: set[asyncio.Future] = set() + self._began: dict[str, float] = {} devtools.listen(self._event) async def attach(self) -> None: @@ -321,20 +326,45 @@ def _on_Browser_downloadWillBegin(self, params: dict, session: Optional[str]) -> self.downloads[params["guid"]] = {"guid": params["guid"], "url": params.get("url", ""), "filename": params.get("suggestedFilename", ""), "state": "inProgress", "started_at": _now()} + self._began[params["guid"]] = time.time() def _on_Browser_downloadProgress(self, params: dict, session: Optional[str]) -> None: if (download := self.downloads.get(params["guid"])) is None: return download["state"] = params["state"] if params["state"] == "completed": - path = params.get("filePath") or str(self.downloads_dir / download["filename"]) - self._spawn(self._hashed(download, Path(path))) + path = Path(params["filePath"]) if params.get("filePath") else self._saved_as(download, params) + if path is None: + download["ended_at"] = _now() + return + download["path"] = str(path) + self._spawn(self._hashed(download, path)) elif params["state"] == "canceled": download["ended_at"] = _now() + def _saved_as(self, download: dict, progress: dict) -> Optional[Path]: + """Where Chrome saved a completed download it gives no path for: of the files in ``downloads_dir`` named as it + names a download of its suggested name (``report.bin``, ``report (1).bin``, ...), the newest written since the + download began that no other download in the record holds, of the size it received when one is; none when + nothing is.""" + name = download["filename"] + stem, ext = os.path.splitext(name) + named = re.compile(re.escape(stem) + r"( \(\d+\))?" + re.escape(ext)) + held = {other.get("path") for other in self.downloads.values() if other is not download} + began, size = self._began.get(download["guid"], 0.0) - CLOCK_SLACK_S, progress.get("receivedBytes") + candidates = [] + for path in self.downloads_dir.iterdir() if self.downloads_dir.is_dir() else (): + try: + stat = path.stat() + except OSError: + continue + if named.fullmatch(path.name) and str(path) not in held and stat.st_mtime >= began: + candidates.append((size is not None and stat.st_size == size, stat.st_mtime, path)) + return max(candidates)[2] if candidates else None + async def _hashed(self, download: dict, path: Path) -> None: size, sha256 = await asyncio.to_thread(_size_and_sha256, path) - download.update(path=str(path), size=size, sha256=sha256, ended_at=_now()) + download.update(size=size, sha256=sha256, ended_at=_now()) def _spawn(self, work: Awaitable) -> None: task = asyncio.ensure_future(work) diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 1f99a80af..8261d4c6f 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -541,6 +541,30 @@ async def test_its_control_records_a_completed_download_with_its_size_and_sha256 assert done["sha256"] == hashlib.sha256(b"%PDF-1.7 report").hexdigest() +@pytest.mark.asyncio +async def test_downloads_of_one_name_chrome_gives_no_path_for_are_each_hashed_at_the_file_it_saved(tmp_path): + devtools = StubDevTools() + control = BrowserControl(devtools, tmp_path, "agent") + (tmp_path / "report.bin").write_bytes(b"older, from before") + os.utime(tmp_path / "report.bin", (0, 0)) + + contents = {"g1": b"the first report", "g2": b"the second, longer report"} + for guid, saved_as in (("g1", "report (1).bin"), ("g2", "report (2).bin")): + devtools.send("Browser.downloadWillBegin", {"guid": guid, "url": "https://a.test/report.bin", + "suggestedFilename": "report.bin"}) + (tmp_path / saved_as).write_bytes(contents[guid]) + devtools.send("Browser.downloadProgress", {"guid": guid, "state": "completed", + "receivedBytes": len(contents[guid])}) + for _ in range(int(WAIT_S / POLL_S)): + if all("sha256" in download for download in control.downloads.values()): + break + await asyncio.sleep(POLL_S) + + assert [(d["path"], d["sha256"]) for d in control.downloads.values()] == [ + (str(tmp_path / name), hashlib.sha256(contents[guid]).hexdigest()) + for guid, name in (("g1", "report (1).bin"), ("g2", "report (2).bin"))] + + @pytest.mark.asyncio async def test_its_control_says_the_tab_whose_page_has_focus_is_active_else_the_first_visible_one(tmp_path): states = {"s-a": ["hidden", False], "s-b": ["visible", False], "s-c": ["visible", True]} From 72f36d073e8463a8e9397252886e0f6fc9ab603c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:31:40 -0800 Subject: [PATCH 10/12] fix(computer): leave Chrome's own download behavior, and take each download's path from Chrome The control connection set Browser.setDownloadBehavior to allow, under which Chrome overwrites a download of a name it already saved; on its own Chrome keeps both, report.bin and report (1).bin. It now asks only for the events, with behavior default, so downloads go where the managed policies send them, as Chrome saves them. Chrome reports each completed download's filePath, which the record takes; with none it records no path rather than a guess, so the name-matching fallback is gone, and BrowserControl no longer takes a downloads folder. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/website_browser.py | 48 +++++-------------- .../tests/test_website_browser.py | 36 +++----------- 2 files changed, 18 insertions(+), 66 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 49a9d484c..083cfb251 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -13,9 +13,7 @@ import json import logging import os -import re import shutil -import time from collections.abc import AsyncIterator from datetime import datetime, timezone from pathlib import Path @@ -48,8 +46,7 @@ ACTIVE_CHECK_TIMEOUT_S = 2.0 #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 -#: How much earlier than a download's start a file it was saved as may say it was written, for clocks' rounding. -CLOCK_SLACK_S = 1.0 + #: Who answered a dialog when agentenv didn't. AGENT = "agent" #: The targets agentenv's connection follows. @@ -190,8 +187,8 @@ class BrowserControl: ``dialogs`` says to: always leaving a page that asks before it's left, so no answer of agentenv's blocks a navigation.""" - def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> None: - self.devtools, self.downloads_dir, self.dialogs = devtools, downloads, dialogs + def __init__(self, devtools: DevToolsPipe, dialogs: str) -> None: + self.devtools, self.dialogs = devtools, dialogs #: When this record starts: a server that restarts starts another. self.since = _now() self.tabs: dict[str, dict] = {} @@ -204,18 +201,17 @@ def __init__(self, devtools: DevToolsPipe, downloads: Path, dialogs: str) -> Non self._open_dialogs: dict[str, dict] = {} self._answering: set[str] = set() self._tasks: set[asyncio.Future] = set() - self._began: dict[str, float] = {} devtools.listen(self._event) async def attach(self) -> None: - """Follow a browser just launched: any tabs the one before left open closed by its server's stop, and every - page this one has or opens attached, its downloads going to ``downloads_dir`` with their events.""" + """Follow a browser just launched: any tabs the one before left open closed by its server's stop, every page + this one has or opens attached, and its downloads reported, which go where its own policies say, as Chrome + downloads unattended: into ``~/Downloads``, a name downloaded again kept beside the first.""" self.ended(STOP) await self.devtools.call("Target.setDiscoverTargets", {"discover": True, "filter": PAGES}) await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True, "filter": PAGES}) - await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "allow", "eventsEnabled": True, - "downloadPath": str(self.downloads_dir)}) + await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "default", "eventsEnabled": True}) def ended(self, how: str) -> None: """The browser is gone, ``how`` it ended: its open tabs are closed by that, and its sessions with them.""" @@ -326,42 +322,20 @@ def _on_Browser_downloadWillBegin(self, params: dict, session: Optional[str]) -> self.downloads[params["guid"]] = {"guid": params["guid"], "url": params.get("url", ""), "filename": params.get("suggestedFilename", ""), "state": "inProgress", "started_at": _now()} - self._began[params["guid"]] = time.time() def _on_Browser_downloadProgress(self, params: dict, session: Optional[str]) -> None: if (download := self.downloads.get(params["guid"])) is None: return download["state"] = params["state"] if params["state"] == "completed": - path = Path(params["filePath"]) if params.get("filePath") else self._saved_as(download, params) - if path is None: + if not params.get("filePath"): download["ended_at"] = _now() return - download["path"] = str(path) - self._spawn(self._hashed(download, path)) + download["path"] = params["filePath"] + self._spawn(self._hashed(download, Path(params["filePath"]))) elif params["state"] == "canceled": download["ended_at"] = _now() - def _saved_as(self, download: dict, progress: dict) -> Optional[Path]: - """Where Chrome saved a completed download it gives no path for: of the files in ``downloads_dir`` named as it - names a download of its suggested name (``report.bin``, ``report (1).bin``, ...), the newest written since the - download began that no other download in the record holds, of the size it received when one is; none when - nothing is.""" - name = download["filename"] - stem, ext = os.path.splitext(name) - named = re.compile(re.escape(stem) + r"( \(\d+\))?" + re.escape(ext)) - held = {other.get("path") for other in self.downloads.values() if other is not download} - began, size = self._began.get(download["guid"], 0.0) - CLOCK_SLACK_S, progress.get("receivedBytes") - candidates = [] - for path in self.downloads_dir.iterdir() if self.downloads_dir.is_dir() else (): - try: - stat = path.stat() - except OSError: - continue - if named.fullmatch(path.name) and str(path) not in held and stat.st_mtime >= began: - candidates.append((size is not None and stat.st_size == size, stat.st_mtime, path)) - return max(candidates)[2] if candidates else None - async def _hashed(self, download: dict, path: Path) -> None: size, sha256 = await asyncio.to_thread(_size_and_sha256, path) download.update(size=size, sha256=sha256, ended_at=_now()) @@ -440,7 +414,7 @@ def __init__(self) -> None: for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) #: agentenv's control of it over that connection, and the record it keeps. - self.control = BrowserControl(self.devtools, Path.home() / DOWNLOADS, self.settings.dialogs) + self.control = BrowserControl(self.devtools, self.settings.dialogs) self.devtools.on_close(self._pipe_closed) @classmethod diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 8261d4c6f..d21dd495c 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -468,7 +468,7 @@ def _page(target, url="about:blank", title=""): async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_leaves_out_its_hidden_pages(tmp_path): devtools = StubDevTools({"Target.createTarget": {"targetId": "hidden"}, "Target.attachToTarget": {"sessionId": "s-hidden"}}) - control = BrowserControl(devtools, tmp_path, "agent") + control = BrowserControl(devtools, "agent") await control.attach() for target in ("a", "b", "c"): @@ -489,6 +489,7 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs][-1] == ("a", "stop") assert [call[0] for call in devtools.calls[:3]] == ["Target.setDiscoverTargets", "Target.setAutoAttach", "Browser.setDownloadBehavior"] + assert devtools.calls[2][1] == {"behavior": "default", "eventsEnabled": True} @pytest.mark.asyncio @@ -501,7 +502,7 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_says_and_records_who_did( tmp_path, dialogs, kind, sent, answer, by): devtools = StubDevTools() - control = BrowserControl(devtools, tmp_path, dialogs) + control = BrowserControl(devtools, dialogs) devtools.send("Target.targetCreated", _page("a")) devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a")}) @@ -522,14 +523,15 @@ async def test_its_control_answers_a_dialog_on_its_pages_session_as_its_setting_ @pytest.mark.asyncio async def test_its_control_records_a_completed_download_with_its_size_and_sha256(tmp_path): devtools = StubDevTools() - control = BrowserControl(devtools, tmp_path, "agent") + control = BrowserControl(devtools, "agent") (tmp_path / "report.pdf").write_bytes(b"%PDF-1.7 report") devtools.send("Browser.downloadWillBegin", {"guid": "g", "url": "https://a.test/report.pdf", "suggestedFilename": "report.pdf"}) devtools.send("Browser.downloadProgress", {"guid": "g", "state": "inProgress", "receivedBytes": 3}) in_progress = dict(control.downloads["g"]) - devtools.send("Browser.downloadProgress", {"guid": "g", "state": "completed"}) + devtools.send("Browser.downloadProgress", {"guid": "g", "state": "completed", + "filePath": str(tmp_path / "report.pdf")}) for _ in range(int(WAIT_S / POLL_S)): if "sha256" in control.downloads["g"]: break @@ -541,35 +543,11 @@ async def test_its_control_records_a_completed_download_with_its_size_and_sha256 assert done["sha256"] == hashlib.sha256(b"%PDF-1.7 report").hexdigest() -@pytest.mark.asyncio -async def test_downloads_of_one_name_chrome_gives_no_path_for_are_each_hashed_at_the_file_it_saved(tmp_path): - devtools = StubDevTools() - control = BrowserControl(devtools, tmp_path, "agent") - (tmp_path / "report.bin").write_bytes(b"older, from before") - os.utime(tmp_path / "report.bin", (0, 0)) - - contents = {"g1": b"the first report", "g2": b"the second, longer report"} - for guid, saved_as in (("g1", "report (1).bin"), ("g2", "report (2).bin")): - devtools.send("Browser.downloadWillBegin", {"guid": guid, "url": "https://a.test/report.bin", - "suggestedFilename": "report.bin"}) - (tmp_path / saved_as).write_bytes(contents[guid]) - devtools.send("Browser.downloadProgress", {"guid": guid, "state": "completed", - "receivedBytes": len(contents[guid])}) - for _ in range(int(WAIT_S / POLL_S)): - if all("sha256" in download for download in control.downloads.values()): - break - await asyncio.sleep(POLL_S) - - assert [(d["path"], d["sha256"]) for d in control.downloads.values()] == [ - (str(tmp_path / name), hashlib.sha256(contents[guid]).hexdigest()) - for guid, name in (("g1", "report (1).bin"), ("g2", "report (2).bin"))] - - @pytest.mark.asyncio async def test_its_control_says_the_tab_whose_page_has_focus_is_active_else_the_first_visible_one(tmp_path): states = {"s-a": ["hidden", False], "s-b": ["visible", False], "s-c": ["visible", True]} devtools = StubDevTools({"Runtime.evaluate": lambda params, session: {"result": {"value": states[session]}}}) - control = BrowserControl(devtools, tmp_path, "agent") + control = BrowserControl(devtools, "agent") for target in ("a", "b", "c"): devtools.send("Target.targetCreated", _page(target)) devtools.send("Target.attachedToTarget", {"sessionId": f"s-{target}", **_page(target)}) From 6969e5000d80f5fecae5de8ec7869bc1ca3f8f80 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:53:44 -0800 Subject: [PATCH 11/12] fix(computer): data/get leaves out a file a running app removes before it's read data/get lists the files under its paths and then reads or uploads each, while the app keeps running. Chromium deletes a BrowserMetrics file soon after it starts, and CI's gate 3 probe read the profile just then, so the whole export failed with ENOENT. A file gone by the time it's read is now left out, inline or uploaded. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/dataplane.py | 29 ++++++++----- .../tests/test_computer_dataplane.py | 43 +++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py index 01f704429..c1c98b42c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/dataplane.py @@ -15,7 +15,7 @@ from pydantic import TypeAdapter, ValidationError from ..agent_env_environment import add_data, get_data -from ..transfers import NamespaceUploader, ReadObject, RelativePath, WriteNamespaceGrant, download +from ..transfers import NamespaceUploader, ReadObject, RelativePath, TransferError, WriteNamespaceGrant, download from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path if TYPE_CHECKING: @@ -64,22 +64,31 @@ async def _computer_data_get(self, write_namespace: Optional[WriteNamespaceGrant async def file_parts(home: Path, paths: list[str], write_namespace: Optional[WriteNamespaceGrant], leaving_out: list[str] = ()) -> list: """A part for each file under ``paths`` of ``home`` but under none of ``leaving_out``, naming its path relative to - ``home``: uploaded under ``write_namespace`` when there is one, else inline.""" + ``home``: uploaded under ``write_namespace`` when there is one, else inline. A file a running app removes before it's + read is left out.""" files = [(relative, source) for relative, source in _data_files(home, paths) if not any(PurePosixPath(relative).is_relative_to(owned) for owned in leaving_out)] + parts = [] if write_namespace is not None: uploader = NamespaceUploader(write_namespace) - parts = [] for relative, source in files: - await uploader.upload(relative, source) + try: + await uploader.upload(relative, source) + except (TransferError, OSError): + if source.exists(): + raise + continue parts.append(uploaded_file_part(relative, name=source.name, mime_type=_mime(source))) return parts - return [ - FilePart(file=FileWithBytes(bytes=base64.b64encode(source.read_bytes()).decode("ascii"), - name=source.name, mimeType=_mime(source)), - metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}}) - for relative, source in files - ] + for relative, source in files: + try: + data = source.read_bytes() + except FileNotFoundError: + continue + parts.append(FilePart(file=FileWithBytes(bytes=base64.b64encode(data).decode("ascii"), name=source.name, + mimeType=_mime(source)), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}})) + return parts async def part_bytes(part: FilePart) -> bytes: diff --git a/packages/agentenv-protocol/tests/test_computer_dataplane.py b/packages/agentenv-protocol/tests/test_computer_dataplane.py index f6a565fc8..b48992d2c 100644 --- a/packages/agentenv-protocol/tests/test_computer_dataplane.py +++ b/packages/agentenv-protocol/tests/test_computer_dataplane.py @@ -16,7 +16,9 @@ from computer_fake import FakeDriver, http_client, rpc # noqa: E402 from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI # noqa: E402 +from agentenv_protocol.computer import dataplane # noqa: E402 from agentenv_protocol.computer.dataplane import DEFAULT_DATA_PATHS # noqa: E402 +from agentenv_protocol.transfers import TransferError # noqa: E402 @pytest.fixture @@ -168,6 +170,47 @@ async def test_data_get_uploads_under_a_write_namespace_grant(home): "metadata": {DATA_OBJECTS_EXTENSION_URI: {"path": "notes.md"}}}] +@pytest.mark.asyncio +async def test_data_get_leaves_out_a_file_removed_after_its_listed(home): + (home / "app").mkdir() + (home / "app" / "kept.txt").write_text("kept") + gone = home / "app" / "gone.pma" + gone.write_text("metrics") + listed = dataplane._data_files + + def listed_then_removed(*args): + found = listed(*args) + gone.unlink() + return found + + with patch("agentenv_protocol.computer.dataplane._data_files", side_effect=listed_then_removed): + async with http_client(FakeDriver()) as http: + parts = (await http.post("/agentenv", json=rpc("data/get", {"paths": ["app"]}))).json()["result"]["parts"] + assert [p["metadata"][DATA_OBJECTS_EXTENSION_URI]["path"] for p in parts] == ["app/kept.txt"] + + +@pytest.mark.asyncio +async def test_data_get_under_a_grant_leaves_out_a_file_removed_before_its_uploaded(home): + (home / "kept.txt").write_text("kept") + grant = {"root_path": "runs/1", "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + "max_objects": 5, "max_object_bytes": 100, "max_total_bytes": 500, + "write": {"kind": "http-post-policy", "url": "https://store.test/", "fields": {}, + "path_field": "key", "file_field": "file"}} + + async def upload(relative, source): + if relative == "gone.txt": + raise TransferError("invalid_transfer", "The transfer source is unavailable.") + + with patch("agentenv_protocol.computer.dataplane._data_files", + return_value=[("gone.txt", home / "gone.txt"), ("kept.txt", home / "kept.txt")]), \ + patch("agentenv_protocol.computer.dataplane.NamespaceUploader") as uploader: + uploader.return_value.upload = upload + async with http_client(FakeDriver()) as http: + parts = (await http.post("/agentenv", json=rpc("data/get", {"write_namespace": grant, "paths": ["kept.txt"]})) + ).json()["result"]["parts"] + assert [p["metadata"][DATA_OBJECTS_EXTENSION_URI]["path"] for p in parts] == ["kept.txt"] + + @pytest.mark.asyncio async def test_a_failed_write_leaves_no_temporary_file(home): with patch("agentenv_protocol.computer._linux.os.replace", side_effect=OSError("disk full")): From c568d9c0bcf10398ca0e2c0f9abe424684e13b12 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:09:20 -0800 Subject: [PATCH 12/12] test(computer): name the chromium record test's waits Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/chromium_app_test.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index c4628ab98..4ca5b9701 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -32,6 +32,8 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] +#: A page server's time to listen, the record's wait and poll, and the Control+Tab presses for one tab change. +SERVE_START_S, RECORD_WAIT_S, RECORD_POLL_S, TAB_PRESSES = 3, 60, 1, 3 ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] ALPHA = "data:text/html,Alpha page

Alpha

" BRAVO = "data:text/html,Bravo page

Bravo

" @@ -136,7 +138,7 @@ async def test_it_runs_chrome_itself_on_its_pipe_answers_prompts_as_the_deploy_s browser = await _browser_process(deployed) serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_PERMISSIONS, "60"], timeout_s=90)) - await asyncio.sleep(3) + await asyncio.sleep(SERVE_START_S) await _keys(deployed, ["F5"]) permissions = await _chromium_windows(deployed, "denied") await serving @@ -192,7 +194,7 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with card = await _app(deployed, WELL_KNOWN_PATH, "chromium") serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_RECORDED, "90"], timeout_s=120)) - await asyncio.sleep(3) + await asyncio.sleep(SERVE_START_S) await _keys(deployed, ["F5"]) accepted = await _chromium_windows(deployed, "accepted") await _keys(deployed, ["Control", "t"], ["Control", "w"]) @@ -265,28 +267,28 @@ def _hashed(record: dict) -> list[dict]: async def _record(deployed, card: dict, until) -> dict: - """The browser's record, once ``until`` says it has what's waited for, or as it last was within a minute.""" + """The browser's record, once ``until`` says it has what's waited for, or as it last was within ``RECORD_WAIT_S``.""" path = protocol_v1.find_extension_method(card, BROWSER, "record")["endpoint"] - for _ in range(60): + for _ in range(RECORD_WAIT_S // RECORD_POLL_S): try: record = await _app(deployed, path, "chromium") if until(record): return record except httpx.HTTPError: pass - await asyncio.sleep(1) + await asyncio.sleep(RECORD_POLL_S) return record async def _tab_to(deployed, title: str) -> list[str]: """Control+Tab pressed as an agent presses it, and again while the window still shows the tab before, which a press can reach before the next tab opens; the window titles once one starts with ``title``.""" - for attempt in range(3): + for attempt in range(TAB_PRESSES): await _keys(deployed, ["Control", "Tab"]) try: return await _chromium_windows(deployed, title, seconds=20) except TimeoutError: - if attempt == 2: + if attempt == TAB_PRESSES - 1: raise