From 2f4522c6b73e0db667722e589f68e982384b3879 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 09:58:47 -0800 Subject: [PATCH 1/6] feat(computer): WebsiteBrowserEnvironment, the base class for website browsers A website browser subclasses agentenv_protocol.website_browser's WebsiteBrowserEnvironment, a computer app, and agentenv owns the rest, inside a computer and alone: - The flags after the subclass's command: its profile, the debugging port on loopback, no first run or default-browser check, --lang, and the site proxy when AGENTENV_SITE_PROXY names one. Alone, loopback goes through the proxy too, and the browser refuses to start without one. In a computer, loopback is bypassed so the apps beside it stay reachable. Alone it also runs headless (headful on the open web) at its window size. declaration() refuses a subclass command that carries any of these, or --remote-allow-origins. - BrowserSettings (start_urls, window, locale, time_zone, as TZ), the write_policies install step, @cdp with Chromium's DevToolsActivePort default and a ready check on that address, browser/v1 params on the card, and the skill. The app SDK gains what a browser alone needs, with every other app as before: ComputerAppEnvironment's _launch_for and _on_desktop hooks; an optional Launch.env, left out of stored declarations when unset; and an AppRuntime that waits for X only when DISPLAY is set and adds the menu launcher only on a desktop. A fixture browser bundle tests it in a computer on the shipped Chrome for Testing build. FD-4937 Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 42 ++- .../src/agentenv_protocol/computer/app.py | 15 +- .../agentenv_protocol/computer/app_runtime.py | 18 +- .../src/agentenv_protocol/website_browser.py | 211 +++++++++++++++ .../tests/test_website_browser.py | 251 ++++++++++++++++++ tst/integration/computer/conftest.py | 12 +- .../website-browser/envs/browser/browser.py | 48 ++++ .../website-browser/envs/browser/env.toml | 4 + .../website-browser/envs/desktop/env.toml | 4 + .../website-browser/tasks/desktop.json | 3 + .../computer/website_browser_test.py | 135 ++++++++++ 11 files changed, 729 insertions(+), 14 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website_browser.py create mode 100644 packages/agentenv-protocol/tests/test_website_browser.py create mode 100644 tst/integration/computer/website-browser/envs/browser/browser.py create mode 100644 tst/integration/computer/website-browser/envs/browser/env.toml create mode 100644 tst/integration/computer/website-browser/envs/desktop/env.toml create mode 100644 tst/integration/computer/website-browser/tasks/desktop.json create mode 100644 tst/integration/computer/website_browser_test.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 8b40aa5c..12212928 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -687,7 +687,7 @@ class MailApp(ComputerAppEnvironment): - **`launch=Launch(...)`** says how it starts in the desktop session: with it (`at="boot"`) or when the agent opens it (`"demand"`). `ready=Ready(...)` says when it's ready, within `timeout_s` of its launch: a window matching `launch.window` by its class, instance or title, a port answering on - loopback, or a command exiting 0. `@ready_check` marks a method for readiness that needs logic (a + loopback, or a command exiting 0. `env` sets variables in the app's environment, over its server's. `@ready_check` marks a method for readiness that needs logic (a sync one runs in a thread), and `@install_step(scope=...)` one a computer runs as it installs the app, after its run scripts, offline and without the class's `__init__`. Each runs on every OS the app installs on, or on those its `os=` names (`@install_step(os="linux")`, `@ready_check(os=[...])`); @@ -716,9 +716,10 @@ desktop session, and `ready` (GET: an `AppReadiness` of `ready`, `problems`, `la and `timeout_s`), which runs the app's readiness checks then, at once, each within 10 s. A server that can't start, or whose start in the session fails (adding the launcher, say), writes why to `.agentenv/server.failure` in the folder and exits, and the driver reports that in its place. Once -the session is up the server adds the app's launcher to the session user's menu, which runs `python --m agentenv_protocol.computer.launch ` to ask the server, so the server starts every copy of -the app, and launches a `boot` app, once per boot. The driver finds each app by the socket in its +the X display is up, when `DISPLAY` names one, the server adds the app's launcher to the session user's +menu, which runs `python -m agentenv_protocol.computer.launch ` to ask the server, so the +server starts every copy of the app, and launches a `boot` app, once per boot. An app that runs with +no desktop, a website browser alone, gets no launcher. The driver finds each app by the socket in its folder: the computer is ready once every app's server answers and each `boot` app, and each app with no launch, says it's ready; a `demand` app counts once something launches it. What the server does in the session (`agentenv_protocol.computer.session`) is Linux's. @@ -739,6 +740,39 @@ requests to the app's server. env stores, and `python -m agentenv_protocol.computer._declare ` prints it as JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. +### Website browsers + +`agentenv_protocol.website_browser.WebsiteBrowserEnvironment` is the computer app class for a Chromium +website browser, run inside a computer or alone. A subclass declares its build as any app does, its +`Launch` running the browser itself. It sets `version`, its build, and `policy_dir`, where that build +reads managed policies; `policies` adds its own over agentenv's, and `profile` is its profile folder +under the session user's home (`.config/chromium` by default), whose contents from its install steps +are the profile's template. The base class adds: + +- **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in + tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`); `window`, + 1280 × 800, alone only; `locale`; and `time_zone`, set as `TZ` in its launch's `env`. +- **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, + `--remote-debugging-port=0`, `--no-first-run`, `--no-default-browser-check` and `--lang`. With + `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off. + Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`, + `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go + through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`, + except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy. + `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, + which agentenv never sets, so the debugging port refuses a WebSocket that carries a page's Origin. +- **Alone, no desktop**: its launch matches and waits on no window, its server adds no launcher and, + without `DISPLAY`, waits for no display, and its ready checks say when it's ready. +- **`write_policies`**, an install step that writes agentenv's unattended-use policies (`POLICIES`) + and the subclass's to `agentenv.json` in `policy_dir`. +- **`@cdp`**, which marks the method returning its debugging address. The default reads + `DevToolsActivePort` from the profile, and `devtools_address()` calls it each time, so it's right + after a relaunch. The ready check `debugging_address_answers` asks that address's `/json/version`. +- **`browser/v1`** on its card, params only: `chromium` (its `version`), `clock: "real"` and, alone, + `window`. +- **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in + `~/Downloads`, folders it makes. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 6772fc08..73113371 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -169,6 +169,8 @@ class Launch(BaseModel): #: Matches the app's window by its WM_CLASS ``class`` or ``instance`` and its ``title``, as given. window: Optional[dict[str, str]] = None ready: Optional[Ready] = None + #: Variables set in the app's environment, over its server's. + env: Optional[dict[str, str]] = None @field_validator("window") @classmethod @@ -399,17 +401,26 @@ def __init__(self) -> None: self.settings = cls.settings_on(on).model_validate(values) self.build_settings = _built_settings(cls, on) declared = cls._installs.get(on) - self.launch = declared.launch if declared is not None else None + launch = declared.launch if declared is not None else None + self.launch = self._launch_for(launch) if launch is not None else None self.paths = list(declared.paths) if declared is not None else [] ready = self.launch.ready if self.launch is not None else None self._computer_app = AppRuntime( self.launch, {fn.__name__: getattr(self, fn.__name__) for fn in _methods(cls) if hasattr(fn, _READY_CHECK_ATTR) and _runs_on(getattr(fn, _READY_CHECK_ATTR), on)}, - ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S) + ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop()) self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, cls.data_timeout_s) + def _launch_for(self, launch: Launch) -> Launch: + """The launch its server uses, from the one declared for its OS: as declared.""" + return launch + + def _on_desktop(self) -> bool: + """Whether it runs in a computer's desktop session, which lists it in its menu.""" + return True + @classmethod def settings_on(cls, os_type: Optional[str]) -> type[BaseModel]: """Its ``Settings`` on ``os_type``, with the fields its OS adds.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 89c5bcd8..2783ac1c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -48,21 +48,26 @@ class AppRuntime: """An app's server's state: its ``launch``, its ``@ready_check`` methods by name, bound to the app, how long it may take to be ready, why its last launch failed, and each sync check still running in its thread.""" - def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float) -> None: + def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float, *, + desktop: bool = True) -> None: self.launch = launch self.ready_checks = ready_checks self.timeout_s = timeout_s + self.desktop = desktop self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} async def start(self) -> None: - """Once the desktop session is up, add the app's menu launcher and launch a boot app, unless the computer has - launched it since it booted, at boot or on demand: a server that restarts leaves the app be.""" + """Once its X display is up, when it has one, add the app's menu launcher on a desktop and launch a boot app, + unless the computer has launched it since it booted, at boot or on demand: a server that restarts leaves the + app be.""" if self.launch is None: return - await session.wait_until_up() - session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) + if os.environ.get("DISPLAY"): + await session.wait_until_up() + if self.desktop: + session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) if self.launch.at == "boot" and _launched_at() is None: try: await self.launch_app() @@ -71,7 +76,8 @@ async def start(self) -> None: async def launch_app(self) -> int: """Start the app from its folder, marked as launched; its pid.""" - pid = await session.launch(self.launch.command, Path.cwd(), {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) + pid = await session.launch(self.launch.command, Path.cwd(), + {**(self.launch.env or {}), "ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) self.launch_failed = None PID.parent.mkdir(parents=True, exist_ok=True) PID.write_text(str(pid)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py new file mode 100644 index 00000000..36845776 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -0,0 +1,211 @@ +"""WebsiteBrowserEnvironment: a website browser as a computer app, run inside a computer or alone. A subclass says how +its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's +command, the unattended-use policies, the debugging address, the settings every browser takes, its skill and +``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy come from variables +agentenv sets, never from its settings.""" +from __future__ import annotations + +import inspect +import json +import os +from pathlib import Path +from typing import Any, Callable, ClassVar +from urllib.parse import urlsplit + +import httpx +from pydantic import BaseModel, Field, field_validator + +from .agent_env_environment import agent_skill +from .computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check +from .types import EnvironmentExtension + +ALONE_ENV = "AGENTENV_BROWSER_ALONE" +OPEN_WEB_ENV = "AGENTENV_BROWSER_OPEN_WEB" +SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" +BROWSER = "urn:agentenv:browser/v1" +#: The flags agentenv sets, by prefix, which a subclass's command may not carry. agentenv never sets +#: ``--remote-allow-origins``, so the debugging port refuses a WebSocket that carries a page's Origin. +OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", + "--headless", "--window-size", "--lang", "--force-webrtc-ip-handling-policy") +#: Alone, loopback goes through the proxy too, so no page reaches the debugging port. In a computer, the apps on +#: loopback beside it stay reachable; ``<-loopback>`` removes Chromium's own rule, so ``*.localhost`` names still don't. +ALONE_BYPASS = "<-loopback>" +COMPUTER_BYPASS = "<-loopback>;127.0.0.1;[::1];localhost" +#: Managed policies for unattended use: no sign-in, sync, password manager, autofill, metrics, promotions, translation, +#: background mode, component updates, default-browser check, download prompt or warning bar for its flags. +POLICIES: dict[str, Any] = { + "BrowserSignin": 0, + "SyncDisabled": True, + "PasswordManagerEnabled": False, + "AutofillSettings": [{"url_pattern": "*", "blocked_types": ["all"]}], + "MetricsReportingEnabled": False, + "PromotionalTabsEnabled": False, + "TranslateEnabled": False, + "BackgroundModeEnabled": False, + "ComponentUpdatesEnabled": False, + "DefaultBrowserSettingEnabled": False, + "PromptForDownloadLocation": False, + "CommandLineFlagSecurityWarningsEnabled": False, + "DownloadDirectory": "${user_home}/Downloads", +} +#: Where an agent's files to upload are, and where downloads land, under the session user's home. +UPLOADS = "Uploads" +DOWNLOADS = "Downloads" +SKILL = f"""# Website browser + +- Open sites by their real names, such as `https://slack.com`: the environment serves its sites under those names. +- Files to upload are in `~/{UPLOADS}`; downloads land in `~/{DOWNLOADS}`. +""" +_CDP_ATTR = "_aee_cdp" + + +def cdp(fn: Callable) -> Callable: + """Mark the method that returns the browser's own debugging address, a ``ws://`` URL where it runs; + ``devtools_address`` calls it each time, so the address is right after a relaunch.""" + setattr(fn, _CDP_ATTR, True) + return fn + + +class Window(BaseModel): + width: int = Field(1280, gt=0) + height: int = Field(800, gt=0) + + +class BrowserSettings(BaseModel): + """The settings every website browser takes, per deploy; a browser's own ``Settings`` adds to them.""" + + #: Opened in tabs at launch, the first active. + start_urls: list[str] = Field(default_factory=lambda: ["about:blank"]) + #: Its window when it runs alone; in a computer the screen sets it. + window: Window = Field(default_factory=Window) + locale: str = "en-US" + #: The IANA time zone its pages see. + time_zone: str = "UTC" + + @field_validator("start_urls") + @classmethod + def _no_flags(cls, urls: list[str]) -> list[str]: + if flags := [url for url in urls if url.startswith("-")]: + raise ValueError(f"start_urls are URLs, not flags: {flags}") + return urls + + +class WebsiteBrowserEnvironment(ComputerAppEnvironment): + """Subclass it as any computer app, with ``@computer_app`` and ``@linux``, its launch running the browser itself, + and set ``version``, its build, and ``policy_dir``, where that build reads managed policies. ``policies`` adds its + own to agentenv's, and ``profile`` is its profile folder under the session user's home, holding what its install + steps put there.""" + + Settings = BrowserSettings + version: ClassVar[str] = "" + policy_dir: ClassVar[Path] = Path("/etc/chromium/policies/managed") + policies: ClassVar[dict[str, Any]] = {} + profile: ClassVar[str] = ".config/chromium" + + def __init__(self) -> None: + super().__init__() + for folder in (UPLOADS, DOWNLOADS): + (Path.home() / folder).mkdir(exist_ok=True) + + @classmethod + def declaration(cls) -> AppDeclaration: + """What the class declares; ValueError for a launch whose command carries a flag agentenv owns.""" + for declared in cls._installs.values(): + if declared.launch is not None: + _refuse_owned(cls.__name__, declared.launch) + return super().declaration() + + def profile_dir(self) -> Path: + return Path.home() / self.profile + + def agentenv_flags(self) -> list[str]: + """The flags agentenv owns. A browser alone goes out only through its site proxy, so it won't launch without + one; in a computer it goes through one when the computer has one, loopback aside.""" + settings, alone, proxy = self.settings, _alone(), os.environ.get(SITE_PROXY_ENV) + flags = [f"--user-data-dir={self.profile_dir()}", "--remote-debugging-port=0", "--no-first-run", + "--no-default-browser-check", f"--lang={settings.locale}"] + if proxy: + flags += [f"--proxy-server={proxy}", f"--proxy-bypass-list={ALONE_BYPASS if alone else COMPUTER_BYPASS}", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"] + elif alone: + raise ValueError(f"a website browser alone goes out only through its site proxy, and {SITE_PROXY_ENV} " + "names none") + if alone: + flags.append(f"--window-size={settings.window.width},{settings.window.height}") + if not _open_web(): + flags.append("--headless=new") + return flags + + def _launch_for(self, launch: Launch) -> Launch: + """The subclass's launch with agentenv's flags and the start tabs after its command, and the time zone in its + environment; alone, with no window to match or wait for, so its ready checks say when it's ready.""" + _refuse_owned(type(self).__name__, launch) + update: dict[str, Any] = {"command": [*launch.command, *self.agentenv_flags(), *self.settings.start_urls], + "env": {**(launch.env or {}), "TZ": self.settings.time_zone}} + if _alone(): + update.update(window=None, ready=None) + return launch.model_copy(update=update) + + def _on_desktop(self) -> bool: + return not _alone() + + @install_step(scope="system") + def write_policies(self) -> None: + """agentenv's unattended-use policies, with the subclass's over them, where its build reads them.""" + self.policy_dir.mkdir(parents=True, exist_ok=True) + (self.policy_dir / "agentenv.json").write_text(json.dumps({**POLICIES, **self.policies}, indent=1) + "\n", + encoding="utf-8") + + @cdp + def devtools(self) -> str: + """Chromium's own: the port it picked and the browser's path, as it writes them to ``DevToolsActivePort`` in + its profile.""" + port, path = (self.profile_dir() / "DevToolsActivePort").read_text().split()[:2] + return f"ws://127.0.0.1:{port}{path}" + + async def devtools_address(self) -> str: + """What the method marked ``@cdp`` returns now; FileNotFoundError until the browser has written its address.""" + for klass in type(self).__mro__: + for name, member in vars(klass).items(): + if getattr(member, _CDP_ATTR, False): + address = getattr(self, name)() + return await address if inspect.isawaitable(address) else address + raise TypeError(f"{type(self).__name__} marks no method @cdp") + + @ready_check + async def debugging_address_answers(self) -> bool: + try: + address = urlsplit(await self.devtools_address()) + except (FileNotFoundError, ValueError): + return False + try: + async with httpx.AsyncClient(timeout=5, trust_env=False) as http: + return (await http.get(f"http://{address.netloc}/json/version")).status_code == 200 + except httpx.HTTPError: + return False + + def declared_extensions(self) -> list[EnvironmentExtension]: + params: dict[str, Any] = {"chromium": self.version, "clock": "real"} + if _alone(): + params["window"] = self.settings.window.model_dump() + browser = EnvironmentExtension( + uri=BROWSER, params=params, + description="A website browser agentenv runs: its build, the clock its pages see and, alone, its window.") + return [*super().declared_extensions(), browser] + + @agent_skill + def browser_skill(self) -> tuple[str, str]: + return "How to use this website browser.", SKILL + + +def _alone() -> bool: + return os.environ.get(ALONE_ENV) == "1" + + +def _open_web() -> bool: + return os.environ.get(OPEN_WEB_ENV) == "1" + + +def _refuse_owned(name: str, launch: Launch) -> None: + if owned := [arg for arg in launch.command if arg.startswith(OWNED_FLAGS)]: + raise ValueError(f"{name}'s launch sets {', '.join(owned)}, which agentenv sets for every website browser") diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py new file mode 100644 index 00000000..4e580740 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -0,0 +1,251 @@ +"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, and the proxy rule +for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings; +its launch run with no desktop when alone; the debugging address read again each time and checked for readiness; its +policies, card params and skill; and an app that isn't a browser launched and declared as before.""" +from __future__ import annotations + +import asyncio +import json + +import pytest + +from agentenv_protocol.computer import APP_SETTINGS_ENV, ComputerAppEnvironment, Launch, Ready, app, linux, session +from agentenv_protocol.computer import app_runtime +from agentenv_protocol.computer.platform import Platform +from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, + POLICIES, SITE_PROXY_ENV, SKILL, WebsiteBrowserEnvironment, cdp) + +PROXY = "http://127.0.0.1:3128" +OWNED = ["--user-data-dir", "--remote-debugging-port=0", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] + + +@linux(launch=Launch(command=["chromium/chrome", "--no-sandbox"], window={"class": "Chromium"}, + ready=Ready(window=True, timeout_s=30))) +class Browser(WebsiteBrowserEnvironment): + version = "156.0.8078.4" + policies = {"DefaultGeolocationSetting": 2} + + +@linux(launch=Launch(command=["mail"], window={"class": "Mail"}, ready=Ready(window=True))) +class Mail(ComputerAppEnvironment): + pass + + +@pytest.fixture(autouse=True) +def browser_home(monkeypatch, tmp_path): + """A browser on Linux, its session user's home under ``tmp_path``, inside a computer with no site proxy.""" + monkeypatch.setattr(app, "detect", lambda: Platform(os="linux")) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + monkeypatch.setenv("ENVIRONMENT_NAME", "chromium") + for name in (ALONE_ENV, OPEN_WEB_ENV, SITE_PROXY_ENV, APP_SETTINGS_ENV): + monkeypatch.delenv(name, raising=False) + return tmp_path / "home" + + +def _settings(monkeypatch, **values) -> None: + monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values})) + + +def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home): + launch = Browser().launch + + profile = f"--user-data-dir={browser_home / '.config/chromium'}" + assert launch.command == ["chromium/chrome", "--no-sandbox", profile, *OWNED[1:], "about:blank"] + assert launch.env == {"TZ": "UTC"} + assert launch.window == {"class": "Chromium"} and launch.ready == Ready(window=True, timeout_s=30) + + +def test_in_a_computer_with_a_site_proxy_only_loopback_bypasses_it(monkeypatch): + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + command = Browser().launch.command + + assert [f"--proxy-server={PROXY}", f"--proxy-bypass-list={COMPUTER_BYPASS}", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"] == command[7:10] + assert not any(flag.startswith(("--headless", "--window-size")) for flag in command) + + +def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_waits_on_no_window(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + _settings(monkeypatch, window={"width": 1024, "height": 768}) + + browser = Browser() + launch = browser.launch + + assert f"--proxy-bypass-list={ALONE_BYPASS}" in launch.command + assert launch.command[-3:] == ["--window-size=1024,768", "--headless=new", "about:blank"] + assert launch.window is None and launch.ready is None + assert Launch.model_validate(launch.model_dump()) == launch + assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == app.DEFAULT_TIMEOUT_S + + +def test_alone_on_the_open_web_it_runs_headful(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(OPEN_WEB_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + command = Browser().launch.command + + assert "--window-size=1280,800" in command and not any(flag.startswith("--headless") for flag in command) + + +def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + + with pytest.raises(ValueError, match=SITE_PROXY_ENV): + Browser() + + +@pytest.mark.parametrize("flag", ["--remote-debugging-port=9222", "--remote-allow-origins=*", "--headless", + "--proxy-server=http://elsewhere:3128", "--user-data-dir=/tmp/profile"]) +def test_a_subclass_whose_command_carries_a_flag_agentenv_owns_is_refused_when_declared(flag): + @linux(launch=Launch(command=["chromium/chrome", flag])) + class Widened(WebsiteBrowserEnvironment): + pass + + with pytest.raises(ValueError, match="agentenv sets for every website browser"): + Widened.declaration() + + +def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_url_is_a_flag(monkeypatch): + _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app"], locale="fr-FR", + time_zone="Europe/Paris") + launch = Browser().launch + _settings(monkeypatch, start_urls=["--remote-allow-origins=*"]) + + assert launch.command[-2:] == ["https://slack.com", "https://linear.app"] and "--lang=fr-FR" in launch.command + assert launch.env == {"TZ": "Europe/Paris"} + with pytest.raises(ValueError, match="start_urls are URLs, not flags"): + Browser() + + +@pytest.mark.asyncio +async def test_alone_its_server_launches_it_at_once_in_its_environment_with_no_display_wait_or_launcher( + monkeypatch, tmp_path): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.chdir(tmp_path) + launched, waited, launchers = [], [], [] + + async def launch(argv, cwd, env): + launched.append(env) + return 4242 + + async def up(): + waited.append(True) + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append(name)) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + + await Browser()._computer_app.start() + + assert launched == [{"TZ": "UTC", "ENVIRONMENT_NAME": "chromium"}] and waited == [] and launchers == [] + + +@pytest.mark.asyncio +async def test_in_a_computer_its_server_waits_for_the_display_and_adds_its_launcher(monkeypatch, tmp_path): + monkeypatch.setenv("DISPLAY", ":1") + monkeypatch.chdir(tmp_path) + waited, launchers = [], [] + + async def launch(argv, cwd, env): + return 4242 + + async def up(): + waited.append(True) + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append(name)) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + + await Browser()._computer_app.start() + + assert waited == [True] and launchers == ["chromium"] + + +@pytest.mark.asyncio +async def test_its_debugging_address_is_read_again_each_time_and_answers_its_ready_check(browser_home): + browser = Browser() + profile = browser_home / ".config/chromium" + profile.mkdir(parents=True) + with pytest.raises(FileNotFoundError): + await browser.devtools_address() + unwritten = await browser.debugging_address_answers() + + server = await asyncio.start_server(_json_version, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + (profile / "DevToolsActivePort").write_text("1\n/devtools/browser/first\n") + first = await browser.devtools_address() + (profile / "DevToolsActivePort").write_text(f"{port}\n/devtools/browser/second\n") + async with server: + relaunched = await browser.devtools_address() + answers = await browser.debugging_address_answers() + + assert unwritten is False + assert first == "ws://127.0.0.1:1/devtools/browser/first" + assert relaunched == f"ws://127.0.0.1:{port}/devtools/browser/second" and answers is True + assert "debugging_address_answers" in browser._computer_app.ready_checks + + +@pytest.mark.asyncio +async def test_a_subclass_that_marks_its_own_method_cdp_gives_that_address(): + class Elsewhere(Browser): + @cdp + async def own(self) -> str: + return "ws://127.0.0.1:9333/devtools/browser/own" + + assert await Elsewhere().devtools_address() == "ws://127.0.0.1:9333/devtools/browser/own" + + +def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads_them(monkeypatch, tmp_path): + monkeypatch.setattr(Browser, "policy_dir", tmp_path / "policies") + + Browser.__new__(Browser).write_policies() + + written = json.loads((tmp_path / "policies" / "agentenv.json").read_text()) + assert written == {**POLICIES, "DefaultGeolocationSetting": 2} + assert written["DownloadDirectory"] == "${user_home}/Downloads" + assert [step.name for step in Browser.declaration().install_steps] == ["write_policies"] + + +def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): + in_computer = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + alone = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + + assert in_computer.params == {"chromium": "156.0.8078.4", "clock": "real"} + assert alone.params == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}} + + +@pytest.mark.asyncio +async def test_its_skill_names_sites_by_their_names_and_its_upload_and_download_folders_which_it_makes(browser_home): + browser = Browser() + + skill = await browser.browser_skill("website-browser") + + assert Browser.declaration().skill.method == "browser_skill" + assert skill["description"] == "How to use this website browser." and skill["skill_md"].endswith(SKILL) + assert (browser_home / "Uploads").is_dir() and (browser_home / "Downloads").is_dir() + + +def test_an_app_that_isnt_a_browser_launches_and_declares_as_before(): + mail = Mail() + stored = Mail.declaration().model_dump(mode="json", exclude_none=True) + + assert mail.launch is Mail._installs["linux"].launch and mail._computer_app.desktop is True + assert stored["os"]["linux"]["launch"] == {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, + "ready": {"window": True, "timeout_s": 60.0}} + + +async def _json_version(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + await reader.readuntil(b"\r\n\r\n") + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\n\r\n{}") + await writer.drain() + writer.close() diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index ff25cc3f..5d14ba95 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,6 +1,6 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, the cua-driver test bundle's and the chromium test bundle's, each composed once -for a module's tests; +shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's and the website-browser +test bundle's, each composed once for a module's tests; ``state``, local stores for a test that runs a bundle; and ``docker``, which runs the docker CLI for them.""" import logging @@ -27,6 +27,7 @@ BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" CUA_DRIVER = Path(__file__).parent / "cua-driver" CHROMIUM = Path(__file__).parent / "chromium" +WEBSITE_BROWSER = Path(__file__).parent / "website-browser" @pytest.fixture(scope="session") @@ -59,6 +60,13 @@ def chromium_desktop(tmp_path_factory): yield from _composed(tmp_path_factory.mktemp("chromium-desktop"), parse_bundle(CHROMIUM)) +@pytest.fixture(scope="module") +def website_browser_desktop(tmp_path_factory): + """The website-browser test bundle's desktop, the ubuntu-24.04 base and the reference driver with its fixture + website browser, composed as ``desktop`` is.""" + yield from _composed(tmp_path_factory.mktemp("website-browser-desktop"), parse_bundle(WEBSITE_BROWSER)) + + def _composed(root: Path, bundle): patch = pytest.MonkeyPatch() patch.chdir(root) diff --git a/tst/integration/computer/website-browser/envs/browser/browser.py b/tst/integration/computer/website-browser/envs/browser/browser.py new file mode 100644 index 00000000..0f74539e --- /dev/null +++ b/tst/integration/computer/website-browser/envs/browser/browser.py @@ -0,0 +1,48 @@ +"""A website browser for the WebsiteBrowserEnvironment tests: the Chrome for Testing build agentenv's chromium ships, +launched by its command itself so agentenv's flags reach it, with a file its install step leaves in its profile as the +profile's template, which it keeps as its data, so a reset relaunches it on that template.""" +import subprocess +from pathlib import Path + +from agentenv_protocol.computer import Download, Launch, Packages, Ready, computer_app, install_step, linux +from agentenv_protocol.website_browser import WebsiteBrowserEnvironment + +VERSION = "156.0.8078.4" +_ARCHIVES = {"amd64": ("linux64/chrome-linux64.zip", + "43fd31766c7533a02dda2ded5ad50c2f861cd5400be7509c0abad71129f79d79"), + "arm64": ("linux-arm64/chrome-linux-arm64.zip", + "910a12fff6170202195f72ef3d353801c7063d301ab80b1f51b77d791fda60ce")} +DOWNLOADS = [Download(url=f"https://storage.googleapis.com/chrome-for-testing-public/{VERSION}/{path}", sha256=sha256, + to=f"chromium/chrome-{arch}.zip", arch=arch) + for arch, (path, sha256) in _ARCHIVES.items()] +PACKAGES = ["fonts-liberation", "libasound2t64", "libatk-bridge2.0-0t64", "libatk1.0-0t64", "libatspi2.0-0t64", + "libcairo2", "libcups2t64", "libdbus-1-3", "libdrm2", "libexpat1", "libgbm1", "libglib2.0-0t64", + "libgtk-3-0t64", "libnspr4", "libnss3", "libpango-1.0-0", "libvulkan1", "libx11-6", "libxcb1", + "libxcomposite1", "libxdamage1", "libxext6", "libxfixes3", "libxkbcommon0", "libxrandr2", "libxshmfence1", + "unzip"] +PROFILE = ".config/chromium" +TEMPLATE = "agentenv-template.txt" + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@linux(packages=Packages(apt=PACKAGES), downloads=DOWNLOADS, paths=[PROFILE], + launch=Launch(command=["chromium/chrome/chrome", "--no-sandbox", "--disable-dev-shm-usage", "--class=Chromium"], + window={"class": "Chromium"}, ready=Ready(window=True))) +class FixtureBrowser(WebsiteBrowserEnvironment): + version = VERSION + policy_dir = Path("/etc/opt/chrome_for_testing/policies/managed") + + @install_step(scope="system") + def unpack(self) -> None: + folder = Path.cwd() / "chromium" + [archive] = folder.glob("chrome-*.zip") + subprocess.run(["unzip", "-q", str(archive), "-d", str(folder / "unpacked")], check=True) + [top] = (folder / "unpacked").iterdir() + top.rename(folder / "chrome") + (folder / "unpacked").rmdir() + archive.unlink() + + @install_step(scope="session") + def profile_template(self) -> None: + (Path.home() / PROFILE).mkdir(parents=True, exist_ok=True) + (Path.home() / PROFILE / TEMPLATE).write_text("from the profile template\n") diff --git a/tst/integration/computer/website-browser/envs/browser/env.toml b/tst/integration/computer/website-browser/envs/browser/env.toml new file mode 100644 index 00000000..21707fc6 --- /dev/null +++ b/tst/integration/computer/website-browser/envs/browser/env.toml @@ -0,0 +1,4 @@ +type = "computer_app" +module = "browser.py:FixtureBrowser" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/website-browser/envs/desktop/env.toml b/tst/integration/computer/website-browser/envs/desktop/env.toml new file mode 100644 index 00000000..1a71716b --- /dev/null +++ b/tst/integration/computer/website-browser/envs/desktop/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" +apps = ["browser"] diff --git a/tst/integration/computer/website-browser/tasks/desktop.json b/tst/integration/computer/website-browser/tasks/desktop.json new file mode 100644 index 00000000..67f3b315 --- /dev/null +++ b/tst/integration/computer/website-browser/tasks/desktop.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"} +] diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py new file mode 100644 index 00000000..b086a7fa --- /dev/null +++ b/tst/integration/computer/website_browser_test.py @@ -0,0 +1,135 @@ +"""WebsiteBrowserEnvironment in a computer, as the website-browser test bundle's desktop composes its fixture browser on +the ubuntu-24.04 base: gate 3 makes this machine's arch ready with it, which its debugging-address check is part of; +deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale and time zone, through +no proxy, with agentenv's policies where its build reads them, and opens a page on loopback; its card declares +browser/v1; and a reset relaunches it on its profile template, at a debugging address its ready check finds. Needs +Docker and the local registry on localhost:5000.""" + +import asyncio +import json +import platform +import shlex + +import httpx +import pytest + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import apps +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] +LOOPBACK = "http://127.0.0.1:8765/" +SERVE_LOOPBACK = ("mkdir -p /tmp/loopback && printf 'Loopback page' > /tmp/loopback/index.html && " + "cd /tmp/loopback && setsid nohup python3 -m http.server 8765 --bind 127.0.0.1 " + ">/dev/null 2>&1 < /dev/null &") +# Opens its URL in a new tab through Chromium's HTTP debugging endpoints, then prints that tab's title once it has one. +OPEN_TAB = """import json, sys, time, urllib.request as u +port, url = sys.argv[1:] +direct = u.build_opener(u.ProxyHandler({})) +for _ in range(30): + try: + direct.open(url, timeout=5) + break + except OSError: + time.sleep(1) +direct.open(u.Request(f"http://127.0.0.1:{port}/json/new?{url}", method="PUT"), timeout=10) +for _ in range(30): + titles = [t["title"] for t in json.load(direct.open(f"http://127.0.0.1:{port}/json/list", timeout=10)) + if t.get("url") == url and t["title"] != url] + if titles: + print(titles[0]) + break + time.sleep(1) +""" + + +def test_gate_3_makes_this_machines_arch_ready_with_the_fixture_browser(website_browser_desktop): + results = website_browser_desktop.metadata["composed"]["container"] + + assert results[ARCH]["status"] == "ready" and results[ARCH]["probe"]["passed"], results[ARCH] + assert [app.environment_name for app in website_browser_desktop.composed_apps()] == ["browser"] + + +@pytest.mark.asyncio +async def test_chromium_runs_with_agentenvs_flags_and_a_reset_relaunches_it_on_its_template(website_browser_desktop): + with namespace_routing(): + deployed = await Env.get(website_browser_desktop.id).deploy( + env_params={"app_settings": {"browser": {"locale": "fr-FR", "time_zone": "Europe/Paris"}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + argv, environ = await _browser_process(deployed) + port = await _devtools_port(deployed) + policies = await inside.run( + deployed, ["cat", "/etc/opt/chrome_for_testing/policies/managed/agentenv.json"], timeout_s=30) + await inside.run(deployed, ["sh", "-c", SERVE_LOOPBACK], timeout_s=30) + title = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, LOOPBACK], timeout_s=90) + card = await _app(deployed, WELL_KNOWN_PATH) + await protocol_v1.reset_data(f"{deployed.environment_url}/svc/browser", headers=deployed.control_headers) + relaunched = await _devtools_port(deployed, other_than=port) + readiness = await _ready(deployed, card) + template = await inside.run(deployed, ["sh", "-c", "cat ~/.config/chromium/agentenv-template.txt"], + timeout_s=30) + finally: + report = await teardown_run(context) + + assert argv[0].endswith("chromium/chrome/chrome") and argv[1:4] == ["--no-sandbox", "--disable-dev-shm-usage", + "--class=Chromium"] + assert argv[4].startswith("--user-data-dir=") and argv[4].endswith("/.config/chromium") + assert argv[5:9] == ["--remote-debugging-port=0", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR"] + assert argv[9:] == ["about:blank"] and "TZ=Europe/Paris" in environ + assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( + policies["stdout"]).items() + assert title["stdout"].strip() == "Loopback page", title + assert protocol_v1.extension_params(card, BROWSER) == {"chromium": "156.0.8078.4", "clock": "real"} + assert relaunched != port and readiness["ready"] is True, readiness + assert template["stdout"] == "from the profile template\n" + assert not report.failed + + +async def _browser_process(deployed) -> tuple[list[str], list[str]]: + """The browser process's argv and environment, once it runs: the oldest process with agentenv's debugging flag.""" + find = "pid=$(pgrep -o -f -- --remote-debugging-port=0) && tr '\\0' '\\n' < /proc/$pid/{}" + for _ in range(60): + argv = await inside.run(deployed, ["sh", "-c", find.format("cmdline")], timeout_s=30) + if argv["exit_code"] == 0 and argv["stdout"]: + environ = await inside.run(deployed, ["sh", "-c", find.format("environ")], timeout_s=30) + return argv["stdout"].splitlines(), environ["stdout"].splitlines() + await asyncio.sleep(1) + raise TimeoutError("no Chromium process with --remote-debugging-port=0") + + +async def _devtools_port(deployed, other_than: str | None = None) -> str: + """The port in the profile's DevToolsActivePort, once Chromium has written one other than ``other_than``.""" + for _ in range(60): + read = await inside.run(deployed, ["sh", "-c", "head -1 ~/.config/chromium/DevToolsActivePort"], timeout_s=30) + port = read["stdout"].strip() + if read["exit_code"] == 0 and port and port != other_than: + return port + await asyncio.sleep(1) + raise TimeoutError(f"no DevToolsActivePort other than {shlex.quote(str(other_than))}") + + +async def _app(deployed, path: str) -> dict: + async with httpx.AsyncClient(timeout=30) as http: + response = await http.get(f"{deployed.environment_url}/svc/browser{path}", headers=deployed.control_headers) + response.raise_for_status() + return response.json() + + +async def _ready(deployed, card: dict) -> dict: + """The app's readiness once it says it's ready, or as it last said within a minute.""" + for _ in range(60): + readiness = await _app(deployed, apps.endpoint(card, "ready")) + if readiness["ready"]: + return readiness + await asyncio.sleep(1) + return readiness From c830cb9c774d39bb31e62b093eea7c1bf252957f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 10:24:27 -0800 Subject: [PATCH 2/6] test(computer): the website browser test reads Chromium the way it runs on the desktop - Its argv is split on whitespace, since Chromium rewrites its process title as one line. - The time zone is read from a page, Intl's, since that rewrite also clobbers /proc//environ. - The loopback pages are served by the script that opens them, and its tab is brought to the front: on the desktop a background tab doesn't finish loading. FD-4937 Co-Authored-By: Claude Opus 5.5 --- .../computer/website_browser_test.py | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index b086a7fa..8b2c5083 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -1,7 +1,7 @@ """WebsiteBrowserEnvironment in a computer, as the website-browser test bundle's desktop composes its fixture browser on the ubuntu-24.04 base: gate 3 makes this machine's arch ready with it, which its debugging-address check is part of; -deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale and time zone, through -no proxy, with agentenv's policies where its build reads them, and opens a page on loopback; its card declares +deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale, through no proxy, +with agentenv's policies where its build reads them, and opens pages on loopback, which see the deploy's time zone; its card declares browser/v1; and a reset relaunches it on its profile template, at a debugging address its ready check finds. Needs Docker and the local registry on localhost:5000.""" @@ -27,24 +27,33 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] -LOOPBACK = "http://127.0.0.1:8765/" -SERVE_LOOPBACK = ("mkdir -p /tmp/loopback && printf 'Loopback page' > /tmp/loopback/index.html && " - "cd /tmp/loopback && setsid nohup python3 -m http.server 8765 --bind 127.0.0.1 " - ">/dev/null 2>&1 < /dev/null &") -# Opens its URL in a new tab through Chromium's HTTP debugging endpoints, then prints that tab's title once it has one. -OPEN_TAB = """import json, sys, time, urllib.request as u -port, url = sys.argv[1:] +# Serves two pages on loopback while it opens the one at its path in a new tab, through Chromium's HTTP debugging +# endpoints, and prints that tab's title once the page has set one: "/" names itself, "/tz.html" the time zone it sees. +# The tab is brought to the front, since on the desktop a background tab doesn't finish loading. +OPEN_TAB = """import http.server, json, sys, threading, time, urllib.request as u +port, path = sys.argv[1:] +PAGES = {"/": b"Loopback page", + "/tz.html": b"loading"} +class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = PAGES.get(self.path, b"") + self.send_response(200 if body else 404) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = http.server.ThreadingHTTPServer(("127.0.0.1", 8765), Page) +threading.Thread(target=server.serve_forever, daemon=True).start() +url = f"http://127.0.0.1:8765{path}" direct = u.build_opener(u.ProxyHandler({})) -for _ in range(30): - try: - direct.open(url, timeout=5) - break - except OSError: - time.sleep(1) -direct.open(u.Request(f"http://127.0.0.1:{port}/json/new?{url}", method="PUT"), timeout=10) +tab = json.load(direct.open(u.Request(f"http://127.0.0.1:{port}/json/new?{url}", method="PUT"), timeout=10)) +direct.open(f"http://127.0.0.1:{port}/json/activate/{tab['id']}", timeout=10) for _ in range(30): titles = [t["title"] for t in json.load(direct.open(f"http://127.0.0.1:{port}/json/list", timeout=10)) - if t.get("url") == url and t["title"] != url] + if t.get("url") == url and t["title"] != "loading" and not url.removeprefix("http://").startswith(t["title"])] if titles: print(titles[0]) break @@ -66,12 +75,12 @@ async def test_chromium_runs_with_agentenvs_flags_and_a_reset_relaunches_it_on_i env_params={"app_settings": {"browser": {"locale": "fr-FR", "time_zone": "Europe/Paris"}}}) context = TaskStepContext(deployed_envs=[deployed]) try: - argv, environ = await _browser_process(deployed) + argv = await _browser_argv(deployed) port = await _devtools_port(deployed) policies = await inside.run( deployed, ["cat", "/etc/opt/chrome_for_testing/policies/managed/agentenv.json"], timeout_s=30) - await inside.run(deployed, ["sh", "-c", SERVE_LOOPBACK], timeout_s=30) - title = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, LOOPBACK], timeout_s=90) + title = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, "/"], timeout_s=90) + zone = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, "/tz.html"], timeout_s=90) card = await _app(deployed, WELL_KNOWN_PATH) await protocol_v1.reset_data(f"{deployed.environment_url}/svc/browser", headers=deployed.control_headers) relaunched = await _devtools_port(deployed, other_than=port) @@ -85,24 +94,25 @@ async def test_chromium_runs_with_agentenvs_flags_and_a_reset_relaunches_it_on_i "--class=Chromium"] assert argv[4].startswith("--user-data-dir=") and argv[4].endswith("/.config/chromium") assert argv[5:9] == ["--remote-debugging-port=0", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR"] - assert argv[9:] == ["about:blank"] and "TZ=Europe/Paris" in environ + assert argv[9:] == ["about:blank"] assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( policies["stdout"]).items() assert title["stdout"].strip() == "Loopback page", title + assert zone["stdout"].strip() == "Europe/Paris", zone assert protocol_v1.extension_params(card, BROWSER) == {"chromium": "156.0.8078.4", "clock": "real"} assert relaunched != port and readiness["ready"] is True, readiness assert template["stdout"] == "from the profile template\n" assert not report.failed -async def _browser_process(deployed) -> tuple[list[str], list[str]]: - """The browser process's argv and environment, once it runs: the oldest process with agentenv's debugging flag.""" - find = "pid=$(pgrep -o -f -- --remote-debugging-port=0) && tr '\\0' '\\n' < /proc/$pid/{}" +async def _browser_argv(deployed) -> list[str]: + """The browser process's argv, once it runs: the oldest process with agentenv's debugging flag. Chromium rewrites + its title as one line, so its argv is split on whitespace, which none of these arguments hold.""" + find = "pid=$(pgrep -o -f -- --remote-debugging-port=0) && tr '\\0' ' ' < /proc/$pid/cmdline" for _ in range(60): - argv = await inside.run(deployed, ["sh", "-c", find.format("cmdline")], timeout_s=30) - if argv["exit_code"] == 0 and argv["stdout"]: - environ = await inside.run(deployed, ["sh", "-c", find.format("environ")], timeout_s=30) - return argv["stdout"].splitlines(), environ["stdout"].splitlines() + argv = await inside.run(deployed, ["sh", "-c", find], timeout_s=30) + if argv["exit_code"] == 0 and argv["stdout"].strip(): + return argv["stdout"].split() await asyncio.sleep(1) raise TimeoutError("no Chromium process with --remote-debugging-port=0") From c4393b7a7da8aada4b471bd4e2cfe415e227047f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 11:21:04 -0800 Subject: [PATCH 3/6] fix(computer): a website browser alone keeps the port and command its ready waits on Alone, the launch dropped all of its ready, not just the window wait, so a subclass that also waits on a port or a command lost those checks. It now drops only the window, and the runtime takes the ready timeout from the launch as declared, so the timeout holds even when nothing else is left to wait on. The ready check's wait on the debugging address is devtools_timeout_s. FD-4937 Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/app.py | 5 +++-- .../src/agentenv_protocol/website_browser.py | 11 ++++++++--- .../tests/test_website_browser.py | 18 +++++++++++++++++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 73113371..d5752b8f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -404,7 +404,7 @@ def __init__(self) -> None: launch = declared.launch if declared is not None else None self.launch = self._launch_for(launch) if launch is not None else None self.paths = list(declared.paths) if declared is not None else [] - ready = self.launch.ready if self.launch is not None else None + ready = launch.ready if launch is not None else None self._computer_app = AppRuntime( self.launch, {fn.__name__: getattr(self, fn.__name__) for fn in _methods(cls) if hasattr(fn, _READY_CHECK_ATTR) and _runs_on(getattr(fn, _READY_CHECK_ATTR), on)}, @@ -414,7 +414,8 @@ def __init__(self) -> None: cls.data_timeout_s) def _launch_for(self, launch: Launch) -> Launch: - """The launch its server uses, from the one declared for its OS: as declared.""" + """The launch its server uses, from the one declared for its OS, whose ``ready`` timeout it keeps: as + declared.""" return launch def _on_desktop(self) -> bool: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 36845776..05c6aa4d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -101,6 +101,8 @@ class WebsiteBrowserEnvironment(ComputerAppEnvironment): policy_dir: ClassVar[Path] = Path("/etc/chromium/policies/managed") policies: ClassVar[dict[str, Any]] = {} profile: ClassVar[str] = ".config/chromium" + #: How long its ready check waits on the debugging address's answer. + devtools_timeout_s: ClassVar[float] = 5.0 def __init__(self) -> None: super().__init__() @@ -138,12 +140,15 @@ def agentenv_flags(self) -> list[str]: def _launch_for(self, launch: Launch) -> Launch: """The subclass's launch with agentenv's flags and the start tabs after its command, and the time zone in its - environment; alone, with no window to match or wait for, so its ready checks say when it's ready.""" + environment; alone, with no window to match or wait for, its ``ready`` keeping its port and command, and + none when that leaves it nothing, so its ready checks say when it's ready.""" _refuse_owned(type(self).__name__, launch) update: dict[str, Any] = {"command": [*launch.command, *self.agentenv_flags(), *self.settings.start_urls], "env": {**(launch.env or {}), "TZ": self.settings.time_zone}} if _alone(): - update.update(window=None, ready=None) + ready = launch.ready + waits = ready is not None and (ready.port is not None or bool(ready.command)) + update.update(window=None, ready=ready.model_copy(update={"window": False}) if waits else None) return launch.model_copy(update=update) def _on_desktop(self) -> bool: @@ -179,7 +184,7 @@ async def debugging_address_answers(self) -> bool: except (FileNotFoundError, ValueError): return False try: - async with httpx.AsyncClient(timeout=5, trust_env=False) as http: + async with httpx.AsyncClient(timeout=self.devtools_timeout_s, trust_env=False) as http: return (await http.get(f"http://{address.netloc}/json/version")).status_code == 200 except httpx.HTTPError: return False diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 4e580740..4ca90216 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -78,7 +78,23 @@ def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_w assert launch.command[-3:] == ["--window-size=1024,768", "--headless=new", "about:blank"] assert launch.window is None and launch.ready is None assert Launch.model_validate(launch.model_dump()) == launch - assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == app.DEFAULT_TIMEOUT_S + assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == 30 + + +def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeout(monkeypatch): + @linux(launch=Launch(command=["chromium/chrome"], window={"class": "Chromium"}, + ready=Ready(window=True, port=9000, command=["true"], timeout_s=45))) + class Waiting(WebsiteBrowserEnvironment): + pass + + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + waiting = Waiting() + + assert waiting.launch.ready == Ready(window=False, port=9000, command=["true"], timeout_s=45) + assert Launch.model_validate(waiting.launch.model_dump()) == waiting.launch + assert waiting._computer_app.timeout_s == 45 def test_alone_on_the_open_web_it_runs_headful(monkeypatch): From 70d0d0c051c2af9d86db1863018803515f0cec5b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 12:14:23 -0800 Subject: [PATCH 4/6] feat(computer): a website browser runs on a debugging pipe, not a port Chromium now runs with --remote-debugging-pipe instead of --remote-debugging-port=0. Its app server launches it with a new pipe's ends as fds 3 and 4 and keeps the other ends as self.devtools, a DevToolsPipe. No port listens and no DevToolsActivePort is written, so nothing else in a computer, such as an agent with a terminal, can drive the browser over raw CDP and bypass the way in its task allows. The ready check asks Browser.getVersion over the pipe. @cdp and devtools_address() are gone, since there is no address. Agents get theirs from the front door (M8.6). - session.launch takes fds, moved into place by agentenv_protocol.computer._with_fds, so the server runs no Python between fork and exec. AppRuntime asks the app for them before each launch (_launch_fds) and closes its copies after. - relaunch_on_start: a restarted server stops a browser it finds running and launches it again, since only the launching server holds the pipe. - Headless Chromium takes one start URL on its command line, so it opens the rest over the pipe, behind the first, and is ready once they're open. FD-4937 Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 19 +- .../agentenv_protocol/computer/_with_fds.py | 26 +++ .../src/agentenv_protocol/computer/app.py | 11 +- .../agentenv_protocol/computer/app_runtime.py | 25 ++- .../src/agentenv_protocol/computer/session.py | 11 +- .../src/agentenv_protocol/website_browser.py | 169 +++++++++++++----- .../tests/test_website_browser.py | 152 +++++++++++----- .../computer/website_browser_test.py | 149 +++++++++------ 8 files changed, 402 insertions(+), 160 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 12212928..58ebb64c 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -750,24 +750,29 @@ under the session user's home (`.config/chromium` by default), whose contents fr are the profile's template. The base class adds: - **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in - tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`); `window`, + tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless + the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, 1280 × 800, alone only; `locale`; and `time_zone`, set as `TZ` in its launch's `env`. - **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, - `--remote-debugging-port=0`, `--no-first-run`, `--no-default-browser-check` and `--lang`. With + `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check` and `--lang`. With `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off. Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`, `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`, except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy. - `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, - which agentenv never sets, so the debugging port refuses a WebSocket that carries a page's Origin. + `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or + any other way to open a debugging port. - **Alone, no desktop**: its launch matches and waits on no window, its server adds no launcher and, without `DISPLAY`, waits for no display, and its ready checks say when it's ready. - **`write_policies`**, an install step that writes agentenv's unattended-use policies (`POLICIES`) and the subclass's to `agentenv.json` in `policy_dir`. -- **`@cdp`**, which marks the method returning its debugging address. The default reads - `DevToolsActivePort` from the profile, and `devtools_address()` calls it each time, so it's right - after a relaunch. The ready check `debugging_address_answers` asks that address's `/json/version`. +- **A debugging pipe, not a port.** Its server launches it with a new pipe's ends as its fds 3 and 4 + (`session.launch`'s `fds`, moved into place by `agentenv_protocol.computer._with_fds`) and keeps the + other ends as `self.devtools`, a `DevToolsPipe`: `await self.devtools.call(method, params)` returns + the result or raises `DevToolsError`, and `ConnectionError` with no browser connected. No port + listens and no `DevToolsActivePort` is written, so nothing else in the computer can reach it over + CDP. Since only the server that launched it holds the pipe, a server that restarts stops it and + launches it again. The ready check `devtools_answers` asks `Browser.getVersion` over the pipe. - **`browser/v1`** on its card, params only: `chromium` (its `version`), `clock: "real"` and, alone, `window`. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py new file mode 100644 index 00000000..58470e5f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py @@ -0,0 +1,26 @@ +"""Run a program with fds it inherits moved to the numbers it expects: ``python -m +agentenv_protocol.computer._with_fds = ... -- ``. ``session.launch`` runs an app launched with ``fds`` +through it, so its server runs no Python between its fork and its exec.""" +from __future__ import annotations + +import fcntl +import os +import sys + + +def main(argv: list[str]) -> None: + split = argv.index("--") + moves = [tuple(int(number) for number in pair.split("=")) for pair in argv[:split]] + # Each source copied above every number named first, so no copy, source or target is another. + above = max(number for move in moves for number in move) + 1 + copies = [(fcntl.fcntl(source, fcntl.F_DUPFD_CLOEXEC, above), target) for source, target in moves] + for source, _ in moves: + os.close(source) + for copy, target in copies: + os.dup2(copy, target) + os.close(copy) + os.execvp(argv[split + 1], argv[split + 1:]) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index d5752b8f..890c0af2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -408,7 +408,8 @@ def __init__(self) -> None: self._computer_app = AppRuntime( self.launch, {fn.__name__: getattr(self, fn.__name__) for fn in _methods(cls) if hasattr(fn, _READY_CHECK_ATTR) and _runs_on(getattr(fn, _READY_CHECK_ATTR), on)}, - ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop()) + ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop(), + launch_fds=self._launch_fds, relaunch_on_start=self._relaunch_on_start()) self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, cls.data_timeout_s) @@ -422,6 +423,14 @@ def _on_desktop(self) -> bool: """Whether it runs in a computer's desktop session, which lists it in its menu.""" return True + async def _launch_fds(self) -> dict[int, int]: + """The fds each launch hands it, by the number each gets in it, made new for that launch: none.""" + return {} + + def _relaunch_on_start(self) -> bool: + """Whether only the server that launched it can talk to it, so a server that restarts launches it again.""" + return False + @classmethod def settings_on(cls, os_type: Optional[str]) -> type[BaseModel]: """Its ``Settings`` on ``os_type``, with the fields its OS adds.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 2783ac1c..a8acad62 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -49,11 +49,14 @@ class AppRuntime: take to be ready, why its last launch failed, and each sync check still running in its thread.""" def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float, *, - desktop: bool = True) -> None: + desktop: bool = True, launch_fds: Optional[Callable[[], Awaitable[dict[int, int]]]] = None, + relaunch_on_start: bool = False) -> None: self.launch = launch self.ready_checks = ready_checks self.timeout_s = timeout_s self.desktop = desktop + self.launch_fds = launch_fds + self.relaunch_on_start = relaunch_on_start self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} @@ -61,23 +64,33 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[] async def start(self) -> None: """Once its X display is up, when it has one, add the app's menu launcher on a desktop and launch a boot app, unless the computer has launched it since it booted, at boot or on demand: a server that restarts leaves the - app be.""" + app be, or with ``relaunch_on_start``, for an app only the server that launched it can talk to, stops it and + launches it again.""" if self.launch is None: return if os.environ.get("DISPLAY"): await session.wait_until_up() if self.desktop: session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) - if self.launch.at == "boot" and _launched_at() is None: + if self.launch.at == "boot" and (_launched_at() is None or self.relaunch_on_start): try: + if self.relaunch_on_start: + await self.stop_app() await self.launch_app() except Exception as e: self.launch_failed = f"{type(e).__name__}: {e}" async def launch_app(self) -> int: - """Start the app from its folder, marked as launched; its pid.""" - pid = await session.launch(self.launch.command, Path.cwd(), - {**(self.launch.env or {}), "ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) + """Start the app from its folder, marked as launched, with the fds the app gives it, whose copies here are + closed once it has them; its pid.""" + fds = await self.launch_fds() if self.launch_fds is not None else {} + try: + pid = await session.launch(self.launch.command, Path.cwd(), + {**(self.launch.env or {}), "ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}, + **({"fds": fds} if fds else {})) + finally: + for fd in fds.values(): + os.close(fd) self.launch_failed = None PID.parent.mkdir(parents=True, exist_ok=True) PID.write_text(str(pid)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 1c8cdb72..77b7e7e8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -7,7 +7,9 @@ import os import re import signal +import sys from pathlib import Path +from typing import Optional #: How often a starting session is looked for. POLL_S = 0.2 @@ -22,12 +24,15 @@ async def wait_until_up() -> None: await asyncio.sleep(POLL_S) -async def launch(argv: list[str], cwd: Path, env: dict[str, str]) -> int: +async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[dict[int, int]] = None) -> int: """Start ``argv`` in ``cwd``, in a session of its own so it outlives the server, with the server's environment and - ``env``; its pid.""" + ``env``, and ``fds``, the server's fds it inherits by the number each gets in it; its pid.""" + if fds: + argv = [sys.executable, "-m", "agentenv_protocol.computer._with_fds", + *(f"{source}={target}" for target, source in fds.items()), "--", *argv] process = await asyncio.create_subprocess_exec( *argv, cwd=cwd, env={**os.environ, **env}, start_new_session=True, stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, pass_fds=tuple((fds or {}).values())) reaped = asyncio.ensure_future(process.wait()) _running.add(reaped) reaped.add_done_callback(_running.discard) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 05c6aa4d..50bdd766 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -1,18 +1,17 @@ """WebsiteBrowserEnvironment: a website browser as a computer app, run inside a computer or alone. A subclass says how its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's -command, the unattended-use policies, the debugging address, the settings every browser takes, its skill and -``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy come from variables -agentenv sets, never from its settings.""" +command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser +takes, its skill and ``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy +come from variables agentenv sets, never from its settings.""" from __future__ import annotations -import inspect +import asyncio +import itertools import json import os from pathlib import Path -from typing import Any, Callable, ClassVar -from urllib.parse import urlsplit +from typing import Any, ClassVar, Optional -import httpx from pydantic import BaseModel, Field, field_validator from .agent_env_environment import agent_skill @@ -23,12 +22,12 @@ OPEN_WEB_ENV = "AGENTENV_BROWSER_OPEN_WEB" SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" BROWSER = "urn:agentenv:browser/v1" -#: The flags agentenv sets, by prefix, which a subclass's command may not carry. agentenv never sets -#: ``--remote-allow-origins``, so the debugging port refuses a WebSocket that carries a page's Origin. +#: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging +#: port, since the browser is reached only over its pipe. OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", "--headless", "--window-size", "--lang", "--force-webrtc-ip-handling-policy") -#: Alone, loopback goes through the proxy too, so no page reaches the debugging port. In a computer, the apps on -#: loopback beside it stay reachable; ``<-loopback>`` removes Chromium's own rule, so ``*.localhost`` names still don't. +#: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` +#: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" COMPUTER_BYPASS = "<-loopback>;127.0.0.1;[::1];localhost" #: Managed policies for unattended use: no sign-in, sync, password manager, autofill, metrics, promotions, translation, @@ -56,14 +55,78 @@ - Open sites by their real names, such as `https://slack.com`: the environment serves its sites under those names. - Files to upload are in `~/{UPLOADS}`; downloads land in `~/{DOWNLOADS}`. """ -_CDP_ATTR = "_aee_cdp" +#: The fds a Chromium on ``--remote-debugging-pipe`` reads its CDP messages from and writes its answers to. +PIPE_IN, PIPE_OUT = 3, 4 +#: The largest CDP message the pipe takes, such as a screenshot's. +MAX_MESSAGE_BYTES = 256 * 1024 * 1024 -def cdp(fn: Callable) -> Callable: - """Mark the method that returns the browser's own debugging address, a ``ws://`` URL where it runs; - ``devtools_address`` calls it each time, so the address is right after a relaunch.""" - setattr(fn, _CDP_ATTR, True) - return fn +class DevToolsError(RuntimeError): + """A CDP call the browser answered with an error.""" + + +class DevToolsPipe: + """A CDP connection over a Chromium's debugging pipe, each message JSON ended by a NUL. It connects anew at every + launch; a call with no browser connected, or whose browser exits before answering, raises ConnectionError.""" + + def __init__(self) -> None: + self._ids = itertools.count(1) + self._waiting: dict[int, asyncio.Future] = {} + self._writer: Optional[asyncio.WriteTransport] = None + self._reading: Optional[asyncio.Task] = None + + async def connect(self, to_browser: int, from_browser: int) -> None: + """Take over this end of a new pipe, ending any connection to a browser launched before.""" + self.close() + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader(limit=MAX_MESSAGE_BYTES) + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), os.fdopen(from_browser, "rb", 0)) + self._writer, _ = await loop.connect_write_pipe(asyncio.Protocol, os.fdopen(to_browser, "wb", 0)) + self._reading = asyncio.ensure_future(self._read(reader)) + + async def call(self, method: str, params: Optional[dict] = None, *, session_id: Optional[str] = None, + timeout_s: float = 30.0) -> dict: + """``method``'s result, or DevToolsError with the browser's message.""" + if self._writer is None or self._writer.is_closing(): + raise ConnectionError("no browser is connected over its debugging pipe") + message_id = next(self._ids) + answer = asyncio.get_running_loop().create_future() + self._waiting[message_id] = answer + message = {"id": message_id, "method": method, "params": params or {}, + **({"sessionId": session_id} if session_id else {})} + self._writer.write(json.dumps(message).encode() + b"\0") + try: + answered = await asyncio.wait_for(answer, timeout_s) + finally: + self._waiting.pop(message_id, None) + if "error" in answered: + raise DevToolsError(f"{method}: {answered['error'].get('message', answered['error'])}") + return answered.get("result", {}) + + def close(self) -> None: + if self._reading is not None: + self._reading.cancel() + if self._writer is not None: + self._writer.close() + self._fail_waiting() + self._writer = self._reading = None + + async def _read(self, reader: asyncio.StreamReader) -> None: + try: + while True: + message = json.loads((await reader.readuntil(b"\0"))[:-1]) + answer = self._waiting.get(message.get("id")) + if answer is not None and not answer.done(): + answer.set_result(message) + except (asyncio.IncompleteReadError, ConnectionError): + if self._writer is not None: + self._writer.close() + self._fail_waiting() + + def _fail_waiting(self) -> None: + for answer in self._waiting.values(): + if not answer.done(): + answer.set_exception(ConnectionError("the browser closed its debugging pipe")) class Window(BaseModel): @@ -101,10 +164,13 @@ class WebsiteBrowserEnvironment(ComputerAppEnvironment): policy_dir: ClassVar[Path] = Path("/etc/chromium/policies/managed") policies: ClassVar[dict[str, Any]] = {} profile: ClassVar[str] = ".config/chromium" - #: How long its ready check waits on the debugging address's answer. + #: How long its ready check waits on the browser's answer over its pipe. devtools_timeout_s: ClassVar[float] = 5.0 def __init__(self) -> None: + #: Its CDP connection, over the pipe its server launches it with. + self.devtools = DevToolsPipe() + self._start_tabs: Optional[asyncio.Future] = None super().__init__() for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) @@ -124,7 +190,7 @@ def agentenv_flags(self) -> list[str]: """The flags agentenv owns. A browser alone goes out only through its site proxy, so it won't launch without one; in a computer it goes through one when the computer has one, loopback aside.""" settings, alone, proxy = self.settings, _alone(), os.environ.get(SITE_PROXY_ENV) - flags = [f"--user-data-dir={self.profile_dir()}", "--remote-debugging-port=0", "--no-first-run", + flags = [f"--user-data-dir={self.profile_dir()}", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", f"--lang={settings.locale}"] if proxy: flags += [f"--proxy-server={proxy}", f"--proxy-bypass-list={ALONE_BYPASS if alone else COMPUTER_BYPASS}", @@ -134,8 +200,8 @@ def agentenv_flags(self) -> list[str]: "names none") if alone: flags.append(f"--window-size={settings.window.width},{settings.window.height}") - if not _open_web(): - flags.append("--headless=new") + if _headless(): + flags.append("--headless=new") return flags def _launch_for(self, launch: Launch) -> Launch: @@ -143,7 +209,7 @@ def _launch_for(self, launch: Launch) -> Launch: environment; alone, with no window to match or wait for, its ``ready`` keeping its port and command, and none when that leaves it nothing, so its ready checks say when it's ready.""" _refuse_owned(type(self).__name__, launch) - update: dict[str, Any] = {"command": [*launch.command, *self.agentenv_flags(), *self.settings.start_urls], + update: dict[str, Any] = {"command": [*launch.command, *self.agentenv_flags(), *self._command_line_tabs()], "env": {**(launch.env or {}), "TZ": self.settings.time_zone}} if _alone(): ready = launch.ready @@ -154,6 +220,30 @@ def _launch_for(self, launch: Launch) -> Launch: def _on_desktop(self) -> bool: return not _alone() + def _command_line_tabs(self) -> list[str]: + """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" + return self.settings.start_urls[:1] if _headless() else self.settings.start_urls + + async def _launch_fds(self) -> dict[int, int]: + """A new debugging pipe for this launch: the browser's ends as its fds 3 and 4, this server's ends its CDP + connection, over which the start tabs its command line can't open are opened.""" + to_browser_read, to_browser_write = os.pipe() + from_browser_read, from_browser_write = os.pipe() + await self.devtools.connect(to_browser_write, from_browser_read) + rest = self.settings.start_urls[len(self._command_line_tabs()):] + self._start_tabs = asyncio.ensure_future(self._open_tabs(rest)) if rest else None + if self._start_tabs is not None: + self._start_tabs.add_done_callback(lambda done: done.cancelled() or done.exception()) + return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} + + async def _open_tabs(self, urls: list[str]) -> None: + """Each of ``urls`` in a tab behind the active one, in order.""" + for url in urls: + await self.devtools.call("Target.createTarget", {"url": url, "background": True}) + + def _relaunch_on_start(self) -> bool: + return True + @install_step(scope="system") def write_policies(self) -> None: """agentenv's unattended-use policies, with the subclass's over them, where its build reads them.""" @@ -161,33 +251,15 @@ def write_policies(self) -> None: (self.policy_dir / "agentenv.json").write_text(json.dumps({**POLICIES, **self.policies}, indent=1) + "\n", encoding="utf-8") - @cdp - def devtools(self) -> str: - """Chromium's own: the port it picked and the browser's path, as it writes them to ``DevToolsActivePort`` in - its profile.""" - port, path = (self.profile_dir() / "DevToolsActivePort").read_text().split()[:2] - return f"ws://127.0.0.1:{port}{path}" - - async def devtools_address(self) -> str: - """What the method marked ``@cdp`` returns now; FileNotFoundError until the browser has written its address.""" - for klass in type(self).__mro__: - for name, member in vars(klass).items(): - if getattr(member, _CDP_ATTR, False): - address = getattr(self, name)() - return await address if inspect.isawaitable(address) else address - raise TypeError(f"{type(self).__name__} marks no method @cdp") - @ready_check - async def debugging_address_answers(self) -> bool: - try: - address = urlsplit(await self.devtools_address()) - except (FileNotFoundError, ValueError): - return False + async def devtools_answers(self) -> bool: + """It answers over its pipe and has opened its start tabs.""" try: - async with httpx.AsyncClient(timeout=self.devtools_timeout_s, trust_env=False) as http: - return (await http.get(f"http://{address.netloc}/json/version")).status_code == 200 - except httpx.HTTPError: + await self.devtools.call("Browser.getVersion", timeout_s=self.devtools_timeout_s) + except (ConnectionError, DevToolsError, asyncio.TimeoutError): return False + tabs = self._start_tabs + return tabs is None or (tabs.done() and not tabs.cancelled() and tabs.exception() is None) def declared_extensions(self) -> list[EnvironmentExtension]: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} @@ -211,6 +283,11 @@ def _open_web() -> bool: return os.environ.get(OPEN_WEB_ENV) == "1" +def _headless() -> bool: + """Alone it runs headless, but headful on Xvfb for the open web, whose sites block headless browsers.""" + return _alone() and not _open_web() + + def _refuse_owned(name: str, launch: Launch) -> None: if owned := [arg for arg in launch.command if arg.startswith(OWNED_FLAGS)]: raise ValueError(f"{name}'s launch sets {', '.join(owned)}, which agentenv sets for every website browser") diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 4ca90216..7820e761 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -1,11 +1,13 @@ """WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, and the proxy rule for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings; -its launch run with no desktop when alone; the debugging address read again each time and checked for readiness; its -policies, card params and skill; and an app that isn't a browser launched and declared as before.""" +its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready +check asks, and a browser launched again when its server restarts; its policies, card params and skill; and an app that +isn't a browser launched and declared as before.""" from __future__ import annotations import asyncio import json +import sys import pytest @@ -13,10 +15,30 @@ from agentenv_protocol.computer import app_runtime from agentenv_protocol.computer.platform import Platform from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, - POLICIES, SITE_PROXY_ENV, SKILL, WebsiteBrowserEnvironment, cdp) + POLICIES, SITE_PROXY_ENV, SKILL, DevToolsError, + WebsiteBrowserEnvironment) PROXY = "http://127.0.0.1:3128" -OWNED = ["--user-data-dir", "--remote-debugging-port=0", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] +OWNED = ["--user-data-dir", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] +# A browser that answers CDP on its debugging pipe, fds 3 and 4, as a Chromium does: Browser.getVersion with its product, +# Target.createTarget by writing the tab's URL to tabs.txt beside it, anything else with an error. +FAKE_BROWSER = """import json, os +buffer = b"" +while chunk := os.read(3, 65536): + buffer += chunk + while b"\\0" in buffer: + raw, buffer = buffer.split(b"\\0", 1) + message = json.loads(raw) + if message["method"] == "Browser.getVersion": + answer = {"result": {"product": "Fake/1"}} + elif message["method"] == "Target.createTarget" and message["params"]["background"]: + with open("tabs.txt", "a") as tabs: + tabs.write(message["params"]["url"] + "\\n") + answer = {"result": {"targetId": message["params"]["url"]}} + else: + answer = {"error": {"code": -32601, "message": "unknown method"}} + os.write(4, json.dumps({"id": message["id"], **answer}).encode() + b"\\0") +""" @linux(launch=Launch(command=["chromium/chrome", "--no-sandbox"], window={"class": "Chromium"}, @@ -146,8 +168,8 @@ async def test_alone_its_server_launches_it_at_once_in_its_environment_with_no_d monkeypatch.chdir(tmp_path) launched, waited, launchers = [], [], [] - async def launch(argv, cwd, env): - launched.append(env) + async def launch(argv, cwd, env, fds=None): + launched.append((env, sorted(fds or {}))) return 4242 async def up(): @@ -160,7 +182,7 @@ async def up(): await Browser()._computer_app.start() - assert launched == [{"TZ": "UTC", "ENVIRONMENT_NAME": "chromium"}] and waited == [] and launchers == [] + assert launched == [({"TZ": "UTC", "ENVIRONMENT_NAME": "chromium"}, [3, 4])] and waited == [] and launchers == [] @pytest.mark.asyncio @@ -169,7 +191,7 @@ async def test_in_a_computer_its_server_waits_for_the_display_and_adds_its_launc monkeypatch.chdir(tmp_path) waited, launchers = [], [] - async def launch(argv, cwd, env): + async def launch(argv, cwd, env, fds=None): return 4242 async def up(): @@ -185,38 +207,92 @@ async def up(): assert waited == [True] and launchers == ["chromium"] +@linux(launch=Launch(command=[sys.executable, "-c", FAKE_BROWSER], ready=Ready(command=["true"]))) +class FakeBrowser(WebsiteBrowserEnvironment): + pass + + @pytest.mark.asyncio -async def test_its_debugging_address_is_read_again_each_time_and_answers_its_ready_check(browser_home): - browser = Browser() - profile = browser_home / ".config/chromium" - profile.mkdir(parents=True) - with pytest.raises(FileNotFoundError): - await browser.devtools_address() - unwritten = await browser.debugging_address_answers() - - server = await asyncio.start_server(_json_version, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - (profile / "DevToolsActivePort").write_text("1\n/devtools/browser/first\n") - first = await browser.devtools_address() - (profile / "DevToolsActivePort").write_text(f"{port}\n/devtools/browser/second\n") - async with server: - relaunched = await browser.devtools_address() - answers = await browser.debugging_address_answers() - - assert unwritten is False - assert first == "ws://127.0.0.1:1/devtools/browser/first" - assert relaunched == f"ws://127.0.0.1:{port}/devtools/browser/second" and answers is True - assert "debugging_address_answers" in browser._computer_app.ready_checks +async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_ready_check_asks(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + browser = FakeBrowser() + runtime = browser._computer_app + unlaunched = await browser.devtools_answers() + + await runtime.launch_app() + version = await browser.devtools.call("Browser.getVersion") + with pytest.raises(DevToolsError, match="unknown method"): + await browser.devtools.call("Page.navigate", {"url": "about:blank"}) + launched = await browser.devtools_answers() + await runtime.stop_app() + stopped = await browser.devtools_answers() + await runtime.launch_app() + relaunched = await browser.devtools_answers() + await runtime.stop_app() + browser.devtools.close() + + assert unlaunched is False and version == {"product": "Fake/1"} and launched is True + assert stopped is False and relaunched is True + assert "devtools_answers" in runtime.ready_checks @pytest.mark.asyncio -async def test_a_subclass_that_marks_its_own_method_cdp_gives_that_address(): - class Elsewhere(Browser): - @cdp - async def own(self) -> str: - return "ws://127.0.0.1:9333/devtools/browser/own" +async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app", "https://github.com"]) + browser = FakeBrowser() - assert await Elsewhere().devtools_address() == "ws://127.0.0.1:9333/devtools/browser/own" + await browser._computer_app.launch_app() + for _ in range(50): + if ready := await browser.devtools_answers(): + break + await asyncio.sleep(0.1) + await browser._computer_app.stop_app() + browser.devtools.close() + + assert browser.launch.command[-2:] == ["--headless=new", "https://slack.com"] + assert ready is True and (tmp_path / "tabs.txt").read_text() == "https://linear.app\nhttps://github.com\n" + + +@pytest.mark.asyncio +async def test_a_restarted_server_launches_a_browser_again_and_leaves_any_other_app_be(monkeypatch, tmp_path): + monkeypatch.setenv("DISPLAY", ":1") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + launched, stopped = [], [] + + async def launch(argv, cwd, env, fds=None): + launched.append((argv[0], sorted(fds or {}))) + return 4242 + + async def stop(pid, timeout_s): + stopped.append(pid) + + async def up(): + pass + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "stop", stop) + monkeypatch.setattr(session, "alive", lambda pid: True) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) + (tmp_path / "launched").mkdir() + (tmp_path / "launched" / tmp_path.name).touch() + (tmp_path / ".agentenv").mkdir() + (tmp_path / ".agentenv" / "app.pid").write_text("4241") + + await Mail()._computer_app.start() + left_alone = list(launched) + browser = Browser() + await browser._computer_app.start() + browser.devtools.close() + + assert left_alone == [] and stopped == [4241] + assert launched == [("chromium/chrome", [3, 4])] def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads_them(monkeypatch, tmp_path): @@ -259,9 +335,3 @@ def test_an_app_that_isnt_a_browser_launches_and_declares_as_before(): assert stored["os"]["linux"]["launch"] == {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True, "timeout_s": 60.0}} - -async def _json_version(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - await reader.readuntil(b"\r\n\r\n") - writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\n\r\n{}") - await writer.drain() - writer.close() diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index 8b2c5083..303e204b 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -1,17 +1,20 @@ """WebsiteBrowserEnvironment in a computer, as the website-browser test bundle's desktop composes its fixture browser on -the ubuntu-24.04 base: gate 3 makes this machine's arch ready with it, which its debugging-address check is part of; -deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale, through no proxy, -with agentenv's policies where its build reads them, and opens pages on loopback, which see the deploy's time zone; its card declares -browser/v1; and a reset relaunches it on its profile template, at a debugging address its ready check finds. Needs -Docker and the local registry on localhost:5000.""" +the ubuntu-24.04 base: gate 3 makes this machine's arch ready with it, which its check over the debugging pipe is part +of; deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale, through no proxy, +with agentenv's policies where its build reads them and with no debugging port anything in the computer could find; its +first start tab sees the deploy's time zone, and a page on loopback loads; its card declares browser/v1; and a reset +relaunches it on its profile template, on a new pipe its ready check answers over. Needs Docker and the local registry +on localhost:5000.""" import asyncio import json import platform -import shlex +import re import httpx import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client from agentenv_protocol import client as protocol_v1 from agentenv_protocol.computer import apps @@ -27,18 +30,15 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] -# Serves two pages on loopback while it opens the one at its path in a new tab, through Chromium's HTTP debugging -# endpoints, and prints that tab's title once the page has set one: "/" names itself, "/tz.html" the time zone it sees. -# The tab is brought to the front, since on the desktop a background tab doesn't finish loading. -OPEN_TAB = """import http.server, json, sys, threading, time, urllib.request as u -port, path = sys.argv[1:] -PAGES = {"/": b"Loopback page", - "/tz.html": b"loading"} +TIME_ZONE_PAGE = ("data:text/html,loading" + "") +LOOPBACK = "http://127.0.0.1:8765/" +# Serves a page on loopback for the seconds it's given. +SERVE_LOOPBACK = """import http.server, sys, threading, time class Page(http.server.BaseHTTPRequestHandler): def do_GET(self): - body = PAGES.get(self.path, b"") - self.send_response(200 if body else 404) + body = b"Loopback page" + self.send_response(200) self.send_header("Content-Type", "text/html") self.send_header("Content-Length", str(len(body))) self.end_headers() @@ -47,18 +47,36 @@ def log_message(self, *args): pass server = http.server.ThreadingHTTPServer(("127.0.0.1", 8765), Page) threading.Thread(target=server.serve_forever, daemon=True).start() -url = f"http://127.0.0.1:8765{path}" -direct = u.build_opener(u.ProxyHandler({})) -tab = json.load(direct.open(u.Request(f"http://127.0.0.1:{port}/json/new?{url}", method="PUT"), timeout=10)) -direct.open(f"http://127.0.0.1:{port}/json/activate/{tab['id']}", timeout=10) -for _ in range(30): - titles = [t["title"] for t in json.load(direct.open(f"http://127.0.0.1:{port}/json/list", timeout=10)) - if t.get("url") == url and t["title"] != "loading" and not url.removeprefix("http://").startswith(t["title"])] - if titles: - print(titles[0]) - break - time.sleep(1) +time.sleep(float(sys.argv[1])) """ +# The browser's pid and argv, and how many TCP sockets anything in its session listens on: its oldest process with +# agentenv's pipe flag, whose title Chromium rewrites as one line. +BROWSER_PROCESS = """import glob, json, os, subprocess +pid = int(subprocess.run(["pgrep", "-o", "-f", "--", "--remote-debugging-pipe"], capture_output=True, text=True, + check=True).stdout) +listening = set() +for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in open(table).read().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + listening.add(fields[9]) +sockets = set() +for proc in glob.glob("/proc/[0-9]*"): + try: + if os.getsid(int(proc.rsplit("/", 1)[1])) != os.getsid(pid): + continue + for fd in os.listdir(f"{proc}/fd"): + link = os.readlink(f"{proc}/fd/{fd}") + if link.startswith("socket:["): + sockets.add(link[8:-1]) + except OSError: + continue +argv = open(f"/proc/{pid}/cmdline", "rb").read().replace(b"\\0", b" ").decode().split() +print(json.dumps({"pid": pid, "argv": argv, "listening": len(listening & sockets)})) +""" +# Each window the window manager manages, its class and its title. +WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' + 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') def test_gate_3_makes_this_machines_arch_ready_with_the_fixture_browser(website_browser_desktop): @@ -69,63 +87,82 @@ def test_gate_3_makes_this_machines_arch_ready_with_the_fixture_browser(website_ @pytest.mark.asyncio -async def test_chromium_runs_with_agentenvs_flags_and_a_reset_relaunches_it_on_its_template(website_browser_desktop): +async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaunches_it_on_its_template( + website_browser_desktop): + settings = {"locale": "fr-FR", "time_zone": "Europe/Paris", "start_urls": [TIME_ZONE_PAGE, LOOPBACK]} with namespace_routing(): - deployed = await Env.get(website_browser_desktop.id).deploy( - env_params={"app_settings": {"browser": {"locale": "fr-FR", "time_zone": "Europe/Paris"}}}) + deployed = await Env.get(website_browser_desktop.id).deploy(env_params={"app_settings": {"browser": settings}}) context = TaskStepContext(deployed_envs=[deployed]) try: - argv = await _browser_argv(deployed) - port = await _devtools_port(deployed) + zone = await _window_titled(deployed, "Europe/Paris") + browser = await _browser_process(deployed) + port_file = await inside.run(deployed, ["sh", "-c", "test -e ~/.config/chromium/DevToolsActivePort"], + timeout_s=30) policies = await inside.run( deployed, ["cat", "/etc/opt/chrome_for_testing/policies/managed/agentenv.json"], timeout_s=30) - title = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, "/"], timeout_s=90) - zone = await inside.run(deployed, ["python3", "-c", OPEN_TAB, port, "/tz.html"], timeout_s=90) + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_LOOPBACK, "60"], + timeout_s=90)) + await asyncio.sleep(3) + await _keys(deployed, ["Control", "Tab"], ["F5"]) + loopback = await _window_titled(deployed, "Loopback page") + await serving card = await _app(deployed, WELL_KNOWN_PATH) - await protocol_v1.reset_data(f"{deployed.environment_url}/svc/browser", headers=deployed.control_headers) - relaunched = await _devtools_port(deployed, other_than=port) readiness = await _ready(deployed, card) + await protocol_v1.reset_data(f"{deployed.environment_url}/svc/browser", headers=deployed.control_headers) + relaunched = await _browser_process(deployed, other_than=browser["pid"]) + relaunched_readiness = await _ready(deployed, card) template = await inside.run(deployed, ["sh", "-c", "cat ~/.config/chromium/agentenv-template.txt"], timeout_s=30) finally: report = await teardown_run(context) + argv = browser["argv"] assert argv[0].endswith("chromium/chrome/chrome") and argv[1:4] == ["--no-sandbox", "--disable-dev-shm-usage", "--class=Chromium"] assert argv[4].startswith("--user-data-dir=") and argv[4].endswith("/.config/chromium") - assert argv[5:9] == ["--remote-debugging-port=0", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR"] - assert argv[9:] == ["about:blank"] + assert argv[5:9] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR"] + assert argv[9:] == [TIME_ZONE_PAGE, LOOPBACK] + assert browser["listening"] == 0 and port_file["exit_code"] != 0 + assert zone.startswith("Europe/Paris") and loopback.startswith("Loopback page") assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( policies["stdout"]).items() - assert title["stdout"].strip() == "Loopback page", title - assert zone["stdout"].strip() == "Europe/Paris", zone assert protocol_v1.extension_params(card, BROWSER) == {"chromium": "156.0.8078.4", "clock": "real"} - assert relaunched != port and readiness["ready"] is True, readiness + assert readiness["ready"] is True, readiness + assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 + assert relaunched_readiness["ready"] is True, relaunched_readiness assert template["stdout"] == "from the profile template\n" assert not report.failed -async def _browser_argv(deployed) -> list[str]: - """The browser process's argv, once it runs: the oldest process with agentenv's debugging flag. Chromium rewrites - its title as one line, so its argv is split on whitespace, which none of these arguments hold.""" - find = "pid=$(pgrep -o -f -- --remote-debugging-port=0) && tr '\\0' ' ' < /proc/$pid/cmdline" +async def _window_titled(deployed, title: str) -> str: + """The title of the computer's Chromium window once one starts with ``title``.""" for _ in range(60): - argv = await inside.run(deployed, ["sh", "-c", find], timeout_s=30) - if argv["exit_code"] == 0 and argv["stdout"].strip(): - return argv["stdout"].split() + listed = (await inside.run(deployed, ["sh", "-c", WINDOWS], timeout_s=30))["stdout"] + classes = re.findall(r'WM_CLASS\(STRING\) = .*"([^"]*)"\n_NET_WM_NAME\(UTF8_STRING\) = "(.*)"', listed) + if found := [name for cls, name in classes if cls == "Chromium" and name.startswith(title)]: + return found[0] await asyncio.sleep(1) - raise TimeoutError("no Chromium process with --remote-debugging-port=0") + raise TimeoutError(f"no Chromium window titled {title!r}: {listed}") + + +async def _keys(deployed, *chords: list[str]) -> None: + """Each chord pressed through the computer's own tools, as an agent presses it.""" + async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + for keys in chords: + pressed = await session.call_tool("computer_key", {"keys": keys}) + assert not pressed.isError, pressed -async def _devtools_port(deployed, other_than: str | None = None) -> str: - """The port in the profile's DevToolsActivePort, once Chromium has written one other than ``other_than``.""" +async def _browser_process(deployed, other_than: int | None = None) -> dict: + """The browser process's pid, argv and listening sockets, once one runs other than ``other_than``.""" for _ in range(60): - read = await inside.run(deployed, ["sh", "-c", "head -1 ~/.config/chromium/DevToolsActivePort"], timeout_s=30) - port = read["stdout"].strip() - if read["exit_code"] == 0 and port and port != other_than: - return port + found = await inside.run(deployed, ["python3", "-c", BROWSER_PROCESS], timeout_s=30) + if found["exit_code"] == 0 and (process := json.loads(found["stdout"]))["pid"] != other_than: + return process await asyncio.sleep(1) - raise TimeoutError(f"no DevToolsActivePort other than {shlex.quote(str(other_than))}") + raise TimeoutError(f"no Chromium on --remote-debugging-pipe other than {other_than}: {found}") async def _app(deployed, path: str) -> dict: From 4aafd2c3114fde2e19e15d6f02910a0ea5c08ceb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 12:23:34 -0800 Subject: [PATCH 5/6] fix(computer): run the fd helper by path, so it loads only the standard library python -m agentenv_protocol.computer._with_fds imported the protocol package and the whole computer SDK, pydantic included, in the child before it exec'd the app, with the pipe fds open. Each launch was slower, and a failed import there broke it. session.launch now runs _with_fds.py by path with -I -S: about 64 ms here, against about 236 ms for importing the computer package. The helper also tolerates no moves. FD-4937 Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 2 +- .../src/agentenv_protocol/computer/_with_fds.py | 8 ++++---- .../src/agentenv_protocol/computer/session.py | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 58ebb64c..1269289f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -767,7 +767,7 @@ are the profile's template. The base class adds: - **`write_policies`**, an install step that writes agentenv's unattended-use policies (`POLICIES`) and the subclass's to `agentenv.json` in `policy_dir`. - **A debugging pipe, not a port.** Its server launches it with a new pipe's ends as its fds 3 and 4 - (`session.launch`'s `fds`, moved into place by `agentenv_protocol.computer._with_fds`) and keeps the + (`session.launch`'s `fds`, moved into place by `computer/_with_fds.py`, run alone with `-I -S`) and keeps the other ends as `self.devtools`, a `DevToolsPipe`: `await self.devtools.call(method, params)` returns the result or raises `DevToolsError`, and `ConnectionError` with no browser connected. No port listens and no `DevToolsActivePort` is written, so nothing else in the computer can reach it over diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py index 58470e5f..6914d52a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py @@ -1,6 +1,6 @@ -"""Run a program with fds it inherits moved to the numbers it expects: ``python -m -agentenv_protocol.computer._with_fds = ... -- ``. ``session.launch`` runs an app launched with ``fds`` -through it, so its server runs no Python between its fork and its exec.""" +"""Run a program with fds it inherits moved to the numbers it expects: ``python -I -S _with_fds.py = ... -- +``. ``session.launch`` runs an app launched with ``fds`` through it, by path, so its server runs no Python between +its fork and its exec and this loads nothing but the standard library.""" from __future__ import annotations import fcntl @@ -12,7 +12,7 @@ def main(argv: list[str]) -> None: split = argv.index("--") moves = [tuple(int(number) for number in pair.split("=")) for pair in argv[:split]] # Each source copied above every number named first, so no copy, source or target is another. - above = max(number for move in moves for number in move) + 1 + above = max((number for move in moves for number in move), default=2) + 1 copies = [(fcntl.fcntl(source, fcntl.F_DUPFD_CLOEXEC, above), target) for source, target in moves] for source, _ in moves: os.close(source) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 77b7e7e8..f9ca0d26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -13,6 +13,8 @@ #: How often a starting session is looked for. POLL_S = 0.2 +#: Moves the fds an app is launched with into place, run on its own so it loads nothing but the standard library. +WITH_FDS = Path(__file__).with_name("_with_fds.py") #: The processes launched into the session, kept until they exit so each is reaped. _running: set[asyncio.Task] = set() @@ -28,8 +30,8 @@ async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[ """Start ``argv`` in ``cwd``, in a session of its own so it outlives the server, with the server's environment and ``env``, and ``fds``, the server's fds it inherits by the number each gets in it; its pid.""" if fds: - argv = [sys.executable, "-m", "agentenv_protocol.computer._with_fds", - *(f"{source}={target}" for target, source in fds.items()), "--", *argv] + argv = [sys.executable, "-I", "-S", str(WITH_FDS), *(f"{source}={target}" for target, source in fds.items()), + "--", *argv] process = await asyncio.create_subprocess_exec( *argv, cwd=cwd, env={**os.environ, **env}, start_new_session=True, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, pass_fds=tuple((fds or {}).values())) From a4b417a66f1a28090c4b5548ea8e9215acc629f9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 12:41:26 -0800 Subject: [PATCH 6/6] fix(computer): a restarted server relaunches an app it launched on demand too With relaunch_on_start, an app that was launched since boot, at boot or on demand, is stopped and launched again when its server restarts, so a browser launched on demand gets a new pipe instead of being stranded. One that was never launched stays unlaunched. DevToolsPipe's default wait is the named call_timeout_s. FD-4937 Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_runtime.py | 8 +++-- .../src/agentenv_protocol/website_browser.py | 7 +++-- .../tests/test_website_browser.py | 30 +++++++++++++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index a8acad62..071eda60 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -65,16 +65,18 @@ async def start(self) -> None: """Once its X display is up, when it has one, add the app's menu launcher on a desktop and launch a boot app, unless the computer has launched it since it booted, at boot or on demand: a server that restarts leaves the app be, or with ``relaunch_on_start``, for an app only the server that launched it can talk to, stops it and - launches it again.""" + launches it again, whether it was launched at boot or on demand.""" if self.launch is None: return if os.environ.get("DISPLAY"): await session.wait_until_up() if self.desktop: session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) - if self.launch.at == "boot" and (_launched_at() is None or self.relaunch_on_start): + launched = _launched_at() is not None + relaunch = self.relaunch_on_start and launched + if relaunch or (self.launch.at == "boot" and not launched): try: - if self.relaunch_on_start: + if relaunch: await self.stop_app() await self.launch_app() except Exception as e: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 50bdd766..27597160 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -69,6 +69,9 @@ class DevToolsPipe: """A CDP connection over a Chromium's debugging pipe, each message JSON ended by a NUL. It connects anew at every launch; a call with no browser connected, or whose browser exits before answering, raises ConnectionError.""" + #: How long a call waits for the browser's answer when it names no wait of its own. + call_timeout_s: ClassVar[float] = 30.0 + def __init__(self) -> None: self._ids = itertools.count(1) self._waiting: dict[int, asyncio.Future] = {} @@ -85,7 +88,7 @@ async def connect(self, to_browser: int, from_browser: int) -> None: self._reading = asyncio.ensure_future(self._read(reader)) async def call(self, method: str, params: Optional[dict] = None, *, session_id: Optional[str] = None, - timeout_s: float = 30.0) -> dict: + timeout_s: Optional[float] = None) -> dict: """``method``'s result, or DevToolsError with the browser's message.""" if self._writer is None or self._writer.is_closing(): raise ConnectionError("no browser is connected over its debugging pipe") @@ -96,7 +99,7 @@ async def call(self, method: str, params: Optional[dict] = None, *, session_id: **({"sessionId": session_id} if session_id else {})} self._writer.write(json.dumps(message).encode() + b"\0") try: - answered = await asyncio.wait_for(answer, timeout_s) + answered = await asyncio.wait_for(answer, self.call_timeout_s if timeout_s is None else timeout_s) finally: self._waiting.pop(message_id, None) if "error" in answered: diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 7820e761..b6d7aa2d 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -53,6 +53,11 @@ class Mail(ComputerAppEnvironment): pass +@linux(launch=Launch(at="demand", command=["chromium/chrome"], window={"class": "Chromium"}, ready=Ready(window=True))) +class OnDemandBrowser(WebsiteBrowserEnvironment): + pass + + @pytest.fixture(autouse=True) def browser_home(monkeypatch, tmp_path): """A browser on Linux, its session user's home under ``tmp_path``, inside a computer with no site proxy.""" @@ -287,12 +292,31 @@ async def up(): await Mail()._computer_app.start() left_alone = list(launched) - browser = Browser() + browser, on_demand = Browser(), OnDemandBrowser() await browser._computer_app.start() + await on_demand._computer_app.start() browser.devtools.close() + on_demand.devtools.close() + + assert left_alone == [] and stopped == [4241, 4242] + assert launched == [("chromium/chrome", [3, 4]), ("chromium/chrome", [3, 4])] + + +@pytest.mark.asyncio +async def test_a_browser_on_demand_that_was_never_launched_is_left_unlaunched_by_its_server(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + launched = [] + + async def launch(argv, cwd, env, fds=None): + launched.append(argv) + return 4242 + + monkeypatch.setattr(session, "launch", launch) + + await OnDemandBrowser()._computer_app.start() - assert left_alone == [] and stopped == [4241] - assert launched == [("chromium/chrome", [3, 4])] + assert launched == [] def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads_them(monkeypatch, tmp_path):