diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 8b40aa5c..1269289f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -687,7 +687,7 @@ class MailApp(ComputerAppEnvironment): - **`launch=Launch(...)`** says how it starts in the desktop session: with it (`at="boot"`) or when the agent opens it (`"demand"`). `ready=Ready(...)` says when it's ready, within `timeout_s` of its launch: a window matching `launch.window` by its class, instance or title, a port answering on - loopback, or a command exiting 0. `@ready_check` marks a method for readiness that needs logic (a + loopback, or a command exiting 0. `env` sets variables in the app's environment, over its server's. `@ready_check` marks a method for readiness that needs logic (a sync one runs in a thread), and `@install_step(scope=...)` one a computer runs as it installs the app, after its run scripts, offline and without the class's `__init__`. Each runs on every OS the app installs on, or on those its `os=` names (`@install_step(os="linux")`, `@ready_check(os=[...])`); @@ -716,9 +716,10 @@ desktop session, and `ready` (GET: an `AppReadiness` of `ready`, `problems`, `la and `timeout_s`), which runs the app's readiness checks then, at once, each within 10 s. A server that can't start, or whose start in the session fails (adding the launcher, say), writes why to `.agentenv/server.failure` in the folder and exits, and the driver reports that in its place. Once -the session is up the server adds the app's launcher to the session user's menu, which runs `python --m agentenv_protocol.computer.launch ` to ask the server, so the server starts every copy of -the app, and launches a `boot` app, once per boot. The driver finds each app by the socket in its +the X display is up, when `DISPLAY` names one, the server adds the app's launcher to the session user's +menu, which runs `python -m agentenv_protocol.computer.launch ` to ask the server, so the +server starts every copy of the app, and launches a `boot` app, once per boot. An app that runs with +no desktop, a website browser alone, gets no launcher. The driver finds each app by the socket in its folder: the computer is ready once every app's server answers and each `boot` app, and each app with no launch, says it's ready; a `demand` app counts once something launches it. What the server does in the session (`agentenv_protocol.computer.session`) is Linux's. @@ -739,6 +740,44 @@ requests to the app's server. env stores, and `python -m agentenv_protocol.computer._declare ` prints it as JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. +### Website browsers + +`agentenv_protocol.website_browser.WebsiteBrowserEnvironment` is the computer app class for a Chromium +website browser, run inside a computer or alone. A subclass declares its build as any app does, its +`Launch` running the browser itself. It sets `version`, its build, and `policy_dir`, where that build +reads managed policies; `policies` adds its own over agentenv's, and `profile` is its profile folder +under the session user's home (`.config/chromium` by default), whose contents from its install steps +are the profile's template. The base class adds: + +- **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in + tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless + the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, + 1280 × 800, alone only; `locale`; and `time_zone`, set as `TZ` in its launch's `env`. +- **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, + `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check` and `--lang`. With + `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off. + Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`, + `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go + through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`, + except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy. + `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or + any other way to open a debugging port. +- **Alone, no desktop**: its launch matches and waits on no window, its server adds no launcher and, + without `DISPLAY`, waits for no display, and its ready checks say when it's ready. +- **`write_policies`**, an install step that writes agentenv's unattended-use policies (`POLICIES`) + and the subclass's to `agentenv.json` in `policy_dir`. +- **A debugging pipe, not a port.** Its server launches it with a new pipe's ends as its fds 3 and 4 + (`session.launch`'s `fds`, moved into place by `computer/_with_fds.py`, run alone with `-I -S`) and keeps the + other ends as `self.devtools`, a `DevToolsPipe`: `await self.devtools.call(method, params)` returns + the result or raises `DevToolsError`, and `ConnectionError` with no browser connected. No port + listens and no `DevToolsActivePort` is written, so nothing else in the computer can reach it over + CDP. Since only the server that launched it holds the pipe, a server that restarts stops it and + launches it again. The ready check `devtools_answers` asks `Browser.getVersion` over the pipe. +- **`browser/v1`** on its card, params only: `chromium` (its `version`), `clock: "real"` and, alone, + `window`. +- **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in + `~/Downloads`, folders it makes. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py new file mode 100644 index 00000000..6914d52a --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_with_fds.py @@ -0,0 +1,26 @@ +"""Run a program with fds it inherits moved to the numbers it expects: ``python -I -S _with_fds.py = ... -- +``. ``session.launch`` runs an app launched with ``fds`` through it, by path, so its server runs no Python between +its fork and its exec and this loads nothing but the standard library.""" +from __future__ import annotations + +import fcntl +import os +import sys + + +def main(argv: list[str]) -> None: + split = argv.index("--") + moves = [tuple(int(number) for number in pair.split("=")) for pair in argv[:split]] + # Each source copied above every number named first, so no copy, source or target is another. + above = max((number for move in moves for number in move), default=2) + 1 + copies = [(fcntl.fcntl(source, fcntl.F_DUPFD_CLOEXEC, above), target) for source, target in moves] + for source, _ in moves: + os.close(source) + for copy, target in copies: + os.dup2(copy, target) + os.close(copy) + os.execvp(argv[split + 1], argv[split + 1:]) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 6772fc08..890c0af2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -169,6 +169,8 @@ class Launch(BaseModel): #: Matches the app's window by its WM_CLASS ``class`` or ``instance`` and its ``title``, as given. window: Optional[dict[str, str]] = None ready: Optional[Ready] = None + #: Variables set in the app's environment, over its server's. + env: Optional[dict[str, str]] = None @field_validator("window") @classmethod @@ -399,17 +401,36 @@ def __init__(self) -> None: self.settings = cls.settings_on(on).model_validate(values) self.build_settings = _built_settings(cls, on) declared = cls._installs.get(on) - self.launch = declared.launch if declared is not None else None + launch = declared.launch if declared is not None else None + self.launch = self._launch_for(launch) if launch is not None else None self.paths = list(declared.paths) if declared is not None else [] - ready = self.launch.ready if self.launch is not None else None + ready = launch.ready if launch is not None else None self._computer_app = AppRuntime( self.launch, {fn.__name__: getattr(self, fn.__name__) for fn in _methods(cls) if hasattr(fn, _READY_CHECK_ATTR) and _runs_on(getattr(fn, _READY_CHECK_ATTR), on)}, - ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S) + ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop(), + launch_fds=self._launch_fds, relaunch_on_start=self._relaunch_on_start()) self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, cls.data_timeout_s) + def _launch_for(self, launch: Launch) -> Launch: + """The launch its server uses, from the one declared for its OS, whose ``ready`` timeout it keeps: as + declared.""" + return launch + + def _on_desktop(self) -> bool: + """Whether it runs in a computer's desktop session, which lists it in its menu.""" + return True + + async def _launch_fds(self) -> dict[int, int]: + """The fds each launch hands it, by the number each gets in it, made new for that launch: none.""" + return {} + + def _relaunch_on_start(self) -> bool: + """Whether only the server that launched it can talk to it, so a server that restarts launches it again.""" + return False + @classmethod def settings_on(cls, os_type: Optional[str]) -> type[BaseModel]: """Its ``Settings`` on ``os_type``, with the fields its OS adds.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 89c5bcd8..071eda60 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -48,30 +48,51 @@ class AppRuntime: """An app's server's state: its ``launch``, its ``@ready_check`` methods by name, bound to the app, how long it may take to be ready, why its last launch failed, and each sync check still running in its thread.""" - def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float) -> None: + def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float, *, + desktop: bool = True, launch_fds: Optional[Callable[[], Awaitable[dict[int, int]]]] = None, + relaunch_on_start: bool = False) -> None: self.launch = launch self.ready_checks = ready_checks self.timeout_s = timeout_s + self.desktop = desktop + self.launch_fds = launch_fds + self.relaunch_on_start = relaunch_on_start self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} async def start(self) -> None: - """Once the desktop session is up, add the app's menu launcher and launch a boot app, unless the computer has - launched it since it booted, at boot or on demand: a server that restarts leaves the app be.""" + """Once its X display is up, when it has one, add the app's menu launcher on a desktop and launch a boot app, + unless the computer has launched it since it booted, at boot or on demand: a server that restarts leaves the + app be, or with ``relaunch_on_start``, for an app only the server that launched it can talk to, stops it and + launches it again, whether it was launched at boot or on demand.""" if self.launch is None: return - await session.wait_until_up() - session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) - if self.launch.at == "boot" and _launched_at() is None: + if os.environ.get("DISPLAY"): + await session.wait_until_up() + if self.desktop: + session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) + launched = _launched_at() is not None + relaunch = self.relaunch_on_start and launched + if relaunch or (self.launch.at == "boot" and not launched): try: + if relaunch: + await self.stop_app() await self.launch_app() except Exception as e: self.launch_failed = f"{type(e).__name__}: {e}" async def launch_app(self) -> int: - """Start the app from its folder, marked as launched; its pid.""" - pid = await session.launch(self.launch.command, Path.cwd(), {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) + """Start the app from its folder, marked as launched, with the fds the app gives it, whose copies here are + closed once it has them; its pid.""" + fds = await self.launch_fds() if self.launch_fds is not None else {} + try: + pid = await session.launch(self.launch.command, Path.cwd(), + {**(self.launch.env or {}), "ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}, + **({"fds": fds} if fds else {})) + finally: + for fd in fds.values(): + os.close(fd) self.launch_failed = None PID.parent.mkdir(parents=True, exist_ok=True) PID.write_text(str(pid)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 1c8cdb72..f9ca0d26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -7,10 +7,14 @@ import os import re import signal +import sys from pathlib import Path +from typing import Optional #: How often a starting session is looked for. POLL_S = 0.2 +#: Moves the fds an app is launched with into place, run on its own so it loads nothing but the standard library. +WITH_FDS = Path(__file__).with_name("_with_fds.py") #: The processes launched into the session, kept until they exit so each is reaped. _running: set[asyncio.Task] = set() @@ -22,12 +26,15 @@ async def wait_until_up() -> None: await asyncio.sleep(POLL_S) -async def launch(argv: list[str], cwd: Path, env: dict[str, str]) -> int: +async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[dict[int, int]] = None) -> int: """Start ``argv`` in ``cwd``, in a session of its own so it outlives the server, with the server's environment and - ``env``; its pid.""" + ``env``, and ``fds``, the server's fds it inherits by the number each gets in it; its pid.""" + if fds: + argv = [sys.executable, "-I", "-S", str(WITH_FDS), *(f"{source}={target}" for target, source in fds.items()), + "--", *argv] process = await asyncio.create_subprocess_exec( *argv, cwd=cwd, env={**os.environ, **env}, start_new_session=True, stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, pass_fds=tuple((fds or {}).values())) reaped = asyncio.ensure_future(process.wait()) _running.add(reaped) reaped.add_done_callback(_running.discard) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py new file mode 100644 index 00000000..27597160 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -0,0 +1,296 @@ +"""WebsiteBrowserEnvironment: a website browser as a computer app, run inside a computer or alone. A subclass says how +its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's +command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser +takes, its skill and ``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy +come from variables agentenv sets, never from its settings.""" +from __future__ import annotations + +import asyncio +import itertools +import json +import os +from pathlib import Path +from typing import Any, ClassVar, Optional + +from pydantic import BaseModel, Field, field_validator + +from .agent_env_environment import agent_skill +from .computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check +from .types import EnvironmentExtension + +ALONE_ENV = "AGENTENV_BROWSER_ALONE" +OPEN_WEB_ENV = "AGENTENV_BROWSER_OPEN_WEB" +SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" +BROWSER = "urn:agentenv:browser/v1" +#: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging +#: port, since the browser is reached only over its pipe. +OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", + "--headless", "--window-size", "--lang", "--force-webrtc-ip-handling-policy") +#: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` +#: removes Chromium's own rule, so ``*.localhost`` names still don't. +ALONE_BYPASS = "<-loopback>" +COMPUTER_BYPASS = "<-loopback>;127.0.0.1;[::1];localhost" +#: Managed policies for unattended use: no sign-in, sync, password manager, autofill, metrics, promotions, translation, +#: background mode, component updates, default-browser check, download prompt or warning bar for its flags. +POLICIES: dict[str, Any] = { + "BrowserSignin": 0, + "SyncDisabled": True, + "PasswordManagerEnabled": False, + "AutofillSettings": [{"url_pattern": "*", "blocked_types": ["all"]}], + "MetricsReportingEnabled": False, + "PromotionalTabsEnabled": False, + "TranslateEnabled": False, + "BackgroundModeEnabled": False, + "ComponentUpdatesEnabled": False, + "DefaultBrowserSettingEnabled": False, + "PromptForDownloadLocation": False, + "CommandLineFlagSecurityWarningsEnabled": False, + "DownloadDirectory": "${user_home}/Downloads", +} +#: Where an agent's files to upload are, and where downloads land, under the session user's home. +UPLOADS = "Uploads" +DOWNLOADS = "Downloads" +SKILL = f"""# Website browser + +- Open sites by their real names, such as `https://slack.com`: the environment serves its sites under those names. +- Files to upload are in `~/{UPLOADS}`; downloads land in `~/{DOWNLOADS}`. +""" +#: The fds a Chromium on ``--remote-debugging-pipe`` reads its CDP messages from and writes its answers to. +PIPE_IN, PIPE_OUT = 3, 4 +#: The largest CDP message the pipe takes, such as a screenshot's. +MAX_MESSAGE_BYTES = 256 * 1024 * 1024 + + +class DevToolsError(RuntimeError): + """A CDP call the browser answered with an error.""" + + +class DevToolsPipe: + """A CDP connection over a Chromium's debugging pipe, each message JSON ended by a NUL. It connects anew at every + launch; a call with no browser connected, or whose browser exits before answering, raises ConnectionError.""" + + #: How long a call waits for the browser's answer when it names no wait of its own. + call_timeout_s: ClassVar[float] = 30.0 + + def __init__(self) -> None: + self._ids = itertools.count(1) + self._waiting: dict[int, asyncio.Future] = {} + self._writer: Optional[asyncio.WriteTransport] = None + self._reading: Optional[asyncio.Task] = None + + async def connect(self, to_browser: int, from_browser: int) -> None: + """Take over this end of a new pipe, ending any connection to a browser launched before.""" + self.close() + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader(limit=MAX_MESSAGE_BYTES) + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), os.fdopen(from_browser, "rb", 0)) + self._writer, _ = await loop.connect_write_pipe(asyncio.Protocol, os.fdopen(to_browser, "wb", 0)) + self._reading = asyncio.ensure_future(self._read(reader)) + + async def call(self, method: str, params: Optional[dict] = None, *, session_id: Optional[str] = None, + timeout_s: Optional[float] = None) -> dict: + """``method``'s result, or DevToolsError with the browser's message.""" + if self._writer is None or self._writer.is_closing(): + raise ConnectionError("no browser is connected over its debugging pipe") + message_id = next(self._ids) + answer = asyncio.get_running_loop().create_future() + self._waiting[message_id] = answer + message = {"id": message_id, "method": method, "params": params or {}, + **({"sessionId": session_id} if session_id else {})} + self._writer.write(json.dumps(message).encode() + b"\0") + try: + answered = await asyncio.wait_for(answer, self.call_timeout_s if timeout_s is None else timeout_s) + finally: + self._waiting.pop(message_id, None) + if "error" in answered: + raise DevToolsError(f"{method}: {answered['error'].get('message', answered['error'])}") + return answered.get("result", {}) + + def close(self) -> None: + if self._reading is not None: + self._reading.cancel() + if self._writer is not None: + self._writer.close() + self._fail_waiting() + self._writer = self._reading = None + + async def _read(self, reader: asyncio.StreamReader) -> None: + try: + while True: + message = json.loads((await reader.readuntil(b"\0"))[:-1]) + answer = self._waiting.get(message.get("id")) + if answer is not None and not answer.done(): + answer.set_result(message) + except (asyncio.IncompleteReadError, ConnectionError): + if self._writer is not None: + self._writer.close() + self._fail_waiting() + + def _fail_waiting(self) -> None: + for answer in self._waiting.values(): + if not answer.done(): + answer.set_exception(ConnectionError("the browser closed its debugging pipe")) + + +class Window(BaseModel): + width: int = Field(1280, gt=0) + height: int = Field(800, gt=0) + + +class BrowserSettings(BaseModel): + """The settings every website browser takes, per deploy; a browser's own ``Settings`` adds to them.""" + + #: Opened in tabs at launch, the first active. + start_urls: list[str] = Field(default_factory=lambda: ["about:blank"]) + #: Its window when it runs alone; in a computer the screen sets it. + window: Window = Field(default_factory=Window) + locale: str = "en-US" + #: The IANA time zone its pages see. + time_zone: str = "UTC" + + @field_validator("start_urls") + @classmethod + def _no_flags(cls, urls: list[str]) -> list[str]: + if flags := [url for url in urls if url.startswith("-")]: + raise ValueError(f"start_urls are URLs, not flags: {flags}") + return urls + + +class WebsiteBrowserEnvironment(ComputerAppEnvironment): + """Subclass it as any computer app, with ``@computer_app`` and ``@linux``, its launch running the browser itself, + and set ``version``, its build, and ``policy_dir``, where that build reads managed policies. ``policies`` adds its + own to agentenv's, and ``profile`` is its profile folder under the session user's home, holding what its install + steps put there.""" + + Settings = BrowserSettings + version: ClassVar[str] = "" + policy_dir: ClassVar[Path] = Path("/etc/chromium/policies/managed") + policies: ClassVar[dict[str, Any]] = {} + profile: ClassVar[str] = ".config/chromium" + #: How long its ready check waits on the browser's answer over its pipe. + devtools_timeout_s: ClassVar[float] = 5.0 + + def __init__(self) -> None: + #: Its CDP connection, over the pipe its server launches it with. + self.devtools = DevToolsPipe() + self._start_tabs: Optional[asyncio.Future] = None + super().__init__() + for folder in (UPLOADS, DOWNLOADS): + (Path.home() / folder).mkdir(exist_ok=True) + + @classmethod + def declaration(cls) -> AppDeclaration: + """What the class declares; ValueError for a launch whose command carries a flag agentenv owns.""" + for declared in cls._installs.values(): + if declared.launch is not None: + _refuse_owned(cls.__name__, declared.launch) + return super().declaration() + + def profile_dir(self) -> Path: + return Path.home() / self.profile + + def agentenv_flags(self) -> list[str]: + """The flags agentenv owns. A browser alone goes out only through its site proxy, so it won't launch without + one; in a computer it goes through one when the computer has one, loopback aside.""" + settings, alone, proxy = self.settings, _alone(), os.environ.get(SITE_PROXY_ENV) + flags = [f"--user-data-dir={self.profile_dir()}", "--remote-debugging-pipe", "--no-first-run", + "--no-default-browser-check", f"--lang={settings.locale}"] + if proxy: + flags += [f"--proxy-server={proxy}", f"--proxy-bypass-list={ALONE_BYPASS if alone else COMPUTER_BYPASS}", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"] + elif alone: + raise ValueError(f"a website browser alone goes out only through its site proxy, and {SITE_PROXY_ENV} " + "names none") + if alone: + flags.append(f"--window-size={settings.window.width},{settings.window.height}") + if _headless(): + flags.append("--headless=new") + return flags + + def _launch_for(self, launch: Launch) -> Launch: + """The subclass's launch with agentenv's flags and the start tabs after its command, and the time zone in its + environment; alone, with no window to match or wait for, its ``ready`` keeping its port and command, and + none when that leaves it nothing, so its ready checks say when it's ready.""" + _refuse_owned(type(self).__name__, launch) + update: dict[str, Any] = {"command": [*launch.command, *self.agentenv_flags(), *self._command_line_tabs()], + "env": {**(launch.env or {}), "TZ": self.settings.time_zone}} + if _alone(): + ready = launch.ready + waits = ready is not None and (ready.port is not None or bool(ready.command)) + update.update(window=None, ready=ready.model_copy(update={"window": False}) if waits else None) + return launch.model_copy(update=update) + + def _on_desktop(self) -> bool: + return not _alone() + + def _command_line_tabs(self) -> list[str]: + """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" + return self.settings.start_urls[:1] if _headless() else self.settings.start_urls + + async def _launch_fds(self) -> dict[int, int]: + """A new debugging pipe for this launch: the browser's ends as its fds 3 and 4, this server's ends its CDP + connection, over which the start tabs its command line can't open are opened.""" + to_browser_read, to_browser_write = os.pipe() + from_browser_read, from_browser_write = os.pipe() + await self.devtools.connect(to_browser_write, from_browser_read) + rest = self.settings.start_urls[len(self._command_line_tabs()):] + self._start_tabs = asyncio.ensure_future(self._open_tabs(rest)) if rest else None + if self._start_tabs is not None: + self._start_tabs.add_done_callback(lambda done: done.cancelled() or done.exception()) + return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} + + async def _open_tabs(self, urls: list[str]) -> None: + """Each of ``urls`` in a tab behind the active one, in order.""" + for url in urls: + await self.devtools.call("Target.createTarget", {"url": url, "background": True}) + + def _relaunch_on_start(self) -> bool: + return True + + @install_step(scope="system") + def write_policies(self) -> None: + """agentenv's unattended-use policies, with the subclass's over them, where its build reads them.""" + self.policy_dir.mkdir(parents=True, exist_ok=True) + (self.policy_dir / "agentenv.json").write_text(json.dumps({**POLICIES, **self.policies}, indent=1) + "\n", + encoding="utf-8") + + @ready_check + async def devtools_answers(self) -> bool: + """It answers over its pipe and has opened its start tabs.""" + try: + await self.devtools.call("Browser.getVersion", timeout_s=self.devtools_timeout_s) + except (ConnectionError, DevToolsError, asyncio.TimeoutError): + return False + tabs = self._start_tabs + return tabs is None or (tabs.done() and not tabs.cancelled() and tabs.exception() is None) + + def declared_extensions(self) -> list[EnvironmentExtension]: + params: dict[str, Any] = {"chromium": self.version, "clock": "real"} + if _alone(): + params["window"] = self.settings.window.model_dump() + browser = EnvironmentExtension( + uri=BROWSER, params=params, + description="A website browser agentenv runs: its build, the clock its pages see and, alone, its window.") + return [*super().declared_extensions(), browser] + + @agent_skill + def browser_skill(self) -> tuple[str, str]: + return "How to use this website browser.", SKILL + + +def _alone() -> bool: + return os.environ.get(ALONE_ENV) == "1" + + +def _open_web() -> bool: + return os.environ.get(OPEN_WEB_ENV) == "1" + + +def _headless() -> bool: + """Alone it runs headless, but headful on Xvfb for the open web, whose sites block headless browsers.""" + return _alone() and not _open_web() + + +def _refuse_owned(name: str, launch: Launch) -> None: + if owned := [arg for arg in launch.command if arg.startswith(OWNED_FLAGS)]: + raise ValueError(f"{name}'s launch sets {', '.join(owned)}, which agentenv sets for every website browser") diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py new file mode 100644 index 00000000..b6d7aa2d --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -0,0 +1,361 @@ +"""WebsiteBrowserEnvironment: agentenv's flags after a subclass's command, in a computer and alone, and the proxy rule +for each; a subclass refused for carrying one of them; the start tabs, the time zone and the window from its settings; +its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready +check asks, and a browser launched again when its server restarts; its policies, card params and skill; and an app that +isn't a browser launched and declared as before.""" +from __future__ import annotations + +import asyncio +import json +import sys + +import pytest + +from agentenv_protocol.computer import APP_SETTINGS_ENV, ComputerAppEnvironment, Launch, Ready, app, linux, session +from agentenv_protocol.computer import app_runtime +from agentenv_protocol.computer.platform import Platform +from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, + POLICIES, SITE_PROXY_ENV, SKILL, DevToolsError, + WebsiteBrowserEnvironment) + +PROXY = "http://127.0.0.1:3128" +OWNED = ["--user-data-dir", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] +# A browser that answers CDP on its debugging pipe, fds 3 and 4, as a Chromium does: Browser.getVersion with its product, +# Target.createTarget by writing the tab's URL to tabs.txt beside it, anything else with an error. +FAKE_BROWSER = """import json, os +buffer = b"" +while chunk := os.read(3, 65536): + buffer += chunk + while b"\\0" in buffer: + raw, buffer = buffer.split(b"\\0", 1) + message = json.loads(raw) + if message["method"] == "Browser.getVersion": + answer = {"result": {"product": "Fake/1"}} + elif message["method"] == "Target.createTarget" and message["params"]["background"]: + with open("tabs.txt", "a") as tabs: + tabs.write(message["params"]["url"] + "\\n") + answer = {"result": {"targetId": message["params"]["url"]}} + else: + answer = {"error": {"code": -32601, "message": "unknown method"}} + os.write(4, json.dumps({"id": message["id"], **answer}).encode() + b"\\0") +""" + + +@linux(launch=Launch(command=["chromium/chrome", "--no-sandbox"], window={"class": "Chromium"}, + ready=Ready(window=True, timeout_s=30))) +class Browser(WebsiteBrowserEnvironment): + version = "156.0.8078.4" + policies = {"DefaultGeolocationSetting": 2} + + +@linux(launch=Launch(command=["mail"], window={"class": "Mail"}, ready=Ready(window=True))) +class Mail(ComputerAppEnvironment): + pass + + +@linux(launch=Launch(at="demand", command=["chromium/chrome"], window={"class": "Chromium"}, ready=Ready(window=True))) +class OnDemandBrowser(WebsiteBrowserEnvironment): + pass + + +@pytest.fixture(autouse=True) +def browser_home(monkeypatch, tmp_path): + """A browser on Linux, its session user's home under ``tmp_path``, inside a computer with no site proxy.""" + monkeypatch.setattr(app, "detect", lambda: Platform(os="linux")) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + (tmp_path / "home").mkdir() + monkeypatch.setenv("ENVIRONMENT_NAME", "chromium") + for name in (ALONE_ENV, OPEN_WEB_ENV, SITE_PROXY_ENV, APP_SETTINGS_ENV): + monkeypatch.delenv(name, raising=False) + return tmp_path / "home" + + +def _settings(monkeypatch, **values) -> None: + monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values})) + + +def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home): + launch = Browser().launch + + profile = f"--user-data-dir={browser_home / '.config/chromium'}" + assert launch.command == ["chromium/chrome", "--no-sandbox", profile, *OWNED[1:], "about:blank"] + assert launch.env == {"TZ": "UTC"} + assert launch.window == {"class": "Chromium"} and launch.ready == Ready(window=True, timeout_s=30) + + +def test_in_a_computer_with_a_site_proxy_only_loopback_bypasses_it(monkeypatch): + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + command = Browser().launch.command + + assert [f"--proxy-server={PROXY}", f"--proxy-bypass-list={COMPUTER_BYPASS}", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"] == command[7:10] + assert not any(flag.startswith(("--headless", "--window-size")) for flag in command) + + +def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_waits_on_no_window(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + _settings(monkeypatch, window={"width": 1024, "height": 768}) + + browser = Browser() + launch = browser.launch + + assert f"--proxy-bypass-list={ALONE_BYPASS}" in launch.command + assert launch.command[-3:] == ["--window-size=1024,768", "--headless=new", "about:blank"] + assert launch.window is None and launch.ready is None + assert Launch.model_validate(launch.model_dump()) == launch + assert browser._computer_app.desktop is False and browser._computer_app.timeout_s == 30 + + +def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeout(monkeypatch): + @linux(launch=Launch(command=["chromium/chrome"], window={"class": "Chromium"}, + ready=Ready(window=True, port=9000, command=["true"], timeout_s=45))) + class Waiting(WebsiteBrowserEnvironment): + pass + + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + waiting = Waiting() + + assert waiting.launch.ready == Ready(window=False, port=9000, command=["true"], timeout_s=45) + assert Launch.model_validate(waiting.launch.model_dump()) == waiting.launch + assert waiting._computer_app.timeout_s == 45 + + +def test_alone_on_the_open_web_it_runs_headful(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(OPEN_WEB_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + + command = Browser().launch.command + + assert "--window-size=1280,800" in command and not any(flag.startswith("--headless") for flag in command) + + +def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch): + monkeypatch.setenv(ALONE_ENV, "1") + + with pytest.raises(ValueError, match=SITE_PROXY_ENV): + Browser() + + +@pytest.mark.parametrize("flag", ["--remote-debugging-port=9222", "--remote-allow-origins=*", "--headless", + "--proxy-server=http://elsewhere:3128", "--user-data-dir=/tmp/profile"]) +def test_a_subclass_whose_command_carries_a_flag_agentenv_owns_is_refused_when_declared(flag): + @linux(launch=Launch(command=["chromium/chrome", flag])) + class Widened(WebsiteBrowserEnvironment): + pass + + with pytest.raises(ValueError, match="agentenv sets for every website browser"): + Widened.declaration() + + +def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_url_is_a_flag(monkeypatch): + _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app"], locale="fr-FR", + time_zone="Europe/Paris") + launch = Browser().launch + _settings(monkeypatch, start_urls=["--remote-allow-origins=*"]) + + assert launch.command[-2:] == ["https://slack.com", "https://linear.app"] and "--lang=fr-FR" in launch.command + assert launch.env == {"TZ": "Europe/Paris"} + with pytest.raises(ValueError, match="start_urls are URLs, not flags"): + Browser() + + +@pytest.mark.asyncio +async def test_alone_its_server_launches_it_at_once_in_its_environment_with_no_display_wait_or_launcher( + monkeypatch, tmp_path): + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.chdir(tmp_path) + launched, waited, launchers = [], [], [] + + async def launch(argv, cwd, env, fds=None): + launched.append((env, sorted(fds or {}))) + return 4242 + + async def up(): + waited.append(True) + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append(name)) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + + await Browser()._computer_app.start() + + assert launched == [({"TZ": "UTC", "ENVIRONMENT_NAME": "chromium"}, [3, 4])] and waited == [] and launchers == [] + + +@pytest.mark.asyncio +async def test_in_a_computer_its_server_waits_for_the_display_and_adds_its_launcher(monkeypatch, tmp_path): + monkeypatch.setenv("DISPLAY", ":1") + monkeypatch.chdir(tmp_path) + waited, launchers = [], [] + + async def launch(argv, cwd, env, fds=None): + return 4242 + + async def up(): + waited.append(True) + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append(name)) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + + await Browser()._computer_app.start() + + assert waited == [True] and launchers == ["chromium"] + + +@linux(launch=Launch(command=[sys.executable, "-c", FAKE_BROWSER], ready=Ready(command=["true"]))) +class FakeBrowser(WebsiteBrowserEnvironment): + pass + + +@pytest.mark.asyncio +async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_ready_check_asks(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + browser = FakeBrowser() + runtime = browser._computer_app + unlaunched = await browser.devtools_answers() + + await runtime.launch_app() + version = await browser.devtools.call("Browser.getVersion") + with pytest.raises(DevToolsError, match="unknown method"): + await browser.devtools.call("Page.navigate", {"url": "about:blank"}) + launched = await browser.devtools_answers() + await runtime.stop_app() + stopped = await browser.devtools_answers() + await runtime.launch_app() + relaunched = await browser.devtools_answers() + await runtime.stop_app() + browser.devtools.close() + + assert unlaunched is False and version == {"product": "Fake/1"} and launched is True + assert stopped is False and relaunched is True + assert "devtools_answers" in runtime.ready_checks + + +@pytest.mark.asyncio +async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app", "https://github.com"]) + browser = FakeBrowser() + + await browser._computer_app.launch_app() + for _ in range(50): + if ready := await browser.devtools_answers(): + break + await asyncio.sleep(0.1) + await browser._computer_app.stop_app() + browser.devtools.close() + + assert browser.launch.command[-2:] == ["--headless=new", "https://slack.com"] + assert ready is True and (tmp_path / "tabs.txt").read_text() == "https://linear.app\nhttps://github.com\n" + + +@pytest.mark.asyncio +async def test_a_restarted_server_launches_a_browser_again_and_leaves_any_other_app_be(monkeypatch, tmp_path): + monkeypatch.setenv("DISPLAY", ":1") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + launched, stopped = [], [] + + async def launch(argv, cwd, env, fds=None): + launched.append((argv[0], sorted(fds or {}))) + return 4242 + + async def stop(pid, timeout_s): + stopped.append(pid) + + async def up(): + pass + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "stop", stop) + monkeypatch.setattr(session, "alive", lambda pid: True) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) + (tmp_path / "launched").mkdir() + (tmp_path / "launched" / tmp_path.name).touch() + (tmp_path / ".agentenv").mkdir() + (tmp_path / ".agentenv" / "app.pid").write_text("4241") + + await Mail()._computer_app.start() + left_alone = list(launched) + browser, on_demand = Browser(), OnDemandBrowser() + await browser._computer_app.start() + await on_demand._computer_app.start() + browser.devtools.close() + on_demand.devtools.close() + + assert left_alone == [] and stopped == [4241, 4242] + assert launched == [("chromium/chrome", [3, 4]), ("chromium/chrome", [3, 4])] + + +@pytest.mark.asyncio +async def test_a_browser_on_demand_that_was_never_launched_is_left_unlaunched_by_its_server(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + launched = [] + + async def launch(argv, cwd, env, fds=None): + launched.append(argv) + return 4242 + + monkeypatch.setattr(session, "launch", launch) + + await OnDemandBrowser()._computer_app.start() + + assert launched == [] + + +def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads_them(monkeypatch, tmp_path): + monkeypatch.setattr(Browser, "policy_dir", tmp_path / "policies") + + Browser.__new__(Browser).write_policies() + + written = json.loads((tmp_path / "policies" / "agentenv.json").read_text()) + assert written == {**POLICIES, "DefaultGeolocationSetting": 2} + assert written["DownloadDirectory"] == "${user_home}/Downloads" + assert [step.name for step in Browser.declaration().install_steps] == ["write_policies"] + + +def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): + in_computer = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + monkeypatch.setenv(ALONE_ENV, "1") + monkeypatch.setenv(SITE_PROXY_ENV, PROXY) + alone = next(e for e in Browser().declared_extensions() if e.uri == BROWSER) + + assert in_computer.params == {"chromium": "156.0.8078.4", "clock": "real"} + assert alone.params == {"chromium": "156.0.8078.4", "clock": "real", "window": {"width": 1280, "height": 800}} + + +@pytest.mark.asyncio +async def test_its_skill_names_sites_by_their_names_and_its_upload_and_download_folders_which_it_makes(browser_home): + browser = Browser() + + skill = await browser.browser_skill("website-browser") + + assert Browser.declaration().skill.method == "browser_skill" + assert skill["description"] == "How to use this website browser." and skill["skill_md"].endswith(SKILL) + assert (browser_home / "Uploads").is_dir() and (browser_home / "Downloads").is_dir() + + +def test_an_app_that_isnt_a_browser_launches_and_declares_as_before(): + mail = Mail() + stored = Mail.declaration().model_dump(mode="json", exclude_none=True) + + assert mail.launch is Mail._installs["linux"].launch and mail._computer_app.desktop is True + assert stored["os"]["linux"]["launch"] == {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, + "ready": {"window": True, "timeout_s": 60.0}} + diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index ff25cc3f..5d14ba95 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,6 +1,6 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, the cua-driver test bundle's and the chromium test bundle's, each composed once -for a module's tests; +shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's and the website-browser +test bundle's, each composed once for a module's tests; ``state``, local stores for a test that runs a bundle; and ``docker``, which runs the docker CLI for them.""" import logging @@ -27,6 +27,7 @@ BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" CUA_DRIVER = Path(__file__).parent / "cua-driver" CHROMIUM = Path(__file__).parent / "chromium" +WEBSITE_BROWSER = Path(__file__).parent / "website-browser" @pytest.fixture(scope="session") @@ -59,6 +60,13 @@ def chromium_desktop(tmp_path_factory): yield from _composed(tmp_path_factory.mktemp("chromium-desktop"), parse_bundle(CHROMIUM)) +@pytest.fixture(scope="module") +def website_browser_desktop(tmp_path_factory): + """The website-browser test bundle's desktop, the ubuntu-24.04 base and the reference driver with its fixture + website browser, composed as ``desktop`` is.""" + yield from _composed(tmp_path_factory.mktemp("website-browser-desktop"), parse_bundle(WEBSITE_BROWSER)) + + def _composed(root: Path, bundle): patch = pytest.MonkeyPatch() patch.chdir(root) diff --git a/tst/integration/computer/website-browser/envs/browser/browser.py b/tst/integration/computer/website-browser/envs/browser/browser.py new file mode 100644 index 00000000..0f74539e --- /dev/null +++ b/tst/integration/computer/website-browser/envs/browser/browser.py @@ -0,0 +1,48 @@ +"""A website browser for the WebsiteBrowserEnvironment tests: the Chrome for Testing build agentenv's chromium ships, +launched by its command itself so agentenv's flags reach it, with a file its install step leaves in its profile as the +profile's template, which it keeps as its data, so a reset relaunches it on that template.""" +import subprocess +from pathlib import Path + +from agentenv_protocol.computer import Download, Launch, Packages, Ready, computer_app, install_step, linux +from agentenv_protocol.website_browser import WebsiteBrowserEnvironment + +VERSION = "156.0.8078.4" +_ARCHIVES = {"amd64": ("linux64/chrome-linux64.zip", + "43fd31766c7533a02dda2ded5ad50c2f861cd5400be7509c0abad71129f79d79"), + "arm64": ("linux-arm64/chrome-linux-arm64.zip", + "910a12fff6170202195f72ef3d353801c7063d301ab80b1f51b77d791fda60ce")} +DOWNLOADS = [Download(url=f"https://storage.googleapis.com/chrome-for-testing-public/{VERSION}/{path}", sha256=sha256, + to=f"chromium/chrome-{arch}.zip", arch=arch) + for arch, (path, sha256) in _ARCHIVES.items()] +PACKAGES = ["fonts-liberation", "libasound2t64", "libatk-bridge2.0-0t64", "libatk1.0-0t64", "libatspi2.0-0t64", + "libcairo2", "libcups2t64", "libdbus-1-3", "libdrm2", "libexpat1", "libgbm1", "libglib2.0-0t64", + "libgtk-3-0t64", "libnspr4", "libnss3", "libpango-1.0-0", "libvulkan1", "libx11-6", "libxcb1", + "libxcomposite1", "libxdamage1", "libxext6", "libxfixes3", "libxkbcommon0", "libxrandr2", "libxshmfence1", + "unzip"] +PROFILE = ".config/chromium" +TEMPLATE = "agentenv-template.txt" + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@linux(packages=Packages(apt=PACKAGES), downloads=DOWNLOADS, paths=[PROFILE], + launch=Launch(command=["chromium/chrome/chrome", "--no-sandbox", "--disable-dev-shm-usage", "--class=Chromium"], + window={"class": "Chromium"}, ready=Ready(window=True))) +class FixtureBrowser(WebsiteBrowserEnvironment): + version = VERSION + policy_dir = Path("/etc/opt/chrome_for_testing/policies/managed") + + @install_step(scope="system") + def unpack(self) -> None: + folder = Path.cwd() / "chromium" + [archive] = folder.glob("chrome-*.zip") + subprocess.run(["unzip", "-q", str(archive), "-d", str(folder / "unpacked")], check=True) + [top] = (folder / "unpacked").iterdir() + top.rename(folder / "chrome") + (folder / "unpacked").rmdir() + archive.unlink() + + @install_step(scope="session") + def profile_template(self) -> None: + (Path.home() / PROFILE).mkdir(parents=True, exist_ok=True) + (Path.home() / PROFILE / TEMPLATE).write_text("from the profile template\n") diff --git a/tst/integration/computer/website-browser/envs/browser/env.toml b/tst/integration/computer/website-browser/envs/browser/env.toml new file mode 100644 index 00000000..21707fc6 --- /dev/null +++ b/tst/integration/computer/website-browser/envs/browser/env.toml @@ -0,0 +1,4 @@ +type = "computer_app" +module = "browser.py:FixtureBrowser" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/website-browser/envs/desktop/env.toml b/tst/integration/computer/website-browser/envs/desktop/env.toml new file mode 100644 index 00000000..1a71716b --- /dev/null +++ b/tst/integration/computer/website-browser/envs/desktop/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" +apps = ["browser"] diff --git a/tst/integration/computer/website-browser/tasks/desktop.json b/tst/integration/computer/website-browser/tasks/desktop.json new file mode 100644 index 00000000..67f3b315 --- /dev/null +++ b/tst/integration/computer/website-browser/tasks/desktop.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"} +] diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py new file mode 100644 index 00000000..303e204b --- /dev/null +++ b/tst/integration/computer/website_browser_test.py @@ -0,0 +1,182 @@ +"""WebsiteBrowserEnvironment in a computer, as the website-browser test bundle's desktop composes its fixture browser on +the ubuntu-24.04 base: gate 3 makes this machine's arch ready with it, which its check over the debugging pipe is part +of; deployed, Chromium runs with agentenv's flags after the fixture's command, on the deploy's locale, through no proxy, +with agentenv's policies where its build reads them and with no debugging port anything in the computer could find; its +first start tab sees the deploy's time zone, and a page on loopback loads; its card declares browser/v1; and a reset +relaunches it on its profile template, on a new pipe its ready check answers over. Needs Docker and the local registry +on localhost:5000.""" + +import asyncio +import json +import platform +import re + +import httpx +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import apps +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] +TIME_ZONE_PAGE = ("data:text/html,loading" + "") +LOOPBACK = "http://127.0.0.1:8765/" +# Serves a page on loopback for the seconds it's given. +SERVE_LOOPBACK = """import http.server, sys, threading, time +class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = b"Loopback page" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = http.server.ThreadingHTTPServer(("127.0.0.1", 8765), Page) +threading.Thread(target=server.serve_forever, daemon=True).start() +time.sleep(float(sys.argv[1])) +""" +# The browser's pid and argv, and how many TCP sockets anything in its session listens on: its oldest process with +# agentenv's pipe flag, whose title Chromium rewrites as one line. +BROWSER_PROCESS = """import glob, json, os, subprocess +pid = int(subprocess.run(["pgrep", "-o", "-f", "--", "--remote-debugging-pipe"], capture_output=True, text=True, + check=True).stdout) +listening = set() +for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in open(table).read().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + listening.add(fields[9]) +sockets = set() +for proc in glob.glob("/proc/[0-9]*"): + try: + if os.getsid(int(proc.rsplit("/", 1)[1])) != os.getsid(pid): + continue + for fd in os.listdir(f"{proc}/fd"): + link = os.readlink(f"{proc}/fd/{fd}") + if link.startswith("socket:["): + sockets.add(link[8:-1]) + except OSError: + continue +argv = open(f"/proc/{pid}/cmdline", "rb").read().replace(b"\\0", b" ").decode().split() +print(json.dumps({"pid": pid, "argv": argv, "listening": len(listening & sockets)})) +""" +# Each window the window manager manages, its class and its title. +WINDOWS = ('for w in $(xprop -root _NET_CLIENT_LIST | sed "s/.*# //; s/,//g"); do ' + 'xprop -id "$w" WM_CLASS _NET_WM_NAME; done') + + +def test_gate_3_makes_this_machines_arch_ready_with_the_fixture_browser(website_browser_desktop): + results = website_browser_desktop.metadata["composed"]["container"] + + assert results[ARCH]["status"] == "ready" and results[ARCH]["probe"]["passed"], results[ARCH] + assert [app.environment_name for app in website_browser_desktop.composed_apps()] == ["browser"] + + +@pytest.mark.asyncio +async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaunches_it_on_its_template( + website_browser_desktop): + settings = {"locale": "fr-FR", "time_zone": "Europe/Paris", "start_urls": [TIME_ZONE_PAGE, LOOPBACK]} + with namespace_routing(): + deployed = await Env.get(website_browser_desktop.id).deploy(env_params={"app_settings": {"browser": settings}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + zone = await _window_titled(deployed, "Europe/Paris") + browser = await _browser_process(deployed) + port_file = await inside.run(deployed, ["sh", "-c", "test -e ~/.config/chromium/DevToolsActivePort"], + timeout_s=30) + policies = await inside.run( + deployed, ["cat", "/etc/opt/chrome_for_testing/policies/managed/agentenv.json"], timeout_s=30) + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_LOOPBACK, "60"], + timeout_s=90)) + await asyncio.sleep(3) + await _keys(deployed, ["Control", "Tab"], ["F5"]) + loopback = await _window_titled(deployed, "Loopback page") + await serving + card = await _app(deployed, WELL_KNOWN_PATH) + readiness = await _ready(deployed, card) + await protocol_v1.reset_data(f"{deployed.environment_url}/svc/browser", headers=deployed.control_headers) + relaunched = await _browser_process(deployed, other_than=browser["pid"]) + relaunched_readiness = await _ready(deployed, card) + template = await inside.run(deployed, ["sh", "-c", "cat ~/.config/chromium/agentenv-template.txt"], + timeout_s=30) + finally: + report = await teardown_run(context) + + argv = browser["argv"] + assert argv[0].endswith("chromium/chrome/chrome") and argv[1:4] == ["--no-sandbox", "--disable-dev-shm-usage", + "--class=Chromium"] + assert argv[4].startswith("--user-data-dir=") and argv[4].endswith("/.config/chromium") + assert argv[5:9] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR"] + assert argv[9:] == [TIME_ZONE_PAGE, LOOPBACK] + assert browser["listening"] == 0 and port_file["exit_code"] != 0 + assert zone.startswith("Europe/Paris") and loopback.startswith("Loopback page") + assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( + policies["stdout"]).items() + assert protocol_v1.extension_params(card, BROWSER) == {"chromium": "156.0.8078.4", "clock": "real"} + assert readiness["ready"] is True, readiness + assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 + assert relaunched_readiness["ready"] is True, relaunched_readiness + assert template["stdout"] == "from the profile template\n" + assert not report.failed + + +async def _window_titled(deployed, title: str) -> str: + """The title of the computer's Chromium window once one starts with ``title``.""" + for _ in range(60): + listed = (await inside.run(deployed, ["sh", "-c", WINDOWS], timeout_s=30))["stdout"] + classes = re.findall(r'WM_CLASS\(STRING\) = .*"([^"]*)"\n_NET_WM_NAME\(UTF8_STRING\) = "(.*)"', listed) + if found := [name for cls, name in classes if cls == "Chromium" and name.startswith(title)]: + return found[0] + await asyncio.sleep(1) + raise TimeoutError(f"no Chromium window titled {title!r}: {listed}") + + +async def _keys(deployed, *chords: list[str]) -> None: + """Each chord pressed through the computer's own tools, as an agent presses it.""" + async with streamablehttp_client(f"{deployed.environment_url}/mcp") as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + for keys in chords: + pressed = await session.call_tool("computer_key", {"keys": keys}) + assert not pressed.isError, pressed + + +async def _browser_process(deployed, other_than: int | None = None) -> dict: + """The browser process's pid, argv and listening sockets, once one runs other than ``other_than``.""" + for _ in range(60): + found = await inside.run(deployed, ["python3", "-c", BROWSER_PROCESS], timeout_s=30) + if found["exit_code"] == 0 and (process := json.loads(found["stdout"]))["pid"] != other_than: + return process + await asyncio.sleep(1) + raise TimeoutError(f"no Chromium on --remote-debugging-pipe other than {other_than}: {found}") + + +async def _app(deployed, path: str) -> dict: + async with httpx.AsyncClient(timeout=30) as http: + response = await http.get(f"{deployed.environment_url}/svc/browser{path}", headers=deployed.control_headers) + response.raise_for_status() + return response.json() + + +async def _ready(deployed, card: dict) -> dict: + """The app's readiness once it says it's ready, or as it last said within a minute.""" + for _ in range(60): + readiness = await _app(deployed, apps.endpoint(card, "ready")) + if readiness["ready"]: + return readiness + await asyncio.sleep(1) + return readiness