diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index fc44a1dcb..b1cec4c44 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -880,14 +880,14 @@ under the session user's home (`.config/chromium` by default), whose contents fr are the profile's template. The base class adds: - **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in - tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless - the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, + tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`); `window`, 1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; `geolocation` and `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who answers a page's JavaScript dialogs: `agent` (the default), or agentenv, accepting (`accept`, a `prompt()` with its default text) or dismissing (`dismiss`) each. Either way agentenv leaves a page that asks before it's left (`beforeunload`), so none of its answers blocks a navigation. -- **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, its +- **The flags agentenv owns**, after the subclass's command and before `about:blank`, the one tab its command line + opens: its profile, its HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check`, `--lang` and, when `/dev/shm` holds under 1 GiB (Docker gives 64 MiB), `--disable-dev-shm-usage`, so it keeps its shared memory in files. With @@ -946,6 +946,8 @@ are the profile's template. The base class adds: - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so alone, it intercepts; - `page/state`, the active tab's id, URL and title and the dialog or file chooser open on it. + - `sites/set {sites}`, the env's own sites, each `{name, domains}`, a `*.` domain matching one label more, which + the deploy hands it before it launches the browser and its server keeps for every launch after. - `cookies/get {urls?}`, `cookies/set {cookies}` and `cookies/delete {name?, domain?, path?}`, on its default context, each cookie as Playwright's `storageState` has it, CDP's `Network.Cookie` fields (`name`, `value`, `domain`, `path`, `expires`, `httpOnly`, `secure`, `sameSite`), so a storage state exported from Playwright loads @@ -968,9 +970,26 @@ are the profile's template. The base class adds: under the `write_namespace` grant when there is one, else inline, as `data/get` returns files. Inline, the whole file is read into memory and sent base64 in one answer, so a large one belongs under a grant. + **Start tabs.** At each launch its start tabs open over its pipe, in a computer and alone alike: the first in the + blank tab its command line opened, the rest in tabs it opens blank after it, each held to the window's size + (headless) before it goes to its URL, so its page sees that size from its first script; then the first is brought + to the front. They load at once, and each settles once it has loaded (its load event fired, its document's final + status read), once it failed (a network error, or on one of the env's sites, as `sites/set` named them, a 4xx or + 5xx status, which settles it at once), or once it has been waited on: every one settles within the browser's + ready timeout of its launch, the launch's setup counted against it, a tab on one of the env's sites waited on what's + left of it and any other on 30 s at most, so `data/reset`, a load and a deploy, which wait that timeout and a margin, + always get its outcome. Its `start_tabs_loaded` ready check passes once all have settled, so `cdp` and a deploy wait + for them. `record`'s `start_tabs` keeps each launch's (`StartTabsLaunch`: `at` and its + `tabs`, each a `StartTab`: `tab`, `url`, `final_url`, `status`, `error`, `load_ms`, `site` — the env's site's + name, null for a live one — and `outcome`, `loaded`, `failed` or `timed_out`), and `data/get`'s state keeps its + latest launch's. A deploy of a computer reads them once it's ready, keeps them in its record and fails, naming + each, if one on the env's sites didn't load; `data/reset` and a load that opens the start tabs again fail + the same way, while a server's restart only records it. + The browser's data plane is a file tree, every file by its path under the session user's home: - `data/get` returns its state as `browser-state.json` (`BrowserState`: `cookies` and `local_storage` - as above, `tabs`, `closed_tabs`, `downloads` and `clock`, `real` with its `now`); its profile's files, + as above, `tabs`, `closed_tabs`, `downloads`, `clock`, `real` with its `now`, and `start_tabs`); its + profile's files, each database with its journal or log, copied in one pass with the browser paused once its state is read, so the files are of one moment, without caches (`Cache`, `Code Cache`, `GPUCache`, the shader and Dawn caches, `CacheStorage`, `ScriptCache`, `Crashpad`) or locks diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 717cac06e..02ca54702 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -446,7 +446,7 @@ def __init__(self) -> None: self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, cls.data_timeout_s, live=cls.data_live, keeps_paths=cls.data_keeps_paths, - resets_failed_add=self._resets_failed_add) + resets_failed_add=self._resets_failed_add, relaunched=self._relaunched) def _launch_for(self, launch: Launch) -> Launch: """The launch its server uses, from the one declared for its OS, whose ``ready`` timeout it keeps: as @@ -473,6 +473,9 @@ def _resets_failed_add(self) -> bool: """Whether a load that fails is followed by a reset, so the app isn't left half loaded: always.""" return True + async def _relaunched(self, op: str) -> None: + """What a data call that stopped the app, ``op``, awaits once it has launched it again: nothing.""" + @classmethod def settings_on(cls, os_type: Optional[str]) -> type[BaseModel]: """Its ``Settings`` on ``os_type``, with the fields its OS adds.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 1898b4f80..00856062f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -11,7 +11,7 @@ import logging import shutil import tempfile -from collections.abc import Callable +from collections.abc import Awaitable, Callable from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Optional @@ -41,7 +41,8 @@ class AppDataPlane: def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[str], timeout_s: float, *, live: frozenset[str] = frozenset(), keeps_paths: bool = False, - resets_failed_add: Callable[[], bool] = lambda: True) -> None: + resets_failed_add: Callable[[], bool] = lambda: True, + relaunched: Optional[Callable[[str], Awaitable[None]]] = None) -> None: self.runtime = runtime self.hooks = hooks self.paths = paths @@ -49,6 +50,8 @@ def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[ #: The operations served on the running app; whether a hook's reset needs the paths as installed; and whether #: a load that fails is followed by a reset now. self.live, self.keeps_paths, self.resets_failed_add = live, keeps_paths, resets_failed_add + #: What a call that stopped the app awaits once it has launched it again, whose error fails the call. + self.relaunched = relaunched self.turn = asyncio.Lock() self.changing: set[asyncio.Task] = set() @@ -99,6 +102,8 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. finally: if running: await self.runtime.launch_app(_CAUSES[op]) + if running and self.relaunched is not None: + await self.relaunched(op) except Exception as e: if not answer.done(): answer.set_exception(e) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py index 904656ebc..bc0e84800 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py @@ -5,19 +5,21 @@ skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's, whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its settings.""" -from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, HASH_CHUNK_BYTES, - PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, - BrowserControl) +from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, FAILED, + HASH_CHUNK_BYTES, LIVE_START_TAB_TIMEOUT_S, LOADED, PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP, + TIMED_OUT, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, BrowserControl) from .devtools import MAX_MESSAGE_BYTES, PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe from .environment import (ALONE_BYPASS, BLOCK, BROWSER, COMPUTER_BYPASS, OWNED_FLAGS, POLICIES, READY_POLL_S, - SHM_MIN_BYTES, SITE_PROXY_ENV, SKILL, BrowserSettings, WebsiteBrowserEnvironment, Window) + SHM_MIN_BYTES, SITE_PROXY_ENV, SITES, SKILL, BrowserSettings, WebsiteBrowserEnvironment, + Window, start_tab_why) from .shapes import (BrowserRecord, ClosedTab, Cookie, CookieToSet, Dialog, Download, LaunchEntry, OpenTab, - OriginStorage, PartitionKey, StorageItem, origin_of) + OriginStorage, PartitionKey, SiteNames, StartTab, StartTabsLaunch, StorageItem, origin_of) __all__ = ["ACTIVE_CHECK_TIMEOUT_S", "AGENT", "AGENTENV", "ALONE_BYPASS", "BLOCK", "BROWSER", "CAPTURE_TIMEOUT_S", - "COMPUTER_BYPASS", "DOWNLOADS", "EXIT", "HASH_CHUNK_BYTES", "MAX_MESSAGE_BYTES", "OWNED_FLAGS", "PAGE_INPUT", - "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", "POLICIES", "READY_POLL_S", "SHM_MIN_BYTES", - "SITE_PROXY_ENV", "SKILL", "STOP", "UNKNOWN", "UNNAVIGATED_CAPTURE_TIMEOUT_S", "UPLOADS", "BrowserControl", - "BrowserSettings", "DevToolsError", "DevToolsPipe", "WebsiteBrowserEnvironment", "Window", - "BrowserRecord", "ClosedTab", "Cookie", "CookieToSet", "Dialog", "Download", "LaunchEntry", "OpenTab", - "OriginStorage", "PartitionKey", "StorageItem", "origin_of"] + "COMPUTER_BYPASS", "DOWNLOADS", "EXIT", "FAILED", "HASH_CHUNK_BYTES", "LIVE_START_TAB_TIMEOUT_S", "LOADED", + "MAX_MESSAGE_BYTES", "OWNED_FLAGS", "PAGE_INPUT", "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", + "POLICIES", "READY_POLL_S", "SHM_MIN_BYTES", "SITE_PROXY_ENV", "SITES", "SKILL", "STOP", "TIMED_OUT", + "UNKNOWN", "UNNAVIGATED_CAPTURE_TIMEOUT_S", "UPLOADS", "BrowserControl", "BrowserSettings", "DevToolsError", + "DevToolsPipe", "WebsiteBrowserEnvironment", "Window", "BrowserRecord", "ClosedTab", "Cookie", "CookieToSet", + "Dialog", "Download", "LaunchEntry", "OpenTab", "OriginStorage", "PartitionKey", "SiteNames", "StartTab", + "StartTabsLaunch", "StorageItem", "origin_of", "start_tab_why"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index b8da02816..c7ea10107 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -8,6 +8,7 @@ import logging import time from collections.abc import AsyncIterator +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any, Awaitable, Callable, Optional @@ -15,7 +16,7 @@ from ..computer.errors import DeviceUnavailable, InvalidArgument from .devtools import DevToolsError, DevToolsPipe -from .shapes import (ClosedTab, Cookie, CookieToSet, Dialog, Download, OpenTab, OriginStorage, StorageItem, +from .shapes import (ClosedTab, Cookie, CookieToSet, Dialog, Download, OpenTab, OriginStorage, StartTab, StorageItem, origin_of) logger = logging.getLogger(__name__) @@ -29,6 +30,10 @@ #: of agentenv's own may take to reach an origin, and how often it's asked meanwhile. PIN_TIMEOUT_S = 10.0 ARRIVAL_POLL_S = 0.05 +#: How long a start tab on a site the env doesn't serve may take to load before it's left loading. +LIVE_START_TAB_TIMEOUT_S = 30.0 +#: How a start tab settled. +LOADED, FAILED, TIMED_OUT = "loaded", "failed", "timed_out" #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 #: Who answered a dialog when agentenv didn't. @@ -80,6 +85,8 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl self._dialog_opened: dict[str, asyncio.Event] = {} self._pinned: dict[str, asyncio.Event] = {} self._tasks: set[asyncio.Future] = set() + #: Each start tab loading, by its session. + self._loading: dict[str, _StartLoad] = {} #: Every http and https origin a page or frame it follows loaded since this server started, and the sessions #: of agentenv's own pages whose requests it answers itself. self.origins: set[str] = set() @@ -108,6 +115,9 @@ def ended(self, how: str) -> None: self._shots.clear() self._dialog_opened.clear() self._pinned.clear() + for load in self._loading.values(): + load.error = load.error or "the browser ended" + load.settled.set() async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" @@ -138,18 +148,84 @@ async def open_tab(self, url: Optional[str] = None, background: bool = False) -> await self._bring_to_front(active) return target - async def reopen_start_tabs(self, urls: list[str]) -> None: - """The start tabs at their ``urls`` again, in place, the first active: each tab open, in opened order, goes to - its URL, and any the browser lacks opens. A page already loaded doesn't read cookies set after it, so a tab - a load signed in shows it only once it loads again.""" - tabs = list(self.tabs) - for index, url in enumerate(urls): - if index < len(tabs): - await self._navigate_tab(tabs[index], url) - else: - tabs.append(await self.open_tab(url)) - if tabs: - await self._bring_to_front(tabs[0]) + async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Optional[str]], own_timeout_s: float, + live_timeout_s: float = LIVE_START_TAB_TIMEOUT_S) -> list[dict]: + """Each of ``urls`` in a start tab, in order, the first then active: in the tabs open now, in opened order, and + in tabs it opens blank after them; each held to the window's size, headless, before it goes to its URL, and + all loading at once. A tab settles once it has loaded, once it has failed, which on one of the env's own sites, + as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a + site, else ``live_timeout_s``. Each tab's outcome, in order, kept though the browser has gone and the first + can't come to the front. A page already loaded doesn't read cookies set after it, so a tab a load signed in + shows it only once it loads again.""" + if not urls: + return [] + targets = list(self.tabs)[:len(urls)] or [await self._first_tab()] + while len(targets) < len(urls): + targets.append((await self.devtools.call("Target.createTarget", + {"url": "about:blank", "background": True}))["targetId"]) + for target in targets: + await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S) + loading = [] + for target, url in zip(targets, urls): + site = site_of(url) + loading.append(await self._start_load(target, url, site)) + outcomes = await asyncio.gather(*(self._settled(target, load, own_timeout_s if load.site is not None + else live_timeout_s) for target, load in zip(targets, loading))) + with contextlib.suppress(ConnectionError, DevToolsError): + await self._bring_to_front(targets[0]) + return outcomes + + async def _first_tab(self) -> str: + """The tab a browser just launched opened, once it has one.""" + deadline = asyncio.get_running_loop().time() + PIN_TIMEOUT_S + while not self.tabs: + if asyncio.get_running_loop().time() > deadline: + raise DeviceUnavailable(f"the browser opened no tab within {PIN_TIMEOUT_S:g} s") + await asyncio.sleep(ARRIVAL_POLL_S) + return next(iter(self.tabs)) + + async def _start_load(self, target: str, url: str, site: Optional[str]) -> _StartLoad: + """The tab ``target`` sent to ``url``, its navigation followed by what the browser says of it. The browser + answers a navigation once it commits, which a page that never answers never does, so its answer is awaited + apart, for as long as the tab is.""" + session = next((s for s, t in self._sessions.items() if t == target), None) + if session is None: + raise DeviceUnavailable(f"the browser's tab {target} has no session to load a start tab in") + load = self._loading[session] = _StartLoad(session, url, site, asyncio.get_running_loop().time()) + try: + await self.devtools.call("Network.enable", session_id=session) + except (ConnectionError, DevToolsError, asyncio.TimeoutError) as e: + load.error = f"{type(e).__name__}: {e}" + load.check() + return load + load.navigating = asyncio.ensure_future(self.devtools.request("Page.navigate", {"url": url}, + session_id=session)) + load.navigating.add_done_callback(load.answered) + return load + + async def _settled(self, target: str, load: _StartLoad, timeout_s: float) -> dict: + """``load``'s outcome once it settles, or once it has been waited on ``timeout_s``.""" + try: + await asyncio.wait_for(load.settled.wait(), timeout_s) + except asyncio.TimeoutError: + pass + finally: + if load.navigating is not None: + load.navigating.cancel() + self._loading.pop(load.session, None) + with contextlib.suppress(ConnectionError, DevToolsError, asyncio.TimeoutError): + await self.devtools.call("Network.disable", session_id=load.session) + document = load.current() or load.loader or "" + status, final_url = load.responses.get(document, (None, None)) + error = load.error or load.failures.get(load.loader or "") or load.failures.get(document) + outcome = FAILED if error or (load.settled.is_set() and status is not None and status >= 400) else \ + LOADED if load.settled.is_set() else TIMED_OUT + if final_url is None and outcome == LOADED: + await self._refresh(target) + final_url = self.tabs.get(target, {}).get("url") + return StartTab(tab=target, url=load.url, final_url=final_url, status=status, error=error, + load_ms=round((asyncio.get_running_loop().time() - load.started) * 1000), site=load.site, + outcome=outcome).model_dump() async def replace_tabs(self, tabs: list) -> None: """``tabs``, each with its ``url`` and whether it's ``active``, opened in order in place of the tabs open now, @@ -161,14 +237,6 @@ async def replace_tabs(self, tabs: list) -> None: if opened: await self._bring_to_front(next((target for target, tab in zip(opened, tabs) if tab.active), opened[0])) - async def _navigate_tab(self, target: str, url: str) -> None: - session = next((s for s, t in self._sessions.items() if t == target), None) - if session is None: - raise DeviceUnavailable(f"the browser's tab {target} has no session to navigate it") - answer = await self.devtools.call("Page.navigate", {"url": url}, session_id=session) - if error := answer.get("errorText"): - raise InvalidArgument(f"the browser couldn't open {url}: {error}") - def forget(self) -> None: """A record that starts now, as a reset leaves the browser: no tabs closed, downloads or dialogs before it, and no origin seen.""" @@ -521,6 +589,26 @@ def _on_Target_targetInfoChanged(self, params: dict, session: Optional[str]) -> def _on_Page_frameNavigated(self, params: dict, session: Optional[str]) -> None: if session in self._sessions: self._saw(params.get("frame", {}).get("url", "")) + frame = params.get("frame", {}) + if (load := self._loading.get(session)) is not None and not frame.get("parentId") and frame.get("loaderId"): + load.commits.append(frame["loaderId"]) + load.check() + + def _on_Network_requestWillBeSent(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and params.get("type") == "Document": + load.requests[params["requestId"]] = params.get("loaderId", "") + + def _on_Network_responseReceived(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and params.get("type") == "Document" and \ + params.get("requestId") in load.requests: + response = params.get("response", {}) + load.responses[load.requests[params["requestId"]]] = (response.get("status"), response.get("url")) + load.check() + + def _on_Network_loadingFailed(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and (loader := load.requests.get(params.get("requestId"))): + load.failures[loader] = params.get("errorText") or "the load failed" + load.check() def _on_Fetch_requestPaused(self, params: dict, session: Optional[str]) -> None: if session in self._serving: @@ -575,6 +663,9 @@ def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None: self._choosers.pop(session, None) if (target := self._sessions.get(session)) is not None: self._spawn(self._refresh(target)) + if (load := self._loading.get(session)) is not None and load.commits: + load.loaded.add(load.commits[-1]) + load.check() def _on_Page_fileChooserOpened(self, params: dict, session: Optional[str]) -> None: if session in self._sessions: @@ -636,6 +727,58 @@ def _finished(self, task: asyncio.Future) -> None: logger.warning(f"the browser's control connection: {type(error).__name__}: {error}") +@dataclass +class _StartLoad: + """What the browser has said of a start tab's navigation, by loader, since it was sent to ``url``: its main frame's + commits in order, the loads that fired after a commit, and its documents' requests, final responses and failures. + Its document is the last its main frame committed from its own navigation on, which a page that sends itself + elsewhere before it loads replaces. It's ``settled`` once its navigation or its document has failed, its document + has loaded or, on one of the env's ``site``, answered an error.""" + session: str + url: str + site: Optional[str] + started: float + loader: Optional[str] = None + error: Optional[str] = None + commits: list[str] = field(default_factory=list) + loaded: set[str] = field(default_factory=set) + requests: dict[str, str] = field(default_factory=dict) + responses: dict[str, tuple[Optional[int], Optional[str]]] = field(default_factory=dict) + failures: dict[str, str] = field(default_factory=dict) + settled: asyncio.Event = field(default_factory=asyncio.Event) + navigating: Optional[asyncio.Future] = None + + def answered(self, navigating: asyncio.Future) -> None: + """The browser's answer to its navigation: its loader, or why it couldn't navigate.""" + if navigating.cancelled(): + return + if (error := navigating.exception()) is not None: + self.error = f"{type(error).__name__}: {error}" + else: + answer = navigating.result() + if "error" in answer: + self.error = str(answer["error"].get("message", answer["error"])) + else: + self.error = answer.get("result", {}).get("errorText") or None + self.loader = answer.get("result", {}).get("loaderId") + self.check() + + def current(self) -> Optional[str]: + """The loader of its document: the last its main frame committed, once its own navigation has; else None.""" + return self.commits[-1] if self.loader is not None and self.loader in self.commits else None + + def check(self) -> None: + if self.error or (self.loader is not None and self.loader in self.failures): + self.settled.set() + return + if (document := self.current()) is None: + return + status = self.responses.get(document, (None, None))[0] + if document in self.failures or document in self.loaded or ( + self.site is not None and status is not None and status >= 400): + self.settled.set() + + def _size_and_sha256(path: Path) -> tuple[int, str]: digest = hashlib.sha256() with path.open("rb") as file: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 51ad3109e..7be58a335 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -9,6 +9,7 @@ import tempfile from pathlib import Path from typing import Any, Awaitable, Callable, ClassVar, Literal, Optional +from urllib.parse import urlsplit from pydantic import BaseModel, Field, ValidationError, field_validator from starlette.requests import Request @@ -27,14 +28,16 @@ from ..transfers import WriteNamespaceGrant from ..types import METHOD_ADD, EnvironmentExtension, IntakeDeclaration, IntakeFormat, intake_extension from . import data -from .control import AGENT, DOWNLOADS, EXIT, STOP, UPLOADS, BrowserControl, _now +from .control import AGENT, DOWNLOADS, EXIT, LIVE_START_TAB_TIMEOUT_S, LOADED, STOP, UPLOADS, BrowserControl, _now from .devtools import PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe -from .shapes import BrowserRecord, BrowserState, CookieToSet, OriginStorage, StateToLoad +from .shapes import BrowserRecord, BrowserState, CookieToSet, OriginStorage, SiteNames, StateToLoad logger = logging.getLogger(__name__) SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" BROWSER = "urn:agentenv:browser/v1" +#: Where its server keeps the env's sites ``sites/set`` hands it, which outlive a relaunch or the server's restart. +SITES = SOCKET.parent / "sites.json" #: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging #: port, since the browser is reached only over its pipe. OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", @@ -46,6 +49,8 @@ BLOCK = 2 #: How often a call that waits for the browser to be ready checks it. READY_POLL_S = 0.5 +#: How much longer than its ready timeout, within which a launch's start tabs settle, a call waits for them. +START_TABS_MARGIN_S = 30.0 #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -126,6 +131,8 @@ def __init__(self) -> None: #: Its CDP connection, over the pipe its server launches it with. self.devtools = DevToolsPipe() self._following: Optional[asyncio.Future] = None + #: The start tabs of each launch since this server started, newest last. + self.start_tab_launches: list[dict] = [] super().__init__() for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) @@ -193,8 +200,9 @@ def _on_desktop(self) -> bool: return not _alone() def _command_line_tabs(self) -> list[str]: - """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" - return self.settings.start_urls[:1] if _alone() else self.settings.start_urls + """The tab its command line opens, blank: its start tabs open over its pipe, so each is held to its window and + followed as it loads.""" + return ["about:blank"] async def _launch_fds(self) -> dict[int, int]: """Before each launch, an empty cache and the deploy's prompt answers in its profile; and a new debugging pipe: @@ -205,7 +213,8 @@ async def _launch_fds(self) -> dict[int, int]: to_browser_read, to_browser_write = os.pipe() from_browser_read, from_browser_write = os.pipe() await self.devtools.connect(to_browser_write, from_browser_read) - self._following = asyncio.ensure_future(self._follow(self.settings.start_urls[len(self._command_line_tabs()):])) + self._following = asyncio.ensure_future(self._follow(self.settings.start_urls, _now(), + asyncio.get_running_loop().time())) self._following.add_done_callback(lambda done: done.cancelled() or done.exception()) return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} @@ -215,11 +224,23 @@ def _pipe_closed(self) -> None: self.control.ended(EXIT if self._computer_app.exited() else STOP) self.relay.ended() - async def _follow(self, urls: list[str]) -> None: - """Its control attached, then each of ``urls`` in a tab behind the active one, in order.""" + async def _follow(self, urls: list[str], at: str, launched: float) -> None: + """Its control attached, then its start tabs at ``urls`` loaded, the launch's outcomes kept.""" await self.control.attach() - for url in urls: - await self.devtools.call("Target.createTarget", {"url": url, "background": True}) + await self._load_start_tabs(urls, at, launched) + + async def _load_start_tabs(self, urls: list[str], at: str, since: Optional[float] = None) -> list[dict]: + """Its start tabs at ``urls``, as the control loads them, every one settled within the browser's ready timeout + of ``since``, its launch's start on the loop's clock, else of now: a tab on one of the env's sites waited on + what's left of it, any other on the live wait at most. A launch's setup counts against it, so its start tabs + settle before anything waiting that timeout and a margin on it gives up. Their outcomes, kept as a launch's + from ``at``.""" + sites, now = _read_sites(), asyncio.get_running_loop().time() + left = max(0.0, self._computer_app.timeout_s - (now - (since if since is not None else now))) + tabs = await self.control.load_start_tabs(urls, lambda url: _site_of(url, sites), left, + min(LIVE_START_TAB_TIMEOUT_S, left)) + self.start_tab_launches.append({"at": at, "tabs": tabs}) + return tabs def _write_prompt_answers(self) -> None: """Its profile's preferences, which it reads as it starts, denying location and notification prompts or @@ -259,11 +280,17 @@ def write_policies(self) -> None: @ready_check async def devtools_answers(self) -> bool: - """It answers over its pipe, its control follows it and it has opened its start tabs.""" + """It answers over its pipe.""" try: await self.devtools.call("Browser.getVersion", timeout_s=self.devtools_timeout_s) except (ConnectionError, DevToolsError, asyncio.TimeoutError): return False + return True + + @ready_check + async def start_tabs_loaded(self) -> bool: + """Its control follows it and every start tab of its launch has settled: loaded, failed or waited on its + time.""" following = self._following return following is not None and following.done() and not following.cancelled() and \ following.exception() is None @@ -284,10 +311,26 @@ async def devtools_answers(self) -> bool: "downloads/get are its downloads, get with its file, uploaded under a write_namespace grant or inline. cdp " "returns a user's CDP address, http_url to discover from and ws_url to connect to, " "with a token agentenv mints in its path: each connection to it gets its own browser-level session behind a " - "guard, and record lists them as cdp_connections, each with the commands the guard refused or changed.")) + "guard, and record lists them as cdp_connections, each with the commands the guard refused or changed. " + "sites/set hands it the env's own sites, by name and domains: a start tab on one must load, its document's " + "status 2xx, or its launch counts it failed, while one on any other site is waited on 30 s at most. record " + "keeps each launch's start_tabs: the URL asked for and ended on, the status, any network error, the load time, " + "the env's site and the outcome, loaded, failed or timed_out.")) async def browser_record(self) -> dict: return BrowserRecord(**await self.control.record(), launches=self._computer_app.launches(), - cdp_connections=list(self.relay.connections)).model_dump() + cdp_connections=list(self.relay.connections), + start_tabs=self.start_tab_launches).model_dump() + + @extension(BROWSER, op="sites/set") + async def browser_sites_set(self, sites: list[dict]) -> dict: + """The env's own sites, each by its name and domains, which it keeps for each launch after: a start tab on one + is waited on strictly.""" + named = _parsed(SiteNames, sites, "sites") + SITES.parent.mkdir(parents=True, exist_ok=True) + written = SITES.with_name(f"{SITES.name}.part") + written.write_text(json.dumps([site.model_dump() for site in named]), encoding="utf-8") + written.replace(SITES) + return {} @extension(BROWSER, op="cdp") async def browser_cdp(self, user: str, request: Request) -> dict: @@ -468,7 +511,8 @@ async def browser_get_data(self, write_namespace: Optional[WriteNamespaceGrant] state = BrowserState(cookies=await self.control.cookies(), local_storage=await self.control.local_storage(), tabs=await self.control.tab_list(), closed_tabs=(record := await self.control.record())["closed_tabs"], - downloads=record["downloads"], clock={"mode": "real", "now": _now()}) + downloads=record["downloads"], clock={"mode": "real", "now": _now()}, + start_tabs=self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) written = {**state.model_dump(), "cookies": [cookie.model_dump(exclude_none=True) for cookie in state.cookies]} (home / ".cache").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=home / ".cache") as staging: @@ -502,6 +546,7 @@ async def browser_add_data(self, parts: list) -> None: await self._computer_app.stop_app() await app_data.in_thread(data.restore_profile, Path.home(), self.profile, load.profile) await self._computer_app.launch_app(METHOD_ADD) + _refuse_failed(await self._start_tabs_settled()) await self._until_ready() applied.append("the profile") if load.files: @@ -517,7 +562,7 @@ async def browser_add_data(self, parts: list) -> None: await self.control.replace_tabs(state.tabs) applied.append("the tabs") elif (state.cookies or state.local_storage) and not agent_acted(): - await self.control.reopen_start_tabs(self.settings.start_urls) + _refuse_failed(await self._load_start_tabs(self.settings.start_urls, _now())) applied.append("the start tabs again") except Exception as e: done = ", ".join(applied) or "nothing" @@ -533,8 +578,26 @@ async def browser_reset_data(self) -> None: await app_data.reset(self.paths) await app_data.in_thread(data.empty, Path.home(), DOWNLOADS, UPLOADS) self.control.forget() + self.start_tab_launches = [] forget_agent_acted() + async def _relaunched(self, op: str) -> None: + """After a reset relaunches it: back once it's ready, InvalidArgument naming a start tab on one of the env's + sites that failed.""" + _refuse_failed(await self._start_tabs_settled()) + await self._until_ready() + + async def _start_tabs_settled(self) -> list[dict]: + """The start tabs of its latest launch once they've settled, waited on its ready timeout and the margin its + setup takes before they load; DeviceUnavailable if they haven't by then.""" + if (following := self._following) is not None: + timeout_s = self._computer_app.timeout_s + START_TABS_MARGIN_S + try: + await asyncio.wait_for(asyncio.shield(following), timeout_s) + except asyncio.TimeoutError: + raise DeviceUnavailable(f"the browser's start tabs hadn't settled after {timeout_s:g} s") from None + return self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else [] + def extension_params(self, uri: str) -> dict: if uri == BROWSER: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} @@ -548,6 +611,44 @@ def browser_skill(self) -> tuple[str, str]: return "How to use this website browser.", SKILL +def _read_sites() -> list[SiteNames]: + """The env's sites ``sites/set`` handed it, none before it did or for a file it can't read.""" + try: + return [SiteNames.model_validate(site) for site in json.loads(SITES.read_text(encoding="utf-8"))] + except (OSError, ValueError, TypeError, ValidationError): + return [] + + +def _site_of(url: str, sites: list[SiteNames]) -> Optional[str]: + """The name of the env's site among ``sites`` that serves ``url``'s host, None for any other.""" + host = (urlsplit(url).hostname or "").rstrip(".") + return next((site.name for site in sites if any(_matches(domain.lower(), host) for domain in site.domains)), None) + + +def _matches(pattern: str, host: str) -> bool: + """Whether ``host`` is the name ``pattern`` names: itself, or for ``*.example.com`` one label more.""" + if pattern.startswith("*."): + head, dot, rest = host.partition(".") + return bool(head) and bool(dot) and rest == pattern[2:] + return host == pattern + + +def _refuse_failed(tabs: list[dict]) -> None: + """InvalidArgument naming each start tab on one of the env's sites that didn't load.""" + if failed := [tab for tab in tabs if tab["site"] is not None and tab["outcome"] != LOADED]: + raise InvalidArgument("; ".join(f"start tab {tab['url']} on the env's site {tab['site']} {start_tab_why(tab)}" + for tab in failed)) + + +def start_tab_why(tab: dict) -> str: + """Why a start tab didn't load, as its outcome says.""" + if tab.get("error"): + return f"didn't load: {tab['error']}" + if tab.get("status") is not None and tab["status"] >= 400: + return f"answered {tab['status']}" + return f"was still loading after {tab['load_ms'] / 1000:g} s" + + def _alone() -> bool: """Whether it runs on a computer with no display, a browser computer's, alone and headless.""" return computer_app.detect().display == "none" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py index d4bfc8a1c..1eb842e4b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py @@ -5,7 +5,7 @@ from typing import Literal, Optional from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator SameSite = Literal["Strict", "Lax", "None"] @@ -152,6 +152,35 @@ class StateToLoad(BaseModel): tabs: Optional[list[TabToOpen]] = None +class SiteNames(BaseModel): + """A site of the env, which ``sites/set`` hands the browser: its ``name`` and the ``domains`` it's served under, a + ``*.`` name matching one label more, as a certificate's wildcard does.""" + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + domains: list[str] = Field(min_length=1) + + +class StartTab(BaseModel): + """How a start tab loaded: the ``url`` asked for, the ``final_url`` it ended on, its document's final HTTP + ``status``, the network ``error`` that kept it from loading, how long it took, the env's ``site`` it's on, None for + a live one, and its ``outcome``: ``loaded``, ``failed`` (an error, or a 4xx or 5xx status) or ``timed_out``.""" + tab: str + url: str + final_url: Optional[str] = None + status: Optional[int] = None + error: Optional[str] = None + load_ms: int + site: Optional[str] = None + outcome: Literal["loaded", "failed", "timed_out"] + + +class StartTabsLaunch(BaseModel): + """The start tabs of one launch of the browser, ``at`` when it began, in ``start_urls`` order.""" + at: str + tabs: list[StartTab] + + class Clock(BaseModel): """The clock the browser's pages see: the computer's own, ``real``, until a browser follows a universe's.""" mode: Literal["real"] @@ -166,6 +195,8 @@ class BrowserState(BaseModel): closed_tabs: list[ClosedTab] downloads: list[Download] clock: Clock + #: The start tabs of its latest launch. + start_tabs: list[StartTab] = [] class BrowserRecord(BaseModel): @@ -178,6 +209,8 @@ class BrowserRecord(BaseModel): dialogs: list[Dialog] launches: list[LaunchEntry] cdp_connections: list[dict] + #: The start tabs of each launch since its server started. + start_tabs: list[StartTabsLaunch] = [] def origin_of(url: str, *, strict: bool = False) -> Optional[str]: diff --git a/packages/agentenv-protocol/tests/test_browser_data.py b/packages/agentenv-protocol/tests/test_browser_data.py index 12a412a9e..aa1f40ae4 100644 --- a/packages/agentenv-protocol/tests/test_browser_data.py +++ b/packages/agentenv-protocol/tests/test_browser_data.py @@ -2,10 +2,12 @@ caches or locks, and its finished downloads; data/add applies such a state while the browser runs, the start tabs loaded again after cookies change, a load's tabs in place of the open ones, and a profile restored with the browser stopped, every part checked first; once an agent has acted, a load can't change the profile or the tabs, a failed one -isn't reset, and one that fails part way says what it applied; data/reset ends the addresses, puts the profile back as -installed, empties the downloads and uploads and starts the record over.""" +isn't reset, and one that fails part way says what it applied; a load whose start tabs on one of the env's sites +don't load fails naming them, as does a reset's relaunch, while a live one never does; data/reset ends the addresses, +puts the profile back as installed, empties the downloads and uploads and starts the record over.""" from __future__ import annotations +import asyncio import base64 import contextlib import json @@ -16,7 +18,7 @@ from agentenv_protocol import website_browser from agentenv_protocol.computer import Launch, Ready, app, linux from agentenv_protocol.computer.app import AGENT_ACTED -from agentenv_protocol.computer.errors import InvalidArgument +from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument from agentenv_protocol.computer.platform import Platform from agentenv_protocol.types import DATA_OBJECTS_EXTENSION_URI, INTAKE_EXTENSION_URI, AddDataRequest from agentenv_protocol.website_browser import WebsiteBrowserEnvironment @@ -34,7 +36,7 @@ class FakeControl: """The browser's control, answering from what a test sets and recording what it's asked to do.""" def __init__(self): - self.did, self.fail = [], None + self.did, self.fail, self.start_tabs_outcome = [], None, "loaded" self.cookie_list = [{"name": "sid", "value": "s1", "domain": "shop.test", "path": "/", "expires": -1, "httpOnly": True, "secure": False, "sameSite": "Lax"}] @@ -66,8 +68,11 @@ async def set_local_storage(self, origins): async def replace_tabs(self, tabs): await self._done("tabs", [(tab.url, tab.active) for tab in tabs]) - async def reopen_start_tabs(self, urls): + async def load_start_tabs(self, urls, site_of, own_timeout_s, live_timeout_s): + self.waits = (own_timeout_s, live_timeout_s) await self._done("start_tabs", urls) + return [{"tab": f"t{index}", "url": url, "final_url": url, "status": 503, "error": None, "load_ms": 40, + "site": site_of(url), "outcome": self.start_tabs_outcome} for index, url in enumerate(urls)] def forget(self): self.did.append(("forget", None)) @@ -106,7 +111,7 @@ def paused(): yield launches.append("continue") - made._computer_app = SimpleNamespace(stop_app=stop_app, launch_app=launch_app, paused=paused) + made._computer_app = SimpleNamespace(stop_app=stop_app, launch_app=launch_app, paused=paused, timeout_s=60.0) made._until_ready = until_ready made.launches = launches return made @@ -140,7 +145,7 @@ async def test_data_get_is_its_state_its_profile_without_caches_or_locks_and_its files = _files(await browser.browser_get_data()) state = json.loads(files.pop("browser-state.json")) - assert set(state) == {"cookies", "local_storage", "tabs", "closed_tabs", "downloads", "clock"} + assert set(state) == {"cookies", "local_storage", "tabs", "closed_tabs", "downloads", "clock", "start_tabs"} assert state["cookies"] == FakeControl().cookie_list and "partitionKey" not in state["cookies"][0] assert state["clock"]["mode"] == "real" and state["tabs"][0]["id"] == "t1" and state["closed_tabs"][0]["id"] == "t0" assert files == {f"{PROFILE}/Default/Cookies": b"db", f"{PROFILE}/Default/Cookies-journal": b"j", @@ -277,6 +282,85 @@ async def test_a_load_that_fails_part_way_says_what_it_had_applied_and_one_befor assert browser._resets_failed_add() is True +@pytest.mark.asyncio +@pytest.mark.parametrize("sites, refused", [([{"name": "shop", "domains": ["shop.test"]}], True), ([], False)], + ids=["own-site", "live"]) +async def test_a_load_whose_start_tab_on_an_env_site_doesnt_load_fails_naming_it_and_a_live_one_doesnt( + browser, sites, refused): + browser.control.start_tabs_outcome = "failed" + await browser.browser_sites_set(sites) + load = browser.browser_add_data(_parts(_file("browser-state.json", _snapshot_state()))) + + if refused: + with pytest.raises(InvalidArgument, match=r"start tab https://shop.test/ on the env's site shop answered 503 " + r"\(the load had applied the cookies, the local storage\)"): + await load + else: + await load + assert browser.start_tab_launches[-1]["tabs"][0]["site"] == ("shop" if refused else None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("set_up_s, waits", [(None, (60, 30)), (45, (15, 15)), (75, (0, 0))], + ids=["a-reopen", "a-launch-whose-setup-took-45-s", "a-launch-past-its-timeout"]) +async def test_start_tabs_settle_within_the_ready_timeout_of_their_launch_its_setup_counted(browser, set_up_s, waits): + loop = asyncio.get_running_loop() + + await browser._load_start_tabs(["https://shop.test/"], "t", None if set_up_s is None else loop.time() - set_up_s) + + own, live = browser.control.waits + assert (round(own), round(live)) == waits + + +@pytest.mark.asyncio +async def test_after_a_reset_relaunches_it_a_start_tab_on_an_env_site_that_failed_fails_the_reset(browser): + def launched(url, site, outcome, **why): + return {"at": "t", "tabs": [{"tab": "t1", "url": url, "final_url": None, "status": None, "error": None, + "load_ms": 60000, "site": site, "outcome": outcome, **why}]} + + browser._following = asyncio.get_running_loop().create_future() + browser._following.set_result(None) + browser.start_tab_launches = [launched("https://slack.com/", None, "timed_out")] + await browser._relaunched("data/reset") + browser.start_tab_launches.append(launched("https://shop.test/", "shop", "timed_out")) + with pytest.raises(InvalidArgument, match="start tab https://shop.test/ on the env's site shop was still loading " + "after 60 s"): + await browser._relaunched("data/reset") + browser.start_tab_launches.append(launched("https://shop.test/", "shop", "failed", + error="net::ERR_CONNECTION_REFUSED")) + with pytest.raises(InvalidArgument, match="didn't load: net::ERR_CONNECTION_REFUSED"): + await browser._relaunched("data/reset") + assert browser.launches == ["ready"] + + +@pytest.mark.asyncio +async def test_after_a_reset_relaunches_it_start_tabs_still_settling_past_its_time_and_margin_fail_the_reset( + browser, monkeypatch): + monkeypatch.setattr(website_browser.environment, "START_TABS_MARGIN_S", 0.05) + browser._computer_app.timeout_s = 0.05 + browser._following = asyncio.get_running_loop().create_future() + + with pytest.raises(DeviceUnavailable, match="start tabs hadn't settled after 0.1 s"): + await browser._relaunched("data/reset") + + +@pytest.mark.asyncio +async def test_sites_set_keeps_the_env_sites_a_start_tab_matches_by_name_or_one_label_under_a_wildcard(browser): + environment = website_browser.environment + before = environment._read_sites() + + await browser.browser_sites_set([{"name": "shop", "domains": ["shop.test"]}, + {"name": "mail", "domains": ["*.mail.test"]}]) + sites = environment._read_sites() + with pytest.raises(InvalidArgument, match=r"sites\[0\]: domains"): + await browser.browser_sites_set([{"name": "shop", "domains": []}]) + + assert before == [] and [site.name for site in environment._read_sites()] == ["shop", "mail"] + assert [environment._site_of(url, sites) for url in ( + "https://shop.test/", "https://SHOP.test./cart", "https://www.mail.test/", "https://a.b.mail.test/", + "https://mail.test/", "https://slack.com/", "about:blank")] == ["shop", "shop", "mail", None, None, None, None] + + @pytest.mark.asyncio async def test_a_state_that_doesnt_fit_is_refused_naming_its_fields_and_never_a_value(browser): bad = _snapshot_state(cookies=[{"name": "sid", "value": "SECRET", "maxAge": 5}]) diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index b4638b324..00b601504 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -252,6 +252,37 @@ async def test_an_export_sends_a_linked_files_contents_from_inside_the_home_and_ assert exported == {"Mail/latest.txt": b"archived"} +@linux(paths=MAIL, launch=Launch(command=["mail"])) +class CheckedMail(ComputerAppEnvironment): + """Mail that, once a data call relaunches it, checks it came back, failing while ``broken``.""" + relaunched: list = [] + broken = False + + async def _relaunched(self, op: str) -> None: + type(self).relaunched.append(op) + if type(self).broken: + raise RuntimeError("it came back broken") + + +@pytest.mark.asyncio +async def test_a_call_that_relaunched_the_app_awaits_its_relaunch_check_whose_failure_fails_the_call(home, desktop): + mail = CheckedMail() + mail._computer_app_data_plane().keep_initial() + await mail._computer_app.launch_app() + + async with _http(mail) as http: + await _rpc(http, "data/reset", {}) + await _rpc(http, "data/get", {}) + CheckedMail.broken = True + broken = await http.post(RPC_PATH, json=_call("data/reset", {})) + await mail._computer_app.stop_app() + + assert CheckedMail.relaunched == ["reset_data", "reset_data"] + assert "it came back broken" in broken.json()["error"]["message"] + assert [event for event, _ in desktop] == ["launch", "stop", "launch", "pause", "continue", "stop", "launch", + "stop"] + + @pytest.mark.asyncio async def test_an_app_that_isnt_running_is_left_stopped(home, desktop): mail = PathsMail() diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 810ec6ff7..c717293eb 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -23,34 +23,66 @@ from agentenv_protocol.computer.platform import Platform from agentenv_protocol.website_browser import (ALONE_BYPASS, BROWSER, COMPUTER_BYPASS, POLICIES, SITE_PROXY_ENV, SKILL, BrowserControl, DevToolsError, WebsiteBrowserEnvironment) +from agentenv_protocol.website_browser.control import STOP PROXY = "http://127.0.0.1:3128" OWNED = ["--user-data-dir", "--disk-cache-dir", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] # A browser that answers CDP on its debugging pipe, fds 3 and 4, as a Chromium does: Browser.getVersion with its -# product, Target.createTarget by writing the tab's URL to tabs.txt beside it, a new window's after "window ", anything -# else with an error. +# product; its first tab, "start", made known and attached to; Target.createTarget by writing the tab's URL, or a new +# window's after "window ", to tabs.txt beside it and attaching to the tab; Page.navigate by writing "navigate +# " there, then committing and loading it; the calls agentenv makes to follow a page with nothing; anything else +# with an error. FAKE_BROWSER = """import json, os -buffer = b"" +buffer, loaders = b"", 0 +def send(method, params, session=None): + os.write(4, json.dumps({"method": method, "params": params, **({"sessionId": session} if session else {})}).encode() + + b"\\0") +def page(target, url="about:blank"): + return {"targetId": target, "type": "page", "url": url, "title": ""} while chunk := os.read(3, 65536): buffer += chunk while b"\\0" in buffer: raw, buffer = buffer.split(b"\\0", 1) message = json.loads(raw) - if message["method"] == "Browser.getVersion": + method, params, session = message["method"], message.get("params") or {}, message.get("sessionId") + events = [] + if method == "Browser.getVersion": answer = {"result": {"product": "Fake/1"}} - elif message["method"] in ("Target.setDiscoverTargets", "Target.setAutoAttach", "Browser.setDownloadBehavior"): + elif method == "Target.setDiscoverTargets": answer = {"result": {}} - if message["method"] == "Target.setDiscoverTargets": - created = {"targetInfo": {"targetId": "start", "type": "page", "url": "about:blank", "title": ""}} - os.write(4, json.dumps({"method": "Target.targetCreated", "params": created}).encode() + b"\\0") - elif message["method"] == "Target.createTarget": + events = [("Target.targetCreated", {"targetInfo": page("start")}, None)] + elif method == "Target.setAutoAttach": + answer = {"result": {}} + events = [("Target.attachedToTarget", {"sessionId": "s-start", "targetInfo": page("start"), + "waitingForDebugger": True}, None)] + elif method == "Target.createTarget": + with open("tabs.txt", "a") as tabs: + tabs.write(("window " if params.get("newWindow") else "") + params["url"] + "\\n") + target = f"t{len(open('tabs.txt').read().splitlines())}" + answer = {"result": {"targetId": target}} + events = [("Target.targetCreated", {"targetInfo": page(target)}, None), + ("Target.attachedToTarget", {"sessionId": "s-" + target, "targetInfo": page(target), + "waitingForDebugger": True}, None)] + elif method == "Page.navigate": + target, loaders = session[2:], loaders + 1 with open("tabs.txt", "a") as tabs: - tabs.write(("window " if message["params"].get("newWindow") else "") + message["params"]["url"] + "\\n") - answer = {"result": {"targetId": message["params"]["url"]}} + tabs.write(f"navigate {target} {params['url']}\\n") + answer = {"result": {"frameId": target, "loaderId": f"L{loaders}"}} + events = [("Page.frameNavigated", {"frame": {"id": target, "loaderId": f"L{loaders}", + "url": params["url"]}}, session), + ("Page.loadEventFired", {}, session)] + elif method == "Target.getTargets": + answer = {"result": {"targetInfos": []}} + elif method in ("Browser.setDownloadBehavior", "Page.enable", "Emulation.setDeviceMetricsOverride", + "Page.setInterceptFileChooserDialog", "Runtime.runIfWaitingForDebugger", "Network.enable", + "Network.disable", "Target.activateTarget", "Page.bringToFront"): + answer = {"result": {}} else: answer = {"error": {"code": -32601, "message": "unknown method"}} os.write(4, json.dumps({"id": message["id"], **answer}).encode() + b"\\0") + for event in events: + send(*event) """ @@ -195,13 +227,15 @@ async def test_each_launch_starts_with_an_empty_cache_and_the_deploys_prompt_ans assert not (browser_home / ".cache/chromium").exists() -def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_url_is_a_flag(monkeypatch): +def test_its_settings_give_its_language_and_time_zone_its_start_tabs_open_over_its_pipe_and_none_is_a_flag( + monkeypatch): _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app"], locale="fr-FR", time_zone="Europe/Paris") launch = Browser().launch _settings(monkeypatch, start_urls=["--remote-allow-origins=*"]) - assert launch.command[-2:] == ["https://slack.com", "https://linear.app"] and "--lang=fr-FR" in launch.command + assert launch.command[-1] == "about:blank" and "--lang=fr-FR" in launch.command + assert not any("slack.com" in arg or "linear.app" in arg for arg in launch.command) assert launch.env == {"TZ": "Europe/Paris"} with pytest.raises(ValueError, match="start_urls are URLs, not flags"): Browser() @@ -271,7 +305,7 @@ async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_rea await runtime.launch_app() version = await browser.devtools.call("Browser.getVersion") with pytest.raises(DevToolsError, match="unknown method"): - await browser.devtools.call("Page.navigate", {"url": "about:blank"}) + await browser.devtools.call("Page.printToPDF") launched = await _answering(browser) await runtime.stop_app() stopped = await browser.devtools_answers() @@ -282,7 +316,7 @@ async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_rea assert unlaunched is False and version == {"product": "Fake/1"} and launched is True assert stopped is False and relaunched is True - assert "devtools_answers" in runtime.ready_checks + assert {"devtools_answers", "start_tabs_loaded"} <= set(runtime.ready_checks) @pytest.mark.asyncio @@ -306,7 +340,8 @@ async def test_a_launch_while_it_runs_opens_a_window_over_its_pipe_and_starts_no @pytest.mark.asyncio -async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): +async def test_headless_its_start_tabs_open_blank_held_to_its_window_then_load_in_order_and_count_it_ready( + monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") _alone(monkeypatch) @@ -316,14 +351,20 @@ async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_res await browser._computer_app.launch_app() for _ in range(50): - if ready := await browser.devtools_answers(): + if ready := await browser.start_tabs_loaded(): break await asyncio.sleep(0.1) await browser._computer_app.stop_app() browser.devtools.close() - assert browser.launch.command[-2:] == ["--headless=new", "https://slack.com"] - assert ready is True and (tmp_path / "tabs.txt").read_text() == "https://linear.app\nhttps://github.com\n" + assert browser.launch.command[-2:] == ["--headless=new", "about:blank"] + assert ready is True and (tmp_path / "tabs.txt").read_text().splitlines() == [ + "about:blank", "about:blank", "navigate start https://slack.com", "navigate t1 https://linear.app", + "navigate t2 https://github.com"] + tabs = browser.start_tab_launches[-1]["tabs"] + assert [(tab["tab"], tab["url"], tab["outcome"], tab["site"]) for tab in tabs] == [ + ("start", "https://slack.com", "loaded", None), ("t1", "https://linear.app", "loaded", None), + ("t2", "https://github.com", "loaded", None)] @pytest.mark.asyncio @@ -452,6 +493,12 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None): answer = self.answers.get(method, {}) return answer(params, session_id) if callable(answer) else answer + async def request(self, method, params=None, *, session_id=None): + try: + return {"result": await self.call(method, params, session_id=session_id)} + except DevToolsError as e: + return {"error": {"message": str(e)}} + def send(self, method, params, session=None): self.listener({"method": method, "params": params, **({"sessionId": session} if session else {})}) @@ -535,6 +582,156 @@ def refused(params, session): "Page.enable", "Runtime.runIfWaitingForDebugger"] +class Navigating(StubDevTools): + """A browser whose first tab is ``start`` and whose new tabs are t1, t2, …, each attached as it opens, answering a + navigation as ``pages`` says by URL: a status and the URL it ends on, then committed and, unless ``no-load`` + follows, loaded, or with ``("redirect", status, url)`` following, sending itself there by script before it loads; + ``refused``, a network error; ``silent``, never answered, as a page that never answers isn't; or ``stale``, a load + event with no commit.""" + + def __init__(self, pages): + super().__init__({"Target.getTargets": {"targetInfos": []}}) + self.pages, self.opened, self.loaders = pages, 0, 0 + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Target.createTarget": + self.calls.append((method, params, session_id)) + self.opened += 1 + target = f"t{self.opened}" + self.send("Target.targetCreated", _page(target)) + self.send("Target.attachedToTarget", {"sessionId": f"s-{target}", "waitingForDebugger": True, + **_page(target)}) + return {"targetId": target} + if method != "Page.navigate": + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + self.calls.append((method, params, session_id)) + page, target = self.pages[params["url"]], session_id[2:] + self.loaders += 1 + loader, request = f"L{self.loaders}", f"r{self.loaders}" + if page == "refused": + return {"frameId": target, "loaderId": loader, "errorText": "net::ERR_CONNECTION_REFUSED"} + if page == "silent": + await asyncio.Event().wait() + if page == "stale": + self.send("Page.loadEventFired", {}, session_id) + return {"frameId": target, "loaderId": loader} + status, final_url, *rest = page + self._document(session_id, target, loader, request, status, final_url) + if rest and isinstance(rest[0], tuple): + _, status, final_url = rest[0] + self._document(session_id, target, f"{loader}-redirect", f"{request}-redirect", status, final_url) + if "no-load" not in rest: + self.send("Page.loadEventFired", {}, session_id) + return {"frameId": target, "loaderId": loader} + + def _document(self, session, target, loader, request, status, url): + document = {"requestId": request, "loaderId": loader, "type": "Document", "frameId": target} + self.send("Network.requestWillBeSent", document, session) + self.send("Network.responseReceived", {**document, "response": {"status": status, "url": url}}, session) + self.send("Page.frameNavigated", {"frame": {"id": target, "loaderId": loader, "url": url}}, session) + + +async def _started(devtools, viewport=(1280, 800)) -> BrowserControl: + """A control following a browser just launched, on its first tab, ``start``.""" + control = BrowserControl(devtools, "agent", viewport) + await control.attach() + devtools.send("Target.targetCreated", _page("start")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-start", "waitingForDebugger": True, **_page("start")}) + return control + + +def _shop(url): + return "shop" if "shop.test" in url else None + + +def _outcomes(tabs): + return [(tab["tab"], tab["url"], tab["final_url"], tab["status"], tab["error"], tab["site"], tab["outcome"]) + for tab in tabs] + + +@pytest.mark.asyncio +async def test_its_start_tabs_load_in_order_in_its_first_tab_then_blank_ones_held_first_and_the_first_is_active(): + devtools = Navigating({"https://shop.test/": (200, "https://shop.test/login"), "https://slack.com/": "silent", + "https://github.com/": (404, "https://github.com/")}) + control = await _started(devtools) + loop = asyncio.get_running_loop() + started = loop.time() + + tabs = await control.load_start_tabs(list(devtools.pages), _shop, own_timeout_s=5, live_timeout_s=0.3) + + assert _outcomes(tabs) == [ + ("start", "https://shop.test/", "https://shop.test/login", 200, None, "shop", "loaded"), + ("t1", "https://slack.com/", None, None, None, None, "timed_out"), + ("t2", "https://github.com/", "https://github.com/", 404, None, None, "failed")] + assert loop.time() - started < 2 + assert list(control.tabs) == ["start", "t1", "t2"] and all(isinstance(tab["load_ms"], int) for tab in tabs) + assert _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": True}, None)] * 2 + for tab_session in ("s-start", "s-t1", "s-t2"): + sent = [method for method, _, on in devtools.calls if on == tab_session] + assert sent.index("Emulation.setDeviceMetricsOverride") < sent.index("Network.enable") < \ + sent.index("Page.navigate") < sent.index("Network.disable"), sent + assert devtools.calls[-2][:2] == ("Target.activateTarget", {"targetId": "start"}) + assert not control._loading + + +@pytest.mark.asyncio +@pytest.mark.parametrize("url, page, outcome, error, status, waited", [ + ("https://shop.test/", (503, "https://shop.test/", "no-load"), "failed", None, 503, False), + ("https://shop.test/", "refused", "failed", "net::ERR_CONNECTION_REFUSED", None, False), + ("https://shop.test/", "silent", "timed_out", None, None, True), + ("https://shop.test/", "stale", "timed_out", None, None, True), + ("https://slack.com/", "silent", "timed_out", None, None, True), + ("https://slack.com/", (503, "https://slack.com/", "no-load"), "timed_out", None, 503, True), + ("https://slack.com/", "refused", "failed", "net::ERR_CONNECTION_REFUSED", None, False), +], ids=["own-error-status-at-once", "own-refused", "own-never-loads", "own-stale-load", "live-never-loads", + "live-error-status-waited", "live-refused"]) +async def test_a_start_tab_settles_strictly_on_an_env_site_and_at_most_its_time_on_a_live_one( + url, page, outcome, error, status, waited): + control = await _started(Navigating({url: page})) + loop = asyncio.get_running_loop() + started = loop.time() + + (tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=0.3 if url.startswith("https://shop") else 5, + live_timeout_s=0.3 if url.startswith("https://slack") else 5) + + assert (tab["outcome"], tab["error"], tab["status"]) == (outcome, error, status) + assert (loop.time() - started >= 0.3) is waited + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redirect, final, status, outcome", [ + (("redirect", 200, "https://shop.test/notes"), "https://shop.test/notes", 200, "loaded"), + (("redirect", 404, "https://shop.test/gone"), "https://shop.test/gone", 404, "failed"), +], ids=["to-a-page", "to-an-error"]) +async def test_a_start_page_that_sends_itself_elsewhere_before_it_loads_is_judged_by_the_page_it_ends_on( + redirect, final, status, outcome): + control = await _started(Navigating({"https://shop.test/": (200, "https://shop.test/", redirect)})) + + (tab,) = await control.load_start_tabs(["https://shop.test/"], _shop, own_timeout_s=5) + + assert (tab["final_url"], tab["status"], tab["outcome"]) == (final, status, outcome) + + +@pytest.mark.asyncio +async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that_and_is_kept_with_its_pipe_gone(): + def gone(params, session): + raise ConnectionError("the browser closed its debugging pipe") + + devtools = Navigating({"https://shop.test/": "silent"}) + devtools.answers["Target.activateTarget"] = gone + control = await _started(devtools) + loading = asyncio.ensure_future(control.load_start_tabs(["https://shop.test/"], _shop, own_timeout_s=5)) + for _ in range(100): + if control._loading: + break + await asyncio.sleep(0.01) + control.ended(STOP) + + (tab,) = await loading + + assert (tab["outcome"], tab["error"]) == ("failed", "the browser ended") + + class Painting(StubDevTools): """A browser whose every tab has focus and paints, but those whose sessions are in ``blank``, which never answer a capture; it records each capture's timeout.""" diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 8631cb011..8da52800a 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -9,7 +9,8 @@ build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists i (`agentenv_protocol.website_browser.WebsiteBrowserEnvironment`): agentenv launches it on a debugging pipe with its own flags, and writes managed policies to `/etc/opt/chrome_for_testing/policies/managed/` for unattended use. It runs without Chromium's sandbox, which a container can't give it. Its env.toml spells out its settings and their defaults -under `[settings]`: `start_urls`, opened at boot, the first tab active; `window`, alone only; `locale`; `time_zone`; and +under `[settings]`: `start_urls`, opened at boot in order, the first tab active, each waited on to load; `window`, +alone only; `locale`; `time_zone`; and `geolocation` and `notifications`, `deny` or `ask`; and `dialogs`, `agent`, `accept` or `dismiss`. A computer env sets its own beside it (`apps = [{ env = "chromium", settings = { start_urls = ["https://example.test/"] } }]`), and a deploy overrides any of them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 2fe499a64..1f1c32974 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -287,7 +287,7 @@ async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, async def run_env_params(self, env_params: Optional[dict], deployed_envs: list[DeployedEnv]) -> Optional[dict]: """With ``site_routing`` and no ``sites``, ``env_params`` naming as its sites the websites the run deployed - before the computer: all of them, or those of the env ids ``site_routing`` lists.""" + before the computer, all of them or those of the env ids ``site_routing`` lists, once each says it's ready.""" routing = (env_params or {}).get("site_routing") if routing is not None and routing is not False and routing is not True and not ( isinstance(routing, list) and all(isinstance(env_id, str) for env_id in routing)): @@ -295,10 +295,12 @@ async def run_env_params(self, env_params: Optional[dict], deployed_envs: list[D f"{routing!r}") if not routing or env_params.get("sites"): return env_params - found = await sites.deployed_sites(deployed_envs, None if routing is True else list(routing)) + only = None if routing is True else list(routing) + found = await sites.deployed_sites(deployed_envs, only) if not found: raise ValueError(f"computer env {self.id!r}: env_params.site_routing reaches the run's websites, and the " "run deployed none before the computer") + await sites.require_ready(deployed_envs, only) return {**env_params, "sites": found} def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optional[dict] = None, diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 97c75ff55..fd236b291 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -20,6 +20,7 @@ import logging import re import shlex +import time from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from pathlib import Path, PurePosixPath @@ -27,12 +28,14 @@ from urllib.parse import urlsplit import httpx +from agentenv_protocol import client as protocol_client +from agentenv_protocol.website import WEBSITE_EXTENSION_URI as WEBSITE from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID -from agent_env.env.env import Env, gateway_url_of +from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of from agent_env.env.envs.computer.utils.site_proxy import is_local, matches from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox from agent_env.store.base import NotFoundError @@ -53,6 +56,9 @@ CLOCK_SKEW = timedelta(minutes=5) CARD_ATTEMPTS = 3 CARD_RETRY_S = 1.0 +#: How long a computer's deploy waits for the run's websites it routes to say they're ready, as each said at its own +#: deploy, and how often it asks meanwhile. +SITES_READY_TIMEOUT_S, SITES_READY_POLL_S = 60.0, 2.0 _LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$") _PROXY_SOURCE = Path(__file__).with_name("site_proxy.py") logger = logging.getLogger(__name__) @@ -460,6 +466,44 @@ async def deployed_sites(deployed_envs: list[DeployedEnv], only: Optional[list[s return _env_qualified(found) +async def require_ready(deployed_envs: list[DeployedEnv], only: Optional[list[str]] = None, *, + timeout_s: float = SITES_READY_TIMEOUT_S, poll_s: float = SITES_READY_POLL_S) -> None: + """Back once each deployed env whose websites a computer routes, those of the env ids ``only`` names or all, says + through ``website/v1``'s ``ready`` that every site of it is ready; RuntimeError naming each one still not, and why, + once ``timeout_s`` has passed. An env that serves no website isn't asked, nor one with no address to ask, and one + with no site serving ``ready`` is ready.""" + waiting = [record for record in deployed_envs + if (only is None or record.env_id in only) and serves_websites(record)] + deadline = time.monotonic() + timeout_s + while True: + why: dict[str, str] = {} + for record in waiting: + try: + readiness = await record.readiness() + except EnvCapabilityUnsupported: + continue + except (httpx.HTTPError, ValueError) as e: + why[record.env_id] = f"{type(e).__name__}: {e}" + continue + if not readiness.ready: + why[record.env_id] = "; ".join(readiness.problems) or "it says it isn't ready" + if not why: + return + if time.monotonic() >= deadline: + raise RuntimeError(f"site routing: the run's websites weren't ready after {timeout_s:g} s: " + + "; ".join(f"{env_id} ({reason})" for env_id, reason in why.items())) + waiting = [record for record in waiting if record.env_id in why] + await asyncio.sleep(poll_s) + + +def serves_websites(record: DeployedEnv) -> bool: + """Whether ``record`` serves websites a computer routes, as ``deployed_sites`` finds them: a named ``website/v1`` + child on its stored card, or a website frontend it lists.""" + card = getattr(record, "environment_card", None) or {} + return bool(getattr(record, "website_frontend_urls", None)) or any( + child.find_extension(WEBSITE) is not None and child.name for child in protocol_client.children(card)) + + def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]: """Each site found, a name that sites of different envs share made ``-`` for each of them, letters, digits, ``-`` and ``_`` kept, so every site of the run has a name of its own.""" diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index bf4897315..da886c5c6 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -13,6 +13,7 @@ from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING from agentenv_protocol.computer.app import COMPUTER_APP from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV +from agentenv_protocol.website_browser import BROWSER, LOADED, start_tab_why from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.attribution import Attribution @@ -201,13 +202,17 @@ async def _finish_boot(env: ComputerEnv, record: DeployedSandboxEnv, sandbox: Sa params: computer_params.ComputerParams, valid_for_s: int, resolution: Optional[str]) -> Optional[dict]: """What a deploy and a reset do once the computer's driver answers, in order: with ``site_routing``, the sites - routed and the held boot apps launched, the computer then ready; then its display checked. What the record keeps - of the routing, or None without it.""" + routed, each browser handed their names and the held boot apps launched, the computer then ready; then its display + checked; then each browser's start tabs read into the record, the boot failing on one on a site of the env's that + didn't load. What the record keeps of the routing, or None without it.""" routed = None if params.site_routing: routed = await _route_sites(env, sandbox, params, valid_for_s) - await _launch_held_apps(env, record, record.environment_url) + await _launch_held_apps(env, record, record.environment_url, params.sites) await _require_display(record, resolution) + if start_tabs := await _start_tabs(record): + record.metadata["computer"]["start_tabs"] = start_tabs + _refuse_failed_start_tabs(record, start_tabs) return routed @@ -220,12 +225,14 @@ async def _route_sites(env: ComputerEnv, sandbox: Sandbox, params: computer_para return {**routed, "valid_for_s": valid_for_s} -async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: str) -> None: +async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: str, + routed: list[computer_params.Site]) -> None: """Launch each app that starts with the computer, whose launch a deploy routing sites holds until the sites' names, - CA and proxy are in place, through its ``computer-app/v1`` ``launch`` once the computer lists it with its server, - then wait until the computer is ready, which its boot only waited to answer; the record keeps the card the computer - answers then, as its boot's card may list an app by its name alone. Each is launched as the child its card lists, - as a computer named for its app, a browser alone, has that name too.""" + CA and proxy are in place, through its ``computer-app/v1`` ``launch`` once the computer lists it with its server, a + website browser first handed the ``routed`` sites, then wait until the computer is ready, which its boot only + waited to answer; the record keeps the card the computer answers then, as its boot's card may list an app by its + name alone. Each is launched as the child its card lists, as a computer named for its app, a browser alone, has + that name too.""" os_type = env.computer.os.type timeout = ready_timeout(env.composed_apps(), os_type) held = [app for app in env.composed_apps() if (launch := app.launch(os_type)) is not None and launch.at == "boot"] @@ -235,11 +242,52 @@ async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: s launchers = {child.name: child for child in protocol_v1.children(record.environment_card) if protocol_v1.find_extension_method(child.card, COMPUTER_APP, "launch") is not None} for app in held: + await _hand_sites(record, launchers[app.environment_name], routed) await record.invoke(COMPUTER_APP, "launch", child=launchers[app.environment_name]) record.environment_card = await ready(url, timeout, record.control_token) record.environment_card_read_at_utc = datetime.now(timezone.utc).isoformat() +async def _hand_sites(record: DeployedSandboxEnv, child: protocol_v1.ChildEnv, + routed: list[computer_params.Site]) -> None: + """The ``routed`` sites, by name and domains, handed to the website browser ``child`` is, through ``browser/v1``'s + ``sites/set``, so its start tabs on them must load; a browser built before it took them is told nothing, its start + tabs on them waited on as any live one's.""" + if child.find_extension(BROWSER) is None: + return + try: + await record.invoke(BROWSER, "sites/set", {"sites": [{"name": site.name, "domains": site.domains} + for site in routed if site.domains]}, child=child) + except EnvCapabilityUnsupported: + logger.warning(f"the website browser {child.name} in {record.env_id} serves no browser/v1 sites/set, so its " + "start tabs on the env's sites are waited on as live ones") + + +async def _start_tabs(record: DeployedSandboxEnv) -> dict[str, list[dict]]: + """Each website browser's start tabs as its latest launch loaded them, by its name: the computer's own card's, for + a browser alone, or a child's; a browser that keeps none, built before it did, left out.""" + card = record.environment_card or {} + browsers = ([(card.get("name") or record.env_id, None)] if protocol_v1.find_extension(card, BROWSER) else []) \ + + [(child.name, child) for child in protocol_v1.children(card) if child.find_extension(BROWSER) is not None] + kept: dict[str, list[dict]] = {} + for name, child in browsers: + try: + launches = (await record.invoke(BROWSER, "record", child=child)).get("start_tabs") or [] + except EnvCapabilityUnsupported: + continue + if launches: + kept[name] = launches[-1]["tabs"] + return kept + + +def _refuse_failed_start_tabs(record: DeployedSandboxEnv, start_tabs: dict[str, list[dict]]) -> None: + """RuntimeError naming each of ``start_tabs``, by browser, on one of the env's sites that didn't load.""" + if failed := [f"{name}'s start tab {tab['url']} on the env's site {tab['site']} {start_tab_why(tab)}" + for name, tabs in start_tabs.items() for tab in tabs + if tab["site"] is not None and tab["outcome"] != LOADED]: + raise RuntimeError(f"computer env {record.env_id!r}: " + "; ".join(failed)) + + async def end_recordings(record: DeployedSandboxEnv, reason: str) -> None: """Have the computer's driver end every recording not yet ended, each uploading what's left and its manifest under the grant it was started with, before the computer goes; a driver that doesn't record has none.""" diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index da4144c84..9934ae261 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -13,8 +13,9 @@ driver's, the agent's address keeps working, and two agents connected at once each drive the browser through their own. browser/v1 reads and sets its cookies, local storage and downloads, and its data plane signs it in, snapshots it, resets it, loads a snapshot back and holds up under chaos; the browser task steps set its cookies and storage and open, activate and -close its tabs by URL; and a task whose deploy_env pins a version of the browser, its latest one with none, passes -preflight on the version pinned and runs its browser steps through task.run. Needs Docker and the local registry on +close its tabs by URL; a task whose deploy_env pins a version of the browser, its latest one with none, passes +preflight on the version pinned and runs its browser steps through task.run; and each start tab's page sees the +window's size from its first script. Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio @@ -54,6 +55,7 @@ from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data, check_browser_data_under_chaos from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state, check_browser_state_under_chaos +from tst.integration.computer.start_tabs import tabs_of from tst.util.a2a_test_agent import put_test_agent pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -126,6 +128,8 @@ def log_message(self, *args): DATA_PAGE = "data:text/html,Home

Home

" #: A page a task's tab steps open, bring to the front and close by its URL. PINNED_PAGE = "data:text/html,pinned" +#: A page whose title is the window's size as its first script sees it, told apart by ``{}``. +SIZED = "data:text/html,{}" # Every TCP port something in the computer listens on, and whether any process is an X server. LISTENING = """import glob ports = set() @@ -412,6 +416,27 @@ async def test_a_task_pinning_a_version_of_the_browser_passes_preflight_on_it_an assert records["sign-out"]["deleted"] == 1 and not report.failed, (records, report) +@pytest.mark.asyncio +async def test_each_start_tabs_page_sees_the_windows_size_from_its_first_script_in_order_the_first_active( + browser_computer): + urls = [SIZED.format(index) for index in range(3)] + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": urls}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + start_tabs = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert [tab["outcome"] for tab in start_tabs] == ["loaded"] * 3 + assert [tab["id"] for tab in tabs] == [tab["tab"] for tab in start_tabs] + assert [(tab["title"], tab["active"]) for tab in tabs] == [("1280x800", True), ("1280x800", False), + ("1280x800", False)] + assert not report.failed + + @pytest.mark.asyncio async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( browser_computer): diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index fb964624f..5e52d5b17 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -10,8 +10,9 @@ accepted for it, and each launch, exit and stop with its cause, across a kill and a restart of its server; and browser/v1 reads and sets its cookies, local storage and downloads; and its data plane signs it in, snapshots it, resets it and loads a snapshot back; and the browser task steps set its cookies and storage and open, activate and close its tabs by URL. -Needs Docker -and the local registry on localhost:5000.""" +Its start tabs open in order, the first active, a deploy recording each one's outcome and going on past a live page +that never answers; and with the run's sites routed to it, a start tab on one must load, judged by the page it +lands on when it sends itself elsewhere before it loads, so a site answering 503 fails the deploy, naming it. Needs Docker and the local registry on localhost:5000.""" import asyncio import hashlib @@ -32,6 +33,7 @@ from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state +from tst.integration.computer.start_tabs import host_pages, tabs_of from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready from agent_env.env.envs.computer.utils import inside from agent_env.store.routing import namespace_routing @@ -185,7 +187,7 @@ async def test_it_runs_chrome_itself_on_its_pipe_answers_prompts_as_the_deploy_s assert argv[0].endswith("chromium/chrome/chrome") and argv[1:3] == ["--no-sandbox", "--class=Chromium"] assert argv[3].endswith("/.config/chromium") and argv[4].endswith("/.cache/chromium") assert argv[5:10] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US", - "--disable-dev-shm-usage"] and argv[10:] == [LOOPBACK] + "--disable-dev-shm-usage"] and argv[10:] == ["about:blank"] assert browser["listening"] == 0 assert permissions == ["denied prompt - Google Chrome for Testing"] assert launched["exit_code"] == 0 and len(windows) == 2 and after["pid"] == browser["pid"] @@ -352,6 +354,55 @@ async def act(): assert not report.failed +@pytest.mark.asyncio +async def test_its_start_tabs_open_in_order_the_first_active_and_a_live_one_that_never_answers_is_recorded_not_fatal( + chromium_desktop): + with host_pages() as port, namespace_routing(): + urls = [f"http://host.docker.internal:{port}/{page}" for page in ("alpha", "bravo", "hang")] + deployed = await Env.get(chromium_desktop.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": urls}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + start_tabs = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert [(tab["url"], tab["status"], tab["site"], tab["outcome"]) for tab in start_tabs] == [ + (urls[0], 200, None, "loaded"), (urls[1], 200, None, "loaded"), (urls[2], None, None, "timed_out")] + assert start_tabs[2]["load_ms"] >= 30_000 and [tab["id"] for tab in tabs] == [tab["tab"] for tab in start_tabs] + assert [(tab["url"], tab["title"], tab["active"]) for tab in tabs[:2]] == [(urls[0], "alpha", True), + (urls[1], "bravo", False)] + assert not tabs[2]["active"] and not report.failed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [200, 503], ids=["loads", "answers-503"]) +async def test_a_start_tab_on_a_routed_site_must_load_where_it_lands_so_a_site_answering_503_fails_the_deploy( + chromium_desktop, status): + with host_pages(site_status=status) as port, namespace_routing(): + params = {"site_routing": True, "sites": [{"name": "shop", "url": f"http://127.0.0.1:{port}/website/shop/", + "domains": ["shop.test"]}], + "app_settings": {"chromium": {"start_urls": ["https://shop.test/"]}}} + if status != 200: + with pytest.raises(RuntimeError, match=r"chromium's start tab https://shop.test/ on the env's site shop " + r"answered 503"): + await Env.get(chromium_desktop.id).deploy(env_params=params) + return + deployed = await Env.get(chromium_desktop.id).deploy(env_params=params) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + (tab,) = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert (tab["url"], tab["final_url"], tab["status"], tab["site"], tab["outcome"]) == ( + "https://shop.test/", "https://shop.test/landed", 200, "shop", "loaded") + assert [(open_tab["url"], open_tab["title"]) for open_tab in tabs] == [("https://shop.test/landed", "landed")] + assert not report.failed + + @pytest.mark.asyncio async def test_its_task_steps_set_cookies_and_storage_open_a_tab_behind_and_activate_and_close_it_by_its_url( chromium_desktop): diff --git a/tst/integration/computer/start_tabs.py b/tst/integration/computer/start_tabs.py new file mode 100644 index 000000000..ffb236f58 --- /dev/null +++ b/tst/integration/computer/start_tabs.py @@ -0,0 +1,64 @@ +"""Pages a computer's start tabs open, served on this machine, which a computer reaches as ``host.docker.internal``: a +site's route, whose home page sends itself elsewhere before it loads, or answers an error; two pages titled by name; +and one that never answers.""" + +import contextlib +import http.server +import threading +import time +from collections.abc import Iterator + +import httpx + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER + +#: How long the page that never answers holds its request, and the image that holds the site's home page from loading. +HANG_S, SLOW_S = 120, 30 +#: The site's home page: it replaces itself with /landed by script as it's read, while an image keeps it from loading. +LEAVING = b"leaving" + + +@contextlib.contextmanager +def host_pages(site_status: int = 200) -> Iterator[int]: + """The pages served on every address of this machine while the block runs, at the port it yields: under + ``/website/shop/``, a site's route, its home page, which sends itself to ``landed`` before it loads, or with any + other ``site_status`` answers it, and ``landed``; ``/alpha`` and ``/bravo``; and ``/hang``, which never answers.""" + + class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path in ("/hang", "/website/shop/slow"): + time.sleep(HANG_S if self.path == "/hang" else SLOW_S) + return + home = self.path == "/website/shop/" + status = site_status if home else 200 + title = self.path.strip("/").rsplit("/", 1)[-1] + body = LEAVING if home and status == 200 else f"{title}".encode() + self.send_response(status) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("0.0.0.0", 0), Page) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield server.server_address[1] + finally: + server.shutdown() + server.server_close() + + +async def tabs_of(deployed, app: str = "chromium") -> list[dict]: + """The open tabs of the browser ``app`` in ``deployed``, as its ``browser/v1`` ``tabs/list`` gives them.""" + async with httpx.AsyncClient(timeout=30, headers=deployed.control_headers) as http: + card = (await http.get(f"{deployed.environment_url}/svc/{app}{WELL_KNOWN_PATH}")).json() + endpoint = protocol_v1.find_extension_method(card, BROWSER, "tabs/list")["endpoint"] + answer = await http.get(f"{deployed.environment_url}/svc/{app}{endpoint}") + answer.raise_for_status() + return answer.json()["tabs"] diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index e20cff8ca..edd6b68ee 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -130,7 +130,7 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun assert argv[4].startswith("--disk-cache-dir=") and argv[4].endswith("/.cache/chromium") assert argv[5:10] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR", "--disable-dev-shm-usage"] - assert argv[10:] == [TIME_ZONE_PAGE, LOOPBACK] + assert argv[10:] == ["about:blank"] assert browser["listening"] == 0 and port_file["exit_code"] != 0 assert zone.startswith("Europe/Paris") and loopback.startswith("Loopback page") assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( @@ -140,8 +140,8 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun "156.0.8078.4", "real", ["cdp", "cookies/delete", "cookies/get", "cookies/set", "dialog/answer", "downloads/get", "downloads/list", "files/choose", "history", "local_storage/clear", "local_storage/get", "local_storage/set", "navigate", "page/capture", "page/input", - "page/state", "page/viewport", "record", "tabs/activate", "tabs/close", "tabs/list", - "tabs/open"]) + "page/state", "page/viewport", "record", "sites/set", "tabs/activate", "tabs/close", + "tabs/list", "tabs/open"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 8a9d4bf0c..c1c863bde 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -27,6 +27,7 @@ from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV from agentenv_protocol.computer.conformance import CheckResult, ProbeReport from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI +from agentenv_protocol.website_browser import BROWSER from click.testing import CliRunner from agent_env.artifact.artifacts.computer import ComputerArtifact @@ -924,7 +925,7 @@ def _sites_gateway(): @pytest.fixture def routes(monkeypatch): """Each site routing a deploy asks for, as (sandbox, sites, options), answering what the proxy's install would; the - gateway serving the notes site's card.""" + gateway serving the notes site's card, its websites ready.""" from agentenv_protocol import client as protocol_v1 made = [] @@ -938,6 +939,7 @@ async def get_card(url, timeout=10, verify=True): return NOTES_CARD monkeypatch.setattr(env_computer_provider.sites, "route", route) + monkeypatch.setattr(env_computer_provider.sites, "require_ready", AsyncMock()) monkeypatch.setattr(protocol_v1, "get_card", get_card) monkeypatch.setattr(env_computer_provider, "ready", AsyncMock(return_value=DRIVER_CARD)) return made @@ -1067,17 +1069,119 @@ async def ready(url, timeout, token): monkeypatch.setattr(env_computer_provider, "ready", ready) monkeypatch.setattr(env_computer_provider, "ready_timeout", lambda apps, os_type: 30.0) record = SimpleNamespace(invoke=invoke, control_token="t0ken", environment_card={"name": "booted"}) - await env_computer_provider._launch_held_apps(env, record, "http://computer") + await env_computer_provider._launch_held_apps(env, record, "http://computer", []) assert order == [("listed", "http://computer", ["mail", "notes"]), (COMPUTER_APP, "launch", "mail", True, "mail"), (COMPUTER_APP, "launch", "notes", True, "mail"), ("ready", "http://computer", "t0ken")] assert record.environment_card == {"name": "ready"} and record.environment_card_read_at_utc order.clear() apps[:] = [apps[1]] - await env_computer_provider._launch_held_apps(env, record, "http://computer") + await env_computer_provider._launch_held_apps(env, record, "http://computer", []) assert order == [("ready", "http://computer", "t0ken")] +def _app_card(name, browser_methods=None): + """A held app's card, launched through computer-app/v1, a website browser when it serves ``browser_methods``.""" + extensions = [{"uri": COMPUTER_APP, "params": {"endpoint": f"/svc/{name}/agentenv/ext/app", + "methods": {"launch": {}}}}] + if browser_methods is not None: + extensions.append({"uri": BROWSER, "params": {"endpoint": f"/svc/{name}/agentenv/ext/browser", + "methods": {method: {} for method in browser_methods}}}) + return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": extensions}} + + +@pytest.mark.asyncio +async def test_each_held_browser_is_handed_the_routed_sites_before_its_launch_and_one_built_before_is_only_launched( + monkeypatch, caplog): + from agent_env.env.env import EnvCapabilityUnsupported + from agent_env.env.envs.computer.utils.params import Site + + calls = [] + apps = [SimpleNamespace(environment_name=name, launch=lambda os_type: SimpleNamespace(at="boot")) + for name in ("chromium", "older", "mail")] + env = SimpleNamespace(computer=SimpleNamespace(os=SimpleNamespace(type="linux")), composed_apps=lambda: apps) + listed_card = {"name": "desk", "children_environments": [ + _app_card("chromium", ["record", "sites/set"]), _app_card("older", ["record"]), _app_card("mail")]} + + async def invoke(uri, method, params=None, *, environment_name=None, child=None): + if method == "sites/set" and child.name == "older": + raise EnvCapabilityUnsupported(uri, method, "desk", None, child.name) + calls.append((uri, method, child.name, params)) + + monkeypatch.setattr(env_computer_provider, "listed", AsyncMock(return_value=listed_card)) + monkeypatch.setattr(env_computer_provider, "ready", AsyncMock(return_value=listed_card)) + monkeypatch.setattr(env_computer_provider, "ready_timeout", lambda apps, os_type: 30.0) + record = SimpleNamespace(invoke=invoke, control_token="t0ken", environment_card={}, env_id="desk") + routed = [Site(name="shop", url="http://gw/website/shop/", domains=["shop.test", "*.shop.test"]), + Site(name="bare", url="http://gw/website/bare/")] + + with caplog.at_level(logging.WARNING): + await env_computer_provider._launch_held_apps(env, record, "http://computer", routed) + + assert calls == [(BROWSER, "sites/set", "chromium", {"sites": [{"name": "shop", "domains": ["shop.test", + "*.shop.test"]}]}), + (COMPUTER_APP, "launch", "chromium", None), (COMPUTER_APP, "launch", "older", None), + (COMPUTER_APP, "launch", "mail", None)] + assert "older in desk serves no browser/v1 sites/set" in caplog.text + + +def _start_tabs_record(card, launches_by_browser): + """A deployed computer whose card is ``card`` and whose browsers' ``record`` keeps ``launches_by_browser``, by the + browser's name, None for the computer's own card; a browser absent from it built before start tabs.""" + from agent_env.env.env import EnvCapabilityUnsupported + + async def invoke(uri, method, params=None, *, environment_name=None, child=None): + name = child.name if child is not None else None + if (uri, method) != (BROWSER, "record") or name not in launches_by_browser: + raise EnvCapabilityUnsupported(uri, method, "desk", None, name) + return {"start_tabs": [{"at": "t", "tabs": tabs} for tabs in launches_by_browser[name]]} + + return SimpleNamespace(invoke=invoke, environment_card=card, env_id="@local/desk") + + +def _tab(url, site=None, outcome="loaded", **fields): + return {"tab": "t1", "url": url, "final_url": url, "status": 200, "error": None, "load_ms": 120, "site": site, + "outcome": outcome, **fields} + + +@pytest.mark.asyncio +async def test_each_browsers_start_tabs_of_its_latest_launch_are_kept_and_one_built_before_them_is_left_out(): + card = {"name": "desk", "children_environments": [_app_card("chromium", ["record"]), _app_card("older", []), + _app_card("mail")]} + record = _start_tabs_record(card, {"chromium": [[_tab("about:blank")], [ + _tab("https://shop.test/", "shop"), _tab("https://slack.com/", outcome="timed_out", status=None)]]}) + alone = _start_tabs_record({"name": "chromium-driver", "capabilities": {"extensions": [ + {"uri": BROWSER, "params": {"endpoint": "/ext/browser", "methods": {"record": {}}}}]}}, + {None: [[_tab("https://linear.app/")]]}) + + assert await env_computer_provider._start_tabs(record) == {"chromium": [ + _tab("https://shop.test/", "shop"), _tab("https://slack.com/", outcome="timed_out", status=None)]} + assert await env_computer_provider._start_tabs(alone) == {"chromium-driver": [_tab("https://linear.app/")]} + + +@pytest.mark.asyncio +async def test_a_start_tab_on_one_of_the_env_sites_that_didnt_load_fails_the_boot_naming_each_once_kept( + monkeypatch): + card = {"name": "desk", "children_environments": [_app_card("chromium", ["record"])]} + tabs = [_tab("https://shop.test/", "shop", "failed", status=503), + _tab("https://mail.test/", "mail", "failed", status=None, error="net::ERR_CONNECTION_REFUSED"), + _tab("https://notes.test/", "notes", "timed_out", status=None, load_ms=60000), + _tab("https://slack.com/", None, "failed", status=500)] + record = _start_tabs_record(card, {"chromium": [tabs]}) + record.metadata = {"computer": {"start_tabs": {"chromium": [_tab("https://shop.test/", "shop")]}}} + monkeypatch.setattr(env_computer_provider, "_require_display", AsyncMock()) + + with pytest.raises(RuntimeError) as failed: + await env_computer_provider._finish_boot(SimpleNamespace(), record, None, + SimpleNamespace(site_routing=False), 0, None) + + assert record.metadata["computer"]["start_tabs"] == {"chromium": tabs} + assert str(failed.value) == ( + "computer env '@local/desk': chromium's start tab https://shop.test/ on the env's site shop answered 503; " + "chromium's start tab https://mail.test/ on the env's site mail didn't load: net::ERR_CONNECTION_REFUSED; " + "chromium's start tab https://notes.test/ on the env's site notes was still loading after 60 s") + + @pytest.mark.asyncio async def test_a_held_app_is_awaited_until_the_driver_lists_it_with_its_server(monkeypatch): """A page driver answers before its browser's server does, and lists the browser by its name alone until then.""" diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 18816f7d4..cb4fa285e 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -497,3 +497,51 @@ async def get_card(url, timeout=10, verify=True): def _status_error(url, code): request = httpx.Request("GET", f"{url}/.well-known/agent-env.json") return httpx.HTTPStatusError(f"{code}", request=request, response=httpx.Response(code, request=request)) + + +class _Website: + """A deployed env serving a website, whose ``readiness`` answers each of ``answers`` in turn, then the last again: a + readiness, an error to raise, or None for a record with no address.""" + + def __init__(self, env_id, *answers, card=None): + self.env_id, self.answers, self.asked = env_id, list(answers), 0 + self.environment_card = card + self.website_frontend_urls = None if card is not None else {"site": f"http://gw/website/{env_id}/"} + + async def readiness(self): + from agent_env.env.env import EnvCapabilityUnsupported + from agentenv_protocol.website import WebsiteReadiness + + answer = self.answers[min(self.asked, len(self.answers) - 1)] + self.asked += 1 + if answer is None: + raise EnvCapabilityUnsupported("urn:agentenv:website/v1", "ready", self.env_id, None) + if isinstance(answer, Exception): + raise answer + return WebsiteReadiness(ready=not answer, problems=answer) + + +@pytest.mark.asyncio +async def test_the_websites_a_computer_routes_are_waited_on_till_ready_one_never_ready_named_and_others_not_asked(): + late = _Website("late", ["notes: warming up"], []) + shop = _Website("shop", []) + other = _Website("other", ["never asked"]) + unaddressed = _Website("unaddressed", None) + + on_its_card = _Website("on-its-card", [], card={"name": "kit", "children_environments": [{ + "name": "notes", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {}}]}}]}) + mcp_only = _Website("mcp-only", httpx.ConnectError("never asked"), card={"name": "mcp", "children_environments": [ + {"name": "tools", "capabilities": {"extensions": []}}]}) + + await sites.require_ready([late, shop, other, unaddressed], ["late", "shop", "unaddressed"], timeout_s=5, + poll_s=0.01) + await sites.require_ready([on_its_card, mcp_only], timeout_s=0.05, poll_s=0.01) + broken = _Website("broken", ["shop: its database is down"]) + unreachable = _Website("unreachable", httpx.ConnectError("refused")) + with pytest.raises(RuntimeError) as never: + await sites.require_ready([broken, unreachable, shop], timeout_s=0.05, poll_s=0.01) + + assert (late.asked, shop.asked, other.asked, unaddressed.asked) == (2, 2, 0, 1) + assert (on_its_card.asked, mcp_only.asked) == (1, 0) + assert str(never.value).startswith("site routing: the run's websites weren't ready after 0.05 s: ") + assert str(never.value).endswith("broken (shop: its database is down); unreachable (ConnectError: refused)")