From cac4d912811bebd00e7efff9f706209b4adeebae Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 7 Nov 2026 09:06:24 -0800 Subject: [PATCH 1/5] feat(browser): start tabs load in order and settle before agents get an address Every start tab now opens over the browser's pipe, in a computer and alone: the first in the blank tab Chromium starts with, the rest in blank tabs after it, each held to the window's size before it goes to its URL, the first brought to the front. They load at once; each settles once it loads, fails (an error status on one of the env's own sites settling it at once) or has waited its time (the ready timeout on an own site, 30 s on a live one), and a start_tabs_loaded ready check passes once all have. Outcomes are kept in browser/v1 record per launch, in data/get and in a computer deploy's record. A deploy routing sites asks each routed website's ready, hands each browser the sites through a new browser/v1 sites/set before launching it, and fails naming any start tab on an own site that didn't load; data/reset (through a new relaunch hook) and a load that reopens the start tabs fail the same way. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 25 ++- .../src/agentenv_protocol/computer/app.py | 5 +- .../agentenv_protocol/computer/app_data.py | 9 +- .../website_browser/__init__.py | 24 +- .../website_browser/control.py | 171 ++++++++++++-- .../website_browser/environment.py | 110 +++++++-- .../website_browser/shapes.py | 35 ++- .../tests/test_browser_data.py | 68 +++++- .../tests/test_computer_app_data.py | 31 +++ .../tests/test_website_browser.py | 210 ++++++++++++++++-- src/agent_env/browsers/README.md | 3 +- src/agent_env/env/envs/computer/env.py | 6 +- .../env/envs/computer/utils/sites.py | 35 ++- .../env_providers/env_computer_provider.py | 60 ++++- .../computer/browser_computer_test.py | 29 ++- tst/integration/computer/chromium_app_test.py | 57 ++++- tst/integration/computer/start_tabs.py | 59 +++++ .../computer/website_browser_test.py | 6 +- tst/unit/env/envs/test_computer_env.py | 105 ++++++++- tst/unit/env/envs/test_computer_sites.py | 39 ++++ 20 files changed, 985 insertions(+), 102 deletions(-) create mode 100644 tst/integration/computer/start_tabs.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 0c569ee12..accb65913 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -878,14 +878,14 @@ under the session user's home (`.config/chromium` by default), whose contents fr are the profile's template. The base class adds: - **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in - tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless - the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`, + tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`); `window`, 1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; `geolocation` and `notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who answers a page's JavaScript dialogs: `agent` (the default), or agentenv, accepting (`accept`, a `prompt()` with its default text) or dismissing (`dismiss`) each. Either way agentenv leaves a page that asks before it's left (`beforeunload`), so none of its answers blocks a navigation. -- **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, its +- **The flags agentenv owns**, after the subclass's command and before `about:blank`, the one tab its command line + opens: its profile, its HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`, `--no-default-browser-check`, `--lang` and, when `/dev/shm` holds under 1 GiB (Docker gives 64 MiB), `--disable-dev-shm-usage`, so it keeps its shared memory in files. With @@ -944,6 +944,8 @@ are the profile's template. The base class adds: - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so alone, it intercepts; - `page/state`, the active tab's id, URL and title and the dialog or file chooser open on it. + - `sites/set {sites}`, the env's own sites, each `{name, domains}`, a `*.` domain matching one label more, which + the deploy hands it before it launches the browser and its server keeps for every launch after. - `cookies/get {urls?}`, `cookies/set {cookies}` and `cookies/delete {name?, domain?, path?}`, on its default context, each cookie as Playwright's `storageState` has it, CDP's `Network.Cookie` fields (`name`, `value`, `domain`, `path`, `expires`, `httpOnly`, `secure`, `sameSite`), so a storage state exported from Playwright loads @@ -966,9 +968,24 @@ are the profile's template. The base class adds: under the `write_namespace` grant when there is one, else inline, as `data/get` returns files. Inline, the whole file is read into memory and sent base64 in one answer, so a large one belongs under a grant. + **Start tabs.** At each launch its start tabs open over its pipe, in a computer and alone alike: the first in the + blank tab its command line opened, the rest in tabs it opens blank after it, each held to the window's size + (headless) before it goes to its URL, so its page sees that size from its first script; then the first is brought + to the front. They load at once, and each settles once it has loaded (its load event fired, its document's final + status read), once it failed (a network error, or on one of the env's sites, as `sites/set` named them, a 4xx or + 5xx status, which settles it at once), or once it has been waited on: as long as the browser may take to be ready + on one of the env's sites, 30 s on any other. Its `start_tabs_loaded` ready check passes once all have settled, so + `cdp` and a deploy wait for them. `record`'s `start_tabs` keeps each launch's (`StartTabsLaunch`: `at` and its + `tabs`, each a `StartTab`: `tab`, `url`, `final_url`, `status`, `error`, `load_ms`, `site` — the env's site's + name, null for a live one — and `outcome`, `loaded`, `failed` or `timed_out`), and `data/get`'s state keeps its + latest launch's. A deploy of a computer reads them once it's ready, keeps them in its record and fails, naming + each, if one on the env's sites didn't load; `data/reset` and a load that opens the start tabs again fail + the same way, while a server's restart only records it. + The browser's data plane is a file tree, every file by its path under the session user's home: - `data/get` returns its state as `browser-state.json` (`BrowserState`: `cookies` and `local_storage` - as above, `tabs`, `closed_tabs`, `downloads` and `clock`, `real` with its `now`); its profile's files, + as above, `tabs`, `closed_tabs`, `downloads`, `clock`, `real` with its `now`, and `start_tabs`); its + profile's files, each database with its journal or log, copied in one pass with the browser paused once its state is read, so the files are of one moment, without caches (`Cache`, `Code Cache`, `GPUCache`, the shader and Dawn caches, `CacheStorage`, `ScriptCache`, `Crashpad`) or locks diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 717cac06e..02ca54702 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -446,7 +446,7 @@ def __init__(self) -> None: self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, cls.data_timeout_s, live=cls.data_live, keeps_paths=cls.data_keeps_paths, - resets_failed_add=self._resets_failed_add) + resets_failed_add=self._resets_failed_add, relaunched=self._relaunched) def _launch_for(self, launch: Launch) -> Launch: """The launch its server uses, from the one declared for its OS, whose ``ready`` timeout it keeps: as @@ -473,6 +473,9 @@ def _resets_failed_add(self) -> bool: """Whether a load that fails is followed by a reset, so the app isn't left half loaded: always.""" return True + async def _relaunched(self, op: str) -> None: + """What a data call that stopped the app, ``op``, awaits once it has launched it again: nothing.""" + @classmethod def settings_on(cls, os_type: Optional[str]) -> type[BaseModel]: """Its ``Settings`` on ``os_type``, with the fields its OS adds.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 397c97699..8bf71edac 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -11,7 +11,7 @@ import logging import shutil import tempfile -from collections.abc import Callable +from collections.abc import Awaitable, Callable from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Optional @@ -40,7 +40,8 @@ class AppDataPlane: def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[str], timeout_s: float, *, live: frozenset[str] = frozenset(), keeps_paths: bool = False, - resets_failed_add: Callable[[], bool] = lambda: True) -> None: + resets_failed_add: Callable[[], bool] = lambda: True, + relaunched: Optional[Callable[[str], Awaitable[None]]] = None) -> None: self.runtime = runtime self.hooks = hooks self.paths = paths @@ -48,6 +49,8 @@ def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[ #: The operations served on the running app; whether a hook's reset needs the paths as installed; and whether #: a load that fails is followed by a reset now. self.live, self.keeps_paths, self.resets_failed_add = live, keeps_paths, resets_failed_add + #: What a call that stopped the app awaits once it has launched it again, whose error fails the call. + self.relaunched = relaunched self.turn = asyncio.Lock() self.changing: set[asyncio.Task] = set() @@ -98,6 +101,8 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. finally: if running: await self.runtime.launch_app(_CAUSES[op]) + if running and self.relaunched is not None: + await self.relaunched(op) except Exception as e: if not answer.done(): answer.set_exception(e) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py index 904656ebc..bc0e84800 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/__init__.py @@ -5,19 +5,21 @@ skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's, whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its settings.""" -from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, HASH_CHUNK_BYTES, - PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, - BrowserControl) +from .control import (ACTIVE_CHECK_TIMEOUT_S, AGENT, AGENTENV, CAPTURE_TIMEOUT_S, DOWNLOADS, EXIT, FAILED, + HASH_CHUNK_BYTES, LIVE_START_TAB_TIMEOUT_S, LOADED, PAGE_INPUT, PAGES, PIN_TIMEOUT_S, STOP, + TIMED_OUT, UNKNOWN, UNNAVIGATED_CAPTURE_TIMEOUT_S, UPLOADS, BrowserControl) from .devtools import MAX_MESSAGE_BYTES, PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe from .environment import (ALONE_BYPASS, BLOCK, BROWSER, COMPUTER_BYPASS, OWNED_FLAGS, POLICIES, READY_POLL_S, - SHM_MIN_BYTES, SITE_PROXY_ENV, SKILL, BrowserSettings, WebsiteBrowserEnvironment, Window) + SHM_MIN_BYTES, SITE_PROXY_ENV, SITES, SKILL, BrowserSettings, WebsiteBrowserEnvironment, + Window, start_tab_why) from .shapes import (BrowserRecord, ClosedTab, Cookie, CookieToSet, Dialog, Download, LaunchEntry, OpenTab, - OriginStorage, PartitionKey, StorageItem, origin_of) + OriginStorage, PartitionKey, SiteNames, StartTab, StartTabsLaunch, StorageItem, origin_of) __all__ = ["ACTIVE_CHECK_TIMEOUT_S", "AGENT", "AGENTENV", "ALONE_BYPASS", "BLOCK", "BROWSER", "CAPTURE_TIMEOUT_S", - "COMPUTER_BYPASS", "DOWNLOADS", "EXIT", "HASH_CHUNK_BYTES", "MAX_MESSAGE_BYTES", "OWNED_FLAGS", "PAGE_INPUT", - "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", "POLICIES", "READY_POLL_S", "SHM_MIN_BYTES", - "SITE_PROXY_ENV", "SKILL", "STOP", "UNKNOWN", "UNNAVIGATED_CAPTURE_TIMEOUT_S", "UPLOADS", "BrowserControl", - "BrowserSettings", "DevToolsError", "DevToolsPipe", "WebsiteBrowserEnvironment", "Window", - "BrowserRecord", "ClosedTab", "Cookie", "CookieToSet", "Dialog", "Download", "LaunchEntry", "OpenTab", - "OriginStorage", "PartitionKey", "StorageItem", "origin_of"] + "COMPUTER_BYPASS", "DOWNLOADS", "EXIT", "FAILED", "HASH_CHUNK_BYTES", "LIVE_START_TAB_TIMEOUT_S", "LOADED", + "MAX_MESSAGE_BYTES", "OWNED_FLAGS", "PAGE_INPUT", "PAGES", "PIN_TIMEOUT_S", "PIPE_IN", "PIPE_OUT", + "POLICIES", "READY_POLL_S", "SHM_MIN_BYTES", "SITE_PROXY_ENV", "SITES", "SKILL", "STOP", "TIMED_OUT", + "UNKNOWN", "UNNAVIGATED_CAPTURE_TIMEOUT_S", "UPLOADS", "BrowserControl", "BrowserSettings", "DevToolsError", + "DevToolsPipe", "WebsiteBrowserEnvironment", "Window", "BrowserRecord", "ClosedTab", "Cookie", "CookieToSet", + "Dialog", "Download", "LaunchEntry", "OpenTab", "OriginStorage", "PartitionKey", "SiteNames", "StartTab", + "StartTabsLaunch", "StorageItem", "origin_of", "start_tab_why"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index b8da02816..fd8a98e9a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -8,6 +8,7 @@ import logging import time from collections.abc import AsyncIterator +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any, Awaitable, Callable, Optional @@ -15,7 +16,7 @@ from ..computer.errors import DeviceUnavailable, InvalidArgument from .devtools import DevToolsError, DevToolsPipe -from .shapes import (ClosedTab, Cookie, CookieToSet, Dialog, Download, OpenTab, OriginStorage, StorageItem, +from .shapes import (ClosedTab, Cookie, CookieToSet, Dialog, Download, OpenTab, OriginStorage, StartTab, StorageItem, origin_of) logger = logging.getLogger(__name__) @@ -29,6 +30,10 @@ #: of agentenv's own may take to reach an origin, and how often it's asked meanwhile. PIN_TIMEOUT_S = 10.0 ARRIVAL_POLL_S = 0.05 +#: How long a start tab on a site the env doesn't serve may take to load before it's left loading. +LIVE_START_TAB_TIMEOUT_S = 30.0 +#: How a start tab settled. +LOADED, FAILED, TIMED_OUT = "loaded", "failed", "timed_out" #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 #: Who answered a dialog when agentenv didn't. @@ -80,6 +85,8 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl self._dialog_opened: dict[str, asyncio.Event] = {} self._pinned: dict[str, asyncio.Event] = {} self._tasks: set[asyncio.Future] = set() + #: Each start tab loading, by its session. + self._loading: dict[str, _StartLoad] = {} #: Every http and https origin a page or frame it follows loaded since this server started, and the sessions #: of agentenv's own pages whose requests it answers itself. self.origins: set[str] = set() @@ -108,6 +115,9 @@ def ended(self, how: str) -> None: self._shots.clear() self._dialog_opened.clear() self._pinned.clear() + for load in self._loading.values(): + load.error = load.error or "the browser ended" + load.settled.set() async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" @@ -138,18 +148,81 @@ async def open_tab(self, url: Optional[str] = None, background: bool = False) -> await self._bring_to_front(active) return target - async def reopen_start_tabs(self, urls: list[str]) -> None: - """The start tabs at their ``urls`` again, in place, the first active: each tab open, in opened order, goes to - its URL, and any the browser lacks opens. A page already loaded doesn't read cookies set after it, so a tab - a load signed in shows it only once it loads again.""" - tabs = list(self.tabs) - for index, url in enumerate(urls): - if index < len(tabs): - await self._navigate_tab(tabs[index], url) - else: - tabs.append(await self.open_tab(url)) - if tabs: - await self._bring_to_front(tabs[0]) + async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Optional[str]], own_timeout_s: float, + live_timeout_s: float = LIVE_START_TAB_TIMEOUT_S) -> list[dict]: + """Each of ``urls`` in a start tab, in order, the first then active: in the tabs open now, in opened order, and + in tabs it opens blank after them; each held to the window's size, headless, before it goes to its URL, and + all loading at once. A tab settles once it has loaded, once it has failed, which on one of the env's own sites, + as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a + site, else ``live_timeout_s``. Each tab's outcome, in order. A page already loaded doesn't read cookies set + after it, so a tab a load signed in shows it only once it loads again.""" + if not urls: + return [] + targets = list(self.tabs)[:len(urls)] or [await self._first_tab()] + while len(targets) < len(urls): + targets.append((await self.devtools.call("Target.createTarget", + {"url": "about:blank", "background": True}))["targetId"]) + for target in targets: + await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S) + loading = [] + for target, url in zip(targets, urls): + site = site_of(url) + loading.append(await self._start_load(target, url, site)) + outcomes = await asyncio.gather(*(self._settled(target, load, own_timeout_s if load.site is not None + else live_timeout_s) for target, load in zip(targets, loading))) + await self._bring_to_front(targets[0]) + return outcomes + + async def _first_tab(self) -> str: + """The tab a browser just launched opened, once it has one.""" + deadline = asyncio.get_running_loop().time() + PIN_TIMEOUT_S + while not self.tabs: + if asyncio.get_running_loop().time() > deadline: + raise DeviceUnavailable(f"the browser opened no tab within {PIN_TIMEOUT_S:g} s") + await asyncio.sleep(ARRIVAL_POLL_S) + return next(iter(self.tabs)) + + async def _start_load(self, target: str, url: str, site: Optional[str]) -> _StartLoad: + """The tab ``target`` sent to ``url``, its navigation followed by what the browser says of it. The browser + answers a navigation once it commits, which a page that never answers never does, so its answer is awaited + apart, for as long as the tab is.""" + session = next((s for s, t in self._sessions.items() if t == target), None) + if session is None: + raise DeviceUnavailable(f"the browser's tab {target} has no session to load a start tab in") + load = self._loading[session] = _StartLoad(session, url, site, asyncio.get_running_loop().time()) + try: + await self.devtools.call("Network.enable", session_id=session) + except (ConnectionError, DevToolsError, asyncio.TimeoutError) as e: + load.error = f"{type(e).__name__}: {e}" + load.check() + return load + load.navigating = asyncio.ensure_future(self.devtools.request("Page.navigate", {"url": url}, + session_id=session)) + load.navigating.add_done_callback(load.answered) + return load + + async def _settled(self, target: str, load: _StartLoad, timeout_s: float) -> dict: + """``load``'s outcome once it settles, or once it has been waited on ``timeout_s``.""" + try: + await asyncio.wait_for(load.settled.wait(), timeout_s) + except asyncio.TimeoutError: + pass + finally: + if load.navigating is not None: + load.navigating.cancel() + self._loading.pop(load.session, None) + with contextlib.suppress(ConnectionError, DevToolsError, asyncio.TimeoutError): + await self.devtools.call("Network.disable", session_id=load.session) + status, final_url = load.responses.get(load.loader or "", (None, None)) + error = load.error or load.failures.get(load.loader or "") + outcome = FAILED if error or (load.settled.is_set() and status is not None and status >= 400) else \ + LOADED if load.settled.is_set() else TIMED_OUT + if final_url is None and outcome == LOADED: + await self._refresh(target) + final_url = self.tabs.get(target, {}).get("url") + return StartTab(tab=target, url=load.url, final_url=final_url, status=status, error=error, + load_ms=round((asyncio.get_running_loop().time() - load.started) * 1000), site=load.site, + outcome=outcome).model_dump() async def replace_tabs(self, tabs: list) -> None: """``tabs``, each with its ``url`` and whether it's ``active``, opened in order in place of the tabs open now, @@ -161,14 +234,6 @@ async def replace_tabs(self, tabs: list) -> None: if opened: await self._bring_to_front(next((target for target, tab in zip(opened, tabs) if tab.active), opened[0])) - async def _navigate_tab(self, target: str, url: str) -> None: - session = next((s for s, t in self._sessions.items() if t == target), None) - if session is None: - raise DeviceUnavailable(f"the browser's tab {target} has no session to navigate it") - answer = await self.devtools.call("Page.navigate", {"url": url}, session_id=session) - if error := answer.get("errorText"): - raise InvalidArgument(f"the browser couldn't open {url}: {error}") - def forget(self) -> None: """A record that starts now, as a reset leaves the browser: no tabs closed, downloads or dialogs before it, and no origin seen.""" @@ -521,6 +586,25 @@ def _on_Target_targetInfoChanged(self, params: dict, session: Optional[str]) -> def _on_Page_frameNavigated(self, params: dict, session: Optional[str]) -> None: if session in self._sessions: self._saw(params.get("frame", {}).get("url", "")) + frame = params.get("frame", {}) + if (load := self._loading.get(session)) is not None and not frame.get("parentId"): + load.committed = frame.get("loaderId") + + def _on_Network_requestWillBeSent(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and params.get("type") == "Document": + load.requests[params["requestId"]] = params.get("loaderId", "") + + def _on_Network_responseReceived(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and params.get("type") == "Document" and \ + params.get("requestId") in load.requests: + response = params.get("response", {}) + load.responses[load.requests[params["requestId"]]] = (response.get("status"), response.get("url")) + load.check() + + def _on_Network_loadingFailed(self, params: dict, session: Optional[str]) -> None: + if (load := self._loading.get(session)) is not None and (loader := load.requests.get(params.get("requestId"))): + load.failures[loader] = params.get("errorText") or "the load failed" + load.check() def _on_Fetch_requestPaused(self, params: dict, session: Optional[str]) -> None: if session in self._serving: @@ -575,6 +659,9 @@ def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None: self._choosers.pop(session, None) if (target := self._sessions.get(session)) is not None: self._spawn(self._refresh(target)) + if (load := self._loading.get(session)) is not None and load.committed is not None: + load.loaded.add(load.committed) + load.check() def _on_Page_fileChooserOpened(self, params: dict, session: Optional[str]) -> None: if session in self._sessions: @@ -636,6 +723,48 @@ def _finished(self, task: asyncio.Future) -> None: logger.warning(f"the browser's control connection: {type(error).__name__}: {error}") +@dataclass +class _StartLoad: + """What the browser has said of a start tab's navigation, by loader, since it was sent to ``url``: its main frame's + last commit, the loads that fired after a commit, and its document's requests, final responses and failures. + It's ``settled`` once its own loader has failed or loaded, or, on one of the env's ``site``, answered an error.""" + session: str + url: str + site: Optional[str] + started: float + loader: Optional[str] = None + error: Optional[str] = None + committed: Optional[str] = None + loaded: set[str] = field(default_factory=set) + requests: dict[str, str] = field(default_factory=dict) + responses: dict[str, tuple[Optional[int], Optional[str]]] = field(default_factory=dict) + failures: dict[str, str] = field(default_factory=dict) + settled: asyncio.Event = field(default_factory=asyncio.Event) + navigating: Optional[asyncio.Future] = None + + def answered(self, navigating: asyncio.Future) -> None: + """The browser's answer to its navigation: its loader, or why it couldn't navigate.""" + if navigating.cancelled(): + return + if (error := navigating.exception()) is not None: + self.error = f"{type(error).__name__}: {error}" + else: + answer = navigating.result() + if "error" in answer: + self.error = str(answer["error"].get("message", answer["error"])) + else: + self.error = answer.get("result", {}).get("errorText") or None + self.loader = answer.get("result", {}).get("loaderId") + self.check() + + def check(self) -> None: + status = self.responses.get(self.loader or "", (None, None))[0] + if self.error or (self.loader is not None and ( + self.loader in self.failures or self.loader in self.loaded + or (self.site is not None and status is not None and status >= 400))): + self.settled.set() + + def _size_and_sha256(path: Path) -> tuple[int, str]: digest = hashlib.sha256() with path.open("rb") as file: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 51ad3109e..f7ba5c634 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -9,6 +9,7 @@ import tempfile from pathlib import Path from typing import Any, Awaitable, Callable, ClassVar, Literal, Optional +from urllib.parse import urlsplit from pydantic import BaseModel, Field, ValidationError, field_validator from starlette.requests import Request @@ -27,14 +28,16 @@ from ..transfers import WriteNamespaceGrant from ..types import METHOD_ADD, EnvironmentExtension, IntakeDeclaration, IntakeFormat, intake_extension from . import data -from .control import AGENT, DOWNLOADS, EXIT, STOP, UPLOADS, BrowserControl, _now +from .control import AGENT, DOWNLOADS, EXIT, LOADED, STOP, UPLOADS, BrowserControl, _now from .devtools import PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe -from .shapes import BrowserRecord, BrowserState, CookieToSet, OriginStorage, StateToLoad +from .shapes import BrowserRecord, BrowserState, CookieToSet, OriginStorage, SiteNames, StateToLoad logger = logging.getLogger(__name__) SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" BROWSER = "urn:agentenv:browser/v1" +#: Where its server keeps the env's sites ``sites/set`` hands it, which outlive a relaunch or the server's restart. +SITES = SOCKET.parent / "sites.json" #: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging #: port, since the browser is reached only over its pipe. OWNED_FLAGS = ("--user-data-dir", "--remote-debugging-", "--remote-allow-origins", "--proxy-", "--no-proxy-server", @@ -126,6 +129,8 @@ def __init__(self) -> None: #: Its CDP connection, over the pipe its server launches it with. self.devtools = DevToolsPipe() self._following: Optional[asyncio.Future] = None + #: The start tabs of each launch since this server started, newest last. + self.start_tab_launches: list[dict] = [] super().__init__() for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) @@ -193,8 +198,9 @@ def _on_desktop(self) -> bool: return not _alone() def _command_line_tabs(self) -> list[str]: - """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" - return self.settings.start_urls[:1] if _alone() else self.settings.start_urls + """The tab its command line opens, blank: its start tabs open over its pipe, so each is held to its window and + followed as it loads.""" + return ["about:blank"] async def _launch_fds(self) -> dict[int, int]: """Before each launch, an empty cache and the deploy's prompt answers in its profile; and a new debugging pipe: @@ -205,7 +211,7 @@ async def _launch_fds(self) -> dict[int, int]: to_browser_read, to_browser_write = os.pipe() from_browser_read, from_browser_write = os.pipe() await self.devtools.connect(to_browser_write, from_browser_read) - self._following = asyncio.ensure_future(self._follow(self.settings.start_urls[len(self._command_line_tabs()):])) + self._following = asyncio.ensure_future(self._follow(self.settings.start_urls, _now())) self._following.add_done_callback(lambda done: done.cancelled() or done.exception()) return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} @@ -215,11 +221,18 @@ def _pipe_closed(self) -> None: self.control.ended(EXIT if self._computer_app.exited() else STOP) self.relay.ended() - async def _follow(self, urls: list[str]) -> None: - """Its control attached, then each of ``urls`` in a tab behind the active one, in order.""" + async def _follow(self, urls: list[str], at: str) -> None: + """Its control attached, then its start tabs at ``urls`` loaded, the launch's outcomes kept.""" await self.control.attach() - for url in urls: - await self.devtools.call("Target.createTarget", {"url": url, "background": True}) + await self._load_start_tabs(urls, at) + + async def _load_start_tabs(self, urls: list[str], at: str) -> list[dict]: + """Its start tabs at ``urls``, as the control loads them, a tab on one of the env's sites waited on for as long + as the browser may take to be ready; their outcomes, kept as a launch's from ``at``.""" + sites = _read_sites() + tabs = await self.control.load_start_tabs(urls, lambda url: _site_of(url, sites), self._computer_app.timeout_s) + self.start_tab_launches.append({"at": at, "tabs": tabs}) + return tabs def _write_prompt_answers(self) -> None: """Its profile's preferences, which it reads as it starts, denying location and notification prompts or @@ -259,11 +272,17 @@ def write_policies(self) -> None: @ready_check async def devtools_answers(self) -> bool: - """It answers over its pipe, its control follows it and it has opened its start tabs.""" + """It answers over its pipe.""" try: await self.devtools.call("Browser.getVersion", timeout_s=self.devtools_timeout_s) except (ConnectionError, DevToolsError, asyncio.TimeoutError): return False + return True + + @ready_check + async def start_tabs_loaded(self) -> bool: + """Its control follows it and every start tab of its launch has settled: loaded, failed or waited on its + time.""" following = self._following return following is not None and following.done() and not following.cancelled() and \ following.exception() is None @@ -284,10 +303,26 @@ async def devtools_answers(self) -> bool: "downloads/get are its downloads, get with its file, uploaded under a write_namespace grant or inline. cdp " "returns a user's CDP address, http_url to discover from and ws_url to connect to, " "with a token agentenv mints in its path: each connection to it gets its own browser-level session behind a " - "guard, and record lists them as cdp_connections, each with the commands the guard refused or changed.")) + "guard, and record lists them as cdp_connections, each with the commands the guard refused or changed. " + "sites/set hands it the env's own sites, by name and domains: a start tab on one must load, its document's " + "status 2xx, or its launch counts it failed, while one on any other site is waited on 30 s at most. record " + "keeps each launch's start_tabs: the URL asked for and ended on, the status, any network error, the load time, " + "the env's site and the outcome, loaded, failed or timed_out.")) async def browser_record(self) -> dict: return BrowserRecord(**await self.control.record(), launches=self._computer_app.launches(), - cdp_connections=list(self.relay.connections)).model_dump() + cdp_connections=list(self.relay.connections), + start_tabs=self.start_tab_launches).model_dump() + + @extension(BROWSER, op="sites/set") + async def browser_sites_set(self, sites: list[dict]) -> dict: + """The env's own sites, each by its name and domains, which it keeps for each launch after: a start tab on one + is waited on strictly.""" + named = _parsed(SiteNames, sites, "sites") + SITES.parent.mkdir(parents=True, exist_ok=True) + written = SITES.with_name(f"{SITES.name}.part") + written.write_text(json.dumps([site.model_dump() for site in named]), encoding="utf-8") + written.replace(SITES) + return {} @extension(BROWSER, op="cdp") async def browser_cdp(self, user: str, request: Request) -> dict: @@ -468,7 +503,8 @@ async def browser_get_data(self, write_namespace: Optional[WriteNamespaceGrant] state = BrowserState(cookies=await self.control.cookies(), local_storage=await self.control.local_storage(), tabs=await self.control.tab_list(), closed_tabs=(record := await self.control.record())["closed_tabs"], - downloads=record["downloads"], clock={"mode": "real", "now": _now()}) + downloads=record["downloads"], clock={"mode": "real", "now": _now()}, + start_tabs=self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) written = {**state.model_dump(), "cookies": [cookie.model_dump(exclude_none=True) for cookie in state.cookies]} (home / ".cache").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=home / ".cache") as staging: @@ -503,6 +539,7 @@ async def browser_add_data(self, parts: list) -> None: await app_data.in_thread(data.restore_profile, Path.home(), self.profile, load.profile) await self._computer_app.launch_app(METHOD_ADD) await self._until_ready() + _refuse_failed(self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) applied.append("the profile") if load.files: await app_data.in_thread(app_data.write_within, Path.home(), load.files, [DOWNLOADS, UPLOADS]) @@ -517,7 +554,7 @@ async def browser_add_data(self, parts: list) -> None: await self.control.replace_tabs(state.tabs) applied.append("the tabs") elif (state.cookies or state.local_storage) and not agent_acted(): - await self.control.reopen_start_tabs(self.settings.start_urls) + _refuse_failed(await self._load_start_tabs(self.settings.start_urls, _now())) applied.append("the start tabs again") except Exception as e: done = ", ".join(applied) or "nothing" @@ -533,8 +570,15 @@ async def browser_reset_data(self) -> None: await app_data.reset(self.paths) await app_data.in_thread(data.empty, Path.home(), DOWNLOADS, UPLOADS) self.control.forget() + self.start_tab_launches = [] forget_agent_acted() + async def _relaunched(self, op: str) -> None: + """After a reset relaunches it: back once its start tabs have settled, or DeviceUnavailable after its ready + timeout; InvalidArgument naming a start tab on one of the env's sites that failed.""" + await self._until_ready() + _refuse_failed(self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) + def extension_params(self, uri: str) -> dict: if uri == BROWSER: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} @@ -548,6 +592,44 @@ def browser_skill(self) -> tuple[str, str]: return "How to use this website browser.", SKILL +def _read_sites() -> list[SiteNames]: + """The env's sites ``sites/set`` handed it, none before it did or for a file it can't read.""" + try: + return [SiteNames.model_validate(site) for site in json.loads(SITES.read_text(encoding="utf-8"))] + except (OSError, ValueError, TypeError, ValidationError): + return [] + + +def _site_of(url: str, sites: list[SiteNames]) -> Optional[str]: + """The name of the env's site among ``sites`` that serves ``url``'s host, None for any other.""" + host = (urlsplit(url).hostname or "").rstrip(".") + return next((site.name for site in sites if any(_matches(domain.lower(), host) for domain in site.domains)), None) + + +def _matches(pattern: str, host: str) -> bool: + """Whether ``host`` is the name ``pattern`` names: itself, or for ``*.example.com`` one label more.""" + if pattern.startswith("*."): + head, dot, rest = host.partition(".") + return bool(head) and bool(dot) and rest == pattern[2:] + return host == pattern + + +def _refuse_failed(tabs: list[dict]) -> None: + """InvalidArgument naming each start tab on one of the env's sites that didn't load.""" + if failed := [tab for tab in tabs if tab["site"] is not None and tab["outcome"] != LOADED]: + raise InvalidArgument("; ".join(f"start tab {tab['url']} on the env's site {tab['site']} {start_tab_why(tab)}" + for tab in failed)) + + +def start_tab_why(tab: dict) -> str: + """Why a start tab didn't load, as its outcome says.""" + if tab.get("error"): + return f"didn't load: {tab['error']}" + if tab.get("status") is not None and tab["status"] >= 400: + return f"answered {tab['status']}" + return f"was still loading after {tab['load_ms'] / 1000:g} s" + + def _alone() -> bool: """Whether it runs on a computer with no display, a browser computer's, alone and headless.""" return computer_app.detect().display == "none" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py index d4bfc8a1c..1eb842e4b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/shapes.py @@ -5,7 +5,7 @@ from typing import Literal, Optional from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator SameSite = Literal["Strict", "Lax", "None"] @@ -152,6 +152,35 @@ class StateToLoad(BaseModel): tabs: Optional[list[TabToOpen]] = None +class SiteNames(BaseModel): + """A site of the env, which ``sites/set`` hands the browser: its ``name`` and the ``domains`` it's served under, a + ``*.`` name matching one label more, as a certificate's wildcard does.""" + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + domains: list[str] = Field(min_length=1) + + +class StartTab(BaseModel): + """How a start tab loaded: the ``url`` asked for, the ``final_url`` it ended on, its document's final HTTP + ``status``, the network ``error`` that kept it from loading, how long it took, the env's ``site`` it's on, None for + a live one, and its ``outcome``: ``loaded``, ``failed`` (an error, or a 4xx or 5xx status) or ``timed_out``.""" + tab: str + url: str + final_url: Optional[str] = None + status: Optional[int] = None + error: Optional[str] = None + load_ms: int + site: Optional[str] = None + outcome: Literal["loaded", "failed", "timed_out"] + + +class StartTabsLaunch(BaseModel): + """The start tabs of one launch of the browser, ``at`` when it began, in ``start_urls`` order.""" + at: str + tabs: list[StartTab] + + class Clock(BaseModel): """The clock the browser's pages see: the computer's own, ``real``, until a browser follows a universe's.""" mode: Literal["real"] @@ -166,6 +195,8 @@ class BrowserState(BaseModel): closed_tabs: list[ClosedTab] downloads: list[Download] clock: Clock + #: The start tabs of its latest launch. + start_tabs: list[StartTab] = [] class BrowserRecord(BaseModel): @@ -178,6 +209,8 @@ class BrowserRecord(BaseModel): dialogs: list[Dialog] launches: list[LaunchEntry] cdp_connections: list[dict] + #: The start tabs of each launch since its server started. + start_tabs: list[StartTabsLaunch] = [] def origin_of(url: str, *, strict: bool = False) -> Optional[str]: diff --git a/packages/agentenv-protocol/tests/test_browser_data.py b/packages/agentenv-protocol/tests/test_browser_data.py index 12a412a9e..9f98d8c72 100644 --- a/packages/agentenv-protocol/tests/test_browser_data.py +++ b/packages/agentenv-protocol/tests/test_browser_data.py @@ -2,8 +2,9 @@ caches or locks, and its finished downloads; data/add applies such a state while the browser runs, the start tabs loaded again after cookies change, a load's tabs in place of the open ones, and a profile restored with the browser stopped, every part checked first; once an agent has acted, a load can't change the profile or the tabs, a failed one -isn't reset, and one that fails part way says what it applied; data/reset ends the addresses, puts the profile back as -installed, empties the downloads and uploads and starts the record over.""" +isn't reset, and one that fails part way says what it applied; a load whose start tabs on one of the env's sites +don't load fails naming them, as does a reset's relaunch, while a live one never does; data/reset ends the addresses, +puts the profile back as installed, empties the downloads and uploads and starts the record over.""" from __future__ import annotations import base64 @@ -34,7 +35,7 @@ class FakeControl: """The browser's control, answering from what a test sets and recording what it's asked to do.""" def __init__(self): - self.did, self.fail = [], None + self.did, self.fail, self.start_tabs_outcome = [], None, "loaded" self.cookie_list = [{"name": "sid", "value": "s1", "domain": "shop.test", "path": "/", "expires": -1, "httpOnly": True, "secure": False, "sameSite": "Lax"}] @@ -66,8 +67,10 @@ async def set_local_storage(self, origins): async def replace_tabs(self, tabs): await self._done("tabs", [(tab.url, tab.active) for tab in tabs]) - async def reopen_start_tabs(self, urls): + async def load_start_tabs(self, urls, site_of, own_timeout_s): await self._done("start_tabs", urls) + return [{"tab": f"t{index}", "url": url, "final_url": url, "status": 503, "error": None, "load_ms": 40, + "site": site_of(url), "outcome": self.start_tabs_outcome} for index, url in enumerate(urls)] def forget(self): self.did.append(("forget", None)) @@ -106,7 +109,7 @@ def paused(): yield launches.append("continue") - made._computer_app = SimpleNamespace(stop_app=stop_app, launch_app=launch_app, paused=paused) + made._computer_app = SimpleNamespace(stop_app=stop_app, launch_app=launch_app, paused=paused, timeout_s=60.0) made._until_ready = until_ready made.launches = launches return made @@ -140,7 +143,7 @@ async def test_data_get_is_its_state_its_profile_without_caches_or_locks_and_its files = _files(await browser.browser_get_data()) state = json.loads(files.pop("browser-state.json")) - assert set(state) == {"cookies", "local_storage", "tabs", "closed_tabs", "downloads", "clock"} + assert set(state) == {"cookies", "local_storage", "tabs", "closed_tabs", "downloads", "clock", "start_tabs"} assert state["cookies"] == FakeControl().cookie_list and "partitionKey" not in state["cookies"][0] assert state["clock"]["mode"] == "real" and state["tabs"][0]["id"] == "t1" and state["closed_tabs"][0]["id"] == "t0" assert files == {f"{PROFILE}/Default/Cookies": b"db", f"{PROFILE}/Default/Cookies-journal": b"j", @@ -277,6 +280,59 @@ async def test_a_load_that_fails_part_way_says_what_it_had_applied_and_one_befor assert browser._resets_failed_add() is True +@pytest.mark.asyncio +@pytest.mark.parametrize("sites, refused", [([{"name": "shop", "domains": ["shop.test"]}], True), ([], False)], + ids=["own-site", "live"]) +async def test_a_load_whose_start_tab_on_an_env_site_doesnt_load_fails_naming_it_and_a_live_one_doesnt( + browser, sites, refused): + browser.control.start_tabs_outcome = "failed" + await browser.browser_sites_set(sites) + load = browser.browser_add_data(_parts(_file("browser-state.json", _snapshot_state()))) + + if refused: + with pytest.raises(InvalidArgument, match=r"start tab https://shop.test/ on the env's site shop answered 503 " + r"\(the load had applied the cookies, the local storage\)"): + await load + else: + await load + assert browser.start_tab_launches[-1]["tabs"][0]["site"] == ("shop" if refused else None) + + +@pytest.mark.asyncio +async def test_after_a_reset_relaunches_it_a_start_tab_on_an_env_site_that_failed_fails_the_reset(browser): + def launched(url, site, outcome, **why): + return {"at": "t", "tabs": [{"tab": "t1", "url": url, "final_url": None, "status": None, "error": None, + "load_ms": 60000, "site": site, "outcome": outcome, **why}]} + + browser.start_tab_launches = [launched("https://slack.com/", None, "timed_out")] + await browser._relaunched("data/reset") + browser.start_tab_launches.append(launched("https://shop.test/", "shop", "timed_out")) + with pytest.raises(InvalidArgument, match="start tab https://shop.test/ on the env's site shop was still loading " + "after 60 s"): + await browser._relaunched("data/reset") + browser.start_tab_launches.append(launched("https://shop.test/", "shop", "failed", + error="net::ERR_CONNECTION_REFUSED")) + with pytest.raises(InvalidArgument, match="didn't load: net::ERR_CONNECTION_REFUSED"): + await browser._relaunched("data/reset") + + +@pytest.mark.asyncio +async def test_sites_set_keeps_the_env_sites_a_start_tab_matches_by_name_or_one_label_under_a_wildcard(browser): + environment = website_browser.environment + before = environment._read_sites() + + await browser.browser_sites_set([{"name": "shop", "domains": ["shop.test"]}, + {"name": "mail", "domains": ["*.mail.test"]}]) + sites = environment._read_sites() + with pytest.raises(InvalidArgument, match=r"sites\[0\]: domains"): + await browser.browser_sites_set([{"name": "shop", "domains": []}]) + + assert before == [] and [site.name for site in environment._read_sites()] == ["shop", "mail"] + assert [environment._site_of(url, sites) for url in ( + "https://shop.test/", "https://SHOP.test./cart", "https://www.mail.test/", "https://a.b.mail.test/", + "https://mail.test/", "https://slack.com/", "about:blank")] == ["shop", "shop", "mail", None, None, None, None] + + @pytest.mark.asyncio async def test_a_state_that_doesnt_fit_is_refused_naming_its_fields_and_never_a_value(browser): bad = _snapshot_state(cookies=[{"name": "sid", "value": "SECRET", "maxAge": 5}]) diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index 99e18ec0c..472ae81f3 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -239,6 +239,37 @@ async def test_an_export_sends_a_linked_files_contents_from_inside_the_home_and_ assert exported == {"Mail/latest.txt": b"archived"} +@linux(paths=MAIL, launch=Launch(command=["mail"])) +class CheckedMail(ComputerAppEnvironment): + """Mail that, once a data call relaunches it, checks it came back, failing while ``broken``.""" + relaunched: list = [] + broken = False + + async def _relaunched(self, op: str) -> None: + type(self).relaunched.append(op) + if type(self).broken: + raise RuntimeError("it came back broken") + + +@pytest.mark.asyncio +async def test_a_call_that_relaunched_the_app_awaits_its_relaunch_check_whose_failure_fails_the_call(home, desktop): + mail = CheckedMail() + mail._computer_app_data_plane().keep_initial() + await mail._computer_app.launch_app() + + async with _http(mail) as http: + await _rpc(http, "data/reset", {}) + await _rpc(http, "data/get", {}) + CheckedMail.broken = True + broken = await http.post(RPC_PATH, json=_call("data/reset", {})) + await mail._computer_app.stop_app() + + assert CheckedMail.relaunched == ["reset_data", "reset_data"] + assert "it came back broken" in broken.json()["error"]["message"] + assert [event for event, _ in desktop] == ["launch", "stop", "launch", "pause", "continue", "stop", "launch", + "stop"] + + @pytest.mark.asyncio async def test_an_app_that_isnt_running_is_left_stopped(home, desktop): mail = PathsMail() diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 810ec6ff7..3b6843e0a 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -23,34 +23,66 @@ from agentenv_protocol.computer.platform import Platform from agentenv_protocol.website_browser import (ALONE_BYPASS, BROWSER, COMPUTER_BYPASS, POLICIES, SITE_PROXY_ENV, SKILL, BrowserControl, DevToolsError, WebsiteBrowserEnvironment) +from agentenv_protocol.website_browser.control import STOP PROXY = "http://127.0.0.1:3128" OWNED = ["--user-data-dir", "--disk-cache-dir", "--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US"] # A browser that answers CDP on its debugging pipe, fds 3 and 4, as a Chromium does: Browser.getVersion with its -# product, Target.createTarget by writing the tab's URL to tabs.txt beside it, a new window's after "window ", anything -# else with an error. +# product; its first tab, "start", made known and attached to; Target.createTarget by writing the tab's URL, or a new +# window's after "window ", to tabs.txt beside it and attaching to the tab; Page.navigate by writing "navigate +# " there, then committing and loading it; the calls agentenv makes to follow a page with nothing; anything else +# with an error. FAKE_BROWSER = """import json, os -buffer = b"" +buffer, loaders = b"", 0 +def send(method, params, session=None): + os.write(4, json.dumps({"method": method, "params": params, **({"sessionId": session} if session else {})}).encode() + + b"\\0") +def page(target, url="about:blank"): + return {"targetId": target, "type": "page", "url": url, "title": ""} while chunk := os.read(3, 65536): buffer += chunk while b"\\0" in buffer: raw, buffer = buffer.split(b"\\0", 1) message = json.loads(raw) - if message["method"] == "Browser.getVersion": + method, params, session = message["method"], message.get("params") or {}, message.get("sessionId") + events = [] + if method == "Browser.getVersion": answer = {"result": {"product": "Fake/1"}} - elif message["method"] in ("Target.setDiscoverTargets", "Target.setAutoAttach", "Browser.setDownloadBehavior"): + elif method == "Target.setDiscoverTargets": answer = {"result": {}} - if message["method"] == "Target.setDiscoverTargets": - created = {"targetInfo": {"targetId": "start", "type": "page", "url": "about:blank", "title": ""}} - os.write(4, json.dumps({"method": "Target.targetCreated", "params": created}).encode() + b"\\0") - elif message["method"] == "Target.createTarget": + events = [("Target.targetCreated", {"targetInfo": page("start")}, None)] + elif method == "Target.setAutoAttach": + answer = {"result": {}} + events = [("Target.attachedToTarget", {"sessionId": "s-start", "targetInfo": page("start"), + "waitingForDebugger": True}, None)] + elif method == "Target.createTarget": + with open("tabs.txt", "a") as tabs: + tabs.write(("window " if params.get("newWindow") else "") + params["url"] + "\\n") + target = f"t{len(open('tabs.txt').read().splitlines())}" + answer = {"result": {"targetId": target}} + events = [("Target.targetCreated", {"targetInfo": page(target)}, None), + ("Target.attachedToTarget", {"sessionId": "s-" + target, "targetInfo": page(target), + "waitingForDebugger": True}, None)] + elif method == "Page.navigate": + target, loaders = session[2:], loaders + 1 with open("tabs.txt", "a") as tabs: - tabs.write(("window " if message["params"].get("newWindow") else "") + message["params"]["url"] + "\\n") - answer = {"result": {"targetId": message["params"]["url"]}} + tabs.write(f"navigate {target} {params['url']}\\n") + answer = {"result": {"frameId": target, "loaderId": f"L{loaders}"}} + events = [("Page.frameNavigated", {"frame": {"id": target, "loaderId": f"L{loaders}", + "url": params["url"]}}, session), + ("Page.loadEventFired", {}, session)] + elif method == "Target.getTargets": + answer = {"result": {"targetInfos": []}} + elif method in ("Browser.setDownloadBehavior", "Page.enable", "Emulation.setDeviceMetricsOverride", + "Page.setInterceptFileChooserDialog", "Runtime.runIfWaitingForDebugger", "Network.enable", + "Network.disable", "Target.activateTarget", "Page.bringToFront"): + answer = {"result": {}} else: answer = {"error": {"code": -32601, "message": "unknown method"}} os.write(4, json.dumps({"id": message["id"], **answer}).encode() + b"\\0") + for event in events: + send(*event) """ @@ -195,13 +227,15 @@ async def test_each_launch_starts_with_an_empty_cache_and_the_deploys_prompt_ans assert not (browser_home / ".cache/chromium").exists() -def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_url_is_a_flag(monkeypatch): +def test_its_settings_give_its_language_and_time_zone_its_start_tabs_open_over_its_pipe_and_none_is_a_flag( + monkeypatch): _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app"], locale="fr-FR", time_zone="Europe/Paris") launch = Browser().launch _settings(monkeypatch, start_urls=["--remote-allow-origins=*"]) - assert launch.command[-2:] == ["https://slack.com", "https://linear.app"] and "--lang=fr-FR" in launch.command + assert launch.command[-1] == "about:blank" and "--lang=fr-FR" in launch.command + assert not any("slack.com" in arg or "linear.app" in arg for arg in launch.command) assert launch.env == {"TZ": "Europe/Paris"} with pytest.raises(ValueError, match="start_urls are URLs, not flags"): Browser() @@ -271,7 +305,7 @@ async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_rea await runtime.launch_app() version = await browser.devtools.call("Browser.getVersion") with pytest.raises(DevToolsError, match="unknown method"): - await browser.devtools.call("Page.navigate", {"url": "about:blank"}) + await browser.devtools.call("Page.printToPDF") launched = await _answering(browser) await runtime.stop_app() stopped = await browser.devtools_answers() @@ -282,7 +316,7 @@ async def test_each_launch_gets_a_new_pipe_its_server_talks_cdp_over_and_its_rea assert unlaunched is False and version == {"product": "Fake/1"} and launched is True assert stopped is False and relaunched is True - assert "devtools_answers" in runtime.ready_checks + assert {"devtools_answers", "start_tabs_loaded"} <= set(runtime.ready_checks) @pytest.mark.asyncio @@ -306,7 +340,8 @@ async def test_a_launch_while_it_runs_opens_a_window_over_its_pipe_and_starts_no @pytest.mark.asyncio -async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): +async def test_headless_its_start_tabs_open_blank_held_to_its_window_then_load_in_order_and_count_it_ready( + monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") _alone(monkeypatch) @@ -316,14 +351,20 @@ async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_res await browser._computer_app.launch_app() for _ in range(50): - if ready := await browser.devtools_answers(): + if ready := await browser.start_tabs_loaded(): break await asyncio.sleep(0.1) await browser._computer_app.stop_app() browser.devtools.close() - assert browser.launch.command[-2:] == ["--headless=new", "https://slack.com"] - assert ready is True and (tmp_path / "tabs.txt").read_text() == "https://linear.app\nhttps://github.com\n" + assert browser.launch.command[-2:] == ["--headless=new", "about:blank"] + assert ready is True and (tmp_path / "tabs.txt").read_text().splitlines() == [ + "about:blank", "about:blank", "navigate start https://slack.com", "navigate t1 https://linear.app", + "navigate t2 https://github.com"] + tabs = browser.start_tab_launches[-1]["tabs"] + assert [(tab["tab"], tab["url"], tab["outcome"], tab["site"]) for tab in tabs] == [ + ("start", "https://slack.com", "loaded", None), ("t1", "https://linear.app", "loaded", None), + ("t2", "https://github.com", "loaded", None)] @pytest.mark.asyncio @@ -452,6 +493,12 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None): answer = self.answers.get(method, {}) return answer(params, session_id) if callable(answer) else answer + async def request(self, method, params=None, *, session_id=None): + try: + return {"result": await self.call(method, params, session_id=session_id)} + except DevToolsError as e: + return {"error": {"message": str(e)}} + def send(self, method, params, session=None): self.listener({"method": method, "params": params, **({"sessionId": session} if session else {})}) @@ -535,6 +582,131 @@ def refused(params, session): "Page.enable", "Runtime.runIfWaitingForDebugger"] +class Navigating(StubDevTools): + """A browser whose first tab is ``start`` and whose new tabs are t1, t2, …, each attached as it opens, answering a + navigation as ``pages`` says by URL: a status and the URL it ends on, then committed and, unless ``no-load`` + follows, loaded; ``refused``, a network error; ``silent``, never answered, as a page that never answers isn't; or + ``stale``, a load event with no commit.""" + + def __init__(self, pages): + super().__init__({"Target.getTargets": {"targetInfos": []}}) + self.pages, self.opened, self.loaders = pages, 0, 0 + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Target.createTarget": + self.calls.append((method, params, session_id)) + self.opened += 1 + target = f"t{self.opened}" + self.send("Target.targetCreated", _page(target)) + self.send("Target.attachedToTarget", {"sessionId": f"s-{target}", "waitingForDebugger": True, + **_page(target)}) + return {"targetId": target} + if method != "Page.navigate": + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + self.calls.append((method, params, session_id)) + page, target = self.pages[params["url"]], session_id[2:] + self.loaders += 1 + loader, request = f"L{self.loaders}", f"r{self.loaders}" + if page == "refused": + return {"frameId": target, "loaderId": loader, "errorText": "net::ERR_CONNECTION_REFUSED"} + if page == "silent": + await asyncio.Event().wait() + if page == "stale": + self.send("Page.loadEventFired", {}, session_id) + return {"frameId": target, "loaderId": loader} + status, final_url, *rest = page + document = {"requestId": request, "loaderId": loader, "type": "Document", "frameId": target} + self.send("Network.requestWillBeSent", document, session_id) + self.send("Network.responseReceived", {**document, "response": {"status": status, "url": final_url}}, + session_id) + self.send("Page.frameNavigated", {"frame": {"id": target, "loaderId": loader, "url": final_url}}, session_id) + if "no-load" not in rest: + self.send("Page.loadEventFired", {}, session_id) + return {"frameId": target, "loaderId": loader} + + +async def _started(devtools, viewport=(1280, 800)) -> BrowserControl: + """A control following a browser just launched, on its first tab, ``start``.""" + control = BrowserControl(devtools, "agent", viewport) + await control.attach() + devtools.send("Target.targetCreated", _page("start")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-start", "waitingForDebugger": True, **_page("start")}) + return control + + +def _shop(url): + return "shop" if "shop.test" in url else None + + +def _outcomes(tabs): + return [(tab["tab"], tab["url"], tab["final_url"], tab["status"], tab["error"], tab["site"], tab["outcome"]) + for tab in tabs] + + +@pytest.mark.asyncio +async def test_its_start_tabs_load_in_order_in_its_first_tab_then_blank_ones_held_first_and_the_first_is_active(): + devtools = Navigating({"https://shop.test/": (200, "https://shop.test/login"), "https://slack.com/": "silent", + "https://github.com/": (404, "https://github.com/")}) + control = await _started(devtools) + loop = asyncio.get_running_loop() + started = loop.time() + + tabs = await control.load_start_tabs(list(devtools.pages), _shop, own_timeout_s=5, live_timeout_s=0.3) + + assert _outcomes(tabs) == [ + ("start", "https://shop.test/", "https://shop.test/login", 200, None, "shop", "loaded"), + ("t1", "https://slack.com/", None, None, None, None, "timed_out"), + ("t2", "https://github.com/", "https://github.com/", 404, None, None, "failed")] + assert loop.time() - started < 2 + assert list(control.tabs) == ["start", "t1", "t2"] and all(isinstance(tab["load_ms"], int) for tab in tabs) + assert _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": True}, None)] * 2 + for session in ("s-start", "s-t1", "s-t2"): + sent = [method for method, _, on in devtools.calls if on == session] + assert sent.index("Emulation.setDeviceMetricsOverride") < sent.index("Network.enable") < \ + sent.index("Page.navigate") < sent.index("Network.disable"), sent + assert devtools.calls[-2][:2] == ("Target.activateTarget", {"targetId": "start"}) + assert not control._loading + + +@pytest.mark.asyncio +@pytest.mark.parametrize("url, page, outcome, error, status, waited", [ + ("https://shop.test/", (503, "https://shop.test/", "no-load"), "failed", None, 503, False), + ("https://shop.test/", "refused", "failed", "net::ERR_CONNECTION_REFUSED", None, False), + ("https://shop.test/", "silent", "timed_out", None, None, True), + ("https://shop.test/", "stale", "timed_out", None, None, True), + ("https://slack.com/", "silent", "timed_out", None, None, True), + ("https://slack.com/", (503, "https://slack.com/", "no-load"), "timed_out", None, 503, True), + ("https://slack.com/", "refused", "failed", "net::ERR_CONNECTION_REFUSED", None, False), +], ids=["own-error-status-at-once", "own-refused", "own-never-loads", "own-stale-load", "live-never-loads", + "live-error-status-waited", "live-refused"]) +async def test_a_start_tab_settles_strictly_on_an_env_site_and_at_most_its_time_on_a_live_one( + url, page, outcome, error, status, waited): + control = await _started(Navigating({url: page})) + loop = asyncio.get_running_loop() + started = loop.time() + + (tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=0.3 if url.startswith("https://shop") else 5, + live_timeout_s=0.3 if url.startswith("https://slack") else 5) + + assert (tab["outcome"], tab["error"], tab["status"]) == (outcome, error, status) + assert (loop.time() - started >= 0.3) is waited + + +@pytest.mark.asyncio +async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that(): + control = await _started(Navigating({"https://shop.test/": "silent"})) + loading = asyncio.ensure_future(control.load_start_tabs(["https://shop.test/"], _shop, own_timeout_s=5)) + for _ in range(100): + if control._loading: + break + await asyncio.sleep(0.01) + control.ended(STOP) + + (tab,) = await loading + + assert (tab["outcome"], tab["error"]) == ("failed", "the browser ended") + + class Painting(StubDevTools): """A browser whose every tab has focus and paints, but those whose sessions are in ``blank``, which never answer a capture; it records each capture's timeout.""" diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 8631cb011..8da52800a 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -9,7 +9,8 @@ build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists i (`agentenv_protocol.website_browser.WebsiteBrowserEnvironment`): agentenv launches it on a debugging pipe with its own flags, and writes managed policies to `/etc/opt/chrome_for_testing/policies/managed/` for unattended use. It runs without Chromium's sandbox, which a container can't give it. Its env.toml spells out its settings and their defaults -under `[settings]`: `start_urls`, opened at boot, the first tab active; `window`, alone only; `locale`; `time_zone`; and +under `[settings]`: `start_urls`, opened at boot in order, the first tab active, each waited on to load; `window`, +alone only; `locale`; `time_zone`; and `geolocation` and `notifications`, `deny` or `ask`; and `dialogs`, `agent`, `accept` or `dismiss`. A computer env sets its own beside it (`apps = [{ env = "chromium", settings = { start_urls = ["https://example.test/"] } }]`), and a deploy overrides any of them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 435d152b9..92933e3be 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -285,7 +285,7 @@ async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, async def run_env_params(self, env_params: Optional[dict], deployed_envs: list[DeployedEnv]) -> Optional[dict]: """With ``site_routing`` and no ``sites``, ``env_params`` naming as its sites the websites the run deployed - before the computer: all of them, or those of the env ids ``site_routing`` lists.""" + before the computer, all of them or those of the env ids ``site_routing`` lists, once each says it's ready.""" routing = (env_params or {}).get("site_routing") if routing is not None and routing is not False and routing is not True and not ( isinstance(routing, list) and all(isinstance(env_id, str) for env_id in routing)): @@ -293,10 +293,12 @@ async def run_env_params(self, env_params: Optional[dict], deployed_envs: list[D f"{routing!r}") if not routing or env_params.get("sites"): return env_params - found = await sites.deployed_sites(deployed_envs, None if routing is True else list(routing)) + only = None if routing is True else list(routing) + found = await sites.deployed_sites(deployed_envs, only) if not found: raise ValueError(f"computer env {self.id!r}: env_params.site_routing reaches the run's websites, and the " "run deployed none before the computer") + await sites.require_ready(deployed_envs, only) return {**env_params, "sites": found} def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optional[dict] = None, diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 97c75ff55..b2bb6519a 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -20,6 +20,7 @@ import logging import re import shlex +import time from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from pathlib import Path, PurePosixPath @@ -32,7 +33,7 @@ from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID -from agent_env.env.env import Env, gateway_url_of +from agent_env.env.env import Env, EnvCapabilityUnsupported, gateway_url_of from agent_env.env.envs.computer.utils.site_proxy import is_local, matches from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox from agent_env.store.base import NotFoundError @@ -53,6 +54,9 @@ CLOCK_SKEW = timedelta(minutes=5) CARD_ATTEMPTS = 3 CARD_RETRY_S = 1.0 +#: How long a computer's deploy waits for the run's websites it routes to say they're ready, as each said at its own +#: deploy, and how often it asks meanwhile. +SITES_READY_TIMEOUT_S, SITES_READY_POLL_S = 60.0, 2.0 _LABEL = re.compile(r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$") _PROXY_SOURCE = Path(__file__).with_name("site_proxy.py") logger = logging.getLogger(__name__) @@ -460,6 +464,35 @@ async def deployed_sites(deployed_envs: list[DeployedEnv], only: Optional[list[s return _env_qualified(found) +async def require_ready(deployed_envs: list[DeployedEnv], only: Optional[list[str]] = None, *, + timeout_s: float = SITES_READY_TIMEOUT_S, poll_s: float = SITES_READY_POLL_S) -> None: + """Back once each deployed env whose websites a computer routes, those of the env ids ``only`` names or all, says + through ``website/v1``'s ``ready`` that every site of it is ready; RuntimeError naming each one still not, and why, + once ``timeout_s`` has passed. An env with no address to ask isn't waited on, nor one with no site serving + ``ready``, which is ready.""" + waiting = [record for record in deployed_envs if only is None or record.env_id in only] + deadline = time.monotonic() + timeout_s + while True: + why: dict[str, str] = {} + for record in waiting: + try: + readiness = await record.readiness() + except EnvCapabilityUnsupported: + continue + except (httpx.HTTPError, ValueError) as e: + why[record.env_id] = f"{type(e).__name__}: {e}" + continue + if not readiness.ready: + why[record.env_id] = "; ".join(readiness.problems) or "it says it isn't ready" + if not why: + return + if time.monotonic() >= deadline: + raise RuntimeError(f"site routing: the run's websites weren't ready after {timeout_s:g} s: " + + "; ".join(f"{env_id} ({reason})" for env_id, reason in why.items())) + waiting = [record for record in waiting if record.env_id in why] + await asyncio.sleep(poll_s) + + def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]: """Each site found, a name that sites of different envs share made ``-`` for each of them, letters, digits, ``-`` and ``_`` kept, so every site of the run has a name of its own.""" diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 14897bc5f..5f5c85498 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -13,6 +13,7 @@ from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING from agentenv_protocol.computer.app import COMPUTER_APP from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV +from agentenv_protocol.website_browser import BROWSER, LOADED, start_tab_why from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.attribution import Attribution @@ -191,13 +192,16 @@ async def _finish_boot(env: ComputerEnv, record: DeployedSandboxEnv, sandbox: Sa params: computer_params.ComputerParams, valid_for_s: int, resolution: Optional[str]) -> Optional[dict]: """What a deploy and a reset do once the computer's driver answers, in order: with ``site_routing``, the sites - routed and the held boot apps launched, the computer then ready; then its display checked. What the record keeps - of the routing, or None without it.""" + routed, each browser handed their names and the held boot apps launched, the computer then ready; then its display + checked; then each browser's start tabs read into the record, the boot failing on one on a site of the env's that + didn't load. What the record keeps of the routing, or None without it.""" routed = None if params.site_routing: routed = await _route_sites(env, sandbox, params, valid_for_s) - await _launch_held_apps(env, record, record.environment_url) + await _launch_held_apps(env, record, record.environment_url, params.sites) await _require_display(record, resolution) + if start_tabs := await _start_tabs(record): + record.metadata["computer"]["start_tabs"] = start_tabs return routed @@ -210,12 +214,14 @@ async def _route_sites(env: ComputerEnv, sandbox: Sandbox, params: computer_para return {**routed, "valid_for_s": valid_for_s} -async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: str) -> None: +async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: str, + routed: list[computer_params.Site]) -> None: """Launch each app that starts with the computer, whose launch a deploy routing sites holds until the sites' names, - CA and proxy are in place, through its ``computer-app/v1`` ``launch`` once the computer lists it with its server, - then wait until the computer is ready, which its boot only waited to answer; the record keeps the card the computer - answers then, as its boot's card may list an app by its name alone. Each is launched as the child its card lists, - as a computer named for its app, a browser alone, has that name too.""" + CA and proxy are in place, through its ``computer-app/v1`` ``launch`` once the computer lists it with its server, a + website browser first handed the ``routed`` sites, then wait until the computer is ready, which its boot only + waited to answer; the record keeps the card the computer answers then, as its boot's card may list an app by its + name alone. Each is launched as the child its card lists, as a computer named for its app, a browser alone, has + that name too.""" os_type = env.computer.os.type timeout = ready_timeout(env.composed_apps(), os_type) held = [app for app in env.composed_apps() if (launch := app.launch(os_type)) is not None and launch.at == "boot"] @@ -225,11 +231,49 @@ async def _launch_held_apps(env: ComputerEnv, record: DeployedSandboxEnv, url: s launchers = {child.name: child for child in protocol_v1.children(record.environment_card) if protocol_v1.find_extension_method(child.card, COMPUTER_APP, "launch") is not None} for app in held: + await _hand_sites(record, launchers[app.environment_name], routed) await record.invoke(COMPUTER_APP, "launch", child=launchers[app.environment_name]) record.environment_card = await ready(url, timeout, record.control_token) record.environment_card_read_at_utc = datetime.now(timezone.utc).isoformat() +async def _hand_sites(record: DeployedSandboxEnv, child: protocol_v1.ChildEnv, + routed: list[computer_params.Site]) -> None: + """The ``routed`` sites, by name and domains, handed to the website browser ``child`` is, through ``browser/v1``'s + ``sites/set``, so its start tabs on them must load; a browser built before it took them is told nothing, its start + tabs on them waited on as any live one's.""" + if child.find_extension(BROWSER) is None: + return + try: + await record.invoke(BROWSER, "sites/set", {"sites": [{"name": site.name, "domains": site.domains} + for site in routed if site.domains]}, child=child) + except EnvCapabilityUnsupported: + logger.warning(f"the website browser {child.name} in {record.env_id} serves no browser/v1 sites/set, so its " + "start tabs on the env's sites are waited on as live ones") + + +async def _start_tabs(record: DeployedSandboxEnv) -> dict[str, list[dict]]: + """Each website browser's start tabs as its latest launch loaded them, by its name: the computer's own card's, for + a browser alone, or a child's; a browser that keeps none, built before it did, left out. RuntimeError naming each + start tab on one of the env's sites that didn't load.""" + card = record.environment_card or {} + browsers = ([(card.get("name") or record.env_id, None)] if protocol_v1.find_extension(card, BROWSER) else []) \ + + [(child.name, child) for child in protocol_v1.children(card) if child.find_extension(BROWSER) is not None] + kept: dict[str, list[dict]] = {} + for name, child in browsers: + try: + launches = (await record.invoke(BROWSER, "record", child=child)).get("start_tabs") or [] + except EnvCapabilityUnsupported: + continue + if launches: + kept[name] = launches[-1]["tabs"] + if failed := [f"{name}'s start tab {tab['url']} on the env's site {tab['site']} {start_tab_why(tab)}" + for name, tabs in kept.items() for tab in tabs + if tab["site"] is not None and tab["outcome"] != LOADED]: + raise RuntimeError(f"computer env {record.env_id!r}: " + "; ".join(failed)) + return kept + + async def end_recordings(record: DeployedSandboxEnv, reason: str) -> None: """Have the computer's driver end every recording not yet ended, each uploading what's left and its manifest under the grant it was started with, before the computer goes; a driver that doesn't record has none.""" diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index da4144c84..9934ae261 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -13,8 +13,9 @@ driver's, the agent's address keeps working, and two agents connected at once each drive the browser through their own. browser/v1 reads and sets its cookies, local storage and downloads, and its data plane signs it in, snapshots it, resets it, loads a snapshot back and holds up under chaos; the browser task steps set its cookies and storage and open, activate and -close its tabs by URL; and a task whose deploy_env pins a version of the browser, its latest one with none, passes -preflight on the version pinned and runs its browser steps through task.run. Needs Docker and the local registry on +close its tabs by URL; a task whose deploy_env pins a version of the browser, its latest one with none, passes +preflight on the version pinned and runs its browser steps through task.run; and each start tab's page sees the +window's size from its first script. Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio @@ -54,6 +55,7 @@ from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data, check_browser_data_under_chaos from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state, check_browser_state_under_chaos +from tst.integration.computer.start_tabs import tabs_of from tst.util.a2a_test_agent import put_test_agent pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -126,6 +128,8 @@ def log_message(self, *args): DATA_PAGE = "data:text/html,Home

Home

" #: A page a task's tab steps open, bring to the front and close by its URL. PINNED_PAGE = "data:text/html,pinned" +#: A page whose title is the window's size as its first script sees it, told apart by ``{}``. +SIZED = "data:text/html,{}" # Every TCP port something in the computer listens on, and whether any process is an X server. LISTENING = """import glob ports = set() @@ -412,6 +416,27 @@ async def test_a_task_pinning_a_version_of_the_browser_passes_preflight_on_it_an assert records["sign-out"]["deleted"] == 1 and not report.failed, (records, report) +@pytest.mark.asyncio +async def test_each_start_tabs_page_sees_the_windows_size_from_its_first_script_in_order_the_first_active( + browser_computer): + urls = [SIZED.format(index) for index in range(3)] + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": urls}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + start_tabs = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert [tab["outcome"] for tab in start_tabs] == ["loaded"] * 3 + assert [tab["id"] for tab in tabs] == [tab["tab"] for tab in start_tabs] + assert [(tab["title"], tab["active"]) for tab in tabs] == [("1280x800", True), ("1280x800", False), + ("1280x800", False)] + assert not report.failed + + @pytest.mark.asyncio async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( browser_computer): diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index fb964624f..78669d7d9 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -10,8 +10,9 @@ accepted for it, and each launch, exit and stop with its cause, across a kill and a restart of its server; and browser/v1 reads and sets its cookies, local storage and downloads; and its data plane signs it in, snapshots it, resets it and loads a snapshot back; and the browser task steps set its cookies and storage and open, activate and close its tabs by URL. -Needs Docker -and the local registry on localhost:5000.""" +Its start tabs open in order, the first active, a deploy recording each one's outcome and going on past a live page +that never answers; and with the run's sites routed to it, a start tab on one must load, so a site answering 503 +fails the deploy, naming it. Needs Docker and the local registry on localhost:5000.""" import asyncio import hashlib @@ -32,6 +33,7 @@ from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state +from tst.integration.computer.start_tabs import host_pages, tabs_of from tst.integration.computer.website_browser_test import _app, _browser_process, _keys, _ready from agent_env.env.envs.computer.utils import inside from agent_env.store.routing import namespace_routing @@ -185,7 +187,7 @@ async def test_it_runs_chrome_itself_on_its_pipe_answers_prompts_as_the_deploy_s assert argv[0].endswith("chromium/chrome/chrome") and argv[1:3] == ["--no-sandbox", "--class=Chromium"] assert argv[3].endswith("/.config/chromium") and argv[4].endswith("/.cache/chromium") assert argv[5:10] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=en-US", - "--disable-dev-shm-usage"] and argv[10:] == [LOOPBACK] + "--disable-dev-shm-usage"] and argv[10:] == ["about:blank"] assert browser["listening"] == 0 assert permissions == ["denied prompt - Google Chrome for Testing"] assert launched["exit_code"] == 0 and len(windows) == 2 and after["pid"] == browser["pid"] @@ -352,6 +354,55 @@ async def act(): assert not report.failed +@pytest.mark.asyncio +async def test_its_start_tabs_open_in_order_the_first_active_and_a_live_one_that_never_answers_is_recorded_not_fatal( + chromium_desktop): + with host_pages() as port, namespace_routing(): + urls = [f"http://host.docker.internal:{port}/{page}" for page in ("alpha", "bravo", "hang")] + deployed = await Env.get(chromium_desktop.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": urls}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + start_tabs = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert [(tab["url"], tab["status"], tab["site"], tab["outcome"]) for tab in start_tabs] == [ + (urls[0], 200, None, "loaded"), (urls[1], 200, None, "loaded"), (urls[2], None, None, "timed_out")] + assert start_tabs[2]["load_ms"] >= 30_000 and [tab["id"] for tab in tabs] == [tab["tab"] for tab in start_tabs] + assert [(tab["url"], tab["title"], tab["active"]) for tab in tabs[:2]] == [(urls[0], "alpha", True), + (urls[1], "bravo", False)] + assert not tabs[2]["active"] and not report.failed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [200, 503], ids=["loads", "answers-503"]) +async def test_a_start_tab_on_a_routed_site_must_load_so_a_site_answering_an_error_fails_the_deploy( + chromium_desktop, status): + with host_pages(site_status=status) as port, namespace_routing(): + params = {"site_routing": True, "sites": [{"name": "shop", "url": f"http://127.0.0.1:{port}/website/shop/", + "domains": ["shop.test"]}], + "app_settings": {"chromium": {"start_urls": ["https://shop.test/"]}}} + if status != 200: + with pytest.raises(RuntimeError, match=r"chromium's start tab https://shop.test/ on the env's site shop " + r"answered 503"): + await Env.get(chromium_desktop.id).deploy(env_params=params) + return + deployed = await Env.get(chromium_desktop.id).deploy(env_params=params) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + finally: + report = await teardown_run(context) + + (tab,) = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert (tab["url"], tab["final_url"], tab["status"], tab["site"], tab["outcome"]) == ( + "https://shop.test/", "https://shop.test/", 200, "shop", "loaded") + assert [(open_tab["url"], open_tab["title"]) for open_tab in tabs] == [("https://shop.test/", "shop")] + assert not report.failed + + @pytest.mark.asyncio async def test_its_task_steps_set_cookies_and_storage_open_a_tab_behind_and_activate_and_close_it_by_its_url( chromium_desktop): diff --git a/tst/integration/computer/start_tabs.py b/tst/integration/computer/start_tabs.py new file mode 100644 index 000000000..f2eb4fa73 --- /dev/null +++ b/tst/integration/computer/start_tabs.py @@ -0,0 +1,59 @@ +"""Pages a computer's start tabs open, served on this machine, which a computer reaches as ``host.docker.internal``: a +site's route, answering its status, two pages titled by name, and one that never answers.""" + +import contextlib +import http.server +import threading +import time +from collections.abc import Iterator + +import httpx + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER + +#: How long the page that never answers holds its request. +HANG_S = 120 + + +@contextlib.contextmanager +def host_pages(site_status: int = 200) -> Iterator[int]: + """The pages served on every address of this machine while the block runs, at the port it yields: ``/website/ + shop/``, a site's route, answering ``site_status``; ``/alpha`` and ``/bravo``; and ``/hang``, which never + answers.""" + + class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/hang": + time.sleep(HANG_S) + return + status = site_status if self.path.startswith("/website/shop/") else 200 + body = f"{self.path.strip('/').rsplit('/', 1)[-1] or 'shop'}".encode() + self.send_response(status) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("0.0.0.0", 0), Page) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield server.server_address[1] + finally: + server.shutdown() + server.server_close() + + +async def tabs_of(deployed, app: str = "chromium") -> list[dict]: + """The open tabs of the browser ``app`` in ``deployed``, as its ``browser/v1`` ``tabs/list`` gives them.""" + async with httpx.AsyncClient(timeout=30, headers=deployed.control_headers) as http: + card = (await http.get(f"{deployed.environment_url}/svc/{app}{WELL_KNOWN_PATH}")).json() + endpoint = protocol_v1.find_extension_method(card, BROWSER, "tabs/list")["endpoint"] + answer = await http.get(f"{deployed.environment_url}/svc/{app}{endpoint}") + answer.raise_for_status() + return answer.json()["tabs"] diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index e20cff8ca..edd6b68ee 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -130,7 +130,7 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun assert argv[4].startswith("--disk-cache-dir=") and argv[4].endswith("/.cache/chromium") assert argv[5:10] == ["--remote-debugging-pipe", "--no-first-run", "--no-default-browser-check", "--lang=fr-FR", "--disable-dev-shm-usage"] - assert argv[10:] == [TIME_ZONE_PAGE, LOOPBACK] + assert argv[10:] == ["about:blank"] assert browser["listening"] == 0 and port_file["exit_code"] != 0 assert zone.startswith("Europe/Paris") and loopback.startswith("Loopback page") assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( @@ -140,8 +140,8 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun "156.0.8078.4", "real", ["cdp", "cookies/delete", "cookies/get", "cookies/set", "dialog/answer", "downloads/get", "downloads/list", "files/choose", "history", "local_storage/clear", "local_storage/get", "local_storage/set", "navigate", "page/capture", "page/input", - "page/state", "page/viewport", "record", "tabs/activate", "tabs/close", "tabs/list", - "tabs/open"]) + "page/state", "page/viewport", "record", "sites/set", "tabs/activate", "tabs/close", + "tabs/list", "tabs/open"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index a8a363820..f286cd638 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -25,6 +25,7 @@ from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_READY, COMPUTER_RECORDING from agentenv_protocol.computer.app import COMPUTER_APP from agentenv_protocol.computer.app_runtime import HOLD_BOOT_LAUNCH_ENV +from agentenv_protocol.website_browser import BROWSER from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI, ENABLE_TOOL_EXTENSION_URI from click.testing import CliRunner @@ -827,7 +828,7 @@ def _sites_gateway(): @pytest.fixture def routes(monkeypatch): """Each site routing a deploy asks for, as (sandbox, sites, options), answering what the proxy's install would; the - gateway serving the notes site's card.""" + gateway serving the notes site's card, its websites ready.""" from agentenv_protocol import client as protocol_v1 made = [] @@ -841,6 +842,7 @@ async def get_card(url, timeout=10, verify=True): return NOTES_CARD monkeypatch.setattr(env_computer_provider.sites, "route", route) + monkeypatch.setattr(env_computer_provider.sites, "require_ready", AsyncMock()) monkeypatch.setattr(protocol_v1, "get_card", get_card) monkeypatch.setattr(env_computer_provider, "ready", AsyncMock(return_value=DRIVER_CARD)) return made @@ -970,17 +972,114 @@ async def ready(url, timeout, token): monkeypatch.setattr(env_computer_provider, "ready", ready) monkeypatch.setattr(env_computer_provider, "ready_timeout", lambda apps, os_type: 30.0) record = SimpleNamespace(invoke=invoke, control_token="t0ken", environment_card={"name": "booted"}) - await env_computer_provider._launch_held_apps(env, record, "http://computer") + await env_computer_provider._launch_held_apps(env, record, "http://computer", []) assert order == [("listed", "http://computer", ["mail", "notes"]), (COMPUTER_APP, "launch", "mail", True, "mail"), (COMPUTER_APP, "launch", "notes", True, "mail"), ("ready", "http://computer", "t0ken")] assert record.environment_card == {"name": "ready"} and record.environment_card_read_at_utc order.clear() apps[:] = [apps[1]] - await env_computer_provider._launch_held_apps(env, record, "http://computer") + await env_computer_provider._launch_held_apps(env, record, "http://computer", []) assert order == [("ready", "http://computer", "t0ken")] +def _app_card(name, browser_methods=None): + """A held app's card, launched through computer-app/v1, a website browser when it serves ``browser_methods``.""" + extensions = [{"uri": COMPUTER_APP, "params": {"endpoint": f"/svc/{name}/agentenv/ext/app", + "methods": {"launch": {}}}}] + if browser_methods is not None: + extensions.append({"uri": BROWSER, "params": {"endpoint": f"/svc/{name}/agentenv/ext/browser", + "methods": {method: {} for method in browser_methods}}}) + return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": extensions}} + + +@pytest.mark.asyncio +async def test_each_held_browser_is_handed_the_routed_sites_before_its_launch_and_one_built_before_is_only_launched( + monkeypatch, caplog): + from agent_env.env.env import EnvCapabilityUnsupported + from agent_env.env.envs.computer.utils.params import Site + + calls = [] + apps = [SimpleNamespace(environment_name=name, launch=lambda os_type: SimpleNamespace(at="boot")) + for name in ("chromium", "older", "mail")] + env = SimpleNamespace(computer=SimpleNamespace(os=SimpleNamespace(type="linux")), composed_apps=lambda: apps) + listed_card = {"name": "desk", "children_environments": [ + _app_card("chromium", ["record", "sites/set"]), _app_card("older", ["record"]), _app_card("mail")]} + + async def invoke(uri, method, params=None, *, environment_name=None, child=None): + if method == "sites/set" and child.name == "older": + raise EnvCapabilityUnsupported(uri, method, "desk", None, child.name) + calls.append((uri, method, child.name, params)) + + monkeypatch.setattr(env_computer_provider, "listed", AsyncMock(return_value=listed_card)) + monkeypatch.setattr(env_computer_provider, "ready", AsyncMock(return_value=listed_card)) + monkeypatch.setattr(env_computer_provider, "ready_timeout", lambda apps, os_type: 30.0) + record = SimpleNamespace(invoke=invoke, control_token="t0ken", environment_card={}, env_id="desk") + routed = [Site(name="shop", url="http://gw/website/shop/", domains=["shop.test", "*.shop.test"]), + Site(name="bare", url="http://gw/website/bare/")] + + with caplog.at_level(logging.WARNING): + await env_computer_provider._launch_held_apps(env, record, "http://computer", routed) + + assert calls == [(BROWSER, "sites/set", "chromium", {"sites": [{"name": "shop", "domains": ["shop.test", + "*.shop.test"]}]}), + (COMPUTER_APP, "launch", "chromium", None), (COMPUTER_APP, "launch", "older", None), + (COMPUTER_APP, "launch", "mail", None)] + assert "older in desk serves no browser/v1 sites/set" in caplog.text + + +def _start_tabs_record(card, launches_by_browser): + """A deployed computer whose card is ``card`` and whose browsers' ``record`` keeps ``launches_by_browser``, by the + browser's name, None for the computer's own card; a browser absent from it built before start tabs.""" + from agent_env.env.env import EnvCapabilityUnsupported + + async def invoke(uri, method, params=None, *, environment_name=None, child=None): + name = child.name if child is not None else None + if (uri, method) != (BROWSER, "record") or name not in launches_by_browser: + raise EnvCapabilityUnsupported(uri, method, "desk", None, name) + return {"start_tabs": [{"at": "t", "tabs": tabs} for tabs in launches_by_browser[name]]} + + return SimpleNamespace(invoke=invoke, environment_card=card, env_id="@local/desk") + + +def _tab(url, site=None, outcome="loaded", **fields): + return {"tab": "t1", "url": url, "final_url": url, "status": 200, "error": None, "load_ms": 120, "site": site, + "outcome": outcome, **fields} + + +@pytest.mark.asyncio +async def test_each_browsers_start_tabs_of_its_latest_launch_are_kept_and_one_built_before_them_is_left_out(): + card = {"name": "desk", "children_environments": [_app_card("chromium", ["record"]), _app_card("older", []), + _app_card("mail")]} + record = _start_tabs_record(card, {"chromium": [[_tab("about:blank")], [ + _tab("https://shop.test/", "shop"), _tab("https://slack.com/", outcome="timed_out", status=None)]]}) + alone = _start_tabs_record({"name": "chromium-driver", "capabilities": {"extensions": [ + {"uri": BROWSER, "params": {"endpoint": "/ext/browser", "methods": {"record": {}}}}]}}, + {None: [[_tab("https://linear.app/")]]}) + + assert await env_computer_provider._start_tabs(record) == {"chromium": [ + _tab("https://shop.test/", "shop"), _tab("https://slack.com/", outcome="timed_out", status=None)]} + assert await env_computer_provider._start_tabs(alone) == {"chromium-driver": [_tab("https://linear.app/")]} + + +@pytest.mark.asyncio +async def test_a_start_tab_on_one_of_the_env_sites_that_didnt_load_fails_the_boot_naming_each_one(): + card = {"name": "desk", "children_environments": [_app_card("chromium", ["record"])]} + record = _start_tabs_record(card, {"chromium": [[ + _tab("https://shop.test/", "shop", "failed", status=503), + _tab("https://mail.test/", "mail", "failed", status=None, error="net::ERR_CONNECTION_REFUSED"), + _tab("https://notes.test/", "notes", "timed_out", status=None, load_ms=60000), + _tab("https://slack.com/", None, "failed", status=500)]]}) + + with pytest.raises(RuntimeError) as failed: + await env_computer_provider._start_tabs(record) + + assert str(failed.value) == ( + "computer env '@local/desk': chromium's start tab https://shop.test/ on the env's site shop answered 503; " + "chromium's start tab https://mail.test/ on the env's site mail didn't load: net::ERR_CONNECTION_REFUSED; " + "chromium's start tab https://notes.test/ on the env's site notes was still loading after 60 s") + + @pytest.mark.asyncio async def test_a_held_app_is_awaited_until_the_driver_lists_it_with_its_server(monkeypatch): """A page driver answers before its browser's server does, and lists the browser by its name alone until then.""" diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 18816f7d4..813c11c78 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -497,3 +497,42 @@ async def get_card(url, timeout=10, verify=True): def _status_error(url, code): request = httpx.Request("GET", f"{url}/.well-known/agent-env.json") return httpx.HTTPStatusError(f"{code}", request=request, response=httpx.Response(code, request=request)) + + +class _Website: + """A deployed env whose ``readiness`` answers each of ``answers`` in turn, then the last again: a readiness, an + error to raise, or None for a record with no address.""" + + def __init__(self, env_id, *answers): + self.env_id, self.answers, self.asked = env_id, list(answers), 0 + + async def readiness(self): + from agent_env.env.env import EnvCapabilityUnsupported + from agentenv_protocol.website import WebsiteReadiness + + answer = self.answers[min(self.asked, len(self.answers) - 1)] + self.asked += 1 + if answer is None: + raise EnvCapabilityUnsupported("urn:agentenv:website/v1", "ready", self.env_id, None) + if isinstance(answer, Exception): + raise answer + return WebsiteReadiness(ready=not answer, problems=answer) + + +@pytest.mark.asyncio +async def test_the_websites_a_computer_routes_are_waited_on_till_ready_and_one_never_ready_is_named(): + late = _Website("late", ["notes: warming up"], []) + shop = _Website("shop", []) + other = _Website("other", ["never asked"]) + unaddressed = _Website("unaddressed", None) + + await sites.require_ready([late, shop, other, unaddressed], ["late", "shop", "unaddressed"], timeout_s=5, + poll_s=0.01) + broken = _Website("broken", ["shop: its database is down"]) + unreachable = _Website("unreachable", httpx.ConnectError("refused")) + with pytest.raises(RuntimeError) as never: + await sites.require_ready([broken, unreachable, shop], timeout_s=0.05, poll_s=0.01) + + assert (late.asked, shop.asked, other.asked, unaddressed.asked) == (2, 2, 0, 1) + assert str(never.value).startswith("site routing: the run's websites weren't ready after 0.05 s: ") + assert str(never.value).endswith("broken (shop: its database is down); unreachable (ConnectError: refused)") From a9d95f8b6e10d778c0423bea4d52329a5f602203 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 7 Nov 2026 09:21:20 -0800 Subject: [PATCH 2/5] fix(browser): a start page that sends itself elsewhere is judged by where it lands, and failed start tabs are named and kept A start tab now follows its main frame to the last document committed from its own navigation on, so a page that replaces itself by script before it loads settles on that page's load and status rather than timing out. After a relaunch, data/reset and a profile load wait on the start tabs themselves, their ready timeout and a setup margin, so a failed one is named rather than reported as the browser not being ready. A boot keeps its start tabs' outcomes in its record before it fails on one. Also renames a test loop variable that shadowed an import. Co-Authored-By: Claude Opus 5.5 --- .../website_browser/control.py | 38 ++++++++++++------- .../website_browser/environment.py | 22 +++++++++-- .../tests/test_browser_data.py | 17 ++++++++- .../tests/test_website_browser.py | 38 ++++++++++++++----- .../env_providers/env_computer_provider.py | 12 ++++-- tst/unit/env/envs/test_computer_env.py | 19 ++++++---- 6 files changed, 108 insertions(+), 38 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index fd8a98e9a..a10dbf850 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -213,8 +213,9 @@ async def _settled(self, target: str, load: _StartLoad, timeout_s: float) -> dic self._loading.pop(load.session, None) with contextlib.suppress(ConnectionError, DevToolsError, asyncio.TimeoutError): await self.devtools.call("Network.disable", session_id=load.session) - status, final_url = load.responses.get(load.loader or "", (None, None)) - error = load.error or load.failures.get(load.loader or "") + document = load.current() or load.loader or "" + status, final_url = load.responses.get(document, (None, None)) + error = load.error or load.failures.get(load.loader or "") or load.failures.get(document) outcome = FAILED if error or (load.settled.is_set() and status is not None and status >= 400) else \ LOADED if load.settled.is_set() else TIMED_OUT if final_url is None and outcome == LOADED: @@ -587,8 +588,9 @@ def _on_Page_frameNavigated(self, params: dict, session: Optional[str]) -> None: if session in self._sessions: self._saw(params.get("frame", {}).get("url", "")) frame = params.get("frame", {}) - if (load := self._loading.get(session)) is not None and not frame.get("parentId"): - load.committed = frame.get("loaderId") + if (load := self._loading.get(session)) is not None and not frame.get("parentId") and frame.get("loaderId"): + load.commits.append(frame["loaderId"]) + load.check() def _on_Network_requestWillBeSent(self, params: dict, session: Optional[str]) -> None: if (load := self._loading.get(session)) is not None and params.get("type") == "Document": @@ -659,8 +661,8 @@ def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None: self._choosers.pop(session, None) if (target := self._sessions.get(session)) is not None: self._spawn(self._refresh(target)) - if (load := self._loading.get(session)) is not None and load.committed is not None: - load.loaded.add(load.committed) + if (load := self._loading.get(session)) is not None and load.commits: + load.loaded.add(load.commits[-1]) load.check() def _on_Page_fileChooserOpened(self, params: dict, session: Optional[str]) -> None: @@ -726,15 +728,17 @@ def _finished(self, task: asyncio.Future) -> None: @dataclass class _StartLoad: """What the browser has said of a start tab's navigation, by loader, since it was sent to ``url``: its main frame's - last commit, the loads that fired after a commit, and its document's requests, final responses and failures. - It's ``settled`` once its own loader has failed or loaded, or, on one of the env's ``site``, answered an error.""" + commits in order, the loads that fired after a commit, and its documents' requests, final responses and failures. + Its document is the last its main frame committed from its own navigation on, which a page that sends itself + elsewhere before it loads replaces. It's ``settled`` once its navigation or its document has failed, its document + has loaded or, on one of the env's ``site``, answered an error.""" session: str url: str site: Optional[str] started: float loader: Optional[str] = None error: Optional[str] = None - committed: Optional[str] = None + commits: list[str] = field(default_factory=list) loaded: set[str] = field(default_factory=set) requests: dict[str, str] = field(default_factory=dict) responses: dict[str, tuple[Optional[int], Optional[str]]] = field(default_factory=dict) @@ -757,11 +761,19 @@ def answered(self, navigating: asyncio.Future) -> None: self.loader = answer.get("result", {}).get("loaderId") self.check() + def current(self) -> Optional[str]: + """The loader of its document: the last its main frame committed, once its own navigation has; else None.""" + return self.commits[-1] if self.loader is not None and self.loader in self.commits else None + def check(self) -> None: - status = self.responses.get(self.loader or "", (None, None))[0] - if self.error or (self.loader is not None and ( - self.loader in self.failures or self.loader in self.loaded - or (self.site is not None and status is not None and status >= 400))): + if self.error or (self.loader is not None and self.loader in self.failures): + self.settled.set() + return + if (document := self.current()) is None: + return + status = self.responses.get(document, (None, None))[0] + if document in self.failures or document in self.loaded or ( + self.site is not None and status is not None and status >= 400): self.settled.set() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index f7ba5c634..75c6b40fd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -49,6 +49,9 @@ BLOCK = 2 #: How often a call that waits for the browser to be ready checks it. READY_POLL_S = 0.5 +#: How much longer than its ready timeout a call waits for a launch's start tabs, for the browser's setup before they +#: load. +START_TABS_MARGIN_S = 30.0 #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -538,8 +541,8 @@ async def browser_add_data(self, parts: list) -> None: await self._computer_app.stop_app() await app_data.in_thread(data.restore_profile, Path.home(), self.profile, load.profile) await self._computer_app.launch_app(METHOD_ADD) + _refuse_failed(await self._start_tabs_settled()) await self._until_ready() - _refuse_failed(self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) applied.append("the profile") if load.files: await app_data.in_thread(app_data.write_within, Path.home(), load.files, [DOWNLOADS, UPLOADS]) @@ -574,10 +577,21 @@ async def browser_reset_data(self) -> None: forget_agent_acted() async def _relaunched(self, op: str) -> None: - """After a reset relaunches it: back once its start tabs have settled, or DeviceUnavailable after its ready - timeout; InvalidArgument naming a start tab on one of the env's sites that failed.""" + """After a reset relaunches it: back once it's ready, InvalidArgument naming a start tab on one of the env's + sites that failed.""" + _refuse_failed(await self._start_tabs_settled()) await self._until_ready() - _refuse_failed(self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else []) + + async def _start_tabs_settled(self) -> list[dict]: + """The start tabs of its latest launch once they've settled, waited on its ready timeout and the margin its + setup takes before they load; DeviceUnavailable if they haven't by then.""" + if (following := self._following) is not None: + timeout_s = self._computer_app.timeout_s + START_TABS_MARGIN_S + try: + await asyncio.wait_for(asyncio.shield(following), timeout_s) + except asyncio.TimeoutError: + raise DeviceUnavailable(f"the browser's start tabs hadn't settled after {timeout_s:g} s") from None + return self.start_tab_launches[-1]["tabs"] if self.start_tab_launches else [] def extension_params(self, uri: str) -> dict: if uri == BROWSER: diff --git a/packages/agentenv-protocol/tests/test_browser_data.py b/packages/agentenv-protocol/tests/test_browser_data.py index 9f98d8c72..87f3f6c24 100644 --- a/packages/agentenv-protocol/tests/test_browser_data.py +++ b/packages/agentenv-protocol/tests/test_browser_data.py @@ -7,6 +7,7 @@ puts the profile back as installed, empties the downloads and uploads and starts the record over.""" from __future__ import annotations +import asyncio import base64 import contextlib import json @@ -17,7 +18,7 @@ from agentenv_protocol import website_browser from agentenv_protocol.computer import Launch, Ready, app, linux from agentenv_protocol.computer.app import AGENT_ACTED -from agentenv_protocol.computer.errors import InvalidArgument +from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument from agentenv_protocol.computer.platform import Platform from agentenv_protocol.types import DATA_OBJECTS_EXTENSION_URI, INTAKE_EXTENSION_URI, AddDataRequest from agentenv_protocol.website_browser import WebsiteBrowserEnvironment @@ -304,6 +305,8 @@ def launched(url, site, outcome, **why): return {"at": "t", "tabs": [{"tab": "t1", "url": url, "final_url": None, "status": None, "error": None, "load_ms": 60000, "site": site, "outcome": outcome, **why}]} + browser._following = asyncio.get_running_loop().create_future() + browser._following.set_result(None) browser.start_tab_launches = [launched("https://slack.com/", None, "timed_out")] await browser._relaunched("data/reset") browser.start_tab_launches.append(launched("https://shop.test/", "shop", "timed_out")) @@ -314,6 +317,18 @@ def launched(url, site, outcome, **why): error="net::ERR_CONNECTION_REFUSED")) with pytest.raises(InvalidArgument, match="didn't load: net::ERR_CONNECTION_REFUSED"): await browser._relaunched("data/reset") + assert browser.launches == ["ready"] + + +@pytest.mark.asyncio +async def test_after_a_reset_relaunches_it_start_tabs_still_settling_past_its_time_and_margin_fail_the_reset( + browser, monkeypatch): + monkeypatch.setattr(website_browser.environment, "START_TABS_MARGIN_S", 0.05) + browser._computer_app.timeout_s = 0.05 + browser._following = asyncio.get_running_loop().create_future() + + with pytest.raises(DeviceUnavailable, match="start tabs hadn't settled after 0.1 s"): + await browser._relaunched("data/reset") @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 3b6843e0a..1d960f135 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -585,8 +585,9 @@ def refused(params, session): class Navigating(StubDevTools): """A browser whose first tab is ``start`` and whose new tabs are t1, t2, …, each attached as it opens, answering a navigation as ``pages`` says by URL: a status and the URL it ends on, then committed and, unless ``no-load`` - follows, loaded; ``refused``, a network error; ``silent``, never answered, as a page that never answers isn't; or - ``stale``, a load event with no commit.""" + follows, loaded, or with ``("redirect", status, url)`` following, sending itself there by script before it loads; + ``refused``, a network error; ``silent``, never answered, as a page that never answers isn't; or ``stale``, a load + event with no commit.""" def __init__(self, pages): super().__init__({"Target.getTargets": {"targetInfos": []}}) @@ -615,15 +616,20 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None): self.send("Page.loadEventFired", {}, session_id) return {"frameId": target, "loaderId": loader} status, final_url, *rest = page - document = {"requestId": request, "loaderId": loader, "type": "Document", "frameId": target} - self.send("Network.requestWillBeSent", document, session_id) - self.send("Network.responseReceived", {**document, "response": {"status": status, "url": final_url}}, - session_id) - self.send("Page.frameNavigated", {"frame": {"id": target, "loaderId": loader, "url": final_url}}, session_id) + self._document(session_id, target, loader, request, status, final_url) + if rest and isinstance(rest[0], tuple): + _, status, final_url = rest[0] + self._document(session_id, target, f"{loader}-redirect", f"{request}-redirect", status, final_url) if "no-load" not in rest: self.send("Page.loadEventFired", {}, session_id) return {"frameId": target, "loaderId": loader} + def _document(self, session, target, loader, request, status, url): + document = {"requestId": request, "loaderId": loader, "type": "Document", "frameId": target} + self.send("Network.requestWillBeSent", document, session) + self.send("Network.responseReceived", {**document, "response": {"status": status, "url": url}}, session) + self.send("Page.frameNavigated", {"frame": {"id": target, "loaderId": loader, "url": url}}, session) + async def _started(devtools, viewport=(1280, 800)) -> BrowserControl: """A control following a browser just launched, on its first tab, ``start``.""" @@ -660,8 +666,8 @@ async def test_its_start_tabs_load_in_order_in_its_first_tab_then_blank_ones_hel assert loop.time() - started < 2 assert list(control.tabs) == ["start", "t1", "t2"] and all(isinstance(tab["load_ms"], int) for tab in tabs) assert _sent(devtools, "Target.createTarget") == [({"url": "about:blank", "background": True}, None)] * 2 - for session in ("s-start", "s-t1", "s-t2"): - sent = [method for method, _, on in devtools.calls if on == session] + for tab_session in ("s-start", "s-t1", "s-t2"): + sent = [method for method, _, on in devtools.calls if on == tab_session] assert sent.index("Emulation.setDeviceMetricsOverride") < sent.index("Network.enable") < \ sent.index("Page.navigate") < sent.index("Network.disable"), sent assert devtools.calls[-2][:2] == ("Target.activateTarget", {"targetId": "start"}) @@ -692,6 +698,20 @@ async def test_a_start_tab_settles_strictly_on_an_env_site_and_at_most_its_time_ assert (loop.time() - started >= 0.3) is waited +@pytest.mark.asyncio +@pytest.mark.parametrize("redirect, final, status, outcome", [ + (("redirect", 200, "https://shop.test/notes"), "https://shop.test/notes", 200, "loaded"), + (("redirect", 404, "https://shop.test/gone"), "https://shop.test/gone", 404, "failed"), +], ids=["to-a-page", "to-an-error"]) +async def test_a_start_page_that_sends_itself_elsewhere_before_it_loads_is_judged_by_the_page_it_ends_on( + redirect, final, status, outcome): + control = await _started(Navigating({"https://shop.test/": (200, "https://shop.test/", redirect)})) + + (tab,) = await control.load_start_tabs(["https://shop.test/"], _shop, own_timeout_s=5) + + assert (tab["final_url"], tab["status"], tab["outcome"]) == (final, status, outcome) + + @pytest.mark.asyncio async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that(): control = await _started(Navigating({"https://shop.test/": "silent"})) diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 5f5c85498..906aed324 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -202,6 +202,7 @@ async def _finish_boot(env: ComputerEnv, record: DeployedSandboxEnv, sandbox: Sa await _require_display(record, resolution) if start_tabs := await _start_tabs(record): record.metadata["computer"]["start_tabs"] = start_tabs + _refuse_failed_start_tabs(record, start_tabs) return routed @@ -254,8 +255,7 @@ async def _hand_sites(record: DeployedSandboxEnv, child: protocol_v1.ChildEnv, async def _start_tabs(record: DeployedSandboxEnv) -> dict[str, list[dict]]: """Each website browser's start tabs as its latest launch loaded them, by its name: the computer's own card's, for - a browser alone, or a child's; a browser that keeps none, built before it did, left out. RuntimeError naming each - start tab on one of the env's sites that didn't load.""" + a browser alone, or a child's; a browser that keeps none, built before it did, left out.""" card = record.environment_card or {} browsers = ([(card.get("name") or record.env_id, None)] if protocol_v1.find_extension(card, BROWSER) else []) \ + [(child.name, child) for child in protocol_v1.children(card) if child.find_extension(BROWSER) is not None] @@ -267,11 +267,15 @@ async def _start_tabs(record: DeployedSandboxEnv) -> dict[str, list[dict]]: continue if launches: kept[name] = launches[-1]["tabs"] + return kept + + +def _refuse_failed_start_tabs(record: DeployedSandboxEnv, start_tabs: dict[str, list[dict]]) -> None: + """RuntimeError naming each of ``start_tabs``, by browser, on one of the env's sites that didn't load.""" if failed := [f"{name}'s start tab {tab['url']} on the env's site {tab['site']} {start_tab_why(tab)}" - for name, tabs in kept.items() for tab in tabs + for name, tabs in start_tabs.items() for tab in tabs if tab["site"] is not None and tab["outcome"] != LOADED]: raise RuntimeError(f"computer env {record.env_id!r}: " + "; ".join(failed)) - return kept async def end_recordings(record: DeployedSandboxEnv, reason: str) -> None: diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index f286cd638..62c34d762 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1063,17 +1063,22 @@ async def test_each_browsers_start_tabs_of_its_latest_launch_are_kept_and_one_bu @pytest.mark.asyncio -async def test_a_start_tab_on_one_of_the_env_sites_that_didnt_load_fails_the_boot_naming_each_one(): +async def test_a_start_tab_on_one_of_the_env_sites_that_didnt_load_fails_the_boot_naming_each_once_kept( + monkeypatch): card = {"name": "desk", "children_environments": [_app_card("chromium", ["record"])]} - record = _start_tabs_record(card, {"chromium": [[ - _tab("https://shop.test/", "shop", "failed", status=503), - _tab("https://mail.test/", "mail", "failed", status=None, error="net::ERR_CONNECTION_REFUSED"), - _tab("https://notes.test/", "notes", "timed_out", status=None, load_ms=60000), - _tab("https://slack.com/", None, "failed", status=500)]]}) + tabs = [_tab("https://shop.test/", "shop", "failed", status=503), + _tab("https://mail.test/", "mail", "failed", status=None, error="net::ERR_CONNECTION_REFUSED"), + _tab("https://notes.test/", "notes", "timed_out", status=None, load_ms=60000), + _tab("https://slack.com/", None, "failed", status=500)] + record = _start_tabs_record(card, {"chromium": [tabs]}) + record.metadata = {"computer": {"start_tabs": {"chromium": [_tab("https://shop.test/", "shop")]}}} + monkeypatch.setattr(env_computer_provider, "_require_display", AsyncMock()) with pytest.raises(RuntimeError) as failed: - await env_computer_provider._start_tabs(record) + await env_computer_provider._finish_boot(SimpleNamespace(), record, None, + SimpleNamespace(site_routing=False), 0, None) + assert record.metadata["computer"]["start_tabs"] == {"chromium": tabs} assert str(failed.value) == ( "computer env '@local/desk': chromium's start tab https://shop.test/ on the env's site shop answered 503; " "chromium's start tab https://mail.test/ on the env's site mail didn't load: net::ERR_CONNECTION_REFUSED; " From f82c632fe874b9275a758c72cb534765a89d9bd1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 7 Nov 2026 15:46:11 -0800 Subject: [PATCH 3/5] fix(computer): ask only the run's envs that serve websites whether they're ready, and test a routed start page that redirects A computer routing the run's websites waited on every deployed env's readiness, so an env serving none that couldn't be reached failed the deploy after 60 s; it now asks only the envs whose websites it routes. The routed-site e2e's home page now replaces itself by script before it loads, held from loading by a slow image, so the start tab is judged by the page it lands on in a real Chromium. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/sites.py | 17 ++++++++++--- tst/integration/computer/chromium_app_test.py | 10 ++++---- tst/integration/computer/start_tabs.py | 25 +++++++++++-------- tst/unit/env/envs/test_computer_sites.py | 17 ++++++++++--- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index b2bb6519a..fd236b291 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -28,6 +28,8 @@ from urllib.parse import urlsplit import httpx +from agentenv_protocol import client as protocol_client +from agentenv_protocol.website import WEBSITE_EXTENSION_URI as WEBSITE from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec @@ -468,9 +470,10 @@ async def require_ready(deployed_envs: list[DeployedEnv], only: Optional[list[st timeout_s: float = SITES_READY_TIMEOUT_S, poll_s: float = SITES_READY_POLL_S) -> None: """Back once each deployed env whose websites a computer routes, those of the env ids ``only`` names or all, says through ``website/v1``'s ``ready`` that every site of it is ready; RuntimeError naming each one still not, and why, - once ``timeout_s`` has passed. An env with no address to ask isn't waited on, nor one with no site serving - ``ready``, which is ready.""" - waiting = [record for record in deployed_envs if only is None or record.env_id in only] + once ``timeout_s`` has passed. An env that serves no website isn't asked, nor one with no address to ask, and one + with no site serving ``ready`` is ready.""" + waiting = [record for record in deployed_envs + if (only is None or record.env_id in only) and serves_websites(record)] deadline = time.monotonic() + timeout_s while True: why: dict[str, str] = {} @@ -493,6 +496,14 @@ async def require_ready(deployed_envs: list[DeployedEnv], only: Optional[list[st await asyncio.sleep(poll_s) +def serves_websites(record: DeployedEnv) -> bool: + """Whether ``record`` serves websites a computer routes, as ``deployed_sites`` finds them: a named ``website/v1`` + child on its stored card, or a website frontend it lists.""" + card = getattr(record, "environment_card", None) or {} + return bool(getattr(record, "website_frontend_urls", None)) or any( + child.find_extension(WEBSITE) is not None and child.name for child in protocol_client.children(card)) + + def _env_qualified(found: list[tuple[str, dict]]) -> list[dict]: """Each site found, a name that sites of different envs share made ``-`` for each of them, letters, digits, ``-`` and ``_`` kept, so every site of the run has a name of its own.""" diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 78669d7d9..5e52d5b17 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -11,8 +11,8 @@ cookies, local storage and downloads; and its data plane signs it in, snapshots it, resets it and loads a snapshot back; and the browser task steps set its cookies and storage and open, activate and close its tabs by URL. Its start tabs open in order, the first active, a deploy recording each one's outcome and going on past a live page -that never answers; and with the run's sites routed to it, a start tab on one must load, so a site answering 503 -fails the deploy, naming it. Needs Docker and the local registry on localhost:5000.""" +that never answers; and with the run's sites routed to it, a start tab on one must load, judged by the page it +lands on when it sends itself elsewhere before it loads, so a site answering 503 fails the deploy, naming it. Needs Docker and the local registry on localhost:5000.""" import asyncio import hashlib @@ -378,7 +378,7 @@ async def test_its_start_tabs_open_in_order_the_first_active_and_a_live_one_that @pytest.mark.asyncio @pytest.mark.parametrize("status", [200, 503], ids=["loads", "answers-503"]) -async def test_a_start_tab_on_a_routed_site_must_load_so_a_site_answering_an_error_fails_the_deploy( +async def test_a_start_tab_on_a_routed_site_must_load_where_it_lands_so_a_site_answering_503_fails_the_deploy( chromium_desktop, status): with host_pages(site_status=status) as port, namespace_routing(): params = {"site_routing": True, "sites": [{"name": "shop", "url": f"http://127.0.0.1:{port}/website/shop/", @@ -398,8 +398,8 @@ async def test_a_start_tab_on_a_routed_site_must_load_so_a_site_answering_an_err (tab,) = deployed.metadata["computer"]["start_tabs"]["chromium"] assert (tab["url"], tab["final_url"], tab["status"], tab["site"], tab["outcome"]) == ( - "https://shop.test/", "https://shop.test/", 200, "shop", "loaded") - assert [(open_tab["url"], open_tab["title"]) for open_tab in tabs] == [("https://shop.test/", "shop")] + "https://shop.test/", "https://shop.test/landed", 200, "shop", "loaded") + assert [(open_tab["url"], open_tab["title"]) for open_tab in tabs] == [("https://shop.test/landed", "landed")] assert not report.failed diff --git a/tst/integration/computer/start_tabs.py b/tst/integration/computer/start_tabs.py index f2eb4fa73..ffb236f58 100644 --- a/tst/integration/computer/start_tabs.py +++ b/tst/integration/computer/start_tabs.py @@ -1,5 +1,6 @@ """Pages a computer's start tabs open, served on this machine, which a computer reaches as ``host.docker.internal``: a -site's route, answering its status, two pages titled by name, and one that never answers.""" +site's route, whose home page sends itself elsewhere before it loads, or answers an error; two pages titled by name; +and one that never answers.""" import contextlib import http.server @@ -13,23 +14,27 @@ from agentenv_protocol.types import WELL_KNOWN_PATH from agentenv_protocol.website_browser import BROWSER -#: How long the page that never answers holds its request. -HANG_S = 120 +#: How long the page that never answers holds its request, and the image that holds the site's home page from loading. +HANG_S, SLOW_S = 120, 30 +#: The site's home page: it replaces itself with /landed by script as it's read, while an image keeps it from loading. +LEAVING = b"leaving" @contextlib.contextmanager def host_pages(site_status: int = 200) -> Iterator[int]: - """The pages served on every address of this machine while the block runs, at the port it yields: ``/website/ - shop/``, a site's route, answering ``site_status``; ``/alpha`` and ``/bravo``; and ``/hang``, which never - answers.""" + """The pages served on every address of this machine while the block runs, at the port it yields: under + ``/website/shop/``, a site's route, its home page, which sends itself to ``landed`` before it loads, or with any + other ``site_status`` answers it, and ``landed``; ``/alpha`` and ``/bravo``; and ``/hang``, which never answers.""" class Page(http.server.BaseHTTPRequestHandler): def do_GET(self): - if self.path == "/hang": - time.sleep(HANG_S) + if self.path in ("/hang", "/website/shop/slow"): + time.sleep(HANG_S if self.path == "/hang" else SLOW_S) return - status = site_status if self.path.startswith("/website/shop/") else 200 - body = f"{self.path.strip('/').rsplit('/', 1)[-1] or 'shop'}".encode() + home = self.path == "/website/shop/" + status = site_status if home else 200 + title = self.path.strip("/").rsplit("/", 1)[-1] + body = LEAVING if home and status == 200 else f"{title}".encode() self.send_response(status) self.send_header("Content-Type", "text/html") self.send_header("Content-Length", str(len(body))) diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 813c11c78..cb4fa285e 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -500,11 +500,13 @@ def _status_error(url, code): class _Website: - """A deployed env whose ``readiness`` answers each of ``answers`` in turn, then the last again: a readiness, an - error to raise, or None for a record with no address.""" + """A deployed env serving a website, whose ``readiness`` answers each of ``answers`` in turn, then the last again: a + readiness, an error to raise, or None for a record with no address.""" - def __init__(self, env_id, *answers): + def __init__(self, env_id, *answers, card=None): self.env_id, self.answers, self.asked = env_id, list(answers), 0 + self.environment_card = card + self.website_frontend_urls = None if card is not None else {"site": f"http://gw/website/{env_id}/"} async def readiness(self): from agent_env.env.env import EnvCapabilityUnsupported @@ -520,19 +522,26 @@ async def readiness(self): @pytest.mark.asyncio -async def test_the_websites_a_computer_routes_are_waited_on_till_ready_and_one_never_ready_is_named(): +async def test_the_websites_a_computer_routes_are_waited_on_till_ready_one_never_ready_named_and_others_not_asked(): late = _Website("late", ["notes: warming up"], []) shop = _Website("shop", []) other = _Website("other", ["never asked"]) unaddressed = _Website("unaddressed", None) + on_its_card = _Website("on-its-card", [], card={"name": "kit", "children_environments": [{ + "name": "notes", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": {}}]}}]}) + mcp_only = _Website("mcp-only", httpx.ConnectError("never asked"), card={"name": "mcp", "children_environments": [ + {"name": "tools", "capabilities": {"extensions": []}}]}) + await sites.require_ready([late, shop, other, unaddressed], ["late", "shop", "unaddressed"], timeout_s=5, poll_s=0.01) + await sites.require_ready([on_its_card, mcp_only], timeout_s=0.05, poll_s=0.01) broken = _Website("broken", ["shop: its database is down"]) unreachable = _Website("unreachable", httpx.ConnectError("refused")) with pytest.raises(RuntimeError) as never: await sites.require_ready([broken, unreachable, shop], timeout_s=0.05, poll_s=0.01) assert (late.asked, shop.asked, other.asked, unaddressed.asked) == (2, 2, 0, 1) + assert (on_its_card.asked, mcp_only.asked) == (1, 0) assert str(never.value).startswith("site routing: the run's websites weren't ready after 0.05 s: ") assert str(never.value).endswith("broken (shop: its database is down); unreachable (ConnectError: refused)") From aa4a2929be229e98546c54f901d68fc46b9b9a12 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 7 Nov 2026 16:05:03 -0800 Subject: [PATCH 4/5] fix(browser): a launch's start tabs are kept when the browser is gone before the first comes to the front Bringing the first start tab to the front over a pipe the browser closed raised, so the outcomes the tabs had settled on, "the browser ended", were never kept for the launch. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/website_browser/control.py | 8 +++++--- packages/agentenv-protocol/tests/test_website_browser.py | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index a10dbf850..c7ea10107 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -154,8 +154,9 @@ async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Option in tabs it opens blank after them; each held to the window's size, headless, before it goes to its URL, and all loading at once. A tab settles once it has loaded, once it has failed, which on one of the env's own sites, as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a - site, else ``live_timeout_s``. Each tab's outcome, in order. A page already loaded doesn't read cookies set - after it, so a tab a load signed in shows it only once it loads again.""" + site, else ``live_timeout_s``. Each tab's outcome, in order, kept though the browser has gone and the first + can't come to the front. A page already loaded doesn't read cookies set after it, so a tab a load signed in + shows it only once it loads again.""" if not urls: return [] targets = list(self.tabs)[:len(urls)] or [await self._first_tab()] @@ -170,7 +171,8 @@ async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Option loading.append(await self._start_load(target, url, site)) outcomes = await asyncio.gather(*(self._settled(target, load, own_timeout_s if load.site is not None else live_timeout_s) for target, load in zip(targets, loading))) - await self._bring_to_front(targets[0]) + with contextlib.suppress(ConnectionError, DevToolsError): + await self._bring_to_front(targets[0]) return outcomes async def _first_tab(self) -> str: diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 1d960f135..c717293eb 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -713,8 +713,13 @@ async def test_a_start_page_that_sends_itself_elsewhere_before_it_loads_is_judge @pytest.mark.asyncio -async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that(): - control = await _started(Navigating({"https://shop.test/": "silent"})) +async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that_and_is_kept_with_its_pipe_gone(): + def gone(params, session): + raise ConnectionError("the browser closed its debugging pipe") + + devtools = Navigating({"https://shop.test/": "silent"}) + devtools.answers["Target.activateTarget"] = gone + control = await _started(devtools) loading = asyncio.ensure_future(control.load_start_tabs(["https://shop.test/"], _shop, own_timeout_s=5)) for _ in range(100): if control._loading: From defe9d32bf87c41b3f8c7a396854cf1a6e7971cd Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 7 Nov 2026 16:48:33 -0800 Subject: [PATCH 5/5] refactor(browser): a launch's start tabs settle within its ready timeout, its setup counted A start tab on one of the env's sites waited the whole ready timeout from its own navigation, after the launch's setup, while a deploy and data/reset wait that timeout and a margin from the launch, so a slow setup could let them give up first with no outcome. Every start tab now settles within the ready timeout of its launch, an own-site one on what's left of it and a live one on 30 s at most, so one constant decides it. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 8 +++-- .../website_browser/environment.py | 29 +++++++++++-------- .../tests/test_browser_data.py | 15 +++++++++- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 06bac550c..b1cec4c44 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -975,9 +975,11 @@ are the profile's template. The base class adds: (headless) before it goes to its URL, so its page sees that size from its first script; then the first is brought to the front. They load at once, and each settles once it has loaded (its load event fired, its document's final status read), once it failed (a network error, or on one of the env's sites, as `sites/set` named them, a 4xx or - 5xx status, which settles it at once), or once it has been waited on: as long as the browser may take to be ready - on one of the env's sites, 30 s on any other. Its `start_tabs_loaded` ready check passes once all have settled, so - `cdp` and a deploy wait for them. `record`'s `start_tabs` keeps each launch's (`StartTabsLaunch`: `at` and its + 5xx status, which settles it at once), or once it has been waited on: every one settles within the browser's + ready timeout of its launch, the launch's setup counted against it, a tab on one of the env's sites waited on what's + left of it and any other on 30 s at most, so `data/reset`, a load and a deploy, which wait that timeout and a margin, + always get its outcome. Its `start_tabs_loaded` ready check passes once all have settled, so `cdp` and a deploy wait + for them. `record`'s `start_tabs` keeps each launch's (`StartTabsLaunch`: `at` and its `tabs`, each a `StartTab`: `tab`, `url`, `final_url`, `status`, `error`, `load_ms`, `site` — the env's site's name, null for a live one — and `outcome`, `loaded`, `failed` or `timed_out`), and `data/get`'s state keeps its latest launch's. A deploy of a computer reads them once it's ready, keeps them in its record and fails, naming diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 75c6b40fd..7be58a335 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -28,7 +28,7 @@ from ..transfers import WriteNamespaceGrant from ..types import METHOD_ADD, EnvironmentExtension, IntakeDeclaration, IntakeFormat, intake_extension from . import data -from .control import AGENT, DOWNLOADS, EXIT, LOADED, STOP, UPLOADS, BrowserControl, _now +from .control import AGENT, DOWNLOADS, EXIT, LIVE_START_TAB_TIMEOUT_S, LOADED, STOP, UPLOADS, BrowserControl, _now from .devtools import PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe from .shapes import BrowserRecord, BrowserState, CookieToSet, OriginStorage, SiteNames, StateToLoad @@ -49,8 +49,7 @@ BLOCK = 2 #: How often a call that waits for the browser to be ready checks it. READY_POLL_S = 0.5 -#: How much longer than its ready timeout a call waits for a launch's start tabs, for the browser's setup before they -#: load. +#: How much longer than its ready timeout, within which a launch's start tabs settle, a call waits for them. START_TABS_MARGIN_S = 30.0 #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. @@ -214,7 +213,8 @@ async def _launch_fds(self) -> dict[int, int]: to_browser_read, to_browser_write = os.pipe() from_browser_read, from_browser_write = os.pipe() await self.devtools.connect(to_browser_write, from_browser_read) - self._following = asyncio.ensure_future(self._follow(self.settings.start_urls, _now())) + self._following = asyncio.ensure_future(self._follow(self.settings.start_urls, _now(), + asyncio.get_running_loop().time())) self._following.add_done_callback(lambda done: done.cancelled() or done.exception()) return {PIPE_IN: to_browser_read, PIPE_OUT: from_browser_write} @@ -224,16 +224,21 @@ def _pipe_closed(self) -> None: self.control.ended(EXIT if self._computer_app.exited() else STOP) self.relay.ended() - async def _follow(self, urls: list[str], at: str) -> None: + async def _follow(self, urls: list[str], at: str, launched: float) -> None: """Its control attached, then its start tabs at ``urls`` loaded, the launch's outcomes kept.""" await self.control.attach() - await self._load_start_tabs(urls, at) - - async def _load_start_tabs(self, urls: list[str], at: str) -> list[dict]: - """Its start tabs at ``urls``, as the control loads them, a tab on one of the env's sites waited on for as long - as the browser may take to be ready; their outcomes, kept as a launch's from ``at``.""" - sites = _read_sites() - tabs = await self.control.load_start_tabs(urls, lambda url: _site_of(url, sites), self._computer_app.timeout_s) + await self._load_start_tabs(urls, at, launched) + + async def _load_start_tabs(self, urls: list[str], at: str, since: Optional[float] = None) -> list[dict]: + """Its start tabs at ``urls``, as the control loads them, every one settled within the browser's ready timeout + of ``since``, its launch's start on the loop's clock, else of now: a tab on one of the env's sites waited on + what's left of it, any other on the live wait at most. A launch's setup counts against it, so its start tabs + settle before anything waiting that timeout and a margin on it gives up. Their outcomes, kept as a launch's + from ``at``.""" + sites, now = _read_sites(), asyncio.get_running_loop().time() + left = max(0.0, self._computer_app.timeout_s - (now - (since if since is not None else now))) + tabs = await self.control.load_start_tabs(urls, lambda url: _site_of(url, sites), left, + min(LIVE_START_TAB_TIMEOUT_S, left)) self.start_tab_launches.append({"at": at, "tabs": tabs}) return tabs diff --git a/packages/agentenv-protocol/tests/test_browser_data.py b/packages/agentenv-protocol/tests/test_browser_data.py index 87f3f6c24..aa1f40ae4 100644 --- a/packages/agentenv-protocol/tests/test_browser_data.py +++ b/packages/agentenv-protocol/tests/test_browser_data.py @@ -68,7 +68,8 @@ async def set_local_storage(self, origins): async def replace_tabs(self, tabs): await self._done("tabs", [(tab.url, tab.active) for tab in tabs]) - async def load_start_tabs(self, urls, site_of, own_timeout_s): + async def load_start_tabs(self, urls, site_of, own_timeout_s, live_timeout_s): + self.waits = (own_timeout_s, live_timeout_s) await self._done("start_tabs", urls) return [{"tab": f"t{index}", "url": url, "final_url": url, "status": 503, "error": None, "load_ms": 40, "site": site_of(url), "outcome": self.start_tabs_outcome} for index, url in enumerate(urls)] @@ -299,6 +300,18 @@ async def test_a_load_whose_start_tab_on_an_env_site_doesnt_load_fails_naming_it assert browser.start_tab_launches[-1]["tabs"][0]["site"] == ("shop" if refused else None) +@pytest.mark.asyncio +@pytest.mark.parametrize("set_up_s, waits", [(None, (60, 30)), (45, (15, 15)), (75, (0, 0))], + ids=["a-reopen", "a-launch-whose-setup-took-45-s", "a-launch-past-its-timeout"]) +async def test_start_tabs_settle_within_the_ready_timeout_of_their_launch_its_setup_counted(browser, set_up_s, waits): + loop = asyncio.get_running_loop() + + await browser._load_start_tabs(["https://shop.test/"], "t", None if set_up_s is None else loop.time() - set_up_s) + + own, live = browser.control.waits + assert (round(own), round(live)) == waits + + @pytest.mark.asyncio async def test_after_a_reset_relaunches_it_a_start_tab_on_an_env_site_that_failed_fails_the_reset(browser): def launched(url, site, outcome, **why):