From 4dd5308b63ab30b416197aa204dd799042ee00fe Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Tue, 27 Oct 2026 15:29:26 -0700 Subject: [PATCH 1/4] feat(website_browser): record the browser with computer-recording/v1 (FD-4497) The website-browser image runs an idle recorder beside Playwright MCP. It serves a card declaring computer-recording/v1, and on start streams the agent's page (DevTools screencast) into the protocol's Recordings, which encodes, uploads and seals it as for a computer env. The gateway reads the browser's card from the recorder's port (MCPServerConfig.card_port); an image without the recorder is omitted from the card as before. start_computer_recording takes an optional environment_name to target a child env of the deployed one. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/multi_env.py | 4 +- .../env/envs/website_browser/Dockerfile | 14 +- .../env/envs/website_browser/__init__.py | 4 + .../env/envs/website_browser/entrypoint.sh | 10 + .../env/envs/website_browser/recorder.py | 304 ++++++++++++++++++ .../env_providers/env_gateway_provider.py | 5 +- .../task_steps/computer_recording.py | 39 ++- .../env/envs/test_website_browser_recorder.py | 208 ++++++++++++ .../env_gateway_provider_test.py | 23 ++ tst/unit/task_step/computer_recording_test.py | 40 +++ 10 files changed, 632 insertions(+), 19 deletions(-) create mode 100644 src/agent_env/env/envs/website_browser/recorder.py create mode 100644 tst/unit/env/envs/test_website_browser_recorder.py diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index ec163dc50..b157834d4 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -39,6 +39,7 @@ def _snapshot_after_load_default() -> bool: ) from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.website import WebsiteEnv +from agent_env.env.envs.website_browser import WEBSITE_BROWSER_RECORDER_PORT from agent_env.store.ids import derive_id from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER from agent_env.attribution import Attribution @@ -615,7 +616,8 @@ async def _load_from_snapshot(self, snapshot) -> None: ] if website_configs: website_browser_env = Env.get(config.default_website_browser_env_id) - mcp_servers = list(mcp_servers) + [MCPServerConfig(image=website_browser_env.docker_image_artifact.image_name, environment_name=website_browser_env.environment_name)] + mcp_servers = list(mcp_servers) + [MCPServerConfig(image=website_browser_env.docker_image_artifact.image_name, environment_name=website_browser_env.environment_name, + card_port=WEBSITE_BROWSER_RECORDER_PORT)] compose_content = gw.create_docker_compose( mcp_servers=mcp_servers, diff --git a/src/agent_env/env/envs/website_browser/Dockerfile b/src/agent_env/env/envs/website_browser/Dockerfile index 83b24f35d..c9f9c9e06 100644 --- a/src/agent_env/env/envs/website_browser/Dockerfile +++ b/src/agent_env/env/envs/website_browser/Dockerfile @@ -1,9 +1,10 @@ FROM public.ecr.aws/docker/library/node:22-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d058f9dff4 # Chromium runtime dependencies for headless mode, and python3 for the socket probe the gateway's compose -# health-checks every MCP server with. +# health-checks every MCP server with and for the recorder. RUN apt-get update && apt-get install -y --no-install-recommends \ - python3-minimal \ + python3 \ + python3-venv \ libnss3 \ libnspr4 \ libatk1.0-0 \ @@ -38,7 +39,14 @@ RUN npm init -y && \ npm install @playwright/mcp@${PLAYWRIGHT_MCP_VERSION} && \ npx playwright install chromium -COPY entrypoint.sh /app/entrypoint.sh +# The recorder (recorder.py): computer-recording/v1 of the browser's page, idle until a recording starts. +ARG AGENTENV_PROTOCOL_VERSION=0.1.382 +ARG WEBSOCKETS_VERSION=15.0.1 +RUN python3 -m venv /opt/recorder && \ + /opt/recorder/bin/pip install --no-cache-dir \ + "agentenv-framework-protocol[computer]==${AGENTENV_PROTOCOL_VERSION}" "websockets==${WEBSOCKETS_VERSION}" + +COPY recorder.py entrypoint.sh /app/ RUN chmod +x /app/entrypoint.sh CMD ["/app/entrypoint.sh"] diff --git a/src/agent_env/env/envs/website_browser/__init__.py b/src/agent_env/env/envs/website_browser/__init__.py index ef99d7b18..f355e25da 100644 --- a/src/agent_env/env/envs/website_browser/__init__.py +++ b/src/agent_env/env/envs/website_browser/__init__.py @@ -6,7 +6,11 @@ # The environment_name used when creating MCPServerEnv. WEBSITE_BROWSER_ENVIRONMENT_NAME = "website-browser" +# The port the image's recorder serves the browser's env card on (recorder.py, entrypoint.sh's RECORDER_PORT). +WEBSITE_BROWSER_RECORDER_PORT = 18766 + __all__ = [ "WEBSITE_BROWSER_IMAGE_TAG", "WEBSITE_BROWSER_ENVIRONMENT_NAME", + "WEBSITE_BROWSER_RECORDER_PORT", ] diff --git a/src/agent_env/env/envs/website_browser/entrypoint.sh b/src/agent_env/env/envs/website_browser/entrypoint.sh index 912a2736d..337885849 100644 --- a/src/agent_env/env/envs/website_browser/entrypoint.sh +++ b/src/agent_env/env/envs/website_browser/entrypoint.sh @@ -3,6 +3,16 @@ set -euo pipefail # MCP_PORT and MCP_HOST are injected by docker-compose (gateway convention). # PLAYWRIGHT_MCP_VERSION is baked into the image at build time. + +# The recorder serves the browser's card on RECORDER_PORT, restarted whenever it exits; the MCP server starts once +# it listens, so the card is there by the time the server is healthy, or after RECORDER_WAIT_S without it. +export RECORDER_PORT="${RECORDER_PORT:-18766}" +(while true; do /opt/recorder/bin/python /app/recorder.py || true; sleep 1; done) & +for _ in $(seq 1 "$(( ${RECORDER_WAIT_S:-20} * 10 ))"); do + (exec 3<>"/dev/tcp/127.0.0.1/${RECORDER_PORT}") 2>/dev/null && break + sleep 0.1 +done + exec npx @playwright/mcp@${PLAYWRIGHT_MCP_VERSION} \ --browser chromium \ --headless \ diff --git a/src/agent_env/env/envs/website_browser/recorder.py b/src/agent_env/env/envs/website_browser/recorder.py new file mode 100644 index 000000000..ce519e09c --- /dev/null +++ b/src/agent_env/env/envs/website_browser/recorder.py @@ -0,0 +1,304 @@ +"""The website browser's recorder: computer-recording/v1 of the page Playwright MCP drives, from Chrome DevTools +screencast frames, on a card and port of its own beside the MCP server. It is idle until a recording starts, which +finds the browser through the remote debugging port Playwright opens for an ``--isolated`` Chromium and follows the +page the agent is on. It runs standalone in the website browser's image, so it imports only +``agentenv-framework-protocol[computer]`` and, to talk to the browser, ``websockets``.""" +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import io +import itertools +import json +import logging +import os +from collections import OrderedDict +from collections.abc import AsyncIterator, Callable +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, AsyncContextManager, Optional + +import httpx +from PIL import Image as PILImage + +from agentenv_protocol.agent_env_environment import ( + AgentEnvEnvironment, + AgentEnvStarletteApplication, + environment_card, + extension, +) +from agentenv_protocol.computer.card import COMPUTER_RECORDING +from agentenv_protocol.computer.recording import RecordedFrame, RecordHints, Recording, Recordings, recording_params + +logger = logging.getLogger("website-browser-recorder") + +#: The port the recorder serves its card on, unless RECORDER_PORT says another; the gateway reads the browser's card +#: there (agent_env.env.envs.website_browser.WEBSITE_BROWSER_RECORDER_PORT). +DEFAULT_PORT = 18766 +#: How often the browser is looked for while a recording has none. +FIND_S = 0.5 +#: A page that hasn't changed is filmed again this often, so a still page doesn't split the recording at a gap. +HOLD_S = 1.0 +#: How long one DevTools command may take. +CALL_S = 10.0 +JPEG_QUALITY = 80 + + +class CdpError(Exception): + """A DevTools command the browser refused, or one it can no longer answer.""" + + +class Cdp: + """One DevTools connection to the browser: commands answered by id, and everything else, in order, as events.""" + + def __init__(self, ws: Any) -> None: + self._ws = ws + self._ids = itertools.count(1) + self._waiting: dict[int, asyncio.Future] = {} + self._events: asyncio.Queue = asyncio.Queue() + + async def call(self, method: str, params: Optional[dict] = None, session: Optional[str] = None) -> dict: + request_id = next(self._ids) + answer = asyncio.get_running_loop().create_future() + self._waiting[request_id] = answer + message = {"id": request_id, "method": method, "params": params or {}} + if session is not None: + message["sessionId"] = session + try: + await self._ws.send(json.dumps(message)) + return await asyncio.wait_for(answer, CALL_S) + finally: + self._waiting.pop(request_id, None) + + async def events(self) -> AsyncIterator[tuple[str, dict, Optional[str]]]: + """(method, params, session) of each event, until the connection closes.""" + while (event := await self._events.get()) is not None: + yield event + + async def read(self) -> None: + """Hand each message to its command or the events, until the connection closes.""" + try: + async for raw in self._ws: + message = json.loads(raw) + if "id" in message: + answer = self._waiting.get(message["id"]) + if answer is not None and not answer.done(): + if "error" in message: + answer.set_exception(CdpError(message["error"].get("message", message["error"]))) + else: + answer.set_result(message.get("result") or {}) + elif "method" in message: + self._events.put_nowait((message["method"], message.get("params") or {}, message.get("sessionId"))) + finally: + for answer in self._waiting.values(): + if not answer.done(): + answer.set_exception(CdpError("the browser closed the connection")) + self._events.put_nowait(None) + + +@contextlib.asynccontextmanager +async def connect_cdp(port: int) -> AsyncIterator[Cdp]: + """A DevTools connection to the browser debugging on ``port`` of this host.""" + from websockets.asyncio.client import connect + + async with httpx.AsyncClient() as client: + version = await client.get(f"http://127.0.0.1:{port}/json/version", timeout=CALL_S) + url = version.raise_for_status().json()["webSocketDebuggerUrl"] + async with connect(url, max_size=None, ping_interval=None, compression=None, open_timeout=CALL_S) as ws: + cdp = Cdp(ws) + reader = asyncio.create_task(cdp.read()) + try: + yield cdp + finally: + reader.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await reader + + +def find_browser(proc: Path = Path("/proc")) -> Optional[int]: + """The remote debugging port of the newest Chromium browser process here, or None while there is none.""" + newest: Optional[tuple[int, int]] = None + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + argv = (entry / "cmdline").read_bytes().split(b"\0") + started = int((entry / "stat").read_text().rsplit(")", 1)[1].split()[19]) + except (OSError, ValueError, IndexError): + continue + if any(arg.startswith(b"--type=") for arg in argv): # a renderer, GPU or utility process + continue + port = next((arg.split(b"=", 1)[1] for arg in argv if arg.startswith(b"--remote-debugging-port=")), b"") + if port.isdigit() and int(port) > 0 and (newest is None or started > newest[0]): + newest = (started, int(port)) + return newest[1] if newest else None + + +class Screencast: + """The screencast of the page the agent is on, in the newest browser: the page opened last, or the last one to + navigate. ``latest`` is its last frame as JPEG, or None while there is no page.""" + + def __init__(self, max_edge: Optional[int], find: Callable[[], Optional[int]], + connect: Callable[[int], AsyncContextManager[Cdp]]) -> None: + self.max_edge = max_edge + self.latest: Optional[bytes] = None + self.frames = 0 + self.page: Optional[str] = None + self._find, self._connect = find, connect + self._session: Optional[str] = None + self._pages: OrderedDict[str, str] = OrderedDict() + + async def run(self) -> None: + """Watch the browser, and the next one when it goes, until cancelled.""" + while True: + port = await asyncio.to_thread(self._find) + if port is not None: + try: + async with self._connect(port) as cdp: + await self.watch(cdp) + except (OSError, CdpError, httpx.HTTPError, KeyError, ValueError) as e: + logger.info("the browser on port %d: %s: %s", port, type(e).__name__, e) + except Exception as e: # the websockets library's own errors, a closed connection among them + logger.warning("the browser on port %d went: %s: %s", port, type(e).__name__, e) + self.latest, self.page, self._session = None, None, None + await asyncio.sleep(FIND_S) + + async def watch(self, cdp: Cdp) -> None: + """Follow the browser's pages over ``cdp`` until it closes.""" + self._pages.clear() + await cdp.call("Target.setDiscoverTargets", {"discover": True}) # announces each page there already is + async for method, params, session in cdp.events(): + await self._event(cdp, method, params, session) + + async def _event(self, cdp: Cdp, method: str, params: dict, session: Optional[str]) -> None: + if method == "Page.screencastFrame": + if session is not None and session == self._session: + self.latest, self.frames = base64.b64decode(params["data"]), self.frames + 1 + with contextlib.suppress(CdpError, asyncio.TimeoutError): + await cdp.call("Page.screencastFrameAck", {"sessionId": params["sessionId"]}, session) + elif method in ("Target.targetCreated", "Target.targetInfoChanged"): + info = params.get("targetInfo") or {} + target, url = info.get("targetId"), info.get("url") + if info.get("type") != "page" or not target: + return + if method == "Target.targetCreated" or self._pages.get(target) != url: + self._pages[target] = url + self._pages.move_to_end(target) + await self._show(cdp, target) + elif method == "Target.targetDestroyed": + target = params.get("targetId") + if self._pages.pop(target, None) is not None and target == self.page: + await self._show(cdp, next(reversed(self._pages), None)) + elif method == "Target.detachedFromTarget" and params.get("sessionId") == self._session: + self.page, self._session = None, None + await self._show(cdp, next(reversed(self._pages), None)) + + async def _show(self, cdp: Cdp, target: Optional[str]) -> None: + """Screencast ``target`` instead; the last frame stays until its first arrives.""" + if target == self.page and self._session is not None: + return + old, self.page, self._session = self._session, target, None + if old is not None: + with contextlib.suppress(CdpError, asyncio.TimeoutError): + await cdp.call("Target.detachFromTarget", {"sessionId": old}) + if target is None: + self.latest = None + return + options: dict[str, Any] = {"format": "jpeg", "quality": JPEG_QUALITY, "everyNthFrame": 1} + if self.max_edge is not None: + options.update(maxWidth=self.max_edge, maxHeight=self.max_edge) + try: + session = (await cdp.call("Target.attachToTarget", {"targetId": target, "flatten": True}))["sessionId"] + await cdp.call("Page.startScreencast", options, session) + except (CdpError, asyncio.TimeoutError, KeyError) as e: # the page went meanwhile + logger.info("screencasting page %s: %s", target, e) + if self.page == target: + self.page = None + return + if self.page == target: + self._session = session + + +@environment_card(name="website-browser") +class BrowserRecorder(AgentEnvEnvironment): + """computer-recording/v1 of the website browser's page; the browser's tools stay Playwright MCP's.""" + + recording = Recording() + + def __init__(self, find: Callable[[], Optional[int]] = find_browser, + connect: Callable[[int], AsyncContextManager[Cdp]] = connect_cdp) -> None: + self._find, self._connect = find, connect + self.recordings = Recordings(_Host(self)) + + def extension_params(self, uri: str) -> dict: + if uri == COMPUTER_RECORDING: + return {**recording_params(self.recording, streams=True), "cursor": False} + return {} + + @extension(COMPUTER_RECORDING, op="start", + description="Record the browser's page: start returns a recording whose segments upload through its " + "grant as they close, and end seals it and returns its manifest; repeats answer the same.") + async def _recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, + fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, + upload: Optional[dict] = None) -> dict: + return await self.recordings.start(request_id, scope, window, fps, max_edge, False, upload) + + @extension(COMPUTER_RECORDING, op="end") + async def _recording_end(self, recording_id: str, upload: Optional[dict] = None, reason: str = "end") -> dict: + return await self.recordings.end(recording_id, upload, reason) + + async def frames(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: + """The page at up to ``fps``: a frame each time it changes, and the last one again after ``HOLD_S``.""" + screencast = Screencast(hints.max_edge, self._find, self._connect) + watching = asyncio.create_task(screencast.run()) + loop = asyncio.get_running_loop() + interval, due = 1 / hints.fps, loop.time() + shown: Optional[bytes] = None + shown_at = 0.0 + try: + while True: + due = max(due + interval, loop.time()) + await asyncio.sleep(due - loop.time()) + if watching.done(): + watching.result() + frame = screencast.latest + if frame is not None and (frame is not shown or loop.time() - shown_at >= HOLD_S): + shown, shown_at = frame, loop.time() + yield RecordedFrame(PILImage.open(io.BytesIO(frame)), datetime.now(timezone.utc)) + finally: + watching.cancel() + + +class _Host: + """What Recordings reads of a computer's runtime: the limits, and a stream to record from.""" + + def __init__(self, recorder: BrowserRecorder) -> None: + self.driver = recorder + + def has(self, capability: str) -> bool: + return capability == "record" + + def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: + return self.driver.frames(hints) + + +def create_app(recorder: BrowserRecorder) -> Any: + from starlette.applications import Starlette + + app = Starlette() + AgentEnvStarletteApplication(recorder._build_card(), recorder).add_routes_to_app(app) + return app + + +def main() -> None: + import uvicorn + + logging.basicConfig(level=logging.INFO, format="%(asctime)s recorder %(levelname)s %(message)s") + logging.getLogger("httpx").setLevel(logging.WARNING) # a request each time it finds a browser + uvicorn.run(create_app(BrowserRecorder()), host=os.environ.get("MCP_HOST", "0.0.0.0"), + port=int(os.environ.get("RECORDER_PORT", DEFAULT_PORT)), log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 1ae581b74..761976edf 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -34,6 +34,7 @@ DB_WEB_PORT, PGWEB_SERVICE_NAME, ) +from agent_env.env.envs.website_browser import WEBSITE_BROWSER_RECORDER_PORT from agent_env.providers.env_providers.constants import ( AGENT_ENV_WEBSITE_BACKEND_PORT, AGENT_ENV_WEBSITE_BACKEND_SUFFIX, @@ -88,6 +89,7 @@ class MCPServerConfig: image: str environment_name: str extra_env_vars: dict[str, str] | None = None + card_port: int | None = None # where its env card is served, when not on the MCP port (the website browser's recorder) @dataclass @@ -452,7 +454,7 @@ def create_docker_compose( rest_proxy_urls = [] for server in mcp_servers: rest_proxy_urls.append( - f"mcp-{server.environment_name}=http://{server.environment_name}:{gateway_port}" + f"mcp-{server.environment_name}=http://{server.environment_name}:{server.card_port or gateway_port}" ) if website_configs: website_urls = ",".join( @@ -728,6 +730,7 @@ async def _deploy_via_vm( browser_server = MCPServerConfig( image=website_browser_env.docker_image_artifact.image_name, environment_name=website_browser_env.environment_name, + card_port=WEBSITE_BROWSER_RECORDER_PORT, ) mcp_servers = list(mcp_servers) + [browser_server] mcp_server_images = list(mcp_server_images) + [website_browser_env.docker_image_artifact] diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index ffc9a850a..628ac2552 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -1,5 +1,6 @@ -"""Record a computer env's display. ``start_computer_recording`` starts a recording on a deployed env whose card -declares computer-recording/v1, with a grant to upload its segments under a namespace of its own; +"""Record a computer env's display. ``start_computer_recording`` starts a recording on a deployed env whose card, or +whose child env's card (``environment_name``), declares computer-recording/v1, with a grant to upload its segments +under a namespace of its own; ``end_computer_recording`` ends it and registers its segments and manifest as one FileArtifactUniverse. Teardown ends whatever a run left open (``end_open_recordings``).""" from __future__ import annotations @@ -39,29 +40,35 @@ class StartComputerRecordingTaskStep(TaskStep): - """Start recording a deployed env's display, under ``name``. Best effort by default.""" + """Start recording a deployed env's display, or its child env ``environment_name``'s, under ``name``. Best effort + by default.""" type: ClassVar[str] = "start_computer_recording" entity_refs = (EntityRef.env("env_id"),) def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: Optional[int] = None, - max_edge: Optional[int] = None, cursor: bool = True, + max_edge: Optional[int] = None, cursor: bool = True, environment_name: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) if not env_id or not name: raise ValueError("start_computer_recording needs an env_id and a name for the recording") self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor + self.environment_name = environment_name def to_dict(self) -> dict: - return {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, + data = {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor} + if self.environment_name is not None: # left out otherwise, so a step without one serializes as before + data["environment_name"] = self.environment_name + return data @classmethod def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: base = cls._base_from_dict(data) base["fail_task_on_error"] = data.get("fail_task_on_error", False) return cls(**base, env_id=data["env_id"], name=data["name"], fps=data.get("fps"), - max_edge=data.get("max_edge"), cursor=data.get("cursor", True)) + max_edge=data.get("max_edge"), cursor=data.get("cursor", True), + environment_name=data.get("environment_name")) async def execute(self, context: TaskStepContext) -> TaskStepContext: recordings = context.metadata.setdefault(RECORDINGS_KEY, {}) @@ -77,10 +84,10 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: try: deployed = _deployed(context, self.env_id) namespace = _namespace(request_id) - grant = _grant(deployed, namespace) + grant = _grant(deployed, namespace, self.environment_name) started = await deployed.invoke(COMPUTER_RECORDING, "start", { "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, - "upload": grant}) + "upload": grant}, environment_name=self.environment_name) except Exception as e: recordings[self.name] = {"env_id": self.env_id, "state": "failed", "error": f"{type(e).__name__}: {e}"[:500]} logger.warning("start_computer_recording %s on %s: %s", self.name, self.env_id, e) @@ -90,6 +97,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: recordings[self.name] = { "env_id": self.env_id, "recording_id": started["recording_id"], "started_at": started["started_at"], "namespace": namespace, "state": "open"} + if self.environment_name is not None: + recordings[self.name]["environment_name"] = self.environment_name logger.info("start_computer_recording %s on %s: %s", self.name, self.env_id, started["recording_id"]) return context @@ -149,9 +158,10 @@ async def end_recording(context: TaskStepContext, name: str, reason: str) -> Non if entry.get("state") != "open": return try: - deployed = _deployed(context, entry["env_id"]) + deployed, environment_name = _deployed(context, entry["env_id"]), entry.get("environment_name") manifest = RecordingManifest.model_validate(await deployed.invoke(COMPUTER_RECORDING, "end", { - "recording_id": entry["recording_id"], "upload": _grant(deployed, entry["namespace"]), "reason": reason})) + "recording_id": entry["recording_id"], "upload": _grant(deployed, entry["namespace"], environment_name), + "reason": reason}, environment_name=environment_name)) except Exception as e: logger.warning("end_computer_recording %s: %s; registering what its namespace holds", name, e) entry["error"] = f"{type(e).__name__}: {e}"[:500] @@ -187,10 +197,11 @@ def _namespace(request_id: str) -> str: return config.get_object_store().object_url(f"{config.get_artifact_key_prefix()}{_KEY_PREFIX}/{key}") -def _grant(deployed, namespace: str) -> dict: - """A grant that uploads under ``namespace``, lasting the env's ``max_duration_s`` and a margin, or the longest - the store issues.""" - params = (find_extension(deployed.environment_card or {}, COMPUTER_RECORDING) or {}).get("params") or {} +def _grant(deployed, namespace: str, environment_name: Optional[str] = None) -> dict: + """A grant that uploads under ``namespace``, lasting the recorded env's ``max_duration_s`` and a margin, or the + longest the store issues.""" + card = deployed.environment_card if environment_name is None else deployed.get_child_env_card(environment_name) + params = (find_extension(card or {}, COMPUTER_RECORDING) or {}).get("params") or {} store = get_config().get_object_store() sandbox_type = getattr(deployed, "sandbox_type", None) if not issues_grants_to(store, sandbox_type): diff --git a/tst/unit/env/envs/test_website_browser_recorder.py b/tst/unit/env/envs/test_website_browser_recorder.py new file mode 100644 index 000000000..6c56dfafa --- /dev/null +++ b/tst/unit/env/envs/test_website_browser_recorder.py @@ -0,0 +1,208 @@ +"""The website browser's recorder: it finds the browser Playwright MCP launched, follows the page the agent is on +through DevTools target events, holds a still page's frame so the recording doesn't split, and serves +computer-recording/v1 on a card of its own that the gateway nests as the website-browser child env.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import itertools +import re +from pathlib import Path + +import httpx +import pytest + +av = pytest.importorskip("av") + +from PIL import Image as PILImage # noqa: E402 +from agentenv_protocol import client as protocol_client # noqa: E402 +from agentenv_protocol.computer.card import COMPUTER_RECORDING # noqa: E402 +from agentenv_protocol.computer.recording import RecordHints # noqa: E402 + +from agent_env.env.envs import website_browser # noqa: E402 +from agent_env.env.envs.website_browser import recorder # noqa: E402 +from agent_env.env.gateway.constants import GATEWAY_EXTENSIONS # noqa: E402 +from agent_env.env.gateway.rest_proxy import rewrite_child_card # noqa: E402 + +START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +def _jpeg(color: tuple, size: tuple = (320, 200)) -> str: + out = io.BytesIO() + PILImage.new("RGB", size, color).save(out, "JPEG") + return base64.b64encode(out.getvalue()).decode() + + +class _Browser: + """A DevTools connection standing in for Chromium's: each attach is a new session, and what's emitted is read in + order until ``close``.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, dict, str | None]] = [] + self._events: asyncio.Queue = asyncio.Queue() + self._sessions = itertools.count(1) + + async def call(self, method, params=None, session=None): + self.calls.append((method, params or {}, session)) + if method == "Target.attachToTarget": + return {"sessionId": f"{params['targetId']}-{next(self._sessions)}"} + return {} + + async def events(self): + while (event := await self._events.get()) is not None: + yield event + + def emit(self, method: str, params: dict, session: str | None = None) -> None: + self._events.put_nowait((method, params, session)) + + def page(self, target: str, url: str, created: bool = True, **info) -> None: + method = "Target.targetCreated" if created else "Target.targetInfoChanged" + self.emit(method, {"targetInfo": {"targetId": target, "type": "page", "url": url, **info}}) + + def frame(self, session: str, data: str) -> None: + self.emit("Page.screencastFrame", {"data": data, "sessionId": 7, "metadata": {}}, session) + + def close(self) -> None: + self._events.put_nowait(None) + + def attached(self) -> list[str]: + return [params["targetId"] for method, params, _ in self.calls if method == "Target.attachToTarget"] + + +def _proc(root: Path, pid: int, argv: list[str], started: int | None) -> None: + entry = root / str(pid) + entry.mkdir() + (entry / "cmdline").write_bytes(b"\0".join(a.encode() for a in argv) + b"\0") + if started is not None: + (entry / "stat").write_text(f"{pid} (chrome) S " + "0 " * 18 + f"{started} 0 0") + + +def test_the_browser_is_the_newest_chromium_with_a_debugging_port(tmp_path): + _proc(tmp_path, 1, ["node", "cli.js", "--port", "18765"], 10) + _proc(tmp_path, 40, ["chrome", "--remote-debugging-port=1111", "--headless"], 100) + _proc(tmp_path, 41, ["chrome", "--type=renderer", "--remote-debugging-port=3333"], 300) + _proc(tmp_path, 50, ["chrome", "--headless", "--remote-debugging-port=2222"], 200) + _proc(tmp_path, 51, ["chrome", "--remote-debugging-port=0"], 400) + _proc(tmp_path, 60, ["chrome", "--remote-debugging-port=4444"], None) # exited while it was read + (tmp_path / "self").mkdir() + assert recorder.find_browser(tmp_path) == 2222 + assert recorder.find_browser(tmp_path / "1") is None + + +@pytest.mark.asyncio +async def test_the_screencast_follows_new_and_navigated_pages_and_falls_back_when_one_closes(): + browser, red, blue = _Browser(), _jpeg((255, 0, 0)), _jpeg((0, 0, 255)) + screencast = recorder.Screencast(640, find=lambda: None, connect=None) + browser.page("A", "about:blank") + browser.emit("Target.targetCreated", {"targetInfo": {"targetId": "F", "type": "iframe", "url": "x"}}) + browser.frame("A-1", red) + browser.page("B", "about:blank", openerId="A") + browser.frame("A-1", blue) # A's last frame, after the switch: acknowledged, not shown + watching = asyncio.create_task(screencast.watch(browser)) + await asyncio.sleep(0.05) + assert (screencast.page, screencast.latest, screencast.frames) == ("B", base64.b64decode(red), 1) + + browser.frame("B-2", blue) + browser.page("B", "about:blank", created=False, attached=True) # nothing about it changed that matters + browser.page("A", "https://shop.test/cart", created=False) + await asyncio.sleep(0.05) + assert (screencast.page, screencast.latest) == ("A", base64.b64decode(blue)) + + browser.emit("Target.targetDestroyed", {"targetId": "A"}) + await asyncio.sleep(0.05) + assert screencast.page == "B" + browser.emit("Target.targetDestroyed", {"targetId": "B"}) + browser.close() + await watching + + assert browser.attached() == ["A", "B", "A", "B"] + assert (screencast.page, screencast.latest) == (None, None) + started = [(params, session) for method, params, session in browser.calls if method == "Page.startScreencast"] + assert started[0] == ({"format": "jpeg", "quality": 80, "everyNthFrame": 1, "maxWidth": 640, "maxHeight": 640}, + "A-1") + assert [session for method, _, session in browser.calls if method == "Page.screencastFrameAck"] == ["A-1", "A-1", + "B-2"] + assert [params["sessionId"] for method, params, _ in browser.calls if method == "Target.detachFromTarget"] == [ + "A-1", "B-2", "A-3", "B-4"] + + +def _recorder(browser: _Browser) -> recorder.BrowserRecorder: + class _Connection: + async def __aenter__(self): + return browser + + async def __aexit__(self, *exc): + return False + + return recorder.BrowserRecorder(find=lambda: 9222, connect=lambda port: _Connection()) + + +@pytest.mark.asyncio +async def test_a_still_page_is_filmed_again_after_the_hold_and_nothing_is_filmed_without_a_page(monkeypatch, home): + monkeypatch.setattr(recorder, "HOLD_S", 0.2) + browser = _Browser() + frames = _recorder(browser).frames(RecordHints("display", 20, None, False, 15, home)) + first = asyncio.ensure_future(anext(frames)) + await asyncio.sleep(0.2) + assert not first.done() # no page yet + browser.page("A", "https://shop.test/") + browser.frame("A-1", _jpeg((0, 255, 0))) + one = await asyncio.wait_for(first, 1) + two = await asyncio.wait_for(anext(frames), 1) + assert 0.15 <= (two.captured_at - one.captured_at).total_seconds() < 0.5 + assert two.image.convert("RGB").getpixel((5, 5))[1] > 200 and two.image.size == (320, 200) + await frames.aclose() + + +@pytest.mark.asyncio +async def test_a_recording_through_the_card_is_an_mp4_of_the_page(home): + browser = _Browser() + app = recorder.create_app(_recorder(browser)) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://recorder") as http: + card = (await http.get("/.well-known/agent-env.json")).json() + started = (await http.post(START, json={"request_id": "run-1:start", "fps": 10, "cursor": True})).json() + browser.page("A", "https://shop.test/") + for shade in range(0, 250, 25): + browser.frame("A-1", _jpeg((shade, 0, 255 - shade))) + await asyncio.sleep(0.1) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + + assert card["name"] == "website-browser" and card["capabilities"]["operations"] == [] + assert [e["uri"] for e in card["capabilities"]["extensions"]] == [COMPUTER_RECORDING] + params = card["capabilities"]["extensions"][0]["params"] + assert params["cursor"] is False and params["source"] == "stream" and set(params["methods"]) == {"start", "end"} + assert (manifest["ended_by"], manifest["cursor"], manifest["source"]) == ("end", False, "stream") + (segment,) = manifest["segments"] + assert (segment["width"], segment["height"], segment["uploaded"]) == (320, 200, False) + with av.open(str(home / ".agentenv-recordings" / segment["path"])) as video: + decoded = list(video.decode(video=0)) + assert len(decoded) == segment["frames"] >= 8 and decoded[0].width == 320 + + +@pytest.mark.asyncio +async def test_the_gateway_nests_the_card_as_a_child_env_whose_recording_is_invoked_under_its_proxy(monkeypatch): + card = recorder.create_app(_recorder(_Browser())) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=card), base_url="http://recorder") as http: + served = (await http.get("/.well-known/agent-env.json")).json() + composed = {"name": "env", "capabilities": {"extensions": GATEWAY_EXTENSIONS}, + "children_environments": [rewrite_child_card("mcp-website-browser", served, GATEWAY_EXTENSIONS)]} + child = protocol_client.find_child(composed, website_browser.WEBSITE_BROWSER_ENVIRONMENT_NAME) + methods = protocol_client.find_extension(child, COMPUTER_RECORDING)["params"]["methods"] + assert {op: m["endpoint"] for op, m in methods.items()} == { + "start": "/svc/mcp-website-browser/agentenv/ext/computer-recording/start", + "end": "/svc/mcp-website-browser/agentenv/ext/computer-recording/end"} + + +def test_the_image_serves_the_recorder_on_the_port_the_gateway_reads_the_card_on(): + entrypoint = (Path(website_browser.__file__).parent / "entrypoint.sh").read_text() + default = re.search(r'RECORDER_PORT="\$\{RECORDER_PORT:-(\d+)\}"', entrypoint).group(1) + assert int(default) == recorder.DEFAULT_PORT == website_browser.WEBSITE_BROWSER_RECORDER_PORT + assert "--isolated" in entrypoint # the remote debugging port the recorder finds comes with it diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 38a9a5c0f..c162fa482 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -1075,6 +1075,27 @@ def test_compose_maps_extra_hosts_into_the_gateway(extra_hosts): assert mapped == ({GATEWAY_SERVICE_NAME: list(extra_hosts)} if extra_hosts else {}) +def test_compose_reads_a_servers_card_on_its_card_port_and_its_mcp_on_the_mcp_port(): + """The website browser serves its card from its recorder's port: the gateway's REST proxy, which it reads the card + through, points there, while its MCP connection and health check stay on the MCP port.""" + from agent_env.env.envs.service_db import ServiceDBConfig + + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[MCPServerConfig(image="mcp-slack", environment_name="slack"), + MCPServerConfig(image="browser", environment_name="website-browser", card_port=18766)], + gateway_image="agent-gateway", + gateway_port=18765, + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance(), + ) + + services = yaml.safe_load(compose)["services"] + gateway = dict(item.split("=", 1) for item in services[GATEWAY_SERVICE_NAME]["environment"]) + assert gateway["REST_PROXY_URLS"] == "mcp-slack=http://slack:18765,mcp-website-browser=http://website-browser:18766" + assert (gateway["INTERNAL_MCP_SERVERS"], gateway["INTERNAL_MCP_PORT"]) == ("slack,website-browser", "18765") + assert "18765" in services["website-browser"]["healthcheck"]["test"][1] + + # --- sidecar rendering -------------------------------------------------------- class _FakeImageArtifact: @@ -1328,6 +1349,8 @@ def _fake_env_get(env_id, *a, **k): names = state_provider.prepare.call_args.args[0] assert set(names) == {"slack", "shop", "website-browser"} assert compose.call_args.kwargs["mcp_server_name"] == "crm" + browser = next(s for s in compose.call_args.kwargs["mcp_servers"] if s.environment_name == "website-browser") + assert browser.card_port == 18766 # its card is the recorder's @pytest.mark.asyncio diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 978ad99b2..be2be1910 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -188,6 +188,9 @@ async def test_a_teardown_sandboxes_step_ends_the_recordings_of_its_envs_only(st def test_the_steps_round_trip_and_refuse_what_they_cant_run(): start = _start(fps=5, max_edge=1280, cursor=False, fail_task_on_error=True) assert StartComputerRecordingTaskStep.from_dict(start.to_dict()).to_dict() == start.to_dict() + assert "environment_name" not in start.to_dict() + child = _start(environment_name="website-browser") + assert StartComputerRecordingTaskStep.from_dict(child.to_dict()).environment_name == "website-browser" assert EndComputerRecordingTaskStep.from_dict(_end().to_dict()).to_dict() == _end().to_dict() assert StartComputerRecordingTaskStep.from_dict({"id": "s", "env_id": "e", "name": "n"}).fail_task_on_error is False with pytest.raises(ValueError, match="needs an env_id and a name"): @@ -273,3 +276,40 @@ async def broken(context, name, reason): await recording_steps.end_open_recordings(context) recordings = context.metadata[RECORDINGS_KEY] assert (recordings["a"]["state"], recordings["b"]["state"]) == ("failed", "ended") + + +class _Gateway(_Computer): + """A gateway env whose child env ``website-browser`` records, and whose own card declares no recording.""" + + def __init__(self, store) -> None: + super().__init__(store, env_id="shop") + self.environment_card = {"name": "shop", "capabilities": {"extensions": []}, + "children_environments": [{**_CARD, "name": "website-browser"}]} + self.children: list = [] + + def get_child_env_card(self, environment_name): + return next((c for c in self.environment_card["children_environments"] if c["name"] == environment_name), None) + + async def invoke(self, uri, method, params=None, *, environment_name=None, **kwargs): + self.children.append(environment_name) + return await super().invoke(uri, method, params, **kwargs) + + +@pytest.mark.asyncio +async def test_a_child_env_is_recorded_through_its_own_card_from_start_to_end(store): + gateway = _Gateway(store) + context = await _start(env_id="shop", environment_name="website-browser").execute(_context(gateway)) + assert context.metadata[RECORDINGS_KEY]["desk"]["environment_name"] == "website-browser" + entry = (await _end().execute(context)).metadata[RECORDINGS_KEY]["desk"] + assert entry["state"] == "ended" and gateway.children == ["website-browser", "website-browser"] + (_, start), (_, end) = gateway.calls + assert start["upload"]["root_path"] == end["upload"]["root_path"] + + +@pytest.mark.asyncio +async def test_a_start_without_a_child_env_reads_the_envs_own_card_as_before(store): + context = await _start(env_id="shop").execute(_context(_Gateway(store))) + entry = context.metadata[RECORDINGS_KEY]["desk"] + assert entry["state"] == "failed" and "max_duration_s" in entry["error"] # its own card declares no recording + context = await _start().execute(_context(_Computer(store))) + assert "environment_name" not in context.metadata[RECORDINGS_KEY]["desk"] From 9574c862896ab93b1d543906268f0d24e9b64ba9 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Tue, 27 Oct 2026 16:00:37 -0700 Subject: [PATCH 2/4] build(website_browser): the recorder takes a protocol floor, as the agent images do, not an exact pin (FD-4497) Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/website_browser/Dockerfile | 6 ++---- .../env/envs/website_browser/recorder-requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 src/agent_env/env/envs/website_browser/recorder-requirements.txt diff --git a/src/agent_env/env/envs/website_browser/Dockerfile b/src/agent_env/env/envs/website_browser/Dockerfile index c9f9c9e06..aa51f9ee3 100644 --- a/src/agent_env/env/envs/website_browser/Dockerfile +++ b/src/agent_env/env/envs/website_browser/Dockerfile @@ -40,11 +40,9 @@ RUN npm init -y && \ npx playwright install chromium # The recorder (recorder.py): computer-recording/v1 of the browser's page, idle until a recording starts. -ARG AGENTENV_PROTOCOL_VERSION=0.1.382 -ARG WEBSOCKETS_VERSION=15.0.1 +COPY recorder-requirements.txt /app/ RUN python3 -m venv /opt/recorder && \ - /opt/recorder/bin/pip install --no-cache-dir \ - "agentenv-framework-protocol[computer]==${AGENTENV_PROTOCOL_VERSION}" "websockets==${WEBSOCKETS_VERSION}" + /opt/recorder/bin/pip install --no-cache-dir -r /app/recorder-requirements.txt COPY recorder.py entrypoint.sh /app/ RUN chmod +x /app/entrypoint.sh diff --git a/src/agent_env/env/envs/website_browser/recorder-requirements.txt b/src/agent_env/env/envs/website_browser/recorder-requirements.txt new file mode 100644 index 000000000..57ae4a2e3 --- /dev/null +++ b/src/agent_env/env/envs/website_browser/recorder-requirements.txt @@ -0,0 +1,4 @@ +# The recorder installs the newest agentenv-framework-protocol at or above this floor, the first with computer-recording/v1; +# raise it when the recorder needs a newer protocol. +agentenv-framework-protocol[computer]>=0.1.382 +websockets==15.0.1 From a00e65919e835ad2ba9cd8d7a2c0752c8b716303 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Tue, 27 Oct 2026 16:51:40 -0700 Subject: [PATCH 3/4] build(release): build the website browser after the protocol is published, as its recorder installs it (FD-4497) A release builds the infra images before it publishes the protocol, and a failed infra image keeps both packages off PyPI, so an infra image that needs the protocol the release ships would block every release. The browser now builds in the after-protocol group with the agents' images, and the build retries while PyPI's index doesn't yet list a protocol floor, as it does for a pin. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 4 ++-- .github/scripts/infra_images.py | 11 ++++++----- tst/unit/test_infra_build_script.py | 10 ++++++++++ tst/unit/test_infra_images_script.py | 9 +++++---- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 4826777d2..b89dfe608 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -50,8 +50,8 @@ # Docker Hub send. _RATE_LIMITED = re.compile(r"429 Too Many Requests|toomanyrequests", re.IGNORECASE) # What pip in an agent image's build answers for the agentenv-framework-protocol its release published moments earlier, -# until PyPI's index, which a CDN caches, lists it. -_NOT_YET_SERVED = re.compile(r"No matching distribution found for agentenv-framework-protocol(\[[^\]]*\])?==") +# until PyPI's index, which a CDN caches, lists it: pinned (an agent's), or a floor it names (the browser's recorder). +_NOT_YET_SERVED = re.compile(r"No matching distribution found for agentenv-framework-protocol(\[[^\]]*\])?(==|>=)") # How long to wait before each try again, so a burst of builds starting together spreads out. _RETRY_WAITS_SECONDS = (30, 60, 120) diff --git a/.github/scripts/infra_images.py b/.github/scripts/infra_images.py index 2df8146c5..6d2c52ae6 100644 --- a/.github/scripts/infra_images.py +++ b/.github/scripts/infra_images.py @@ -5,8 +5,8 @@ ``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, ``--missing`` only those the registry hasn't published: it has no image under the tag, or refuses to show one, as GHCR does for a package that doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, and ``--only agents`` -only the agents' images, which a release builds after it publishes the protocol, as an agent's protocol floor may name -it. ``write PATH`` records the digest the registry serves under every +only the images that install the protocol, the agents' and the website browser's, which a release builds after it +publishes the protocol, as their protocol floor may name it. ``write PATH`` records the digest the registry serves under every image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously otherwise, and always anonymously by ``public``. @@ -26,9 +26,10 @@ from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest REPO = Path(__file__).resolve().parents[2] -# "agents" is the shipped bundles' images, the agents' and an example env's, which install the protocol a release -# publishes. -_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in BUNDLE_IMAGES), "agents": BUNDLE_IMAGES} +# "agents" is the images that install the protocol a release publishes: the shipped bundles', the agents' and an +# example env's, and the website browser's, whose recorder installs it. +_AFTER_PROTOCOL = BUNDLE_IMAGES + tuple(image for image in PUBLISHED_IMAGES if image.name == "agentenv-website-browser") +_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL), "agents": _AFTER_PROTOCOL} def main() -> int: diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index ca0fafc39..160e54464 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -279,6 +279,16 @@ def test_retry_tries_again_while_pypi_doesnt_list_the_protocol_a_build_installs( in capsys.readouterr().out) +def test_retry_tries_again_while_pypi_doesnt_list_the_protocol_floor_a_build_names(script, monkeypatch, capsys): + """The website browser's recorder names a protocol floor, which its release may be the first to publish.""" + missing = "ERROR: No matching distribution found for agentenv-framework-protocol[computer]>=0.1.382\n" + runs, waits = _attempts(script, monkeypatch, [(missing, 1), ("#13 DONE 4.1s\n", 0)]) + + assert _run(script, monkeypatch, "retry", "docker", "build", ".") == 0 + + assert len(runs) == 2 and waits == [script._RETRY_WAITS_SECONDS[0]] + + def test_retry_doesnt_try_again_a_build_missing_another_package(script, monkeypatch, capsys): runs, waits = _attempts(script, monkeypatch, [("ERROR: No matching distribution found for nonexistent-pkg==1.0\n", 1)]) diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 97c549040..22fb1969b 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -51,16 +51,17 @@ def test_plan_lists_every_image_with_its_checkout_paths_and_tag(script, monkeypa def test_plan_only_lists_the_infra_images_or_the_agents_images(script, monkeypatch, capsys): - """The release builds the infra images before it publishes the protocol, and the bundles', which install it, - after.""" + """The release builds the infra images before it publishes the protocol, and the images that install it, the + bundles' and the website browser's, after.""" planned = {} for only in ("infra", "agents"): assert _run(script, monkeypatch, "plan", "--only", only) == 0 planned[only] = [entry["name"] for entry in json.loads(capsys.readouterr().out)] bundles = [image.name for image in BUNDLE_IMAGES] - assert planned == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in bundles], - "agents": bundles} + after = [*bundles, "agentenv-website-browser"] # the browser's recorder installs the protocol too + assert planned == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in after], + "agents": after} assert "agentenv-gateway" in planned["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-mail-mcp"] From f811cbc203a4dc5706c72ad827caed941ae579fe Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Tue, 27 Oct 2026 16:59:49 -0700 Subject: [PATCH 4/4] fix(website_browser): a page that closes before its screencast starts hands back to the newest page left (FD-4497) Co-Authored-By: Claude Opus 5.5 --- .../env/envs/website_browser/recorder.py | 4 +++- .../env/envs/test_website_browser_recorder.py | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/envs/website_browser/recorder.py b/src/agent_env/env/envs/website_browser/recorder.py index ce519e09c..a849a10fa 100644 --- a/src/agent_env/env/envs/website_browser/recorder.py +++ b/src/agent_env/env/envs/website_browser/recorder.py @@ -213,8 +213,10 @@ async def _show(self, cdp: Cdp, target: Optional[str]) -> None: await cdp.call("Page.startScreencast", options, session) except (CdpError, asyncio.TimeoutError, KeyError) as e: # the page went meanwhile logger.info("screencasting page %s: %s", target, e) - if self.page == target: + if self.page == target: # nothing took over since: back to the newest page left self.page = None + self._pages.pop(target, None) + await self._show(cdp, next(reversed(self._pages), None)) return if self.page == target: self._session = session diff --git a/tst/unit/env/envs/test_website_browser_recorder.py b/tst/unit/env/envs/test_website_browser_recorder.py index 6c56dfafa..255f45654 100644 --- a/tst/unit/env/envs/test_website_browser_recorder.py +++ b/tst/unit/env/envs/test_website_browser_recorder.py @@ -47,12 +47,15 @@ class _Browser: def __init__(self) -> None: self.calls: list[tuple[str, dict, str | None]] = [] + self.gone: set[str] = set() # pages that close before an attach reaches them self._events: asyncio.Queue = asyncio.Queue() self._sessions = itertools.count(1) async def call(self, method, params=None, session=None): self.calls.append((method, params or {}, session)) if method == "Target.attachToTarget": + if params["targetId"] in self.gone: + raise recorder.CdpError(f"No target with given id found: {params['targetId']}") return {"sessionId": f"{params['targetId']}-{next(self._sessions)}"} return {} @@ -134,6 +137,23 @@ async def test_the_screencast_follows_new_and_navigated_pages_and_falls_back_whe "A-1", "B-2", "A-3", "B-4"] +@pytest.mark.asyncio +async def test_a_page_that_closes_before_its_screencast_starts_hands_back_to_the_page_before_it(): + browser, red = _Browser(), _jpeg((255, 0, 0)) + screencast = recorder.Screencast(None, find=lambda: None, connect=None) + browser.gone.add("B") + browser.page("A", "https://shop.test/") + browser.frame("A-1", red) + browser.page("B", "about:blank", openerId="A") # a popup that closes while the attach is on its way + browser.emit("Target.targetDestroyed", {"targetId": "B"}) + browser.close() + await screencast.watch(browser) + + assert browser.attached() == ["A", "B", "A"] + assert (screencast.page, screencast.latest) == ("A", base64.b64decode(red)) + assert screencast._session == "A-2" + + def _recorder(browser: _Browser) -> recorder.BrowserRecorder: class _Connection: async def __aenter__(self):