diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 93779fc3a..b6cc8c649 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -790,8 +790,10 @@ are the profile's template. The base class adds: the pipe rather than a second copy, which would hand its window to the first and exit. The ready check `devtools_answers` asks `Browser.getVersion` over the pipe. - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches - to every page with flattened sessions and keeps a record, `since` its server started: open tabs in - opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and + to every page with flattened sessions, telling each page it attached to as it opened, a popup say, to run, and keeps a + record, `since` its server started: open tabs in opened order, each with its URL and title as the browser has them + when the record is read (Chrome sends no event for a title), the one whose page has focus `active`; `closed_tabs`, + each as of its page's last load, with `closed_at` and `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 083cfb251..a5a435caf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -260,13 +260,26 @@ async def active_tab(self) -> Optional[str]: return visible async def record(self) -> dict: - """The record ``since`` it started: open ``tabs``, each with whether it's ``active``, ``closed_tabs``, - ``downloads`` and ``dialogs``.""" + """The record ``since`` it started: open ``tabs``, each with its URL and title now and whether it's ``active``, + ``closed_tabs``, ``downloads`` and ``dialogs``.""" + await self._refresh() active = await self.active_tab() return {"since": self.since, "tabs": [{**tab, "active": tab["id"] == active} for tab in self.tabs.values()], "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()), "dialogs": list(self.dialog_log)} + async def _refresh(self, target: Optional[str] = None) -> None: + """The URL and title of the tab ``target``, or of every open tab, as the browser has them now: it sends no + event when a page's title changes.""" + try: + infos = (await self.devtools.call("Target.getTargets", {"filter": PAGES}, + timeout_s=ACTIVE_CHECK_TIMEOUT_S))["targetInfos"] + except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError): + return + for info in infos: + if target in (None, info["targetId"]) and (tab := self.tabs.get(info["targetId"])) is not None: + tab.update(url=info.get("url", tab["url"]), title=info.get("title", tab["title"])) + def _event(self, message: dict) -> None: handler = getattr(self, "_on_" + message.get("method", "").replace(".", "_"), None) if handler is not None: @@ -295,10 +308,16 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N if params["targetInfo"].get("type") == "page": self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"] self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"])) + # A page attached as it opens, a popup say, waits for this before it navigates. + self._spawn(self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=params["sessionId"])) def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: self._sessions.pop(params["sessionId"], None) + def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None: + if (target := self._sessions.get(session)) is not None: + self._spawn(self._refresh(target)) + def _on_Page_javascriptDialogOpening(self, params: dict, session: Optional[str]) -> None: dialog = {"tab": self._sessions.get(session), "type": params.get("type"), "message": params.get("message", ""), "url": params.get("url", ""), "opened_at": _now(), "answer": None, "by": None} diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index d21dd495c..c4ce5bbe8 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -3,7 +3,8 @@ its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready check asks, and a browser launched again when its server restarts; its shared memory in files only on a small /dev/shm; an empty cache and the deploy's prompt answers in its profile at each launch; its policies, card params and -skill; and an app that isn't a browser launched and declared as before.""" +skill; its record's titles read from the browser, which sends no event for a page's; and an app that isn't a browser +launched and declared as before.""" from __future__ import annotations import asyncio @@ -492,6 +493,40 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_ assert devtools.calls[2][1] == {"behavior": "default", "eventsEnabled": True} +@pytest.mark.asyncio +async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to_as_it_opened_navigates(tmp_path): + devtools = StubDevTools() + control = BrowserControl(devtools, "agent") + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0) + + assert ("Page.enable", {}, "s-popup") in devtools.calls + assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls + + +@pytest.mark.asyncio +async def test_its_record_reads_each_tabs_title_when_read_and_when_its_page_loads_since_no_event_says(tmp_path): + now = {"a": "Inbox", "b": "Draft"} + devtools = StubDevTools({"Target.getTargets": lambda params, session: {"targetInfos": [ + _page(target, f"https://mail.test/{target}", title)["targetInfo"] for target, title in now.items()]}}) + control = BrowserControl(devtools, "agent") + await control.attach() + for target in ("a", "b"): + devtools.send("Target.targetCreated", _page(target, f"https://mail.test/{target}", f"mail.test/{target}")) + devtools.send("Target.attachedToTarget", {"sessionId": f"s-{target}", **_page(target)}) + devtools.send("Page.loadEventFired", {"timestamp": 1}, session="s-b") + await asyncio.sleep(0) + del now["b"] + devtools.send("Target.targetDestroyed", {"targetId": "b"}) + now["a"] = "Inbox (1)" + record = await control.record() + + assert [(tab["id"], tab["title"]) for tab in record["tabs"]] == [("a", "Inbox (1)")] + assert [(tab["id"], tab["title"]) for tab in record["closed_tabs"]] == [("b", "Draft")] + assert ("Target.getTargets", {"filter": [{"type": "page"}]}, None) in devtools.calls + + @pytest.mark.asyncio @pytest.mark.parametrize("dialogs, kind, sent, answer, by", [ ("accept", "prompt", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"), diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 4ca5b9701..aac3fb21f 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -59,6 +59,27 @@ def log_message(self, *args): threading.Thread(target=server.serve_forever, daemon=True).start() time.sleep(float(sys.argv[1])) """ +POPPING = "http://127.0.0.1:8768/" +# Serves, on loopback for the seconds it's given, a page that opens /popped in a popup once, when Enter is pressed. +SERVE_POPPING = """import http.server, sys, threading, time +class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/popped": + body = b"Popped page

Popped

" + else: + body = (b"Opener page") + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = http.server.ThreadingHTTPServer(("127.0.0.1", 8768), Page) +threading.Thread(target=server.serve_forever, daemon=True).start() +time.sleep(float(sys.argv[1])) +""" RECORDED = "http://127.0.0.1:8767/" # Serves, on loopback for the seconds it's given, a page that asks to confirm and then downloads report.bin, whose # content differs at each request: the nth is _report(n). @@ -184,6 +205,32 @@ async def test_a_desktop_that_gives_it_defaults_deploys_with_them_and_a_deploys_ assert not report.failed +@pytest.mark.asyncio +async def test_a_popup_a_page_opens_loads_with_its_control_attached(chromium_desktop): + with namespace_routing(): + deployed = await Env.get(chromium_desktop.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [POPPING]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + card = await _app(deployed, WELL_KNOWN_PATH, "chromium") + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_POPPING, "60"], + timeout_s=90)) + await asyncio.sleep(SERVE_START_S) + await _keys(deployed, ["F5"]) + await _chromium_windows(deployed, "Opener page") + await _keys(deployed, ["Enter"]) + popped = await _record(deployed, card, lambda r: any(tab["title"] == "Popped page" for tab in r["tabs"])) + windows = await _chromium_windows(deployed, "Popped page") + serving.cancel() + finally: + report = await teardown_run(context) + + assert [(tab["url"], tab["title"]) for tab in popped["tabs"]] == [ + (POPPING, "Opener page"), (f"{POPPING}popped", "Popped page")], popped + assert any(title.startswith("Popped page") for title in windows), windows + assert not report.failed + + @pytest.mark.asyncio async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with_its_cause(chromium_desktop): settings = {"start_urls": [RECORDED], "dialogs": "accept"}