feat(agents): add a Browser Use agent

.github/dependabot.yml

@@ -40,6 +40,7 @@ updates:
       - "/src/agent_env/env/gateway"
       - "/src/agent_env/env/envs/service_db"
       - "/src/agent_env/env/envs/website_browser"
+      - "/src/agent_env/agents/agents/browser-use"
       - "/src/agent_env/agents/agents/claude-code"
       - "/src/agent_env/agents/agents/codex"
       - "/src/agent_env/agents/agents/gemini"

.github/scripts/infra_build.py

@@ -31,6 +31,7 @@
     "agentenv-db-web": "src/agent_env/env/envs/service_db/Dockerfile.db-web",
     "agentenv-db-mcp": "src/agent_env/env/envs/service_db/Dockerfile.db-mcp",
     "agentenv-website-browser": "src/agent_env/env/envs/website_browser/Dockerfile",
+    "agentenv-browser-use": "src/agent_env/agents/agents/browser-use/Dockerfile",
     "agentenv-claude-code": "src/agent_env/agents/agents/claude-code/Dockerfile",
     "agentenv-codex": "src/agent_env/agents/agents/codex/Dockerfile",
     "agentenv-gemini": "src/agent_env/agents/agents/gemini/Dockerfile",
@@ -72,6 +73,8 @@
                                  " && ls -d /root/.cache/ms-playwright/chromium-*"],
     "agentenv-claude-code": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
                              'claude --version && python -c "import a2a_server"'],
+    "agentenv-browser-use": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
+                             'node -e "import(\'@browser_use/pi\')" && python -c "import a2a_server"'],
     "agentenv-codex": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
                        'codex --version && mcp-remote --help >/dev/null && python -c "import a2a_server"'],
     "agentenv-gemini": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",

src/agent_env/agents/README.md

@@ -71,6 +71,14 @@ a relay that adds task attribution, and takes its model from the prompt, else `o
 `install/v1` too, into a container or onto a Linux host with glibc. Its folder's `README.md` covers the extensions
 it implements. Run its smoke task with `agent-env run agents --task letta-code-smoke`.
 
+`agents/browser-use/` runs [Browser Use Pi](https://github.com/browser-use/browser-use-pi) behind the protocol's A2A
+agent framework: each turn is one `node agent.mjs` run that drives a Browser Use Cloud browser, or the one `cdp_url`
+names, with the task env's MCP tools beside it, and a context's turns continue one conversation. Name it as
+`browser-use`, its alias, or by its id, `@local/agentenv-framework/agents/browser-use`. It calls the `[model]`
+endpoint as an OpenAI-compatible one and takes its model from the prompt, and its browser key from
+`BROWSER_USE_API_KEY`. Its folder's `README.md` covers the extensions it implements. Run its smoke task with
+`agent-env run agents --task browser-use-smoke`.
+
 `agents/openhands/` runs the [OpenHands](https://github.com/OpenHands/software-agent-sdk) agent behind the
 protocol's A2A agent framework, driving the OpenHands SDK in process rather than a CLI: each turn is one run of a
 persisted SDK conversation, the task env's MCP servers go to its `mcp_config`, and its SDK events are the turn's

src/agent_env/agents/agents/browser-use/Dockerfile

@@ -0,0 +1,16 @@
+# docker build -t browser-use-a2a src/agent_env/agents/agents/browser-use
+FROM mirror.gcr.io/library/node:22-bookworm-slim@sha256:4b150b48b4116e2a22f2c39cc27025fbf4f153e2078fd3b14d159fdb5c7279da AS node
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci --ignore-scripts --omit=dev --no-audit --no-fund --loglevel=error
+
+FROM mirror.gcr.io/library/python:3.12-slim-bookworm@sha256:e697c81897302a4c4333bade000d98038fbb21eb064f8ee1fd42d02ebcd9d337
+COPY --from=node /usr/local/bin/node /usr/local/bin/node
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt && useradd --create-home agent
+COPY --from=node /app/node_modules node_modules
+COPY a2a_server.py agent.mjs package.json ./
+USER agent
+EXPOSE 8000
+CMD ["python3", "a2a_server.py"]

src/agent_env/agents/agents/browser-use/README.md

@@ -0,0 +1,26 @@
+# Browser Use A2A agent
+
+[Browser Use Pi](https://github.com/browser-use/browser-use-pi) wrapped with `agentenv_protocol.a2a_agent`, shipped in
+the `agents` bundle as `@local/agentenv-framework/agents/browser-use`, with alias `browser-use`. Browser Use Pi drives
+a real browser over CDP from a persistent JavaScript REPL. `package-lock.json` pins `@browser_use/pi` to 0.1.0.
+
+Each turn runs `node agent.mjs` once. It calls the `[model]` endpoint as an OpenAI-compatible one
+(`LITELLM_BASE_URL`, `LITELLM_API_KEY`, `LITELLM_EXTRA_HEADERS`), and the task env's MCP tools are the agent's tools
+next to the browser, named `<server>__<tool>`. The browser is a [Browser Use Cloud](https://cloud.browser-use.com)
+browser, which needs `BROWSER_USE_API_KEY`, or the Chrome DevTools endpoint `cdp_url` names. A context's turns
+continue one conversation through a saved history file; each turn opens a fresh browser.
+
+```bash
+agent-env a2a-agent put --id browser-use --dockerfile src/agent_env/agents/agents/browser-use/Dockerfile \
+    --default-model gpt-5.4-mini --env-var BROWSER_USE_API_KEY=...
+agent-env run agents --task browser-use-smoke
+```
+
+| Extension | Implementation |
+|---|---|
+| `agent-config/v1` | `model` (else `BROWSER_USE_A2A_MODEL`), `effort`, `system_prompt` (added to Browser Use Pi's instructions), `max_steps`, `cdp_url`, `timeout_seconds` |
+| `mcp-config/v1` | SDK default; each server's tools are listed per turn and called over streamable HTTP |
+| `trajectory/v1` | Browser Use Pi's agent events, live, format `browser-use-pi-events/1`, without the streamed `message_update` / `tool_execution_update` deltas |
+
+A run that stops early (`max_steps`, `timeout`, `cost_limit`, ...) fails with code `browser_use.<status>` and its
+last text. Skills, snapshots, peers and `install/v1` are not implemented, so the validator reports them unsupported.

src/agent_env/agents/agents/browser-use/a2a_server.py

@@ -0,0 +1,151 @@
+"""Serves Browser Use Pi as an A2A agent: each turn runs ``agent.mjs`` once, and a context's turns share one
+conversation through a saved history file. ``AGENT_CARD`` is what agent-env's install/v1 reads."""
+
+import asyncio
+import contextlib
+import hashlib
+import json
+import os
+import signal
+from pathlib import Path
+from typing import Literal
+
+from agentenv_protocol.a2a_agent import (
+    MCP_CONFIG_V1,
+    TRAJECTORY_V1,
+    AgentConfig,
+    AgentEnvAgent,
+    AgentIdentity,
+    TaskRequest,
+    TaskResult,
+    TextPart,
+    Usage,
+    a2a_agent,
+    enable,
+)
+
+HERE = Path(__file__).parent
+HOME = Path(os.environ.get("BROWSER_USE_A2A_HOME", Path.home() / ".browser-use-a2a"))
+# Leaves the agent time to deliver what it has before the turn's deadline kills it.
+DELIVERY_SECONDS = 30
+STOP_SECONDS = 5
+# Streamed token deltas; each message's end event carries the whole message.
+_UNRECORDED_EVENTS = frozenset({"message_update", "tool_execution_update"})
+
+
+class BrowserUseConfig(AgentConfig):
+    model: str | None = None
+    effort: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None
+    system_prompt: str | None = None
+    max_steps: int = 40
+    # A Chrome DevTools endpoint to drive instead of a Browser Use Cloud browser.
+    cdp_url: str | None = None
+    timeout_seconds: int = 1800
+
+
+@a2a_agent(
+    identity=AgentIdentity(
+        name="browser-use",
+        description="Browser Use Pi: drives a real browser over CDP from a persistent JavaScript REPL.",
+        version="1.0.0",
+        input_modes=("text",),
+    ),
+    config=BrowserUseConfig,
+    config_description="Configure Browser Use Pi: model, effort, system prompt, step budget and browser.",
+    extensions=(
+        MCP_CONFIG_V1,
+        enable(TRAJECTORY_V1, live=True, description="Browser Use Pi's agent events, live."),
+    ),
+)
+class BrowserUseAgent(AgentEnvAgent):
+    def __init__(self, command: tuple[str, ...] = ("node", str(HERE / "agent.mjs"))) -> None:
+        self.command = command
+
+    async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult:
+        config = request.config
+        model = config.model or os.environ.get("BROWSER_USE_A2A_MODEL")
+        if not model:
+            return TaskResult.failure("browser_use.no_model", "no model is configured; set 'model' through agent-config")
+        session = request.session_ref or hashlib.sha256(request.context_id.encode()).hexdigest()[:32]
+        workspace = request.workspace or HOME / "workspaces" / session
+        workspace.mkdir(parents=True, exist_ok=True)
+        turn = {
+            "prompt": "\n\n".join(part.text for part in request.parts if isinstance(part, TextPart)),
+            "model": model,
+            "effort": config.effort,
+            "systemPrompt": config.system_prompt,
+            "maxSteps": config.max_steps,
+            "timeoutMs": max(config.timeout_seconds - DELIVERY_SECONDS, 1) * 1000,
+            "cdpUrl": config.cdp_url,
+            "mcpServers": {name: {"url": s["url"], "headers": s.get("headers") or {}} for name, s in request.mcp_servers.items()},
+            "workspace": str(workspace),
+            # Browser Use Pi refuses a history saved under another model, so a model change starts a new conversation.
+            "historyFile": str(HOME / "sessions" / f"{session}-{hashlib.sha256(model.encode()).hexdigest()[:8]}.json"),
+        }
+        (HOME / "sessions").mkdir(parents=True, exist_ok=True)
+        request.trajectory.set_format("browser-use-pi-events/1")
+
+        process = await asyncio.create_subprocess_exec(
+            *self.command,
+            stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+            start_new_session=True, limit=64 * 1024 * 1024,
+        )
+        stderr = asyncio.create_task(process.stderr.read())
+        result = None
+        tool_calls = 0
+        try:
+            async with asyncio.timeout(config.timeout_seconds):
+                process.stdin.write(json.dumps(turn).encode())
+                process.stdin.close()
+                async for line in process.stdout:
+                    if not line.startswith(b"{"):
+                        continue
+                    message = json.loads(line)
+                    if message["type"] == "result":
+                        result = message["result"]
+                    elif message["event"]["type"] not in _UNRECORDED_EVENTS:
+                        request.trajectory.append(message["event"])
+                        tool_calls += message["event"]["type"] == "tool_execution_end"
+                await process.wait()
+        except TimeoutError:
+            return TaskResult.failure("browser_use.timeout", f"no answer within {config.timeout_seconds}s")
+        finally:
+            # On a timeout or a cancel, the agent stops its browser, then goes with everything it started.
+            if process.returncode is None:
+                with contextlib.suppress(ProcessLookupError):
+                    os.killpg(process.pid, signal.SIGTERM)
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(asyncio.shield(process.wait()), STOP_SECONDS)
+                with contextlib.suppress(ProcessLookupError):
+                    os.killpg(process.pid, signal.SIGKILL)
+                await process.wait()
+
+        if result is None:
+            error = (await stderr).decode(errors="replace")[-4000:] or f"agent.mjs exited {process.returncode}"
+            return TaskResult.failure("browser_use.crashed", error, error_type="infra_error")
+        usage = result["usage"]
+        builder = TaskResult.builder()
+        if result["status"] == "completed":
+            builder.succeeded()
+        else:
+            builder.failed(f"browser_use.{result['status']}", result.get("error") or result["status"])
+        return (
+            builder.add_text(result["text"] or result["status"])
+            .session_ref(session)
+            .usage(Usage(
+                tool_call_count=tool_calls,
+                input_tokens=usage["input"],
+                output_tokens=usage["output"],
+                total_tokens=usage["totalTokens"],
+                provider_details={"steps": result["steps"], "model": result["model"]},
+            ))
+            .build()
+        )
+
+
+agent = BrowserUseAgent()
+app = agent.create_app()
+AGENT_CARD = app.state.agentenv_a2a.card
+
+if __name__ == "__main__":
+    agent.serve()

src/agent_env/agents/agents/browser-use/agent.mjs

@@ -0,0 +1,77 @@
+// One turn of Browser Use Pi: reads the turn as JSON on stdin, prints each event, then the result, as a JSON line.
+import { Browser, BrowserUse, Type } from '@browser_use/pi';
+import { createModels, createProvider, envApiKeyAuth } from '@earendil-works/pi-ai';
+import { openAICompletionsApi } from '@earendil-works/pi-ai/api/openai-completions.lazy';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { existsSync } from 'node:fs';
+import { text } from 'node:stream/consumers';
+
+const turn = JSON.parse(await text(process.stdin));
+const print = (line) => process.stdout.write(JSON.stringify(line) + '\n');
+
+// The turn's model, served by agent-env's OpenAI-compatible endpoint.
+const models = createModels();
+models.setProvider(createProvider({
+  id: 'agentenv',
+  baseUrl: process.env.LITELLM_BASE_URL,
+  auth: { apiKey: envApiKeyAuth('agent-env model key', ['LITELLM_API_KEY']) },
+  models: [{
+    id: turn.model, name: turn.model, api: 'openai-completions', provider: 'agentenv',
+    baseUrl: process.env.LITELLM_BASE_URL, headers: JSON.parse(process.env.LITELLM_EXTRA_HEADERS || '{}'), reasoning: Boolean(turn.effort), input: ['text', 'image'],
+    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 200_000, maxTokens: 32_000,
+  }],
+  api: openAICompletionsApi(),
+}));
+
+// Each env's MCP tools become the agent's tools, named <server>__<tool>.
+const clients = [];
+const tools = [];
+for (const [server, { url, headers }] of Object.entries(turn.mcpServers)) {
+  const client = new Client({ name: 'browser-use', version: '1.0.0' });
+  await client.connect(new StreamableHTTPClientTransport(new URL(url), { requestInit: { headers } }));
+  clients.push(client);
+  let page = {};
+  do {
+    page = await client.listTools({ cursor: page.nextCursor });
+    for (const tool of page.tools) addTool(client, server, tool);
+  } while (page.nextCursor);
+}
+
+function addTool(client, server, tool) {
+  const base = `${server}__${tool.name}`.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 60);
+  let name = base;
+  for (let n = 2; tools.some((t) => t.name === name); n++) name = `${base}_${n}`;
+  tools.push({
+    name, label: name, description: tool.description ?? '', parameters: Type.Unsafe(tool.inputSchema),
+    execute: async (_id, args, signal) => {
+      const result = await client.callTool({ name: tool.name, arguments: args }, undefined, { signal });
+      const content = result.content.map((part) => part.type === 'image' ? part : { type: 'text', text: part.text ?? JSON.stringify(part) });
+      if (result.isError) throw new Error(content.map((part) => part.text ?? '').join('\n'));
+      return { content, details: result.structuredContent ?? null };
+    },
+  });
+}
+
+const agent = await BrowserUse.create({
+  model: `agentenv/${turn.model}`,
+  models,
+  reasoning: turn.effort,
+  instructions: turn.systemPrompt,
+  tools,
+  browser: turn.cdpUrl ? Browser.chrome({ cdpUrl: turn.cdpUrl }) : Browser.cloud({ apiKey: process.env.BROWSER_USE_API_KEY, timeoutMinutes: Math.ceil(turn.timeoutMs / 60_000) + 1 }),
+  workspace: turn.workspace,
+  historyFile: existsSync(turn.historyFile) ? turn.historyFile : undefined,
+  telemetry: false,
+});
+// A cancelled turn stops its browser before it goes.
+process.once('SIGTERM', () => agent.close().finally(() => process.exit(143)));
+try {
+  const options = { maxSteps: turn.maxSteps, timeoutMs: turn.timeoutMs, onEvent: (event) => print({ type: 'event', event }) };
+  const result = existsSync(turn.historyFile) ? await agent.followUp(turn.prompt, options) : await agent.run(turn.prompt, options);
+  await agent.saveHistory(turn.historyFile);
+  print({ type: 'result', result });
+} finally {
+  await agent.close();
+  await Promise.allSettled(clients.map((client) => client.close()));
+}