browser and computer tools in the sandbox

CHANGELOG.md

@@ -2,7 +2,34 @@
 
 ## [Unreleased]
 
+### Added
+- `browser` and `computer`: agents drive a web browser and a virtual screen inside their sandbox
+  (`spec/2026-10-07-browser-computer-use`). A profile's `display` (`env.yaml` schema version 2)
+  starts the display image's screen and Chromium; `browser` acts on accessibility-snapshot refs,
+  `computer` on screen pixels, both run as the display's own user, and their screenshots reach
+  the model as images. The sandbox keeps no network: the browser reaches loopback and files only.
+  No agent may set `os_user: swarmdisplay`, the display's own user.
+  `deploy/images/display` builds the image (`mise run display:build`).
+- Tool results may carry images (`ToolResult.images`, `ToolMessage.images`, `ImageRef`), stored
+  as blobs and referenced by hash. `RequestOptions.max_images` (default 3) keeps a request to
+  the context's latest images; `before_model_request` may change it.
+- model-gateway accepts `image_url` parts (inline PNG `data:` URLs) in tool messages and moves
+  them into a user message after the tool messages before sending upstream, after reasoning
+  passback.
+- `Exec.user` and `Exec.collect`: a sandbox tool may run as another user and have sandboxd read
+  files out of the sandbox after the call. `SandboxTool.output` turns sandboxd's result into the
+  agent's.
+- The `.eval` export embeds a run's images: tool results inline, contexts as attachments, with
+  `[image omitted]` where a request left one out.
+
 ### Changed
+- Event schema version 10: tool events carry `images`, model events `max_images`, `exec`
+  observations `collected`. Older runs read as before.
+- `env.yaml` schema version 2 adds a profile's `display`. Version 1 files load as before.
+- A fork at a turn after a run's first `browser` or `computer` call is refused
+  (`FAILED_PRECONDITION`): a fork restores files, not a browser.
+- `GatewaySession` takes `images`, where it reads the images requests carry;
+  `check_transcript` takes `images` to rebuild them.
 - Models are chosen when a case is submitted, not written in the case
   (`spec/2026-10-06-run-time-models`). `case.yaml` schema version 4: an agent names a
   `model_slot` (default `default`) instead of a `model`, and agents in one slot run one model.

console/CHANGELOG.md

@@ -5,6 +5,8 @@
 ### Added
 - The case flow draws no sandbox for an agent with `sandbox: none` (case version 5), and its
   details say `none`. New case's template is `schema_version: 5`.
+- The case flow shows a profile's `display`: its screen size as a chip, and the size and first
+  page among its settings.
 - A case's Files tab opens on a flow: `case.yaml` and its env file, pending edits included,
   drawn as variant axes, the task, agents, channels, extensions, sandboxes, profiles, canaries,
   and scorers, with what connects them. A click shows a part's settings, its connections, its

console/src/lib/caseGraph.test.ts

@@ -271,3 +271,32 @@ test("paths expand over the axes they name", () => {
   expect(expandPath("p/${variant.x}_${variant.y}.md", axes)).toEqual(["p/a_1.md", "p/a_2.md", "p/b_1.md", "p/b_2.md"])
   expect(expandPath("p/${variant.z}.md", axes)).toEqual(["p/${variant.z}.md"])
 })
+
+test("a profile's display shows its size and first page", () => {
+  const text = `schema_version: 4
+id: c
+workspace: w
+swarm:
+  agents:
+    - { id: a, prompt: a.md, tools: [browser, computer], sandbox_profile: desk }
+task: { input: t.md }
+`
+  const env = `schema_version: 2
+sandbox_profiles:
+  default: { image: busybox }
+  desk:
+    image: swarmeval/display:dev
+    display: { width: 1280, url: "http://127.0.0.1:8080/" }
+`
+  const g = caseGraph(
+    new Map([
+      ["case.yaml", text],
+      ["env.yaml", env],
+    ])
+  )
+  const desk = g.nodes.find((n) => n.id === "profile:desk")!
+  expect(desk.chips).toContain("display 1280x768")
+  expect(desk.fields).toContainEqual(["display", "1280x768 · http://127.0.0.1:8080/"])
+  const plain = g.nodes.find((n) => n.id === "profile:default")!
+  expect(plain.fields.map(([k]) => k)).not.toContain("display")
+})

console/src/lib/caseGraph.ts

@@ -381,6 +381,11 @@ export function caseGraph(texts: ReadonlyMap<string, string | undefined>): CaseG
     const mounts = list(p.fs).map(obj)
     const limits = Object.entries(obj(p.limits)).map(([k, v]) => `${k} ${inline(v)}`)
     const copies = list(p.files).map(obj)
+    // A profile's screen, which `browser` and `computer` drive; sizes default to 1024x768.
+    const display =
+      p.display === undefined
+        ? ""
+        : `${inline(obj(p.display).width ?? 1024)}x${inline(obj(p.display).height ?? 768)}`
     nodes.push({
       id: `profile:${name}`,
       kind: "profile",
@@ -389,6 +394,7 @@ export function caseGraph(texts: ReadonlyMap<string, string | undefined>): CaseG
       chips: [
         ...mounts.map((m) => `${str(m.path)}${m.protected ? " protected" : ""}`),
         ...limits,
+        ...(display ? [`display ${display}`] : []),
       ],
       fields: rows(
         ["image", str(p.image)],
@@ -399,6 +405,7 @@ export function caseGraph(texts: ReadonlyMap<string, string | undefined>): CaseG
             .join(" · "),
         ]),
         ["limits", limits.join(", ")],
+        ["display", display && [display, str(obj(p.display).url)].filter(Boolean).join(" · ")],
         ...copies.map((f): [string, string] => ["copies", `${str(f.from)} → ${str(f.to)}`])
       ),
       source: envYaml?.source(["sandbox_profiles", name]),

deploy/images/display/Dockerfile

@@ -0,0 +1,33 @@
+# syntax=docker/dockerfile:1
+# The display image: a virtual screen, a browser, and `swarm-display`, which the `computer` and
+# `browser` tools drive. Cases extend it with `FROM swarmeval/display` and add their own apps,
+# started from /etc/swarm-display/start.d/. Contract: README.md beside this file. Build from the
+# repo root:
+#   docker build -f deploy/images/display/Dockerfile -t swarmeval/display:dev deploy/images/display
+
+FROM python:3.12-slim-bookworm
+
+# Xvfb is the screen; matchbox keeps every window maximized without decorations; xdotool moves
+# the mouse and types; xauth writes the screen's cookie. No terminal: the only programs on the
+# screen are the browser and what a case adds.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        xvfb xauth xdotool matchbox-window-manager fonts-dejavu-core fonts-noto-cjk \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PLAYWRIGHT_BROWSERS_PATH=/opt/swarm-display/browsers
+RUN pip install --no-cache-dir playwright==1.63.0 pillow==12.3.0 \
+    && playwright install --with-deps --no-shell chromium \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+# The display user owns the display's state; agents' users cannot read it, so they reach the
+# screen and the browser only through the tools.
+RUN useradd --uid 990 --create-home --shell /usr/sbin/nologin swarmdisplay \
+    && install -d -o swarmdisplay -g swarmdisplay -m 0700 /run/swarm-display \
+    && install -d -m 1777 /tmp/.X11-unix \
+    && install -d -m 0755 /etc/swarm-display/start.d
+
+COPY swarm_display /opt/swarm-display/swarm_display
+COPY swarm-display /usr/local/bin/swarm-display
+RUN chmod 0755 /usr/local/bin/swarm-display \
+    && python -m compileall -q /opt/swarm-display/swarm_display

deploy/images/display/README.md

@@ -0,0 +1,74 @@
+# swarmeval/display
+
+The sandbox image for the `browser` and `computer` tools: a virtual screen, Chromium on it, and
+`swarm-display`, which drives both. A case gets it through a profile's
+[`display`](../../../docs/case-format.md#display); sandboxd starts it
+([docs/services/sandboxd.md](../../../docs/services/sandboxd.md#display)); the tools are in
+[docs/agent-runtime.md](../../../docs/agent-runtime.md#tools). Why it is shaped this way:
+[spec/2026-10-07-browser-computer-use](../../../spec/2026-10-07-browser-computer-use/README.md).
+
+```sh
+mise run display:build      # docker build -t swarmeval/display:dev deploy/images/display
+```
+
+Building downloads Debian packages and Playwright's Chromium; running needs no network.
+
+## What it holds
+
+| Piece | Version | Role |
+|---|---|---|
+| `python:3.12-slim-bookworm` | | Base, and the Python `swarm-display` runs on |
+| Xvfb, xauth | Debian | The screen, `:1`, with a cookie only `swarmdisplay` can read |
+| matchbox-window-manager | Debian | Keeps every window maximized, without decorations |
+| xdotool | Debian | Mouse and keyboard for `computer` |
+| Playwright, its Chromium | 1.63.0 | The browser, driven through a pipe: no debugging port |
+| Pillow | 12.3.0 | Screenshots of the whole screen |
+| DejaVu, Noto CJK fonts | Debian | Text in pages, Chinese and Japanese included |
+
+No terminal or file manager: the screen shows the browser and whatever a case adds.
+
+## Contract
+
+What sandboxd and the tools rely on. An image `FROM swarmeval/display` keeps it as long as it
+does not remove these:
+
+- The user `swarmdisplay` (uid 990) and `/run/swarm-display`, owned by it, mode `0700`.
+- `/usr/local/bin/swarm-display`:
+  - `start --width W --height H [--url U]`: starts the daemon (screen, window manager, start
+    scripts, browser), waits until the browser is up, prints the daemon's pid, and exits 0.
+    sandboxd runs it as `swarmdisplay` when it creates the sandbox.
+  - `computer <json>` and `browser <json>`: one action, as the tools describe it. The text goes
+    to stdout; an image, if the action took one, to `/run/swarm-display/out/screenshot.png`,
+    which sandboxd collects and removes. Exit status 1 means the action failed; the reason is on
+    stderr. They must run as `swarmdisplay`.
+- `sh`, `sleep`, and `tr`, which every sandbox image needs.
+- `USER` left unset (root, with no capabilities in a sandbox). Agents without an `os_user` run
+  as the image's user, so sandboxd refuses an image whose user is `swarmdisplay`.
+
+## Adding a case's apps
+
+Put executables in `/etc/swarm-display/start.d/`. The daemon runs them in name order, as
+`swarmdisplay`, after the screen is up and before the browser opens, each with 30 seconds to
+return; a non-zero exit fails the sandbox. Start servers in the background:
+
+```dockerfile
+FROM swarmeval/display:dev
+COPY shop /opt/shop
+COPY <<'EOF' /etc/swarm-display/start.d/10-shop
+#!/bin/sh
+cd /opt/shop && nohup python3 -m http.server 8080 --bind 127.0.0.1 >/dev/null 2>&1 &
+EOF
+RUN chmod 0755 /etc/swarm-display/start.d/10-shop
+```
+
+and point the profile's `display.url` at `http://127.0.0.1:8080/`. Their processes run as
+`swarmdisplay`, so sandboxd counts them as the display's: they are not reported as `proc.*`
+events.
+
+## Limits
+
+- The display's state, Chromium's profile included, lives in `/run/swarm-display`, outside every
+  key path: nothing the browser writes is a file change, and downloads are not kept.
+- Dialogs (`alert`, `confirm`) are dismissed.
+- The browser locates elements by snapshot ref through the `aria-ref=` selector. Playwright 1.64
+  makes `page.get_by_ref` public; move to it when upgrading.

deploy/images/display/swarm-display

@@ -0,0 +1,3 @@
+#!/bin/sh
+# The display's command line: `start`, `computer`, `browser`. See README.md.
+PYTHONPATH=/opt/swarm-display exec /usr/local/bin/python3 -s -m swarm_display "$@"

deploy/images/display/swarm_display/__init__.py

@@ -0,0 +1,20 @@
+"""The display stack of a sandbox: a virtual screen, a browser on it, and the daemon that drives
+both for the `computer` and `browser` tools (deploy/images/display/README.md).
+
+Everything here runs inside the sandbox as the user `swarmdisplay`.
+"""
+
+from pathlib import Path
+
+STATE = Path("/run/swarm-display")
+SOCKET = STATE / "daemon.sock"
+LOG = STATE / "daemon.log"
+SCREENSHOT = STATE / "out" / "screenshot.png"
+"""Written by an action that has an image; sandboxd collects and removes it after the call."""
+DISPLAY = ":1"
+XAUTHORITY = STATE / "Xauthority"
+START_DIR = Path("/etc/swarm-display/start.d")
+
+
+class ActionError(Exception):
+    """The action could not be done; the message is what the agent sees."""

deploy/images/display/swarm_display/__main__.py

@@ -0,0 +1,117 @@
+"""`swarm-display start | computer <json> | browser <json> | daemon`.
+
+`start` launches the daemon, waits until the screen and the browser are up, prints the daemon's
+pid, and exits; sandboxd runs it once, when it creates the sandbox. `computer` and `browser`
+send one action to the daemon and print its text; an image, if any, is at `SCREENSHOT`. Exit
+status 1 means the action failed, and the reason is on stderr.
+"""
+
+import argparse
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+
+from swarm_display import LOG, SCREENSHOT, SOCKET, STATE
+
+START_TIMEOUT_S = 50.0
+ACTION_TIMEOUT_S = 180.0
+
+
+def _start(width: int, height: int, url: str) -> int:
+    if SOCKET.exists():
+        print(f"the display is already running ({SOCKET} exists)", file=sys.stderr)
+        return 1
+    for sub in ("home", "tmp", "out", "profile"):
+        (STATE / sub).mkdir(mode=0o700, exist_ok=True)
+    argv = [sys.executable, "-s", "-m", "swarm_display", "daemon"]
+    argv += ["--width", str(width), "--height", str(height), "--url", url]
+    with open(LOG, "ab") as log:
+        daemon = subprocess.Popen(
+            argv,
+            stdin=subprocess.DEVNULL,
+            stdout=log,
+            stderr=log,
+            start_new_session=True,
+            cwd=STATE,
+        )
+    deadline = time.monotonic() + START_TIMEOUT_S
+    while time.monotonic() < deadline:
+        if daemon.poll() is not None:
+            print(f"the display daemon exited {daemon.returncode}:\n{_log_tail()}", file=sys.stderr)
+            return 1
+        if SOCKET.exists():
+            print(daemon.pid)
+            return 0
+        time.sleep(0.1)
+    daemon.kill()
+    print(f"the display was not up after {START_TIMEOUT_S:.0f}s:\n{_log_tail()}", file=sys.stderr)
+    return 1
+
+
+def _log_tail() -> str:
+    try:
+        return LOG.read_text(errors="replace")[-4000:]
+    except FileNotFoundError:
+        return "(no log)"
+
+
+def _act(tool: str, raw: str) -> int:
+    try:
+        args = json.loads(raw)
+    except ValueError as err:
+        print(f"arguments are not JSON: {err}", file=sys.stderr)
+        return 1
+    try:
+        SCREENSHOT.unlink(missing_ok=True)
+    except PermissionError:
+        print(f"swarm-display {tool} must run as the display's user, swarmdisplay", file=sys.stderr)
+        return 1
+    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as conn:
+        conn.settimeout(ACTION_TIMEOUT_S)
+        try:
+            conn.connect(str(SOCKET))
+        except OSError as err:
+            print(f"the display is not running ({SOCKET}: {err})", file=sys.stderr)
+            return 1
+        conn.sendall(json.dumps({"tool": tool, "args": args}).encode() + b"\n")
+        with conn.makefile("rb") as reply:
+            line = reply.readline()
+    if not line:
+        print("the display daemon closed the connection without an answer", file=sys.stderr)
+        return 1
+    answer = json.loads(line)
+    if not answer["ok"]:
+        print(answer["text"], file=sys.stderr)
+        return 1
+    sys.stdout.write(answer["text"])
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="swarm-display")
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("start", "daemon"):
+        p = sub.add_parser(name)
+        p.add_argument("--width", type=int, required=True)
+        p.add_argument("--height", type=int, required=True)
+        p.add_argument("--url", default="about:blank")
+    for name in ("computer", "browser"):
+        sub.add_parser(name).add_argument("arguments")
+    opts = parser.parse_args()
+    os.umask(0o077)
+    match opts.command:
+        case "start":
+            return _start(opts.width, opts.height, opts.url)
+        case "daemon":
+            from swarm_display.daemon import run
+
+            return run(opts.width, opts.height, opts.url)
+        case tool:
+            return _act(tool, opts.arguments)
+
+
+if __name__ == "__main__":
+    sys.exit(main())