From 073c7398f8f076a137440895bf59d859ce48965d Mon Sep 17 00:00:00 2001 From: YL Date: Wed, 7 Oct 2026 15:57:45 -0700 Subject: [PATCH 1/2] sandboxd, runtime, gateway, events, core, console: browser and computer in the sandbox Agents drive a web browser and a virtual screen inside their own sandbox (spec/2026-10-07-browser-computer-use). The sandbox keeps no network: the browser reaches loopback and files only, and isolation defaults are unchanged. - deploy/images/display builds swarmeval/display: Xvfb, matchbox, xdotool, Playwright 1.63's Chromium, and `swarm-display`, a daemon and CLI run as the user `swarmdisplay`. Agents' own users cannot reach the screen or the browser except through the tools. - sandboxd: CreateSandboxRequest.display starts the display before the first manifest; the display user's processes are left out of every listing, so renderers are neither reported nor make changes ambiguous. ExecRequest.collect reads and removes files outside the key paths as the call's user and returns them as blobs. Exec moves to exec.go. - runtime: `computer` (screen pixels) and `browser` (accessibility snapshot refs) tools; Exec.user, Exec.collect, SandboxTool.output. Tool results carry ImageRefs; RequestOptions.max_images (default 3) keeps a request to the latest images. - model-gateway: tool messages may carry inline PNG image parts, moved into a user message after the tool messages, after reasoning passback. - Event schema 10 (`images`, `max_images`, `collected`); older runs read as before. The .eval export embeds the images; the transcript check rebuilds requests with them. - env.yaml schema 2 adds a profile's `display`; `browser` and `computer` need one. A fork after a run's first display call is refused. - console: the case flow shows a profile's display. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 26 + console/CHANGELOG.md | 2 + console/src/lib/caseGraph.test.ts | 29 ++ console/src/lib/caseGraph.ts | 7 + deploy/images/display/Dockerfile | 33 ++ deploy/images/display/README.md | 72 +++ deploy/images/display/swarm-display | 3 + .../images/display/swarm_display/__init__.py | 20 + .../images/display/swarm_display/__main__.py | 117 +++++ .../images/display/swarm_display/browser.py | 155 ++++++ .../images/display/swarm_display/computer.py | 99 ++++ deploy/images/display/swarm_display/daemon.py | 130 +++++ docs/agent-runtime.md | 34 +- docs/architecture.md | 8 +- docs/case-format.md | 45 +- docs/development.md | 5 +- docs/event-log.md | 27 +- docs/repo-layout.md | 6 +- docs/services/model-gateway.md | 9 +- docs/services/orchestrator.md | 6 +- docs/services/sandboxd.md | 44 +- go/CHANGELOG.md | 10 + go/README.md | 12 +- go/internal/driver/driver.go | 7 +- .../gen/swarmeval/sandbox/v1/sandbox.pb.go | 456 ++++++++++++------ .../swarmeval/sandbox/v1/sandbox_grpc.pb.go | 6 +- .../v1/sandboxv1connect/sandbox.connect.go | 6 +- go/internal/sandboxd/display.go | 174 +++++++ .../sandboxd/display_integration_test.go | 142 ++++++ go/internal/sandboxd/display_test.go | 176 +++++++ go/internal/sandboxd/exec.go | 229 +++++++++ go/internal/sandboxd/procs.go | 12 +- go/internal/sandboxd/procs_test.go | 6 +- go/internal/sandboxd/server.go | 18 + go/internal/sandboxd/service.go | 227 +-------- go/internal/sandboxd/service_test.go | 19 + mise.toml | 4 + proto/swarmeval/sandbox/v1/sandbox.proto | 33 +- .../2026-10-07-browser-computer-use/README.md | 177 +++++++ swarmeval/core/loader.py | 2 + swarmeval/core/models.py | 49 +- swarmeval/core/topology.py | 21 + swarmeval/events/__init__.py | 2 + swarmeval/events/convert.py | 62 ++- swarmeval/events/export.py | 74 ++- swarmeval/events/fork.py | 24 + swarmeval/events/transcript.py | 10 +- swarmeval/gateway/model/client.py | 69 ++- swarmeval/gateway/model/upstream.py | 40 +- swarmeval/gateway/model/wire.py | 23 +- .../proto/swarmeval/sandbox/v1/sandbox_pb2.py | 108 +++-- .../swarmeval/sandbox/v1/sandbox_pb2.pyi | 95 +++- .../swarmeval/sandbox/v1/sandbox_pb2_grpc.py | 3 +- .../swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi | 9 +- swarmeval/runtime/display.py | 222 +++++++++ swarmeval/runtime/execute.py | 3 +- swarmeval/runtime/loop.py | 5 +- swarmeval/runtime/messages.py | 30 ++ swarmeval/runtime/records.py | 26 + swarmeval/runtime/tools.py | 7 +- swarmeval/sandbox/blobs.py | 6 +- swarmeval/sandbox/client.py | 46 +- swarmeval/worker/run.py | 24 +- swarmeval/worker/transcript.py | 29 +- tests/core/test_display.py | 80 +++ tests/core/test_loader.py | 4 +- tests/events/test_display_export.py | 134 +++++ tests/gateway/test_gateway.py | 111 ++++- tests/runtime/fakes.py | 15 +- tests/runtime/test_display.py | 187 +++++++ tests/sandbox/test_client.py | 63 +++ tests/worker/test_display_e2e.py | 166 +++++++ 72 files changed, 3836 insertions(+), 504 deletions(-) create mode 100644 deploy/images/display/Dockerfile create mode 100644 deploy/images/display/README.md create mode 100755 deploy/images/display/swarm-display create mode 100644 deploy/images/display/swarm_display/__init__.py create mode 100644 deploy/images/display/swarm_display/__main__.py create mode 100644 deploy/images/display/swarm_display/browser.py create mode 100644 deploy/images/display/swarm_display/computer.py create mode 100644 deploy/images/display/swarm_display/daemon.py create mode 100644 go/internal/sandboxd/display.go create mode 100644 go/internal/sandboxd/display_integration_test.go create mode 100644 go/internal/sandboxd/display_test.go create mode 100644 go/internal/sandboxd/exec.go create mode 100644 spec/2026-10-07-browser-computer-use/README.md create mode 100644 swarmeval/runtime/display.py create mode 100644 tests/core/test_display.py create mode 100644 tests/events/test_display_export.py create mode 100644 tests/runtime/test_display.py create mode 100644 tests/worker/test_display_e2e.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4000f1b..5173b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,33 @@ ## [Unreleased] +### Added +- `browser` and `computer`: agents drive a web browser and a virtual screen inside their sandbox + (`spec/2026-10-07-browser-computer-use`). A profile's `display` (`env.yaml` schema version 2) + starts the display image's screen and Chromium; `browser` acts on accessibility-snapshot refs, + `computer` on screen pixels, both run as the display's own user, and their screenshots reach + the model as images. The sandbox keeps no network: the browser reaches loopback and files only. + `deploy/images/display` builds the image (`mise run display:build`). +- Tool results may carry images (`ToolResult.images`, `ToolMessage.images`, `ImageRef`), stored + as blobs and referenced by hash. `RequestOptions.max_images` (default 3) keeps a request to + the context's latest images; `before_model_request` may change it. +- model-gateway accepts `image_url` parts (inline PNG `data:` URLs) in tool messages and moves + them into a user message after the tool messages before sending upstream, after reasoning + passback. +- `Exec.user` and `Exec.collect`: a sandbox tool may run as another user and have sandboxd read + files out of the sandbox after the call. `SandboxTool.output` turns sandboxd's result into the + agent's. +- The `.eval` export embeds a run's images: tool results inline, contexts as attachments, with + `[image omitted]` where a request left one out. + ### Changed +- Event schema version 10: tool events carry `images`, model events `max_images`, `exec` + observations `collected`. Older runs read as before. +- `env.yaml` schema version 2 adds a profile's `display`. Version 1 files load as before. +- A fork at a turn after a run's first `browser` or `computer` call is refused + (`FAILED_PRECONDITION`): a fork restores files, not a browser. +- `GatewaySession` takes `images`, where it reads the images requests carry; + `check_transcript` takes `images` to rebuild them. - Models are chosen when a case is submitted, not written in the case (`spec/2026-10-06-run-time-models`). `case.yaml` schema version 4: an agent names a `model_slot` (default `default`) instead of a `model`, and agents in one slot run one model. diff --git a/console/CHANGELOG.md b/console/CHANGELOG.md index 6a19ec5..e2521e2 100644 --- a/console/CHANGELOG.md +++ b/console/CHANGELOG.md @@ -5,6 +5,8 @@ ### Added - The case flow draws no sandbox for an agent with `sandbox: none` (case version 5), and its details say `none`. New case's template is `schema_version: 5`. +- The case flow shows a profile's `display`: its screen size as a chip, and the size and first + page among its settings. - A case's Files tab opens on a flow: `case.yaml` and its env file, pending edits included, drawn as variant axes, the task, agents, channels, extensions, sandboxes, profiles, canaries, and scorers, with what connects them. A click shows a part's settings, its connections, its diff --git a/console/src/lib/caseGraph.test.ts b/console/src/lib/caseGraph.test.ts index c1a4431..6c3f875 100644 --- a/console/src/lib/caseGraph.test.ts +++ b/console/src/lib/caseGraph.test.ts @@ -271,3 +271,32 @@ test("paths expand over the axes they name", () => { expect(expandPath("p/${variant.x}_${variant.y}.md", axes)).toEqual(["p/a_1.md", "p/a_2.md", "p/b_1.md", "p/b_2.md"]) expect(expandPath("p/${variant.z}.md", axes)).toEqual(["p/${variant.z}.md"]) }) + +test("a profile's display shows its size and first page", () => { + const text = `schema_version: 4 +id: c +workspace: w +swarm: + agents: + - { id: a, prompt: a.md, tools: [browser, computer], sandbox_profile: desk } +task: { input: t.md } +` + const env = `schema_version: 2 +sandbox_profiles: + default: { image: busybox } + desk: + image: swarmeval/display:dev + display: { width: 1280, url: "http://127.0.0.1:8080/" } +` + const g = caseGraph( + new Map([ + ["case.yaml", text], + ["env.yaml", env], + ]) + ) + const desk = g.nodes.find((n) => n.id === "profile:desk")! + expect(desk.chips).toContain("display 1280x768") + expect(desk.fields).toContainEqual(["display", "1280x768 · http://127.0.0.1:8080/"]) + const plain = g.nodes.find((n) => n.id === "profile:default")! + expect(plain.fields.map(([k]) => k)).not.toContain("display") +}) diff --git a/console/src/lib/caseGraph.ts b/console/src/lib/caseGraph.ts index f607246..aef8961 100644 --- a/console/src/lib/caseGraph.ts +++ b/console/src/lib/caseGraph.ts @@ -381,6 +381,11 @@ export function caseGraph(texts: ReadonlyMap): CaseG const mounts = list(p.fs).map(obj) const limits = Object.entries(obj(p.limits)).map(([k, v]) => `${k} ${inline(v)}`) const copies = list(p.files).map(obj) + // A profile's screen, which `browser` and `computer` drive; sizes default to 1024x768. + const display = + p.display === undefined + ? "" + : `${inline(obj(p.display).width ?? 1024)}x${inline(obj(p.display).height ?? 768)}` nodes.push({ id: `profile:${name}`, kind: "profile", @@ -389,6 +394,7 @@ export function caseGraph(texts: ReadonlyMap): CaseG chips: [ ...mounts.map((m) => `${str(m.path)}${m.protected ? " protected" : ""}`), ...limits, + ...(display ? [`display ${display}`] : []), ], fields: rows( ["image", str(p.image)], @@ -399,6 +405,7 @@ export function caseGraph(texts: ReadonlyMap): CaseG .join(" · "), ]), ["limits", limits.join(", ")], + ["display", display && [display, str(obj(p.display).url)].filter(Boolean).join(" · ")], ...copies.map((f): [string, string] => ["copies", `${str(f.from)} → ${str(f.to)}`]) ), source: envYaml?.source(["sandbox_profiles", name]), diff --git a/deploy/images/display/Dockerfile b/deploy/images/display/Dockerfile new file mode 100644 index 0000000..5add9d2 --- /dev/null +++ b/deploy/images/display/Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 +# The display image: a virtual screen, a browser, and `swarm-display`, which the `computer` and +# `browser` tools drive. Cases extend it with `FROM swarmeval/display` and add their own apps, +# started from /etc/swarm-display/start.d/. Contract: README.md beside this file. Build from the +# repo root: +# docker build -f deploy/images/display/Dockerfile -t swarmeval/display:dev deploy/images/display + +FROM python:3.12-slim-bookworm + +# Xvfb is the screen; matchbox keeps every window maximized without decorations; xdotool moves +# the mouse and types; xauth writes the screen's cookie. No terminal: the only programs on the +# screen are the browser and what a case adds. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb xauth xdotool matchbox-window-manager fonts-dejavu-core fonts-noto-cjk \ + && rm -rf /var/lib/apt/lists/* + +ENV PLAYWRIGHT_BROWSERS_PATH=/opt/swarm-display/browsers +RUN pip install --no-cache-dir playwright==1.63.0 pillow==12.3.0 \ + && playwright install --with-deps --no-shell chromium \ + && rm -rf /var/lib/apt/lists/* /root/.cache + +# The display user owns the display's state; agents' users cannot read it, so they reach the +# screen and the browser only through the tools. +RUN useradd --uid 990 --create-home --shell /usr/sbin/nologin swarmdisplay \ + && install -d -o swarmdisplay -g swarmdisplay -m 0700 /run/swarm-display \ + && install -d -m 1777 /tmp/.X11-unix \ + && install -d -m 0755 /etc/swarm-display/start.d + +COPY swarm_display /opt/swarm-display/swarm_display +COPY swarm-display /usr/local/bin/swarm-display +RUN chmod 0755 /usr/local/bin/swarm-display \ + && python -m compileall -q /opt/swarm-display/swarm_display diff --git a/deploy/images/display/README.md b/deploy/images/display/README.md new file mode 100644 index 0000000..2281580 --- /dev/null +++ b/deploy/images/display/README.md @@ -0,0 +1,72 @@ +# swarmeval/display + +The sandbox image for the `browser` and `computer` tools: a virtual screen, Chromium on it, and +`swarm-display`, which drives both. A case gets it through a profile's +[`display`](../../../docs/case-format.md#display); sandboxd starts it +([docs/services/sandboxd.md](../../../docs/services/sandboxd.md#display)); the tools are in +[docs/agent-runtime.md](../../../docs/agent-runtime.md#tools). Why it is shaped this way: +[spec/2026-10-07-browser-computer-use](../../../spec/2026-10-07-browser-computer-use/README.md). + +```sh +mise run display:build # docker build -t swarmeval/display:dev deploy/images/display +``` + +Building downloads Debian packages and Playwright's Chromium; running needs no network. + +## What it holds + +| Piece | Version | Role | +|---|---|---| +| `python:3.12-slim-bookworm` | | Base, and the Python `swarm-display` runs on | +| Xvfb, xauth | Debian | The screen, `:1`, with a cookie only `swarmdisplay` can read | +| matchbox-window-manager | Debian | Keeps every window maximized, without decorations | +| xdotool | Debian | Mouse and keyboard for `computer` | +| Playwright, its Chromium | 1.63.0 | The browser, driven through a pipe: no debugging port | +| Pillow | 12.3.0 | Screenshots of the whole screen | +| DejaVu, Noto CJK fonts | Debian | Text in pages, Chinese and Japanese included | + +No terminal or file manager: the screen shows the browser and whatever a case adds. + +## Contract + +What sandboxd and the tools rely on. An image `FROM swarmeval/display` keeps it as long as it +does not remove these: + +- The user `swarmdisplay` (uid 990) and `/run/swarm-display`, owned by it, mode `0700`. +- `/usr/local/bin/swarm-display`: + - `start --width W --height H [--url U]`: starts the daemon (screen, window manager, start + scripts, browser), waits until the browser is up, prints the daemon's pid, and exits 0. + sandboxd runs it as `swarmdisplay` when it creates the sandbox. + - `computer ` and `browser `: one action, as the tools describe it. The text goes + to stdout; an image, if the action took one, to `/run/swarm-display/out/screenshot.png`, + which sandboxd collects and removes. Exit status 1 means the action failed; the reason is on + stderr. They must run as `swarmdisplay`. +- `sh`, `sleep`, and `tr`, which every sandbox image needs. + +## Adding a case's apps + +Put executables in `/etc/swarm-display/start.d/`. The daemon runs them in name order, as +`swarmdisplay`, after the screen is up and before the browser opens, each with 30 seconds to +return; a non-zero exit fails the sandbox. Start servers in the background: + +```dockerfile +FROM swarmeval/display:dev +COPY shop /opt/shop +COPY <<'EOF' /etc/swarm-display/start.d/10-shop +#!/bin/sh +cd /opt/shop && nohup python3 -m http.server 8080 --bind 127.0.0.1 >/dev/null 2>&1 & +EOF +RUN chmod 0755 /etc/swarm-display/start.d/10-shop +``` + +and point the profile's `display.url` at `http://127.0.0.1:8080/`. Their processes run as +`swarmdisplay`, so sandboxd counts them as the display's: they are not reported as `proc.*` +events. + +## Limits + +- The display's state, Chromium's profile included, lives in `/run/swarm-display`, outside every + key path: nothing the browser writes is a file change, and downloads are not kept. +- Dialogs (`alert`, `confirm`) are dismissed. +- The browser locates elements by snapshot ref through the `aria-ref=` selector. Playwright 1.64 + makes `page.get_by_ref` public; move to it when upgrading. diff --git a/deploy/images/display/swarm-display b/deploy/images/display/swarm-display new file mode 100755 index 0000000..a1a0ef9 --- /dev/null +++ b/deploy/images/display/swarm-display @@ -0,0 +1,3 @@ +#!/bin/sh +# The display's command line: `start`, `computer`, `browser`. See README.md. +PYTHONPATH=/opt/swarm-display exec /usr/local/bin/python3 -s -m swarm_display "$@" diff --git a/deploy/images/display/swarm_display/__init__.py b/deploy/images/display/swarm_display/__init__.py new file mode 100644 index 0000000..f0ee773 --- /dev/null +++ b/deploy/images/display/swarm_display/__init__.py @@ -0,0 +1,20 @@ +"""The display stack of a sandbox: a virtual screen, a browser on it, and the daemon that drives +both for the `computer` and `browser` tools (deploy/images/display/README.md). + +Everything here runs inside the sandbox as the user `swarmdisplay`. +""" + +from pathlib import Path + +STATE = Path("/run/swarm-display") +SOCKET = STATE / "daemon.sock" +LOG = STATE / "daemon.log" +SCREENSHOT = STATE / "out" / "screenshot.png" +"""Written by an action that has an image; sandboxd collects and removes it after the call.""" +DISPLAY = ":1" +XAUTHORITY = STATE / "Xauthority" +START_DIR = Path("/etc/swarm-display/start.d") + + +class ActionError(Exception): + """The action could not be done; the message is what the agent sees.""" diff --git a/deploy/images/display/swarm_display/__main__.py b/deploy/images/display/swarm_display/__main__.py new file mode 100644 index 0000000..e59f53f --- /dev/null +++ b/deploy/images/display/swarm_display/__main__.py @@ -0,0 +1,117 @@ +"""`swarm-display start | computer | browser | daemon`. + +`start` launches the daemon, waits until the screen and the browser are up, prints the daemon's +pid, and exits; sandboxd runs it once, when it creates the sandbox. `computer` and `browser` +send one action to the daemon and print its text; an image, if any, is at `SCREENSHOT`. Exit +status 1 means the action failed, and the reason is on stderr. +""" + +import argparse +import json +import os +import socket +import subprocess +import sys +import time + +from swarm_display import LOG, SCREENSHOT, SOCKET, STATE + +START_TIMEOUT_S = 50.0 +ACTION_TIMEOUT_S = 180.0 + + +def _start(width: int, height: int, url: str) -> int: + if SOCKET.exists(): + print(f"the display is already running ({SOCKET} exists)", file=sys.stderr) + return 1 + for sub in ("home", "tmp", "out", "profile"): + (STATE / sub).mkdir(mode=0o700, exist_ok=True) + argv = [sys.executable, "-s", "-m", "swarm_display", "daemon"] + argv += ["--width", str(width), "--height", str(height), "--url", url] + with open(LOG, "ab") as log: + daemon = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=log, + start_new_session=True, + cwd=STATE, + ) + deadline = time.monotonic() + START_TIMEOUT_S + while time.monotonic() < deadline: + if daemon.poll() is not None: + print(f"the display daemon exited {daemon.returncode}:\n{_log_tail()}", file=sys.stderr) + return 1 + if SOCKET.exists(): + print(daemon.pid) + return 0 + time.sleep(0.1) + daemon.kill() + print(f"the display was not up after {START_TIMEOUT_S:.0f}s:\n{_log_tail()}", file=sys.stderr) + return 1 + + +def _log_tail() -> str: + try: + return LOG.read_text(errors="replace")[-4000:] + except FileNotFoundError: + return "(no log)" + + +def _act(tool: str, raw: str) -> int: + try: + args = json.loads(raw) + except ValueError as err: + print(f"arguments are not JSON: {err}", file=sys.stderr) + return 1 + try: + SCREENSHOT.unlink(missing_ok=True) + except PermissionError: + print(f"swarm-display {tool} must run as the display's user, swarmdisplay", file=sys.stderr) + return 1 + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as conn: + conn.settimeout(ACTION_TIMEOUT_S) + try: + conn.connect(str(SOCKET)) + except OSError as err: + print(f"the display is not running ({SOCKET}: {err})", file=sys.stderr) + return 1 + conn.sendall(json.dumps({"tool": tool, "args": args}).encode() + b"\n") + with conn.makefile("rb") as reply: + line = reply.readline() + if not line: + print("the display daemon closed the connection without an answer", file=sys.stderr) + return 1 + answer = json.loads(line) + if not answer["ok"]: + print(answer["text"], file=sys.stderr) + return 1 + sys.stdout.write(answer["text"]) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(prog="swarm-display") + sub = parser.add_subparsers(dest="command", required=True) + for name in ("start", "daemon"): + p = sub.add_parser(name) + p.add_argument("--width", type=int, required=True) + p.add_argument("--height", type=int, required=True) + p.add_argument("--url", default="about:blank") + for name in ("computer", "browser"): + sub.add_parser(name).add_argument("arguments") + opts = parser.parse_args() + os.umask(0o077) + match opts.command: + case "start": + return _start(opts.width, opts.height, opts.url) + case "daemon": + from swarm_display.daemon import run + + return run(opts.width, opts.height, opts.url) + case tool: + return _act(tool, opts.arguments) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/images/display/swarm_display/browser.py b/deploy/images/display/swarm_display/browser.py new file mode 100644 index 0000000..2c04425 --- /dev/null +++ b/deploy/images/display/swarm_display/browser.py @@ -0,0 +1,155 @@ +"""`browser`: Chromium on the screen, driven through Playwright. + +Every action answers with the active tab's URL, title, and an accessibility snapshot in which +each element carries a `[ref=eN]`; actions on an element name that ref. The window is the same +one `computer` sees, so the two tools can be mixed. +""" + +import asyncio +from typing import Any + +from playwright.async_api import BrowserContext, Locator, Page, async_playwright + +from swarm_display import SCREENSHOT, STATE, ActionError + +SNAPSHOT_CHARS = 40_000 +ACTION_TIMEOUT_MS = 10_000 +NAVIGATION_TIMEOUT_MS = 30_000 +SCROLL_STEP_PX = 400 + + +class Browser: + def __init__(self, context: BrowserContext) -> None: + self.context = context + self.active = context.pages[0] + context.on("page", self._opened) + context.set_default_timeout(ACTION_TIMEOUT_MS) + context.set_default_navigation_timeout(NAVIGATION_TIMEOUT_MS) + + @classmethod + async def launch(cls, width: int, height: int, url: str) -> "Browser": + playwright = await async_playwright().start() + context = await playwright.chromium.launch_persistent_context( + user_data_dir=STATE / "profile", + headless=False, + no_viewport=True, + # gVisor or runc is the sandbox; Chromium's own needs namespaces a sandbox lacks. + chromium_sandbox=False, + args=[ + "--window-position=0,0", + f"--window-size={width},{height}", + "--start-maximized", + "--disable-dev-shm-usage", + "--disable-gpu", + "--no-first-run", + "--disable-breakpad", + ], + ) + browser = cls(context) + if url != "about:blank": + await browser.active.goto(url, wait_until="domcontentloaded") + return browser + + def _opened(self, page: Page) -> None: + self.active = page + + async def act(self, args: dict[str, Any]) -> str: + action = args.get("action") + page = self.active + match action: + case "navigate": + await page.goto(_text(args, "url"), wait_until="domcontentloaded") + case "back": + await page.go_back(wait_until="domcontentloaded") + case "forward": + await page.go_forward(wait_until="domcontentloaded") + case "reload": + await page.reload(wait_until="domcontentloaded") + case "snapshot" | "screenshot": + pass + case "click": + await _ref(page, args).click() + case "hover": + await _ref(page, args).hover() + case "type": + target = _ref(page, args) + await target.fill(_text(args, "text")) + if args.get("submit"): + await target.press("Enter") + case "select": + values = args.get("values") + if not isinstance(values, list): + raise ActionError("`values` must be a list of option values or labels") + await _ref(page, args).select_option([str(v) for v in values]) + case "press": + await page.keyboard.press(_text(args, "key")) + case "scroll": + steps = int(args.get("amount", 3)) * SCROLL_STEP_PX + dx, dy = {"up": (0, -steps), "down": (0, steps), "left": (-steps, 0)}.get( + args.get("direction", "down"), (steps, 0) + ) + await page.mouse.wheel(dx, dy) + case "wait": + await asyncio.sleep(float(args.get("seconds", 1))) + case "tabs": + pass + case "tab_new": + page = await self.context.new_page() + self.active = page + if args.get("url"): + await page.goto(_text(args, "url"), wait_until="domcontentloaded") + case "tab_select": + self.active = self._tab(args) + await self.active.bring_to_front() + case "tab_close": + closing = self._tab(args) if "index" in args else page + if len(self.context.pages) == 1: + raise ActionError("this is the last tab; open another before closing it") + await closing.close() + self.active = self.context.pages[-1] + await self.active.bring_to_front() + case _: + raise ActionError(f"unknown browser action {action!r}") + if action == "screenshot" or args.get("screenshot"): + await self.active.screenshot(path=SCREENSHOT, type="png") + return await self._describe() + + def _tab(self, args: dict[str, Any]) -> Page: + index = args.get("index") + pages = self.context.pages + if not isinstance(index, int) or not 1 <= index <= len(pages): + raise ActionError(f"`index` must be a tab number from 1 to {len(pages)}") + return pages[index - 1] + + async def _describe(self) -> str: + page = self.active + pages = self.context.pages + lines = [f"Tab {pages.index(page) + 1} of {len(pages)}"] + if len(pages) > 1: + for i, p in enumerate(pages, 1): + lines.append(f" {i}. {await p.title()} - {p.url}") + lines += [f"URL: {page.url}", f"Title: {await page.title()}", ""] + snapshot = await page.aria_snapshot(mode="ai") + if len(snapshot) > SNAPSHOT_CHARS: + snapshot = ( + snapshot[:SNAPSHOT_CHARS] + + f"\n[snapshot cut at {SNAPSHOT_CHARS} of {len(snapshot)} characters; " + "scroll, or act on what is shown]" + ) + lines.append(snapshot) + return "\n".join(lines) + "\n" + + +def _text(args: dict[str, Any], name: str) -> str: + value = args.get(name) + if not isinstance(value, str) or not value: + raise ActionError(f"`{name}` is required") + return value + + +def _ref(page: Page, args: dict[str, Any]) -> Locator: + """The element a snapshot named. Playwright 1.64's `page.get_by_ref` replaces the selector.""" + ref = _text(args, "ref") + if not ref.isalnum(): + raise ActionError(f"`ref` {ref!r} is not a ref from the snapshot, such as `e12`") + return page.locator(f"aria-ref={ref}") diff --git a/deploy/images/display/swarm_display/computer.py b/deploy/images/display/swarm_display/computer.py new file mode 100644 index 0000000..735b5bc --- /dev/null +++ b/deploy/images/display/swarm_display/computer.py @@ -0,0 +1,99 @@ +"""`computer`: the mouse, the keyboard, and screenshots of the whole screen, in screen pixels. + +The worker validates arguments before they get here (swarmeval/runtime/display.py); this module +checks only what it needs to act safely, such as coordinates inside the screen. +""" + +import asyncio +from typing import Any + +from PIL import ImageGrab + +from swarm_display import DISPLAY, SCREENSHOT, ActionError + +SETTLE_S = 0.4 +"""Time for the screen to redraw after an action, before its screenshot.""" +TYPE_DELAY_MS = 12 +BUTTONS = {"left": "1", "middle": "2", "right": "3"} +WHEEL = {"up": "4", "down": "5", "left": "6", "right": "7"} + + +class Computer: + def __init__(self, width: int, height: int) -> None: + self.width = width + self.height = height + + async def act(self, args: dict[str, Any]) -> str: + action = args.get("action") + match action: + case "screenshot": + pass + case "click": + x, y = self._point(args, "coordinate") + button = BUTTONS[args.get("button", "left")] + count = str(args.get("count", 1)) + await _xdotool("mousemove", "--sync", x, y, "click", "--repeat", count, button) + case "mouse_move": + await _xdotool("mousemove", "--sync", *self._point(args, "coordinate")) + case "drag": + x0, y0 = self._point(args, "start_coordinate") + x1, y1 = self._point(args, "coordinate") + await _xdotool("mousemove", "--sync", x0, y0, "mousedown", "1") + await _xdotool("mousemove", "--sync", x1, y1, "mouseup", "1") + case "type": + await _xdotool("type", "--delay", str(TYPE_DELAY_MS), "--", str(args["text"])) + case "key": + await _xdotool("key", "--", *str(args["text"]).split()) + case "scroll": + if "coordinate" in args: + await _xdotool("mousemove", "--sync", *self._point(args, "coordinate")) + wheel = WHEEL[args["direction"]] + await _xdotool("click", "--repeat", str(args.get("amount", 3)), wheel) + case "wait": + await asyncio.sleep(float(args.get("seconds", 1))) + case "cursor_position": + out = await _xdotool("getmouselocation", "--shell") + pos = dict(line.split("=", 1) for line in out.splitlines() if "=" in line) + return f"The mouse is at ({pos['X']}, {pos['Y']}) on a {self._size()} screen.\n" + case _: + raise ActionError(f"unknown computer action {action!r}") + if action != "screenshot": + await asyncio.sleep(SETTLE_S) + await asyncio.to_thread(self._screenshot) + return f"Did `{action}`. Screenshot of the {self._size()} screen attached.\n" + + def _size(self) -> str: + return f"{self.width}x{self.height}" + + def _point(self, args: dict[str, Any], name: str) -> tuple[str, str]: + point = args.get(name) + if not isinstance(point, list) or len(point) != 2: + raise ActionError(f"`{name}` must be [x, y]") + x, y = point + if not (isinstance(x, int) and isinstance(y, int)): + raise ActionError(f"`{name}` must be two integers, got {point!r}") + if not (0 <= x < self.width and 0 <= y < self.height): + raise ActionError( + f"`{name}` {point!r} is off the {self._size()} screen: x runs 0 to " + f"{self.width - 1}, y 0 to {self.height - 1}" + ) + return str(x), str(y) + + def _screenshot(self) -> None: + image = ImageGrab.grab(xdisplay=DISPLAY) + partial = SCREENSHOT.with_suffix(".partial") + image.save(partial, format="PNG") + partial.replace(SCREENSHOT) + + +async def _xdotool(*argv: str) -> str: + proc = await asyncio.create_subprocess_exec( + "xdotool", + *argv, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await proc.communicate() + if proc.returncode: + raise ActionError(f"xdotool {' '.join(argv)} failed: {err.decode(errors='replace')}") + return out.decode(errors="replace") diff --git a/deploy/images/display/swarm_display/daemon.py b/deploy/images/display/swarm_display/daemon.py new file mode 100644 index 0000000..91612a9 --- /dev/null +++ b/deploy/images/display/swarm_display/daemon.py @@ -0,0 +1,130 @@ +"""The daemon: starts the screen, a window manager, the case's start scripts, and the browser, +then answers one action at a time on a unix socket only `swarmdisplay` can open. + +The browser is driven through Playwright's pipe, so no debugging port is open in the sandbox. The +screen needs the cookie in `XAUTHORITY`, which only `swarmdisplay` can read. +""" + +import asyncio +import json +import os +import secrets +import subprocess +import time +from typing import Any + +from playwright.async_api import Error as PlaywrightError + +from swarm_display import DISPLAY, SOCKET, START_DIR, STATE, XAUTHORITY, ActionError +from swarm_display.browser import Browser +from swarm_display.computer import Computer + +START_SCRIPT_TIMEOUT_S = 30.0 + + +def _environment() -> None: + os.environ.update( + { + "DISPLAY": DISPLAY, + "XAUTHORITY": str(XAUTHORITY), + "HOME": str(STATE / "home"), + "TMPDIR": str(STATE / "tmp"), + "XDG_CONFIG_HOME": str(STATE / "home" / ".config"), + "XDG_CACHE_HOME": str(STATE / "home" / ".cache"), + "XDG_RUNTIME_DIR": str(STATE / "tmp"), + } + ) + + +def _screen(width: int, height: int) -> None: + cookie = secrets.token_hex(16) + XAUTHORITY.touch(mode=0o600) + subprocess.run(["xauth", "-q", "-f", str(XAUTHORITY), "add", DISPLAY, ".", cookie], check=True) + subprocess.Popen( + [ + "Xvfb", + DISPLAY, + "-screen", + "0", + f"{width}x{height}x24", + "-nolisten", + "tcp", + "-auth", + str(XAUTHORITY), + "-dpi", + "96", + ], + stdin=subprocess.DEVNULL, + ) + deadline = time.monotonic() + 15 + while subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True).returncode: + if time.monotonic() > deadline: + raise RuntimeError("Xvfb did not accept connections within 15s") + time.sleep(0.1) + subprocess.Popen( + ["matchbox-window-manager", "-use_titlebar", "no", "-use_cursor", "yes"], + stdin=subprocess.DEVNULL, + ) + + +def _start_scripts() -> None: + if not START_DIR.is_dir(): + return + for script in sorted(START_DIR.iterdir()): + if script.is_file() and os.access(script, os.X_OK): + print(f"running {script}", flush=True) + subprocess.run( + [str(script)], stdin=subprocess.DEVNULL, check=True, timeout=START_SCRIPT_TIMEOUT_S + ) + + +async def _serve(computer: Computer, browser: Browser) -> None: + lock = asyncio.Lock() + + async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + # Anything but an ActionError or a browser error is a bug: asyncio logs it to the daemon + # log, the connection closes unanswered, and the next action is served as usual. + try: + line = await reader.readline() + async with lock: + answer = await _answer(json.loads(line), computer, browser) + writer.write(json.dumps(answer).encode() + b"\n") + await writer.drain() + finally: + writer.close() + + server = await asyncio.start_unix_server(handle, path=str(SOCKET)) + os.chmod(SOCKET, 0o600) + async with server: + await server.serve_forever() + + +async def _answer(request: dict[str, Any], computer: Computer, browser: Browser) -> dict[str, Any]: + tool, args = request["tool"], request["args"] + try: + if not isinstance(args, dict): + raise ActionError("arguments must be a JSON object") + if tool == "computer": + text = await computer.act(args) + elif tool == "browser": + text = await browser.act(args) + else: + raise ActionError(f"unknown tool {tool!r}") + except ActionError as err: + return {"ok": False, "text": str(err)} + except PlaywrightError as err: + return {"ok": False, "text": f"The browser refused: {err.message}"} + return {"ok": True, "text": text} + + +def run(width: int, height: int, url: str) -> int: + _environment() + _screen(width, height) + _start_scripts() + + async def main() -> None: + browser = await Browser.launch(width, height, url) + await _serve(Computer(width, height), browser) + + asyncio.run(main()) + return 0 diff --git a/docs/agent-runtime.md b/docs/agent-runtime.md index 9cfc2b3..1a7f95f 100644 --- a/docs/agent-runtime.md +++ b/docs/agent-runtime.md @@ -103,11 +103,11 @@ message or a new generation, so a request can always be rebuilt from `messages`. | Kind | Where it runs | Defined by | |---|---|---| -| `SandboxTool` | sandboxd, in the calling agent's sandbox. Its `build` only turns arguments into an `Exec` | The runtime, or an extension with `runs_in="sandbox"` | +| `SandboxTool` | sandboxd, in the calling agent's sandbox. Its `build` only turns arguments into an `Exec`, which may name the user to run as (instead of the agent's `os_user`) and files to `collect`; its optional `output` turns sandboxd's result into what the agent sees (`exec_output` when unset) | The runtime, or an extension with `runs_in="sandbox"` | | `WorkerTool` | The worker, with only its extension's `HookContext`. It must not do its own I/O | An extension with `runs_in="worker"` | | `WebTool` | The worker, through the run's `WebClient`. Its `build` only turns arguments into a `WebRequest` | The runtime: `web_request` | -The runtime provides three tools itself: +The runtime provides five tools itself: - `shell` (`swarmeval.runtime.tools.SHELL`): `cmd` runs with `sh -c` in the agent's sandbox, with `timeout_s` from 0 to 600 seconds, 60 by default. The agent sees stdout then stderr, each cut @@ -119,11 +119,39 @@ The runtime provides three tools itself: `headers`, `body`, and `timeout_s` up to 120, 30 by default. Sent by the worker, never the sandbox; the agent sees the status line, headers, and the body cut at 64 KiB. Only for agents that list it. See [orchestrator.md](services/orchestrator.md#web_request). +- `computer` and `browser` (`swarmeval.runtime.display`, handed to the loop as `DISPLAY_TOOLS`): + the screen and the web browser of a sandbox whose profile has a + [`display`](case-format.md#display). Both run `swarm-display` in the sandbox as the user + `swarmdisplay`, not the agent's `os_user`, and collect the screenshot an action takes, which + reaches the agent as an image ([below](#images)). `computer`: `screenshot`, `click` + (`coordinate`, `button`, `count`), `mouse_move`, `drag` (`start_coordinate`, `coordinate`), + `type` and `key` (`text`, xdotool key names for `key`), `scroll` (`direction`, `amount`, + optional `coordinate`), `wait` (`seconds`), `cursor_position`. Coordinates are `[x, y]` screen + pixels. Every action but `cursor_position` returns a screenshot. `browser`: `navigate` + (`url`), `back`, `forward`, `reload`, `snapshot`, `screenshot`, `click` and `hover` (`ref`), + `type` (`ref`, `text`, `submit`), `select` (`ref`, `values`), `press` (`key`), `scroll`, + `wait`, `tabs`, `tab_new` (`url`), `tab_select` and `tab_close` (`index`). Every action + returns the active tab's URL, title, and an accessibility snapshot in which each element + carries a `[ref=eN]`; `screenshot`, or any action with `screenshot: true`, adds an image of + the page. Arguments are checked against the actions' required fields before anything runs. A `RuntimeTool` runs in the worker without I/O and returns its result with the events it causes, which commit with the tool call. `BUILTIN_TOOL_NAMES` lists every runtime tool name; pass it to `load_extensions` so no extension reuses one. +### Images + +A tool result may carry images: `ToolResult.images` and `ToolMessage.images`, each an `ImageRef` +(`sha256`, `media_type` `image/png`, `width`, `height`). The bytes are in the blob store, uploaded +before the tool's event commits; contexts and events hold only the reference. An +`after_tool_result` hook may drop or replace images like any other part of the result. Only tool +results carry images. + +`RequestOptions.max_images` (default 3) caps the images a request carries: the context's last +`max_images` go with it, and each earlier one is replaced by the text `[image omitted]` +(`visible_images`). `before_model_request` may change it. It is recorded on the model event, and +the export expands the request's input with the same rule. + Arguments are validated against the tool's pydantic model. Invalid arguments, an unknown tool, a non-zero exit, and a timeout all become error results the agent sees, and each is recorded. A call counts as unknown unless the request that produced it offered the tool, so a tool that @@ -261,7 +289,7 @@ agents that use the sandbox), `token`, and where it is planted: `hostname`, `env | `on_resume` | Observe | `ResumeInfo`: `fork`, `source_run_id`, `at_seq`, `fidelity` (`fs_restored` or `fs_partial`) | `None`. Runs instead of `on_run_start` when a run goes on from another run's state, after the state is restored and before the first turn; extension state is already the source's | | `before_turn` | Gate | `TurnInfo` | `Proceed`, `Skip`, `Inject(messages)`, `Stop(reason)` | | `compact_context` | Transform | Current messages | `None`, or a new, non-empty message tuple (new generation). An empty tuple fails the run | -| `before_model_request` | Transform | `RequestOptions` | `RequestOptions`. Tools may only be narrowed | +| `before_model_request` | Transform | `RequestOptions` | `RequestOptions`. Tools may only be narrowed. `max_images` may change ([Images](#images)) | | `after_model_response` | Transform | `AssistantMessage` | `AssistantMessage` | | `before_tool_call` | Gate | `ToolCall` | `Allow`, `Rewrite(arguments)`, `Block(result)` | | `after_tool_result` | Transform | `ToolResult` | `ToolResult` | diff --git a/docs/architecture.md b/docs/architecture.md index 33dac88..8d26dd7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -162,7 +162,9 @@ flowchart LR the call's `ToolEvent`, committed before the agent sees the result, like any other tool result. Detail: [services/orchestrator.md](services/orchestrator.md#web_request). - File and process events are collected by `sandboxd` after each tool call: a diff of the key paths, - which are mounted as host volumes, and the processes still alive. They come back with the tool + which are mounted as host volumes, and the processes still alive. Screenshots the `browser` and + `computer` tools take are files sandboxd collects from outside the key paths after the call; + they reach the model as images and are stored as blobs. They come back with the tool result and commit in the same transaction as its `ToolEvent`. Reads that write nothing are not recorded; a decoy file counts as hit when its canary content shows up anywhere. Syscall-level audit is an opt-in interface backed by gVisor Runtime Monitoring. @@ -207,6 +209,10 @@ place with pyarrow and queries them with DuckDB in process ([analysis.md](servic [services/orchestrator.md](services/orchestrator.md#web_request). - The agent loop runs outside the sandbox. Containers only execute tool calls, so model keys, logs, and recovery state are out of the agent's reach. +- A sandbox whose profile has a display runs a screen and a browser inside it, as the user + `swarmdisplay`. The browser has the sandbox's network, so it reaches only loopback and files. + Agents' own users reach the screen and the browser only through the `browser` and `computer` + tools. Detail: [services/sandboxd.md](services/sandboxd.md#display). - Extensions run in the worker, with its privileges. A case's own extensions (`case:` files) are case author code, not agent code, and run only where the deployment allows it (`--allow-case-code`, off by default). Sandbox isolation does not cover them. diff --git a/docs/case-format.md b/docs/case-format.md index e484ce0..5a78ae0 100644 --- a/docs/case-format.md +++ b/docs/case-format.md @@ -99,7 +99,7 @@ An agent: | `model_slot` | no | The [model slot](#model-slots) the agent runs in. Default `default` | | `prompt` | yes | System prompt file | | `task` | no | First user message file. Overrides `task.input` | -| `tools` | no | Tool names: `shell`, `send_message`, `web_request`, or one an extension provides. The run fails to start if nothing provides one. `web_request` is the agent's only way to the internet | +| `tools` | no | Tool names: `shell`, `send_message`, `web_request`, `browser`, `computer`, or one an extension provides. The run fails to start if nothing provides one. `web_request` is the agent's only way to the internet. `browser` and `computer` need a sandbox whose profile has a [`display`](#display) | | `sandbox` | no | Join a shared instance from `env.yaml`, or, from version 5, `none` for [no sandbox](#sandboxes) | | `sandbox_profile` | no | Profile of the agent's private sandbox. Exclusive with `sandbox` | | `os_user` | no | Unix user the agent's commands run as; not with `sandbox: none`. sandboxd adds it to the sandbox's image with a private home under `/home`, unless the image has it already. Where it may write besides its home is up to the image | @@ -143,8 +143,8 @@ console's run form asks models for. | Key | Meaning | |---|---| -| `schema_version` | `1` | -| `sandbox_profiles` | Name → `image`, `fs` mounts (`path`, `mode: rw \| ro`, `protected`), `limits` (`cpu`, `memory`, `pids`, `disk`), `files` (below) | +| `schema_version` | `1` or `2` | +| `sandbox_profiles` | Name → `image`, `fs` mounts (`path`, `mode: rw \| ro`, `protected`), `limits` (`cpu`, `memory`, `pids`, `disk`), `files` (below), `display` ([below](#display), version 2) | | `sandboxes` | Name → `profile`. Declare only instances that agents share | | `canaries` | Files holding a token generated per run, below | @@ -342,6 +342,40 @@ rejects the following: - A shared instance named like an agent that has a private sandbox. - From version 5, a shared instance named `none`, which no agent can join. +## Display + +A profile with `display` gives its sandboxes a virtual screen with a web browser on it, which the +`browser` and `computer` tools drive ([agent-runtime.md](agent-runtime.md#tools)). It needs +`env.yaml` version 2 and an image built from `swarmeval/display` +([deploy/images/display](../deploy/images/display/README.md)). + +```yaml +schema_version: 2 +sandbox_profiles: + desktop: + image: swarmeval/display:dev # or an image FROM it, with the case's own apps + fs: [{ path: /workspace }] + limits: { memory: 2gib, pids: 512 } + files: [{ from: site, to: /workspace }] + display: { width: 1024, height: 768, url: "file:///workspace/index.html" } +``` + +| Key | Default | Meaning | +|---|---|---| +| `width`, `height` | `1024`, `768` | The screen in pixels, 320×240 to 1920×1200. At 1024×768 neither Claude nor GPT scales a screenshot, so the coordinates a model reads off it are screen pixels | +| `url` | `about:blank` | The page the browser opens first. Sandboxes have no network: pages come from `file://` or a server inside the sandbox | + +- The display runs as the user `swarmdisplay` and keeps its state in `/run/swarm-display`. A key + path there, above it, or at `/` is rejected. Agents' own users cannot reach the screen or the + browser except through the two tools. +- Programs the case needs on the screen or behind the browser, such as a web shop on + `127.0.0.1:8080`, start from executables in the image's `/etc/swarm-display/start.d/`, before + the browser opens. +- Agents sharing a sandbox share its screen. +- The browser is the memory of the run: a [fork](services/orchestrator.md#forks) restores files + but not the screen, so a fork after an agent's first `browser` or `computer` call is refused. +- Chromium needs memory: give the profile at least `memory: 1gib` and a few hundred `pids`. + ## Variants The cartesian product of the chosen [models](#model-slots), slot by slot, then of `variants:` @@ -451,7 +485,10 @@ error says how to move a case: set `schema_version: 4`, remove each agent's `mod | 1 | The format as first released; `models` filled each case's `model` variant axis. No longer loads | | 2 | `models` fills model slots, as a list (the `default` slot) or by slot; entries take `models` | -`env.yaml` is at version 1. +| `env.yaml` | Adds | +|---|---| +| 1 | The format as first released | +| 2 | A profile's [`display`](#display) | ## Not accepted yet diff --git a/docs/development.md b/docs/development.md index aad855a..85605bf 100644 --- a/docs/development.md +++ b/docs/development.md @@ -31,10 +31,11 @@ mise run sync | `mise run lint` | `ruff format --check .` and `ruff check .` | | `mise run typecheck` | `pyright` in strict mode over `swarmeval/` and `tests/` | | `mise run test` | `pytest` (asyncio mode `auto`), without tests marked `docker` or `browser` | -| `mise run test:docker` | `pytest -m docker`: the Postgres store, migrations, and export against a throwaway `postgres:18-alpine` and `rustfs/rustfs` from testcontainers, the sandboxd client against a sandboxd it builds from `go/` (needs `busybox:latest`), the public API through an `edge` it builds from `go/`, and mutual TLS between the Go and Python services with certificates from a `swarm-certs` it builds. Run it before a change to `swarmeval/db/`, `swarmeval/events/`, `swarmeval/sandbox/`, `swarmeval/control/`, or edge is done | +| `mise run test:docker` | `pytest -m docker`: the Postgres store, migrations, and export against a throwaway `postgres:18-alpine` and `rustfs/rustfs` from testcontainers, the sandboxd client against a sandboxd it builds from `go/` (needs `busybox:latest`; the `browser` and `computer` test also needs `swarmeval/display:dev` and skips without it), the public API through an `edge` it builds from `go/`, and mutual TLS between the Go and Python services with certificates from a `swarm-certs` it builds. Run it before a change to `swarmeval/db/`, `swarmeval/events/`, `swarmeval/sandbox/`, `swarmeval/control/`, or edge is done | | `mise run go:lint` | `golangci-lint run` over `go/`, integration tests included | | `mise run go:test` | `go test ./...` in `go/` | -| `mise run go:test-integration` | sandboxd against the local docker daemon (needs `busybox:latest` and `python:3.12-slim`), and edge against a throwaway `postgres:18-alpine` from testcontainers-go | +| `mise run go:test-integration` | sandboxd against the local docker daemon (needs `busybox:latest` and `python:3.12-slim`; the display test also `swarmeval/display:dev`, and skips without it), and edge against a throwaway `postgres:18-alpine` from testcontainers-go | +| `mise run display:build` | Builds `swarmeval/display:dev`, the [display image](../deploy/images/display/README.md) cases with `browser` and `computer` run on. Downloads Debian packages and Playwright's Chromium | | `mise run console:install` | `pnpm install --frozen-lockfile` in `console/`. The other console tasks run it first | | `mise run console:check` | The console's `tsc -b`, `eslint`, and `vitest` unit tests | | `mise run console:build` | Builds the console into `go/internal/edge/webui/static/` (not committed), where edge embeds it from. Build edge after it; an edge built without it serves the API only | diff --git a/docs/event-log.md b/docs/event-log.md index 52d2703..89b6dfa 100644 --- a/docs/event-log.md +++ b/docs/event-log.md @@ -56,9 +56,13 @@ The hash chain is not in the payload, because the hash covers the payload. It li `prev_hash` / `hash` columns and is added to `metadata.swarmeval` on export. Kind-specific fields also go under `metadata.swarmeval`. A `ModelEvent` carries `input` (the -`gen` / `len` reference, below), the offered `tools`, and `raw_tool_arguments`, the argument -text the model produced. A `ToolEvent` carries `raw_arguments`, `executed_arguments`, -`blocked_by`, and `exec`. Inspect keeps tool arguments as a parsed object. Arguments that are not +`gen` / `len` reference, below), the offered `tools`, `raw_tool_arguments`, the argument +text the model produced, and `max_images`, how many of the context's latest images the request +carried (schema version 10). A `ToolEvent` carries `raw_arguments`, `executed_arguments`, +`blocked_by`, `exec` (with `collected`, the files sandboxd read out of the sandbox after the call, +from version 10), and `images` when the result has any: `{sha256, media_type, width, height}` +each, the bytes in the [blob store](#large-objects). `result` stays the text; the export adds the +images ([below](#export)). Inspect keeps tool arguments as a parsed object. Arguments that are not a JSON object, or hold a value JSON cannot carry exactly (NaN, a number that overflows a 64-bit float, an integer beyond ±(2**53 − 1)), are stored as `{}`, and the raw text is kept. @@ -219,7 +223,7 @@ event, `InfoEvent(source="swarmeval.transcript_check")`, so both exports carry i | Check | What must hold | Mismatch names | |---|---|---| | `context` | Walking each agent's generations in `idx` order: generation 0 starts with the case's system prompt and task; a later generation starts with the `after` of a `compact_context` intervention. After that, an assistant message at index *i* of generation *g* is the response of the agent's model call built from `(g, i)`; the tool messages after it are the results of that call's `ToolEvent`s (`parent_id`), in `seq` order, with the call's id; a user message is a `msg.deliver` to this agent (as the bus words it), an injection into it (`ctx.actions.inject`), or a `before_turn` `Inject`. Where an `after_model_response` / `after_tool_result` intervention rewrote the event, the last rewrite's `after` is what must appear instead | The model or tool event the message was compared with; none when no event could explain it. A generation whose start nothing explains is reported once and not walked further | -| `request` | Each agent model call's request, rebuilt from the stored context (`gen`, `len`), the tools it offered, and its sampling options, hashes (sha256 of the wire body the worker sends) to the gateway's `request_sha256` | The model event | +| `request` | Each agent model call's request, rebuilt from the stored context (`gen`, `len`), the tools it offered, its sampling options, and the images it carried (read from the blob store, `max_images`), hashes (sha256 of the wire body the worker sends) to the gateway's `request_sha256` | The model event | | `response` | Each model call's response equals the gateway's `upstream_response_json` normalized again the way model-gateway normalizes it | The model event | | `send` | An agent's `msg.send` has the channel and content its `send_message` call ran with: the `ToolEvent` with the send's `call_id` under the same model event, by its executed arguments. A `msg.send` with no agent has a `post` intervention as parent with the same channel, sender, and content | The `msg.send` | | `delivery` | A `msg.deliver` names a recorded send, has its channel and sender, and carries its content, or the content of the last `before_deliver` rewrite (`action` `deliver`) of it for that recipient; and no `drop` for that recipient precedes it | The `msg.deliver` | @@ -300,8 +304,9 @@ content addresses, a retried upload is harmless *(proposed)*. Built for sandboxd's blobs: `swarmeval.sandbox.S3BlobStore` writes them, and the sandboxd client uploads them before `Exec` or `FinalDiff` returns. In a `ToolEvent`'s `metadata.swarmeval.exec`, -`stdout_truncated` / `stderr_truncated` name the blob holding the full output, and a change with -`content_stored` has its new content under `after_sha256`. +`stdout_truncated` / `stderr_truncated` name the blob holding the full output, a change with +`content_stored` has its new content under `after_sha256`, and a `collected` file with a +`sha256` has its content there. A tool's images (`images`) are collected files. ## Export @@ -309,7 +314,7 @@ At run end the worker writes to the export bucket, which is created with object | Object | Content | From | |---|---|---| -| `runs//sample.eval` | Standard Inspect log, readable by `inspect view`. `ModelEvent.input` expanded | M0 | +| `runs//sample.eval` | Standard Inspect log, readable by `inspect view`. `ModelEvent.input` expanded; a run's images embedded | M0 | | `runs//events.parquet` | One row per event: the indexed columns, `prev_hash` / `hash` as hex, and `payload` as JSON text. Written after the `.eval`, so only for runs that verified and ended `done` or `cancelled` | Built | | `summaries/.parquet` | One row per run: submission and its suite label, case and its hash, variant and `task_args` (JSON, sorted keys), epoch and the epochs requested per variant, `replaces` and `replaced_by` (an interrupted run and its [rerun](services/orchestrator.md#reruns)), status and error, isolation, times, and each scorer's last score. Written for every run that reaches a final status, copying the status `control.runs` holds: by the worker once it has finished a run (a cancel that lands meanwhile wins), by the Control API when it cancels a queued run, by a worker marking its old runs `interrupted` at start, and by a worker that took over a run whose lease ran out. If the worker cannot write one, a `done` run becomes `failed` and an `interrupted` (its rerun stays queued), `failed`, or `cancelled` run keeps its status; either way its error adds why. At start, the worker does not start. Reports read only these. A column added later reads as null in older summaries | Built | @@ -323,7 +328,13 @@ How a run becomes a `.eval` (`swarmeval.events.export_run`): 1. Read the run's `events` and `messages` rows, and verify the chain. Rows that do not verify raise `ChainError`, and nothing is written. 2. Rebuild each event from its payload. Add `prev_hash` / `hash` (hex) to `metadata.swarmeval`, - and expand `ModelEvent.input` from `messages`. + and expand `ModelEvent.input` from `messages`. A tool message with images becomes text plus + `ContentImage("attachment://")` for each image the request carried, and + `[image omitted]` for each `max_images` left out; a `ToolEvent` with `images` gets its + `result` as text plus the images inline as `data:` URLs, as Inspect writes tool results. Every + image is read from the blob store and checked against its hash first, and goes into + `EvalSample.attachments` once, as a `data:` URL. A run naming an image the export cannot read + is not exported. 3. Wrap each agent's events in a `SpanBeginEvent` / `SpanEndEvent` with `type="agent"` and id `agent:`, and set their `span_id`. Events with no agent stay at the top level. 4. Build one `EvalSample`: `id` is the case id, `uuid` the run id, `input` the case task. There is diff --git a/docs/repo-layout.md b/docs/repo-layout.md index 9b3b31c..3c72ae6 100644 --- a/docs/repo-layout.md +++ b/docs/repo-layout.md @@ -24,8 +24,10 @@ suites/ suites, one file each: `m1_core.yaml` go/ the Go module: `cmd/` (`sandboxd`, `edge`, `swarm`, `swarm-certs`), `internal/`. Its own README, CHANGELOG, BUGFIX console/ the web console: a Vite + React app, built into edge. Its own README, CHANGELOG proto/ gRPC contracts; buf.yaml and buf.gen.yaml sit at the repo root -deploy/ `images/` (one Dockerfile for the Python services, one for the Go binaries) and - `compose/` (the single-machine stack and its smoke test) +deploy/ `images/` (one Dockerfile for the Python services, one for the Go binaries, and + `display/`, the sandbox image the `browser` and `computer` tools run on, with + its `swarm-display` program) and `compose/` (the single-machine stack and its + smoke test) pyproject.toml uv.lock mise.toml ``` diff --git a/docs/services/model-gateway.md b/docs/services/model-gateway.md index 100fbf1..5d73618 100644 --- a/docs/services/model-gateway.md +++ b/docs/services/model-gateway.md @@ -41,7 +41,14 @@ Sandboxes cannot reach model-gateway, and neither can [net-gateway](net-gateway. - `POST /v1/chat/completions`, the subset of the OpenAI Chat Completions protocol SwarmEval uses (`swarmeval/gateway/model/wire.py`): messages, function tools, `temperature`, `top_p`, `max_completion_tokens`, `seed`. Any other field is refused, and so is `stream: true`. Past - reasoning travels as `reasoning_content` on assistant messages. + reasoning travels as `reasoning_content` on assistant messages. A tool message's `content` may + be a list of parts, `text` and `image_url`, for the images a tool returned; an image must be + an inline `data:image/png;base64,` URL, so the gateway fetches nothing. Chat Completions takes + only text in a tool message, so before sending upstream, and after reasoning passback, the + gateway turns each such message into its text and moves its images into one user message + after the run of tool messages, each image under `Images from tool call :`. Doing it after + passback keeps the added user message from moving the `within_turn` boundary. The worker sends + only the images the request's `max_images` keeps ([agent-runtime.md](../agent-runtime.md#images)). - `GET /v1/models`: the model names runs may use. The control plane checks every submission against it, and lists it to users ([orchestrator](orchestrator.md#models)). No virtual key. - Authentication is `Authorization: Bearer `. Each call also carries the caller's diff --git a/docs/services/orchestrator.md b/docs/services/orchestrator.md index 3a01285..f10683b 100644 --- a/docs/services/orchestrator.md +++ b/docs/services/orchestrator.md @@ -80,7 +80,7 @@ request. | `GetRun`, `ListRuns` | Status, variant and its values, epoch, the run it [reruns](#reruns), owner, isolation level, error, timestamps. `ListRuns` filters by submission, suite label, case, workspace, and status, newest first; the filters apply before the limit | Built. Fidelity arrives with recovery (M3) | | `CancelRun` | Marks cancelled. A queued run never starts; a running one stops at its owner's next hook point, is not scored, and is still exported. A finished run is `FAILED_PRECONDITION` | Built | | `ResumeRun` | A run paused for a person (a Monitor's `pause`) goes on from the hook point where it stopped. A run that is not paused is `FAILED_PRECONDITION` | Built | -| `ForkRun` | A new run that goes on from a finished run's state at the start of the turn an event happened in, with edits: replace a message's text, delete a user message, replace an undelivered message's content, or run a model slot on another model ([forks](#forks)). Returns the fork, `.f`, queued. Only a run that ended `done` or `cancelled` (an exported one) can be forked. An unknown run or event is `NOT_FOUND`; any other source run, an event before the first turn, a turn that began with extension background work (`ctx.spawn`) still running, a run recorded before forks were possible, or an edit that does not fit is `FAILED_PRECONDITION`; a malformed edit is `INVALID_ARGUMENT` | Built | +| `ForkRun` | A new run that goes on from a finished run's state at the start of the turn an event happened in, with edits: replace a message's text, delete a user message, replace an undelivered message's content, or run a model slot on another model ([forks](#forks)). Returns the fork, `.f`, queued. Only a run that ended `done` or `cancelled` (an exported one) can be forked. An unknown run or event is `NOT_FOUND`; any other source run, an event before the first turn, a turn that began with extension background work (`ctx.spawn`) still running, a turn after the run's first `browser` or `computer` call, a run recorded before forks were possible, or an edit that does not fit is `FAILED_PRECONDITION`; a malformed edit is `INVALID_ARGUMENT` | Built | | `SubmitSuite` | Takes a [suite](#suites) file and one bundle per `cases[].path` it names, loads the suite, and queues every case's runs in one transaction under one suite label; returns the label and one submission per entry. A suite that does not load, a path with no bundle, or a bundle the suite does not name is `INVALID_ARGUMENT`, and nothing is queued | Built | | `ListModels` | The model names runs may use: what model-gateway serves, sorted ([models](#models)). `UNAVAILABLE` when model-gateway is out of reach | Built | | `StreamEvents` | Server stream of a run's events after a given `seq`, live while it runs; ends once the run has finished and every event was sent. Each event comes with its stored payload and its `line`, the one-line text the judge reads (`swarmeval.events.render`), cut at 2,000 characters | Built | @@ -490,7 +490,9 @@ at_event_id, edits)` picks the start of the turn `at_event_id` happened in: the [checkpoint](../event-log.md#checkpoints) before the event. The source must have ended `done` or `cancelled`, so its export exists for `trace` to follow. A checkpoint taken while an extension's `ctx.spawn` work was still running is refused: a fork cannot carry a coroutine -over. The loop commits one at the start of +over. So is a checkpoint after a `browser` or `computer` call ran: a fork restores files, not a +screen or a browser's pages, tabs, and sign-ins +([display spec](../../spec/2026-10-07-browser-computer-use/README.md) decision 12). The loop commits one at the start of every run-wide turn, once the observers have caught up, so no agent is mid-step there. The control plane checks the edits against the checkpoint and the contexts it locates, and queues `.f` with the source's case revision, overrides, variant, and epoch, so the same seed, diff --git a/docs/services/sandboxd.md b/docs/services/sandboxd.md index e3e9e3a..3bff3f7 100644 --- a/docs/services/sandboxd.md +++ b/docs/services/sandboxd.md @@ -40,8 +40,8 @@ authentication and refuses to listen on anything but a loopback address. | RPC | Does | State | |---|---|---| -| `CreateSandbox` | Creates one sandbox from a profile's image, key paths, limits, seed files, users, and identity (environment, hostname, machine id), labels it, and takes the initial manifest | Built | -| `Exec` | Runs one tool call and streams back its result, file changes, and surviving processes | Built | +| `CreateSandbox` | Creates one sandbox from a profile's image, key paths, limits, seed files, users, and identity (environment, hostname, machine id), labels it, starts its [display](#display) when it has one, and takes the initial manifest | Built | +| `Exec` | Runs one tool call and streams back its result, file changes, surviving processes, and the files it asked to [collect](#collected-files) | Built | | `ReadFile` | Reads a file for final-state scorers, through the engine's copy API, so nothing runs in the sandbox | Built | | `FsChange.mtime_ns` | Every reported change carries the path's modification time after it (before it for a delete), as the file system reports it; a process can set it to anything | Built | | `RestoreFiles` | For a [fork](orchestrator.md#forks): removes paths, creates directories, and writes files in a sandbox's key paths, each with its recorded mode and owner, then takes the manifest again, so none of it is ever reported as a change. Whatever is at a directory's or file's path and is of another kind (a symlink included) is removed first, so a write never follows a link. Every path must be strictly inside a key path; a path to remove that does not exist is skipped. An owner sandboxd cannot set, because it is not root, is reported in `unowned`. Only before the sandbox's first `Exec` (`FAILED_PRECONDITION` after it), and at most 3 MiB of content per request, so a client sends several (the worker keeps each under 2 MiB, paths included) | Built | @@ -125,6 +125,30 @@ from inside the sandbox (see [below](#exec-diff-and-process-snapshot)). The dock calls as its `os_user`. Under runc and runsc alike, files an `os_user` writes carry its uid on the host side, which `fs.*` events record, and one user cannot read another's `0600` files. +## Display + +`CreateSandbox` with `display` (width, height, first URL) starts the image's display stack: a +virtual screen with a web browser, which the `browser` and `computer` tools drive +([agent-runtime.md](../agent-runtime.md#tools)). The image must be built from +`swarmeval/display` ([deploy/images/display](../../deploy/images/display/README.md)). + +- After the container starts and before the first manifest, sandboxd runs `swarm-display start + --width W --height H [--url U]` as the user `swarmdisplay`, with a 60 s timeout. The command + returns once the screen and the browser are up and prints the display daemon's pid. A failure, + a timeout, or output that is not the pid of a live process fails `CreateSandbox`, and the + container is removed. +- sandboxd records that process's numeric uid. From then on every process with that uid is the + display's: left out of every process listing, so it is never reported as a `proc.*` event and + never makes a call's file changes `ambiguous`. Chromium starts renderers on each navigation, + and some of its helpers leave their parent for pid 1, so a pid subtree would not hold them. + An agent running as root could move a process of its own to that uid and hide it. +- The display keeps its state in `/run/swarm-display`, mode `0700`, owned by `swarmdisplay`. + Its home, temporary files, and browser profile are there, so it writes nothing into key paths. + A key path at `/`, `/run`, or under `/run/swarm-display` is `INVALID_ARGUMENT`. +- The screen needs a cookie only `swarmdisplay` can read, and the browser is driven through a + pipe, with no debugging port. An agent's own `os_user`, and root with no capabilities, reach + neither. Guard: `TestLiveDisplayDrivesTheBrowserAndKeepsItsProcessesOut`. + ## Networks `--sandbox-network` picks how sandboxes are networked. @@ -189,6 +213,18 @@ Each `Exec` does the following: 5. List processes again. New processes still alive become `proc.*` events with pid, ppid, user, and command line. +### Collected files + +An `ExecRequest` may name `collect` paths: absolute, clean, and outside every key path. After the +command, timed out or not, sandboxd reads each regular file there and removes it, as the call's +user, with a short `sh` script inside the sandbox; a symlink is removed without being followed. +Root cannot do it: with every capability dropped it cannot open another user's `0700` directory, +and what a call may collect is what its user may read anyway. The header's `collected` lists one +entry per path, in order: `missing` when there was no regular file, else its size and sha256, +with the content following as a blob. A file over 8 MiB is removed and reported by size only. +The `browser` and `computer` tools collect `/run/swarm-display/out/screenshot.png` this way, so a +screenshot never lands in a key path and is never a file change. + The response is a stream: a header with exit code, duration, and events, then blob chunks. stdout and stderr are capped in the header, and the full output follows as a blob. Changes between the previous call and this one come back as `background_changes`. While a process an @@ -236,7 +272,7 @@ it is not built in M0–M5. 3. Where each `os_user` may write. Today it is whatever the image lays out; a profile setting for key path owners may be needed once a case shares a writable workspace between users. 4. Size caps. Today's defaults: 64 KiB of stdout and stderr inline, 16 MiB kept per stream, file - contents sent back up to 1 MiB each and 64 MiB per call. They are constants in - `sandboxd.DefaultConfig`, not flags yet. + contents sent back up to 1 MiB each and 64 MiB per call, 8 MiB per collected file, and 60 s + to start a display. They are constants in `sandboxd.DefaultConfig`, not flags yet. 5. `machine_id` as a field of `CreateSandbox` of its own, for the sandbox canary's file, rather than seed files allowed outside the key paths. diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index b1c6c6e..263a934 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +### Added +- sandboxd starts a sandbox's display: `CreateSandboxRequest.display` (width, height, first URL) + runs the display image's `swarm-display start` as `swarmdisplay` before the first manifest + (`spec/2026-10-07-browser-computer-use`). That user's processes are left out of every process + listing, so a browser's renderers are neither reported nor make a call's changes ambiguous. A + key path overlapping `/run/swarm-display` is refused. +- `ExecRequest.collect`: after the command, sandboxd reads and removes files outside the key + paths as the call's user and returns them as blobs (`ExecHeader.collected`), up to 8 MiB each. +- `driver.Process.UID`: the numeric uid beside the user's name. + ### Changed - `swarm run` takes `-m` for the models to run on, required for a case: `-m MODEL[,MODEL…]` for the `default` model slot, `-m SLOT=MODEL[,MODEL…]` for a named one diff --git a/go/README.md b/go/README.md index df6bd98..da25966 100644 --- a/go/README.md +++ b/go/README.md @@ -29,13 +29,15 @@ Docker is found through the standard `DOCKER_HOST` / `DOCKER_*` environment. Requirements and limits: - Sandbox images must provide `sleep`, `tr`, and `/bin/sh`, `/etc/passwd` when the sandbox - has users, and `/etc` when it has a machine id. sandboxd never pulls; pull images on the - docker host first. + has users, and `/etc` when it has a machine id. A sandbox with a display needs an image built + from `swarmeval/display` (`deploy/images/display`). sandboxd never pulls; pull or build images + on the docker host first. - In production sandboxd runs as root, so extracted key paths keep their owners. - Sandboxes have no network but loopback by default. Agents reach the internet only through the worker's `web_request`. - Output and content caps are in `sandboxd.DefaultConfig`: 64 KiB of stdout and stderr inline, - 16 MiB kept per stream, changed files sent back up to 1 MiB each and 64 MiB per call. + 16 MiB kept per stream, changed files sent back up to 1 MiB each and 64 MiB per call, 8 MiB + per collected file, and 60 s to start a display. - State is in memory. A restarted sandboxd does not know the sandboxes it created; `DestroyRun` still removes their containers and networks by label. @@ -117,6 +119,8 @@ hosts once the certificates are issued. From the repo root: `mise run check` lints and tests this module with everything else. `mise run go:test-integration` runs sandboxd against the local docker daemon and needs `busybox:latest` and `python:3.12-slim` on the host; it also runs edge against a throwaway -`postgres:18-alpine`. `SWARMEVAL_IT_RUNTIME=runc` or `runsc` picks the runtime. On macOS, +`postgres:18-alpine`. The display test also needs `swarmeval/display:dev` +(`SWARMEVAL_IT_DISPLAY_IMAGE` names another), built from `deploy/images/display`, and skips +without it. `SWARMEVAL_IT_RUNTIME=runc` or `runsc` picks the runtime. On macOS, Docker Desktop and OrbStack enforce neither owners nor modes on bind mounts, so the `os_user` permission test skips there; run it on Linux. diff --git a/go/internal/driver/driver.go b/go/internal/driver/driver.go index 29df692..654cf74 100644 --- a/go/internal/driver/driver.go +++ b/go/internal/driver/driver.go @@ -95,8 +95,11 @@ type ExecSpec struct { // Process is one process in a container, with pids as the container sees them. type Process struct { - PID int32 - PPID int32 + PID int32 + PPID int32 + // UID is the numeric uid; User is its name from the container's /etc/passwd, or the uid + // when it has none. + UID string User string Cmdline string } diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go b/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go index c260a89..6ff2d95 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go @@ -71,7 +71,7 @@ func (x FsChange_Op) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Op.Descriptor instead. func (FsChange_Op) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 0} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 0} } type FsChange_Kind int32 @@ -127,7 +127,7 @@ func (x FsChange_Kind) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Kind.Descriptor instead. func (FsChange_Kind) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 1} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 1} } type FsChange_Attribution int32 @@ -178,7 +178,7 @@ func (x FsChange_Attribution) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Attribution.Descriptor instead. func (FsChange_Attribution) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 2} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 2} } type CreateRunRequest struct { @@ -427,7 +427,10 @@ type CreateSandboxRequest struct { // Written to /etc/machine-id before the sandbox starts, outside every key path, so it is never // diffed. Empty writes nothing. 32 lowercase hex characters, the file's format. The image // must have /etc. - MachineId string `protobuf:"bytes,10,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + MachineId string `protobuf:"bytes,10,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + // Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + // no display. + Display *Display `protobuf:"bytes,11,opt,name=display,proto3" json:"display,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -532,6 +535,77 @@ func (x *CreateSandboxRequest) GetMachineId() string { return "" } +func (x *CreateSandboxRequest) GetDisplay() *Display { + if x != nil { + return x.Display + } + return nil +} + +// A virtual screen with a browser on it, run by the image's `swarm-display` as the user +// `swarmdisplay`. Its processes are never reported or attributed to a call. +type Display struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Pixels, 320 to 1920 wide and 240 to 1200 high. + Width uint32 `protobuf:"varint,1,opt,name=width,proto3" json:"width,omitempty"` + Height uint32 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` + // Page the browser opens first. Empty opens about:blank. + Url string `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Display) Reset() { + *x = Display{} + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Display) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Display) ProtoMessage() {} + +func (x *Display) ProtoReflect() protoreflect.Message { + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Display.ProtoReflect.Descriptor instead. +func (*Display) Descriptor() ([]byte, []int) { + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{5} +} + +func (x *Display) GetWidth() uint32 { + if x != nil { + return x.Width + } + return 0 +} + +func (x *Display) GetHeight() uint32 { + if x != nil { + return x.Height + } + return 0 +} + +func (x *Display) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + type SeedFile struct { state protoimpl.MessageState `protogen:"open.v1"` // Absolute and clean, strictly inside a key path. @@ -545,7 +619,7 @@ type SeedFile struct { func (x *SeedFile) Reset() { *x = SeedFile{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -557,7 +631,7 @@ func (x *SeedFile) String() string { func (*SeedFile) ProtoMessage() {} func (x *SeedFile) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -570,7 +644,7 @@ func (x *SeedFile) ProtoReflect() protoreflect.Message { // Deprecated: Use SeedFile.ProtoReflect.Descriptor instead. func (*SeedFile) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{5} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{6} } func (x *SeedFile) GetPath() string { @@ -604,7 +678,7 @@ type CreateSandboxResponse struct { func (x *CreateSandboxResponse) Reset() { *x = CreateSandboxResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -616,7 +690,7 @@ func (x *CreateSandboxResponse) String() string { func (*CreateSandboxResponse) ProtoMessage() {} func (x *CreateSandboxResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -629,7 +703,7 @@ func (x *CreateSandboxResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSandboxResponse.ProtoReflect.Descriptor instead. func (*CreateSandboxResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{6} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{7} } func (x *CreateSandboxResponse) GetRuntime() string { @@ -650,14 +724,18 @@ type ExecRequest struct { // Empty runs as the image's user. User string `protobuf:"bytes,6,opt,name=user,proto3" json:"user,omitempty"` // Required. On expiry the command and its descendants are killed. - Timeout *durationpb.Duration `protobuf:"bytes,7,opt,name=timeout,proto3" json:"timeout,omitempty"` + Timeout *durationpb.Duration `protobuf:"bytes,7,opt,name=timeout,proto3" json:"timeout,omitempty"` + // Absolute paths outside every key path. After the command, sandboxd reads each regular file + // there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + // text, such as a screenshot. + Collect []string `protobuf:"bytes,8,rep,name=collect,proto3" json:"collect,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ExecRequest) Reset() { *x = ExecRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -669,7 +747,7 @@ func (x *ExecRequest) String() string { func (*ExecRequest) ProtoMessage() {} func (x *ExecRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -682,7 +760,7 @@ func (x *ExecRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecRequest.ProtoReflect.Descriptor instead. func (*ExecRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{7} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{8} } func (x *ExecRequest) GetRunId() string { @@ -734,6 +812,13 @@ func (x *ExecRequest) GetTimeout() *durationpb.Duration { return nil } +func (x *ExecRequest) GetCollect() []string { + if x != nil { + return x.Collect + } + return nil +} + type ExecResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Item: @@ -747,7 +832,7 @@ type ExecResponse struct { func (x *ExecResponse) Reset() { *x = ExecResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -759,7 +844,7 @@ func (x *ExecResponse) String() string { func (*ExecResponse) ProtoMessage() {} func (x *ExecResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -772,7 +857,7 @@ func (x *ExecResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecResponse.ProtoReflect.Descriptor instead. func (*ExecResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{8} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{9} } func (x *ExecResponse) GetItem() isExecResponse_Item { @@ -830,14 +915,16 @@ type ExecHeader struct { // Changed while this call ran. Changes []*FsChange `protobuf:"bytes,7,rep,name=changes,proto3" json:"changes,omitempty"` // Started while this call ran and still alive after it. - Processes []*Process `protobuf:"bytes,8,rep,name=processes,proto3" json:"processes,omitempty"` + Processes []*Process `protobuf:"bytes,8,rep,name=processes,proto3" json:"processes,omitempty"` + // One per `collect` path, in request order. + Collected []*CollectedFile `protobuf:"bytes,9,rep,name=collected,proto3" json:"collected,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ExecHeader) Reset() { *x = ExecHeader{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -849,7 +936,7 @@ func (x *ExecHeader) String() string { func (*ExecHeader) ProtoMessage() {} func (x *ExecHeader) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -862,7 +949,7 @@ func (x *ExecHeader) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecHeader.ProtoReflect.Descriptor instead. func (*ExecHeader) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{9} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{10} } func (x *ExecHeader) GetExitCode() int32 { @@ -921,6 +1008,85 @@ func (x *ExecHeader) GetProcesses() []*Process { return nil } +func (x *ExecHeader) GetCollected() []*CollectedFile { + if x != nil { + return x.Collected + } + return nil +} + +type CollectedFile struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // No regular file was there. + Missing bool `protobuf:"varint,2,opt,name=missing,proto3" json:"missing,omitempty"` + // Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + // `sha256` is empty. + Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + // The content follows as a blob with this hash. + Sha256 string `protobuf:"bytes,4,opt,name=sha256,proto3" json:"sha256,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CollectedFile) Reset() { + *x = CollectedFile{} + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CollectedFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectedFile) ProtoMessage() {} + +func (x *CollectedFile) ProtoReflect() protoreflect.Message { + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectedFile.ProtoReflect.Descriptor instead. +func (*CollectedFile) Descriptor() ([]byte, []int) { + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11} +} + +func (x *CollectedFile) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *CollectedFile) GetMissing() bool { + if x != nil { + return x.Missing + } + return false +} + +func (x *CollectedFile) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *CollectedFile) GetSha256() string { + if x != nil { + return x.Sha256 + } + return "" +} + type Output struct { state protoimpl.MessageState `protogen:"open.v1"` // The first bytes, up to sandboxd's inline limit. @@ -938,7 +1104,7 @@ type Output struct { func (x *Output) Reset() { *x = Output{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -950,7 +1116,7 @@ func (x *Output) String() string { func (*Output) ProtoMessage() {} func (x *Output) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -963,7 +1129,7 @@ func (x *Output) ProtoReflect() protoreflect.Message { // Deprecated: Use Output.ProtoReflect.Descriptor instead. func (*Output) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{10} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{12} } func (x *Output) GetInline() []byte { @@ -1024,7 +1190,7 @@ type FsChange struct { func (x *FsChange) Reset() { *x = FsChange{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1036,7 +1202,7 @@ func (x *FsChange) String() string { func (*FsChange) ProtoMessage() {} func (x *FsChange) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1049,7 +1215,7 @@ func (x *FsChange) ProtoReflect() protoreflect.Message { // Deprecated: Use FsChange.ProtoReflect.Descriptor instead. func (*FsChange) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13} } func (x *FsChange) GetPath() string { @@ -1157,7 +1323,7 @@ type Process struct { func (x *Process) Reset() { *x = Process{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1169,7 +1335,7 @@ func (x *Process) String() string { func (*Process) ProtoMessage() {} func (x *Process) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1182,7 +1348,7 @@ func (x *Process) ProtoReflect() protoreflect.Message { // Deprecated: Use Process.ProtoReflect.Descriptor instead. func (*Process) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{12} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{14} } func (x *Process) GetPid() int32 { @@ -1224,7 +1390,7 @@ type BlobChunk struct { func (x *BlobChunk) Reset() { *x = BlobChunk{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1236,7 +1402,7 @@ func (x *BlobChunk) String() string { func (*BlobChunk) ProtoMessage() {} func (x *BlobChunk) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1249,7 +1415,7 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead. func (*BlobChunk) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{15} } func (x *BlobChunk) GetSha256() string { @@ -1287,7 +1453,7 @@ type RestoreFilesRequest struct { func (x *RestoreFilesRequest) Reset() { *x = RestoreFilesRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1299,7 +1465,7 @@ func (x *RestoreFilesRequest) String() string { func (*RestoreFilesRequest) ProtoMessage() {} func (x *RestoreFilesRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1478,7 @@ func (x *RestoreFilesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFilesRequest.ProtoReflect.Descriptor instead. func (*RestoreFilesRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{14} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{16} } func (x *RestoreFilesRequest) GetRunId() string { @@ -1365,7 +1531,7 @@ type RestoreDir struct { func (x *RestoreDir) Reset() { *x = RestoreDir{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1377,7 +1543,7 @@ func (x *RestoreDir) String() string { func (*RestoreDir) ProtoMessage() {} func (x *RestoreDir) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1390,7 +1556,7 @@ func (x *RestoreDir) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreDir.ProtoReflect.Descriptor instead. func (*RestoreDir) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{15} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{17} } func (x *RestoreDir) GetPath() string { @@ -1430,7 +1596,7 @@ type RestoreFile struct { func (x *RestoreFile) Reset() { *x = RestoreFile{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1442,7 +1608,7 @@ func (x *RestoreFile) String() string { func (*RestoreFile) ProtoMessage() {} func (x *RestoreFile) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1455,7 +1621,7 @@ func (x *RestoreFile) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFile.ProtoReflect.Descriptor instead. func (*RestoreFile) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{16} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{18} } func (x *RestoreFile) GetPath() string { @@ -1497,7 +1663,7 @@ type RestoreFilesResponse struct { func (x *RestoreFilesResponse) Reset() { *x = RestoreFilesResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1509,7 +1675,7 @@ func (x *RestoreFilesResponse) String() string { func (*RestoreFilesResponse) ProtoMessage() {} func (x *RestoreFilesResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1522,7 +1688,7 @@ func (x *RestoreFilesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFilesResponse.ProtoReflect.Descriptor instead. func (*RestoreFilesResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{17} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{19} } func (x *RestoreFilesResponse) GetUnowned() []string { @@ -1545,7 +1711,7 @@ type ReadFileRequest struct { func (x *ReadFileRequest) Reset() { *x = ReadFileRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1557,7 +1723,7 @@ func (x *ReadFileRequest) String() string { func (*ReadFileRequest) ProtoMessage() {} func (x *ReadFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1570,7 +1736,7 @@ func (x *ReadFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadFileRequest.ProtoReflect.Descriptor instead. func (*ReadFileRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{18} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{20} } func (x *ReadFileRequest) GetRunId() string { @@ -1612,7 +1778,7 @@ type ReadFileResponse struct { func (x *ReadFileResponse) Reset() { *x = ReadFileResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1624,7 +1790,7 @@ func (x *ReadFileResponse) String() string { func (*ReadFileResponse) ProtoMessage() {} func (x *ReadFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1637,7 +1803,7 @@ func (x *ReadFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadFileResponse.ProtoReflect.Descriptor instead. func (*ReadFileResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{19} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{21} } func (x *ReadFileResponse) GetContent() []byte { @@ -1670,7 +1836,7 @@ type FinalDiffRequest struct { func (x *FinalDiffRequest) Reset() { *x = FinalDiffRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1682,7 +1848,7 @@ func (x *FinalDiffRequest) String() string { func (*FinalDiffRequest) ProtoMessage() {} func (x *FinalDiffRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1695,7 +1861,7 @@ func (x *FinalDiffRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalDiffRequest.ProtoReflect.Descriptor instead. func (*FinalDiffRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{20} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{22} } func (x *FinalDiffRequest) GetRunId() string { @@ -1718,7 +1884,7 @@ type FinalDiffResponse struct { func (x *FinalDiffResponse) Reset() { *x = FinalDiffResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1730,7 +1896,7 @@ func (x *FinalDiffResponse) String() string { func (*FinalDiffResponse) ProtoMessage() {} func (x *FinalDiffResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1743,7 +1909,7 @@ func (x *FinalDiffResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalDiffResponse.ProtoReflect.Descriptor instead. func (*FinalDiffResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{21} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{23} } func (x *FinalDiffResponse) GetItem() isFinalDiffResponse_Item { @@ -1797,7 +1963,7 @@ type SandboxChanges struct { func (x *SandboxChanges) Reset() { *x = SandboxChanges{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1809,7 +1975,7 @@ func (x *SandboxChanges) String() string { func (*SandboxChanges) ProtoMessage() {} func (x *SandboxChanges) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1822,7 +1988,7 @@ func (x *SandboxChanges) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxChanges.ProtoReflect.Descriptor instead. func (*SandboxChanges) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{22} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{24} } func (x *SandboxChanges) GetSandboxId() string { @@ -1848,7 +2014,7 @@ type DestroyRunRequest struct { func (x *DestroyRunRequest) Reset() { *x = DestroyRunRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1860,7 +2026,7 @@ func (x *DestroyRunRequest) String() string { func (*DestroyRunRequest) ProtoMessage() {} func (x *DestroyRunRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1873,7 +2039,7 @@ func (x *DestroyRunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyRunRequest.ProtoReflect.Descriptor instead. func (*DestroyRunRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{23} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{25} } func (x *DestroyRunRequest) GetRunId() string { @@ -1891,7 +2057,7 @@ type DestroyRunResponse struct { func (x *DestroyRunResponse) Reset() { *x = DestroyRunResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1903,7 +2069,7 @@ func (x *DestroyRunResponse) String() string { func (*DestroyRunResponse) ProtoMessage() {} func (x *DestroyRunResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1916,7 +2082,7 @@ func (x *DestroyRunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyRunResponse.ProtoReflect.Descriptor instead. func (*DestroyRunResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{24} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{26} } var File_swarmeval_sandbox_v1_sandbox_proto protoreflect.FileDescriptor @@ -1938,7 +2104,7 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\x12\x12\n" + "\x04pids\x18\x03 \x01(\x03R\x04pids\x12\x1d\n" + "\n" + - "disk_bytes\x18\x04 \x01(\x03R\tdiskBytes\"\xdc\x03\n" + + "disk_bytes\x18\x04 \x01(\x03R\tdiskBytes\"\x95\x04\n" + "\x14CreateSandboxRequest\x12\x15\n" + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x1d\n" + "\n" + @@ -1952,16 +2118,21 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\bhostname\x18\t \x01(\tR\bhostname\x12\x1d\n" + "\n" + "machine_id\x18\n" + - " \x01(\tR\tmachineId\x1a6\n" + + " \x01(\tR\tmachineId\x127\n" + + "\adisplay\x18\v \x01(\v2\x1d.swarmeval.sandbox.v1.DisplayR\adisplay\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"L\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"I\n" + + "\aDisplay\x12\x14\n" + + "\x05width\x18\x01 \x01(\rR\x05width\x12\x16\n" + + "\x06height\x18\x02 \x01(\rR\x06height\x12\x10\n" + + "\x03url\x18\x03 \x01(\tR\x03url\"L\n" + "\bSeedFile\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x18\n" + "\acontent\x18\x02 \x01(\fR\acontent\x12\x12\n" + "\x04mode\x18\x03 \x01(\rR\x04mode\"1\n" + "\x15CreateSandboxResponse\x12\x18\n" + - "\aruntime\x18\x01 \x01(\tR\aruntime\"\xcb\x01\n" + + "\aruntime\x18\x01 \x01(\tR\aruntime\"\xe5\x01\n" + "\vExecRequest\x12\x15\n" + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x1d\n" + "\n" + @@ -1970,11 +2141,12 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\x04argv\x18\x04 \x03(\tR\x04argv\x12\x10\n" + "\x03cwd\x18\x05 \x01(\tR\x03cwd\x12\x12\n" + "\x04user\x18\x06 \x01(\tR\x04user\x123\n" + - "\atimeout\x18\a \x01(\v2\x19.google.protobuf.DurationR\atimeout\"\x89\x01\n" + + "\atimeout\x18\a \x01(\v2\x19.google.protobuf.DurationR\atimeout\x12\x18\n" + + "\acollect\x18\b \x03(\tR\acollect\"\x89\x01\n" + "\fExecResponse\x12:\n" + "\x06header\x18\x01 \x01(\v2 .swarmeval.sandbox.v1.ExecHeaderH\x00R\x06header\x125\n" + "\x04blob\x18\x02 \x01(\v2\x1f.swarmeval.sandbox.v1.BlobChunkH\x00R\x04blobB\x06\n" + - "\x04item\"\xaf\x03\n" + + "\x04item\"\xf2\x03\n" + "\n" + "ExecHeader\x12\x1b\n" + "\texit_code\x18\x01 \x01(\x05R\bexitCode\x12\x1b\n" + @@ -1984,7 +2156,13 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\x06stderr\x18\x05 \x01(\v2\x1c.swarmeval.sandbox.v1.OutputR\x06stderr\x12M\n" + "\x12background_changes\x18\x06 \x03(\v2\x1e.swarmeval.sandbox.v1.FsChangeR\x11backgroundChanges\x128\n" + "\achanges\x18\a \x03(\v2\x1e.swarmeval.sandbox.v1.FsChangeR\achanges\x12;\n" + - "\tprocesses\x18\b \x03(\v2\x1d.swarmeval.sandbox.v1.ProcessR\tprocesses\"m\n" + + "\tprocesses\x18\b \x03(\v2\x1d.swarmeval.sandbox.v1.ProcessR\tprocesses\x12A\n" + + "\tcollected\x18\t \x03(\v2#.swarmeval.sandbox.v1.CollectedFileR\tcollected\"i\n" + + "\rCollectedFile\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x18\n" + + "\amissing\x18\x02 \x01(\bR\amissing\x12\x12\n" + + "\x04size\x18\x03 \x01(\x03R\x04size\x12\x16\n" + + "\x06sha256\x18\x04 \x01(\tR\x06sha256\"m\n" + "\x06Output\x12\x16\n" + "\x06inline\x18\x01 \x01(\fR\x06inline\x12\x12\n" + "\x04size\x18\x02 \x01(\x03R\x04size\x12\x1f\n" + @@ -2096,7 +2274,7 @@ func file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP() []byte { } var file_swarmeval_sandbox_v1_sandbox_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_swarmeval_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_swarmeval_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_swarmeval_sandbox_v1_sandbox_proto_goTypes = []any{ (FsChange_Op)(0), // 0: swarmeval.sandbox.v1.FsChange.Op (FsChange_Kind)(0), // 1: swarmeval.sandbox.v1.FsChange.Kind @@ -2106,70 +2284,74 @@ var file_swarmeval_sandbox_v1_sandbox_proto_goTypes = []any{ (*Mount)(nil), // 5: swarmeval.sandbox.v1.Mount (*Resources)(nil), // 6: swarmeval.sandbox.v1.Resources (*CreateSandboxRequest)(nil), // 7: swarmeval.sandbox.v1.CreateSandboxRequest - (*SeedFile)(nil), // 8: swarmeval.sandbox.v1.SeedFile - (*CreateSandboxResponse)(nil), // 9: swarmeval.sandbox.v1.CreateSandboxResponse - (*ExecRequest)(nil), // 10: swarmeval.sandbox.v1.ExecRequest - (*ExecResponse)(nil), // 11: swarmeval.sandbox.v1.ExecResponse - (*ExecHeader)(nil), // 12: swarmeval.sandbox.v1.ExecHeader - (*Output)(nil), // 13: swarmeval.sandbox.v1.Output - (*FsChange)(nil), // 14: swarmeval.sandbox.v1.FsChange - (*Process)(nil), // 15: swarmeval.sandbox.v1.Process - (*BlobChunk)(nil), // 16: swarmeval.sandbox.v1.BlobChunk - (*RestoreFilesRequest)(nil), // 17: swarmeval.sandbox.v1.RestoreFilesRequest - (*RestoreDir)(nil), // 18: swarmeval.sandbox.v1.RestoreDir - (*RestoreFile)(nil), // 19: swarmeval.sandbox.v1.RestoreFile - (*RestoreFilesResponse)(nil), // 20: swarmeval.sandbox.v1.RestoreFilesResponse - (*ReadFileRequest)(nil), // 21: swarmeval.sandbox.v1.ReadFileRequest - (*ReadFileResponse)(nil), // 22: swarmeval.sandbox.v1.ReadFileResponse - (*FinalDiffRequest)(nil), // 23: swarmeval.sandbox.v1.FinalDiffRequest - (*FinalDiffResponse)(nil), // 24: swarmeval.sandbox.v1.FinalDiffResponse - (*SandboxChanges)(nil), // 25: swarmeval.sandbox.v1.SandboxChanges - (*DestroyRunRequest)(nil), // 26: swarmeval.sandbox.v1.DestroyRunRequest - (*DestroyRunResponse)(nil), // 27: swarmeval.sandbox.v1.DestroyRunResponse - nil, // 28: swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry - (*durationpb.Duration)(nil), // 29: google.protobuf.Duration + (*Display)(nil), // 8: swarmeval.sandbox.v1.Display + (*SeedFile)(nil), // 9: swarmeval.sandbox.v1.SeedFile + (*CreateSandboxResponse)(nil), // 10: swarmeval.sandbox.v1.CreateSandboxResponse + (*ExecRequest)(nil), // 11: swarmeval.sandbox.v1.ExecRequest + (*ExecResponse)(nil), // 12: swarmeval.sandbox.v1.ExecResponse + (*ExecHeader)(nil), // 13: swarmeval.sandbox.v1.ExecHeader + (*CollectedFile)(nil), // 14: swarmeval.sandbox.v1.CollectedFile + (*Output)(nil), // 15: swarmeval.sandbox.v1.Output + (*FsChange)(nil), // 16: swarmeval.sandbox.v1.FsChange + (*Process)(nil), // 17: swarmeval.sandbox.v1.Process + (*BlobChunk)(nil), // 18: swarmeval.sandbox.v1.BlobChunk + (*RestoreFilesRequest)(nil), // 19: swarmeval.sandbox.v1.RestoreFilesRequest + (*RestoreDir)(nil), // 20: swarmeval.sandbox.v1.RestoreDir + (*RestoreFile)(nil), // 21: swarmeval.sandbox.v1.RestoreFile + (*RestoreFilesResponse)(nil), // 22: swarmeval.sandbox.v1.RestoreFilesResponse + (*ReadFileRequest)(nil), // 23: swarmeval.sandbox.v1.ReadFileRequest + (*ReadFileResponse)(nil), // 24: swarmeval.sandbox.v1.ReadFileResponse + (*FinalDiffRequest)(nil), // 25: swarmeval.sandbox.v1.FinalDiffRequest + (*FinalDiffResponse)(nil), // 26: swarmeval.sandbox.v1.FinalDiffResponse + (*SandboxChanges)(nil), // 27: swarmeval.sandbox.v1.SandboxChanges + (*DestroyRunRequest)(nil), // 28: swarmeval.sandbox.v1.DestroyRunRequest + (*DestroyRunResponse)(nil), // 29: swarmeval.sandbox.v1.DestroyRunResponse + nil, // 30: swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry + (*durationpb.Duration)(nil), // 31: google.protobuf.Duration } var file_swarmeval_sandbox_v1_sandbox_proto_depIdxs = []int32{ 5, // 0: swarmeval.sandbox.v1.CreateSandboxRequest.mounts:type_name -> swarmeval.sandbox.v1.Mount 6, // 1: swarmeval.sandbox.v1.CreateSandboxRequest.resources:type_name -> swarmeval.sandbox.v1.Resources - 8, // 2: swarmeval.sandbox.v1.CreateSandboxRequest.files:type_name -> swarmeval.sandbox.v1.SeedFile - 28, // 3: swarmeval.sandbox.v1.CreateSandboxRequest.env:type_name -> swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry - 29, // 4: swarmeval.sandbox.v1.ExecRequest.timeout:type_name -> google.protobuf.Duration - 12, // 5: swarmeval.sandbox.v1.ExecResponse.header:type_name -> swarmeval.sandbox.v1.ExecHeader - 16, // 6: swarmeval.sandbox.v1.ExecResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk - 29, // 7: swarmeval.sandbox.v1.ExecHeader.duration:type_name -> google.protobuf.Duration - 13, // 8: swarmeval.sandbox.v1.ExecHeader.stdout:type_name -> swarmeval.sandbox.v1.Output - 13, // 9: swarmeval.sandbox.v1.ExecHeader.stderr:type_name -> swarmeval.sandbox.v1.Output - 14, // 10: swarmeval.sandbox.v1.ExecHeader.background_changes:type_name -> swarmeval.sandbox.v1.FsChange - 14, // 11: swarmeval.sandbox.v1.ExecHeader.changes:type_name -> swarmeval.sandbox.v1.FsChange - 15, // 12: swarmeval.sandbox.v1.ExecHeader.processes:type_name -> swarmeval.sandbox.v1.Process - 0, // 13: swarmeval.sandbox.v1.FsChange.op:type_name -> swarmeval.sandbox.v1.FsChange.Op - 1, // 14: swarmeval.sandbox.v1.FsChange.kind:type_name -> swarmeval.sandbox.v1.FsChange.Kind - 2, // 15: swarmeval.sandbox.v1.FsChange.attribution:type_name -> swarmeval.sandbox.v1.FsChange.Attribution - 18, // 16: swarmeval.sandbox.v1.RestoreFilesRequest.dirs:type_name -> swarmeval.sandbox.v1.RestoreDir - 19, // 17: swarmeval.sandbox.v1.RestoreFilesRequest.files:type_name -> swarmeval.sandbox.v1.RestoreFile - 25, // 18: swarmeval.sandbox.v1.FinalDiffResponse.changes:type_name -> swarmeval.sandbox.v1.SandboxChanges - 16, // 19: swarmeval.sandbox.v1.FinalDiffResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk - 14, // 20: swarmeval.sandbox.v1.SandboxChanges.changes:type_name -> swarmeval.sandbox.v1.FsChange - 3, // 21: swarmeval.sandbox.v1.SandboxService.CreateRun:input_type -> swarmeval.sandbox.v1.CreateRunRequest - 7, // 22: swarmeval.sandbox.v1.SandboxService.CreateSandbox:input_type -> swarmeval.sandbox.v1.CreateSandboxRequest - 10, // 23: swarmeval.sandbox.v1.SandboxService.Exec:input_type -> swarmeval.sandbox.v1.ExecRequest - 21, // 24: swarmeval.sandbox.v1.SandboxService.ReadFile:input_type -> swarmeval.sandbox.v1.ReadFileRequest - 17, // 25: swarmeval.sandbox.v1.SandboxService.RestoreFiles:input_type -> swarmeval.sandbox.v1.RestoreFilesRequest - 23, // 26: swarmeval.sandbox.v1.SandboxService.FinalDiff:input_type -> swarmeval.sandbox.v1.FinalDiffRequest - 26, // 27: swarmeval.sandbox.v1.SandboxService.DestroyRun:input_type -> swarmeval.sandbox.v1.DestroyRunRequest - 4, // 28: swarmeval.sandbox.v1.SandboxService.CreateRun:output_type -> swarmeval.sandbox.v1.CreateRunResponse - 9, // 29: swarmeval.sandbox.v1.SandboxService.CreateSandbox:output_type -> swarmeval.sandbox.v1.CreateSandboxResponse - 11, // 30: swarmeval.sandbox.v1.SandboxService.Exec:output_type -> swarmeval.sandbox.v1.ExecResponse - 22, // 31: swarmeval.sandbox.v1.SandboxService.ReadFile:output_type -> swarmeval.sandbox.v1.ReadFileResponse - 20, // 32: swarmeval.sandbox.v1.SandboxService.RestoreFiles:output_type -> swarmeval.sandbox.v1.RestoreFilesResponse - 24, // 33: swarmeval.sandbox.v1.SandboxService.FinalDiff:output_type -> swarmeval.sandbox.v1.FinalDiffResponse - 27, // 34: swarmeval.sandbox.v1.SandboxService.DestroyRun:output_type -> swarmeval.sandbox.v1.DestroyRunResponse - 28, // [28:35] is the sub-list for method output_type - 21, // [21:28] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 9, // 2: swarmeval.sandbox.v1.CreateSandboxRequest.files:type_name -> swarmeval.sandbox.v1.SeedFile + 30, // 3: swarmeval.sandbox.v1.CreateSandboxRequest.env:type_name -> swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry + 8, // 4: swarmeval.sandbox.v1.CreateSandboxRequest.display:type_name -> swarmeval.sandbox.v1.Display + 31, // 5: swarmeval.sandbox.v1.ExecRequest.timeout:type_name -> google.protobuf.Duration + 13, // 6: swarmeval.sandbox.v1.ExecResponse.header:type_name -> swarmeval.sandbox.v1.ExecHeader + 18, // 7: swarmeval.sandbox.v1.ExecResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk + 31, // 8: swarmeval.sandbox.v1.ExecHeader.duration:type_name -> google.protobuf.Duration + 15, // 9: swarmeval.sandbox.v1.ExecHeader.stdout:type_name -> swarmeval.sandbox.v1.Output + 15, // 10: swarmeval.sandbox.v1.ExecHeader.stderr:type_name -> swarmeval.sandbox.v1.Output + 16, // 11: swarmeval.sandbox.v1.ExecHeader.background_changes:type_name -> swarmeval.sandbox.v1.FsChange + 16, // 12: swarmeval.sandbox.v1.ExecHeader.changes:type_name -> swarmeval.sandbox.v1.FsChange + 17, // 13: swarmeval.sandbox.v1.ExecHeader.processes:type_name -> swarmeval.sandbox.v1.Process + 14, // 14: swarmeval.sandbox.v1.ExecHeader.collected:type_name -> swarmeval.sandbox.v1.CollectedFile + 0, // 15: swarmeval.sandbox.v1.FsChange.op:type_name -> swarmeval.sandbox.v1.FsChange.Op + 1, // 16: swarmeval.sandbox.v1.FsChange.kind:type_name -> swarmeval.sandbox.v1.FsChange.Kind + 2, // 17: swarmeval.sandbox.v1.FsChange.attribution:type_name -> swarmeval.sandbox.v1.FsChange.Attribution + 20, // 18: swarmeval.sandbox.v1.RestoreFilesRequest.dirs:type_name -> swarmeval.sandbox.v1.RestoreDir + 21, // 19: swarmeval.sandbox.v1.RestoreFilesRequest.files:type_name -> swarmeval.sandbox.v1.RestoreFile + 27, // 20: swarmeval.sandbox.v1.FinalDiffResponse.changes:type_name -> swarmeval.sandbox.v1.SandboxChanges + 18, // 21: swarmeval.sandbox.v1.FinalDiffResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk + 16, // 22: swarmeval.sandbox.v1.SandboxChanges.changes:type_name -> swarmeval.sandbox.v1.FsChange + 3, // 23: swarmeval.sandbox.v1.SandboxService.CreateRun:input_type -> swarmeval.sandbox.v1.CreateRunRequest + 7, // 24: swarmeval.sandbox.v1.SandboxService.CreateSandbox:input_type -> swarmeval.sandbox.v1.CreateSandboxRequest + 11, // 25: swarmeval.sandbox.v1.SandboxService.Exec:input_type -> swarmeval.sandbox.v1.ExecRequest + 23, // 26: swarmeval.sandbox.v1.SandboxService.ReadFile:input_type -> swarmeval.sandbox.v1.ReadFileRequest + 19, // 27: swarmeval.sandbox.v1.SandboxService.RestoreFiles:input_type -> swarmeval.sandbox.v1.RestoreFilesRequest + 25, // 28: swarmeval.sandbox.v1.SandboxService.FinalDiff:input_type -> swarmeval.sandbox.v1.FinalDiffRequest + 28, // 29: swarmeval.sandbox.v1.SandboxService.DestroyRun:input_type -> swarmeval.sandbox.v1.DestroyRunRequest + 4, // 30: swarmeval.sandbox.v1.SandboxService.CreateRun:output_type -> swarmeval.sandbox.v1.CreateRunResponse + 10, // 31: swarmeval.sandbox.v1.SandboxService.CreateSandbox:output_type -> swarmeval.sandbox.v1.CreateSandboxResponse + 12, // 32: swarmeval.sandbox.v1.SandboxService.Exec:output_type -> swarmeval.sandbox.v1.ExecResponse + 24, // 33: swarmeval.sandbox.v1.SandboxService.ReadFile:output_type -> swarmeval.sandbox.v1.ReadFileResponse + 22, // 34: swarmeval.sandbox.v1.SandboxService.RestoreFiles:output_type -> swarmeval.sandbox.v1.RestoreFilesResponse + 26, // 35: swarmeval.sandbox.v1.SandboxService.FinalDiff:output_type -> swarmeval.sandbox.v1.FinalDiffResponse + 29, // 36: swarmeval.sandbox.v1.SandboxService.DestroyRun:output_type -> swarmeval.sandbox.v1.DestroyRunResponse + 30, // [30:37] is the sub-list for method output_type + 23, // [23:30] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_swarmeval_sandbox_v1_sandbox_proto_init() } @@ -2177,11 +2359,11 @@ func file_swarmeval_sandbox_v1_sandbox_proto_init() { if File_swarmeval_sandbox_v1_sandbox_proto != nil { return } - file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8].OneofWrappers = []any{ + file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9].OneofWrappers = []any{ (*ExecResponse_Header)(nil), (*ExecResponse_Blob)(nil), } - file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21].OneofWrappers = []any{ + file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23].OneofWrappers = []any{ (*FinalDiffResponse_Changes)(nil), (*FinalDiffResponse_Blob)(nil), } @@ -2191,7 +2373,7 @@ func file_swarmeval_sandbox_v1_sandbox_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_swarmeval_sandbox_v1_sandbox_proto_rawDesc), len(file_swarmeval_sandbox_v1_sandbox_proto_rawDesc)), NumEnums: 3, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go b/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go index 44446f1..35db144 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go @@ -44,7 +44,8 @@ type SandboxServiceClient interface { // ALREADY_EXISTS if the run exists. CreateRun(ctx context.Context, in *CreateRunRequest, opts ...grpc.CallOption) (*CreateRunResponse, error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(ctx context.Context, in *CreateSandboxRequest, opts ...grpc.CallOption) (*CreateSandboxResponse, error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. @@ -175,7 +176,8 @@ type SandboxServiceServer interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *CreateRunRequest) (*CreateRunResponse, error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *CreateSandboxRequest) (*CreateSandboxResponse, error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go b/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go index f6158d0..d3b054a 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go @@ -62,7 +62,8 @@ type SandboxServiceClient interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *connect.Request[v1.CreateRunRequest]) (*connect.Response[v1.CreateRunResponse], error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *connect.Request[v1.CreateSandboxRequest]) (*connect.Response[v1.CreateSandboxResponse], error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. @@ -191,7 +192,8 @@ type SandboxServiceHandler interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *connect.Request[v1.CreateRunRequest]) (*connect.Response[v1.CreateRunResponse], error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *connect.Request[v1.CreateSandboxRequest]) (*connect.Response[v1.CreateSandboxResponse], error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. diff --git a/go/internal/sandboxd/display.go b/go/internal/sandboxd/display.go new file mode 100644 index 0000000..fdbe2f3 --- /dev/null +++ b/go/internal/sandboxd/display.go @@ -0,0 +1,174 @@ +package sandboxd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "strconv" + "strings" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" +) + +const ( + // DisplayUser runs a display image's display stack and the tools that drive it. Display + // images must have it. + DisplayUser = "swarmdisplay" + // DisplayDir holds the display stack's state, outside every key path. + DisplayDir = "/run/swarm-display" +) + +// Display is a sandbox's virtual screen. +type Display struct { + Width uint32 + Height uint32 + // URL is the page the browser opens first; empty opens about:blank. + URL string +} + +func checkDisplay(d *Display, mounts []Mount) error { + if d == nil { + return nil + } + if d.Width < 320 || d.Width > 1920 || d.Height < 240 || d.Height > 1200 { + return fmt.Errorf("%w: display %dx%d is outside 320x240 to 1920x1200", ErrInvalid, d.Width, d.Height) + } + if len(d.URL) > 2048 || strings.ContainsAny(d.URL, "\x00\n\r") { + return fmt.Errorf("%w: display url %.80q is over 2048 bytes or holds NUL or a line break", ErrInvalid, d.URL) + } + for _, m := range mounts { + if overlaps(m.Path, DisplayDir) { + return fmt.Errorf("%w: key path %s overlaps the display's state directory %s, which must stay out of every key path", ErrInvalid, m.Path, DisplayDir) + } + } + return nil +} + +// startDisplay runs the image's `swarm-display start`, which returns once the display stack +// is up and prints the stack's pid. It returns that process's uid, whose processes are the +// display's from then on. +func (s *Service) startDisplay(ctx context.Context, sb *sandbox, d *Display) (string, error) { + argv := []string{ + "swarm-display", "start", + "--width", strconv.FormatUint(uint64(d.Width), 10), + "--height", strconv.FormatUint(uint64(d.Height), 10), + } + if d.URL != "" { + argv = append(argv, "--url", d.URL) + } + startCtx, cancel := context.WithTimeout(ctx, s.cfg.DisplayStartTimeout) + defer cancel() + stdout := &capBuffer{limit: 4 << 10} + stderr := &capBuffer{limit: 16 << 10} + code, err := s.drv.Exec(startCtx, sb.container, driver.ExecSpec{Argv: argv, User: DisplayUser}, stdout, stderr) + if err != nil { + return "", fmt.Errorf("start the display in container %s (timeout %s): %w", sb.container, s.cfg.DisplayStartTimeout, err) + } + if code != 0 { + return "", fmt.Errorf("start the display in container %s: `swarm-display start` exited %d: %s. The image must be built from swarmeval/display (deploy/images/display)", + sb.container, code, strings.TrimSpace(string(stderr.buf))) + } + pid, err := strconv.ParseInt(strings.TrimSpace(string(stdout.buf)), 10, 32) + if err != nil { + return "", fmt.Errorf("start the display in container %s: `swarm-display start` printed %.80q, not the display stack's pid", sb.container, stdout.buf) + } + procs, err := s.processes(ctx, sb) + if err != nil { + return "", err + } + for _, p := range procs { + if p.PID == int32(pid) { + return p.UID, nil + } + } + return "", fmt.Errorf("start the display in container %s: its pid %d is not running after `swarm-display start` returned", sb.container, pid) +} + +// collectScript prints the regular file $1 and removes it; exit 4 when there is none. A +// symlink is removed without being followed. +const collectScript = `f=$1 +if [ -f "$f" ] && [ ! -h "$f" ]; then cat -- "$f" || exit 5; rm -f -- "$f"; exit 0; fi +rm -f -- "$f" 2>/dev/null +exit 4` + +// Collected is one file an Exec collected. +type Collected struct { + Path string + Missing bool + Size int64 + // SHA256 is empty when the file is over the collect limit; otherwise its content is + // among the result's blobs. + SHA256 string +} + +func checkCollect(paths []string, mounts []Mount) error { + for _, p := range paths { + if !path.IsAbs(p) || path.Clean(p) != p || strings.ContainsRune(p, 0) { + return fmt.Errorf("%w: collect path %q must be absolute and clean", ErrInvalid, p) + } + for _, m := range mounts { + if overlaps(p, m.Path) { + return fmt.Errorf("%w: collect path %s overlaps key path %s; collected files live outside key paths", ErrInvalid, p, m.Path) + } + } + } + return nil +} + +// collect reads and removes each path, in order, as the call's user: root has no capability to +// read another user's private directory, and what a call may collect is what its user may read. +func (s *Service) collect(ctx context.Context, sb *sandbox, user string, paths []string, blobs *[]Blob) ([]Collected, error) { + out := make([]Collected, 0, len(paths)) + for _, p := range paths { + readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) + stdout := &capBuffer{limit: int(s.cfg.CollectLimit)} + stderr := &capBuffer{limit: 4 << 10} + argv := []string{"/bin/sh", "-c", collectScript, "sh", p} + code, err := s.drv.Exec(readCtx, sb.container, driver.ExecSpec{Argv: argv, User: user}, stdout, stderr) + cancel() + if err != nil { + return nil, fmt.Errorf("collect %s from container %s: %w", p, sb.container, err) + } + switch code { + case 0: + case 4: + out = append(out, Collected{Path: p, Missing: true}) + continue + default: + return nil, fmt.Errorf("collect %s from container %s: exit %d: %s", p, sb.container, code, strings.TrimSpace(string(stderr.buf))) + } + c := Collected{Path: p, Size: stdout.size} + if stdout.size <= s.cfg.CollectLimit { + sum := sha256.Sum256(stdout.buf) + c.SHA256 = hex.EncodeToString(sum[:]) + *blobs = append(*blobs, Blob{SHA256: c.SHA256, Data: stdout.buf}) + } + out = append(out, c) + } + return out, nil +} + +// overlaps tells whether one of two clean absolute paths is the other or under it. +func overlaps(a, b string) bool { + return a == "/" || b == "/" || under(a, b) || under(b, a) +} + +// withoutDisplay drops the display user's processes: they belong to the display, not to a call. +func (sb *sandbox) withoutDisplay(procs []driver.Process) []driver.Process { + if sb.displayUID == "" { + return procs + } + kept := procs[:0:0] + for _, p := range procs { + if p.UID != sb.displayUID { + kept = append(kept, p) + } + } + return kept +} + +// defaultDisplayStartTimeout bounds `swarm-display start`, which waits for the browser. +const defaultDisplayStartTimeout = 60 * time.Second diff --git a/go/internal/sandboxd/display_integration_test.go b/go/internal/sandboxd/display_integration_test.go new file mode 100644 index 0000000..7d2aaef --- /dev/null +++ b/go/internal/sandboxd/display_integration_test.go @@ -0,0 +1,142 @@ +//go:build integration + +package sandboxd + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" + "github.com/jackeydou/DUNE/go/internal/driver/docker" +) + +// displayImage is built from deploy/images/display, which needs the internet, so the test +// skips when the docker host does not have it. +func displayImage(t *testing.T, drv *docker.Driver) string { + t.Helper() + img := os.Getenv("SWARMEVAL_IT_DISPLAY_IMAGE") + if img == "" { + img = "swarmeval/display:dev" + } + rc, err := drv.ExportImagePath(context.Background(), img, "/etc/swarm-display", map[string]string{driver.LabelManaged: "true"}) + if err != nil { + t.Skipf("display image %s is not usable (%v); build it with `docker build -f deploy/images/display/Dockerfile -t %s deploy/images/display`", img, err, img) + } + _ = rc.Close() + return img +} + +const page = `Shop +` + +func TestLiveDisplayDrivesTheBrowserAndKeepsItsProcessesOut(t *testing.T) { + ctx := context.Background() + drv, err := docker.New(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = drv.Close() }) + img := displayImage(t, drv) + state, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := DefaultConfig(state) + if rt := os.Getenv("SWARMEVAL_IT_RUNTIME"); rt != "" { + cfg.Runtime = rt + } + svc := New(cfg, drv, slog.New(slog.DiscardHandler)) + run := "it_display" + t.Cleanup(func() { + if err := svc.DestroyRun(context.Background(), run); err != nil { + t.Errorf("destroy: %v", err) + } + }) + if err := svc.CreateRun(ctx, run, []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err = svc.CreateSandbox(ctx, CreateRequest{ + RunID: run, SandboxID: "desk", Image: img, + Mounts: []Mount{{Path: "/workspace"}, {Path: "/tmp"}}, + Resources: driver.Resources{MemoryBytes: 2 << 30, Pids: 512}, + Files: []SeedFile{{Path: "/workspace/index.html", Content: []byte(page), Mode: 0o644}}, + Users: []string{"agent"}, + Display: &Display{Width: 1024, Height: 768, URL: "file:///workspace/index.html"}, + }) + if err != nil { + t.Fatal(err) + } + shot := []string{DisplayDir + "/out/screenshot.png"} + call := func(id, user string, argv []string, collect []string) ExecResult { + t.Helper() + res, err := svc.Exec(ctx, ExecRequest{RunID: run, SandboxID: "desk", CallID: id, Argv: argv, User: user, Timeout: time.Minute, Collect: collect}) + if err != nil { + t.Fatal(err) + } + return res + } + png := func(res ExecResult) []byte { + t.Helper() + if len(res.Collected) != 1 || res.Collected[0].SHA256 == "" { + t.Fatalf("collected = %+v; stderr %s", res.Collected, res.Stderr.Inline) + } + for _, b := range res.Blobs { + if b.SHA256 == res.Collected[0].SHA256 { + return b.Data + } + } + t.Fatal("the collected screenshot is not among the blobs") + return nil + } + + res := call("call_1", DisplayUser, []string{"swarm-display", "browser", `{"action":"snapshot","screenshot":true}`}, shot) + if res.ExitCode != 0 || !strings.Contains(string(res.Stdout.Inline), `button "Pay"`) { + t.Fatalf("snapshot exit %d:\n%s\n%s", res.ExitCode, res.Stdout.Inline, res.Stderr.Inline) + } + if !bytes.HasPrefix(png(res), []byte("\x89PNG\r\n\x1a\n")) { + t.Fatal("the browser screenshot is not a PNG") + } + ref := refOf(t, string(res.Stdout.Inline), `button "Pay"`) + res = call("call_2", DisplayUser, []string{"swarm-display", "browser", `{"action":"click","ref":"` + ref + `"}`}, shot) + if res.ExitCode != 0 || !strings.Contains(string(res.Stdout.Inline), "Title: paid") { + t.Fatalf("click exit %d:\n%s\n%s", res.ExitCode, res.Stdout.Inline, res.Stderr.Inline) + } + if !res.Collected[0].Missing { + t.Fatalf("a click without `screenshot` collected %+v", res.Collected) + } + res = call("call_3", DisplayUser, []string{"swarm-display", "computer", `{"action":"click","coordinate":[500,400]}`}, shot) + if res.ExitCode != 0 || !bytes.HasPrefix(png(res), []byte("\x89PNG")) { + t.Fatalf("computer click exit %d: %s", res.ExitCode, res.Stderr.Inline) + } + if len(res.Processes) != 0 || len(res.Changes) != 0 || len(res.Background) != 0 { + t.Fatalf("display calls reported processes %+v, changes %+v, background %+v", res.Processes, res.Changes, res.Background) + } + + res = call("call_4", "agent", []string{"sh", "-c", "DISPLAY=:1 xdotool getmouselocation; ls /run/swarm-display"}, nil) + if res.ExitCode == 0 { + t.Fatalf("the agent's user reached the display:\n%s", res.Stdout.Inline) + } + res = call("call_5", "agent", []string{"sh", "-c", "echo x > /workspace/note"}, nil) + if len(res.Changes) != 1 || res.Changes[0].Ambiguous { + t.Fatalf("changes = %+v; the browser's processes must not make the call's own write ambiguous", res.Changes) + } +} + +// refOf finds the ref of the snapshot line holding what. +func refOf(t *testing.T, snapshot, what string) string { + t.Helper() + for line := range strings.Lines(snapshot) { + if i := strings.Index(line, "[ref="); strings.Contains(line, what) && i >= 0 { + ref, _, _ := strings.Cut(line[i+len("[ref="):], "]") + return ref + } + } + t.Fatalf("no %s in the snapshot:\n%s", what, snapshot) + return "" +} diff --git a/go/internal/sandboxd/display_test.go b/go/internal/sandboxd/display_test.go new file mode 100644 index 0000000..1324388 --- /dev/null +++ b/go/internal/sandboxd/display_test.go @@ -0,0 +1,176 @@ +package sandboxd + +import ( + "context" + "errors" + "io" + "log/slog" + "slices" + "testing" + + "github.com/jackeydou/DUNE/go/internal/driver" +) + +// newDisplayFixture creates a sandbox with a display; the fake's display stack is pid 50, run +// by swarmdisplay, beside the container's own sleep. +func newDisplayFixture(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + idle := driver.Process{PID: 1, PPID: 0, User: "root", Cmdline: "sleep infinity"} + stack := driver.Process{PID: 50, PPID: 1, User: DisplayUser, Cmdline: "swarm-display daemon"} + f.drv.procs = []driver.Process{idle} + f.drv.display = func(stdout io.Writer) (int, error) { + f.drv.procs = []driver.Process{idle, stack} + _, err := io.WriteString(stdout, "50\n") + return 0, err + } + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", + Mounts: []Mount{{Path: "/workspace"}}, + Display: &Display{Width: 1024, Height: 768, URL: "http://127.0.0.1:8080/"}, + }) + if err != nil { + t.Fatal(err) + } + return f +} + +func TestDisplayStartsAsTheDisplayUserWithItsSize(t *testing.T) { + f := newDisplayFixture(t) + + start := f.drv.execs[0] + want := []string{"swarm-display", "start", "--width", "1024", "--height", "768", "--url", "http://127.0.0.1:8080/"} + if start.User != DisplayUser || !slices.Equal(start.Argv, want) { + t.Fatalf("start = %+v, want %v as %s", start, want, DisplayUser) + } +} + +func TestDisplayProcessesAreNeitherReportedNorAttributed(t *testing.T) { + f := newDisplayFixture(t) + idle := f.drv.procs[0] + stack := f.drv.procs[1] + renderer := driver.Process{PID: 61, PPID: 50, User: DisplayUser, Cmdline: "chromium --type=renderer"} + server := driver.Process{PID: 70, PPID: 1, User: "agent", Cmdline: "python -m http.server"} + + f.drv.handler = func(context.Context, driver.ExecSpec, io.Writer, io.Writer) (int, error) { + f.drv.procs = []driver.Process{idle, stack, renderer, server} + return 0, nil + } + res, err := f.svc.Exec(context.Background(), ExecRequest{RunID: "run_2", SandboxID: "desk", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9}) + if err != nil { + t.Fatal(err) + } + + if len(res.Processes) != 1 || res.Processes[0].PID != 70 { + t.Fatalf("processes = %+v; the renderer is the display's, the server the call's", res.Processes) + } +} + +func TestDisplayStartFailureRemovesTheContainer(t *testing.T) { + cases := map[string]func(io.Writer) (int, error){ + "exit": func(io.Writer) (int, error) { return 1, nil }, + "no pid": func(w io.Writer) (int, error) { _, err := io.WriteString(w, "ready\n"); return 0, err }, + "gone": func(w io.Writer) (int, error) { _, err := io.WriteString(w, "99\n"); return 0, err }, + } + for name, play := range cases { + f := newFixture(t) + f.drv.display = play + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", Display: &Display{Width: 1024, Height: 768}, + }) + if err == nil { + t.Errorf("%s: CreateSandbox succeeded", name) + continue + } + if !slices.Contains(f.drv.removed, driver.ContainerID("c_desk")) { + t.Errorf("%s: container not removed after %v", name, err) + } + } +} + +func TestDisplayIsRefusedWhenItCannotWork(t *testing.T) { + cases := map[string]CreateRequest{ + "too small": {Display: &Display{Width: 100, Height: 768}}, + "too large": {Display: &Display{Width: 1024, Height: 4000}}, + "url with newline": {Display: &Display{Width: 1024, Height: 768, URL: "http://x/\nevil"}}, + "key path at its dir": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: DisplayDir}}}, + "key path above it": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/run"}}}, + "key path at root": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/"}}}, + } + for name, req := range cases { + svc := New(DefaultConfig(t.TempDir()), &fakeDriver{}, slog.New(slog.DiscardHandler)) + req.RunID, req.SandboxID, req.Image = "run_1", "desk", "img" + if err := svc.CreateRun(context.Background(), "run_1", []string{"desk"}); err != nil { + t.Fatal(err) + } + if _, err := svc.CreateSandbox(context.Background(), req); !errors.Is(err, ErrInvalid) { + t.Errorf("%s: err = %v, want ErrInvalid", name, err) + } + } +} + +func TestExecCollectsAndRemovesFiles(t *testing.T) { + f := newFixture(t) + png := []byte("\x89PNG fake") + f.drv.files = map[string][]byte{"/run/swarm-display/out/screenshot.png": png} + + res, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{"/run/swarm-display/out/screenshot.png", "/run/swarm-display/out/none.png"}, + }) + if err != nil { + t.Fatal(err) + } + + if len(res.Collected) != 2 { + t.Fatalf("collected = %+v", res.Collected) + } + got, missing := res.Collected[0], res.Collected[1] + if got.Missing || got.Size != int64(len(png)) || got.SHA256 == "" || !missing.Missing { + t.Fatalf("collected = %+v", res.Collected) + } + i := slices.IndexFunc(res.Blobs, func(b Blob) bool { return b.SHA256 == got.SHA256 }) + if i < 0 || string(res.Blobs[i].Data) != string(png) { + t.Fatalf("blobs = %v; the collected file must follow as a blob", res.Blobs) + } + if _, left := f.drv.files["/run/swarm-display/out/screenshot.png"]; left { + t.Fatal("collected file was not removed") + } +} + +func TestExecCollectOverTheLimitIsRemovedButNotSent(t *testing.T) { + f := newFixture(t) + f.svc.cfg.CollectLimit = 4 + f.drv.files = map[string][]byte{"/run/out.png": []byte("too large")} + + res, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{"/run/out.png"}, + }) + if err != nil { + t.Fatal(err) + } + + if c := res.Collected[0]; c.Missing || c.Size != 9 || c.SHA256 != "" { + t.Fatalf("collected = %+v", c) + } +} + +func TestExecRefusesCollectPathsInKeyPaths(t *testing.T) { + f := newFixture(t) + for _, p := range []string{"/workspace/shot.png", "/", "relative.png", "/run/../workspace/x"} { + _, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{p}, + }) + if !errors.Is(err, ErrInvalid) { + t.Errorf("collect %q: err = %v, want ErrInvalid", p, err) + } + } +} diff --git a/go/internal/sandboxd/exec.go b/go/internal/sandboxd/exec.go new file mode 100644 index 0000000..fa6b91e --- /dev/null +++ b/go/internal/sandboxd/exec.go @@ -0,0 +1,229 @@ +package sandboxd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" + "github.com/jackeydou/DUNE/go/internal/fsdiff" +) + +// ExecRequest is one tool call. +type ExecRequest struct { + RunID string + SandboxID string + CallID string + Argv []string + Cwd string + User string + Timeout time.Duration + // Collect lists files outside the key paths to read, remove, and return after the command. + Collect []string +} + +// Change is a file change with its attribution. +type Change struct { + fsdiff.Change + Protected bool + Ambiguous bool + Candidates []string + // Content is set when the new content is among the result's blobs. + Content bool +} + +// ExecResult is everything one call produced. +type ExecResult struct { + ExitCode int + TimedOut bool + Duration time.Duration + Stdout Output + Stderr Output + Background []Change + Changes []Change + Processes []driver.Process + Collected []Collected + Blobs []Blob +} + +// Exec runs one call and reports its output, the changes around it, and what it left +// running. +func (s *Service) Exec(ctx context.Context, req ExecRequest) (ExecResult, error) { + if len(req.Argv) == 0 { + return ExecResult{}, fmt.Errorf("%w: call %s has an empty argv", ErrInvalid, req.CallID) + } + if req.Timeout <= 0 { + return ExecResult{}, fmt.Errorf("%w: call %s has no timeout", ErrInvalid, req.CallID) + } + if req.CallID == "" { + return ExecResult{}, fmt.Errorf("%w: call id is required for attribution", ErrInvalid) + } + sb, err := s.lookup(req.RunID, req.SandboxID) + if err != nil { + return ExecResult{}, err + } + if err := checkCollect(req.Collect, sb.mounts); err != nil { + return ExecResult{}, fmt.Errorf("call %s: %w", req.CallID, err) + } + sb.mu.Lock() + defer sb.mu.Unlock() + sb.execd = true + + pre, err := fsdiff.Walk(sb.roots, sb.manifest) + if err != nil { + return ExecResult{}, err + } + baseline, err := s.processes(ctx, sb) + if err != nil { + return ExecResult{}, err + } + candidates := sb.liveCalls(baseline) + budget := s.cfg.ContentBudget + var blobs []Blob + background := s.attribute(sb, fsdiff.Diff(sb.manifest, pre), true, candidates, &budget, &blobs) + + stdout := &capBuffer{limit: s.cfg.OutputLimit} + stderrBuf := &capBuffer{limit: s.cfg.OutputLimit} + stderr := &pidWriter{next: stderrBuf} + start := time.Now() + execCtx, cancel := context.WithTimeout(ctx, req.Timeout) + code, err := s.drv.Exec(execCtx, sb.container, driver.ExecSpec{Argv: wrapArgv(req.Argv), Cwd: req.Cwd, User: req.User}, stdout, stderr) + cancel() + duration := time.Since(start) + timedOut := false + switch { + case err == nil: + case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: + timedOut, code = true, -1 + if err := s.killTree(ctx, sb, req, stderr.pid); err != nil { + return ExecResult{}, err + } + default: + return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) + } + if err := stderr.flush(); err != nil { + return ExecResult{}, err + } + collected, err := s.collect(ctx, sb, req.User, req.Collect, &blobs) + if err != nil { + return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) + } + + post, err := fsdiff.Walk(sb.roots, pre) + if err != nil { + return ExecResult{}, err + } + after, err := s.processes(ctx, sb) + if err != nil { + return ExecResult{}, err + } + windowCandidates := candidates + if len(candidates) > 0 { + windowCandidates = append(slices.Clone(candidates), req.CallID) + } + changes := s.attribute(sb, fsdiff.Diff(pre, post), len(candidates) > 0, windowCandidates, &budget, &blobs) + started := sb.recordProcesses(baseline, after, req.CallID) + sb.manifest = post + + outBlob := func(c *capBuffer) Output { + out, blob := c.output(s.cfg.InlineLimit) + if blob != nil { + blobs = append(blobs, *blob) + } + return out + } + return ExecResult{ + ExitCode: code, + TimedOut: timedOut, + Duration: duration, + Stdout: outBlob(stdout), + Stderr: outBlob(stderrBuf), + Background: background, + Changes: changes, + Processes: started, + Collected: collected, + Blobs: blobs, + }, nil +} + +// killTree kills a timed-out command and its descendants from inside the sandbox, as the +// call's own user: with every capability dropped, only the same uid may signal them. +func (s *Service) killTree(ctx context.Context, sb *sandbox, req ExecRequest, pid string) error { + if pid == "" { + s.log.Warn("timed-out call printed no pid; its processes are left running and will be reported", + "run_id", req.RunID, "sandbox_id", req.SandboxID, "call_id", req.CallID) + return nil + } + killCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) + defer cancel() + argv := []string{"/bin/sh", "-c", killTreeScript, "sh", pid} + if _, err := s.drv.Exec(killCtx, sb.container, driver.ExecSpec{Argv: argv, User: req.User}, io.Discard, io.Discard); err != nil { + return fmt.Errorf("kill timed-out call %s (pid %s) in sandbox %s: %w", req.CallID, pid, req.SandboxID, err) + } + return nil +} + +func (s *Service) attribute(sb *sandbox, diff []fsdiff.Change, ambiguous bool, candidates []string, budget *int64, blobs *[]Blob) []Change { + out := make([]Change, len(diff)) + for i, c := range diff { + out[i] = Change{ + Change: c, + Protected: protected(sb.mounts, c.Path), + Ambiguous: ambiguous, + Candidates: candidates, + } + if c.Op == fsdiff.OpDelete || c.After.Kind != fsdiff.KindFile || c.After.Size > s.cfg.ContentLimit || c.After.Size > *budget { + continue + } + data, err := os.ReadFile(c.After.HostPath) + if err != nil { + continue // Gone or replaced since the walk; the hash still stands. + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != c.After.SHA256 { + continue + } + *budget -= int64(len(data)) + *blobs = append(*blobs, Blob{SHA256: c.After.SHA256, Data: data}) + out[i].Content = true + } + return out +} + +// liveCalls lists the calls whose processes are still alive, sorted. +func (sb *sandbox) liveCalls(procs []driver.Process) []string { + seen := map[string]bool{} + for _, p := range procs { + if call, ok := sb.live[p.PID]; ok { + seen[call] = true + } + } + return slices.Sorted(maps.Keys(seen)) +} + +// recordProcesses returns processes alive after the call that were not alive before it, +// and forgets processes that have exited. +func (sb *sandbox) recordProcesses(before, after []driver.Process, callID string) []driver.Process { + existed := map[int32]bool{} + for _, p := range before { + existed[p.PID] = true + } + alive := map[int32]bool{} + var started []driver.Process + for _, p := range after { + alive[p.PID] = true + if !existed[p.PID] { + started = append(started, p) + sb.live[p.PID] = callID + } + } + maps.DeleteFunc(sb.live, func(pid int32, _ string) bool { return !alive[pid] }) + return started +} diff --git a/go/internal/sandboxd/procs.go b/go/internal/sandboxd/procs.go index 0fbaafe..fb9146d 100644 --- a/go/internal/sandboxd/procs.go +++ b/go/internal/sandboxd/procs.go @@ -36,7 +36,7 @@ for d in /proc/[0-9]*; do done` // processes lists what runs in the sandbox, with pids as the sandbox sees them: the same pids -// killTree signals. It runs as root, which reads every process's /proc entries without any +// killTree signals. The display's processes are left out. It runs as root, which reads every process's /proc entries without any // capability. func (s *Service) processes(ctx context.Context, sb *sandbox) ([]driver.Process, error) { listCtx, cancel := context.WithTimeout(ctx, s.cfg.HelperTimeout) @@ -56,7 +56,11 @@ func (s *Service) processes(ctx context.Context, sb *sandbox) ([]driver.Process, return nil, fmt.Errorf("process listing of container %s is %d bytes, over the %d byte limit: the sandbox holds too many processes or too long command lines. Set a pids limit in the profile", sb.container, stdout.size, s.cfg.ProcessListLimit) } - return parseProcesses(string(stdout.buf)) + procs, err := parseProcesses(string(stdout.buf)) + if err != nil { + return nil, err + } + return sb.withoutDisplay(procs), nil } // parseProcesses reads processScript's output. Names come from the sandbox's /etc/passwd, which @@ -86,13 +90,13 @@ func parseProcesses(out string) ([]driver.Process, error) { if err := errors.Join(err1, err2, err3); err != nil { return nil, fmt.Errorf("process listing line %q: %w", line, err) } - procs = append(procs, driver.Process{PID: int32(pid), PPID: int32(ppid), User: fields[2], Cmdline: strings.TrimRight(fields[3], " ")}) + procs = append(procs, driver.Process{PID: int32(pid), PPID: int32(ppid), UID: fields[2], User: fields[2], Cmdline: strings.TrimRight(fields[3], " ")}) default: return nil, fmt.Errorf("process listing line %q is neither a U nor a P line", line) } } for i, p := range procs { - if name, ok := names[p.User]; ok { + if name, ok := names[p.UID]; ok { procs[i].User = name } } diff --git a/go/internal/sandboxd/procs_test.go b/go/internal/sandboxd/procs_test.go index 218590a..33a8c4a 100644 --- a/go/internal/sandboxd/procs_test.go +++ b/go/internal/sandboxd/procs_test.go @@ -21,9 +21,9 @@ func TestParseProcessesNamesUsersAndKeepsCommandLines(t *testing.T) { t.Fatal(err) } want := []driver.Process{ - {PID: 1, PPID: 0, User: "root", Cmdline: "/sbin/docker-init -- sleep infinity"}, - {PID: 14, PPID: 1, User: "qa", Cmdline: "python3 -m http.server 8000"}, - {PID: 20, PPID: 14, User: "4242", Cmdline: "[kworker]"}, + {PID: 1, PPID: 0, UID: "0", User: "root", Cmdline: "/sbin/docker-init -- sleep infinity"}, + {PID: 14, PPID: 1, UID: "1001", User: "qa", Cmdline: "python3 -m http.server 8000"}, + {PID: 20, PPID: 14, UID: "4242", User: "4242", Cmdline: "[kworker]"}, } if !slices.Equal(procs, want) { t.Fatalf("procs = %+v\nwant %+v", procs, want) diff --git a/go/internal/sandboxd/server.go b/go/internal/sandboxd/server.go index 6393cdf..934f36b 100644 --- a/go/internal/sandboxd/server.go +++ b/go/internal/sandboxd/server.go @@ -68,6 +68,7 @@ func (s *Server) CreateSandbox(ctx context.Context, req *sandboxv1.CreateSandbox Env: req.GetEnv(), Hostname: req.GetHostname(), MachineID: req.GetMachineId(), + Display: display(req.GetDisplay()), }) if err != nil { return nil, s.status("CreateSandbox", err) @@ -87,6 +88,7 @@ func (s *Server) Exec(req *sandboxv1.ExecRequest, stream grpc.ServerStreamingSer Cwd: req.GetCwd(), User: req.GetUser(), Timeout: req.GetTimeout().AsDuration(), + Collect: req.GetCollect(), }) if err != nil { return s.status("Exec", err) @@ -100,6 +102,7 @@ func (s *Server) Exec(req *sandboxv1.ExecRequest, stream grpc.ServerStreamingSer BackgroundChanges: changes(res.Background), Changes: changes(res.Changes), Processes: processes(res.Processes), + Collected: collected(res.Collected), } if err := stream.Send(&sandboxv1.ExecResponse{Item: &sandboxv1.ExecResponse_Header{Header: header}}); err != nil { return err @@ -200,6 +203,21 @@ func sendBlobs(blobs []Blob, send func(*sandboxv1.BlobChunk) error) error { return nil } +func display(d *sandboxv1.Display) *Display { + if d == nil { + return nil + } + return &Display{Width: d.GetWidth(), Height: d.GetHeight(), URL: d.GetUrl()} +} + +func collected(cs []Collected) []*sandboxv1.CollectedFile { + out := make([]*sandboxv1.CollectedFile, len(cs)) + for i, c := range cs { + out[i] = &sandboxv1.CollectedFile{Path: c.Path, Missing: c.Missing, Size: c.Size, Sha256: c.SHA256} + } + return out +} + func output(o Output) *sandboxv1.Output { return &sandboxv1.Output{Inline: o.Inline, Size: o.Size, BlobSha256: o.BlobSHA256, Capped: o.Capped} } diff --git a/go/internal/sandboxd/service.go b/go/internal/sandboxd/service.go index 65d8bef..193dab2 100644 --- a/go/internal/sandboxd/service.go +++ b/go/internal/sandboxd/service.go @@ -5,14 +5,10 @@ package sandboxd import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" - "io" "io/fs" "log/slog" - "maps" "net/netip" "os" "path" @@ -71,6 +67,10 @@ type Config struct { // ProcessListLimit caps a sandbox's process listing. A sandbox controls how many processes // it has and how long their command lines are. ProcessListLimit int + // CollectLimit caps each file an Exec collects. A larger file is removed but not sent. + CollectLimit int64 + // DisplayStartTimeout bounds starting a sandbox's display stack. + DisplayStartTimeout time.Duration } // DefaultConfig returns the limits sandboxd uses unless flags override them. @@ -89,6 +89,9 @@ func DefaultConfig(stateDir string) Config { SeedLimit: 1 << 20, RestoreLimit: 3 << 20, ProcessListLimit: 4 << 20, + CollectLimit: 8 << 20, + + DisplayStartTimeout: defaultDisplayStartTimeout, } } @@ -119,6 +122,9 @@ type sandbox struct { live map[int32]string // execd is set by the first Exec; RestoreFiles is refused after it. execd bool + // displayUID is the uid running the display stack, empty without a display. Its + // processes are left out of every listing. + displayUID string } // New returns a Service. cfg.StateDir must exist. @@ -147,6 +153,8 @@ type CreateRequest struct { Env map[string]string Hostname string MachineID string + // Display, when set, starts the image's display stack before the first manifest. + Display *Display } // SeedFile is written into a key path before the first manifest. @@ -178,6 +186,9 @@ func (s *Service) CreateSandbox(ctx context.Context, req CreateRequest) (string, if err := checkIdentity(req, mounts); err != nil { return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } + if err := checkDisplay(req.Display, mounts); err != nil { + return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) + } net, err := s.network(req.RunID, req.SandboxID) if err != nil { return "", err @@ -268,6 +279,13 @@ func (s *Service) create(ctx context.Context, req CreateRequest, sb *sandbox, ru return fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } sb.container = id + if req.Display != nil { + uid, err := s.startDisplay(ctx, sb, req.Display) + if err != nil { + return errors.Join(fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err), s.drv.Remove(context.WithoutCancel(ctx), id)) + } + sb.displayUID = uid + } manifest, err := fsdiff.Walk(sb.roots, nil) if err != nil { return errors.Join(err, s.drv.Remove(context.WithoutCancel(ctx), id)) @@ -310,207 +328,6 @@ func (s *Service) runtime(ctx context.Context) (string, error) { return "runc", nil } -// ExecRequest is one tool call. -type ExecRequest struct { - RunID string - SandboxID string - CallID string - Argv []string - Cwd string - User string - Timeout time.Duration -} - -// Change is a file change with its attribution. -type Change struct { - fsdiff.Change - Protected bool - Ambiguous bool - Candidates []string - // Content is set when the new content is among the result's blobs. - Content bool -} - -// ExecResult is everything one call produced. -type ExecResult struct { - ExitCode int - TimedOut bool - Duration time.Duration - Stdout Output - Stderr Output - Background []Change - Changes []Change - Processes []driver.Process - Blobs []Blob -} - -// Exec runs one call and reports its output, the changes around it, and what it left -// running. -func (s *Service) Exec(ctx context.Context, req ExecRequest) (ExecResult, error) { - if len(req.Argv) == 0 { - return ExecResult{}, fmt.Errorf("%w: call %s has an empty argv", ErrInvalid, req.CallID) - } - if req.Timeout <= 0 { - return ExecResult{}, fmt.Errorf("%w: call %s has no timeout", ErrInvalid, req.CallID) - } - if req.CallID == "" { - return ExecResult{}, fmt.Errorf("%w: call id is required for attribution", ErrInvalid) - } - sb, err := s.lookup(req.RunID, req.SandboxID) - if err != nil { - return ExecResult{}, err - } - sb.mu.Lock() - defer sb.mu.Unlock() - sb.execd = true - - pre, err := fsdiff.Walk(sb.roots, sb.manifest) - if err != nil { - return ExecResult{}, err - } - baseline, err := s.processes(ctx, sb) - if err != nil { - return ExecResult{}, err - } - candidates := sb.liveCalls(baseline) - budget := s.cfg.ContentBudget - var blobs []Blob - background := s.attribute(sb, fsdiff.Diff(sb.manifest, pre), true, candidates, &budget, &blobs) - - stdout := &capBuffer{limit: s.cfg.OutputLimit} - stderrBuf := &capBuffer{limit: s.cfg.OutputLimit} - stderr := &pidWriter{next: stderrBuf} - start := time.Now() - execCtx, cancel := context.WithTimeout(ctx, req.Timeout) - code, err := s.drv.Exec(execCtx, sb.container, driver.ExecSpec{Argv: wrapArgv(req.Argv), Cwd: req.Cwd, User: req.User}, stdout, stderr) - cancel() - duration := time.Since(start) - timedOut := false - switch { - case err == nil: - case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: - timedOut, code = true, -1 - if err := s.killTree(ctx, sb, req, stderr.pid); err != nil { - return ExecResult{}, err - } - default: - return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) - } - if err := stderr.flush(); err != nil { - return ExecResult{}, err - } - - post, err := fsdiff.Walk(sb.roots, pre) - if err != nil { - return ExecResult{}, err - } - after, err := s.processes(ctx, sb) - if err != nil { - return ExecResult{}, err - } - windowCandidates := candidates - if len(candidates) > 0 { - windowCandidates = append(slices.Clone(candidates), req.CallID) - } - changes := s.attribute(sb, fsdiff.Diff(pre, post), len(candidates) > 0, windowCandidates, &budget, &blobs) - started := sb.recordProcesses(baseline, after, req.CallID) - sb.manifest = post - - outBlob := func(c *capBuffer) Output { - out, blob := c.output(s.cfg.InlineLimit) - if blob != nil { - blobs = append(blobs, *blob) - } - return out - } - return ExecResult{ - ExitCode: code, - TimedOut: timedOut, - Duration: duration, - Stdout: outBlob(stdout), - Stderr: outBlob(stderrBuf), - Background: background, - Changes: changes, - Processes: started, - Blobs: blobs, - }, nil -} - -// killTree kills a timed-out command and its descendants from inside the sandbox, as the -// call's own user: with every capability dropped, only the same uid may signal them. -func (s *Service) killTree(ctx context.Context, sb *sandbox, req ExecRequest, pid string) error { - if pid == "" { - s.log.Warn("timed-out call printed no pid; its processes are left running and will be reported", - "run_id", req.RunID, "sandbox_id", req.SandboxID, "call_id", req.CallID) - return nil - } - killCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) - defer cancel() - argv := []string{"/bin/sh", "-c", killTreeScript, "sh", pid} - if _, err := s.drv.Exec(killCtx, sb.container, driver.ExecSpec{Argv: argv, User: req.User}, io.Discard, io.Discard); err != nil { - return fmt.Errorf("kill timed-out call %s (pid %s) in sandbox %s: %w", req.CallID, pid, req.SandboxID, err) - } - return nil -} - -func (s *Service) attribute(sb *sandbox, diff []fsdiff.Change, ambiguous bool, candidates []string, budget *int64, blobs *[]Blob) []Change { - out := make([]Change, len(diff)) - for i, c := range diff { - out[i] = Change{ - Change: c, - Protected: protected(sb.mounts, c.Path), - Ambiguous: ambiguous, - Candidates: candidates, - } - if c.Op == fsdiff.OpDelete || c.After.Kind != fsdiff.KindFile || c.After.Size > s.cfg.ContentLimit || c.After.Size > *budget { - continue - } - data, err := os.ReadFile(c.After.HostPath) - if err != nil { - continue // Gone or replaced since the walk; the hash still stands. - } - sum := sha256.Sum256(data) - if hex.EncodeToString(sum[:]) != c.After.SHA256 { - continue - } - *budget -= int64(len(data)) - *blobs = append(*blobs, Blob{SHA256: c.After.SHA256, Data: data}) - out[i].Content = true - } - return out -} - -// liveCalls lists the calls whose processes are still alive, sorted. -func (sb *sandbox) liveCalls(procs []driver.Process) []string { - seen := map[string]bool{} - for _, p := range procs { - if call, ok := sb.live[p.PID]; ok { - seen[call] = true - } - } - return slices.Sorted(maps.Keys(seen)) -} - -// recordProcesses returns processes alive after the call that were not alive before it, -// and forgets processes that have exited. -func (sb *sandbox) recordProcesses(before, after []driver.Process, callID string) []driver.Process { - existed := map[int32]bool{} - for _, p := range before { - existed[p.PID] = true - } - alive := map[int32]bool{} - var started []driver.Process - for _, p := range after { - alive[p.PID] = true - if !existed[p.PID] { - started = append(started, p) - sb.live[p.PID] = callID - } - } - maps.DeleteFunc(sb.live, func(pid int32, _ string) bool { return !alive[pid] }) - return started -} - // SandboxChanges are one sandbox's changes found by FinalDiff. type SandboxChanges struct { SandboxID string diff --git a/go/internal/sandboxd/service_test.go b/go/internal/sandboxd/service_test.go index 96784f1..278a2c5 100644 --- a/go/internal/sandboxd/service_test.go +++ b/go/internal/sandboxd/service_test.go @@ -35,6 +35,10 @@ type fakeDriver struct { networks []driver.NetworkSpec netsGone []driver.NetworkID failNet string // CreateNetwork fails for this name + // files are regular files outside the key paths, for collectScript. + files map[string][]byte + // display plays `swarm-display start`: its output and the processes it leaves. + display func(stdout io.Writer) (int, error) } func (f *fakeDriver) Runtimes(context.Context) ([]string, error) { return f.runtimes, nil } @@ -100,9 +104,24 @@ func (f *fakeDriver) CreateContainer(_ context.Context, spec driver.ContainerSpe } func (f *fakeDriver) Exec(ctx context.Context, _ driver.ContainerID, spec driver.ExecSpec, stdout, stderr io.Writer) (int, error) { + if spec.Argv[0] == "swarm-display" { + f.mu.Lock() + f.execs = append(f.execs, spec) + f.mu.Unlock() + return f.display(stdout) + } if spec.Argv[2] == processScript { return f.listing(stdout) } + if spec.Argv[2] == collectScript { + data, ok := f.files[spec.Argv[4]] + if !ok { + return 4, nil + } + delete(f.files, spec.Argv[4]) + _, err := stdout.Write(data) + return 0, err + } f.mu.Lock() f.execs = append(f.execs, spec) f.mu.Unlock() diff --git a/mise.toml b/mise.toml index 2d422b0..1e63351 100644 --- a/mise.toml +++ b/mise.toml @@ -48,6 +48,10 @@ description = "sandboxd against the local docker daemon (needs busybox:latest an dir = "go" run = "go test -tags integration -count=1 ./..." +[tasks."display:build"] +description = "Build swarmeval/display:dev, the sandbox image for the browser and computer tools. Downloads Debian packages and Chromium" +run = "docker build -f deploy/images/display/Dockerfile -t swarmeval/display:dev deploy/images/display" + [tasks."deploy:smoke"] description = "The compose stack with a recorded model backend: a case created, run, and read through the CLI and the API. Needs docker" run = "deploy/compose/smoke.sh" diff --git a/proto/swarmeval/sandbox/v1/sandbox.proto b/proto/swarmeval/sandbox/v1/sandbox.proto index cc9dc01..4569f86 100644 --- a/proto/swarmeval/sandbox/v1/sandbox.proto +++ b/proto/swarmeval/sandbox/v1/sandbox.proto @@ -19,7 +19,8 @@ service SandboxService { rpc CreateRun(CreateRunRequest) returns (CreateRunResponse); // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. rpc CreateSandbox(CreateSandboxRequest) returns (CreateSandboxResponse); // Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -92,6 +93,19 @@ message CreateSandboxRequest { // diffed. Empty writes nothing. 32 lowercase hex characters, the file's format. The image // must have /etc. string machine_id = 10; + // Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + // no display. + Display display = 11; +} + +// A virtual screen with a browser on it, run by the image's `swarm-display` as the user +// `swarmdisplay`. Its processes are never reported or attributed to a call. +message Display { + // Pixels, 320 to 1920 wide and 240 to 1200 high. + uint32 width = 1; + uint32 height = 2; + // Page the browser opens first. Empty opens about:blank. + string url = 3; } message SeedFile { @@ -118,6 +132,10 @@ message ExecRequest { string user = 6; // Required. On expiry the command and its descendants are killed. google.protobuf.Duration timeout = 7; + // Absolute paths outside every key path. After the command, sandboxd reads each regular file + // there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + // text, such as a screenshot. + repeated string collect = 8; } message ExecResponse { @@ -141,6 +159,19 @@ message ExecHeader { repeated FsChange changes = 7; // Started while this call ran and still alive after it. repeated Process processes = 8; + // One per `collect` path, in request order. + repeated CollectedFile collected = 9; +} + +message CollectedFile { + string path = 1; + // No regular file was there. + bool missing = 2; + // Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + // `sha256` is empty. + int64 size = 3; + // The content follows as a blob with this hash. + string sha256 = 4; } message Output { diff --git a/spec/2026-10-07-browser-computer-use/README.md b/spec/2026-10-07-browser-computer-use/README.md new file mode 100644 index 0000000..5330160 --- /dev/null +++ b/spec/2026-10-07-browser-computer-use/README.md @@ -0,0 +1,177 @@ +# 沙箱里的 browser 和 computer：agent 操作图形桌面和浏览器 + +## Status + +accepted · 2026-10-07 · 方案（沙箱内显示栈，只访问沙箱内）、截图进上下文、事件格式升版本且不为旧格式写兼容层、 +坐标约定、fork 处理由提出人选定（2026-10-07）；其余决定是起草人的建议。 + +## Request + +提出人（2026-10-07）："我需要能让 agent 运行的时候给 sandbox 支持上 browser 和 computer 使用的能力。"在三个方案 +（A：显示栈装在沙箱里，只访问沙箱内；B：单独的浏览器出口容器上公网；C：等 net-gateway）里选了 A："按照方案 A +就可以，让 agent 在 sandbox 里面可以使用 browser 和 computer"。 + +提问时确认的四点（2026-10-07）： + +- browser 也要能截图。事件格式升版本；"如果兼容要写很多的代码就不兼容了"。 +- 先写 spec，然后直接实现。 +- 截图喂给 API 模型（Claude、GPT），用屏幕的实际像素坐标。 +- 用过 browser 或 computer 的 run，在那之后的事件上拒绝 fork。 + +## Decisions + +### 一、沙箱里的显示栈 + +#### 1. 显示栈装在沙箱里，隔离不变 + +沙箱仍然是 `--network none`。浏览器只能访问沙箱内的 `localhost` 和 `file://`，访问不了公网，也访问不了别的沙箱。 +隔离默认值（AGENTS.md "Isolation defaults"）一点不变，所以不需要另外签字。上公网的浏览是方案 B / C，不在这里做。 + +#### 2. profile 的 `display` 打开显示栈 + +`env.yaml` 的 sandbox profile 加一个可选的 `display`： + +```yaml +sandbox_profiles: + desktop: + image: swarmeval/display:dev + fs: [{ path: /workspace }] + limits: { memory: 2gib, pids: 512 } + display: { width: 1024, height: 768, url: "http://127.0.0.1:8080/" } +``` + +- `width`、`height` 默认 1024×768。这个尺寸 Claude 不会缩放，GPT 的 high detail 也不会缩放，所以模型看到的像素 + 坐标就是屏幕坐标（提出人选定用像素坐标）。上限 1920×1200。 +- `url` 可选：浏览器启动后打开的第一页，默认 `about:blank`。 +- 只加键，不改已有键。按 [case 格式的版本规则](../../docs/case-format.md#versioning)，新字段要升版本： + `env.yaml` 升到 2，`display` 只在 2 里有效，版本 1 照常读。 + +#### 3. 显示镜像的约定 + +`deploy/images/display/` 构建 `swarmeval/display`：Debian、Xvfb、一个让每个窗口都最大化的窗口管理器 +（matchbox）、xdotool、Playwright 1.63 和它的 Chromium，再加 `swarm-display` 程序。case 作者用 +`FROM swarmeval/display` 加自己的应用。镜像必须提供： + +- 用户 `swarmdisplay`，以及只属于它、权限 `0700` 的 `/run/swarm-display`。 +- `swarm-display start|computer|browser`。 +- 沙箱镜像本来就要求的 `sh`、`sleep`、`tr`。 + +`/etc/swarm-display/start.d/` 里的可执行文件在浏览器打开之前依次以 `swarmdisplay` 运行，用来启动 case 自己的应用， +比如 `127.0.0.1:8080` 上的网店。 + +#### 4. sandboxd 在建沙箱时启动它 + +`CreateSandboxRequest` 加 `display`（宽、高、起始 URL）。设置了它时，sandboxd 在容器启动之后、取第一份 manifest +之前，以 `swarmdisplay` 运行 `swarm-display start`。这个命令等显示栈就绪后退出，在 stdout 上打印守护进程的 pid。 +超时、退出码非 0、或者输出不是一个 pid，都让 `CreateSandbox` 失败。 + +显示栈的 `HOME`、`TMPDIR`、浏览器 profile 都放在 `/run/swarm-display` 下，不在任何 key path 里，所以不产生文件 +变更事件。一个 key path 如果是 `/run/swarm-display`、在它下面、或者包含它，就拒绝。 + +#### 5. 显示用户的进程不算任何调用的进程 + +浏览器每次导航都会起新的渲染进程。要是按现在的规则，它们会记成某次调用留下的进程，之后所有调用的文件变更都会 +变成 `ambiguous`。所以 sandboxd 记下守护进程的数字 uid（从 `start` 打印的 pid 查），之后这个 uid 的进程既不进 +`processes`，也不参与归属判断。 + +代价：以 root 运行的 agent 可以把自己的进程切到这个 uid 下藏起来。不过 root agent 本来就能做很多事，这一点写进 +文档。镜像里没有终端程序，所以 `computer` 能起的程序只有 Chromium 和 case 作者自己加的。 + +### 二、工具 + +#### 6. `computer` 和 `browser` 是 SandboxTool，以显示用户运行 + +两个都是内置工具，agent 在 `tools` 里列出才有。agent 的沙箱 profile 没有 `display` 时，case 加载失败。 + +- `Exec` 加 `user`：工具自己指定运行用户，覆盖 agent 的 `os_user`。这两个工具都以 `swarmdisplay` 运行 + `swarm-display computer|browser `。 +- `swarm-display` 通过 `/run/swarm-display` 里的 unix socket 把请求交给守护进程。X 服务器有 cookie 认证，浏览器由 + 守护进程通过 pipe 控制（Playwright 默认），不开 CDP 端口。所以非 root 的 agent 绕过这两个工具就碰不到屏幕和 + 浏览器。 + +**`computer`**：`screenshot`、`click`（`button`、`count`）、`mouse_move`、`drag`、`type`、`key`、`scroll`、`wait`、 +`cursor_position`。坐标是 `[x, y]` 屏幕像素。除了 `cursor_position`，每个动作之后都返回一张截图，文字里写屏幕尺寸。 + +**`browser`**：`navigate`、`back`、`forward`、`reload`、`snapshot`、`click`、`type`、`select`、`press`、`hover`、 +`scroll`、`wait`、`tabs`、`tab_new`、`tab_select`、`tab_close`、`screenshot`。 + +- 每个动作都返回 URL、标题，以及 Playwright `aria_snapshot(mode="ai")` 产生的页面快照，快照里每个元素带 + `[ref=eN]`。元素用 `ref` 指定，内部通过 `aria-ref=` 选择器定位。Playwright 1.64 发布后换成 `page.get_by_ref`。 +- `screenshot` 动作，或者任何动作加 `screenshot: true`，会额外附上视口的截图（提出人要求 browser 也能截图）。 + +#### 7. 截图怎么出沙箱：`Exec.collect` + +`ExecRequest` 加 `collect`：命令结束后，sandboxd 以这次调用的用户读出这些路径上的普通文件，读完就删掉，作为 blob 流回去； +`ExecHeader.collected` 列出每个文件的路径、大小、sha256，以及是否缺失、是否超过上限（8 MiB）。 + +以调用的用户读，是因为沙箱 drop 了所有 capability，root 读不了别的用户的 `0700` 目录；而且一次调用能收走的，本来就 +是它的用户能读到的。两个工具都收 `/run/swarm-display/out/screenshot.png`。截图不写进 key path，所以不产生文件变更事件，agent 也看不到。 +worker 在边界上检查每张截图：PNG 签名、IHDR 里的宽高、尺寸上限；不合格的截图变成工具错误。 + +### 三、图片进上下文 + +#### 8. 消息带图片引用，不带图片字节 + +- `ToolResult` 和 `ToolMessage` 加 `images`：`ImageRef(sha256, media_type, width, height)` 的元组。图片字节在 + blob store 里，按哈希引用。`messages` 行和事件里只有引用。 +- `after_tool_result` hook 可以改写或去掉图片；改写照常记成 `intervention`。 +- 只有工具结果带图片。用户消息、频道消息、系统提示不带。 + +#### 9. 每次请求最多带最近 3 张图 + +截图累积很快。`RequestOptions` 加 `max_images`，默认 3：一次请求只带上下文里最后 3 张图，更早的图换成文字 +`[image omitted]`。`before_model_request` 可以改这个值；它记在 `ModelEvent` 里，导出时按同样的规则展开 `input`， +所以 `.eval` 里展示的就是模型实际看到的内容。 + +#### 10. gateway 协议：图片放在 tool 消息里，发给后端时挪到 user 消息 + +- worker 到 gateway：`tool` 消息的 `content` 可以是内容块列表（`text`、`image_url` 带 `data:image/png;base64,` + URL）。这是 SwarmEval 自己子集的扩展，gateway 照常校验。 +- gateway 到后端：OpenAI Chat Completions 的 tool 消息只能放文本。所以 gateway 在做完 reasoning passback **之后**， + 把 tool 消息里的图片挪到紧跟在这一串 tool 消息后面的一条 user 消息里，每张图前面标上它属于哪次工具调用。放在 + passback 之后，是因为 `within_turn` 以"最后一条 user 消息"为界，挪出来的 user 消息不能把这个边界移动。 +- worker 组请求时从 blob store 取图片；同一个 run 里同一个哈希只取一次。 + +#### 11. 事件格式 10，不写兼容层 + +- `metadata.swarmeval.schema_version` 升到 10。新增：`ToolEvent` 的 `images`；`ModelEvent` 的 `max_images`； + `exec` 观测里的 `collected`。 +- 新字段都是可选的，所以旧 run 不需要额外代码就照常能读。提出人同意，需要大量兼容代码时可以不兼容；这里不需要。 +- `ToolEvent.result` 在 Postgres 里仍然是文本，读它的地方（detect、analysis、transcript）都不用改。 +- 导出 `.eval` 时，带图的 `ToolEvent.result` 和 `ChatMessageTool` 变成 `ContentText` 加 + `ContentImage("attachment://")`，图片作为 `data:` URI 放进 `EvalSample.attachments`，每张图只放一次。 + 这样 Inspect View 可以直接看到截图，`.eval` 不依赖 blob store。 + +### 四、fork + +#### 12. 用过显示工具之后，不能再 fork + +fork 只恢复 key path 里的文件，浏览器和桌面的内存状态恢复不了。所以，如果 fork 点之前有执行过的 `browser` 或 +`computer` 调用（`executed_arguments` 不为空），就拒绝这次 fork，错误信息里说明原因（提出人选定）。在第一次 +显示工具调用之前 fork 照常可以。 + +## Rejected + +- **浏览器放在 worker 里，或者单独的出口容器（方案 B）。** 要放宽隔离默认值，还会多一个事件来源。提出人选了 A。 +- **截图走 stdout。** stdout 会转成文本，截图又要再存一份 blob；而且工具自己的文字输出就没地方放了。 +- **用 `ReadFile`（docker copy API）读截图。** gVisor 下，copy API 看不到容器 rootfs 覆盖层里的写入。 +- **截图写进 key path。** 会产生文件变更事件，agent 也能读到、能改。 +- **按 ref 定位只用公开 API（`boxes=True` 加鼠标坐标）。** 元素在视口外时会点错。 +- **tool 结果的图片在 worker 里就拆成 user 消息。** 会移动 `within_turn` 的 passback 边界；而且事件里存的上下文 + 和模型实际看到的不一致。 +- **所有截图都发给模型。** 上下文会无限增长。 +- **只按 pid 子树排除显示栈的进程。** Chromium 的一些辅助进程会脱离父进程、被重新挂到 pid 1 上，这样就排除不掉。 + +## Open questions + +（无） + +## Plan + +1. proto 和 sandboxd：`display`、`collect`、按显示 uid 排除进程。Go 单元测试，以及用真实显示镜像跑的集成测试。 +2. 显示镜像和 `swarm-display`。 +3. Python：消息和记录、两个工具、sandbox 客户端、gateway 协议和后端转换、事件格式 10、导出、fork 拒绝、case 加载。 +4. 文档：sandboxd、agent-runtime、case-format、event-log、model-gateway、architecture，以及 CHANGELOG。 + +退出条件：`mise run check` 通过；在 docker 里用显示镜像跑通 `computer` 截图和点击、`browser` 打开沙箱内的页面 +并点击 ref；`.eval` 里的截图能解析出来。 diff --git a/swarmeval/core/loader.py b/swarmeval/core/loader.py index d29dd57..25ba379 100644 --- a/swarmeval/core/loader.py +++ b/swarmeval/core/loader.py @@ -35,6 +35,7 @@ SandboxPlan, check_canaries, check_crossing, + check_displays, plan_sandboxes, seed_files, ) @@ -261,6 +262,7 @@ def _variant( env_where = f"{env_path}" + (f" (variant {values})" if values else "") sandboxes = plan_sandboxes(case, env, env_path) check_canaries(env, sandboxes, env_where) + check_displays(case, env, sandboxes, where) seeds = seed_files(env, sandboxes, files.tree, env_where) scripts: dict[str, str] = {} for scorer in case.scorers: diff --git a/swarmeval/core/models.py b/swarmeval/core/models.py index 30f5938..5ec6a9e 100644 --- a/swarmeval/core/models.py +++ b/swarmeval/core/models.py @@ -24,6 +24,7 @@ ) from swarmeval.detect.defs import DetectorDef +from swarmeval.runtime.display import DISPLAY_STATE, DISPLAY_TOOLS from swarmeval.runtime.extensions import ExtensionUse from swarmeval.runtime.tools import BUILTIN_TOOLS, SandboxTool @@ -41,7 +42,8 @@ DEFAULT_SLOT = "default" """The model slot of an agent that names none.""" -ENV_SCHEMA_VERSIONS = frozenset({1}) +ENV_SCHEMA_VERSIONS = frozenset({1, 2}) +"""`env.yaml` versions read. Version 2 adds a profile's `display`.""" Name = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_]*$", max_length=63)] """Ids of cases, agents, channels, sandboxes, profiles, and variant axes. They end up in @@ -336,7 +338,9 @@ def has_sandbox(self, agent: AgentDef) -> bool: @model_validator(mode="after") def _sandboxless_agents_run_nothing(self) -> Self: - sandbox_tools = {t.name for t in BUILTIN_TOOLS if isinstance(t, SandboxTool)} + sandbox_tools = { + t.name for t in (*BUILTIN_TOOLS, *DISPLAY_TOOLS) if isinstance(t, SandboxTool) + } for agent in self.swarm.agents: if self.has_sandbox(agent): continue @@ -398,17 +402,47 @@ class FileCopy(Strict): """Where it lands. A directory's contents go under it.""" +class Display(Strict): + """A virtual screen with a browser on it (docs/case-format.md#display). Needs an image built + from `swarmeval/display`.""" + + width: Annotated[int, Field(ge=320, le=1920)] = 1024 + height: Annotated[int, Field(ge=240, le=1200)] = 768 + url: Annotated[str, Field(min_length=1, max_length=2048, pattern=r"^[^\x00\r\n]*$")] | None = ( + None + ) + """Page the browser opens first; `about:blank` when unset.""" + + class SandboxProfile(Strict): image: Annotated[str, Field(min_length=1)] fs: tuple[Mount, ...] = () limits: ResourceLimits = ResourceLimits() files: tuple[FileCopy, ...] = () + display: Display | None = None @model_validator(mode="after") def _unique_mounts(self) -> Self: _no_duplicates("mount path", [m.path for m in self.fs]) return self + @model_validator(mode="after") + def _display_state_outside_key_paths(self) -> Self: + if self.display is None: + return self + for m in self.fs: + p = m.path + if ( + p == "/" + or DISPLAY_STATE.startswith(p + "/") + or (p + "/").startswith(DISPLAY_STATE + "/") + ): + raise ValueError( + f"key path `{p}` overlaps `{DISPLAY_STATE}`, where the display keeps its " + "state out of the agent's reach. Mount a path outside it." + ) + return self + class SandboxInstance(Strict): profile: Name @@ -444,6 +478,17 @@ class EnvFile(Strict): version 5, one with `sandbox: none` gets none.""" canaries: tuple[CanaryDef, ...] = () + @model_validator(mode="after") + def _display_needs_version_2(self) -> Self: + if self.schema_version < 2: + for name, profile in self.sandbox_profiles.items(): + if profile.display is not None: + raise ValueError( + f"profile `{name}` has `display`, which `schema_version: 2` added. Set " + "`schema_version: 2`." + ) + return self + @model_validator(mode="after") def _unique_canaries(self) -> Self: _no_duplicates("canary", [c.id for c in self.canaries]) diff --git a/swarmeval/core/topology.py b/swarmeval/core/topology.py index 93b7787..c70518c 100644 --- a/swarmeval/core/topology.py +++ b/swarmeval/core/topology.py @@ -10,6 +10,7 @@ from swarmeval.core.errors import CaseError from swarmeval.core.models import NO_SANDBOX, NO_SANDBOX_VERSION, CaseFile, EnvFile +from swarmeval.runtime.display import DISPLAY_TOOL_NAMES Tree = Callable[[str, str], list[tuple[str, Path]]] """A case path and the field naming it → each file under it, relative to it, and its path.""" @@ -52,6 +53,26 @@ def check_crossing( ) +def check_displays( + case: CaseFile, env: EnvFile, sandboxes: Mapping[str, SandboxPlan], where: str +) -> None: + """`browser` and `computer` drive a sandbox's display, so the agent's profile needs one. An + agent with no sandbox cannot list them (`CaseFile`).""" + tools = {a.id: a.tools for a in case.swarm.agents} + for plan in sandboxes.values(): + if env.sandbox_profiles[plan.profile].display is not None: + continue + for agent_id in plan.agents: + wanted = [t for t in tools[agent_id] if t in DISPLAY_TOOL_NAMES] + if wanted: + raise CaseError( + f"{where}: agent `{agent_id}` lists {', '.join(f'`{t}`' for t in wanted)}, " + f"but its sandbox `{plan.id}` has profile `{plan.profile}`, which has no " + "`display`. Add `display: {}` to the profile and use an image built from " + "swarmeval/display." + ) + + def check_canaries(env: EnvFile, sandboxes: Mapping[str, SandboxPlan], where: str) -> None: for canary in env.canaries: plan = sandboxes.get(canary.sandbox) diff --git a/swarmeval/events/__init__.py b/swarmeval/events/__init__.py index 9dc123b..841fd09 100644 --- a/swarmeval/events/__init__.py +++ b/swarmeval/events/__init__.py @@ -10,6 +10,7 @@ RunHeader, StoredRun, assemble, + blob_key, export_key, export_run, load_run, @@ -54,6 +55,7 @@ "StoredRun", "VariantLogError", "assemble", + "blob_key", "events_key", "export_events", "export_key", diff --git a/swarmeval/events/convert.py b/swarmeval/events/convert.py index 972bdfe..cba6f89 100644 --- a/swarmeval/events/convert.py +++ b/swarmeval/events/convert.py @@ -8,6 +8,7 @@ import json import math import shlex +from collections.abc import Sequence from dataclasses import dataclass from typing import Any, Literal @@ -27,6 +28,7 @@ ChatMessageTool, ChatMessageUser, Content, + ContentImage, ContentReasoning, ContentText, GenerateConfig, @@ -42,12 +44,15 @@ from pydantic import JsonValue from swarmeval.runtime.messages import ( + IMAGE_OMITTED, AssistantMessage, ChatMessage, + ImageRef, SystemMessage, ToolCall, ToolMessage, UserMessage, + visible_images, ) from swarmeval.runtime.records import ( Exec, @@ -61,9 +66,14 @@ ToolCallRecord, ) -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 """Version of the `metadata.swarmeval` extension. Bump it when any field below changes shape. +10: tool events may carry `images` (`ImageRef`s: a blob hash, media type, and size) from the +`computer` and `browser` tools; model events carry `max_images`, how many of the context's +latest images the request sent; `exec` observations carry `collected`, the files sandboxd read +out of the sandbox after the call. Version 9 runs have none of these and read as before. + 9: a fork's `intervention` (`hook: fork`) may be `replace_model`, whose `after` names the model slot and the models before and after; such a fork's agents of that slot run the new model. A run's `task_args` name its models as `model.`. Version 8 runs have neither and read as @@ -245,6 +255,7 @@ def _model_event(record: ModelCallRecord) -> tuple[ModelEvent, dict[str, JsonVal # What the model produced, byte for byte: Inspect's tool calls hold parsed arguments. "raw_tool_arguments": {c.id: c.arguments for c in record.response.tool_calls}, "gateway": record.gateway.model_dump(mode="json"), + "max_images": options.max_images, } return event, extra @@ -273,6 +284,8 @@ def _tool_event(record: ToolCallRecord) -> tuple[ToolEvent, dict[str, JsonValue] extra["exec"] = _exec_observations(record.exec_result) if record.web is not None: extra["web"] = record.web.model_dump(mode="json", exclude={"body"}) + if result.images: + extra["images"] = [i.model_dump(mode="json") for i in result.images] return event, extra @@ -362,7 +375,24 @@ def from_inspect_assistant( ) -def to_inspect_message(message: ChatMessage) -> InspectMessage: +ATTACHMENT = "attachment://" +"""Inspect's reference into `EvalSample.attachments`. Images are keyed by their sha256, and the +export puts each one's bytes there once.""" + + +def image_content( + images: Sequence[ImageRef], shown: Sequence[bool] +) -> list[ContentText | ContentImage]: + """Images as Inspect content; one not `shown` to the model is the text it saw instead.""" + return [ + ContentImage(image=f"{ATTACHMENT}{i.sha256}") if show else ContentText(text=IMAGE_OMITTED) + for i, show in zip(images, shown, strict=True) + ] + + +def to_inspect_message(message: ChatMessage, shown: Sequence[bool] = ()) -> InspectMessage: + """`shown` says, per image of a tool message, whether the request carried it; images past + its end were carried.""" match message: case SystemMessage(): return ChatMessageSystem(content=message.content) @@ -372,6 +402,28 @@ def to_inspect_message(message: ChatMessage) -> InspectMessage: return to_inspect_assistant(message) case ToolMessage(): error = ToolCallError("unknown", message.content) if message.is_error else None - return ChatMessageTool( - content=message.content, tool_call_id=message.tool_call_id, error=error - ) + content: str | list[Content] = message.content + if message.images: + visible = [*shown, *([True] * (len(message.images) - len(shown)))] + content = [ + ContentText(text=message.content), + *image_content(message.images, visible), + ] + return ChatMessageTool(content=content, tool_call_id=message.tool_call_id, error=error) + + +def to_inspect_context( + messages: Sequence[ChatMessage], max_images: int | None +) -> list[InspectMessage]: + """A request's context as the model saw it: with `max_images`, only the latest images; `None` + (events before schema 10) shows every image.""" + if max_images is None: + return [to_inspect_message(m) for m in messages] + keep = visible_images(messages, max_images) + return [ + to_inspect_message( + m, + [(i, j) in keep for j in range(len(m.images))] if isinstance(m, ToolMessage) else (), + ) + for i, m in enumerate(messages) + ] diff --git a/swarmeval/events/export.py b/swarmeval/events/export.py index cbab283..bed7e74 100644 --- a/swarmeval/events/export.py +++ b/swarmeval/events/export.py @@ -5,14 +5,17 @@ """ import asyncio +import base64 import copy +import dataclasses +import hashlib import tempfile from collections import defaultdict -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from importlib.metadata import version from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, cast from inspect_ai.event import ( Event, @@ -22,6 +25,7 @@ ScoreEvent, SpanBeginEvent, SpanEndEvent, + ToolEvent, ) from inspect_ai.log import ( EvalConfig, @@ -36,7 +40,7 @@ EvalSpec, write_eval_log, ) -from inspect_ai.model import ModelUsage +from inspect_ai.model import ContentImage, ContentText, ModelUsage from inspect_ai.scorer import Score from pyarrow import NativeFile from pyarrow.fs import S3FileSystem @@ -46,13 +50,14 @@ from swarmeval.db import control_runs, events, messages, run_specs from swarmeval.events.chain import ChainRow, ChainStart, verify -from swarmeval.events.convert import SCHEMA_VERSION, to_inspect_message -from swarmeval.runtime.messages import ChatMessage +from swarmeval.events.convert import SCHEMA_VERSION, to_inspect_context +from swarmeval.runtime.messages import ChatMessage, ImageRef, ToolMessage from swarmeval.runtime.records import LifecycleRecord _EVENT = TypeAdapter[Event](Event) _MESSAGE = TypeAdapter[ChatMessage](ChatMessage) _LIFECYCLE = TypeAdapter[LifecycleRecord](LifecycleRecord) +_IMAGES = TypeAdapter[list[ImageRef]](list[ImageRef]) @dataclass(frozen=True) @@ -84,6 +89,8 @@ class StoredRun: forked_from: str | None = None chain_start: ChainStart | None = None """For a fork: where its chain links into its source's.""" + images: Mapping[str, bytes] = dataclasses.field(default_factory=dict[str, bytes]) + """Every image `image_hashes` names, by sha256. The export embeds them.""" @dataclass(frozen=True) @@ -128,6 +135,39 @@ def export_key(run_id: str) -> str: return f"runs/{run_id}/sample.eval" +def blob_key(sha256: str) -> str: + """Where the blob store keeps content with this hash (docs/event-log.md#large-objects).""" + return f"blobs/sha256/{sha256}" + + +def image_hashes(run: StoredRun) -> set[str]: + """Every image a run's contexts or tool events name.""" + hashes = { + i.sha256 + for context in run.messages.values() + for m in context + if isinstance(m, ToolMessage) + for i in m.images + } + for row in run.events: + ours = cast(dict[str, Any], row.payload)["metadata"]["swarmeval"] + hashes.update(i.sha256 for i in _IMAGES.validate_python(ours.get("images", []))) + return hashes + + +async def load_images(store: ObjectStore, hashes: Iterable[str]) -> dict[str, bytes]: + async def one(sha256: str) -> tuple[str, bytes]: + data = await asyncio.to_thread(store.get, blob_key(sha256)) + if hashlib.sha256(data).hexdigest() != sha256: + raise ValueError( + f"blob {blob_key(sha256)} does not hash to its name. The object store was " + "changed after upload; do not export from it." + ) + return sha256, data + + return dict(await asyncio.gather(*(one(h) for h in sorted(hashes)))) + + async def load_run(engine: AsyncEngine, run_id: str) -> StoredRun: async with engine.connect() as conn: workspace = ( @@ -185,6 +225,12 @@ def assemble(header: RunHeader, run: StoredRun) -> EvalLog: "after the run has started." ) verify(header.run_id, run.events, run.chain_start) + missing = image_hashes(run) - run.images.keys() + if missing: + raise ValueError( + f"run {header.run_id} names {len(missing)} image(s) the export was not given, such " + f"as {min(missing)}. Load them with `load_images`." + ) stored = [_restore(row, run) for row in run.events] sample_events = _with_agent_spans(stored, tuple(header.models)) first, last = stored[0].timestamp, stored[-1].timestamp @@ -204,6 +250,7 @@ def assemble(header: RunHeader, run: StoredRun) -> EvalLog: error=error, limit=_limit(stored), scores=scores or None, + attachments={sha256: _data_url(data) for sha256, data in run.images.items()}, metadata={ "swarmeval": { "schema_version": SCHEMA_VERSION, @@ -265,7 +312,9 @@ def write_eval(log: EvalLog, path: Path) -> None: async def export_run(engine: AsyncEngine, header: RunHeader, store: ObjectStore) -> str: """Writes the run's `.eval` to `store` and returns its key within the bucket.""" - log = assemble(header, await load_run(engine, header.run_id)) + run = await load_run(engine, header.run_id) + run = dataclasses.replace(run, images=await load_images(store, image_hashes(run))) + log = assemble(header, run) key = export_key(header.run_id) await asyncio.to_thread(_upload, log, store, key) return key @@ -289,10 +338,21 @@ def _restore(row: ChainRow, run: StoredRun) -> Event: ref = ours["input"] if ref["gen"] is not None: context = run.messages[(ours["agent_id"], ref["gen"])][: ref["len"]] - event.input = [to_inspect_message(m) for m in context] + event.input = to_inspect_context(context, ours.get("max_images")) + if isinstance(event, ToolEvent) and ours.get("images"): + # Inline, as Inspect writes tool results: it resolves `attachment://` in messages only. + images = _IMAGES.validate_python(ours["images"]) + event.result = [ + ContentText(text=str(event.result)), + *(ContentImage(image=_data_url(run.images[i.sha256])) for i in images), + ] return event +def _data_url(png: bytes) -> str: + return f"data:image/png;base64,{base64.b64encode(png).decode()}" + + def _agent_of(event: Event) -> str | None: assert event.metadata is not None, "every stored event carries metadata.swarmeval" return event.metadata["swarmeval"]["agent_id"] diff --git a/swarmeval/events/fork.py b/swarmeval/events/fork.py index 4a97436..9f8ac83 100644 --- a/swarmeval/events/fork.py +++ b/swarmeval/events/fork.py @@ -16,6 +16,7 @@ from swarmeval.db import canaries, checkpoints, events, messages, run_specs from swarmeval.events.seal import ChainHead +from swarmeval.runtime.display import DISPLAY_TOOL_NAMES from swarmeval.runtime.extensions import CanaryInfo, SandboxCanaryInfo from swarmeval.runtime.messages import ChatMessage from swarmeval.runtime.records import Checkpoint @@ -74,6 +75,29 @@ async def point_at(engine: AsyncEngine, run_id: str, seq: int, what: str = "") - "checkpoints, or the run was recorded before forks were possible. Fork at a " "later event of a `round_robin` or `event_driven` run." ) + display = ( + await conn.execute( + select(events.c.event_id, events.c.payload["function"].astext) + .where( + events.c.run_id == run_id, + events.c.seq <= row.seq, + events.c.type == "tool", + events.c.payload["function"].astext.in_(DISPLAY_TOOL_NAMES), + events.c.payload["metadata"]["swarmeval"]["executed_arguments"].astext.is_not( + None + ), + ) + .order_by(events.c.seq) + .limit(1) + ) + ).one_or_none() + if display is not None: + raise ForkPointError( + f"run `{run_id}` used `{display[1]}` (event `{display[0]}`) before the turn " + f"holding {what or f'seq {seq + 1}'}. A fork restores files but not a browser or " + "a screen, so it cannot go on from there. Fork at an event before the first " + "`browser` or `computer` call." + ) checkpoint = Checkpoint.model_validate(row.state) if checkpoint.spawned: raise ForkPointError( diff --git a/swarmeval/events/transcript.py b/swarmeval/events/transcript.py index bc61b7e..fa204a7 100644 --- a/swarmeval/events/transcript.py +++ b/swarmeval/events/transcript.py @@ -19,7 +19,7 @@ from swarmeval.db import events, messages from swarmeval.events.convert import from_inspect_assistant from swarmeval.events.fork import lineage -from swarmeval.runtime.messages import AssistantMessage, ChatMessage, RequestOptions +from swarmeval.runtime.messages import AssistantMessage, ChatMessage, ImageRef, RequestOptions from swarmeval.runtime.records import ( GatewayRecord, InterventionRecord, @@ -31,6 +31,7 @@ _EVENT = TypeAdapter[Event](Event) _TYPES = ("model", "tool", "swarmeval.intervention", "swarmeval.msg.send", "swarmeval.msg.deliver") _MESSAGE = TypeAdapter[ChatMessage](ChatMessage) +_IMAGES = TypeAdapter[list[ImageRef]](list[ImageRef]) @dataclass(frozen=True) @@ -123,14 +124,16 @@ async def load_transcript(engine: AsyncEngine, run_id: str) -> RunTranscript: case ModelEvent() as event: model_calls.append(_model_call(event_id, agent_id, event)) case ToolEvent() as event: + assert event.metadata is not None, "stored events carry metadata.swarmeval" + ours = event.metadata["swarmeval"] result = ToolResult( call_id=event.id, tool=event.function, content=str(event.result), is_error=bool(event.failed), + images=tuple(_IMAGES.validate_python(ours.get("images", []))), ) - assert event.metadata is not None, "stored events carry metadata.swarmeval" - executed = event.metadata["swarmeval"]["executed_arguments"] + executed = ours["executed_arguments"] tool_results.append( ToolOutcome(event_id, agent_id, parent_id, result, executed) ) @@ -179,6 +182,7 @@ def _model_call(event_id: str, agent_id: str | None, event: ModelEvent) -> Model top_p=config.top_p, max_output_tokens=config.max_tokens, seed=config.seed, + **({"max_images": ours["max_images"]} if "max_images" in ours else {}), ), response=from_inspect_assistant( event.output.choices[0].message, ours["raw_tool_arguments"] diff --git a/swarmeval/gateway/model/client.py b/swarmeval/gateway/model/client.py index f225813..1bbdce6 100644 --- a/swarmeval/gateway/model/client.py +++ b/swarmeval/gateway/model/client.py @@ -11,13 +11,14 @@ """ import asyncio +import base64 import hashlib import json import secrets -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from types import TracebackType -from typing import Literal, Self +from typing import Literal, Protocol, Self import grpc import httpx2 @@ -30,8 +31,12 @@ WireAssistantMessage, WireFunction, WireFunctionCall, + WireImagePart, + WireImageURL, WireMessage, + WirePart, WireSystemMessage, + WireTextPart, WireTool, WireToolCall, WireToolMessage, @@ -40,6 +45,7 @@ from swarmeval.proto.swarmeval.modelgw.v1 import recorder_pb2 as pb from swarmeval.proto.swarmeval.modelgw.v1.recorder_pb2_grpc import RecorderServiceStub from swarmeval.runtime.messages import ( + IMAGE_OMITTED, AssistantMessage, ChatMessage, ModelRequest, @@ -49,6 +55,7 @@ ToolMessage, Usage, UserMessage, + visible_images, ) from swarmeval.runtime.ports import AgentCaller, Caller, ExtensionCaller, RecordedResponse from swarmeval.runtime.records import ( @@ -65,6 +72,12 @@ """model-gateway's status for a call the model backend failed after retries.""" +class ImageSource(Protocol): + async def get(self, sha256: str) -> bytes: + """The image stored under `sha256`, checked against it.""" + ... + + class ModelGatewayError(Exception): """model-gateway failed a call, or broke the recording protocol. `status` is the HTTP status when the gateway answered with an error; 503 means the run is not attached.""" @@ -106,9 +119,11 @@ def __init__( owner_epoch: int, callers: Sequence[Caller], writer: RunWriter, + images: ImageSource, ) -> None: """`http` is based at the gateway's HTTP address and should have no read timeout: a - call lasts as long as the model takes.""" + call lasts as long as the model takes. `images` holds the images tools returned, which + requests carry inline.""" self._http = http self._stub = RecorderServiceStub(channel) self._run_id = run_id @@ -120,6 +135,9 @@ def __init__( self._stream: grpc.aio.StreamStreamCall[pb.AttachRequest, pb.AttachResponse] | None = None self._reader: asyncio.Task[None] | None = None self._broken: BaseException | None = None + self._images = images + self._sent_images: dict[str, bytes] = {} + """The last request's images. The next one mostly carries the same ones.""" async def __aenter__(self) -> Self: stream = self._stub.Attach() @@ -167,7 +185,8 @@ async def generate( name = caller_name(caller) self._calls += 1 call_id = f"call_{self._calls}" - body = to_wire(request).model_dump_json(exclude_none=True).encode() + wire = to_wire(request, await self._fetch_images(request)) + body = wire.model_dump_json(exclude_none=True).encode() pending = _Pending(caller, request, hashlib.sha256(body).hexdigest(), parent_id) self._pending[call_id] = pending headers = { @@ -200,6 +219,19 @@ async def generate( ) return RecordedResponse(response=response, event=event) + async def _fetch_images(self, request: ModelRequest) -> dict[str, bytes]: + wanted = { + m.images[j].sha256 + for i, j in visible_images(request.messages, request.options.max_images) + if isinstance(m := request.messages[i], ToolMessage) + } + fetched = { + sha256: self._sent_images.get(sha256) or await self._images.get(sha256) + for sha256 in sorted(wanted) + } + self._sent_images = fetched + return fetched + async def _read( self, stream: grpc.aio.StreamStreamCall[pb.AttachRequest, pb.AttachResponse] ) -> None: @@ -345,11 +377,36 @@ def _wire_message(message: ChatMessage) -> WireMessage: return WireToolMessage(tool_call_id=tool_call_id, content=content) -def to_wire(request: ModelRequest) -> ChatRequest: +def _with_images( + message: WireToolMessage, tool: ToolMessage, shown: Sequence[bool], images: Mapping[str, bytes] +) -> WireToolMessage: + parts: list[WirePart] = [WireTextPart(text=tool.content)] if tool.content else [] + for image, show in zip(tool.images, shown, strict=True): + if show: + data = base64.b64encode(images[image.sha256]).decode() + url = WireImageURL(url=f"data:{image.media_type};base64,{data}") + parts.append(WireImagePart(image_url=url)) + else: + parts.append(WireTextPart(text=IMAGE_OMITTED)) + return message.model_copy(update={"content": parts}) + + +def to_wire(request: ModelRequest, images: Mapping[str, bytes] | None = None) -> ChatRequest: + """`images` holds the bytes of every image the request shows, by sha256: the context's last + `options.max_images`; earlier ones become `IMAGE_OMITTED`.""" options = request.options + keep = visible_images(request.messages, options.max_images) + messages: list[WireMessage] = [] + for i, m in enumerate(request.messages): + wire = _wire_message(m) + if isinstance(m, ToolMessage) and m.images: + assert isinstance(wire, WireToolMessage) + shown = [(i, j) in keep for j in range(len(m.images))] + wire = _with_images(wire, m, shown, {} if images is None else images) + messages.append(wire) return ChatRequest( model=request.model, - messages=[_wire_message(m) for m in request.messages], + messages=messages, tools=[ WireTool( function=WireFunction( diff --git a/swarmeval/gateway/model/upstream.py b/swarmeval/gateway/model/upstream.py index 843a0da..2b1603d 100644 --- a/swarmeval/gateway/model/upstream.py +++ b/swarmeval/gateway/model/upstream.py @@ -7,7 +7,7 @@ import time from collections.abc import Mapping from dataclasses import dataclass -from typing import cast +from typing import Any, cast import httpx2 from openai import APIConnectionError, APIStatusError, AsyncOpenAI, omit @@ -80,6 +80,34 @@ def passback(messages: list[WireMessage], mode: ReasoningPassback) -> list[WireM return kept +TOOL_IMAGES_LABEL = "Images from tool call {call_id}:" + + +def move_tool_images(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Tool messages with text only, as Chat Completions requires, each run of them followed by a + user message holding their images, each image labelled with its tool call. Runs after + `passback`, so the added user messages do not move its turn boundary.""" + out: list[dict[str, Any]] = [] + images: list[dict[str, Any]] = [] + for message in messages: + if message["role"] != "tool" and images: + out.append({"role": "user", "content": images}) + images = [] + content = message.get("content") + if message["role"] == "tool" and isinstance(content, list): + parts = cast(list[dict[str, Any]], content) + texts = [p["text"] for p in parts if p["type"] == "text"] + pictures = [p for p in parts if p["type"] == "image_url"] + message = {**message, "content": "\n".join(texts)} + if pictures: + label = TOOL_IMAGES_LABEL.format(call_id=message["tool_call_id"]) + images += [{"type": "text", "text": label}, *pictures] + out.append(message) + if images: + out.append({"role": "user", "content": images}) + return out + + class Upstreams: """One SDK client per backend, created on first use.""" @@ -139,10 +167,12 @@ async def complete(self, request: ChatRequest) -> UpstreamResult: }.items() if v is not None } - messages = [ - m.model_dump(exclude_none=True) - for m in passback(request.messages, backend.reasoning_passback) - ] + messages = move_tool_images( + [ + m.model_dump(exclude_none=True) + for m in passback(request.messages, backend.reasoning_passback) + ] + ) tools = [t.model_dump() for t in request.tools or ()] counter = [0] token = _attempts.set(counter) diff --git a/swarmeval/gateway/model/wire.py b/swarmeval/gateway/model/wire.py index f1191f5..9518fa5 100644 --- a/swarmeval/gateway/model/wire.py +++ b/swarmeval/gateway/model/wire.py @@ -41,10 +41,31 @@ class WireAssistantMessage(Wire): tool_calls: list[WireToolCall] | None = None +class WireTextPart(Wire): + type: Literal["text"] = "text" + text: str + + +class WireImageURL(Wire): + url: Annotated[str, Field(pattern=r"^data:image/png;base64,[A-Za-z0-9+/]+={0,2}$")] + """A PNG inline as a `data:` URL; the gateway fetches nothing.""" + + +class WireImagePart(Wire): + type: Literal["image_url"] = "image_url" + image_url: WireImageURL + + +WirePart = Annotated[WireTextPart | WireImagePart, Field(discriminator="type")] + + class WireToolMessage(Wire): role: Literal["tool"] = "tool" tool_call_id: str - content: str + content: str | list[WirePart] + """Parts carry the images a tool returned. Chat Completions takes only text in a tool + message, so the gateway moves the images into a user message for the backend + (`upstream.move_tool_images`).""" WireMessage = Annotated[ diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py index aefd799..571b4f8 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py @@ -25,7 +25,7 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2swarmeval/proto/swarmeval/sandbox/v1/sandbox.proto\x12\x14swarmeval.sandbox.v1\x1a\x1egoogle/protobuf/duration.proto\"7\n\x10\x43reateRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x13\n\x0bsandbox_ids\x18\x02 \x03(\t\"\x13\n\x11\x43reateRunResponse\";\n\x05Mount\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x11\n\tread_only\x18\x02 \x01(\x08\x12\x11\n\tprotected\x18\x03 \x01(\x08\"Q\n\tResources\x12\x0c\n\x04\x63pus\x18\x01 \x01(\x01\x12\x14\n\x0cmemory_bytes\x18\x02 \x01(\x03\x12\x0c\n\x04pids\x18\x03 \x01(\x03\x12\x12\n\ndisk_bytes\x18\x04 \x01(\x03\"\xfc\x02\n\x14\x43reateSandboxRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12+\n\x06mounts\x18\x04 \x03(\x0b\x32\x1b.swarmeval.sandbox.v1.Mount\x12\x32\n\tresources\x18\x05 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.Resources\x12-\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.SeedFile\x12\r\n\x05users\x18\x07 \x03(\t\x12@\n\x03\x65nv\x18\x08 \x03(\x0b\x32\x33.swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry\x12\x10\n\x08hostname\x18\t \x01(\t\x12\x12\n\nmachine_id\x18\n \x01(\t\x1a*\n\x08\x45nvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"7\n\x08SeedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\"(\n\x15\x43reateSandboxResponse\x12\x0f\n\x07runtime\x18\x01 \x01(\t\"\x97\x01\n\x0b\x45xecRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ll_id\x18\x03 \x01(\t\x12\x0c\n\x04\x61rgv\x18\x04 \x03(\t\x12\x0b\n\x03\x63wd\x18\x05 \x01(\t\x12\x0c\n\x04user\x18\x06 \x01(\t\x12*\n\x07timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\"{\n\x0c\x45xecResponse\x12\x32\n\x06header\x18\x01 \x01(\x0b\x32 .swarmeval.sandbox.v1.ExecHeaderH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"\xda\x02\n\nExecHeader\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x11\n\ttimed_out\x18\x02 \x01(\x08\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x06stdout\x18\x04 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12,\n\x06stderr\x18\x05 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12:\n\x12\x62\x61\x63kground_changes\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12/\n\x07\x63hanges\x18\x07 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12\x30\n\tprocesses\x18\x08 \x03(\x0b\x32\x1d.swarmeval.sandbox.v1.Process\"K\n\x06Output\x12\x0e\n\x06inline\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x13\n\x0b\x62lob_sha256\x18\x03 \x01(\t\x12\x0e\n\x06\x63\x61pped\x18\x04 \x01(\x08\"\xe1\x04\n\x08\x46sChange\x12\x0c\n\x04path\x18\x01 \x01(\t\x12-\n\x02op\x18\x02 \x01(\x0e\x32!.swarmeval.sandbox.v1.FsChange.Op\x12\x31\n\x04kind\x18\x03 \x01(\x0e\x32#.swarmeval.sandbox.v1.FsChange.Kind\x12\x0b\n\x03uid\x18\x04 \x01(\r\x12\x0c\n\x04mode\x18\x05 \x01(\r\x12\x0c\n\x04size\x18\x06 \x01(\x03\x12\x15\n\rbefore_sha256\x18\x07 \x01(\t\x12\x14\n\x0c\x61\x66ter_sha256\x18\x08 \x01(\t\x12\x11\n\tprotected\x18\t \x01(\x08\x12?\n\x0b\x61ttribution\x18\n \x01(\x0e\x32*.swarmeval.sandbox.v1.FsChange.Attribution\x12\x17\n\x0f\x63\x61ndidate_calls\x18\x0b \x03(\t\x12\x0f\n\x07\x63ontent\x18\x0c \x01(\x08\x12\x10\n\x08mtime_ns\x18\r \x01(\x03\"E\n\x02Op\x12\x12\n\x0eOP_UNSPECIFIED\x10\x00\x12\r\n\tOP_CREATE\x10\x01\x12\r\n\tOP_MODIFY\x10\x02\x12\r\n\tOP_DELETE\x10\x03\"[\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_FILE\x10\x01\x12\x0c\n\x08KIND_DIR\x10\x02\x12\x10\n\x0cKIND_SYMLINK\x10\x03\x12\x0e\n\nKIND_OTHER\x10\x04\"[\n\x0b\x41ttribution\x12\x1b\n\x17\x41TTRIBUTION_UNSPECIFIED\x10\x00\x12\x14\n\x10\x41TTRIBUTION_CALL\x10\x01\x12\x19\n\x15\x41TTRIBUTION_AMBIGUOUS\x10\x02\"C\n\x07Process\x12\x0b\n\x03pid\x18\x01 \x01(\x05\x12\x0c\n\x04ppid\x18\x02 \x01(\x05\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0f\n\x07\x63mdline\x18\x04 \x01(\t\"7\n\tBlobChunk\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04last\x18\x03 \x01(\x08\"\xab\x01\n\x13RestoreFilesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0e\n\x06remove\x18\x03 \x03(\t\x12.\n\x04\x64irs\x18\x04 \x03(\x0b\x32 .swarmeval.sandbox.v1.RestoreDir\x12\x30\n\x05\x66iles\x18\x05 \x03(\x0b\x32!.swarmeval.sandbox.v1.RestoreFile\"5\n\nRestoreDir\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04mode\x18\x02 \x01(\r\x12\x0b\n\x03uid\x18\x03 \x01(\r\"G\n\x0bRestoreFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\x12\x0b\n\x03uid\x18\x04 \x01(\r\"\'\n\x14RestoreFilesResponse\x12\x0f\n\x07unowned\x18\x01 \x03(\t\"V\n\x0fReadFileRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x11\n\tmax_bytes\x18\x04 \x01(\x03\"D\n\x10ReadFileResponse\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x11\n\ttruncated\x18\x03 \x01(\x08\"\"\n\x10\x46inalDiffRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x85\x01\n\x11\x46inalDiffResponse\x12\x37\n\x07\x63hanges\x18\x01 \x01(\x0b\x32$.swarmeval.sandbox.v1.SandboxChangesH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"U\n\x0eSandboxChanges\x12\x12\n\nsandbox_id\x18\x01 \x01(\t\x12/\n\x07\x63hanges\x18\x02 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\"#\n\x11\x44\x65stroyRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x14\n\x12\x44\x65stroyRunResponse2\xac\x05\n\x0eSandboxService\x12\\\n\tCreateRun\x12&.swarmeval.sandbox.v1.CreateRunRequest\x1a\'.swarmeval.sandbox.v1.CreateRunResponse\x12h\n\rCreateSandbox\x12*.swarmeval.sandbox.v1.CreateSandboxRequest\x1a+.swarmeval.sandbox.v1.CreateSandboxResponse\x12O\n\x04\x45xec\x12!.swarmeval.sandbox.v1.ExecRequest\x1a\".swarmeval.sandbox.v1.ExecResponse0\x01\x12Y\n\x08ReadFile\x12%.swarmeval.sandbox.v1.ReadFileRequest\x1a&.swarmeval.sandbox.v1.ReadFileResponse\x12\x65\n\x0cRestoreFiles\x12).swarmeval.sandbox.v1.RestoreFilesRequest\x1a*.swarmeval.sandbox.v1.RestoreFilesResponse\x12^\n\tFinalDiff\x12&.swarmeval.sandbox.v1.FinalDiffRequest\x1a\'.swarmeval.sandbox.v1.FinalDiffResponse0\x01\x12_\n\nDestroyRun\x12\'.swarmeval.sandbox.v1.DestroyRunRequest\x1a(.swarmeval.sandbox.v1.DestroyRunResponseBJZHgithub.com/jackeydou/DUNE/go/internal/gen/swarmeval/sandbox/v1;sandboxv1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2swarmeval/proto/swarmeval/sandbox/v1/sandbox.proto\x12\x14swarmeval.sandbox.v1\x1a\x1egoogle/protobuf/duration.proto\"7\n\x10\x43reateRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x13\n\x0bsandbox_ids\x18\x02 \x03(\t\"\x13\n\x11\x43reateRunResponse\";\n\x05Mount\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x11\n\tread_only\x18\x02 \x01(\x08\x12\x11\n\tprotected\x18\x03 \x01(\x08\"Q\n\tResources\x12\x0c\n\x04\x63pus\x18\x01 \x01(\x01\x12\x14\n\x0cmemory_bytes\x18\x02 \x01(\x03\x12\x0c\n\x04pids\x18\x03 \x01(\x03\x12\x12\n\ndisk_bytes\x18\x04 \x01(\x03\"\xac\x03\n\x14\x43reateSandboxRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12+\n\x06mounts\x18\x04 \x03(\x0b\x32\x1b.swarmeval.sandbox.v1.Mount\x12\x32\n\tresources\x18\x05 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.Resources\x12-\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.SeedFile\x12\r\n\x05users\x18\x07 \x03(\t\x12@\n\x03\x65nv\x18\x08 \x03(\x0b\x32\x33.swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry\x12\x10\n\x08hostname\x18\t \x01(\t\x12\x12\n\nmachine_id\x18\n \x01(\t\x12.\n\x07\x64isplay\x18\x0b \x01(\x0b\x32\x1d.swarmeval.sandbox.v1.Display\x1a*\n\x08\x45nvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"5\n\x07\x44isplay\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x0b\n\x03url\x18\x03 \x01(\t\"7\n\x08SeedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\"(\n\x15\x43reateSandboxResponse\x12\x0f\n\x07runtime\x18\x01 \x01(\t\"\xa8\x01\n\x0b\x45xecRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ll_id\x18\x03 \x01(\t\x12\x0c\n\x04\x61rgv\x18\x04 \x03(\t\x12\x0b\n\x03\x63wd\x18\x05 \x01(\t\x12\x0c\n\x04user\x18\x06 \x01(\t\x12*\n\x07timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x0f\n\x07\x63ollect\x18\x08 \x03(\t\"{\n\x0c\x45xecResponse\x12\x32\n\x06header\x18\x01 \x01(\x0b\x32 .swarmeval.sandbox.v1.ExecHeaderH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"\x92\x03\n\nExecHeader\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x11\n\ttimed_out\x18\x02 \x01(\x08\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x06stdout\x18\x04 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12,\n\x06stderr\x18\x05 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12:\n\x12\x62\x61\x63kground_changes\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12/\n\x07\x63hanges\x18\x07 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12\x30\n\tprocesses\x18\x08 \x03(\x0b\x32\x1d.swarmeval.sandbox.v1.Process\x12\x36\n\tcollected\x18\t \x03(\x0b\x32#.swarmeval.sandbox.v1.CollectedFile\"L\n\rCollectedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07missing\x18\x02 \x01(\x08\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0e\n\x06sha256\x18\x04 \x01(\t\"K\n\x06Output\x12\x0e\n\x06inline\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x13\n\x0b\x62lob_sha256\x18\x03 \x01(\t\x12\x0e\n\x06\x63\x61pped\x18\x04 \x01(\x08\"\xe1\x04\n\x08\x46sChange\x12\x0c\n\x04path\x18\x01 \x01(\t\x12-\n\x02op\x18\x02 \x01(\x0e\x32!.swarmeval.sandbox.v1.FsChange.Op\x12\x31\n\x04kind\x18\x03 \x01(\x0e\x32#.swarmeval.sandbox.v1.FsChange.Kind\x12\x0b\n\x03uid\x18\x04 \x01(\r\x12\x0c\n\x04mode\x18\x05 \x01(\r\x12\x0c\n\x04size\x18\x06 \x01(\x03\x12\x15\n\rbefore_sha256\x18\x07 \x01(\t\x12\x14\n\x0c\x61\x66ter_sha256\x18\x08 \x01(\t\x12\x11\n\tprotected\x18\t \x01(\x08\x12?\n\x0b\x61ttribution\x18\n \x01(\x0e\x32*.swarmeval.sandbox.v1.FsChange.Attribution\x12\x17\n\x0f\x63\x61ndidate_calls\x18\x0b \x03(\t\x12\x0f\n\x07\x63ontent\x18\x0c \x01(\x08\x12\x10\n\x08mtime_ns\x18\r \x01(\x03\"E\n\x02Op\x12\x12\n\x0eOP_UNSPECIFIED\x10\x00\x12\r\n\tOP_CREATE\x10\x01\x12\r\n\tOP_MODIFY\x10\x02\x12\r\n\tOP_DELETE\x10\x03\"[\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_FILE\x10\x01\x12\x0c\n\x08KIND_DIR\x10\x02\x12\x10\n\x0cKIND_SYMLINK\x10\x03\x12\x0e\n\nKIND_OTHER\x10\x04\"[\n\x0b\x41ttribution\x12\x1b\n\x17\x41TTRIBUTION_UNSPECIFIED\x10\x00\x12\x14\n\x10\x41TTRIBUTION_CALL\x10\x01\x12\x19\n\x15\x41TTRIBUTION_AMBIGUOUS\x10\x02\"C\n\x07Process\x12\x0b\n\x03pid\x18\x01 \x01(\x05\x12\x0c\n\x04ppid\x18\x02 \x01(\x05\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0f\n\x07\x63mdline\x18\x04 \x01(\t\"7\n\tBlobChunk\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04last\x18\x03 \x01(\x08\"\xab\x01\n\x13RestoreFilesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0e\n\x06remove\x18\x03 \x03(\t\x12.\n\x04\x64irs\x18\x04 \x03(\x0b\x32 .swarmeval.sandbox.v1.RestoreDir\x12\x30\n\x05\x66iles\x18\x05 \x03(\x0b\x32!.swarmeval.sandbox.v1.RestoreFile\"5\n\nRestoreDir\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04mode\x18\x02 \x01(\r\x12\x0b\n\x03uid\x18\x03 \x01(\r\"G\n\x0bRestoreFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\x12\x0b\n\x03uid\x18\x04 \x01(\r\"\'\n\x14RestoreFilesResponse\x12\x0f\n\x07unowned\x18\x01 \x03(\t\"V\n\x0fReadFileRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x11\n\tmax_bytes\x18\x04 \x01(\x03\"D\n\x10ReadFileResponse\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x11\n\ttruncated\x18\x03 \x01(\x08\"\"\n\x10\x46inalDiffRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x85\x01\n\x11\x46inalDiffResponse\x12\x37\n\x07\x63hanges\x18\x01 \x01(\x0b\x32$.swarmeval.sandbox.v1.SandboxChangesH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"U\n\x0eSandboxChanges\x12\x12\n\nsandbox_id\x18\x01 \x01(\t\x12/\n\x07\x63hanges\x18\x02 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\"#\n\x11\x44\x65stroyRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x14\n\x12\x44\x65stroyRunResponse2\xac\x05\n\x0eSandboxService\x12\\\n\tCreateRun\x12&.swarmeval.sandbox.v1.CreateRunRequest\x1a\'.swarmeval.sandbox.v1.CreateRunResponse\x12h\n\rCreateSandbox\x12*.swarmeval.sandbox.v1.CreateSandboxRequest\x1a+.swarmeval.sandbox.v1.CreateSandboxResponse\x12O\n\x04\x45xec\x12!.swarmeval.sandbox.v1.ExecRequest\x1a\".swarmeval.sandbox.v1.ExecResponse0\x01\x12Y\n\x08ReadFile\x12%.swarmeval.sandbox.v1.ReadFileRequest\x1a&.swarmeval.sandbox.v1.ReadFileResponse\x12\x65\n\x0cRestoreFiles\x12).swarmeval.sandbox.v1.RestoreFilesRequest\x1a*.swarmeval.sandbox.v1.RestoreFilesResponse\x12^\n\tFinalDiff\x12&.swarmeval.sandbox.v1.FinalDiffRequest\x1a\'.swarmeval.sandbox.v1.FinalDiffResponse0\x01\x12_\n\nDestroyRun\x12\'.swarmeval.sandbox.v1.DestroyRunRequest\x1a(.swarmeval.sandbox.v1.DestroyRunResponseBJZHgithub.com/jackeydou/DUNE/go/internal/gen/swarmeval/sandbox/v1;sandboxv1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -44,55 +44,59 @@ _globals['_RESOURCES']._serialized_start=247 _globals['_RESOURCES']._serialized_end=328 _globals['_CREATESANDBOXREQUEST']._serialized_start=331 - _globals['_CREATESANDBOXREQUEST']._serialized_end=711 - _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_start=669 - _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_end=711 - _globals['_SEEDFILE']._serialized_start=713 - _globals['_SEEDFILE']._serialized_end=768 - _globals['_CREATESANDBOXRESPONSE']._serialized_start=770 - _globals['_CREATESANDBOXRESPONSE']._serialized_end=810 - _globals['_EXECREQUEST']._serialized_start=813 - _globals['_EXECREQUEST']._serialized_end=964 - _globals['_EXECRESPONSE']._serialized_start=966 - _globals['_EXECRESPONSE']._serialized_end=1089 - _globals['_EXECHEADER']._serialized_start=1092 - _globals['_EXECHEADER']._serialized_end=1438 - _globals['_OUTPUT']._serialized_start=1440 - _globals['_OUTPUT']._serialized_end=1515 - _globals['_FSCHANGE']._serialized_start=1518 - _globals['_FSCHANGE']._serialized_end=2127 - _globals['_FSCHANGE_OP']._serialized_start=1872 - _globals['_FSCHANGE_OP']._serialized_end=1941 - _globals['_FSCHANGE_KIND']._serialized_start=1943 - _globals['_FSCHANGE_KIND']._serialized_end=2034 - _globals['_FSCHANGE_ATTRIBUTION']._serialized_start=2036 - _globals['_FSCHANGE_ATTRIBUTION']._serialized_end=2127 - _globals['_PROCESS']._serialized_start=2129 - _globals['_PROCESS']._serialized_end=2196 - _globals['_BLOBCHUNK']._serialized_start=2198 - _globals['_BLOBCHUNK']._serialized_end=2253 - _globals['_RESTOREFILESREQUEST']._serialized_start=2256 - _globals['_RESTOREFILESREQUEST']._serialized_end=2427 - _globals['_RESTOREDIR']._serialized_start=2429 - _globals['_RESTOREDIR']._serialized_end=2482 - _globals['_RESTOREFILE']._serialized_start=2484 - _globals['_RESTOREFILE']._serialized_end=2555 - _globals['_RESTOREFILESRESPONSE']._serialized_start=2557 - _globals['_RESTOREFILESRESPONSE']._serialized_end=2596 - _globals['_READFILEREQUEST']._serialized_start=2598 - _globals['_READFILEREQUEST']._serialized_end=2684 - _globals['_READFILERESPONSE']._serialized_start=2686 - _globals['_READFILERESPONSE']._serialized_end=2754 - _globals['_FINALDIFFREQUEST']._serialized_start=2756 - _globals['_FINALDIFFREQUEST']._serialized_end=2790 - _globals['_FINALDIFFRESPONSE']._serialized_start=2793 - _globals['_FINALDIFFRESPONSE']._serialized_end=2926 - _globals['_SANDBOXCHANGES']._serialized_start=2928 - _globals['_SANDBOXCHANGES']._serialized_end=3013 - _globals['_DESTROYRUNREQUEST']._serialized_start=3015 - _globals['_DESTROYRUNREQUEST']._serialized_end=3050 - _globals['_DESTROYRUNRESPONSE']._serialized_start=3052 - _globals['_DESTROYRUNRESPONSE']._serialized_end=3072 - _globals['_SANDBOXSERVICE']._serialized_start=3075 - _globals['_SANDBOXSERVICE']._serialized_end=3759 + _globals['_CREATESANDBOXREQUEST']._serialized_end=759 + _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_start=717 + _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_end=759 + _globals['_DISPLAY']._serialized_start=761 + _globals['_DISPLAY']._serialized_end=814 + _globals['_SEEDFILE']._serialized_start=816 + _globals['_SEEDFILE']._serialized_end=871 + _globals['_CREATESANDBOXRESPONSE']._serialized_start=873 + _globals['_CREATESANDBOXRESPONSE']._serialized_end=913 + _globals['_EXECREQUEST']._serialized_start=916 + _globals['_EXECREQUEST']._serialized_end=1084 + _globals['_EXECRESPONSE']._serialized_start=1086 + _globals['_EXECRESPONSE']._serialized_end=1209 + _globals['_EXECHEADER']._serialized_start=1212 + _globals['_EXECHEADER']._serialized_end=1614 + _globals['_COLLECTEDFILE']._serialized_start=1616 + _globals['_COLLECTEDFILE']._serialized_end=1692 + _globals['_OUTPUT']._serialized_start=1694 + _globals['_OUTPUT']._serialized_end=1769 + _globals['_FSCHANGE']._serialized_start=1772 + _globals['_FSCHANGE']._serialized_end=2381 + _globals['_FSCHANGE_OP']._serialized_start=2126 + _globals['_FSCHANGE_OP']._serialized_end=2195 + _globals['_FSCHANGE_KIND']._serialized_start=2197 + _globals['_FSCHANGE_KIND']._serialized_end=2288 + _globals['_FSCHANGE_ATTRIBUTION']._serialized_start=2290 + _globals['_FSCHANGE_ATTRIBUTION']._serialized_end=2381 + _globals['_PROCESS']._serialized_start=2383 + _globals['_PROCESS']._serialized_end=2450 + _globals['_BLOBCHUNK']._serialized_start=2452 + _globals['_BLOBCHUNK']._serialized_end=2507 + _globals['_RESTOREFILESREQUEST']._serialized_start=2510 + _globals['_RESTOREFILESREQUEST']._serialized_end=2681 + _globals['_RESTOREDIR']._serialized_start=2683 + _globals['_RESTOREDIR']._serialized_end=2736 + _globals['_RESTOREFILE']._serialized_start=2738 + _globals['_RESTOREFILE']._serialized_end=2809 + _globals['_RESTOREFILESRESPONSE']._serialized_start=2811 + _globals['_RESTOREFILESRESPONSE']._serialized_end=2850 + _globals['_READFILEREQUEST']._serialized_start=2852 + _globals['_READFILEREQUEST']._serialized_end=2938 + _globals['_READFILERESPONSE']._serialized_start=2940 + _globals['_READFILERESPONSE']._serialized_end=3008 + _globals['_FINALDIFFREQUEST']._serialized_start=3010 + _globals['_FINALDIFFREQUEST']._serialized_end=3044 + _globals['_FINALDIFFRESPONSE']._serialized_start=3047 + _globals['_FINALDIFFRESPONSE']._serialized_end=3180 + _globals['_SANDBOXCHANGES']._serialized_start=3182 + _globals['_SANDBOXCHANGES']._serialized_end=3267 + _globals['_DESTROYRUNREQUEST']._serialized_start=3269 + _globals['_DESTROYRUNREQUEST']._serialized_end=3304 + _globals['_DESTROYRUNRESPONSE']._serialized_start=3306 + _globals['_DESTROYRUNRESPONSE']._serialized_end=3326 + _globals['_SANDBOXSERVICE']._serialized_start=3329 + _globals['_SANDBOXSERVICE']._serialized_end=4013 # @@protoc_insertion_point(module_scope) diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi index 663ec85..32a1661 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi @@ -153,6 +153,7 @@ class CreateSandboxRequest(_message.Message): ENV_FIELD_NUMBER: _builtins.int HOSTNAME_FIELD_NUMBER: _builtins.int MACHINE_ID_FIELD_NUMBER: _builtins.int + DISPLAY_FIELD_NUMBER: _builtins.int run_id: _builtins.str sandbox_id: _builtins.str image: _builtins.str @@ -190,6 +191,12 @@ class CreateSandboxRequest(_message.Message): override the image's. Names match [A-Za-z_][A-Za-z0-9_]{0,127}; values hold no NUL. """ + @_builtins.property + def display(self) -> Global___Display: + """Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + no display. + """ + def __init__( self, *, @@ -203,15 +210,47 @@ class CreateSandboxRequest(_message.Message): env: _abc.Mapping[_builtins.str, _builtins.str] | None = ..., hostname: _builtins.str = ..., machine_id: _builtins.str = ..., + display: Global___Display | None = ..., ) -> None: ... - _HasFieldArgType: _TypeAlias = _typing.Literal["resources", b"resources"] # noqa: Y015 + _HasFieldArgType: _TypeAlias = _typing.Literal["display", b"display", "resources", b"resources"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["env", b"env", "files", b"files", "hostname", b"hostname", "image", b"image", "machine_id", b"machine_id", "mounts", b"mounts", "resources", b"resources", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "users", b"users"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["display", b"display", "env", b"env", "files", b"files", "hostname", b"hostname", "image", b"image", "machine_id", b"machine_id", "mounts", b"mounts", "resources", b"resources", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "users", b"users"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... Global___CreateSandboxRequest: _TypeAlias = CreateSandboxRequest # noqa: Y015 +@_typing.final +class Display(_message.Message): + """A virtual screen with a browser on it, run by the image's `swarm-display` as the user + `swarmdisplay`. Its processes are never reported or attributed to a call. + """ + + DESCRIPTOR: _descriptor.Descriptor + + WIDTH_FIELD_NUMBER: _builtins.int + HEIGHT_FIELD_NUMBER: _builtins.int + URL_FIELD_NUMBER: _builtins.int + width: _builtins.int + """Pixels, 320 to 1920 wide and 240 to 1200 high.""" + height: _builtins.int + url: _builtins.str + """Page the browser opens first. Empty opens about:blank.""" + def __init__( + self, + *, + width: _builtins.int = ..., + height: _builtins.int = ..., + url: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["height", b"height", "url", b"url", "width", b"width"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Display: _TypeAlias = Display # noqa: Y015 + @_typing.final class SeedFile(_message.Message): DESCRIPTOR: _descriptor.Descriptor @@ -270,6 +309,7 @@ class ExecRequest(_message.Message): CWD_FIELD_NUMBER: _builtins.int USER_FIELD_NUMBER: _builtins.int TIMEOUT_FIELD_NUMBER: _builtins.int + COLLECT_FIELD_NUMBER: _builtins.int run_id: _builtins.str sandbox_id: _builtins.str call_id: _builtins.str @@ -283,6 +323,13 @@ class ExecRequest(_message.Message): def timeout(self) -> _duration_pb2.Duration: """Required. On expiry the command and its descendants are killed.""" + @_builtins.property + def collect(self) -> _containers.RepeatedScalarFieldContainer[_builtins.str]: + """Absolute paths outside every key path. After the command, sandboxd reads each regular file + there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + text, such as a screenshot. + """ + def __init__( self, *, @@ -293,10 +340,11 @@ class ExecRequest(_message.Message): cwd: _builtins.str = ..., user: _builtins.str = ..., timeout: _duration_pb2.Duration | None = ..., + collect: _abc.Iterable[_builtins.str] | None = ..., ) -> None: ... _HasFieldArgType: _TypeAlias = _typing.Literal["timeout", b"timeout"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["argv", b"argv", "call_id", b"call_id", "cwd", b"cwd", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "timeout", b"timeout", "user", b"user"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["argv", b"argv", "call_id", b"call_id", "collect", b"collect", "cwd", b"cwd", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "timeout", b"timeout", "user", b"user"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... @@ -340,6 +388,7 @@ class ExecHeader(_message.Message): BACKGROUND_CHANGES_FIELD_NUMBER: _builtins.int CHANGES_FIELD_NUMBER: _builtins.int PROCESSES_FIELD_NUMBER: _builtins.int + COLLECTED_FIELD_NUMBER: _builtins.int exit_code: _builtins.int """-1 when timed out.""" timed_out: _builtins.bool @@ -363,6 +412,10 @@ class ExecHeader(_message.Message): def processes(self) -> _containers.RepeatedCompositeFieldContainer[Global___Process]: """Started while this call ran and still alive after it.""" + @_builtins.property + def collected(self) -> _containers.RepeatedCompositeFieldContainer[Global___CollectedFile]: + """One per `collect` path, in request order.""" + def __init__( self, *, @@ -374,15 +427,49 @@ class ExecHeader(_message.Message): background_changes: _abc.Iterable[Global___FsChange] | None = ..., changes: _abc.Iterable[Global___FsChange] | None = ..., processes: _abc.Iterable[Global___Process] | None = ..., + collected: _abc.Iterable[Global___CollectedFile] | None = ..., ) -> None: ... _HasFieldArgType: _TypeAlias = _typing.Literal["duration", b"duration", "stderr", b"stderr", "stdout", b"stdout"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["background_changes", b"background_changes", "changes", b"changes", "duration", b"duration", "exit_code", b"exit_code", "processes", b"processes", "stderr", b"stderr", "stdout", b"stdout", "timed_out", b"timed_out"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["background_changes", b"background_changes", "changes", b"changes", "collected", b"collected", "duration", b"duration", "exit_code", b"exit_code", "processes", b"processes", "stderr", b"stderr", "stdout", b"stdout", "timed_out", b"timed_out"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... Global___ExecHeader: _TypeAlias = ExecHeader # noqa: Y015 +@_typing.final +class CollectedFile(_message.Message): + DESCRIPTOR: _descriptor.Descriptor + + PATH_FIELD_NUMBER: _builtins.int + MISSING_FIELD_NUMBER: _builtins.int + SIZE_FIELD_NUMBER: _builtins.int + SHA256_FIELD_NUMBER: _builtins.int + path: _builtins.str + missing: _builtins.bool + """No regular file was there.""" + size: _builtins.int + """Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + `sha256` is empty. + """ + sha256: _builtins.str + """The content follows as a blob with this hash.""" + def __init__( + self, + *, + path: _builtins.str = ..., + missing: _builtins.bool = ..., + size: _builtins.int = ..., + sha256: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["missing", b"missing", "path", b"path", "sha256", b"sha256", "size", b"size"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___CollectedFile: _TypeAlias = CollectedFile # noqa: Y015 + @_typing.final class Output(_message.Message): DESCRIPTOR: _descriptor.Descriptor diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py index 1159cc6..74e31d6 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py @@ -96,7 +96,8 @@ def CreateRun(self, request, context): def CreateSandbox(self, request, context): """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi index ad6ba86..374ac77 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi @@ -46,7 +46,8 @@ class SandboxServiceStub: """ CreateSandbox: _grpc.UnaryUnaryMultiCallable[_sandbox_pb2.CreateSandboxRequest, _sandbox_pb2.CreateSandboxResponse] """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ Exec: _grpc.UnaryStreamMultiCallable[_sandbox_pb2.ExecRequest, _sandbox_pb2.ExecResponse] """Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -85,7 +86,8 @@ class SandboxServiceAsyncStub(SandboxServiceStub): """ CreateSandbox: _aio.UnaryUnaryMultiCallable[_sandbox_pb2.CreateSandboxRequest, _sandbox_pb2.CreateSandboxResponse] # type: ignore[assignment] """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ Exec: _aio.UnaryStreamMultiCallable[_sandbox_pb2.ExecRequest, _sandbox_pb2.ExecResponse] # type: ignore[assignment] """Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -133,7 +135,8 @@ class SandboxServiceServicer(metaclass=_abc_1.ABCMeta): context: _ServicerContext, ) -> _typing.Union[_sandbox_pb2.CreateSandboxResponse, _abc.Awaitable[_sandbox_pb2.CreateSandboxResponse]]: """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ @_abc_1.abstractmethod diff --git a/swarmeval/runtime/display.py b/swarmeval/runtime/display.py new file mode 100644 index 0000000..d100b49 --- /dev/null +++ b/swarmeval/runtime/display.py @@ -0,0 +1,222 @@ +"""`computer` and `browser`: tools that drive a sandbox's display (docs/agent-runtime.md#tools). + +Both run `swarm-display` inside the sandbox as the display's own user, which only these tools +run as, so an agent's shell cannot reach the screen. An image an action produces is collected by +sandboxd from outside the key paths and reaches the agent as an `ImageRef`. +""" + +import json +from collections.abc import Callable +from typing import Annotated, Literal, Self + +from pydantic import BaseModel, Field, model_validator + +from swarmeval.runtime.messages import ImageRef, ToolCall +from swarmeval.runtime.records import Exec, ExecResult, ToolResult +from swarmeval.runtime.tools import SandboxTool, exec_output + +DISPLAY_USER = "swarmdisplay" +"""Runs the display stack and these tools. Display images have it (deploy/images/display).""" +DISPLAY_STATE = "/run/swarm-display" +"""The display's state directory, kept out of every key path.""" +DISPLAY_TOOL_NAMES = ("computer", "browser") +SCREENSHOT = f"{DISPLAY_STATE}/out/screenshot.png" +ACTION_TIMEOUT_S = 90.0 + +Point = Annotated[ + list[int], Field(min_length=2, max_length=2, description="[x, y] in screen pixels.") +] + + +def _require(args: BaseModel, action: str, needs: dict[str, tuple[str, ...]]) -> None: + missing = [f for f in needs.get(action, ()) if getattr(args, f) is None] + if missing: + names = ", ".join(f"`{f}`" for f in missing) + raise ValueError(f"action `{action}` needs {names}") + + +class ComputerArgs(BaseModel): + action: Literal[ + "screenshot", + "click", + "mouse_move", + "drag", + "type", + "key", + "scroll", + "wait", + "cursor_position", + ] = Field(description="What to do. Every action but `cursor_position` returns a screenshot.") + coordinate: Point | None = Field( + default=None, + description="Where to click or move to (`click`, `mouse_move`, `drag`'s end), or where " + "to scroll (`scroll`, optional).", + ) + start_coordinate: Point | None = Field(default=None, description="Where `drag` starts.") + button: Literal["left", "middle", "right"] = Field(default="left", description="For `click`.") + count: int = Field(default=1, ge=1, le=3, description="Clicks: 2 for a double click.") + text: str | None = Field( + default=None, + max_length=10_000, + description="For `type`, the text to type. For `key`, xdotool key names, combos joined " + "with `+` and several keys separated by spaces: `ctrl+l`, `Return`, `alt+Tab`.", + ) + direction: Literal["up", "down", "left", "right"] | None = Field( + default=None, description="For `scroll`." + ) + amount: int = Field(default=3, ge=1, le=20, description="Wheel clicks for `scroll`.") + seconds: float = Field(default=1.0, gt=0, le=10, description="For `wait`.") + + @model_validator(mode="after") + def _needs(self) -> Self: + _require( + self, + self.action, + { + "click": ("coordinate",), + "mouse_move": ("coordinate",), + "drag": ("start_coordinate", "coordinate"), + "type": ("text",), + "key": ("text",), + "scroll": ("direction",), + }, + ) + return self + + +class BrowserArgs(BaseModel): + action: Literal[ + "navigate", + "back", + "forward", + "reload", + "snapshot", + "screenshot", + "click", + "hover", + "type", + "select", + "press", + "scroll", + "wait", + "tabs", + "tab_new", + "tab_select", + "tab_close", + ] = Field(description="What to do. Every action returns the page snapshot afterwards.") + url: str | None = Field( + default=None, max_length=8192, description="For `navigate`, and optionally `tab_new`." + ) + ref: str | None = Field( + default=None, + pattern=r"^[A-Za-z0-9]{1,32}$", + description="The element to act on, as its `ref` in the latest snapshot, such as `e12` " + "(`click`, `hover`, `type`, `select`).", + ) + text: str | None = Field( + default=None, max_length=10_000, description="For `type`: replaces the field's text." + ) + submit: bool = Field(default=False, description="For `type`: press Enter afterwards.") + values: list[str] | None = Field( + default=None, description="For `select`: option values or labels." + ) + key: str | None = Field( + default=None, + max_length=100, + description="For `press`: a key or combo such as `Enter`, `Escape`, `Control+a`.", + ) + direction: Literal["up", "down", "left", "right"] = Field( + default="down", description="For `scroll`." + ) + amount: int = Field(default=3, ge=1, le=20, description="For `scroll`: screens' worth / 2.") + seconds: float = Field(default=1.0, gt=0, le=10, description="For `wait`.") + index: int | None = Field( + default=None, ge=1, description="Tab number, from 1 (`tab_select`, `tab_close`)." + ) + screenshot: bool = Field( + default=False, + description="Attach a screenshot of the page after the action. `screenshot` always does.", + ) + + @model_validator(mode="after") + def _needs(self) -> Self: + _require( + self, + self.action, + { + "navigate": ("url",), + "click": ("ref",), + "hover": ("ref",), + "type": ("ref", "text"), + "select": ("ref", "values"), + "press": ("key",), + "tab_select": ("index",), + }, + ) + return self + + +def _build(tool: str) -> Callable[[BaseModel], Exec]: + def build(args: BaseModel) -> Exec: + arguments = json.dumps(args.model_dump(exclude_none=True)) + return Exec( + argv=("swarm-display", tool, arguments), + timeout_s=ACTION_TIMEOUT_S, + user=DISPLAY_USER, + collect=(SCREENSHOT,), + ) + + return build + + +def display_output(call: ToolCall, result: ExecResult) -> ToolResult: + """The action's text, with its screenshot as an image when it took one.""" + text = exec_output(call, result) + images: list[ImageRef] = [] + problems: list[str] = [] + for c in result.collected: + if c.missing: + continue + if c.sha256 is None: + problems.append(f"[the screenshot was {c.size} bytes, over sandboxd's limit]") + elif c.png is None: + problems.append("[the display returned a screenshot that is not a PNG image]") + else: + images.append(ImageRef(sha256=c.sha256, width=c.png.width, height=c.png.height)) + content = text.content + "".join(f"\n{p}" for p in problems) + return text.model_copy( + update={ + "content": content, + "is_error": text.is_error or bool(problems), + "images": tuple(images), + } + ) + + +COMPUTER = SandboxTool( + name="computer", + description=( + "Use the mouse and keyboard on your sandbox's screen, a desktop showing a web browser. " + "Coordinates are screen pixels from the top left, as in the screenshots. Each action " + "returns a screenshot taken after it." + ), + args=ComputerArgs, + build=_build("computer"), + output=display_output, +) + +BROWSER = SandboxTool( + name="browser", + description=( + "Drive the web browser on your sandbox's screen. Every action returns the active tab's " + "URL, title, and an accessibility snapshot in which each element has a `[ref=eN]`; act " + "on elements by that ref. Refs change when the page does: use the latest snapshot. The " + "sandbox has no internet; pages are served from inside it." + ), + args=BrowserArgs, + build=_build("browser"), + output=display_output, +) + +DISPLAY_TOOLS = (COMPUTER, BROWSER) +"""Handed to `RunLoop` beside `BUILTIN_TOOLS`.""" diff --git a/swarmeval/runtime/execute.py b/swarmeval/runtime/execute.py index 66e722a..0b09553 100644 --- a/swarmeval/runtime/execute.py +++ b/swarmeval/runtime/execute.py @@ -92,4 +92,5 @@ async def execute( ) from err assert agent.sandbox_id is not None, "checked in _check_tools" result = await self._sandbox.exec(agent.sandbox_id, agent.os_user, command, call_id=call.id) - return Execution(exec_output(call, result), call.arguments, exec_result=result) + output = tool.output or exec_output + return Execution(output(call, result), call.arguments, exec_result=result) diff --git a/swarmeval/runtime/loop.py b/swarmeval/runtime/loop.py index 7b5557d..30902ed 100644 --- a/swarmeval/runtime/loop.py +++ b/swarmeval/runtime/loop.py @@ -577,7 +577,10 @@ async def _tool_step( admitted = await d.after_tool_result(agent.info, result, tool_event.event_id) txn = admitted.txn message = ToolMessage( - tool_call_id=call.id, content=admitted.value.content, is_error=admitted.value.is_error + tool_call_id=call.id, + content=admitted.value.content, + is_error=admitted.value.is_error, + images=admitted.value.images, ) self._admit(txn, agent, [(message, admitted.intervention_id or tool_event.event_id)]) await self._writer.commit(txn) diff --git a/swarmeval/runtime/messages.py b/swarmeval/runtime/messages.py index 7a15ac6..0fd75d1 100644 --- a/swarmeval/runtime/messages.py +++ b/swarmeval/runtime/messages.py @@ -6,6 +6,7 @@ is how the dispatcher detects an intervention. """ +from collections.abc import Sequence from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, JsonValue @@ -39,11 +40,22 @@ class AssistantMessage(Frozen): tool_calls: tuple[ToolCall, ...] = () +class ImageRef(Frozen): + """An image a tool returned. The bytes are in the blob store under `sha256`; messages and + events hold only the reference.""" + + sha256: str + media_type: Literal["image/png"] = "image/png" + width: int + height: int + + class ToolMessage(Frozen): role: Literal["tool"] = "tool" tool_call_id: str content: str is_error: bool = False + images: tuple[ImageRef, ...] = () ChatMessage = Annotated[ @@ -71,6 +83,24 @@ class RequestOptions(Frozen): top_p: float | None = None max_output_tokens: int | None = None seed: int | None = None + max_images: int = Field(default=3, ge=0) + """Images sent with the request: the context's last `max_images`; earlier ones are replaced + by `IMAGE_OMITTED` (`visible_images`).""" + + +IMAGE_OMITTED = "[image omitted]" + + +def visible_images(messages: Sequence[ChatMessage], max_images: int) -> set[tuple[int, int]]: + """`(message index, image index)` of the images a request with `max_images` carries: the + last ones in the context.""" + every = [ + (i, j) + for i, m in enumerate(messages) + if isinstance(m, ToolMessage) + for j in range(len(m.images)) + ] + return set(every[max(0, len(every) - max_images) :]) if max_images else set() class ModelRequest(Frozen): diff --git a/swarmeval/runtime/records.py b/swarmeval/runtime/records.py index 25ddb14..3735083 100644 --- a/swarmeval/runtime/records.py +++ b/swarmeval/runtime/records.py @@ -15,6 +15,7 @@ AssistantMessage, ChatMessage, Frozen, + ImageRef, RequestOptions, ToolCall, Usage, @@ -43,6 +44,7 @@ class ToolResult(Frozen): tool: str content: str is_error: bool = False + images: tuple[ImageRef, ...] = () class Exec(Frozen): @@ -51,6 +53,11 @@ class Exec(Frozen): argv: tuple[str, ...] cwd: str | None = None timeout_s: float = 60.0 + user: str | None = None + """Runs as this user instead of the calling agent's `os_user`, such as the display's.""" + collect: tuple[str, ...] = () + """Files outside the key paths that sandboxd reads, removes, and returns after the command + (docs/services/sandboxd.md#exec-diff-and-process-snapshot).""" class FsChange(Frozen): @@ -93,6 +100,23 @@ class Truncated(Frozen): """The kept output is itself cut at sandboxd's blob limit.""" +class PngInfo(Frozen): + width: int + height: int + + +class CollectedFile(Frozen): + path: str + missing: bool = False + """No regular file was there.""" + size: int = 0 + sha256: str | None = None + """The content is in the blob store under this hash. `None` when missing or over sandboxd's + collect limit.""" + png: PngInfo | None = None + """Set when the content is a PNG image, read from its header by sandboxd's client.""" + + class ExecResult(Frozen): exit_code: int stdout: str @@ -107,6 +131,8 @@ class ExecResult(Frozen): """Changed between the previous call on this sandbox and this one, by background processes. Always ambiguous.""" processes: tuple[ProcessInfo, ...] = () + collected: tuple[CollectedFile, ...] = () + """One per `Exec.collect` path, in order.""" class WebRequest(Frozen): diff --git a/swarmeval/runtime/tools.py b/swarmeval/runtime/tools.py index db6e583..9cdab8b 100644 --- a/swarmeval/runtime/tools.py +++ b/swarmeval/runtime/tools.py @@ -29,6 +29,8 @@ class SandboxTool: args: type[BaseModel] build: Callable[[Any], Exec] owner: str | None = None + output: Callable[[ToolCall, ExecResult], ToolResult] | None = None + """Turns sandboxd's result into what the agent sees; `exec_output` when unset.""" @dataclass(frozen=True) @@ -165,9 +167,10 @@ def _build_web_request(args: WebRequestArgs) -> WebRequest: ) BUILTIN_TOOLS: tuple[Tool, ...] = (SHELL, WEB_REQUEST) -"""Tools the caller hands to `RunLoop`. The loop adds the Message Bus's `send_message` itself.""" +"""Tools the caller hands to `RunLoop`, with `swarmeval.runtime.display.DISPLAY_TOOLS`. The loop +adds the Message Bus's `send_message` itself.""" -BUILTIN_TOOL_NAMES = ("shell", "send_message", "web_request") +BUILTIN_TOOL_NAMES = ("shell", "send_message", "web_request", "computer", "browser") """Every tool name the runtime provides. Extensions may not reuse them.""" diff --git a/swarmeval/sandbox/blobs.py b/swarmeval/sandbox/blobs.py index 790645e..ecb8351 100644 --- a/swarmeval/sandbox/blobs.py +++ b/swarmeval/sandbox/blobs.py @@ -9,11 +9,7 @@ from dataclasses import dataclass from typing import Protocol -from swarmeval.events import ObjectStore - - -def blob_key(sha256: str) -> str: - return f"blobs/sha256/{sha256}" +from swarmeval.events import ObjectStore, blob_key class BlobStore(Protocol): diff --git a/swarmeval/sandbox/client.py b/swarmeval/sandbox/client.py index b8159d4..c01aa19 100644 --- a/swarmeval/sandbox/client.py +++ b/swarmeval/sandbox/client.py @@ -7,6 +7,7 @@ """ import hashlib +import struct from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import timedelta @@ -18,7 +19,15 @@ from swarmeval.core.models import SandboxProfile from swarmeval.proto.swarmeval.sandbox.v1 import sandbox_pb2 as pb from swarmeval.proto.swarmeval.sandbox.v1.sandbox_pb2_grpc import SandboxServiceStub -from swarmeval.runtime.records import Exec, ExecResult, FsChange, ProcessInfo, Truncated +from swarmeval.runtime.records import ( + CollectedFile, + Exec, + ExecResult, + FsChange, + PngInfo, + ProcessInfo, + Truncated, +) from swarmeval.sandbox.blobs import BlobStore @@ -147,6 +156,9 @@ async def create( hostname=hostname, machine_id=machine_id, ) + if profile.display is not None: + d = profile.display + request.display.CopyFrom(pb.Display(width=d.width, height=d.height, url=d.url or "")) try: response = await self._stub.CreateSandbox(request) except grpc.aio.AioRpcError as err: @@ -162,8 +174,9 @@ async def exec( call_id=call_id, argv=command.argv, cwd=command.cwd or "", - user=os_user or "", + user=command.user or os_user or "", timeout=_duration(command.timeout_s), + collect=command.collect, ) where = f"sandbox `{sandbox_id}` call `{call_id}`" stream = self._stub.Exec(request, timeout=command.timeout_s + self._exec_margin_s) @@ -189,7 +202,8 @@ async def exec( f"sandboxd Exec for run `{self._run_id}` {where}: the stream ended without a " "header." ) - await self._store(blobs.finish()) + received = blobs.finish() + await self._store(received) stdout, stdout_truncated = _output(header.stdout) stderr, stderr_truncated = _output(header.stderr) return ExecResult( @@ -206,6 +220,16 @@ async def exec( ProcessInfo(pid=p.pid, ppid=p.ppid, user=_clean(p.user), cmdline=_clean(p.cmdline)) for p in header.processes ), + collected=tuple( + CollectedFile( + path=c.path, + missing=c.missing, + size=c.size, + sha256=c.sha256 or None, + png=png_info(received[c.sha256]) if c.sha256 else None, + ) + for c in header.collected + ), ) async def restore( @@ -354,6 +378,21 @@ def finish(self) -> dict[str, bytes]: return self._done +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +_PNG_MAX_SIDE = 8192 + + +def png_info(data: bytes) -> PngInfo | None: + """The size a PNG's header declares, or `None` for anything that is not a PNG of at most + 8192 pixels a side. The content comes from inside a sandbox.""" + if len(data) < 24 or not data.startswith(_PNG_SIGNATURE) or data[12:16] != b"IHDR": + return None + width, height = struct.unpack(">II", data[16:24]) + if not (0 < width <= _PNG_MAX_SIDE and 0 < height <= _PNG_MAX_SIDE): + return None + return PngInfo(width=width, height=height) + + def _duration(seconds: float) -> Duration: duration = Duration() duration.FromTimedelta(timedelta(seconds=seconds)) @@ -365,6 +404,7 @@ def _expected_blobs(header: pb.ExecHeader) -> list[str]: for change in (*header.changes, *header.background_changes): if change.content: hashes.append(change.after_sha256) + hashes.extend(c.sha256 for c in header.collected if c.sha256) return hashes diff --git a/swarmeval/worker/run.py b/swarmeval/worker/run.py index cfccb01..8b41d8c 100644 --- a/swarmeval/worker/run.py +++ b/swarmeval/worker/run.py @@ -34,6 +34,7 @@ ) from swarmeval.honeypot import PlacedCanary, place, place_sandboxes from swarmeval.runtime import RunConfigError, RunLoop, RunSpec +from swarmeval.runtime.display import DISPLAY_TOOLS from swarmeval.runtime.extensions import ( ExtensionError, ExtensionLoadError, @@ -53,7 +54,7 @@ from swarmeval.worker.fork import last_changes, lineage_views, load_fork, restore from swarmeval.worker.pause import CANCELLED, QueuePauser from swarmeval.worker.probes import IsolationError, ProbeSandbox, check_isolation -from swarmeval.worker.transcript import check_transcript +from swarmeval.worker.transcript import check_transcript, transcript_images log = logging.getLogger(__name__) @@ -232,6 +233,7 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: owner_epoch=run.owner_epoch, callers=callers, writer=writer, + images=blobs, ) ) web = await stack.enter_async_context(HttpWebClient(blobs)) @@ -242,7 +244,7 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: sandbox_executor=sandboxes, pauser=QueuePauser(deps.queue, run.run_id, run.owner_epoch, deps.cancel_poll_s), extensions=extensions, - tools=BUILTIN_TOOLS, + tools=(*BUILTIN_TOOLS, *DISPLAY_TOOLS), web_client=web, fork=fork, ) @@ -265,7 +267,9 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: writer=writer, prior=prior, ).run(committed) - await _check_transcript(deps.engine, spec, loop, writer, last_lifecycle(committed)) + await _check_transcript( + deps.engine, spec, loop, writer, blobs, last_lifecycle(committed) + ) except (ExtensionError, RunConfigError, ScoringError) as err: return Outcome("failed", str(err)) except IsolationError as err: @@ -280,15 +284,23 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: async def _check_transcript( - engine: AsyncEngine, spec: RunSpec, loop: RunLoop, writer: RunWriter, end: str + engine: AsyncEngine, + spec: RunSpec, + loop: RunLoop, + writer: RunWriter, + blobs: S3BlobStore, + end: str, ) -> None: """Commits the transcript check as the run's last event before export, so the exports carry it. A mismatch is a finding about the run, not a failure of it. Its parent is the - last event a mismatch names, or else `end`, the loop's last lifecycle event.""" + last event a mismatch names, or else `end`, the loop's last lifecycle event. Every image + the contexts hold is read into memory for it, since requests carried them inline.""" + transcript = await load_transcript(engine, spec.run_id) check = check_transcript( - await load_transcript(engine, spec.run_id), + transcript, prompts={a.id: initial_context(a) for a in spec.agents}, tools=loop.tool_schemas(), + images={h: await blobs.get(h) for h in sorted(transcript_images(transcript))}, ) parent = check.event_ids[-1] if check.event_ids else end await writer.commit(Transaction(events=[EventDraft(record=check, parent_id=parent)])) diff --git a/swarmeval/worker/transcript.py b/swarmeval/worker/transcript.py index 987c542..28b3f6d 100644 --- a/swarmeval/worker/transcript.py +++ b/swarmeval/worker/transcript.py @@ -81,14 +81,27 @@ class _Sources: equal message in any agent's context.""" +def transcript_images(transcript: RunTranscript) -> set[str]: + """Every image the run's contexts hold, which `check_transcript` needs the bytes of.""" + return { + image.sha256 + for context in transcript.contexts.values() + for message in context + if isinstance(message, ToolMessage) + for image in message.images + } + + def check_transcript( transcript: RunTranscript, *, prompts: Mapping[str, tuple[ChatMessage, ...]], tools: Mapping[str, ToolSchema], + images: Mapping[str, bytes] | None = None, ) -> TranscriptCheckRecord: """`prompts` is each agent's generation 0 before its first turn; `tools` every tool the run - could offer, by name, as requests carried them.""" + could offer, by name, as requests carried them; `images` the bytes of every image + `transcript_images` names, which requests carried inline.""" sources = _index(transcript) mismatches: list[TranscriptMismatch] = [] explained: list[str] = [] @@ -104,7 +117,15 @@ def check_transcript( if call.event_id in inherited: continue requests += 1 - mismatch = _check_request(call, call.agent_id, call.gen, call.length, transcript, tools) + mismatch = _check_request( + call, + call.agent_id, + call.gen, + call.length, + transcript, + tools, + {} if images is None else images, + ) if mismatch is not None: mismatches.append(mismatch) mismatches.extend( @@ -218,6 +239,7 @@ def mismatch(idx: int, event_id: str | None, detail: str) -> TranscriptMismatch: tool_call_id=outcome.result.call_id, content=result.content, is_error=result.is_error, + images=result.images, ) if message != expected: found.append(mismatch(idx, outcome.event_id, _differs("tool call", by))) @@ -271,6 +293,7 @@ def _check_request( length: int, transcript: RunTranscript, tools: Mapping[str, ToolSchema], + images: Mapping[str, bytes], ) -> TranscriptMismatch | None: def mismatch(detail: str) -> TranscriptMismatch: return TranscriptMismatch( @@ -297,7 +320,7 @@ def mismatch(detail: str) -> TranscriptMismatch: tools=tuple(tools[name] for name in call.options.tools), options=call.options, ) - body = to_wire(request).model_dump_json(exclude_none=True).encode() + body = to_wire(request, images).model_dump_json(exclude_none=True).encode() digest = hashlib.sha256(body).hexdigest() if digest != call.gateway.request_sha256: return mismatch( diff --git a/tests/core/test_display.py b/tests/core/test_display.py new file mode 100644 index 0000000..d6d453c --- /dev/null +++ b/tests/core/test_display.py @@ -0,0 +1,80 @@ +from pathlib import Path + +import pytest + +from swarmeval.core import load_case +from swarmeval.core.models import Display +from tests.core.test_loader import base_case, base_env, load_error, write + + +def desktop_env(**display: object) -> dict[str, object]: + env = base_env() + env["schema_version"] = 2 + env["sandbox_profiles"]["desktop"] = { + "image": "swarmeval/display:dev", + "fs": [{"path": "/workspace"}], + "display": display, + } + return env + + +def test_a_profile_display_defaults_to_xga(tmp_path: Path) -> None: + case = base_case() + case["swarm"]["agents"][0].update(tools=["browser", "computer"], sandbox_profile="desktop") + + (variant,) = load_case( + write(tmp_path, case, desktop_env(url="http://127.0.0.1:8080/")) + ).variants + + display = variant.env.sandbox_profiles["desktop"].display + assert display == Display(width=1024, height=768, url="http://127.0.0.1:8080/") + assert variant.env.sandbox_profiles["default"].display is None + + +@pytest.mark.parametrize("tool", ["browser", "computer"]) +def test_a_display_tool_needs_a_profile_with_a_display(tmp_path: Path, tool: str) -> None: + case = base_case() + case["swarm"]["agents"][0]["tools"] = ["shell", tool] + + message = load_error(tmp_path, case, desktop_env()) + + assert f"agent `dev` lists `{tool}`" in message + assert "profile `default`, which has no `display`" in message + + +@pytest.mark.parametrize("path", ["/run", "/run/swarm-display", "/run/swarm-display/out"]) +def test_a_key_path_cannot_overlap_the_display_state(tmp_path: Path, path: str) -> None: + env = desktop_env() + env["sandbox_profiles"]["desktop"]["fs"] = [{"path": path}] # pyright: ignore[reportIndexIssue] + + message = load_error(tmp_path, base_case(), env) + + assert f"key path `{path}` overlaps `/run/swarm-display`" in message + + +@pytest.mark.parametrize( + "display", [{"width": 100}, {"height": 5000}, {"url": "http://x/\nInjected: 1"}, {"depth": 24}] +) +def test_a_display_out_of_range_is_rejected(tmp_path: Path, display: dict[str, object]) -> None: + message = load_error(tmp_path, base_case(), desktop_env(**display)) + + assert "display" in message + + +def test_a_display_needs_env_version_2(tmp_path: Path) -> None: + env = desktop_env() + env["schema_version"] = 1 + + message = load_error(tmp_path, base_case(), env) + + assert "profile `desktop` has `display`, which `schema_version: 2` added" in message + + +def test_an_agent_without_a_sandbox_cannot_list_a_display_tool(tmp_path: Path) -> None: + case = base_case() + case["schema_version"] = 5 + case["swarm"]["agents"][1].update(tools=["browser"], sandbox="none") + + message = load_error(tmp_path, case, desktop_env()) + + assert "agent `qa` has `sandbox: none` but lists `browser`" in message diff --git a/tests/core/test_loader.py b/tests/core/test_loader.py index 6d19c8b..f7380b6 100644 --- a/tests/core/test_loader.py +++ b/tests/core/test_loader.py @@ -197,9 +197,9 @@ def test_missing_schema_version_is_rejected(tmp_path: Path) -> None: def test_unsupported_schema_version_is_rejected(tmp_path: Path) -> None: env = base_env() - env["schema_version"] = 2 + env["schema_version"] = 3 - assert "env.yaml has `schema_version: 2`" in load_error(tmp_path, base_case(), env) + assert "env.yaml has `schema_version: 3`" in load_error(tmp_path, base_case(), env) def test_workspace_is_required(tmp_path: Path) -> None: diff --git a/tests/events/test_display_export.py b/tests/events/test_display_export.py new file mode 100644 index 0000000..c25fd32 --- /dev/null +++ b/tests/events/test_display_export.py @@ -0,0 +1,134 @@ +"""Screenshots through the record: tool events and contexts hold references, the export embeds +the images once, and the transcript check rebuilds requests with them.""" + +import base64 +import hashlib +import struct +from pathlib import Path +from typing import Any, cast + +import pytest +from inspect_ai.event import ModelEvent, ToolEvent +from inspect_ai.log import read_eval_log +from inspect_ai.model import ChatMessageTool, ContentImage, ContentText + +from swarmeval.events import StoredRun, assemble, write_eval +from swarmeval.events.export import image_hashes +from swarmeval.runtime.display import DISPLAY_TOOLS, SCREENSHOT +from swarmeval.runtime.messages import ImageRef, ToolMessage +from swarmeval.runtime.records import CollectedFile, Exec, ExecResult +from swarmeval.runtime.specs import initial_context +from swarmeval.runtime.tools import SHELL +from swarmeval.sandbox.client import png_info +from swarmeval.worker.transcript import check_transcript +from tests.events.test_export import HEADER, stored +from tests.runtime.fakes import ANY_IMAGE, FakeSandbox, Harness, agent, call, harness, reply +from tests.worker.test_transcript import transcript_of, with_context + +SHOTS = 4 + + +def png(n: int) -> bytes: + return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", 13) + b"IHDR" + struct.pack(">II", 1024, n) + + +IMAGES = {hashlib.sha256(png(n)).hexdigest(): png(n) for n in range(1, SHOTS + 1)} + + +async def screenshots() -> Harness: + taken = iter(IMAGES.items()) + + def handler(command: Exec) -> ExecResult: + sha, data = next(taken) + collected = CollectedFile(path=SCREENSHOT, size=len(data), sha256=sha, png=png_info(data)) + return ExecResult(exit_code=0, stdout="Did it.\n", stderr="", collected=(collected,)) + + look = call("computer", '{"action": "screenshot"}') + h = harness( + (agent("a", tools=("computer",)),), + {"a": [*(reply("", look) for _ in range(SHOTS)), reply("done")]}, + sandbox=FakeSandbox(handler=handler), + tools=(SHELL, *DISPLAY_TOOLS), + ) + await h.loop.run() + return h + + +def with_images(run: StoredRun) -> StoredRun: + return StoredRun( + workspace=run.workspace, events=run.events, messages=run.messages, images=IMAGES + ) + + +async def test_the_export_embeds_each_image_once_and_shows_what_the_model_saw( + tmp_path: Path, +) -> None: + h = await screenshots() + run = with_images(stored(h.store)) + assert image_hashes(run) == set(IMAGES) + + path = tmp_path / "sample.eval" + write_eval(assemble(HEADER, run), path) + (sample,) = read_eval_log(path, resolve_attachments=True).samples or [] + + tools = [e for e in sample.events if isinstance(e, ToolEvent)] + first = tools[0].result + assert isinstance(first, list) + assert first[0] == ContentText(text="Did it.\n") + image = first[1] + assert isinstance(image, ContentImage) + assert image.image == f"data:image/png;base64,{base64.b64encode(png(1)).decode()}" + last = [e for e in sample.events if isinstance(e, ModelEvent)][-1] + shown = [m for m in last.input if isinstance(m, ChatMessageTool)] + assert len(shown) == SHOTS + parts = [m.content[1] for m in shown if isinstance(m.content, list)] + assert parts[0] == ContentText(text="[image omitted]"), "only the last 3 images are sent" + assert all(isinstance(p, ContentImage) for p in parts[1:]) + + +async def test_an_export_without_the_images_is_refused() -> None: + h = await screenshots() + + with pytest.raises(ValueError, match="names 4 image"): + assemble(HEADER, stored(h.store)) + + +async def test_tool_events_carry_image_references_under_schema_10() -> None: + h = await screenshots() + run = stored(h.store) + + payloads = [cast(dict[str, Any], r.payload) for r in run.events] + tool = next(p for p in payloads if p["event"] == "tool") + ours = tool["metadata"]["swarmeval"] + assert ours["schema_version"] == 10 + sha = next(iter(IMAGES)) + assert ours["images"] == [ + {"sha256": sha, "media_type": "image/png", "width": 1024, "height": 1} + ] + assert ours["exec"]["collected"][0]["sha256"] == sha + assert tool["result"] == "Did it.\n" + + +async def test_the_transcript_check_rebuilds_requests_with_their_images() -> None: + h = await screenshots() + transcript = transcript_of(h.store) + + def check(images: dict[str, bytes] | None = None) -> bool: + return check_transcript( + transcript, + prompts={"a": initial_context(agent("a", tools=("computer",)))}, + tools=h.loop.tool_schemas(), + images=images or ANY_IMAGE, + ).consistent + + assert check() + message = transcript.contexts[("a", 0)][3] + assert isinstance(message, ToolMessage) + swapped = message.model_copy(update={"images": (ImageRef(sha256="f" * 64, width=1, height=1),)}) + tampered = check_transcript( + with_context(transcript, "a", 3, swapped), + prompts={"a": initial_context(agent("a", tools=("computer",)))}, + tools=h.loop.tool_schemas(), + images=ANY_IMAGE, + ) + assert not tampered.consistent diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 73d8954..e0e4e2d 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -1,7 +1,8 @@ import asyncio +import hashlib import json from collections.abc import AsyncIterator -from dataclasses import dataclass +from dataclasses import dataclass, field import grpc import httpx2 @@ -25,6 +26,7 @@ from swarmeval.runtime.loop import RunLoop from swarmeval.runtime.messages import ( AssistantMessage, + ImageRef, ModelRequest, RequestOptions, SystemMessage, @@ -59,6 +61,16 @@ CALLERS: tuple[Caller, ...] = (DEV, ExtensionCaller("acme.judge")) +class MemoryImages: + def __init__(self) -> None: + self.data: dict[str, bytes] = {} + self.reads: list[str] = [] + + async def get(self, sha256: str) -> bytes: + self.reads.append(sha256) + return self.data[sha256] + + @dataclass class Rig: backend: MockBackend @@ -67,6 +79,7 @@ class Rig: http: httpx2.AsyncClient channel: grpc.aio.Channel attachments: Attachments + images: MemoryImages = field(default_factory=MemoryImages) def session(self, *, run_id: str = "run_1", owner_epoch: int = 1) -> GatewaySession: return GatewaySession( @@ -76,6 +89,7 @@ def session(self, *, run_id: str = "run_1", owner_epoch: int = 1) -> GatewaySess owner_epoch=owner_epoch, callers=CALLERS, writer=self.writer, + images=self.images, ) @@ -379,3 +393,98 @@ async def test_a_call_waiting_for_its_ack_fails_when_the_stream_goes() -> None: with pytest.raises(NotAttachedError, match="closed \\(stream ended\\) before it acknowledged"): await waiting assert attachments.lookup("k") is None + + +def _image(rig: Rig, data: bytes) -> ImageRef: + sha = hashlib.sha256(data).hexdigest() + rig.images.data[sha] = data + return ImageRef(sha256=sha, width=1024, height=768) + + +async def test_tool_images_reach_the_backend_in_a_user_message_after_the_tools(rig: Rig) -> None: + rig.backend.reply(completion("done")) + first, second = _image(rig, b"first png"), _image(rig, b"second png") + current = AssistantMessage( + reasoning="current thoughts", + tool_calls=( + ToolCall(id="c1", name="computer", arguments="{}"), + ToolCall(id="c2", name="computer", arguments="{}"), + ), + ) + + async with rig.session() as session: + await session.generate( + DEV, + request( + UserMessage(content="go"), + current, + ToolMessage(tool_call_id="c1", content="clicked", images=(first,)), + ToolMessage(tool_call_id="c2", content="", images=(second,)), + ), + parent_id=None, + ) + + sent = rig.backend.requests[0]["messages"] + assert [m["role"] for m in sent] == ["user", "assistant", "tool", "tool", "user"] + assert sent[2]["content"] == "clicked" + assert sent[3]["content"] == "" + assert sent[4]["content"] == [ + {"type": "text", "text": "Images from tool call c1:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,Zmlyc3QgcG5n"}}, + {"type": "text", "text": "Images from tool call c2:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,c2Vjb25kIHBuZw=="}}, + ] + # The user message carrying the images comes after `passback`: the turn is still current. + assert sent[1]["reasoning_content"] == "current thoughts" + + +async def test_only_the_latest_images_are_sent(rig: Rig) -> None: + rig.backend.reply(completion("done")) + rig.backend.reply(completion("done")) + old, new = _image(rig, b"old"), _image(rig, b"new") + calls = (ToolCall(id="c1", name="computer", arguments="{}"),) + messages = ( + UserMessage(content="go"), + AssistantMessage(tool_calls=calls), + ToolMessage(tool_call_id="c1", content="one", images=(old,)), + AssistantMessage(tool_calls=(ToolCall(id="c2", name="computer", arguments="{}"),)), + ToolMessage(tool_call_id="c2", content="two", images=(new,)), + ) + base = request(*messages) + one = base.model_copy(update={"options": base.options.model_copy(update={"max_images": 1})}) + + async with rig.session() as session: + await session.generate(DEV, one, parent_id=None) + await session.generate(DEV, one, parent_id=None) + + sent = rig.backend.requests[0]["messages"] + assert sent[2]["content"] == "one\n[image omitted]" + assert [m["role"] for m in sent] == ["user", "assistant", "tool", "assistant", "tool", "user"] + assert sent[4]["content"] == "two" + assert sent[5]["content"][1]["image_url"]["url"] == "data:image/png;base64,bmV3" + assert rig.images.reads == [new.sha256], "the second request reuses the first's images" + + +async def test_a_tool_image_that_is_not_an_inline_png_is_refused(rig: Rig) -> None: + wire = { + "model": "qwen-test", + "messages": [ + {"role": "user", "content": "go"}, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "image_url", "image_url": {"url": "http://evil/x.png"}}], + }, + ], + } + + async with rig.session() as session: + key = session._keys["agent:dev"] # pyright: ignore[reportPrivateUsage] + reply = await rig.http.post( + "/v1/chat/completions", + json=wire, + headers={"Authorization": f"Bearer {key}", CALL_ID_HEADER: "call_1"}, + ) + + assert reply.json()["error"]["code"] == "invalid_request" + assert rig.backend.requests == [] diff --git a/tests/runtime/fakes.py b/tests/runtime/fakes.py index 18b64b4..4f0579a 100644 --- a/tests/runtime/fakes.py +++ b/tests/runtime/fakes.py @@ -3,7 +3,8 @@ import asyncio import hashlib import json -from collections.abc import Callable, Sequence +from collections import defaultdict +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any, Literal @@ -52,7 +53,7 @@ WebRequest, ) from swarmeval.runtime.specs import AgentSpec, Limits, RunSpec -from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL, WEB_REQUEST +from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL, WEB_REQUEST, Tool from swarmeval.runtime.writer import RunWriter from tests.gateway.mock_backend import completion, tool_call @@ -161,10 +162,15 @@ def messages(self, agent_id: str) -> list[ChatMessage]: """A gateway record with placeholder hash and raw response; `honest_record` fills them in.""" +ANY_IMAGE: Mapping[str, bytes] = defaultdict[str, bytes](lambda: b"image") +"""Image bytes for fakes, which never store any: every image reads the same. Pass it to +`check_transcript` too, so the request hashes it recomputes match `honest_record`'s.""" + + def honest_record(request: ModelRequest, response: ModelResponse) -> GatewayRecord: """What model-gateway records for a call it served faithfully: the hash of the body the worker sends, and a backend completion that normalizes to `response`.""" - body = to_wire(request).model_dump_json(exclude_none=True).encode() + body = to_wire(request, ANY_IMAGE).model_dump_json(exclude_none=True).encode() message = response.message raw = completion( message.content, @@ -328,6 +334,7 @@ def harness( fork: ForkStart | None = None, turn_policy: Literal["round_robin", "event_driven", "async"] = "round_robin", latency: dict[str, float] | None = None, + tools: Sequence[Tool] = (SHELL, WEB_REQUEST), ) -> Harness: store = store or FakeStore() pauser = pauser or FakePauser() @@ -356,7 +363,7 @@ def harness( sandbox_executor=sandbox, pauser=pauser, extensions=loaded, - tools=[SHELL, WEB_REQUEST], + tools=list(tools), web_client=web, fork=fork, ) diff --git a/tests/runtime/test_display.py b/tests/runtime/test_display.py new file mode 100644 index 0000000..bf58e5e --- /dev/null +++ b/tests/runtime/test_display.py @@ -0,0 +1,187 @@ +import hashlib +import json +import struct + +import pytest +from pydantic import ValidationError + +from swarmeval.runtime.display import ( + BROWSER, + COMPUTER, + DISPLAY_TOOLS, + SCREENSHOT, + BrowserArgs, + ComputerArgs, + display_output, +) +from swarmeval.runtime.messages import ImageRef, ToolMessage, UserMessage, visible_images +from swarmeval.runtime.records import CollectedFile, Exec, ExecResult, PngInfo, ToolCallRecord +from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL +from swarmeval.sandbox.client import png_info +from tests.runtime.fakes import FakeSandbox, agent, call, harness, reply + +SHOT = "ab" * 32 + + +def png(width: int, height: int) -> bytes: + return ( + b"\x89PNG\r\n\x1a\n" + struct.pack(">I", 13) + b"IHDR" + struct.pack(">II", width, height) + ) + + +def shot(collected: CollectedFile, exit_code: int = 0) -> ExecResult: + return ExecResult( + exit_code=exit_code, stdout="Did `click`.\n", stderr="", collected=(collected,) + ) + + +def test_display_tools_are_builtin_names() -> None: + assert {t.name for t in DISPLAY_TOOLS} <= set(BUILTIN_TOOL_NAMES) + + +@pytest.mark.parametrize( + ("args", "missing"), + [ + ({"action": "click"}, "`coordinate`"), + ({"action": "drag", "coordinate": [1, 2]}, "`start_coordinate`"), + ({"action": "type"}, "`text`"), + ({"action": "scroll"}, "`direction`"), + ], +) +def test_computer_actions_require_their_arguments(args: dict[str, object], missing: str) -> None: + with pytest.raises(ValidationError, match=missing): + ComputerArgs.model_validate(args) + + +def test_computer_points_are_two_integers() -> None: + with pytest.raises(ValidationError): + ComputerArgs.model_validate({"action": "click", "coordinate": [1, 2, 3]}) + with pytest.raises(ValidationError): + ComputerArgs.model_validate({"action": "click", "coordinate": [1.5, 2]}) + + +@pytest.mark.parametrize( + ("args", "missing"), + [ + ({"action": "navigate"}, "`url`"), + ({"action": "type", "ref": "e3"}, "`text`"), + ({"action": "select", "ref": "e3"}, "`values`"), + ({"action": "tab_select"}, "`index`"), + ], +) +def test_browser_actions_require_their_arguments(args: dict[str, object], missing: str) -> None: + with pytest.raises(ValidationError, match=missing): + BrowserArgs.model_validate(args) + + +def test_browser_refs_are_plain_tokens() -> None: + with pytest.raises(ValidationError): + BrowserArgs.model_validate({"action": "click", "ref": "e1] >> css=body"}) + + +def test_display_tools_run_as_the_display_user_and_collect_the_screenshot() -> None: + command = COMPUTER.build(ComputerArgs.model_validate({"action": "click", "coordinate": [3, 4]})) + + assert command.user == "swarmdisplay" + assert command.collect == (SCREENSHOT,) + assert command.argv[:2] == ("swarm-display", "computer") + assert json.loads(command.argv[2]) == { + "action": "click", + "coordinate": [3, 4], + "button": "left", + "count": 1, + "amount": 3, + "seconds": 1.0, + } + assert BROWSER.build(BrowserArgs(action="back")).argv[:2] == ("swarm-display", "browser") + + +def test_a_collected_png_becomes_an_image() -> None: + result = display_output( + call("computer", id="c1"), + shot( + CollectedFile( + path=SCREENSHOT, size=100, sha256=SHOT, png=PngInfo(width=1024, height=768) + ) + ), + ) + + assert not result.is_error + assert result.content == "Did `click`.\n" + assert result.images == (ImageRef(sha256=SHOT, width=1024, height=768),) + + +def test_no_screenshot_means_no_image() -> None: + result = display_output( + call("browser", id="c1"), shot(CollectedFile(path=SCREENSHOT, missing=True)) + ) + + assert (result.is_error, result.images) == (False, ()) + + +@pytest.mark.parametrize( + ("collected", "says"), + [ + (CollectedFile(path=SCREENSHOT, size=9, sha256=SHOT), "not a PNG"), + (CollectedFile(path=SCREENSHOT, size=99_000_000), "over sandboxd's limit"), + ], +) +def test_an_unusable_screenshot_is_an_error_the_agent_sees( + collected: CollectedFile, says: str +) -> None: + result = display_output(call("computer", id="c1"), shot(collected)) + + assert result.is_error + assert says in result.content + assert result.images == () + + +def test_png_info_reads_the_header_and_refuses_the_rest() -> None: + assert png_info(png(1024, 768)) == PngInfo(width=1024, height=768) + assert png_info(b"GIF89a" + bytes(30)) is None + assert png_info(png(0, 768)) is None + assert png_info(png(100_000, 10)) is None + assert png_info(png(10, 10)[:20]) is None + + +def test_visible_images_are_the_latest() -> None: + a, b, c = (ImageRef(sha256=x * 64, width=1, height=1) for x in "abc") + messages = ( + UserMessage(content="go"), + ToolMessage(tool_call_id="1", content="", images=(a, b)), + ToolMessage(tool_call_id="2", content=""), + ToolMessage(tool_call_id="3", content="", images=(c,)), + ) + + assert visible_images(messages, 2) == {(1, 1), (3, 0)} + assert visible_images(messages, 5) == {(1, 0), (1, 1), (3, 0)} + assert visible_images(messages, 0) == set() + + +async def test_a_screenshot_reaches_the_agents_context_and_the_record() -> None: + image = png(1024, 768) + sha = hashlib.sha256(image).hexdigest() + + def handler(command: Exec) -> ExecResult: + assert command.user == "swarmdisplay" + return shot( + CollectedFile(path=SCREENSHOT, size=len(image), sha256=sha, png=png_info(image)) + ) + + h = harness( + (agent(tools=("computer",)),), + {"a": [reply("", call("computer", '{"action": "screenshot"}')), reply("done")]}, + sandbox=FakeSandbox(handler=handler), + tools=(SHELL, *DISPLAY_TOOLS), + ) + + await h.loop.run() + + message = h.store.messages("a")[3] + assert isinstance(message, ToolMessage) + assert message.images == (ImageRef(sha256=sha, width=1024, height=768),) + record = h.store.records("tool")[0].record + assert isinstance(record, ToolCallRecord) + assert record.result.images == message.images + _, second = h.model.requests[1] + assert second.options.max_images == 3 diff --git a/tests/sandbox/test_client.py b/tests/sandbox/test_client.py index 6950116..f7ff159 100644 --- a/tests/sandbox/test_client.py +++ b/tests/sandbox/test_client.py @@ -332,3 +332,66 @@ async def test_restore_batches_paths_and_content_and_reports_unowned_paths(rig: ] assert [(d.path, d.uid) for r in requests for d in r.dirs] == [("/workspace/d", 1000)] assert unowned == ["/workspace/small.txt"] * len(requests) + + +PNG = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x04\x00\x00\x00\x03\x00rest" + + +async def test_a_display_profile_asks_sandboxd_for_a_display(rig: Rig) -> None: + profile = SandboxProfile.model_validate( + {"image": "swarmeval/display:dev", "display": {"width": 1280, "url": "http://127.0.0.1/"}} + ) + + await rig.client.create("box_a", profile) + + (request,) = rig.server.requests + assert request.HasField("display") + assert (request.display.width, request.display.height, request.display.url) == ( + 1280, + 768, + "http://127.0.0.1/", + ) + + +async def test_a_profile_without_a_display_sends_none(rig: Rig) -> None: + await rig.client.create("box_a", SandboxProfile(image="busybox")) + + (request,) = rig.server.requests + assert not request.HasField("display") + + +async def test_exec_runs_as_the_commands_user_and_returns_what_it_collected(rig: Rig) -> None: + rig.server.exec_items = [ + header_item( + exit_code=0, + collected=[ + pb.CollectedFile(path="/run/out/shot.png", size=len(PNG), sha256=sha(PNG)), + pb.CollectedFile(path="/run/out/none.png", missing=True), + ], + ), + *blob_items(PNG), + ] + command = Exec( + argv=("true",), user="swarmdisplay", collect=("/run/out/shot.png", "/run/out/none.png") + ) + + result = await rig.client.exec("box_a", "agent", command, call_id="c1") + + (request,) = rig.server.requests + assert (request.user, list(request.collect)) == ("swarmdisplay", list(command.collect)) + shot, none = result.collected + assert (shot.sha256, shot.png and (shot.png.width, shot.png.height)) == (sha(PNG), (1024, 768)) + assert (none.missing, none.sha256, none.png) == (True, None, None) + assert rig.blobs.stored[sha(PNG)] == PNG + + +async def test_a_collected_file_that_is_not_a_png_has_no_png_info(rig: Rig) -> None: + rig.server.exec_items = [ + header_item(collected=[pb.CollectedFile(path="/run/x", size=3, sha256=sha(b"gif"))]), + *blob_items(b"gif"), + ] + + result = await rig.client.exec("box_a", None, LS, call_id="c1") + + assert result.collected[0].png is None + assert rig.server.requests[0].user == "" diff --git a/tests/worker/test_display_e2e.py b/tests/worker/test_display_e2e.py new file mode 100644 index 0000000..ef7ccdc --- /dev/null +++ b/tests/worker/test_display_e2e.py @@ -0,0 +1,166 @@ +"""`browser` and `computer` through the whole platform: sandboxd starts the display image's +screen, screenshots reach the model inline, the export embeds them, and a fork after the first +display call is refused. Needs the display image (deploy/images/display) on the docker host.""" + +import json +import os +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +import grpc +import pytest +import yaml +from inspect_ai.event import ModelEvent, ToolEvent +from inspect_ai.log import read_eval_log +from inspect_ai.model import ChatMessageTool, ContentImage + +from swarmeval.analysis.exports import load_events_table +from swarmeval.control.bundles import pack +from swarmeval.events import export_key +from swarmeval.proto.swarmeval.control.v1 import control_pb2 as pb +from tests.gateway.mock_backend import completion, tool_call +from tests.models import chosen +from tests.worker.conftest import Platform + +IMAGE = os.environ.get("SWARMEVAL_IT_DISPLAY_IMAGE", "swarmeval/display:dev") + +pytestmark = [ + pytest.mark.docker, + pytest.mark.skipif( + subprocess.run(["docker", "image", "inspect", IMAGE], capture_output=True).returncode != 0, + reason=f"display image {IMAGE} is not on the docker host; build deploy/images/display", + ), +] + +PAGE = """Shop +""" +CASE: dict[str, Any] = { + "schema_version": 4, + "id": "desktop", + "workspace": "ws_e2e", + "swarm": { + "agents": [{"id": "a", "prompt": "a.md", "tools": ["browser", "computer"]}], + "limits": {"max_turns": 6}, + }, + "task": {"input": "task.md"}, +} +ENV: dict[str, Any] = { + "schema_version": 2, + "sandbox_profiles": { + "default": { + "image": IMAGE, + "fs": [{"path": "/workspace"}], + "limits": {"memory": "2gib", "pids": 512}, + "files": [{"from": "site", "to": "/workspace"}], + "display": {"url": "file:///workspace/index.html"}, + } + }, +} + + +def bundle(root: Path) -> bytes: + (root / "site").mkdir() + files = { + "case.yaml": yaml.safe_dump(CASE), + "env.yaml": yaml.safe_dump(ENV), + "a.md": "You are a.", + "task.md": "Pay.", + "site/index.html": PAGE, + } + for rel, text in files.items(): + (root / rel).write_text(text) + return pack(root) + + +def policy(request: dict[str, Any]) -> dict[str, Any]: + turns = sum(1 for m in request["messages"] if m["role"] == "assistant") + if turns == 0: + look = {"action": "snapshot", "screenshot": True} + return completion("", tool_calls=[tool_call("browser", json.dumps(look), id="b1")]) + if turns == 1: + snapshot = next(m["content"] for m in request["messages"] if m["role"] == "tool") + ref = snapshot.split('button "Pay" [ref=')[1].split("]")[0] + click = {"action": "click", "ref": ref} + shot = {"action": "click", "coordinate": [10, 10]} + return completion( + "", + tool_calls=[ + tool_call("browser", json.dumps(click), id="b2"), + tool_call("computer", json.dumps(shot), id="c1"), + ], + ) + return completion("done") + + +async def test_an_agent_drives_the_display_and_sees_its_screenshots( + platform: Platform, tmp_path: Path +) -> None: + platform.backend.respond = policy + (run_id,) = ( + await platform.control.SubmitRuns( + pb.SubmitRunsRequest(models=chosen("mock-model"), case_bundle=bundle(tmp_path)) + ) + ).run_ids + + outcome = (await platform.worker.drain())[run_id] + + assert outcome.status == "done", outcome.error + last = platform.backend.requests[-1]["messages"] + roles = [m["role"] for m in last] + assert roles == [ + "system", + "user", + "assistant", + "tool", + "user", + "assistant", + "tool", + "tool", + "user", + ] + assert "Title: paid" in last[6]["content"] + images = [p for m in (last[4], last[8]) for p in m["content"] if p["type"] == "image_url"] + assert len(images) == 2, "the first snapshot's and the computer click's screenshots" + assert all(p["image_url"]["url"].startswith("data:image/png;base64,iVBOR") for p in images) + + events = (await load_events_table(platform.store, run_id)).to_pylist() + tools = [json.loads(e["payload"]) for e in events if e["type"] == "tool"] + ours = [t["metadata"]["swarmeval"] for t in tools] + assert [len(o.get("images", [])) for o in ours] == [1, 0, 1] + assert [o["exec"]["processes"] for o in ours] == [[], [], []] + assert ours[2]["images"][0]["width"] == 1024 + + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "sample.eval" + path.write_bytes(platform.store.get(export_key(run_id))) + (sample,) = read_eval_log(path, resolve_attachments=True).samples or [] + shown = [e.result for e in sample.events if isinstance(e, ToolEvent)] + assert isinstance(shown[0], list) + assert isinstance(shown[0][1], ContentImage) + final = [e for e in sample.events if isinstance(e, ModelEvent)][-1] + seen = [ + c.image + for m in final.input + if isinstance(m, ChatMessageTool) and isinstance(m.content, list) + for c in m.content + if isinstance(c, ContentImage) + ] + assert len(seen) == 2 + assert all(i.startswith("data:image/png;base64,iVBOR") for i in seen) + + models = [e for e in events if e["type"] == "model"] + before = await platform.control.ForkRun( + pb.ForkRunRequest(run_id=run_id, at_event_id=models[0]["event_id"], actor="ada") + ) + assert before.run.forked_from == run_id, "a fork before the first display call goes on" + with pytest.raises(grpc.aio.AioRpcError) as err: + await platform.control.ForkRun( + pb.ForkRunRequest(run_id=run_id, at_event_id=models[-1]["event_id"], actor="ada") + ) + assert err.value.code() == grpc.StatusCode.FAILED_PRECONDITION + assert "used `browser`" in (err.value.details() or "") + # The fork from before the first display call runs, here, so no later test's worker drains it. + forked = (await platform.worker.drain())[before.run.run_id] + assert forked.status == "done", forked.error From 75ab13d4b5cc2da4c09620075bab3fbb2b8aa0a0 Mon Sep 17 00:00:00 2001 From: YL Date: Wed, 7 Oct 2026 17:10:25 -0700 Subject: [PATCH 2/2] sandboxd, core: no agent runs as the display's user (review on #36) An agent with `os_user: swarmdisplay` ran its shell as the display's user, with the X cookie, the daemon's socket, and the browser profile in reach, around the `browser` and `computer` tools. The loader now refuses that os_user. sandboxd, with a display, refuses it among the sandbox's users, and after starting the display checks the uid a command without a user gets: an image whose `USER` is `swarmdisplay` fails the sandbox. The case format says why a fork before the first display call starts the display before restoring files: in the source, too, the display started from the seed files. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + deploy/images/display/README.md | 2 ++ docs/case-format.md | 6 +++- docs/services/sandboxd.md | 5 ++++ go/CHANGELOG.md | 3 +- go/internal/sandboxd/display.go | 43 ++++++++++++++++++++++++---- go/internal/sandboxd/display_test.go | 25 ++++++++++++++++ go/internal/sandboxd/service.go | 2 +- go/internal/sandboxd/service_test.go | 7 +++++ swarmeval/core/models.py | 13 ++++++++- tests/core/test_display.py | 9 ++++++ 11 files changed, 106 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5173b21..2330b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ starts the display image's screen and Chromium; `browser` acts on accessibility-snapshot refs, `computer` on screen pixels, both run as the display's own user, and their screenshots reach the model as images. The sandbox keeps no network: the browser reaches loopback and files only. + No agent may set `os_user: swarmdisplay`, the display's own user. `deploy/images/display` builds the image (`mise run display:build`). - Tool results may carry images (`ToolResult.images`, `ToolMessage.images`, `ImageRef`), stored as blobs and referenced by hash. `RequestOptions.max_images` (default 3) keeps a request to diff --git a/deploy/images/display/README.md b/deploy/images/display/README.md index 2281580..ca51daa 100644 --- a/deploy/images/display/README.md +++ b/deploy/images/display/README.md @@ -42,6 +42,8 @@ does not remove these: which sandboxd collects and removes. Exit status 1 means the action failed; the reason is on stderr. They must run as `swarmdisplay`. - `sh`, `sleep`, and `tr`, which every sandbox image needs. +- `USER` left unset (root, with no capabilities in a sandbox). Agents without an `os_user` run + as the image's user, so sandboxd refuses an image whose user is `swarmdisplay`. ## Adding a case's apps diff --git a/docs/case-format.md b/docs/case-format.md index 5a78ae0..0ead173 100644 --- a/docs/case-format.md +++ b/docs/case-format.md @@ -367,13 +367,17 @@ sandbox_profiles: - The display runs as the user `swarmdisplay` and keeps its state in `/run/swarm-display`. A key path there, above it, or at `/` is rejected. Agents' own users cannot reach the screen or the - browser except through the two tools. + browser except through the two tools, so no agent may set `os_user: swarmdisplay`, and an + image whose `USER` is `swarmdisplay` fails the sandbox. - Programs the case needs on the screen or behind the browser, such as a web shop on `127.0.0.1:8080`, start from executables in the image's `/etc/swarm-display/start.d/`, before the browser opens. - Agents sharing a sandbox share its screen. - The browser is the memory of the run: a [fork](services/orchestrator.md#forks) restores files but not the screen, so a fork after an agent's first `browser` or `computer` call is refused. + A fork before it starts the display from the seed files, as its source did when its sandbox + was created, and restores the source's file changes after: the start scripts and the first + page see what they saw in the source, and the files are the source's at the fork point. - Chromium needs memory: give the profile at least `memory: 1gib` and a few hundred `pids`. ## Variants diff --git a/docs/services/sandboxd.md b/docs/services/sandboxd.md index 3bff3f7..432656d 100644 --- a/docs/services/sandboxd.md +++ b/docs/services/sandboxd.md @@ -148,6 +148,11 @@ virtual screen with a web browser, which the `browser` and `computer` tools driv - The screen needs a cookie only `swarmdisplay` can read, and the browser is driven through a pipe, with no debugging port. An agent's own `os_user`, and root with no capabilities, reach neither. Guard: `TestLiveDisplayDrivesTheBrowserAndKeepsItsProcessesOut`. +- So no agent may run as the display's user: `CreateSandbox` with a display refuses + `swarmdisplay` among its `users` (the agents' `os_user`s), and, after starting the display, + checks the uid a command run without a user gets, the image's `USER`. If that is the display's + uid, `CreateSandbox` fails with `INVALID_ARGUMENT` and removes the container. The loader + refuses `os_user: swarmdisplay` first. ## Networks diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index 263a934..0fd1193 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -7,7 +7,8 @@ runs the display image's `swarm-display start` as `swarmdisplay` before the first manifest (`spec/2026-10-07-browser-computer-use`). That user's processes are left out of every process listing, so a browser's renderers are neither reported nor make a call's changes ambiguous. A - key path overlapping `/run/swarm-display` is refused. + key path overlapping `/run/swarm-display` is refused, and so is `swarmdisplay` as an agent's + user or as the image's `USER`. - `ExecRequest.collect`: after the command, sandboxd reads and removes files outside the key paths as the call's user and returns them as blobs (`ExecHeader.collected`), up to 8 MiB each. - `driver.Process.UID`: the numeric uid beside the user's name. diff --git a/go/internal/sandboxd/display.go b/go/internal/sandboxd/display.go index fdbe2f3..3f4e5ec 100644 --- a/go/internal/sandboxd/display.go +++ b/go/internal/sandboxd/display.go @@ -5,7 +5,9 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "io" "path" + "slices" "strconv" "strings" "time" @@ -29,10 +31,15 @@ type Display struct { URL string } -func checkDisplay(d *Display, mounts []Mount) error { +func checkDisplay(d *Display, mounts []Mount, users []string) error { if d == nil { return nil } + // Users are the agents' users. The display's runs the screen and the browser, so an agent + // running as it would reach both around the tools. + if slices.Contains(users, DisplayUser) { + return fmt.Errorf("%w: user %s runs the display and cannot be an agent's user", ErrInvalid, DisplayUser) + } if d.Width < 320 || d.Width > 1920 || d.Height < 240 || d.Height > 1200 { return fmt.Errorf("%w: display %dx%d is outside 320x240 to 1920x1200", ErrInvalid, d.Width, d.Height) } @@ -79,12 +86,36 @@ func (s *Service) startDisplay(ctx context.Context, sb *sandbox, d *Display) (st if err != nil { return "", err } - for _, p := range procs { - if p.PID == int32(pid) { - return p.UID, nil - } + i := slices.IndexFunc(procs, func(p driver.Process) bool { return p.PID == int32(pid) }) + if i < 0 { + return "", fmt.Errorf("start the display in container %s: its pid %d is not running after `swarm-display start` returned", sb.container, pid) + } + uid := procs[i].UID + // An agent without an os_user runs as the image's user; that must not be the display's. + fallback, err := s.defaultUID(ctx, sb) + if err != nil { + return "", err + } + if fallback == uid { + return "", fmt.Errorf("%w: the image's default user is uid %s, which runs the display, so agents without an os_user would reach the screen and the browser around the tools. Leave `USER` unset in an image built from swarmeval/display", ErrInvalid, uid) + } + return uid, nil +} + +// defaultUIDScript prints the uid it runs as, with builtins only. +const defaultUIDScript = `while read -r k v _; do case $k in Uid:) echo "$v"; exit 0 ;; esac; done < /proc/self/status +exit 3` + +// defaultUID is the uid a command run without a user gets: the image's `USER`. +func (s *Service) defaultUID(ctx context.Context, sb *sandbox) (string, error) { + idCtx, cancel := context.WithTimeout(ctx, s.cfg.HelperTimeout) + defer cancel() + stdout := &capBuffer{limit: 64} + code, err := s.drv.Exec(idCtx, sb.container, driver.ExecSpec{Argv: []string{"/bin/sh", "-c", defaultUIDScript}}, stdout, io.Discard) + if err != nil || code != 0 { + return "", fmt.Errorf("read the default user of container %s: exit %d: %w", sb.container, code, err) } - return "", fmt.Errorf("start the display in container %s: its pid %d is not running after `swarm-display start` returned", sb.container, pid) + return strings.TrimSpace(string(stdout.buf)), nil } // collectScript prints the regular file $1 and removes it; exit 4 when there is none. A diff --git a/go/internal/sandboxd/display_test.go b/go/internal/sandboxd/display_test.go index 1324388..f11875e 100644 --- a/go/internal/sandboxd/display_test.go +++ b/go/internal/sandboxd/display_test.go @@ -102,6 +102,7 @@ func TestDisplayIsRefusedWhenItCannotWork(t *testing.T) { "key path at its dir": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: DisplayDir}}}, "key path above it": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/run"}}}, "key path at root": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/"}}}, + "agent as display": {Display: &Display{Width: 1024, Height: 768}, Users: []string{"qa", DisplayUser}}, } for name, req := range cases { svc := New(DefaultConfig(t.TempDir()), &fakeDriver{}, slog.New(slog.DiscardHandler)) @@ -174,3 +175,27 @@ func TestExecRefusesCollectPathsInKeyPaths(t *testing.T) { } } } + +func TestDisplayIsRefusedWhenTheImageRunsAsTheDisplayUser(t *testing.T) { + f := newFixture(t) + f.drv.defaultUID = "1000" // the fake lists the first user it meets, swarmdisplay, as 1000 + f.drv.procs = nil + f.drv.display = func(stdout io.Writer) (int, error) { + f.drv.procs = []driver.Process{{PID: 50, PPID: 1, User: DisplayUser, Cmdline: "swarm-display daemon"}} + _, err := io.WriteString(stdout, "50\n") + return 0, err + } + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", Display: &Display{Width: 1024, Height: 768}, + }) + + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v, want ErrInvalid: agents without an os_user would run as the display", err) + } + if !slices.Contains(f.drv.removed, driver.ContainerID("c_desk")) { + t.Fatal("container not removed") + } +} diff --git a/go/internal/sandboxd/service.go b/go/internal/sandboxd/service.go index 193dab2..1e1af1b 100644 --- a/go/internal/sandboxd/service.go +++ b/go/internal/sandboxd/service.go @@ -186,7 +186,7 @@ func (s *Service) CreateSandbox(ctx context.Context, req CreateRequest) (string, if err := checkIdentity(req, mounts); err != nil { return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } - if err := checkDisplay(req.Display, mounts); err != nil { + if err := checkDisplay(req.Display, mounts, req.Users); err != nil { return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } net, err := s.network(req.RunID, req.SandboxID) diff --git a/go/internal/sandboxd/service_test.go b/go/internal/sandboxd/service_test.go index 278a2c5..2495825 100644 --- a/go/internal/sandboxd/service_test.go +++ b/go/internal/sandboxd/service_test.go @@ -2,6 +2,7 @@ package sandboxd import ( "archive/tar" + "cmp" "context" "errors" "fmt" @@ -37,6 +38,8 @@ type fakeDriver struct { failNet string // CreateNetwork fails for this name // files are regular files outside the key paths, for collectScript. files map[string][]byte + // defaultUID is what a command run without a user gets; empty reads as root. + defaultUID string // display plays `swarm-display start`: its output and the processes it leaves. display func(stdout io.Writer) (int, error) } @@ -113,6 +116,10 @@ func (f *fakeDriver) Exec(ctx context.Context, _ driver.ContainerID, spec driver if spec.Argv[2] == processScript { return f.listing(stdout) } + if spec.Argv[2] == defaultUIDScript { + _, err := io.WriteString(stdout, cmp.Or(f.defaultUID, "0")+"\n") + return 0, err + } if spec.Argv[2] == collectScript { data, ok := f.files[spec.Argv[4]] if !ok { diff --git a/swarmeval/core/models.py b/swarmeval/core/models.py index 5ec6a9e..dedba91 100644 --- a/swarmeval/core/models.py +++ b/swarmeval/core/models.py @@ -24,7 +24,7 @@ ) from swarmeval.detect.defs import DetectorDef -from swarmeval.runtime.display import DISPLAY_STATE, DISPLAY_TOOLS +from swarmeval.runtime.display import DISPLAY_STATE, DISPLAY_TOOLS, DISPLAY_USER from swarmeval.runtime.extensions import ExtensionUse from swarmeval.runtime.tools import BUILTIN_TOOLS, SandboxTool @@ -336,6 +336,17 @@ def slots(self) -> tuple[str, ...]: def has_sandbox(self, agent: AgentDef) -> bool: return self.schema_version < NO_SANDBOX_VERSION or agent.sandbox != NO_SANDBOX + @model_validator(mode="after") + def _display_user_is_reserved(self) -> Self: + for agent in self.swarm.agents: + if agent.os_user == DISPLAY_USER: + raise ValueError( + f"agent `{agent.id}` sets `os_user: {DISPLAY_USER}`, the user that runs a " + "sandbox's display: its commands could reach the screen and the browser " + "around the `browser` and `computer` tools. Pick another user." + ) + return self + @model_validator(mode="after") def _sandboxless_agents_run_nothing(self) -> Self: sandbox_tools = { diff --git a/tests/core/test_display.py b/tests/core/test_display.py index d6d453c..52be67d 100644 --- a/tests/core/test_display.py +++ b/tests/core/test_display.py @@ -78,3 +78,12 @@ def test_an_agent_without_a_sandbox_cannot_list_a_display_tool(tmp_path: Path) - message = load_error(tmp_path, case, desktop_env()) assert "agent `qa` has `sandbox: none` but lists `browser`" in message + + +def test_no_agent_may_run_as_the_display_user(tmp_path: Path) -> None: + case = base_case() + case["swarm"]["agents"][0].update(sandbox_profile="desktop", os_user="swarmdisplay") + + message = load_error(tmp_path, case, desktop_env()) + + assert "agent `dev` sets `os_user: swarmdisplay`" in message