diff --git a/CHANGELOG.md b/CHANGELOG.md index 4000f1b..2330b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,34 @@ ## [Unreleased] +### Added +- `browser` and `computer`: agents drive a web browser and a virtual screen inside their sandbox + (`spec/2026-10-07-browser-computer-use`). A profile's `display` (`env.yaml` schema version 2) + starts the display image's screen and Chromium; `browser` acts on accessibility-snapshot refs, + `computer` on screen pixels, both run as the display's own user, and their screenshots reach + the model as images. The sandbox keeps no network: the browser reaches loopback and files only. + No agent may set `os_user: swarmdisplay`, the display's own user. + `deploy/images/display` builds the image (`mise run display:build`). +- Tool results may carry images (`ToolResult.images`, `ToolMessage.images`, `ImageRef`), stored + as blobs and referenced by hash. `RequestOptions.max_images` (default 3) keeps a request to + the context's latest images; `before_model_request` may change it. +- model-gateway accepts `image_url` parts (inline PNG `data:` URLs) in tool messages and moves + them into a user message after the tool messages before sending upstream, after reasoning + passback. +- `Exec.user` and `Exec.collect`: a sandbox tool may run as another user and have sandboxd read + files out of the sandbox after the call. `SandboxTool.output` turns sandboxd's result into the + agent's. +- The `.eval` export embeds a run's images: tool results inline, contexts as attachments, with + `[image omitted]` where a request left one out. + ### Changed +- Event schema version 10: tool events carry `images`, model events `max_images`, `exec` + observations `collected`. Older runs read as before. +- `env.yaml` schema version 2 adds a profile's `display`. Version 1 files load as before. +- A fork at a turn after a run's first `browser` or `computer` call is refused + (`FAILED_PRECONDITION`): a fork restores files, not a browser. +- `GatewaySession` takes `images`, where it reads the images requests carry; + `check_transcript` takes `images` to rebuild them. - Models are chosen when a case is submitted, not written in the case (`spec/2026-10-06-run-time-models`). `case.yaml` schema version 4: an agent names a `model_slot` (default `default`) instead of a `model`, and agents in one slot run one model. diff --git a/console/CHANGELOG.md b/console/CHANGELOG.md index 6a19ec5..e2521e2 100644 --- a/console/CHANGELOG.md +++ b/console/CHANGELOG.md @@ -5,6 +5,8 @@ ### Added - The case flow draws no sandbox for an agent with `sandbox: none` (case version 5), and its details say `none`. New case's template is `schema_version: 5`. +- The case flow shows a profile's `display`: its screen size as a chip, and the size and first + page among its settings. - A case's Files tab opens on a flow: `case.yaml` and its env file, pending edits included, drawn as variant axes, the task, agents, channels, extensions, sandboxes, profiles, canaries, and scorers, with what connects them. A click shows a part's settings, its connections, its diff --git a/console/src/lib/caseGraph.test.ts b/console/src/lib/caseGraph.test.ts index c1a4431..6c3f875 100644 --- a/console/src/lib/caseGraph.test.ts +++ b/console/src/lib/caseGraph.test.ts @@ -271,3 +271,32 @@ test("paths expand over the axes they name", () => { expect(expandPath("p/${variant.x}_${variant.y}.md", axes)).toEqual(["p/a_1.md", "p/a_2.md", "p/b_1.md", "p/b_2.md"]) expect(expandPath("p/${variant.z}.md", axes)).toEqual(["p/${variant.z}.md"]) }) + +test("a profile's display shows its size and first page", () => { + const text = `schema_version: 4 +id: c +workspace: w +swarm: + agents: + - { id: a, prompt: a.md, tools: [browser, computer], sandbox_profile: desk } +task: { input: t.md } +` + const env = `schema_version: 2 +sandbox_profiles: + default: { image: busybox } + desk: + image: swarmeval/display:dev + display: { width: 1280, url: "http://127.0.0.1:8080/" } +` + const g = caseGraph( + new Map([ + ["case.yaml", text], + ["env.yaml", env], + ]) + ) + const desk = g.nodes.find((n) => n.id === "profile:desk")! + expect(desk.chips).toContain("display 1280x768") + expect(desk.fields).toContainEqual(["display", "1280x768 · http://127.0.0.1:8080/"]) + const plain = g.nodes.find((n) => n.id === "profile:default")! + expect(plain.fields.map(([k]) => k)).not.toContain("display") +}) diff --git a/console/src/lib/caseGraph.ts b/console/src/lib/caseGraph.ts index f607246..aef8961 100644 --- a/console/src/lib/caseGraph.ts +++ b/console/src/lib/caseGraph.ts @@ -381,6 +381,11 @@ export function caseGraph(texts: ReadonlyMap): CaseG const mounts = list(p.fs).map(obj) const limits = Object.entries(obj(p.limits)).map(([k, v]) => `${k} ${inline(v)}`) const copies = list(p.files).map(obj) + // A profile's screen, which `browser` and `computer` drive; sizes default to 1024x768. + const display = + p.display === undefined + ? "" + : `${inline(obj(p.display).width ?? 1024)}x${inline(obj(p.display).height ?? 768)}` nodes.push({ id: `profile:${name}`, kind: "profile", @@ -389,6 +394,7 @@ export function caseGraph(texts: ReadonlyMap): CaseG chips: [ ...mounts.map((m) => `${str(m.path)}${m.protected ? " protected" : ""}`), ...limits, + ...(display ? [`display ${display}`] : []), ], fields: rows( ["image", str(p.image)], @@ -399,6 +405,7 @@ export function caseGraph(texts: ReadonlyMap): CaseG .join(" · "), ]), ["limits", limits.join(", ")], + ["display", display && [display, str(obj(p.display).url)].filter(Boolean).join(" · ")], ...copies.map((f): [string, string] => ["copies", `${str(f.from)} → ${str(f.to)}`]) ), source: envYaml?.source(["sandbox_profiles", name]), diff --git a/deploy/images/display/Dockerfile b/deploy/images/display/Dockerfile new file mode 100644 index 0000000..5add9d2 --- /dev/null +++ b/deploy/images/display/Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 +# The display image: a virtual screen, a browser, and `swarm-display`, which the `computer` and +# `browser` tools drive. Cases extend it with `FROM swarmeval/display` and add their own apps, +# started from /etc/swarm-display/start.d/. Contract: README.md beside this file. Build from the +# repo root: +# docker build -f deploy/images/display/Dockerfile -t swarmeval/display:dev deploy/images/display + +FROM python:3.12-slim-bookworm + +# Xvfb is the screen; matchbox keeps every window maximized without decorations; xdotool moves +# the mouse and types; xauth writes the screen's cookie. No terminal: the only programs on the +# screen are the browser and what a case adds. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb xauth xdotool matchbox-window-manager fonts-dejavu-core fonts-noto-cjk \ + && rm -rf /var/lib/apt/lists/* + +ENV PLAYWRIGHT_BROWSERS_PATH=/opt/swarm-display/browsers +RUN pip install --no-cache-dir playwright==1.63.0 pillow==12.3.0 \ + && playwright install --with-deps --no-shell chromium \ + && rm -rf /var/lib/apt/lists/* /root/.cache + +# The display user owns the display's state; agents' users cannot read it, so they reach the +# screen and the browser only through the tools. +RUN useradd --uid 990 --create-home --shell /usr/sbin/nologin swarmdisplay \ + && install -d -o swarmdisplay -g swarmdisplay -m 0700 /run/swarm-display \ + && install -d -m 1777 /tmp/.X11-unix \ + && install -d -m 0755 /etc/swarm-display/start.d + +COPY swarm_display /opt/swarm-display/swarm_display +COPY swarm-display /usr/local/bin/swarm-display +RUN chmod 0755 /usr/local/bin/swarm-display \ + && python -m compileall -q /opt/swarm-display/swarm_display diff --git a/deploy/images/display/README.md b/deploy/images/display/README.md new file mode 100644 index 0000000..ca51daa --- /dev/null +++ b/deploy/images/display/README.md @@ -0,0 +1,74 @@ +# swarmeval/display + +The sandbox image for the `browser` and `computer` tools: a virtual screen, Chromium on it, and +`swarm-display`, which drives both. A case gets it through a profile's +[`display`](../../../docs/case-format.md#display); sandboxd starts it +([docs/services/sandboxd.md](../../../docs/services/sandboxd.md#display)); the tools are in +[docs/agent-runtime.md](../../../docs/agent-runtime.md#tools). Why it is shaped this way: +[spec/2026-10-07-browser-computer-use](../../../spec/2026-10-07-browser-computer-use/README.md). + +```sh +mise run display:build # docker build -t swarmeval/display:dev deploy/images/display +``` + +Building downloads Debian packages and Playwright's Chromium; running needs no network. + +## What it holds + +| Piece | Version | Role | +|---|---|---| +| `python:3.12-slim-bookworm` | | Base, and the Python `swarm-display` runs on | +| Xvfb, xauth | Debian | The screen, `:1`, with a cookie only `swarmdisplay` can read | +| matchbox-window-manager | Debian | Keeps every window maximized, without decorations | +| xdotool | Debian | Mouse and keyboard for `computer` | +| Playwright, its Chromium | 1.63.0 | The browser, driven through a pipe: no debugging port | +| Pillow | 12.3.0 | Screenshots of the whole screen | +| DejaVu, Noto CJK fonts | Debian | Text in pages, Chinese and Japanese included | + +No terminal or file manager: the screen shows the browser and whatever a case adds. + +## Contract + +What sandboxd and the tools rely on. An image `FROM swarmeval/display` keeps it as long as it +does not remove these: + +- The user `swarmdisplay` (uid 990) and `/run/swarm-display`, owned by it, mode `0700`. +- `/usr/local/bin/swarm-display`: + - `start --width W --height H [--url U]`: starts the daemon (screen, window manager, start + scripts, browser), waits until the browser is up, prints the daemon's pid, and exits 0. + sandboxd runs it as `swarmdisplay` when it creates the sandbox. + - `computer ` and `browser `: one action, as the tools describe it. The text goes + to stdout; an image, if the action took one, to `/run/swarm-display/out/screenshot.png`, + which sandboxd collects and removes. Exit status 1 means the action failed; the reason is on + stderr. They must run as `swarmdisplay`. +- `sh`, `sleep`, and `tr`, which every sandbox image needs. +- `USER` left unset (root, with no capabilities in a sandbox). Agents without an `os_user` run + as the image's user, so sandboxd refuses an image whose user is `swarmdisplay`. + +## Adding a case's apps + +Put executables in `/etc/swarm-display/start.d/`. The daemon runs them in name order, as +`swarmdisplay`, after the screen is up and before the browser opens, each with 30 seconds to +return; a non-zero exit fails the sandbox. Start servers in the background: + +```dockerfile +FROM swarmeval/display:dev +COPY shop /opt/shop +COPY <<'EOF' /etc/swarm-display/start.d/10-shop +#!/bin/sh +cd /opt/shop && nohup python3 -m http.server 8080 --bind 127.0.0.1 >/dev/null 2>&1 & +EOF +RUN chmod 0755 /etc/swarm-display/start.d/10-shop +``` + +and point the profile's `display.url` at `http://127.0.0.1:8080/`. Their processes run as +`swarmdisplay`, so sandboxd counts them as the display's: they are not reported as `proc.*` +events. + +## Limits + +- The display's state, Chromium's profile included, lives in `/run/swarm-display`, outside every + key path: nothing the browser writes is a file change, and downloads are not kept. +- Dialogs (`alert`, `confirm`) are dismissed. +- The browser locates elements by snapshot ref through the `aria-ref=` selector. Playwright 1.64 + makes `page.get_by_ref` public; move to it when upgrading. diff --git a/deploy/images/display/swarm-display b/deploy/images/display/swarm-display new file mode 100755 index 0000000..a1a0ef9 --- /dev/null +++ b/deploy/images/display/swarm-display @@ -0,0 +1,3 @@ +#!/bin/sh +# The display's command line: `start`, `computer`, `browser`. See README.md. +PYTHONPATH=/opt/swarm-display exec /usr/local/bin/python3 -s -m swarm_display "$@" diff --git a/deploy/images/display/swarm_display/__init__.py b/deploy/images/display/swarm_display/__init__.py new file mode 100644 index 0000000..f0ee773 --- /dev/null +++ b/deploy/images/display/swarm_display/__init__.py @@ -0,0 +1,20 @@ +"""The display stack of a sandbox: a virtual screen, a browser on it, and the daemon that drives +both for the `computer` and `browser` tools (deploy/images/display/README.md). + +Everything here runs inside the sandbox as the user `swarmdisplay`. +""" + +from pathlib import Path + +STATE = Path("/run/swarm-display") +SOCKET = STATE / "daemon.sock" +LOG = STATE / "daemon.log" +SCREENSHOT = STATE / "out" / "screenshot.png" +"""Written by an action that has an image; sandboxd collects and removes it after the call.""" +DISPLAY = ":1" +XAUTHORITY = STATE / "Xauthority" +START_DIR = Path("/etc/swarm-display/start.d") + + +class ActionError(Exception): + """The action could not be done; the message is what the agent sees.""" diff --git a/deploy/images/display/swarm_display/__main__.py b/deploy/images/display/swarm_display/__main__.py new file mode 100644 index 0000000..e59f53f --- /dev/null +++ b/deploy/images/display/swarm_display/__main__.py @@ -0,0 +1,117 @@ +"""`swarm-display start | computer | browser | daemon`. + +`start` launches the daemon, waits until the screen and the browser are up, prints the daemon's +pid, and exits; sandboxd runs it once, when it creates the sandbox. `computer` and `browser` +send one action to the daemon and print its text; an image, if any, is at `SCREENSHOT`. Exit +status 1 means the action failed, and the reason is on stderr. +""" + +import argparse +import json +import os +import socket +import subprocess +import sys +import time + +from swarm_display import LOG, SCREENSHOT, SOCKET, STATE + +START_TIMEOUT_S = 50.0 +ACTION_TIMEOUT_S = 180.0 + + +def _start(width: int, height: int, url: str) -> int: + if SOCKET.exists(): + print(f"the display is already running ({SOCKET} exists)", file=sys.stderr) + return 1 + for sub in ("home", "tmp", "out", "profile"): + (STATE / sub).mkdir(mode=0o700, exist_ok=True) + argv = [sys.executable, "-s", "-m", "swarm_display", "daemon"] + argv += ["--width", str(width), "--height", str(height), "--url", url] + with open(LOG, "ab") as log: + daemon = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=log, + start_new_session=True, + cwd=STATE, + ) + deadline = time.monotonic() + START_TIMEOUT_S + while time.monotonic() < deadline: + if daemon.poll() is not None: + print(f"the display daemon exited {daemon.returncode}:\n{_log_tail()}", file=sys.stderr) + return 1 + if SOCKET.exists(): + print(daemon.pid) + return 0 + time.sleep(0.1) + daemon.kill() + print(f"the display was not up after {START_TIMEOUT_S:.0f}s:\n{_log_tail()}", file=sys.stderr) + return 1 + + +def _log_tail() -> str: + try: + return LOG.read_text(errors="replace")[-4000:] + except FileNotFoundError: + return "(no log)" + + +def _act(tool: str, raw: str) -> int: + try: + args = json.loads(raw) + except ValueError as err: + print(f"arguments are not JSON: {err}", file=sys.stderr) + return 1 + try: + SCREENSHOT.unlink(missing_ok=True) + except PermissionError: + print(f"swarm-display {tool} must run as the display's user, swarmdisplay", file=sys.stderr) + return 1 + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as conn: + conn.settimeout(ACTION_TIMEOUT_S) + try: + conn.connect(str(SOCKET)) + except OSError as err: + print(f"the display is not running ({SOCKET}: {err})", file=sys.stderr) + return 1 + conn.sendall(json.dumps({"tool": tool, "args": args}).encode() + b"\n") + with conn.makefile("rb") as reply: + line = reply.readline() + if not line: + print("the display daemon closed the connection without an answer", file=sys.stderr) + return 1 + answer = json.loads(line) + if not answer["ok"]: + print(answer["text"], file=sys.stderr) + return 1 + sys.stdout.write(answer["text"]) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(prog="swarm-display") + sub = parser.add_subparsers(dest="command", required=True) + for name in ("start", "daemon"): + p = sub.add_parser(name) + p.add_argument("--width", type=int, required=True) + p.add_argument("--height", type=int, required=True) + p.add_argument("--url", default="about:blank") + for name in ("computer", "browser"): + sub.add_parser(name).add_argument("arguments") + opts = parser.parse_args() + os.umask(0o077) + match opts.command: + case "start": + return _start(opts.width, opts.height, opts.url) + case "daemon": + from swarm_display.daemon import run + + return run(opts.width, opts.height, opts.url) + case tool: + return _act(tool, opts.arguments) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/images/display/swarm_display/browser.py b/deploy/images/display/swarm_display/browser.py new file mode 100644 index 0000000..2c04425 --- /dev/null +++ b/deploy/images/display/swarm_display/browser.py @@ -0,0 +1,155 @@ +"""`browser`: Chromium on the screen, driven through Playwright. + +Every action answers with the active tab's URL, title, and an accessibility snapshot in which +each element carries a `[ref=eN]`; actions on an element name that ref. The window is the same +one `computer` sees, so the two tools can be mixed. +""" + +import asyncio +from typing import Any + +from playwright.async_api import BrowserContext, Locator, Page, async_playwright + +from swarm_display import SCREENSHOT, STATE, ActionError + +SNAPSHOT_CHARS = 40_000 +ACTION_TIMEOUT_MS = 10_000 +NAVIGATION_TIMEOUT_MS = 30_000 +SCROLL_STEP_PX = 400 + + +class Browser: + def __init__(self, context: BrowserContext) -> None: + self.context = context + self.active = context.pages[0] + context.on("page", self._opened) + context.set_default_timeout(ACTION_TIMEOUT_MS) + context.set_default_navigation_timeout(NAVIGATION_TIMEOUT_MS) + + @classmethod + async def launch(cls, width: int, height: int, url: str) -> "Browser": + playwright = await async_playwright().start() + context = await playwright.chromium.launch_persistent_context( + user_data_dir=STATE / "profile", + headless=False, + no_viewport=True, + # gVisor or runc is the sandbox; Chromium's own needs namespaces a sandbox lacks. + chromium_sandbox=False, + args=[ + "--window-position=0,0", + f"--window-size={width},{height}", + "--start-maximized", + "--disable-dev-shm-usage", + "--disable-gpu", + "--no-first-run", + "--disable-breakpad", + ], + ) + browser = cls(context) + if url != "about:blank": + await browser.active.goto(url, wait_until="domcontentloaded") + return browser + + def _opened(self, page: Page) -> None: + self.active = page + + async def act(self, args: dict[str, Any]) -> str: + action = args.get("action") + page = self.active + match action: + case "navigate": + await page.goto(_text(args, "url"), wait_until="domcontentloaded") + case "back": + await page.go_back(wait_until="domcontentloaded") + case "forward": + await page.go_forward(wait_until="domcontentloaded") + case "reload": + await page.reload(wait_until="domcontentloaded") + case "snapshot" | "screenshot": + pass + case "click": + await _ref(page, args).click() + case "hover": + await _ref(page, args).hover() + case "type": + target = _ref(page, args) + await target.fill(_text(args, "text")) + if args.get("submit"): + await target.press("Enter") + case "select": + values = args.get("values") + if not isinstance(values, list): + raise ActionError("`values` must be a list of option values or labels") + await _ref(page, args).select_option([str(v) for v in values]) + case "press": + await page.keyboard.press(_text(args, "key")) + case "scroll": + steps = int(args.get("amount", 3)) * SCROLL_STEP_PX + dx, dy = {"up": (0, -steps), "down": (0, steps), "left": (-steps, 0)}.get( + args.get("direction", "down"), (steps, 0) + ) + await page.mouse.wheel(dx, dy) + case "wait": + await asyncio.sleep(float(args.get("seconds", 1))) + case "tabs": + pass + case "tab_new": + page = await self.context.new_page() + self.active = page + if args.get("url"): + await page.goto(_text(args, "url"), wait_until="domcontentloaded") + case "tab_select": + self.active = self._tab(args) + await self.active.bring_to_front() + case "tab_close": + closing = self._tab(args) if "index" in args else page + if len(self.context.pages) == 1: + raise ActionError("this is the last tab; open another before closing it") + await closing.close() + self.active = self.context.pages[-1] + await self.active.bring_to_front() + case _: + raise ActionError(f"unknown browser action {action!r}") + if action == "screenshot" or args.get("screenshot"): + await self.active.screenshot(path=SCREENSHOT, type="png") + return await self._describe() + + def _tab(self, args: dict[str, Any]) -> Page: + index = args.get("index") + pages = self.context.pages + if not isinstance(index, int) or not 1 <= index <= len(pages): + raise ActionError(f"`index` must be a tab number from 1 to {len(pages)}") + return pages[index - 1] + + async def _describe(self) -> str: + page = self.active + pages = self.context.pages + lines = [f"Tab {pages.index(page) + 1} of {len(pages)}"] + if len(pages) > 1: + for i, p in enumerate(pages, 1): + lines.append(f" {i}. {await p.title()} - {p.url}") + lines += [f"URL: {page.url}", f"Title: {await page.title()}", ""] + snapshot = await page.aria_snapshot(mode="ai") + if len(snapshot) > SNAPSHOT_CHARS: + snapshot = ( + snapshot[:SNAPSHOT_CHARS] + + f"\n[snapshot cut at {SNAPSHOT_CHARS} of {len(snapshot)} characters; " + "scroll, or act on what is shown]" + ) + lines.append(snapshot) + return "\n".join(lines) + "\n" + + +def _text(args: dict[str, Any], name: str) -> str: + value = args.get(name) + if not isinstance(value, str) or not value: + raise ActionError(f"`{name}` is required") + return value + + +def _ref(page: Page, args: dict[str, Any]) -> Locator: + """The element a snapshot named. Playwright 1.64's `page.get_by_ref` replaces the selector.""" + ref = _text(args, "ref") + if not ref.isalnum(): + raise ActionError(f"`ref` {ref!r} is not a ref from the snapshot, such as `e12`") + return page.locator(f"aria-ref={ref}") diff --git a/deploy/images/display/swarm_display/computer.py b/deploy/images/display/swarm_display/computer.py new file mode 100644 index 0000000..735b5bc --- /dev/null +++ b/deploy/images/display/swarm_display/computer.py @@ -0,0 +1,99 @@ +"""`computer`: the mouse, the keyboard, and screenshots of the whole screen, in screen pixels. + +The worker validates arguments before they get here (swarmeval/runtime/display.py); this module +checks only what it needs to act safely, such as coordinates inside the screen. +""" + +import asyncio +from typing import Any + +from PIL import ImageGrab + +from swarm_display import DISPLAY, SCREENSHOT, ActionError + +SETTLE_S = 0.4 +"""Time for the screen to redraw after an action, before its screenshot.""" +TYPE_DELAY_MS = 12 +BUTTONS = {"left": "1", "middle": "2", "right": "3"} +WHEEL = {"up": "4", "down": "5", "left": "6", "right": "7"} + + +class Computer: + def __init__(self, width: int, height: int) -> None: + self.width = width + self.height = height + + async def act(self, args: dict[str, Any]) -> str: + action = args.get("action") + match action: + case "screenshot": + pass + case "click": + x, y = self._point(args, "coordinate") + button = BUTTONS[args.get("button", "left")] + count = str(args.get("count", 1)) + await _xdotool("mousemove", "--sync", x, y, "click", "--repeat", count, button) + case "mouse_move": + await _xdotool("mousemove", "--sync", *self._point(args, "coordinate")) + case "drag": + x0, y0 = self._point(args, "start_coordinate") + x1, y1 = self._point(args, "coordinate") + await _xdotool("mousemove", "--sync", x0, y0, "mousedown", "1") + await _xdotool("mousemove", "--sync", x1, y1, "mouseup", "1") + case "type": + await _xdotool("type", "--delay", str(TYPE_DELAY_MS), "--", str(args["text"])) + case "key": + await _xdotool("key", "--", *str(args["text"]).split()) + case "scroll": + if "coordinate" in args: + await _xdotool("mousemove", "--sync", *self._point(args, "coordinate")) + wheel = WHEEL[args["direction"]] + await _xdotool("click", "--repeat", str(args.get("amount", 3)), wheel) + case "wait": + await asyncio.sleep(float(args.get("seconds", 1))) + case "cursor_position": + out = await _xdotool("getmouselocation", "--shell") + pos = dict(line.split("=", 1) for line in out.splitlines() if "=" in line) + return f"The mouse is at ({pos['X']}, {pos['Y']}) on a {self._size()} screen.\n" + case _: + raise ActionError(f"unknown computer action {action!r}") + if action != "screenshot": + await asyncio.sleep(SETTLE_S) + await asyncio.to_thread(self._screenshot) + return f"Did `{action}`. Screenshot of the {self._size()} screen attached.\n" + + def _size(self) -> str: + return f"{self.width}x{self.height}" + + def _point(self, args: dict[str, Any], name: str) -> tuple[str, str]: + point = args.get(name) + if not isinstance(point, list) or len(point) != 2: + raise ActionError(f"`{name}` must be [x, y]") + x, y = point + if not (isinstance(x, int) and isinstance(y, int)): + raise ActionError(f"`{name}` must be two integers, got {point!r}") + if not (0 <= x < self.width and 0 <= y < self.height): + raise ActionError( + f"`{name}` {point!r} is off the {self._size()} screen: x runs 0 to " + f"{self.width - 1}, y 0 to {self.height - 1}" + ) + return str(x), str(y) + + def _screenshot(self) -> None: + image = ImageGrab.grab(xdisplay=DISPLAY) + partial = SCREENSHOT.with_suffix(".partial") + image.save(partial, format="PNG") + partial.replace(SCREENSHOT) + + +async def _xdotool(*argv: str) -> str: + proc = await asyncio.create_subprocess_exec( + "xdotool", + *argv, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await proc.communicate() + if proc.returncode: + raise ActionError(f"xdotool {' '.join(argv)} failed: {err.decode(errors='replace')}") + return out.decode(errors="replace") diff --git a/deploy/images/display/swarm_display/daemon.py b/deploy/images/display/swarm_display/daemon.py new file mode 100644 index 0000000..91612a9 --- /dev/null +++ b/deploy/images/display/swarm_display/daemon.py @@ -0,0 +1,130 @@ +"""The daemon: starts the screen, a window manager, the case's start scripts, and the browser, +then answers one action at a time on a unix socket only `swarmdisplay` can open. + +The browser is driven through Playwright's pipe, so no debugging port is open in the sandbox. The +screen needs the cookie in `XAUTHORITY`, which only `swarmdisplay` can read. +""" + +import asyncio +import json +import os +import secrets +import subprocess +import time +from typing import Any + +from playwright.async_api import Error as PlaywrightError + +from swarm_display import DISPLAY, SOCKET, START_DIR, STATE, XAUTHORITY, ActionError +from swarm_display.browser import Browser +from swarm_display.computer import Computer + +START_SCRIPT_TIMEOUT_S = 30.0 + + +def _environment() -> None: + os.environ.update( + { + "DISPLAY": DISPLAY, + "XAUTHORITY": str(XAUTHORITY), + "HOME": str(STATE / "home"), + "TMPDIR": str(STATE / "tmp"), + "XDG_CONFIG_HOME": str(STATE / "home" / ".config"), + "XDG_CACHE_HOME": str(STATE / "home" / ".cache"), + "XDG_RUNTIME_DIR": str(STATE / "tmp"), + } + ) + + +def _screen(width: int, height: int) -> None: + cookie = secrets.token_hex(16) + XAUTHORITY.touch(mode=0o600) + subprocess.run(["xauth", "-q", "-f", str(XAUTHORITY), "add", DISPLAY, ".", cookie], check=True) + subprocess.Popen( + [ + "Xvfb", + DISPLAY, + "-screen", + "0", + f"{width}x{height}x24", + "-nolisten", + "tcp", + "-auth", + str(XAUTHORITY), + "-dpi", + "96", + ], + stdin=subprocess.DEVNULL, + ) + deadline = time.monotonic() + 15 + while subprocess.run(["xdotool", "getdisplaygeometry"], capture_output=True).returncode: + if time.monotonic() > deadline: + raise RuntimeError("Xvfb did not accept connections within 15s") + time.sleep(0.1) + subprocess.Popen( + ["matchbox-window-manager", "-use_titlebar", "no", "-use_cursor", "yes"], + stdin=subprocess.DEVNULL, + ) + + +def _start_scripts() -> None: + if not START_DIR.is_dir(): + return + for script in sorted(START_DIR.iterdir()): + if script.is_file() and os.access(script, os.X_OK): + print(f"running {script}", flush=True) + subprocess.run( + [str(script)], stdin=subprocess.DEVNULL, check=True, timeout=START_SCRIPT_TIMEOUT_S + ) + + +async def _serve(computer: Computer, browser: Browser) -> None: + lock = asyncio.Lock() + + async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + # Anything but an ActionError or a browser error is a bug: asyncio logs it to the daemon + # log, the connection closes unanswered, and the next action is served as usual. + try: + line = await reader.readline() + async with lock: + answer = await _answer(json.loads(line), computer, browser) + writer.write(json.dumps(answer).encode() + b"\n") + await writer.drain() + finally: + writer.close() + + server = await asyncio.start_unix_server(handle, path=str(SOCKET)) + os.chmod(SOCKET, 0o600) + async with server: + await server.serve_forever() + + +async def _answer(request: dict[str, Any], computer: Computer, browser: Browser) -> dict[str, Any]: + tool, args = request["tool"], request["args"] + try: + if not isinstance(args, dict): + raise ActionError("arguments must be a JSON object") + if tool == "computer": + text = await computer.act(args) + elif tool == "browser": + text = await browser.act(args) + else: + raise ActionError(f"unknown tool {tool!r}") + except ActionError as err: + return {"ok": False, "text": str(err)} + except PlaywrightError as err: + return {"ok": False, "text": f"The browser refused: {err.message}"} + return {"ok": True, "text": text} + + +def run(width: int, height: int, url: str) -> int: + _environment() + _screen(width, height) + _start_scripts() + + async def main() -> None: + browser = await Browser.launch(width, height, url) + await _serve(Computer(width, height), browser) + + asyncio.run(main()) + return 0 diff --git a/docs/agent-runtime.md b/docs/agent-runtime.md index 9cfc2b3..1a7f95f 100644 --- a/docs/agent-runtime.md +++ b/docs/agent-runtime.md @@ -103,11 +103,11 @@ message or a new generation, so a request can always be rebuilt from `messages`. | Kind | Where it runs | Defined by | |---|---|---| -| `SandboxTool` | sandboxd, in the calling agent's sandbox. Its `build` only turns arguments into an `Exec` | The runtime, or an extension with `runs_in="sandbox"` | +| `SandboxTool` | sandboxd, in the calling agent's sandbox. Its `build` only turns arguments into an `Exec`, which may name the user to run as (instead of the agent's `os_user`) and files to `collect`; its optional `output` turns sandboxd's result into what the agent sees (`exec_output` when unset) | The runtime, or an extension with `runs_in="sandbox"` | | `WorkerTool` | The worker, with only its extension's `HookContext`. It must not do its own I/O | An extension with `runs_in="worker"` | | `WebTool` | The worker, through the run's `WebClient`. Its `build` only turns arguments into a `WebRequest` | The runtime: `web_request` | -The runtime provides three tools itself: +The runtime provides five tools itself: - `shell` (`swarmeval.runtime.tools.SHELL`): `cmd` runs with `sh -c` in the agent's sandbox, with `timeout_s` from 0 to 600 seconds, 60 by default. The agent sees stdout then stderr, each cut @@ -119,11 +119,39 @@ The runtime provides three tools itself: `headers`, `body`, and `timeout_s` up to 120, 30 by default. Sent by the worker, never the sandbox; the agent sees the status line, headers, and the body cut at 64 KiB. Only for agents that list it. See [orchestrator.md](services/orchestrator.md#web_request). +- `computer` and `browser` (`swarmeval.runtime.display`, handed to the loop as `DISPLAY_TOOLS`): + the screen and the web browser of a sandbox whose profile has a + [`display`](case-format.md#display). Both run `swarm-display` in the sandbox as the user + `swarmdisplay`, not the agent's `os_user`, and collect the screenshot an action takes, which + reaches the agent as an image ([below](#images)). `computer`: `screenshot`, `click` + (`coordinate`, `button`, `count`), `mouse_move`, `drag` (`start_coordinate`, `coordinate`), + `type` and `key` (`text`, xdotool key names for `key`), `scroll` (`direction`, `amount`, + optional `coordinate`), `wait` (`seconds`), `cursor_position`. Coordinates are `[x, y]` screen + pixels. Every action but `cursor_position` returns a screenshot. `browser`: `navigate` + (`url`), `back`, `forward`, `reload`, `snapshot`, `screenshot`, `click` and `hover` (`ref`), + `type` (`ref`, `text`, `submit`), `select` (`ref`, `values`), `press` (`key`), `scroll`, + `wait`, `tabs`, `tab_new` (`url`), `tab_select` and `tab_close` (`index`). Every action + returns the active tab's URL, title, and an accessibility snapshot in which each element + carries a `[ref=eN]`; `screenshot`, or any action with `screenshot: true`, adds an image of + the page. Arguments are checked against the actions' required fields before anything runs. A `RuntimeTool` runs in the worker without I/O and returns its result with the events it causes, which commit with the tool call. `BUILTIN_TOOL_NAMES` lists every runtime tool name; pass it to `load_extensions` so no extension reuses one. +### Images + +A tool result may carry images: `ToolResult.images` and `ToolMessage.images`, each an `ImageRef` +(`sha256`, `media_type` `image/png`, `width`, `height`). The bytes are in the blob store, uploaded +before the tool's event commits; contexts and events hold only the reference. An +`after_tool_result` hook may drop or replace images like any other part of the result. Only tool +results carry images. + +`RequestOptions.max_images` (default 3) caps the images a request carries: the context's last +`max_images` go with it, and each earlier one is replaced by the text `[image omitted]` +(`visible_images`). `before_model_request` may change it. It is recorded on the model event, and +the export expands the request's input with the same rule. + Arguments are validated against the tool's pydantic model. Invalid arguments, an unknown tool, a non-zero exit, and a timeout all become error results the agent sees, and each is recorded. A call counts as unknown unless the request that produced it offered the tool, so a tool that @@ -261,7 +289,7 @@ agents that use the sandbox), `token`, and where it is planted: `hostname`, `env | `on_resume` | Observe | `ResumeInfo`: `fork`, `source_run_id`, `at_seq`, `fidelity` (`fs_restored` or `fs_partial`) | `None`. Runs instead of `on_run_start` when a run goes on from another run's state, after the state is restored and before the first turn; extension state is already the source's | | `before_turn` | Gate | `TurnInfo` | `Proceed`, `Skip`, `Inject(messages)`, `Stop(reason)` | | `compact_context` | Transform | Current messages | `None`, or a new, non-empty message tuple (new generation). An empty tuple fails the run | -| `before_model_request` | Transform | `RequestOptions` | `RequestOptions`. Tools may only be narrowed | +| `before_model_request` | Transform | `RequestOptions` | `RequestOptions`. Tools may only be narrowed. `max_images` may change ([Images](#images)) | | `after_model_response` | Transform | `AssistantMessage` | `AssistantMessage` | | `before_tool_call` | Gate | `ToolCall` | `Allow`, `Rewrite(arguments)`, `Block(result)` | | `after_tool_result` | Transform | `ToolResult` | `ToolResult` | diff --git a/docs/architecture.md b/docs/architecture.md index 33dac88..8d26dd7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -162,7 +162,9 @@ flowchart LR the call's `ToolEvent`, committed before the agent sees the result, like any other tool result. Detail: [services/orchestrator.md](services/orchestrator.md#web_request). - File and process events are collected by `sandboxd` after each tool call: a diff of the key paths, - which are mounted as host volumes, and the processes still alive. They come back with the tool + which are mounted as host volumes, and the processes still alive. Screenshots the `browser` and + `computer` tools take are files sandboxd collects from outside the key paths after the call; + they reach the model as images and are stored as blobs. They come back with the tool result and commit in the same transaction as its `ToolEvent`. Reads that write nothing are not recorded; a decoy file counts as hit when its canary content shows up anywhere. Syscall-level audit is an opt-in interface backed by gVisor Runtime Monitoring. @@ -207,6 +209,10 @@ place with pyarrow and queries them with DuckDB in process ([analysis.md](servic [services/orchestrator.md](services/orchestrator.md#web_request). - The agent loop runs outside the sandbox. Containers only execute tool calls, so model keys, logs, and recovery state are out of the agent's reach. +- A sandbox whose profile has a display runs a screen and a browser inside it, as the user + `swarmdisplay`. The browser has the sandbox's network, so it reaches only loopback and files. + Agents' own users reach the screen and the browser only through the `browser` and `computer` + tools. Detail: [services/sandboxd.md](services/sandboxd.md#display). - Extensions run in the worker, with its privileges. A case's own extensions (`case:` files) are case author code, not agent code, and run only where the deployment allows it (`--allow-case-code`, off by default). Sandbox isolation does not cover them. diff --git a/docs/case-format.md b/docs/case-format.md index e484ce0..0ead173 100644 --- a/docs/case-format.md +++ b/docs/case-format.md @@ -99,7 +99,7 @@ An agent: | `model_slot` | no | The [model slot](#model-slots) the agent runs in. Default `default` | | `prompt` | yes | System prompt file | | `task` | no | First user message file. Overrides `task.input` | -| `tools` | no | Tool names: `shell`, `send_message`, `web_request`, or one an extension provides. The run fails to start if nothing provides one. `web_request` is the agent's only way to the internet | +| `tools` | no | Tool names: `shell`, `send_message`, `web_request`, `browser`, `computer`, or one an extension provides. The run fails to start if nothing provides one. `web_request` is the agent's only way to the internet. `browser` and `computer` need a sandbox whose profile has a [`display`](#display) | | `sandbox` | no | Join a shared instance from `env.yaml`, or, from version 5, `none` for [no sandbox](#sandboxes) | | `sandbox_profile` | no | Profile of the agent's private sandbox. Exclusive with `sandbox` | | `os_user` | no | Unix user the agent's commands run as; not with `sandbox: none`. sandboxd adds it to the sandbox's image with a private home under `/home`, unless the image has it already. Where it may write besides its home is up to the image | @@ -143,8 +143,8 @@ console's run form asks models for. | Key | Meaning | |---|---| -| `schema_version` | `1` | -| `sandbox_profiles` | Name → `image`, `fs` mounts (`path`, `mode: rw \| ro`, `protected`), `limits` (`cpu`, `memory`, `pids`, `disk`), `files` (below) | +| `schema_version` | `1` or `2` | +| `sandbox_profiles` | Name → `image`, `fs` mounts (`path`, `mode: rw \| ro`, `protected`), `limits` (`cpu`, `memory`, `pids`, `disk`), `files` (below), `display` ([below](#display), version 2) | | `sandboxes` | Name → `profile`. Declare only instances that agents share | | `canaries` | Files holding a token generated per run, below | @@ -342,6 +342,44 @@ rejects the following: - A shared instance named like an agent that has a private sandbox. - From version 5, a shared instance named `none`, which no agent can join. +## Display + +A profile with `display` gives its sandboxes a virtual screen with a web browser on it, which the +`browser` and `computer` tools drive ([agent-runtime.md](agent-runtime.md#tools)). It needs +`env.yaml` version 2 and an image built from `swarmeval/display` +([deploy/images/display](../deploy/images/display/README.md)). + +```yaml +schema_version: 2 +sandbox_profiles: + desktop: + image: swarmeval/display:dev # or an image FROM it, with the case's own apps + fs: [{ path: /workspace }] + limits: { memory: 2gib, pids: 512 } + files: [{ from: site, to: /workspace }] + display: { width: 1024, height: 768, url: "file:///workspace/index.html" } +``` + +| Key | Default | Meaning | +|---|---|---| +| `width`, `height` | `1024`, `768` | The screen in pixels, 320×240 to 1920×1200. At 1024×768 neither Claude nor GPT scales a screenshot, so the coordinates a model reads off it are screen pixels | +| `url` | `about:blank` | The page the browser opens first. Sandboxes have no network: pages come from `file://` or a server inside the sandbox | + +- The display runs as the user `swarmdisplay` and keeps its state in `/run/swarm-display`. A key + path there, above it, or at `/` is rejected. Agents' own users cannot reach the screen or the + browser except through the two tools, so no agent may set `os_user: swarmdisplay`, and an + image whose `USER` is `swarmdisplay` fails the sandbox. +- Programs the case needs on the screen or behind the browser, such as a web shop on + `127.0.0.1:8080`, start from executables in the image's `/etc/swarm-display/start.d/`, before + the browser opens. +- Agents sharing a sandbox share its screen. +- The browser is the memory of the run: a [fork](services/orchestrator.md#forks) restores files + but not the screen, so a fork after an agent's first `browser` or `computer` call is refused. + A fork before it starts the display from the seed files, as its source did when its sandbox + was created, and restores the source's file changes after: the start scripts and the first + page see what they saw in the source, and the files are the source's at the fork point. +- Chromium needs memory: give the profile at least `memory: 1gib` and a few hundred `pids`. + ## Variants The cartesian product of the chosen [models](#model-slots), slot by slot, then of `variants:` @@ -451,7 +489,10 @@ error says how to move a case: set `schema_version: 4`, remove each agent's `mod | 1 | The format as first released; `models` filled each case's `model` variant axis. No longer loads | | 2 | `models` fills model slots, as a list (the `default` slot) or by slot; entries take `models` | -`env.yaml` is at version 1. +| `env.yaml` | Adds | +|---|---| +| 1 | The format as first released | +| 2 | A profile's [`display`](#display) | ## Not accepted yet diff --git a/docs/development.md b/docs/development.md index aad855a..85605bf 100644 --- a/docs/development.md +++ b/docs/development.md @@ -31,10 +31,11 @@ mise run sync | `mise run lint` | `ruff format --check .` and `ruff check .` | | `mise run typecheck` | `pyright` in strict mode over `swarmeval/` and `tests/` | | `mise run test` | `pytest` (asyncio mode `auto`), without tests marked `docker` or `browser` | -| `mise run test:docker` | `pytest -m docker`: the Postgres store, migrations, and export against a throwaway `postgres:18-alpine` and `rustfs/rustfs` from testcontainers, the sandboxd client against a sandboxd it builds from `go/` (needs `busybox:latest`), the public API through an `edge` it builds from `go/`, and mutual TLS between the Go and Python services with certificates from a `swarm-certs` it builds. Run it before a change to `swarmeval/db/`, `swarmeval/events/`, `swarmeval/sandbox/`, `swarmeval/control/`, or edge is done | +| `mise run test:docker` | `pytest -m docker`: the Postgres store, migrations, and export against a throwaway `postgres:18-alpine` and `rustfs/rustfs` from testcontainers, the sandboxd client against a sandboxd it builds from `go/` (needs `busybox:latest`; the `browser` and `computer` test also needs `swarmeval/display:dev` and skips without it), the public API through an `edge` it builds from `go/`, and mutual TLS between the Go and Python services with certificates from a `swarm-certs` it builds. Run it before a change to `swarmeval/db/`, `swarmeval/events/`, `swarmeval/sandbox/`, `swarmeval/control/`, or edge is done | | `mise run go:lint` | `golangci-lint run` over `go/`, integration tests included | | `mise run go:test` | `go test ./...` in `go/` | -| `mise run go:test-integration` | sandboxd against the local docker daemon (needs `busybox:latest` and `python:3.12-slim`), and edge against a throwaway `postgres:18-alpine` from testcontainers-go | +| `mise run go:test-integration` | sandboxd against the local docker daemon (needs `busybox:latest` and `python:3.12-slim`; the display test also `swarmeval/display:dev`, and skips without it), and edge against a throwaway `postgres:18-alpine` from testcontainers-go | +| `mise run display:build` | Builds `swarmeval/display:dev`, the [display image](../deploy/images/display/README.md) cases with `browser` and `computer` run on. Downloads Debian packages and Playwright's Chromium | | `mise run console:install` | `pnpm install --frozen-lockfile` in `console/`. The other console tasks run it first | | `mise run console:check` | The console's `tsc -b`, `eslint`, and `vitest` unit tests | | `mise run console:build` | Builds the console into `go/internal/edge/webui/static/` (not committed), where edge embeds it from. Build edge after it; an edge built without it serves the API only | diff --git a/docs/event-log.md b/docs/event-log.md index 52d2703..89b6dfa 100644 --- a/docs/event-log.md +++ b/docs/event-log.md @@ -56,9 +56,13 @@ The hash chain is not in the payload, because the hash covers the payload. It li `prev_hash` / `hash` columns and is added to `metadata.swarmeval` on export. Kind-specific fields also go under `metadata.swarmeval`. A `ModelEvent` carries `input` (the -`gen` / `len` reference, below), the offered `tools`, and `raw_tool_arguments`, the argument -text the model produced. A `ToolEvent` carries `raw_arguments`, `executed_arguments`, -`blocked_by`, and `exec`. Inspect keeps tool arguments as a parsed object. Arguments that are not +`gen` / `len` reference, below), the offered `tools`, `raw_tool_arguments`, the argument +text the model produced, and `max_images`, how many of the context's latest images the request +carried (schema version 10). A `ToolEvent` carries `raw_arguments`, `executed_arguments`, +`blocked_by`, `exec` (with `collected`, the files sandboxd read out of the sandbox after the call, +from version 10), and `images` when the result has any: `{sha256, media_type, width, height}` +each, the bytes in the [blob store](#large-objects). `result` stays the text; the export adds the +images ([below](#export)). Inspect keeps tool arguments as a parsed object. Arguments that are not a JSON object, or hold a value JSON cannot carry exactly (NaN, a number that overflows a 64-bit float, an integer beyond ±(2**53 − 1)), are stored as `{}`, and the raw text is kept. @@ -219,7 +223,7 @@ event, `InfoEvent(source="swarmeval.transcript_check")`, so both exports carry i | Check | What must hold | Mismatch names | |---|---|---| | `context` | Walking each agent's generations in `idx` order: generation 0 starts with the case's system prompt and task; a later generation starts with the `after` of a `compact_context` intervention. After that, an assistant message at index *i* of generation *g* is the response of the agent's model call built from `(g, i)`; the tool messages after it are the results of that call's `ToolEvent`s (`parent_id`), in `seq` order, with the call's id; a user message is a `msg.deliver` to this agent (as the bus words it), an injection into it (`ctx.actions.inject`), or a `before_turn` `Inject`. Where an `after_model_response` / `after_tool_result` intervention rewrote the event, the last rewrite's `after` is what must appear instead | The model or tool event the message was compared with; none when no event could explain it. A generation whose start nothing explains is reported once and not walked further | -| `request` | Each agent model call's request, rebuilt from the stored context (`gen`, `len`), the tools it offered, and its sampling options, hashes (sha256 of the wire body the worker sends) to the gateway's `request_sha256` | The model event | +| `request` | Each agent model call's request, rebuilt from the stored context (`gen`, `len`), the tools it offered, its sampling options, and the images it carried (read from the blob store, `max_images`), hashes (sha256 of the wire body the worker sends) to the gateway's `request_sha256` | The model event | | `response` | Each model call's response equals the gateway's `upstream_response_json` normalized again the way model-gateway normalizes it | The model event | | `send` | An agent's `msg.send` has the channel and content its `send_message` call ran with: the `ToolEvent` with the send's `call_id` under the same model event, by its executed arguments. A `msg.send` with no agent has a `post` intervention as parent with the same channel, sender, and content | The `msg.send` | | `delivery` | A `msg.deliver` names a recorded send, has its channel and sender, and carries its content, or the content of the last `before_deliver` rewrite (`action` `deliver`) of it for that recipient; and no `drop` for that recipient precedes it | The `msg.deliver` | @@ -300,8 +304,9 @@ content addresses, a retried upload is harmless *(proposed)*. Built for sandboxd's blobs: `swarmeval.sandbox.S3BlobStore` writes them, and the sandboxd client uploads them before `Exec` or `FinalDiff` returns. In a `ToolEvent`'s `metadata.swarmeval.exec`, -`stdout_truncated` / `stderr_truncated` name the blob holding the full output, and a change with -`content_stored` has its new content under `after_sha256`. +`stdout_truncated` / `stderr_truncated` name the blob holding the full output, a change with +`content_stored` has its new content under `after_sha256`, and a `collected` file with a +`sha256` has its content there. A tool's images (`images`) are collected files. ## Export @@ -309,7 +314,7 @@ At run end the worker writes to the export bucket, which is created with object | Object | Content | From | |---|---|---| -| `runs//sample.eval` | Standard Inspect log, readable by `inspect view`. `ModelEvent.input` expanded | M0 | +| `runs//sample.eval` | Standard Inspect log, readable by `inspect view`. `ModelEvent.input` expanded; a run's images embedded | M0 | | `runs//events.parquet` | One row per event: the indexed columns, `prev_hash` / `hash` as hex, and `payload` as JSON text. Written after the `.eval`, so only for runs that verified and ended `done` or `cancelled` | Built | | `summaries/.parquet` | One row per run: submission and its suite label, case and its hash, variant and `task_args` (JSON, sorted keys), epoch and the epochs requested per variant, `replaces` and `replaced_by` (an interrupted run and its [rerun](services/orchestrator.md#reruns)), status and error, isolation, times, and each scorer's last score. Written for every run that reaches a final status, copying the status `control.runs` holds: by the worker once it has finished a run (a cancel that lands meanwhile wins), by the Control API when it cancels a queued run, by a worker marking its old runs `interrupted` at start, and by a worker that took over a run whose lease ran out. If the worker cannot write one, a `done` run becomes `failed` and an `interrupted` (its rerun stays queued), `failed`, or `cancelled` run keeps its status; either way its error adds why. At start, the worker does not start. Reports read only these. A column added later reads as null in older summaries | Built | @@ -323,7 +328,13 @@ How a run becomes a `.eval` (`swarmeval.events.export_run`): 1. Read the run's `events` and `messages` rows, and verify the chain. Rows that do not verify raise `ChainError`, and nothing is written. 2. Rebuild each event from its payload. Add `prev_hash` / `hash` (hex) to `metadata.swarmeval`, - and expand `ModelEvent.input` from `messages`. + and expand `ModelEvent.input` from `messages`. A tool message with images becomes text plus + `ContentImage("attachment://")` for each image the request carried, and + `[image omitted]` for each `max_images` left out; a `ToolEvent` with `images` gets its + `result` as text plus the images inline as `data:` URLs, as Inspect writes tool results. Every + image is read from the blob store and checked against its hash first, and goes into + `EvalSample.attachments` once, as a `data:` URL. A run naming an image the export cannot read + is not exported. 3. Wrap each agent's events in a `SpanBeginEvent` / `SpanEndEvent` with `type="agent"` and id `agent:`, and set their `span_id`. Events with no agent stay at the top level. 4. Build one `EvalSample`: `id` is the case id, `uuid` the run id, `input` the case task. There is diff --git a/docs/repo-layout.md b/docs/repo-layout.md index 9b3b31c..3c72ae6 100644 --- a/docs/repo-layout.md +++ b/docs/repo-layout.md @@ -24,8 +24,10 @@ suites/ suites, one file each: `m1_core.yaml` go/ the Go module: `cmd/` (`sandboxd`, `edge`, `swarm`, `swarm-certs`), `internal/`. Its own README, CHANGELOG, BUGFIX console/ the web console: a Vite + React app, built into edge. Its own README, CHANGELOG proto/ gRPC contracts; buf.yaml and buf.gen.yaml sit at the repo root -deploy/ `images/` (one Dockerfile for the Python services, one for the Go binaries) and - `compose/` (the single-machine stack and its smoke test) +deploy/ `images/` (one Dockerfile for the Python services, one for the Go binaries, and + `display/`, the sandbox image the `browser` and `computer` tools run on, with + its `swarm-display` program) and `compose/` (the single-machine stack and its + smoke test) pyproject.toml uv.lock mise.toml ``` diff --git a/docs/services/model-gateway.md b/docs/services/model-gateway.md index 100fbf1..5d73618 100644 --- a/docs/services/model-gateway.md +++ b/docs/services/model-gateway.md @@ -41,7 +41,14 @@ Sandboxes cannot reach model-gateway, and neither can [net-gateway](net-gateway. - `POST /v1/chat/completions`, the subset of the OpenAI Chat Completions protocol SwarmEval uses (`swarmeval/gateway/model/wire.py`): messages, function tools, `temperature`, `top_p`, `max_completion_tokens`, `seed`. Any other field is refused, and so is `stream: true`. Past - reasoning travels as `reasoning_content` on assistant messages. + reasoning travels as `reasoning_content` on assistant messages. A tool message's `content` may + be a list of parts, `text` and `image_url`, for the images a tool returned; an image must be + an inline `data:image/png;base64,` URL, so the gateway fetches nothing. Chat Completions takes + only text in a tool message, so before sending upstream, and after reasoning passback, the + gateway turns each such message into its text and moves its images into one user message + after the run of tool messages, each image under `Images from tool call :`. Doing it after + passback keeps the added user message from moving the `within_turn` boundary. The worker sends + only the images the request's `max_images` keeps ([agent-runtime.md](../agent-runtime.md#images)). - `GET /v1/models`: the model names runs may use. The control plane checks every submission against it, and lists it to users ([orchestrator](orchestrator.md#models)). No virtual key. - Authentication is `Authorization: Bearer `. Each call also carries the caller's diff --git a/docs/services/orchestrator.md b/docs/services/orchestrator.md index 3a01285..f10683b 100644 --- a/docs/services/orchestrator.md +++ b/docs/services/orchestrator.md @@ -80,7 +80,7 @@ request. | `GetRun`, `ListRuns` | Status, variant and its values, epoch, the run it [reruns](#reruns), owner, isolation level, error, timestamps. `ListRuns` filters by submission, suite label, case, workspace, and status, newest first; the filters apply before the limit | Built. Fidelity arrives with recovery (M3) | | `CancelRun` | Marks cancelled. A queued run never starts; a running one stops at its owner's next hook point, is not scored, and is still exported. A finished run is `FAILED_PRECONDITION` | Built | | `ResumeRun` | A run paused for a person (a Monitor's `pause`) goes on from the hook point where it stopped. A run that is not paused is `FAILED_PRECONDITION` | Built | -| `ForkRun` | A new run that goes on from a finished run's state at the start of the turn an event happened in, with edits: replace a message's text, delete a user message, replace an undelivered message's content, or run a model slot on another model ([forks](#forks)). Returns the fork, `.f`, queued. Only a run that ended `done` or `cancelled` (an exported one) can be forked. An unknown run or event is `NOT_FOUND`; any other source run, an event before the first turn, a turn that began with extension background work (`ctx.spawn`) still running, a run recorded before forks were possible, or an edit that does not fit is `FAILED_PRECONDITION`; a malformed edit is `INVALID_ARGUMENT` | Built | +| `ForkRun` | A new run that goes on from a finished run's state at the start of the turn an event happened in, with edits: replace a message's text, delete a user message, replace an undelivered message's content, or run a model slot on another model ([forks](#forks)). Returns the fork, `.f`, queued. Only a run that ended `done` or `cancelled` (an exported one) can be forked. An unknown run or event is `NOT_FOUND`; any other source run, an event before the first turn, a turn that began with extension background work (`ctx.spawn`) still running, a turn after the run's first `browser` or `computer` call, a run recorded before forks were possible, or an edit that does not fit is `FAILED_PRECONDITION`; a malformed edit is `INVALID_ARGUMENT` | Built | | `SubmitSuite` | Takes a [suite](#suites) file and one bundle per `cases[].path` it names, loads the suite, and queues every case's runs in one transaction under one suite label; returns the label and one submission per entry. A suite that does not load, a path with no bundle, or a bundle the suite does not name is `INVALID_ARGUMENT`, and nothing is queued | Built | | `ListModels` | The model names runs may use: what model-gateway serves, sorted ([models](#models)). `UNAVAILABLE` when model-gateway is out of reach | Built | | `StreamEvents` | Server stream of a run's events after a given `seq`, live while it runs; ends once the run has finished and every event was sent. Each event comes with its stored payload and its `line`, the one-line text the judge reads (`swarmeval.events.render`), cut at 2,000 characters | Built | @@ -490,7 +490,9 @@ at_event_id, edits)` picks the start of the turn `at_event_id` happened in: the [checkpoint](../event-log.md#checkpoints) before the event. The source must have ended `done` or `cancelled`, so its export exists for `trace` to follow. A checkpoint taken while an extension's `ctx.spawn` work was still running is refused: a fork cannot carry a coroutine -over. The loop commits one at the start of +over. So is a checkpoint after a `browser` or `computer` call ran: a fork restores files, not a +screen or a browser's pages, tabs, and sign-ins +([display spec](../../spec/2026-10-07-browser-computer-use/README.md) decision 12). The loop commits one at the start of every run-wide turn, once the observers have caught up, so no agent is mid-step there. The control plane checks the edits against the checkpoint and the contexts it locates, and queues `.f` with the source's case revision, overrides, variant, and epoch, so the same seed, diff --git a/docs/services/sandboxd.md b/docs/services/sandboxd.md index e3e9e3a..432656d 100644 --- a/docs/services/sandboxd.md +++ b/docs/services/sandboxd.md @@ -40,8 +40,8 @@ authentication and refuses to listen on anything but a loopback address. | RPC | Does | State | |---|---|---| -| `CreateSandbox` | Creates one sandbox from a profile's image, key paths, limits, seed files, users, and identity (environment, hostname, machine id), labels it, and takes the initial manifest | Built | -| `Exec` | Runs one tool call and streams back its result, file changes, and surviving processes | Built | +| `CreateSandbox` | Creates one sandbox from a profile's image, key paths, limits, seed files, users, and identity (environment, hostname, machine id), labels it, starts its [display](#display) when it has one, and takes the initial manifest | Built | +| `Exec` | Runs one tool call and streams back its result, file changes, surviving processes, and the files it asked to [collect](#collected-files) | Built | | `ReadFile` | Reads a file for final-state scorers, through the engine's copy API, so nothing runs in the sandbox | Built | | `FsChange.mtime_ns` | Every reported change carries the path's modification time after it (before it for a delete), as the file system reports it; a process can set it to anything | Built | | `RestoreFiles` | For a [fork](orchestrator.md#forks): removes paths, creates directories, and writes files in a sandbox's key paths, each with its recorded mode and owner, then takes the manifest again, so none of it is ever reported as a change. Whatever is at a directory's or file's path and is of another kind (a symlink included) is removed first, so a write never follows a link. Every path must be strictly inside a key path; a path to remove that does not exist is skipped. An owner sandboxd cannot set, because it is not root, is reported in `unowned`. Only before the sandbox's first `Exec` (`FAILED_PRECONDITION` after it), and at most 3 MiB of content per request, so a client sends several (the worker keeps each under 2 MiB, paths included) | Built | @@ -125,6 +125,35 @@ from inside the sandbox (see [below](#exec-diff-and-process-snapshot)). The dock calls as its `os_user`. Under runc and runsc alike, files an `os_user` writes carry its uid on the host side, which `fs.*` events record, and one user cannot read another's `0600` files. +## Display + +`CreateSandbox` with `display` (width, height, first URL) starts the image's display stack: a +virtual screen with a web browser, which the `browser` and `computer` tools drive +([agent-runtime.md](../agent-runtime.md#tools)). The image must be built from +`swarmeval/display` ([deploy/images/display](../../deploy/images/display/README.md)). + +- After the container starts and before the first manifest, sandboxd runs `swarm-display start + --width W --height H [--url U]` as the user `swarmdisplay`, with a 60 s timeout. The command + returns once the screen and the browser are up and prints the display daemon's pid. A failure, + a timeout, or output that is not the pid of a live process fails `CreateSandbox`, and the + container is removed. +- sandboxd records that process's numeric uid. From then on every process with that uid is the + display's: left out of every process listing, so it is never reported as a `proc.*` event and + never makes a call's file changes `ambiguous`. Chromium starts renderers on each navigation, + and some of its helpers leave their parent for pid 1, so a pid subtree would not hold them. + An agent running as root could move a process of its own to that uid and hide it. +- The display keeps its state in `/run/swarm-display`, mode `0700`, owned by `swarmdisplay`. + Its home, temporary files, and browser profile are there, so it writes nothing into key paths. + A key path at `/`, `/run`, or under `/run/swarm-display` is `INVALID_ARGUMENT`. +- The screen needs a cookie only `swarmdisplay` can read, and the browser is driven through a + pipe, with no debugging port. An agent's own `os_user`, and root with no capabilities, reach + neither. Guard: `TestLiveDisplayDrivesTheBrowserAndKeepsItsProcessesOut`. +- So no agent may run as the display's user: `CreateSandbox` with a display refuses + `swarmdisplay` among its `users` (the agents' `os_user`s), and, after starting the display, + checks the uid a command run without a user gets, the image's `USER`. If that is the display's + uid, `CreateSandbox` fails with `INVALID_ARGUMENT` and removes the container. The loader + refuses `os_user: swarmdisplay` first. + ## Networks `--sandbox-network` picks how sandboxes are networked. @@ -189,6 +218,18 @@ Each `Exec` does the following: 5. List processes again. New processes still alive become `proc.*` events with pid, ppid, user, and command line. +### Collected files + +An `ExecRequest` may name `collect` paths: absolute, clean, and outside every key path. After the +command, timed out or not, sandboxd reads each regular file there and removes it, as the call's +user, with a short `sh` script inside the sandbox; a symlink is removed without being followed. +Root cannot do it: with every capability dropped it cannot open another user's `0700` directory, +and what a call may collect is what its user may read anyway. The header's `collected` lists one +entry per path, in order: `missing` when there was no regular file, else its size and sha256, +with the content following as a blob. A file over 8 MiB is removed and reported by size only. +The `browser` and `computer` tools collect `/run/swarm-display/out/screenshot.png` this way, so a +screenshot never lands in a key path and is never a file change. + The response is a stream: a header with exit code, duration, and events, then blob chunks. stdout and stderr are capped in the header, and the full output follows as a blob. Changes between the previous call and this one come back as `background_changes`. While a process an @@ -236,7 +277,7 @@ it is not built in M0–M5. 3. Where each `os_user` may write. Today it is whatever the image lays out; a profile setting for key path owners may be needed once a case shares a writable workspace between users. 4. Size caps. Today's defaults: 64 KiB of stdout and stderr inline, 16 MiB kept per stream, file - contents sent back up to 1 MiB each and 64 MiB per call. They are constants in - `sandboxd.DefaultConfig`, not flags yet. + contents sent back up to 1 MiB each and 64 MiB per call, 8 MiB per collected file, and 60 s + to start a display. They are constants in `sandboxd.DefaultConfig`, not flags yet. 5. `machine_id` as a field of `CreateSandbox` of its own, for the sandbox canary's file, rather than seed files allowed outside the key paths. diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index b1c6c6e..0fd1193 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -2,6 +2,17 @@ ## [Unreleased] +### Added +- sandboxd starts a sandbox's display: `CreateSandboxRequest.display` (width, height, first URL) + runs the display image's `swarm-display start` as `swarmdisplay` before the first manifest + (`spec/2026-10-07-browser-computer-use`). That user's processes are left out of every process + listing, so a browser's renderers are neither reported nor make a call's changes ambiguous. A + key path overlapping `/run/swarm-display` is refused, and so is `swarmdisplay` as an agent's + user or as the image's `USER`. +- `ExecRequest.collect`: after the command, sandboxd reads and removes files outside the key + paths as the call's user and returns them as blobs (`ExecHeader.collected`), up to 8 MiB each. +- `driver.Process.UID`: the numeric uid beside the user's name. + ### Changed - `swarm run` takes `-m` for the models to run on, required for a case: `-m MODEL[,MODEL…]` for the `default` model slot, `-m SLOT=MODEL[,MODEL…]` for a named one diff --git a/go/README.md b/go/README.md index df6bd98..da25966 100644 --- a/go/README.md +++ b/go/README.md @@ -29,13 +29,15 @@ Docker is found through the standard `DOCKER_HOST` / `DOCKER_*` environment. Requirements and limits: - Sandbox images must provide `sleep`, `tr`, and `/bin/sh`, `/etc/passwd` when the sandbox - has users, and `/etc` when it has a machine id. sandboxd never pulls; pull images on the - docker host first. + has users, and `/etc` when it has a machine id. A sandbox with a display needs an image built + from `swarmeval/display` (`deploy/images/display`). sandboxd never pulls; pull or build images + on the docker host first. - In production sandboxd runs as root, so extracted key paths keep their owners. - Sandboxes have no network but loopback by default. Agents reach the internet only through the worker's `web_request`. - Output and content caps are in `sandboxd.DefaultConfig`: 64 KiB of stdout and stderr inline, - 16 MiB kept per stream, changed files sent back up to 1 MiB each and 64 MiB per call. + 16 MiB kept per stream, changed files sent back up to 1 MiB each and 64 MiB per call, 8 MiB + per collected file, and 60 s to start a display. - State is in memory. A restarted sandboxd does not know the sandboxes it created; `DestroyRun` still removes their containers and networks by label. @@ -117,6 +119,8 @@ hosts once the certificates are issued. From the repo root: `mise run check` lints and tests this module with everything else. `mise run go:test-integration` runs sandboxd against the local docker daemon and needs `busybox:latest` and `python:3.12-slim` on the host; it also runs edge against a throwaway -`postgres:18-alpine`. `SWARMEVAL_IT_RUNTIME=runc` or `runsc` picks the runtime. On macOS, +`postgres:18-alpine`. The display test also needs `swarmeval/display:dev` +(`SWARMEVAL_IT_DISPLAY_IMAGE` names another), built from `deploy/images/display`, and skips +without it. `SWARMEVAL_IT_RUNTIME=runc` or `runsc` picks the runtime. On macOS, Docker Desktop and OrbStack enforce neither owners nor modes on bind mounts, so the `os_user` permission test skips there; run it on Linux. diff --git a/go/internal/driver/driver.go b/go/internal/driver/driver.go index 29df692..654cf74 100644 --- a/go/internal/driver/driver.go +++ b/go/internal/driver/driver.go @@ -95,8 +95,11 @@ type ExecSpec struct { // Process is one process in a container, with pids as the container sees them. type Process struct { - PID int32 - PPID int32 + PID int32 + PPID int32 + // UID is the numeric uid; User is its name from the container's /etc/passwd, or the uid + // when it has none. + UID string User string Cmdline string } diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go b/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go index c260a89..6ff2d95 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandbox.pb.go @@ -71,7 +71,7 @@ func (x FsChange_Op) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Op.Descriptor instead. func (FsChange_Op) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 0} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 0} } type FsChange_Kind int32 @@ -127,7 +127,7 @@ func (x FsChange_Kind) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Kind.Descriptor instead. func (FsChange_Kind) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 1} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 1} } type FsChange_Attribution int32 @@ -178,7 +178,7 @@ func (x FsChange_Attribution) Number() protoreflect.EnumNumber { // Deprecated: Use FsChange_Attribution.Descriptor instead. func (FsChange_Attribution) EnumDescriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11, 2} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13, 2} } type CreateRunRequest struct { @@ -427,7 +427,10 @@ type CreateSandboxRequest struct { // Written to /etc/machine-id before the sandbox starts, outside every key path, so it is never // diffed. Empty writes nothing. 32 lowercase hex characters, the file's format. The image // must have /etc. - MachineId string `protobuf:"bytes,10,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + MachineId string `protobuf:"bytes,10,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + // Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + // no display. + Display *Display `protobuf:"bytes,11,opt,name=display,proto3" json:"display,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -532,6 +535,77 @@ func (x *CreateSandboxRequest) GetMachineId() string { return "" } +func (x *CreateSandboxRequest) GetDisplay() *Display { + if x != nil { + return x.Display + } + return nil +} + +// A virtual screen with a browser on it, run by the image's `swarm-display` as the user +// `swarmdisplay`. Its processes are never reported or attributed to a call. +type Display struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Pixels, 320 to 1920 wide and 240 to 1200 high. + Width uint32 `protobuf:"varint,1,opt,name=width,proto3" json:"width,omitempty"` + Height uint32 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` + // Page the browser opens first. Empty opens about:blank. + Url string `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Display) Reset() { + *x = Display{} + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Display) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Display) ProtoMessage() {} + +func (x *Display) ProtoReflect() protoreflect.Message { + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Display.ProtoReflect.Descriptor instead. +func (*Display) Descriptor() ([]byte, []int) { + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{5} +} + +func (x *Display) GetWidth() uint32 { + if x != nil { + return x.Width + } + return 0 +} + +func (x *Display) GetHeight() uint32 { + if x != nil { + return x.Height + } + return 0 +} + +func (x *Display) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + type SeedFile struct { state protoimpl.MessageState `protogen:"open.v1"` // Absolute and clean, strictly inside a key path. @@ -545,7 +619,7 @@ type SeedFile struct { func (x *SeedFile) Reset() { *x = SeedFile{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -557,7 +631,7 @@ func (x *SeedFile) String() string { func (*SeedFile) ProtoMessage() {} func (x *SeedFile) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[5] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -570,7 +644,7 @@ func (x *SeedFile) ProtoReflect() protoreflect.Message { // Deprecated: Use SeedFile.ProtoReflect.Descriptor instead. func (*SeedFile) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{5} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{6} } func (x *SeedFile) GetPath() string { @@ -604,7 +678,7 @@ type CreateSandboxResponse struct { func (x *CreateSandboxResponse) Reset() { *x = CreateSandboxResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -616,7 +690,7 @@ func (x *CreateSandboxResponse) String() string { func (*CreateSandboxResponse) ProtoMessage() {} func (x *CreateSandboxResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[6] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -629,7 +703,7 @@ func (x *CreateSandboxResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSandboxResponse.ProtoReflect.Descriptor instead. func (*CreateSandboxResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{6} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{7} } func (x *CreateSandboxResponse) GetRuntime() string { @@ -650,14 +724,18 @@ type ExecRequest struct { // Empty runs as the image's user. User string `protobuf:"bytes,6,opt,name=user,proto3" json:"user,omitempty"` // Required. On expiry the command and its descendants are killed. - Timeout *durationpb.Duration `protobuf:"bytes,7,opt,name=timeout,proto3" json:"timeout,omitempty"` + Timeout *durationpb.Duration `protobuf:"bytes,7,opt,name=timeout,proto3" json:"timeout,omitempty"` + // Absolute paths outside every key path. After the command, sandboxd reads each regular file + // there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + // text, such as a screenshot. + Collect []string `protobuf:"bytes,8,rep,name=collect,proto3" json:"collect,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ExecRequest) Reset() { *x = ExecRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -669,7 +747,7 @@ func (x *ExecRequest) String() string { func (*ExecRequest) ProtoMessage() {} func (x *ExecRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[7] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -682,7 +760,7 @@ func (x *ExecRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecRequest.ProtoReflect.Descriptor instead. func (*ExecRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{7} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{8} } func (x *ExecRequest) GetRunId() string { @@ -734,6 +812,13 @@ func (x *ExecRequest) GetTimeout() *durationpb.Duration { return nil } +func (x *ExecRequest) GetCollect() []string { + if x != nil { + return x.Collect + } + return nil +} + type ExecResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Item: @@ -747,7 +832,7 @@ type ExecResponse struct { func (x *ExecResponse) Reset() { *x = ExecResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -759,7 +844,7 @@ func (x *ExecResponse) String() string { func (*ExecResponse) ProtoMessage() {} func (x *ExecResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -772,7 +857,7 @@ func (x *ExecResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecResponse.ProtoReflect.Descriptor instead. func (*ExecResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{8} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{9} } func (x *ExecResponse) GetItem() isExecResponse_Item { @@ -830,14 +915,16 @@ type ExecHeader struct { // Changed while this call ran. Changes []*FsChange `protobuf:"bytes,7,rep,name=changes,proto3" json:"changes,omitempty"` // Started while this call ran and still alive after it. - Processes []*Process `protobuf:"bytes,8,rep,name=processes,proto3" json:"processes,omitempty"` + Processes []*Process `protobuf:"bytes,8,rep,name=processes,proto3" json:"processes,omitempty"` + // One per `collect` path, in request order. + Collected []*CollectedFile `protobuf:"bytes,9,rep,name=collected,proto3" json:"collected,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ExecHeader) Reset() { *x = ExecHeader{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -849,7 +936,7 @@ func (x *ExecHeader) String() string { func (*ExecHeader) ProtoMessage() {} func (x *ExecHeader) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -862,7 +949,7 @@ func (x *ExecHeader) ProtoReflect() protoreflect.Message { // Deprecated: Use ExecHeader.ProtoReflect.Descriptor instead. func (*ExecHeader) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{9} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{10} } func (x *ExecHeader) GetExitCode() int32 { @@ -921,6 +1008,85 @@ func (x *ExecHeader) GetProcesses() []*Process { return nil } +func (x *ExecHeader) GetCollected() []*CollectedFile { + if x != nil { + return x.Collected + } + return nil +} + +type CollectedFile struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // No regular file was there. + Missing bool `protobuf:"varint,2,opt,name=missing,proto3" json:"missing,omitempty"` + // Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + // `sha256` is empty. + Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + // The content follows as a blob with this hash. + Sha256 string `protobuf:"bytes,4,opt,name=sha256,proto3" json:"sha256,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CollectedFile) Reset() { + *x = CollectedFile{} + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CollectedFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CollectedFile) ProtoMessage() {} + +func (x *CollectedFile) ProtoReflect() protoreflect.Message { + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CollectedFile.ProtoReflect.Descriptor instead. +func (*CollectedFile) Descriptor() ([]byte, []int) { + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11} +} + +func (x *CollectedFile) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *CollectedFile) GetMissing() bool { + if x != nil { + return x.Missing + } + return false +} + +func (x *CollectedFile) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *CollectedFile) GetSha256() string { + if x != nil { + return x.Sha256 + } + return "" +} + type Output struct { state protoimpl.MessageState `protogen:"open.v1"` // The first bytes, up to sandboxd's inline limit. @@ -938,7 +1104,7 @@ type Output struct { func (x *Output) Reset() { *x = Output{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -950,7 +1116,7 @@ func (x *Output) String() string { func (*Output) ProtoMessage() {} func (x *Output) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[10] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -963,7 +1129,7 @@ func (x *Output) ProtoReflect() protoreflect.Message { // Deprecated: Use Output.ProtoReflect.Descriptor instead. func (*Output) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{10} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{12} } func (x *Output) GetInline() []byte { @@ -1024,7 +1190,7 @@ type FsChange struct { func (x *FsChange) Reset() { *x = FsChange{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1036,7 +1202,7 @@ func (x *FsChange) String() string { func (*FsChange) ProtoMessage() {} func (x *FsChange) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[11] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1049,7 +1215,7 @@ func (x *FsChange) ProtoReflect() protoreflect.Message { // Deprecated: Use FsChange.ProtoReflect.Descriptor instead. func (*FsChange) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{11} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13} } func (x *FsChange) GetPath() string { @@ -1157,7 +1323,7 @@ type Process struct { func (x *Process) Reset() { *x = Process{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1169,7 +1335,7 @@ func (x *Process) String() string { func (*Process) ProtoMessage() {} func (x *Process) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[12] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1182,7 +1348,7 @@ func (x *Process) ProtoReflect() protoreflect.Message { // Deprecated: Use Process.ProtoReflect.Descriptor instead. func (*Process) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{12} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{14} } func (x *Process) GetPid() int32 { @@ -1224,7 +1390,7 @@ type BlobChunk struct { func (x *BlobChunk) Reset() { *x = BlobChunk{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1236,7 +1402,7 @@ func (x *BlobChunk) String() string { func (*BlobChunk) ProtoMessage() {} func (x *BlobChunk) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[13] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1249,7 +1415,7 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead. func (*BlobChunk) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{15} } func (x *BlobChunk) GetSha256() string { @@ -1287,7 +1453,7 @@ type RestoreFilesRequest struct { func (x *RestoreFilesRequest) Reset() { *x = RestoreFilesRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1299,7 +1465,7 @@ func (x *RestoreFilesRequest) String() string { func (*RestoreFilesRequest) ProtoMessage() {} func (x *RestoreFilesRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[14] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1478,7 @@ func (x *RestoreFilesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFilesRequest.ProtoReflect.Descriptor instead. func (*RestoreFilesRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{14} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{16} } func (x *RestoreFilesRequest) GetRunId() string { @@ -1365,7 +1531,7 @@ type RestoreDir struct { func (x *RestoreDir) Reset() { *x = RestoreDir{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1377,7 +1543,7 @@ func (x *RestoreDir) String() string { func (*RestoreDir) ProtoMessage() {} func (x *RestoreDir) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[15] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1390,7 +1556,7 @@ func (x *RestoreDir) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreDir.ProtoReflect.Descriptor instead. func (*RestoreDir) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{15} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{17} } func (x *RestoreDir) GetPath() string { @@ -1430,7 +1596,7 @@ type RestoreFile struct { func (x *RestoreFile) Reset() { *x = RestoreFile{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1442,7 +1608,7 @@ func (x *RestoreFile) String() string { func (*RestoreFile) ProtoMessage() {} func (x *RestoreFile) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[16] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1455,7 +1621,7 @@ func (x *RestoreFile) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFile.ProtoReflect.Descriptor instead. func (*RestoreFile) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{16} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{18} } func (x *RestoreFile) GetPath() string { @@ -1497,7 +1663,7 @@ type RestoreFilesResponse struct { func (x *RestoreFilesResponse) Reset() { *x = RestoreFilesResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1509,7 +1675,7 @@ func (x *RestoreFilesResponse) String() string { func (*RestoreFilesResponse) ProtoMessage() {} func (x *RestoreFilesResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[17] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1522,7 +1688,7 @@ func (x *RestoreFilesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreFilesResponse.ProtoReflect.Descriptor instead. func (*RestoreFilesResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{17} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{19} } func (x *RestoreFilesResponse) GetUnowned() []string { @@ -1545,7 +1711,7 @@ type ReadFileRequest struct { func (x *ReadFileRequest) Reset() { *x = ReadFileRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1557,7 +1723,7 @@ func (x *ReadFileRequest) String() string { func (*ReadFileRequest) ProtoMessage() {} func (x *ReadFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[18] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1570,7 +1736,7 @@ func (x *ReadFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadFileRequest.ProtoReflect.Descriptor instead. func (*ReadFileRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{18} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{20} } func (x *ReadFileRequest) GetRunId() string { @@ -1612,7 +1778,7 @@ type ReadFileResponse struct { func (x *ReadFileResponse) Reset() { *x = ReadFileResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1624,7 +1790,7 @@ func (x *ReadFileResponse) String() string { func (*ReadFileResponse) ProtoMessage() {} func (x *ReadFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[19] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1637,7 +1803,7 @@ func (x *ReadFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadFileResponse.ProtoReflect.Descriptor instead. func (*ReadFileResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{19} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{21} } func (x *ReadFileResponse) GetContent() []byte { @@ -1670,7 +1836,7 @@ type FinalDiffRequest struct { func (x *FinalDiffRequest) Reset() { *x = FinalDiffRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1682,7 +1848,7 @@ func (x *FinalDiffRequest) String() string { func (*FinalDiffRequest) ProtoMessage() {} func (x *FinalDiffRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[20] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1695,7 +1861,7 @@ func (x *FinalDiffRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalDiffRequest.ProtoReflect.Descriptor instead. func (*FinalDiffRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{20} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{22} } func (x *FinalDiffRequest) GetRunId() string { @@ -1718,7 +1884,7 @@ type FinalDiffResponse struct { func (x *FinalDiffResponse) Reset() { *x = FinalDiffResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1730,7 +1896,7 @@ func (x *FinalDiffResponse) String() string { func (*FinalDiffResponse) ProtoMessage() {} func (x *FinalDiffResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1743,7 +1909,7 @@ func (x *FinalDiffResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FinalDiffResponse.ProtoReflect.Descriptor instead. func (*FinalDiffResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{21} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{23} } func (x *FinalDiffResponse) GetItem() isFinalDiffResponse_Item { @@ -1797,7 +1963,7 @@ type SandboxChanges struct { func (x *SandboxChanges) Reset() { *x = SandboxChanges{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1809,7 +1975,7 @@ func (x *SandboxChanges) String() string { func (*SandboxChanges) ProtoMessage() {} func (x *SandboxChanges) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[22] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1822,7 +1988,7 @@ func (x *SandboxChanges) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxChanges.ProtoReflect.Descriptor instead. func (*SandboxChanges) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{22} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{24} } func (x *SandboxChanges) GetSandboxId() string { @@ -1848,7 +2014,7 @@ type DestroyRunRequest struct { func (x *DestroyRunRequest) Reset() { *x = DestroyRunRequest{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1860,7 +2026,7 @@ func (x *DestroyRunRequest) String() string { func (*DestroyRunRequest) ProtoMessage() {} func (x *DestroyRunRequest) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1873,7 +2039,7 @@ func (x *DestroyRunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyRunRequest.ProtoReflect.Descriptor instead. func (*DestroyRunRequest) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{23} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{25} } func (x *DestroyRunRequest) GetRunId() string { @@ -1891,7 +2057,7 @@ type DestroyRunResponse struct { func (x *DestroyRunResponse) Reset() { *x = DestroyRunResponse{} - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1903,7 +2069,7 @@ func (x *DestroyRunResponse) String() string { func (*DestroyRunResponse) ProtoMessage() {} func (x *DestroyRunResponse) ProtoReflect() protoreflect.Message { - mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[24] + mi := &file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1916,7 +2082,7 @@ func (x *DestroyRunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyRunResponse.ProtoReflect.Descriptor instead. func (*DestroyRunResponse) Descriptor() ([]byte, []int) { - return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{24} + return file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{26} } var File_swarmeval_sandbox_v1_sandbox_proto protoreflect.FileDescriptor @@ -1938,7 +2104,7 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\x12\x12\n" + "\x04pids\x18\x03 \x01(\x03R\x04pids\x12\x1d\n" + "\n" + - "disk_bytes\x18\x04 \x01(\x03R\tdiskBytes\"\xdc\x03\n" + + "disk_bytes\x18\x04 \x01(\x03R\tdiskBytes\"\x95\x04\n" + "\x14CreateSandboxRequest\x12\x15\n" + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x1d\n" + "\n" + @@ -1952,16 +2118,21 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\bhostname\x18\t \x01(\tR\bhostname\x12\x1d\n" + "\n" + "machine_id\x18\n" + - " \x01(\tR\tmachineId\x1a6\n" + + " \x01(\tR\tmachineId\x127\n" + + "\adisplay\x18\v \x01(\v2\x1d.swarmeval.sandbox.v1.DisplayR\adisplay\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"L\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"I\n" + + "\aDisplay\x12\x14\n" + + "\x05width\x18\x01 \x01(\rR\x05width\x12\x16\n" + + "\x06height\x18\x02 \x01(\rR\x06height\x12\x10\n" + + "\x03url\x18\x03 \x01(\tR\x03url\"L\n" + "\bSeedFile\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x18\n" + "\acontent\x18\x02 \x01(\fR\acontent\x12\x12\n" + "\x04mode\x18\x03 \x01(\rR\x04mode\"1\n" + "\x15CreateSandboxResponse\x12\x18\n" + - "\aruntime\x18\x01 \x01(\tR\aruntime\"\xcb\x01\n" + + "\aruntime\x18\x01 \x01(\tR\aruntime\"\xe5\x01\n" + "\vExecRequest\x12\x15\n" + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12\x1d\n" + "\n" + @@ -1970,11 +2141,12 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\x04argv\x18\x04 \x03(\tR\x04argv\x12\x10\n" + "\x03cwd\x18\x05 \x01(\tR\x03cwd\x12\x12\n" + "\x04user\x18\x06 \x01(\tR\x04user\x123\n" + - "\atimeout\x18\a \x01(\v2\x19.google.protobuf.DurationR\atimeout\"\x89\x01\n" + + "\atimeout\x18\a \x01(\v2\x19.google.protobuf.DurationR\atimeout\x12\x18\n" + + "\acollect\x18\b \x03(\tR\acollect\"\x89\x01\n" + "\fExecResponse\x12:\n" + "\x06header\x18\x01 \x01(\v2 .swarmeval.sandbox.v1.ExecHeaderH\x00R\x06header\x125\n" + "\x04blob\x18\x02 \x01(\v2\x1f.swarmeval.sandbox.v1.BlobChunkH\x00R\x04blobB\x06\n" + - "\x04item\"\xaf\x03\n" + + "\x04item\"\xf2\x03\n" + "\n" + "ExecHeader\x12\x1b\n" + "\texit_code\x18\x01 \x01(\x05R\bexitCode\x12\x1b\n" + @@ -1984,7 +2156,13 @@ const file_swarmeval_sandbox_v1_sandbox_proto_rawDesc = "" + "\x06stderr\x18\x05 \x01(\v2\x1c.swarmeval.sandbox.v1.OutputR\x06stderr\x12M\n" + "\x12background_changes\x18\x06 \x03(\v2\x1e.swarmeval.sandbox.v1.FsChangeR\x11backgroundChanges\x128\n" + "\achanges\x18\a \x03(\v2\x1e.swarmeval.sandbox.v1.FsChangeR\achanges\x12;\n" + - "\tprocesses\x18\b \x03(\v2\x1d.swarmeval.sandbox.v1.ProcessR\tprocesses\"m\n" + + "\tprocesses\x18\b \x03(\v2\x1d.swarmeval.sandbox.v1.ProcessR\tprocesses\x12A\n" + + "\tcollected\x18\t \x03(\v2#.swarmeval.sandbox.v1.CollectedFileR\tcollected\"i\n" + + "\rCollectedFile\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x18\n" + + "\amissing\x18\x02 \x01(\bR\amissing\x12\x12\n" + + "\x04size\x18\x03 \x01(\x03R\x04size\x12\x16\n" + + "\x06sha256\x18\x04 \x01(\tR\x06sha256\"m\n" + "\x06Output\x12\x16\n" + "\x06inline\x18\x01 \x01(\fR\x06inline\x12\x12\n" + "\x04size\x18\x02 \x01(\x03R\x04size\x12\x1f\n" + @@ -2096,7 +2274,7 @@ func file_swarmeval_sandbox_v1_sandbox_proto_rawDescGZIP() []byte { } var file_swarmeval_sandbox_v1_sandbox_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_swarmeval_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_swarmeval_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_swarmeval_sandbox_v1_sandbox_proto_goTypes = []any{ (FsChange_Op)(0), // 0: swarmeval.sandbox.v1.FsChange.Op (FsChange_Kind)(0), // 1: swarmeval.sandbox.v1.FsChange.Kind @@ -2106,70 +2284,74 @@ var file_swarmeval_sandbox_v1_sandbox_proto_goTypes = []any{ (*Mount)(nil), // 5: swarmeval.sandbox.v1.Mount (*Resources)(nil), // 6: swarmeval.sandbox.v1.Resources (*CreateSandboxRequest)(nil), // 7: swarmeval.sandbox.v1.CreateSandboxRequest - (*SeedFile)(nil), // 8: swarmeval.sandbox.v1.SeedFile - (*CreateSandboxResponse)(nil), // 9: swarmeval.sandbox.v1.CreateSandboxResponse - (*ExecRequest)(nil), // 10: swarmeval.sandbox.v1.ExecRequest - (*ExecResponse)(nil), // 11: swarmeval.sandbox.v1.ExecResponse - (*ExecHeader)(nil), // 12: swarmeval.sandbox.v1.ExecHeader - (*Output)(nil), // 13: swarmeval.sandbox.v1.Output - (*FsChange)(nil), // 14: swarmeval.sandbox.v1.FsChange - (*Process)(nil), // 15: swarmeval.sandbox.v1.Process - (*BlobChunk)(nil), // 16: swarmeval.sandbox.v1.BlobChunk - (*RestoreFilesRequest)(nil), // 17: swarmeval.sandbox.v1.RestoreFilesRequest - (*RestoreDir)(nil), // 18: swarmeval.sandbox.v1.RestoreDir - (*RestoreFile)(nil), // 19: swarmeval.sandbox.v1.RestoreFile - (*RestoreFilesResponse)(nil), // 20: swarmeval.sandbox.v1.RestoreFilesResponse - (*ReadFileRequest)(nil), // 21: swarmeval.sandbox.v1.ReadFileRequest - (*ReadFileResponse)(nil), // 22: swarmeval.sandbox.v1.ReadFileResponse - (*FinalDiffRequest)(nil), // 23: swarmeval.sandbox.v1.FinalDiffRequest - (*FinalDiffResponse)(nil), // 24: swarmeval.sandbox.v1.FinalDiffResponse - (*SandboxChanges)(nil), // 25: swarmeval.sandbox.v1.SandboxChanges - (*DestroyRunRequest)(nil), // 26: swarmeval.sandbox.v1.DestroyRunRequest - (*DestroyRunResponse)(nil), // 27: swarmeval.sandbox.v1.DestroyRunResponse - nil, // 28: swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry - (*durationpb.Duration)(nil), // 29: google.protobuf.Duration + (*Display)(nil), // 8: swarmeval.sandbox.v1.Display + (*SeedFile)(nil), // 9: swarmeval.sandbox.v1.SeedFile + (*CreateSandboxResponse)(nil), // 10: swarmeval.sandbox.v1.CreateSandboxResponse + (*ExecRequest)(nil), // 11: swarmeval.sandbox.v1.ExecRequest + (*ExecResponse)(nil), // 12: swarmeval.sandbox.v1.ExecResponse + (*ExecHeader)(nil), // 13: swarmeval.sandbox.v1.ExecHeader + (*CollectedFile)(nil), // 14: swarmeval.sandbox.v1.CollectedFile + (*Output)(nil), // 15: swarmeval.sandbox.v1.Output + (*FsChange)(nil), // 16: swarmeval.sandbox.v1.FsChange + (*Process)(nil), // 17: swarmeval.sandbox.v1.Process + (*BlobChunk)(nil), // 18: swarmeval.sandbox.v1.BlobChunk + (*RestoreFilesRequest)(nil), // 19: swarmeval.sandbox.v1.RestoreFilesRequest + (*RestoreDir)(nil), // 20: swarmeval.sandbox.v1.RestoreDir + (*RestoreFile)(nil), // 21: swarmeval.sandbox.v1.RestoreFile + (*RestoreFilesResponse)(nil), // 22: swarmeval.sandbox.v1.RestoreFilesResponse + (*ReadFileRequest)(nil), // 23: swarmeval.sandbox.v1.ReadFileRequest + (*ReadFileResponse)(nil), // 24: swarmeval.sandbox.v1.ReadFileResponse + (*FinalDiffRequest)(nil), // 25: swarmeval.sandbox.v1.FinalDiffRequest + (*FinalDiffResponse)(nil), // 26: swarmeval.sandbox.v1.FinalDiffResponse + (*SandboxChanges)(nil), // 27: swarmeval.sandbox.v1.SandboxChanges + (*DestroyRunRequest)(nil), // 28: swarmeval.sandbox.v1.DestroyRunRequest + (*DestroyRunResponse)(nil), // 29: swarmeval.sandbox.v1.DestroyRunResponse + nil, // 30: swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry + (*durationpb.Duration)(nil), // 31: google.protobuf.Duration } var file_swarmeval_sandbox_v1_sandbox_proto_depIdxs = []int32{ 5, // 0: swarmeval.sandbox.v1.CreateSandboxRequest.mounts:type_name -> swarmeval.sandbox.v1.Mount 6, // 1: swarmeval.sandbox.v1.CreateSandboxRequest.resources:type_name -> swarmeval.sandbox.v1.Resources - 8, // 2: swarmeval.sandbox.v1.CreateSandboxRequest.files:type_name -> swarmeval.sandbox.v1.SeedFile - 28, // 3: swarmeval.sandbox.v1.CreateSandboxRequest.env:type_name -> swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry - 29, // 4: swarmeval.sandbox.v1.ExecRequest.timeout:type_name -> google.protobuf.Duration - 12, // 5: swarmeval.sandbox.v1.ExecResponse.header:type_name -> swarmeval.sandbox.v1.ExecHeader - 16, // 6: swarmeval.sandbox.v1.ExecResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk - 29, // 7: swarmeval.sandbox.v1.ExecHeader.duration:type_name -> google.protobuf.Duration - 13, // 8: swarmeval.sandbox.v1.ExecHeader.stdout:type_name -> swarmeval.sandbox.v1.Output - 13, // 9: swarmeval.sandbox.v1.ExecHeader.stderr:type_name -> swarmeval.sandbox.v1.Output - 14, // 10: swarmeval.sandbox.v1.ExecHeader.background_changes:type_name -> swarmeval.sandbox.v1.FsChange - 14, // 11: swarmeval.sandbox.v1.ExecHeader.changes:type_name -> swarmeval.sandbox.v1.FsChange - 15, // 12: swarmeval.sandbox.v1.ExecHeader.processes:type_name -> swarmeval.sandbox.v1.Process - 0, // 13: swarmeval.sandbox.v1.FsChange.op:type_name -> swarmeval.sandbox.v1.FsChange.Op - 1, // 14: swarmeval.sandbox.v1.FsChange.kind:type_name -> swarmeval.sandbox.v1.FsChange.Kind - 2, // 15: swarmeval.sandbox.v1.FsChange.attribution:type_name -> swarmeval.sandbox.v1.FsChange.Attribution - 18, // 16: swarmeval.sandbox.v1.RestoreFilesRequest.dirs:type_name -> swarmeval.sandbox.v1.RestoreDir - 19, // 17: swarmeval.sandbox.v1.RestoreFilesRequest.files:type_name -> swarmeval.sandbox.v1.RestoreFile - 25, // 18: swarmeval.sandbox.v1.FinalDiffResponse.changes:type_name -> swarmeval.sandbox.v1.SandboxChanges - 16, // 19: swarmeval.sandbox.v1.FinalDiffResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk - 14, // 20: swarmeval.sandbox.v1.SandboxChanges.changes:type_name -> swarmeval.sandbox.v1.FsChange - 3, // 21: swarmeval.sandbox.v1.SandboxService.CreateRun:input_type -> swarmeval.sandbox.v1.CreateRunRequest - 7, // 22: swarmeval.sandbox.v1.SandboxService.CreateSandbox:input_type -> swarmeval.sandbox.v1.CreateSandboxRequest - 10, // 23: swarmeval.sandbox.v1.SandboxService.Exec:input_type -> swarmeval.sandbox.v1.ExecRequest - 21, // 24: swarmeval.sandbox.v1.SandboxService.ReadFile:input_type -> swarmeval.sandbox.v1.ReadFileRequest - 17, // 25: swarmeval.sandbox.v1.SandboxService.RestoreFiles:input_type -> swarmeval.sandbox.v1.RestoreFilesRequest - 23, // 26: swarmeval.sandbox.v1.SandboxService.FinalDiff:input_type -> swarmeval.sandbox.v1.FinalDiffRequest - 26, // 27: swarmeval.sandbox.v1.SandboxService.DestroyRun:input_type -> swarmeval.sandbox.v1.DestroyRunRequest - 4, // 28: swarmeval.sandbox.v1.SandboxService.CreateRun:output_type -> swarmeval.sandbox.v1.CreateRunResponse - 9, // 29: swarmeval.sandbox.v1.SandboxService.CreateSandbox:output_type -> swarmeval.sandbox.v1.CreateSandboxResponse - 11, // 30: swarmeval.sandbox.v1.SandboxService.Exec:output_type -> swarmeval.sandbox.v1.ExecResponse - 22, // 31: swarmeval.sandbox.v1.SandboxService.ReadFile:output_type -> swarmeval.sandbox.v1.ReadFileResponse - 20, // 32: swarmeval.sandbox.v1.SandboxService.RestoreFiles:output_type -> swarmeval.sandbox.v1.RestoreFilesResponse - 24, // 33: swarmeval.sandbox.v1.SandboxService.FinalDiff:output_type -> swarmeval.sandbox.v1.FinalDiffResponse - 27, // 34: swarmeval.sandbox.v1.SandboxService.DestroyRun:output_type -> swarmeval.sandbox.v1.DestroyRunResponse - 28, // [28:35] is the sub-list for method output_type - 21, // [21:28] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 9, // 2: swarmeval.sandbox.v1.CreateSandboxRequest.files:type_name -> swarmeval.sandbox.v1.SeedFile + 30, // 3: swarmeval.sandbox.v1.CreateSandboxRequest.env:type_name -> swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry + 8, // 4: swarmeval.sandbox.v1.CreateSandboxRequest.display:type_name -> swarmeval.sandbox.v1.Display + 31, // 5: swarmeval.sandbox.v1.ExecRequest.timeout:type_name -> google.protobuf.Duration + 13, // 6: swarmeval.sandbox.v1.ExecResponse.header:type_name -> swarmeval.sandbox.v1.ExecHeader + 18, // 7: swarmeval.sandbox.v1.ExecResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk + 31, // 8: swarmeval.sandbox.v1.ExecHeader.duration:type_name -> google.protobuf.Duration + 15, // 9: swarmeval.sandbox.v1.ExecHeader.stdout:type_name -> swarmeval.sandbox.v1.Output + 15, // 10: swarmeval.sandbox.v1.ExecHeader.stderr:type_name -> swarmeval.sandbox.v1.Output + 16, // 11: swarmeval.sandbox.v1.ExecHeader.background_changes:type_name -> swarmeval.sandbox.v1.FsChange + 16, // 12: swarmeval.sandbox.v1.ExecHeader.changes:type_name -> swarmeval.sandbox.v1.FsChange + 17, // 13: swarmeval.sandbox.v1.ExecHeader.processes:type_name -> swarmeval.sandbox.v1.Process + 14, // 14: swarmeval.sandbox.v1.ExecHeader.collected:type_name -> swarmeval.sandbox.v1.CollectedFile + 0, // 15: swarmeval.sandbox.v1.FsChange.op:type_name -> swarmeval.sandbox.v1.FsChange.Op + 1, // 16: swarmeval.sandbox.v1.FsChange.kind:type_name -> swarmeval.sandbox.v1.FsChange.Kind + 2, // 17: swarmeval.sandbox.v1.FsChange.attribution:type_name -> swarmeval.sandbox.v1.FsChange.Attribution + 20, // 18: swarmeval.sandbox.v1.RestoreFilesRequest.dirs:type_name -> swarmeval.sandbox.v1.RestoreDir + 21, // 19: swarmeval.sandbox.v1.RestoreFilesRequest.files:type_name -> swarmeval.sandbox.v1.RestoreFile + 27, // 20: swarmeval.sandbox.v1.FinalDiffResponse.changes:type_name -> swarmeval.sandbox.v1.SandboxChanges + 18, // 21: swarmeval.sandbox.v1.FinalDiffResponse.blob:type_name -> swarmeval.sandbox.v1.BlobChunk + 16, // 22: swarmeval.sandbox.v1.SandboxChanges.changes:type_name -> swarmeval.sandbox.v1.FsChange + 3, // 23: swarmeval.sandbox.v1.SandboxService.CreateRun:input_type -> swarmeval.sandbox.v1.CreateRunRequest + 7, // 24: swarmeval.sandbox.v1.SandboxService.CreateSandbox:input_type -> swarmeval.sandbox.v1.CreateSandboxRequest + 11, // 25: swarmeval.sandbox.v1.SandboxService.Exec:input_type -> swarmeval.sandbox.v1.ExecRequest + 23, // 26: swarmeval.sandbox.v1.SandboxService.ReadFile:input_type -> swarmeval.sandbox.v1.ReadFileRequest + 19, // 27: swarmeval.sandbox.v1.SandboxService.RestoreFiles:input_type -> swarmeval.sandbox.v1.RestoreFilesRequest + 25, // 28: swarmeval.sandbox.v1.SandboxService.FinalDiff:input_type -> swarmeval.sandbox.v1.FinalDiffRequest + 28, // 29: swarmeval.sandbox.v1.SandboxService.DestroyRun:input_type -> swarmeval.sandbox.v1.DestroyRunRequest + 4, // 30: swarmeval.sandbox.v1.SandboxService.CreateRun:output_type -> swarmeval.sandbox.v1.CreateRunResponse + 10, // 31: swarmeval.sandbox.v1.SandboxService.CreateSandbox:output_type -> swarmeval.sandbox.v1.CreateSandboxResponse + 12, // 32: swarmeval.sandbox.v1.SandboxService.Exec:output_type -> swarmeval.sandbox.v1.ExecResponse + 24, // 33: swarmeval.sandbox.v1.SandboxService.ReadFile:output_type -> swarmeval.sandbox.v1.ReadFileResponse + 22, // 34: swarmeval.sandbox.v1.SandboxService.RestoreFiles:output_type -> swarmeval.sandbox.v1.RestoreFilesResponse + 26, // 35: swarmeval.sandbox.v1.SandboxService.FinalDiff:output_type -> swarmeval.sandbox.v1.FinalDiffResponse + 29, // 36: swarmeval.sandbox.v1.SandboxService.DestroyRun:output_type -> swarmeval.sandbox.v1.DestroyRunResponse + 30, // [30:37] is the sub-list for method output_type + 23, // [23:30] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_swarmeval_sandbox_v1_sandbox_proto_init() } @@ -2177,11 +2359,11 @@ func file_swarmeval_sandbox_v1_sandbox_proto_init() { if File_swarmeval_sandbox_v1_sandbox_proto != nil { return } - file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[8].OneofWrappers = []any{ + file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[9].OneofWrappers = []any{ (*ExecResponse_Header)(nil), (*ExecResponse_Blob)(nil), } - file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[21].OneofWrappers = []any{ + file_swarmeval_sandbox_v1_sandbox_proto_msgTypes[23].OneofWrappers = []any{ (*FinalDiffResponse_Changes)(nil), (*FinalDiffResponse_Blob)(nil), } @@ -2191,7 +2373,7 @@ func file_swarmeval_sandbox_v1_sandbox_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_swarmeval_sandbox_v1_sandbox_proto_rawDesc), len(file_swarmeval_sandbox_v1_sandbox_proto_rawDesc)), NumEnums: 3, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go b/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go index 44446f1..35db144 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandbox_grpc.pb.go @@ -44,7 +44,8 @@ type SandboxServiceClient interface { // ALREADY_EXISTS if the run exists. CreateRun(ctx context.Context, in *CreateRunRequest, opts ...grpc.CallOption) (*CreateRunResponse, error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(ctx context.Context, in *CreateSandboxRequest, opts ...grpc.CallOption) (*CreateSandboxResponse, error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. @@ -175,7 +176,8 @@ type SandboxServiceServer interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *CreateRunRequest) (*CreateRunResponse, error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *CreateSandboxRequest) (*CreateSandboxResponse, error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. diff --git a/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go b/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go index f6158d0..d3b054a 100644 --- a/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go +++ b/go/internal/gen/swarmeval/sandbox/v1/sandboxv1connect/sandbox.connect.go @@ -62,7 +62,8 @@ type SandboxServiceClient interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *connect.Request[v1.CreateRunRequest]) (*connect.Response[v1.CreateRunResponse], error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *connect.Request[v1.CreateSandboxRequest]) (*connect.Response[v1.CreateSandboxResponse], error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. @@ -191,7 +192,8 @@ type SandboxServiceHandler interface { // ALREADY_EXISTS if the run exists. CreateRun(context.Context, *connect.Request[v1.CreateRunRequest]) (*connect.Response[v1.CreateRunResponse], error) // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. CreateSandbox(context.Context, *connect.Request[v1.CreateSandboxRequest]) (*connect.Response[v1.CreateSandboxResponse], error) // Runs one tool call. The stream carries exactly one header first, then the blobs the // header refers to, each as one or more chunks. Calls on one sandbox run one at a time. diff --git a/go/internal/sandboxd/display.go b/go/internal/sandboxd/display.go new file mode 100644 index 0000000..3f4e5ec --- /dev/null +++ b/go/internal/sandboxd/display.go @@ -0,0 +1,205 @@ +package sandboxd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "slices" + "strconv" + "strings" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" +) + +const ( + // DisplayUser runs a display image's display stack and the tools that drive it. Display + // images must have it. + DisplayUser = "swarmdisplay" + // DisplayDir holds the display stack's state, outside every key path. + DisplayDir = "/run/swarm-display" +) + +// Display is a sandbox's virtual screen. +type Display struct { + Width uint32 + Height uint32 + // URL is the page the browser opens first; empty opens about:blank. + URL string +} + +func checkDisplay(d *Display, mounts []Mount, users []string) error { + if d == nil { + return nil + } + // Users are the agents' users. The display's runs the screen and the browser, so an agent + // running as it would reach both around the tools. + if slices.Contains(users, DisplayUser) { + return fmt.Errorf("%w: user %s runs the display and cannot be an agent's user", ErrInvalid, DisplayUser) + } + if d.Width < 320 || d.Width > 1920 || d.Height < 240 || d.Height > 1200 { + return fmt.Errorf("%w: display %dx%d is outside 320x240 to 1920x1200", ErrInvalid, d.Width, d.Height) + } + if len(d.URL) > 2048 || strings.ContainsAny(d.URL, "\x00\n\r") { + return fmt.Errorf("%w: display url %.80q is over 2048 bytes or holds NUL or a line break", ErrInvalid, d.URL) + } + for _, m := range mounts { + if overlaps(m.Path, DisplayDir) { + return fmt.Errorf("%w: key path %s overlaps the display's state directory %s, which must stay out of every key path", ErrInvalid, m.Path, DisplayDir) + } + } + return nil +} + +// startDisplay runs the image's `swarm-display start`, which returns once the display stack +// is up and prints the stack's pid. It returns that process's uid, whose processes are the +// display's from then on. +func (s *Service) startDisplay(ctx context.Context, sb *sandbox, d *Display) (string, error) { + argv := []string{ + "swarm-display", "start", + "--width", strconv.FormatUint(uint64(d.Width), 10), + "--height", strconv.FormatUint(uint64(d.Height), 10), + } + if d.URL != "" { + argv = append(argv, "--url", d.URL) + } + startCtx, cancel := context.WithTimeout(ctx, s.cfg.DisplayStartTimeout) + defer cancel() + stdout := &capBuffer{limit: 4 << 10} + stderr := &capBuffer{limit: 16 << 10} + code, err := s.drv.Exec(startCtx, sb.container, driver.ExecSpec{Argv: argv, User: DisplayUser}, stdout, stderr) + if err != nil { + return "", fmt.Errorf("start the display in container %s (timeout %s): %w", sb.container, s.cfg.DisplayStartTimeout, err) + } + if code != 0 { + return "", fmt.Errorf("start the display in container %s: `swarm-display start` exited %d: %s. The image must be built from swarmeval/display (deploy/images/display)", + sb.container, code, strings.TrimSpace(string(stderr.buf))) + } + pid, err := strconv.ParseInt(strings.TrimSpace(string(stdout.buf)), 10, 32) + if err != nil { + return "", fmt.Errorf("start the display in container %s: `swarm-display start` printed %.80q, not the display stack's pid", sb.container, stdout.buf) + } + procs, err := s.processes(ctx, sb) + if err != nil { + return "", err + } + i := slices.IndexFunc(procs, func(p driver.Process) bool { return p.PID == int32(pid) }) + if i < 0 { + return "", fmt.Errorf("start the display in container %s: its pid %d is not running after `swarm-display start` returned", sb.container, pid) + } + uid := procs[i].UID + // An agent without an os_user runs as the image's user; that must not be the display's. + fallback, err := s.defaultUID(ctx, sb) + if err != nil { + return "", err + } + if fallback == uid { + return "", fmt.Errorf("%w: the image's default user is uid %s, which runs the display, so agents without an os_user would reach the screen and the browser around the tools. Leave `USER` unset in an image built from swarmeval/display", ErrInvalid, uid) + } + return uid, nil +} + +// defaultUIDScript prints the uid it runs as, with builtins only. +const defaultUIDScript = `while read -r k v _; do case $k in Uid:) echo "$v"; exit 0 ;; esac; done < /proc/self/status +exit 3` + +// defaultUID is the uid a command run without a user gets: the image's `USER`. +func (s *Service) defaultUID(ctx context.Context, sb *sandbox) (string, error) { + idCtx, cancel := context.WithTimeout(ctx, s.cfg.HelperTimeout) + defer cancel() + stdout := &capBuffer{limit: 64} + code, err := s.drv.Exec(idCtx, sb.container, driver.ExecSpec{Argv: []string{"/bin/sh", "-c", defaultUIDScript}}, stdout, io.Discard) + if err != nil || code != 0 { + return "", fmt.Errorf("read the default user of container %s: exit %d: %w", sb.container, code, err) + } + return strings.TrimSpace(string(stdout.buf)), nil +} + +// collectScript prints the regular file $1 and removes it; exit 4 when there is none. A +// symlink is removed without being followed. +const collectScript = `f=$1 +if [ -f "$f" ] && [ ! -h "$f" ]; then cat -- "$f" || exit 5; rm -f -- "$f"; exit 0; fi +rm -f -- "$f" 2>/dev/null +exit 4` + +// Collected is one file an Exec collected. +type Collected struct { + Path string + Missing bool + Size int64 + // SHA256 is empty when the file is over the collect limit; otherwise its content is + // among the result's blobs. + SHA256 string +} + +func checkCollect(paths []string, mounts []Mount) error { + for _, p := range paths { + if !path.IsAbs(p) || path.Clean(p) != p || strings.ContainsRune(p, 0) { + return fmt.Errorf("%w: collect path %q must be absolute and clean", ErrInvalid, p) + } + for _, m := range mounts { + if overlaps(p, m.Path) { + return fmt.Errorf("%w: collect path %s overlaps key path %s; collected files live outside key paths", ErrInvalid, p, m.Path) + } + } + } + return nil +} + +// collect reads and removes each path, in order, as the call's user: root has no capability to +// read another user's private directory, and what a call may collect is what its user may read. +func (s *Service) collect(ctx context.Context, sb *sandbox, user string, paths []string, blobs *[]Blob) ([]Collected, error) { + out := make([]Collected, 0, len(paths)) + for _, p := range paths { + readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) + stdout := &capBuffer{limit: int(s.cfg.CollectLimit)} + stderr := &capBuffer{limit: 4 << 10} + argv := []string{"/bin/sh", "-c", collectScript, "sh", p} + code, err := s.drv.Exec(readCtx, sb.container, driver.ExecSpec{Argv: argv, User: user}, stdout, stderr) + cancel() + if err != nil { + return nil, fmt.Errorf("collect %s from container %s: %w", p, sb.container, err) + } + switch code { + case 0: + case 4: + out = append(out, Collected{Path: p, Missing: true}) + continue + default: + return nil, fmt.Errorf("collect %s from container %s: exit %d: %s", p, sb.container, code, strings.TrimSpace(string(stderr.buf))) + } + c := Collected{Path: p, Size: stdout.size} + if stdout.size <= s.cfg.CollectLimit { + sum := sha256.Sum256(stdout.buf) + c.SHA256 = hex.EncodeToString(sum[:]) + *blobs = append(*blobs, Blob{SHA256: c.SHA256, Data: stdout.buf}) + } + out = append(out, c) + } + return out, nil +} + +// overlaps tells whether one of two clean absolute paths is the other or under it. +func overlaps(a, b string) bool { + return a == "/" || b == "/" || under(a, b) || under(b, a) +} + +// withoutDisplay drops the display user's processes: they belong to the display, not to a call. +func (sb *sandbox) withoutDisplay(procs []driver.Process) []driver.Process { + if sb.displayUID == "" { + return procs + } + kept := procs[:0:0] + for _, p := range procs { + if p.UID != sb.displayUID { + kept = append(kept, p) + } + } + return kept +} + +// defaultDisplayStartTimeout bounds `swarm-display start`, which waits for the browser. +const defaultDisplayStartTimeout = 60 * time.Second diff --git a/go/internal/sandboxd/display_integration_test.go b/go/internal/sandboxd/display_integration_test.go new file mode 100644 index 0000000..7d2aaef --- /dev/null +++ b/go/internal/sandboxd/display_integration_test.go @@ -0,0 +1,142 @@ +//go:build integration + +package sandboxd + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" + "github.com/jackeydou/DUNE/go/internal/driver/docker" +) + +// displayImage is built from deploy/images/display, which needs the internet, so the test +// skips when the docker host does not have it. +func displayImage(t *testing.T, drv *docker.Driver) string { + t.Helper() + img := os.Getenv("SWARMEVAL_IT_DISPLAY_IMAGE") + if img == "" { + img = "swarmeval/display:dev" + } + rc, err := drv.ExportImagePath(context.Background(), img, "/etc/swarm-display", map[string]string{driver.LabelManaged: "true"}) + if err != nil { + t.Skipf("display image %s is not usable (%v); build it with `docker build -f deploy/images/display/Dockerfile -t %s deploy/images/display`", img, err, img) + } + _ = rc.Close() + return img +} + +const page = `Shop +` + +func TestLiveDisplayDrivesTheBrowserAndKeepsItsProcessesOut(t *testing.T) { + ctx := context.Background() + drv, err := docker.New(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = drv.Close() }) + img := displayImage(t, drv) + state, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := DefaultConfig(state) + if rt := os.Getenv("SWARMEVAL_IT_RUNTIME"); rt != "" { + cfg.Runtime = rt + } + svc := New(cfg, drv, slog.New(slog.DiscardHandler)) + run := "it_display" + t.Cleanup(func() { + if err := svc.DestroyRun(context.Background(), run); err != nil { + t.Errorf("destroy: %v", err) + } + }) + if err := svc.CreateRun(ctx, run, []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err = svc.CreateSandbox(ctx, CreateRequest{ + RunID: run, SandboxID: "desk", Image: img, + Mounts: []Mount{{Path: "/workspace"}, {Path: "/tmp"}}, + Resources: driver.Resources{MemoryBytes: 2 << 30, Pids: 512}, + Files: []SeedFile{{Path: "/workspace/index.html", Content: []byte(page), Mode: 0o644}}, + Users: []string{"agent"}, + Display: &Display{Width: 1024, Height: 768, URL: "file:///workspace/index.html"}, + }) + if err != nil { + t.Fatal(err) + } + shot := []string{DisplayDir + "/out/screenshot.png"} + call := func(id, user string, argv []string, collect []string) ExecResult { + t.Helper() + res, err := svc.Exec(ctx, ExecRequest{RunID: run, SandboxID: "desk", CallID: id, Argv: argv, User: user, Timeout: time.Minute, Collect: collect}) + if err != nil { + t.Fatal(err) + } + return res + } + png := func(res ExecResult) []byte { + t.Helper() + if len(res.Collected) != 1 || res.Collected[0].SHA256 == "" { + t.Fatalf("collected = %+v; stderr %s", res.Collected, res.Stderr.Inline) + } + for _, b := range res.Blobs { + if b.SHA256 == res.Collected[0].SHA256 { + return b.Data + } + } + t.Fatal("the collected screenshot is not among the blobs") + return nil + } + + res := call("call_1", DisplayUser, []string{"swarm-display", "browser", `{"action":"snapshot","screenshot":true}`}, shot) + if res.ExitCode != 0 || !strings.Contains(string(res.Stdout.Inline), `button "Pay"`) { + t.Fatalf("snapshot exit %d:\n%s\n%s", res.ExitCode, res.Stdout.Inline, res.Stderr.Inline) + } + if !bytes.HasPrefix(png(res), []byte("\x89PNG\r\n\x1a\n")) { + t.Fatal("the browser screenshot is not a PNG") + } + ref := refOf(t, string(res.Stdout.Inline), `button "Pay"`) + res = call("call_2", DisplayUser, []string{"swarm-display", "browser", `{"action":"click","ref":"` + ref + `"}`}, shot) + if res.ExitCode != 0 || !strings.Contains(string(res.Stdout.Inline), "Title: paid") { + t.Fatalf("click exit %d:\n%s\n%s", res.ExitCode, res.Stdout.Inline, res.Stderr.Inline) + } + if !res.Collected[0].Missing { + t.Fatalf("a click without `screenshot` collected %+v", res.Collected) + } + res = call("call_3", DisplayUser, []string{"swarm-display", "computer", `{"action":"click","coordinate":[500,400]}`}, shot) + if res.ExitCode != 0 || !bytes.HasPrefix(png(res), []byte("\x89PNG")) { + t.Fatalf("computer click exit %d: %s", res.ExitCode, res.Stderr.Inline) + } + if len(res.Processes) != 0 || len(res.Changes) != 0 || len(res.Background) != 0 { + t.Fatalf("display calls reported processes %+v, changes %+v, background %+v", res.Processes, res.Changes, res.Background) + } + + res = call("call_4", "agent", []string{"sh", "-c", "DISPLAY=:1 xdotool getmouselocation; ls /run/swarm-display"}, nil) + if res.ExitCode == 0 { + t.Fatalf("the agent's user reached the display:\n%s", res.Stdout.Inline) + } + res = call("call_5", "agent", []string{"sh", "-c", "echo x > /workspace/note"}, nil) + if len(res.Changes) != 1 || res.Changes[0].Ambiguous { + t.Fatalf("changes = %+v; the browser's processes must not make the call's own write ambiguous", res.Changes) + } +} + +// refOf finds the ref of the snapshot line holding what. +func refOf(t *testing.T, snapshot, what string) string { + t.Helper() + for line := range strings.Lines(snapshot) { + if i := strings.Index(line, "[ref="); strings.Contains(line, what) && i >= 0 { + ref, _, _ := strings.Cut(line[i+len("[ref="):], "]") + return ref + } + } + t.Fatalf("no %s in the snapshot:\n%s", what, snapshot) + return "" +} diff --git a/go/internal/sandboxd/display_test.go b/go/internal/sandboxd/display_test.go new file mode 100644 index 0000000..f11875e --- /dev/null +++ b/go/internal/sandboxd/display_test.go @@ -0,0 +1,201 @@ +package sandboxd + +import ( + "context" + "errors" + "io" + "log/slog" + "slices" + "testing" + + "github.com/jackeydou/DUNE/go/internal/driver" +) + +// newDisplayFixture creates a sandbox with a display; the fake's display stack is pid 50, run +// by swarmdisplay, beside the container's own sleep. +func newDisplayFixture(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + idle := driver.Process{PID: 1, PPID: 0, User: "root", Cmdline: "sleep infinity"} + stack := driver.Process{PID: 50, PPID: 1, User: DisplayUser, Cmdline: "swarm-display daemon"} + f.drv.procs = []driver.Process{idle} + f.drv.display = func(stdout io.Writer) (int, error) { + f.drv.procs = []driver.Process{idle, stack} + _, err := io.WriteString(stdout, "50\n") + return 0, err + } + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", + Mounts: []Mount{{Path: "/workspace"}}, + Display: &Display{Width: 1024, Height: 768, URL: "http://127.0.0.1:8080/"}, + }) + if err != nil { + t.Fatal(err) + } + return f +} + +func TestDisplayStartsAsTheDisplayUserWithItsSize(t *testing.T) { + f := newDisplayFixture(t) + + start := f.drv.execs[0] + want := []string{"swarm-display", "start", "--width", "1024", "--height", "768", "--url", "http://127.0.0.1:8080/"} + if start.User != DisplayUser || !slices.Equal(start.Argv, want) { + t.Fatalf("start = %+v, want %v as %s", start, want, DisplayUser) + } +} + +func TestDisplayProcessesAreNeitherReportedNorAttributed(t *testing.T) { + f := newDisplayFixture(t) + idle := f.drv.procs[0] + stack := f.drv.procs[1] + renderer := driver.Process{PID: 61, PPID: 50, User: DisplayUser, Cmdline: "chromium --type=renderer"} + server := driver.Process{PID: 70, PPID: 1, User: "agent", Cmdline: "python -m http.server"} + + f.drv.handler = func(context.Context, driver.ExecSpec, io.Writer, io.Writer) (int, error) { + f.drv.procs = []driver.Process{idle, stack, renderer, server} + return 0, nil + } + res, err := f.svc.Exec(context.Background(), ExecRequest{RunID: "run_2", SandboxID: "desk", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9}) + if err != nil { + t.Fatal(err) + } + + if len(res.Processes) != 1 || res.Processes[0].PID != 70 { + t.Fatalf("processes = %+v; the renderer is the display's, the server the call's", res.Processes) + } +} + +func TestDisplayStartFailureRemovesTheContainer(t *testing.T) { + cases := map[string]func(io.Writer) (int, error){ + "exit": func(io.Writer) (int, error) { return 1, nil }, + "no pid": func(w io.Writer) (int, error) { _, err := io.WriteString(w, "ready\n"); return 0, err }, + "gone": func(w io.Writer) (int, error) { _, err := io.WriteString(w, "99\n"); return 0, err }, + } + for name, play := range cases { + f := newFixture(t) + f.drv.display = play + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", Display: &Display{Width: 1024, Height: 768}, + }) + if err == nil { + t.Errorf("%s: CreateSandbox succeeded", name) + continue + } + if !slices.Contains(f.drv.removed, driver.ContainerID("c_desk")) { + t.Errorf("%s: container not removed after %v", name, err) + } + } +} + +func TestDisplayIsRefusedWhenItCannotWork(t *testing.T) { + cases := map[string]CreateRequest{ + "too small": {Display: &Display{Width: 100, Height: 768}}, + "too large": {Display: &Display{Width: 1024, Height: 4000}}, + "url with newline": {Display: &Display{Width: 1024, Height: 768, URL: "http://x/\nevil"}}, + "key path at its dir": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: DisplayDir}}}, + "key path above it": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/run"}}}, + "key path at root": {Display: &Display{Width: 1024, Height: 768}, Mounts: []Mount{{Path: "/"}}}, + "agent as display": {Display: &Display{Width: 1024, Height: 768}, Users: []string{"qa", DisplayUser}}, + } + for name, req := range cases { + svc := New(DefaultConfig(t.TempDir()), &fakeDriver{}, slog.New(slog.DiscardHandler)) + req.RunID, req.SandboxID, req.Image = "run_1", "desk", "img" + if err := svc.CreateRun(context.Background(), "run_1", []string{"desk"}); err != nil { + t.Fatal(err) + } + if _, err := svc.CreateSandbox(context.Background(), req); !errors.Is(err, ErrInvalid) { + t.Errorf("%s: err = %v, want ErrInvalid", name, err) + } + } +} + +func TestExecCollectsAndRemovesFiles(t *testing.T) { + f := newFixture(t) + png := []byte("\x89PNG fake") + f.drv.files = map[string][]byte{"/run/swarm-display/out/screenshot.png": png} + + res, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{"/run/swarm-display/out/screenshot.png", "/run/swarm-display/out/none.png"}, + }) + if err != nil { + t.Fatal(err) + } + + if len(res.Collected) != 2 { + t.Fatalf("collected = %+v", res.Collected) + } + got, missing := res.Collected[0], res.Collected[1] + if got.Missing || got.Size != int64(len(png)) || got.SHA256 == "" || !missing.Missing { + t.Fatalf("collected = %+v", res.Collected) + } + i := slices.IndexFunc(res.Blobs, func(b Blob) bool { return b.SHA256 == got.SHA256 }) + if i < 0 || string(res.Blobs[i].Data) != string(png) { + t.Fatalf("blobs = %v; the collected file must follow as a blob", res.Blobs) + } + if _, left := f.drv.files["/run/swarm-display/out/screenshot.png"]; left { + t.Fatal("collected file was not removed") + } +} + +func TestExecCollectOverTheLimitIsRemovedButNotSent(t *testing.T) { + f := newFixture(t) + f.svc.cfg.CollectLimit = 4 + f.drv.files = map[string][]byte{"/run/out.png": []byte("too large")} + + res, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{"/run/out.png"}, + }) + if err != nil { + t.Fatal(err) + } + + if c := res.Collected[0]; c.Missing || c.Size != 9 || c.SHA256 != "" { + t.Fatalf("collected = %+v", c) + } +} + +func TestExecRefusesCollectPathsInKeyPaths(t *testing.T) { + f := newFixture(t) + for _, p := range []string{"/workspace/shot.png", "/", "relative.png", "/run/../workspace/x"} { + _, err := f.svc.Exec(context.Background(), ExecRequest{ + RunID: "run_1", SandboxID: "box", CallID: "call_1", Argv: []string{"true"}, Timeout: 5e9, + Collect: []string{p}, + }) + if !errors.Is(err, ErrInvalid) { + t.Errorf("collect %q: err = %v, want ErrInvalid", p, err) + } + } +} + +func TestDisplayIsRefusedWhenTheImageRunsAsTheDisplayUser(t *testing.T) { + f := newFixture(t) + f.drv.defaultUID = "1000" // the fake lists the first user it meets, swarmdisplay, as 1000 + f.drv.procs = nil + f.drv.display = func(stdout io.Writer) (int, error) { + f.drv.procs = []driver.Process{{PID: 50, PPID: 1, User: DisplayUser, Cmdline: "swarm-display daemon"}} + _, err := io.WriteString(stdout, "50\n") + return 0, err + } + if err := f.svc.CreateRun(context.Background(), "run_2", []string{"desk"}); err != nil { + t.Fatal(err) + } + _, err := f.svc.CreateSandbox(context.Background(), CreateRequest{ + RunID: "run_2", SandboxID: "desk", Image: "img", Display: &Display{Width: 1024, Height: 768}, + }) + + if !errors.Is(err, ErrInvalid) { + t.Fatalf("err = %v, want ErrInvalid: agents without an os_user would run as the display", err) + } + if !slices.Contains(f.drv.removed, driver.ContainerID("c_desk")) { + t.Fatal("container not removed") + } +} diff --git a/go/internal/sandboxd/exec.go b/go/internal/sandboxd/exec.go new file mode 100644 index 0000000..fa6b91e --- /dev/null +++ b/go/internal/sandboxd/exec.go @@ -0,0 +1,229 @@ +package sandboxd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + "time" + + "github.com/jackeydou/DUNE/go/internal/driver" + "github.com/jackeydou/DUNE/go/internal/fsdiff" +) + +// ExecRequest is one tool call. +type ExecRequest struct { + RunID string + SandboxID string + CallID string + Argv []string + Cwd string + User string + Timeout time.Duration + // Collect lists files outside the key paths to read, remove, and return after the command. + Collect []string +} + +// Change is a file change with its attribution. +type Change struct { + fsdiff.Change + Protected bool + Ambiguous bool + Candidates []string + // Content is set when the new content is among the result's blobs. + Content bool +} + +// ExecResult is everything one call produced. +type ExecResult struct { + ExitCode int + TimedOut bool + Duration time.Duration + Stdout Output + Stderr Output + Background []Change + Changes []Change + Processes []driver.Process + Collected []Collected + Blobs []Blob +} + +// Exec runs one call and reports its output, the changes around it, and what it left +// running. +func (s *Service) Exec(ctx context.Context, req ExecRequest) (ExecResult, error) { + if len(req.Argv) == 0 { + return ExecResult{}, fmt.Errorf("%w: call %s has an empty argv", ErrInvalid, req.CallID) + } + if req.Timeout <= 0 { + return ExecResult{}, fmt.Errorf("%w: call %s has no timeout", ErrInvalid, req.CallID) + } + if req.CallID == "" { + return ExecResult{}, fmt.Errorf("%w: call id is required for attribution", ErrInvalid) + } + sb, err := s.lookup(req.RunID, req.SandboxID) + if err != nil { + return ExecResult{}, err + } + if err := checkCollect(req.Collect, sb.mounts); err != nil { + return ExecResult{}, fmt.Errorf("call %s: %w", req.CallID, err) + } + sb.mu.Lock() + defer sb.mu.Unlock() + sb.execd = true + + pre, err := fsdiff.Walk(sb.roots, sb.manifest) + if err != nil { + return ExecResult{}, err + } + baseline, err := s.processes(ctx, sb) + if err != nil { + return ExecResult{}, err + } + candidates := sb.liveCalls(baseline) + budget := s.cfg.ContentBudget + var blobs []Blob + background := s.attribute(sb, fsdiff.Diff(sb.manifest, pre), true, candidates, &budget, &blobs) + + stdout := &capBuffer{limit: s.cfg.OutputLimit} + stderrBuf := &capBuffer{limit: s.cfg.OutputLimit} + stderr := &pidWriter{next: stderrBuf} + start := time.Now() + execCtx, cancel := context.WithTimeout(ctx, req.Timeout) + code, err := s.drv.Exec(execCtx, sb.container, driver.ExecSpec{Argv: wrapArgv(req.Argv), Cwd: req.Cwd, User: req.User}, stdout, stderr) + cancel() + duration := time.Since(start) + timedOut := false + switch { + case err == nil: + case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: + timedOut, code = true, -1 + if err := s.killTree(ctx, sb, req, stderr.pid); err != nil { + return ExecResult{}, err + } + default: + return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) + } + if err := stderr.flush(); err != nil { + return ExecResult{}, err + } + collected, err := s.collect(ctx, sb, req.User, req.Collect, &blobs) + if err != nil { + return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) + } + + post, err := fsdiff.Walk(sb.roots, pre) + if err != nil { + return ExecResult{}, err + } + after, err := s.processes(ctx, sb) + if err != nil { + return ExecResult{}, err + } + windowCandidates := candidates + if len(candidates) > 0 { + windowCandidates = append(slices.Clone(candidates), req.CallID) + } + changes := s.attribute(sb, fsdiff.Diff(pre, post), len(candidates) > 0, windowCandidates, &budget, &blobs) + started := sb.recordProcesses(baseline, after, req.CallID) + sb.manifest = post + + outBlob := func(c *capBuffer) Output { + out, blob := c.output(s.cfg.InlineLimit) + if blob != nil { + blobs = append(blobs, *blob) + } + return out + } + return ExecResult{ + ExitCode: code, + TimedOut: timedOut, + Duration: duration, + Stdout: outBlob(stdout), + Stderr: outBlob(stderrBuf), + Background: background, + Changes: changes, + Processes: started, + Collected: collected, + Blobs: blobs, + }, nil +} + +// killTree kills a timed-out command and its descendants from inside the sandbox, as the +// call's own user: with every capability dropped, only the same uid may signal them. +func (s *Service) killTree(ctx context.Context, sb *sandbox, req ExecRequest, pid string) error { + if pid == "" { + s.log.Warn("timed-out call printed no pid; its processes are left running and will be reported", + "run_id", req.RunID, "sandbox_id", req.SandboxID, "call_id", req.CallID) + return nil + } + killCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) + defer cancel() + argv := []string{"/bin/sh", "-c", killTreeScript, "sh", pid} + if _, err := s.drv.Exec(killCtx, sb.container, driver.ExecSpec{Argv: argv, User: req.User}, io.Discard, io.Discard); err != nil { + return fmt.Errorf("kill timed-out call %s (pid %s) in sandbox %s: %w", req.CallID, pid, req.SandboxID, err) + } + return nil +} + +func (s *Service) attribute(sb *sandbox, diff []fsdiff.Change, ambiguous bool, candidates []string, budget *int64, blobs *[]Blob) []Change { + out := make([]Change, len(diff)) + for i, c := range diff { + out[i] = Change{ + Change: c, + Protected: protected(sb.mounts, c.Path), + Ambiguous: ambiguous, + Candidates: candidates, + } + if c.Op == fsdiff.OpDelete || c.After.Kind != fsdiff.KindFile || c.After.Size > s.cfg.ContentLimit || c.After.Size > *budget { + continue + } + data, err := os.ReadFile(c.After.HostPath) + if err != nil { + continue // Gone or replaced since the walk; the hash still stands. + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != c.After.SHA256 { + continue + } + *budget -= int64(len(data)) + *blobs = append(*blobs, Blob{SHA256: c.After.SHA256, Data: data}) + out[i].Content = true + } + return out +} + +// liveCalls lists the calls whose processes are still alive, sorted. +func (sb *sandbox) liveCalls(procs []driver.Process) []string { + seen := map[string]bool{} + for _, p := range procs { + if call, ok := sb.live[p.PID]; ok { + seen[call] = true + } + } + return slices.Sorted(maps.Keys(seen)) +} + +// recordProcesses returns processes alive after the call that were not alive before it, +// and forgets processes that have exited. +func (sb *sandbox) recordProcesses(before, after []driver.Process, callID string) []driver.Process { + existed := map[int32]bool{} + for _, p := range before { + existed[p.PID] = true + } + alive := map[int32]bool{} + var started []driver.Process + for _, p := range after { + alive[p.PID] = true + if !existed[p.PID] { + started = append(started, p) + sb.live[p.PID] = callID + } + } + maps.DeleteFunc(sb.live, func(pid int32, _ string) bool { return !alive[pid] }) + return started +} diff --git a/go/internal/sandboxd/procs.go b/go/internal/sandboxd/procs.go index 0fbaafe..fb9146d 100644 --- a/go/internal/sandboxd/procs.go +++ b/go/internal/sandboxd/procs.go @@ -36,7 +36,7 @@ for d in /proc/[0-9]*; do done` // processes lists what runs in the sandbox, with pids as the sandbox sees them: the same pids -// killTree signals. It runs as root, which reads every process's /proc entries without any +// killTree signals. The display's processes are left out. It runs as root, which reads every process's /proc entries without any // capability. func (s *Service) processes(ctx context.Context, sb *sandbox) ([]driver.Process, error) { listCtx, cancel := context.WithTimeout(ctx, s.cfg.HelperTimeout) @@ -56,7 +56,11 @@ func (s *Service) processes(ctx context.Context, sb *sandbox) ([]driver.Process, return nil, fmt.Errorf("process listing of container %s is %d bytes, over the %d byte limit: the sandbox holds too many processes or too long command lines. Set a pids limit in the profile", sb.container, stdout.size, s.cfg.ProcessListLimit) } - return parseProcesses(string(stdout.buf)) + procs, err := parseProcesses(string(stdout.buf)) + if err != nil { + return nil, err + } + return sb.withoutDisplay(procs), nil } // parseProcesses reads processScript's output. Names come from the sandbox's /etc/passwd, which @@ -86,13 +90,13 @@ func parseProcesses(out string) ([]driver.Process, error) { if err := errors.Join(err1, err2, err3); err != nil { return nil, fmt.Errorf("process listing line %q: %w", line, err) } - procs = append(procs, driver.Process{PID: int32(pid), PPID: int32(ppid), User: fields[2], Cmdline: strings.TrimRight(fields[3], " ")}) + procs = append(procs, driver.Process{PID: int32(pid), PPID: int32(ppid), UID: fields[2], User: fields[2], Cmdline: strings.TrimRight(fields[3], " ")}) default: return nil, fmt.Errorf("process listing line %q is neither a U nor a P line", line) } } for i, p := range procs { - if name, ok := names[p.User]; ok { + if name, ok := names[p.UID]; ok { procs[i].User = name } } diff --git a/go/internal/sandboxd/procs_test.go b/go/internal/sandboxd/procs_test.go index 218590a..33a8c4a 100644 --- a/go/internal/sandboxd/procs_test.go +++ b/go/internal/sandboxd/procs_test.go @@ -21,9 +21,9 @@ func TestParseProcessesNamesUsersAndKeepsCommandLines(t *testing.T) { t.Fatal(err) } want := []driver.Process{ - {PID: 1, PPID: 0, User: "root", Cmdline: "/sbin/docker-init -- sleep infinity"}, - {PID: 14, PPID: 1, User: "qa", Cmdline: "python3 -m http.server 8000"}, - {PID: 20, PPID: 14, User: "4242", Cmdline: "[kworker]"}, + {PID: 1, PPID: 0, UID: "0", User: "root", Cmdline: "/sbin/docker-init -- sleep infinity"}, + {PID: 14, PPID: 1, UID: "1001", User: "qa", Cmdline: "python3 -m http.server 8000"}, + {PID: 20, PPID: 14, UID: "4242", User: "4242", Cmdline: "[kworker]"}, } if !slices.Equal(procs, want) { t.Fatalf("procs = %+v\nwant %+v", procs, want) diff --git a/go/internal/sandboxd/server.go b/go/internal/sandboxd/server.go index 6393cdf..934f36b 100644 --- a/go/internal/sandboxd/server.go +++ b/go/internal/sandboxd/server.go @@ -68,6 +68,7 @@ func (s *Server) CreateSandbox(ctx context.Context, req *sandboxv1.CreateSandbox Env: req.GetEnv(), Hostname: req.GetHostname(), MachineID: req.GetMachineId(), + Display: display(req.GetDisplay()), }) if err != nil { return nil, s.status("CreateSandbox", err) @@ -87,6 +88,7 @@ func (s *Server) Exec(req *sandboxv1.ExecRequest, stream grpc.ServerStreamingSer Cwd: req.GetCwd(), User: req.GetUser(), Timeout: req.GetTimeout().AsDuration(), + Collect: req.GetCollect(), }) if err != nil { return s.status("Exec", err) @@ -100,6 +102,7 @@ func (s *Server) Exec(req *sandboxv1.ExecRequest, stream grpc.ServerStreamingSer BackgroundChanges: changes(res.Background), Changes: changes(res.Changes), Processes: processes(res.Processes), + Collected: collected(res.Collected), } if err := stream.Send(&sandboxv1.ExecResponse{Item: &sandboxv1.ExecResponse_Header{Header: header}}); err != nil { return err @@ -200,6 +203,21 @@ func sendBlobs(blobs []Blob, send func(*sandboxv1.BlobChunk) error) error { return nil } +func display(d *sandboxv1.Display) *Display { + if d == nil { + return nil + } + return &Display{Width: d.GetWidth(), Height: d.GetHeight(), URL: d.GetUrl()} +} + +func collected(cs []Collected) []*sandboxv1.CollectedFile { + out := make([]*sandboxv1.CollectedFile, len(cs)) + for i, c := range cs { + out[i] = &sandboxv1.CollectedFile{Path: c.Path, Missing: c.Missing, Size: c.Size, Sha256: c.SHA256} + } + return out +} + func output(o Output) *sandboxv1.Output { return &sandboxv1.Output{Inline: o.Inline, Size: o.Size, BlobSha256: o.BlobSHA256, Capped: o.Capped} } diff --git a/go/internal/sandboxd/service.go b/go/internal/sandboxd/service.go index 65d8bef..1e1af1b 100644 --- a/go/internal/sandboxd/service.go +++ b/go/internal/sandboxd/service.go @@ -5,14 +5,10 @@ package sandboxd import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" - "io" "io/fs" "log/slog" - "maps" "net/netip" "os" "path" @@ -71,6 +67,10 @@ type Config struct { // ProcessListLimit caps a sandbox's process listing. A sandbox controls how many processes // it has and how long their command lines are. ProcessListLimit int + // CollectLimit caps each file an Exec collects. A larger file is removed but not sent. + CollectLimit int64 + // DisplayStartTimeout bounds starting a sandbox's display stack. + DisplayStartTimeout time.Duration } // DefaultConfig returns the limits sandboxd uses unless flags override them. @@ -89,6 +89,9 @@ func DefaultConfig(stateDir string) Config { SeedLimit: 1 << 20, RestoreLimit: 3 << 20, ProcessListLimit: 4 << 20, + CollectLimit: 8 << 20, + + DisplayStartTimeout: defaultDisplayStartTimeout, } } @@ -119,6 +122,9 @@ type sandbox struct { live map[int32]string // execd is set by the first Exec; RestoreFiles is refused after it. execd bool + // displayUID is the uid running the display stack, empty without a display. Its + // processes are left out of every listing. + displayUID string } // New returns a Service. cfg.StateDir must exist. @@ -147,6 +153,8 @@ type CreateRequest struct { Env map[string]string Hostname string MachineID string + // Display, when set, starts the image's display stack before the first manifest. + Display *Display } // SeedFile is written into a key path before the first manifest. @@ -178,6 +186,9 @@ func (s *Service) CreateSandbox(ctx context.Context, req CreateRequest) (string, if err := checkIdentity(req, mounts); err != nil { return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } + if err := checkDisplay(req.Display, mounts, req.Users); err != nil { + return "", fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) + } net, err := s.network(req.RunID, req.SandboxID) if err != nil { return "", err @@ -268,6 +279,13 @@ func (s *Service) create(ctx context.Context, req CreateRequest, sb *sandbox, ru return fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err) } sb.container = id + if req.Display != nil { + uid, err := s.startDisplay(ctx, sb, req.Display) + if err != nil { + return errors.Join(fmt.Errorf("sandbox %s of run %s: %w", req.SandboxID, req.RunID, err), s.drv.Remove(context.WithoutCancel(ctx), id)) + } + sb.displayUID = uid + } manifest, err := fsdiff.Walk(sb.roots, nil) if err != nil { return errors.Join(err, s.drv.Remove(context.WithoutCancel(ctx), id)) @@ -310,207 +328,6 @@ func (s *Service) runtime(ctx context.Context) (string, error) { return "runc", nil } -// ExecRequest is one tool call. -type ExecRequest struct { - RunID string - SandboxID string - CallID string - Argv []string - Cwd string - User string - Timeout time.Duration -} - -// Change is a file change with its attribution. -type Change struct { - fsdiff.Change - Protected bool - Ambiguous bool - Candidates []string - // Content is set when the new content is among the result's blobs. - Content bool -} - -// ExecResult is everything one call produced. -type ExecResult struct { - ExitCode int - TimedOut bool - Duration time.Duration - Stdout Output - Stderr Output - Background []Change - Changes []Change - Processes []driver.Process - Blobs []Blob -} - -// Exec runs one call and reports its output, the changes around it, and what it left -// running. -func (s *Service) Exec(ctx context.Context, req ExecRequest) (ExecResult, error) { - if len(req.Argv) == 0 { - return ExecResult{}, fmt.Errorf("%w: call %s has an empty argv", ErrInvalid, req.CallID) - } - if req.Timeout <= 0 { - return ExecResult{}, fmt.Errorf("%w: call %s has no timeout", ErrInvalid, req.CallID) - } - if req.CallID == "" { - return ExecResult{}, fmt.Errorf("%w: call id is required for attribution", ErrInvalid) - } - sb, err := s.lookup(req.RunID, req.SandboxID) - if err != nil { - return ExecResult{}, err - } - sb.mu.Lock() - defer sb.mu.Unlock() - sb.execd = true - - pre, err := fsdiff.Walk(sb.roots, sb.manifest) - if err != nil { - return ExecResult{}, err - } - baseline, err := s.processes(ctx, sb) - if err != nil { - return ExecResult{}, err - } - candidates := sb.liveCalls(baseline) - budget := s.cfg.ContentBudget - var blobs []Blob - background := s.attribute(sb, fsdiff.Diff(sb.manifest, pre), true, candidates, &budget, &blobs) - - stdout := &capBuffer{limit: s.cfg.OutputLimit} - stderrBuf := &capBuffer{limit: s.cfg.OutputLimit} - stderr := &pidWriter{next: stderrBuf} - start := time.Now() - execCtx, cancel := context.WithTimeout(ctx, req.Timeout) - code, err := s.drv.Exec(execCtx, sb.container, driver.ExecSpec{Argv: wrapArgv(req.Argv), Cwd: req.Cwd, User: req.User}, stdout, stderr) - cancel() - duration := time.Since(start) - timedOut := false - switch { - case err == nil: - case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: - timedOut, code = true, -1 - if err := s.killTree(ctx, sb, req, stderr.pid); err != nil { - return ExecResult{}, err - } - default: - return ExecResult{}, fmt.Errorf("call %s in sandbox %s: %w", req.CallID, req.SandboxID, err) - } - if err := stderr.flush(); err != nil { - return ExecResult{}, err - } - - post, err := fsdiff.Walk(sb.roots, pre) - if err != nil { - return ExecResult{}, err - } - after, err := s.processes(ctx, sb) - if err != nil { - return ExecResult{}, err - } - windowCandidates := candidates - if len(candidates) > 0 { - windowCandidates = append(slices.Clone(candidates), req.CallID) - } - changes := s.attribute(sb, fsdiff.Diff(pre, post), len(candidates) > 0, windowCandidates, &budget, &blobs) - started := sb.recordProcesses(baseline, after, req.CallID) - sb.manifest = post - - outBlob := func(c *capBuffer) Output { - out, blob := c.output(s.cfg.InlineLimit) - if blob != nil { - blobs = append(blobs, *blob) - } - return out - } - return ExecResult{ - ExitCode: code, - TimedOut: timedOut, - Duration: duration, - Stdout: outBlob(stdout), - Stderr: outBlob(stderrBuf), - Background: background, - Changes: changes, - Processes: started, - Blobs: blobs, - }, nil -} - -// killTree kills a timed-out command and its descendants from inside the sandbox, as the -// call's own user: with every capability dropped, only the same uid may signal them. -func (s *Service) killTree(ctx context.Context, sb *sandbox, req ExecRequest, pid string) error { - if pid == "" { - s.log.Warn("timed-out call printed no pid; its processes are left running and will be reported", - "run_id", req.RunID, "sandbox_id", req.SandboxID, "call_id", req.CallID) - return nil - } - killCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.HelperTimeout) - defer cancel() - argv := []string{"/bin/sh", "-c", killTreeScript, "sh", pid} - if _, err := s.drv.Exec(killCtx, sb.container, driver.ExecSpec{Argv: argv, User: req.User}, io.Discard, io.Discard); err != nil { - return fmt.Errorf("kill timed-out call %s (pid %s) in sandbox %s: %w", req.CallID, pid, req.SandboxID, err) - } - return nil -} - -func (s *Service) attribute(sb *sandbox, diff []fsdiff.Change, ambiguous bool, candidates []string, budget *int64, blobs *[]Blob) []Change { - out := make([]Change, len(diff)) - for i, c := range diff { - out[i] = Change{ - Change: c, - Protected: protected(sb.mounts, c.Path), - Ambiguous: ambiguous, - Candidates: candidates, - } - if c.Op == fsdiff.OpDelete || c.After.Kind != fsdiff.KindFile || c.After.Size > s.cfg.ContentLimit || c.After.Size > *budget { - continue - } - data, err := os.ReadFile(c.After.HostPath) - if err != nil { - continue // Gone or replaced since the walk; the hash still stands. - } - sum := sha256.Sum256(data) - if hex.EncodeToString(sum[:]) != c.After.SHA256 { - continue - } - *budget -= int64(len(data)) - *blobs = append(*blobs, Blob{SHA256: c.After.SHA256, Data: data}) - out[i].Content = true - } - return out -} - -// liveCalls lists the calls whose processes are still alive, sorted. -func (sb *sandbox) liveCalls(procs []driver.Process) []string { - seen := map[string]bool{} - for _, p := range procs { - if call, ok := sb.live[p.PID]; ok { - seen[call] = true - } - } - return slices.Sorted(maps.Keys(seen)) -} - -// recordProcesses returns processes alive after the call that were not alive before it, -// and forgets processes that have exited. -func (sb *sandbox) recordProcesses(before, after []driver.Process, callID string) []driver.Process { - existed := map[int32]bool{} - for _, p := range before { - existed[p.PID] = true - } - alive := map[int32]bool{} - var started []driver.Process - for _, p := range after { - alive[p.PID] = true - if !existed[p.PID] { - started = append(started, p) - sb.live[p.PID] = callID - } - } - maps.DeleteFunc(sb.live, func(pid int32, _ string) bool { return !alive[pid] }) - return started -} - // SandboxChanges are one sandbox's changes found by FinalDiff. type SandboxChanges struct { SandboxID string diff --git a/go/internal/sandboxd/service_test.go b/go/internal/sandboxd/service_test.go index 96784f1..2495825 100644 --- a/go/internal/sandboxd/service_test.go +++ b/go/internal/sandboxd/service_test.go @@ -2,6 +2,7 @@ package sandboxd import ( "archive/tar" + "cmp" "context" "errors" "fmt" @@ -35,6 +36,12 @@ type fakeDriver struct { networks []driver.NetworkSpec netsGone []driver.NetworkID failNet string // CreateNetwork fails for this name + // files are regular files outside the key paths, for collectScript. + files map[string][]byte + // defaultUID is what a command run without a user gets; empty reads as root. + defaultUID string + // display plays `swarm-display start`: its output and the processes it leaves. + display func(stdout io.Writer) (int, error) } func (f *fakeDriver) Runtimes(context.Context) ([]string, error) { return f.runtimes, nil } @@ -100,9 +107,28 @@ func (f *fakeDriver) CreateContainer(_ context.Context, spec driver.ContainerSpe } func (f *fakeDriver) Exec(ctx context.Context, _ driver.ContainerID, spec driver.ExecSpec, stdout, stderr io.Writer) (int, error) { + if spec.Argv[0] == "swarm-display" { + f.mu.Lock() + f.execs = append(f.execs, spec) + f.mu.Unlock() + return f.display(stdout) + } if spec.Argv[2] == processScript { return f.listing(stdout) } + if spec.Argv[2] == defaultUIDScript { + _, err := io.WriteString(stdout, cmp.Or(f.defaultUID, "0")+"\n") + return 0, err + } + if spec.Argv[2] == collectScript { + data, ok := f.files[spec.Argv[4]] + if !ok { + return 4, nil + } + delete(f.files, spec.Argv[4]) + _, err := stdout.Write(data) + return 0, err + } f.mu.Lock() f.execs = append(f.execs, spec) f.mu.Unlock() diff --git a/mise.toml b/mise.toml index 2d422b0..1e63351 100644 --- a/mise.toml +++ b/mise.toml @@ -48,6 +48,10 @@ description = "sandboxd against the local docker daemon (needs busybox:latest an dir = "go" run = "go test -tags integration -count=1 ./..." +[tasks."display:build"] +description = "Build swarmeval/display:dev, the sandbox image for the browser and computer tools. Downloads Debian packages and Chromium" +run = "docker build -f deploy/images/display/Dockerfile -t swarmeval/display:dev deploy/images/display" + [tasks."deploy:smoke"] description = "The compose stack with a recorded model backend: a case created, run, and read through the CLI and the API. Needs docker" run = "deploy/compose/smoke.sh" diff --git a/proto/swarmeval/sandbox/v1/sandbox.proto b/proto/swarmeval/sandbox/v1/sandbox.proto index cc9dc01..4569f86 100644 --- a/proto/swarmeval/sandbox/v1/sandbox.proto +++ b/proto/swarmeval/sandbox/v1/sandbox.proto @@ -19,7 +19,8 @@ service SandboxService { rpc CreateRun(CreateRunRequest) returns (CreateRunResponse); // Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - // content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + // content at that path, plus any seed files. With `display` set, the image's display stack is + // started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. rpc CreateSandbox(CreateSandboxRequest) returns (CreateSandboxResponse); // Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -92,6 +93,19 @@ message CreateSandboxRequest { // diffed. Empty writes nothing. 32 lowercase hex characters, the file's format. The image // must have /etc. string machine_id = 10; + // Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + // no display. + Display display = 11; +} + +// A virtual screen with a browser on it, run by the image's `swarm-display` as the user +// `swarmdisplay`. Its processes are never reported or attributed to a call. +message Display { + // Pixels, 320 to 1920 wide and 240 to 1200 high. + uint32 width = 1; + uint32 height = 2; + // Page the browser opens first. Empty opens about:blank. + string url = 3; } message SeedFile { @@ -118,6 +132,10 @@ message ExecRequest { string user = 6; // Required. On expiry the command and its descendants are killed. google.protobuf.Duration timeout = 7; + // Absolute paths outside every key path. After the command, sandboxd reads each regular file + // there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + // text, such as a screenshot. + repeated string collect = 8; } message ExecResponse { @@ -141,6 +159,19 @@ message ExecHeader { repeated FsChange changes = 7; // Started while this call ran and still alive after it. repeated Process processes = 8; + // One per `collect` path, in request order. + repeated CollectedFile collected = 9; +} + +message CollectedFile { + string path = 1; + // No regular file was there. + bool missing = 2; + // Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + // `sha256` is empty. + int64 size = 3; + // The content follows as a blob with this hash. + string sha256 = 4; } message Output { diff --git a/spec/2026-10-07-browser-computer-use/README.md b/spec/2026-10-07-browser-computer-use/README.md new file mode 100644 index 0000000..5330160 --- /dev/null +++ b/spec/2026-10-07-browser-computer-use/README.md @@ -0,0 +1,177 @@ +# 沙箱里的 browser 和 computer:agent 操作图形桌面和浏览器 + +## Status + +accepted · 2026-10-07 · 方案(沙箱内显示栈,只访问沙箱内)、截图进上下文、事件格式升版本且不为旧格式写兼容层、 +坐标约定、fork 处理由提出人选定(2026-10-07);其余决定是起草人的建议。 + +## Request + +提出人(2026-10-07):"我需要能让 agent 运行的时候给 sandbox 支持上 browser 和 computer 使用的能力。"在三个方案 +(A:显示栈装在沙箱里,只访问沙箱内;B:单独的浏览器出口容器上公网;C:等 net-gateway)里选了 A:"按照方案 A +就可以,让 agent 在 sandbox 里面可以使用 browser 和 computer"。 + +提问时确认的四点(2026-10-07): + +- browser 也要能截图。事件格式升版本;"如果兼容要写很多的代码就不兼容了"。 +- 先写 spec,然后直接实现。 +- 截图喂给 API 模型(Claude、GPT),用屏幕的实际像素坐标。 +- 用过 browser 或 computer 的 run,在那之后的事件上拒绝 fork。 + +## Decisions + +### 一、沙箱里的显示栈 + +#### 1. 显示栈装在沙箱里,隔离不变 + +沙箱仍然是 `--network none`。浏览器只能访问沙箱内的 `localhost` 和 `file://`,访问不了公网,也访问不了别的沙箱。 +隔离默认值(AGENTS.md "Isolation defaults")一点不变,所以不需要另外签字。上公网的浏览是方案 B / C,不在这里做。 + +#### 2. profile 的 `display` 打开显示栈 + +`env.yaml` 的 sandbox profile 加一个可选的 `display`: + +```yaml +sandbox_profiles: + desktop: + image: swarmeval/display:dev + fs: [{ path: /workspace }] + limits: { memory: 2gib, pids: 512 } + display: { width: 1024, height: 768, url: "http://127.0.0.1:8080/" } +``` + +- `width`、`height` 默认 1024×768。这个尺寸 Claude 不会缩放,GPT 的 high detail 也不会缩放,所以模型看到的像素 + 坐标就是屏幕坐标(提出人选定用像素坐标)。上限 1920×1200。 +- `url` 可选:浏览器启动后打开的第一页,默认 `about:blank`。 +- 只加键,不改已有键。按 [case 格式的版本规则](../../docs/case-format.md#versioning),新字段要升版本: + `env.yaml` 升到 2,`display` 只在 2 里有效,版本 1 照常读。 + +#### 3. 显示镜像的约定 + +`deploy/images/display/` 构建 `swarmeval/display`:Debian、Xvfb、一个让每个窗口都最大化的窗口管理器 +(matchbox)、xdotool、Playwright 1.63 和它的 Chromium,再加 `swarm-display` 程序。case 作者用 +`FROM swarmeval/display` 加自己的应用。镜像必须提供: + +- 用户 `swarmdisplay`,以及只属于它、权限 `0700` 的 `/run/swarm-display`。 +- `swarm-display start|computer|browser`。 +- 沙箱镜像本来就要求的 `sh`、`sleep`、`tr`。 + +`/etc/swarm-display/start.d/` 里的可执行文件在浏览器打开之前依次以 `swarmdisplay` 运行,用来启动 case 自己的应用, +比如 `127.0.0.1:8080` 上的网店。 + +#### 4. sandboxd 在建沙箱时启动它 + +`CreateSandboxRequest` 加 `display`(宽、高、起始 URL)。设置了它时,sandboxd 在容器启动之后、取第一份 manifest +之前,以 `swarmdisplay` 运行 `swarm-display start`。这个命令等显示栈就绪后退出,在 stdout 上打印守护进程的 pid。 +超时、退出码非 0、或者输出不是一个 pid,都让 `CreateSandbox` 失败。 + +显示栈的 `HOME`、`TMPDIR`、浏览器 profile 都放在 `/run/swarm-display` 下,不在任何 key path 里,所以不产生文件 +变更事件。一个 key path 如果是 `/run/swarm-display`、在它下面、或者包含它,就拒绝。 + +#### 5. 显示用户的进程不算任何调用的进程 + +浏览器每次导航都会起新的渲染进程。要是按现在的规则,它们会记成某次调用留下的进程,之后所有调用的文件变更都会 +变成 `ambiguous`。所以 sandboxd 记下守护进程的数字 uid(从 `start` 打印的 pid 查),之后这个 uid 的进程既不进 +`processes`,也不参与归属判断。 + +代价:以 root 运行的 agent 可以把自己的进程切到这个 uid 下藏起来。不过 root agent 本来就能做很多事,这一点写进 +文档。镜像里没有终端程序,所以 `computer` 能起的程序只有 Chromium 和 case 作者自己加的。 + +### 二、工具 + +#### 6. `computer` 和 `browser` 是 SandboxTool,以显示用户运行 + +两个都是内置工具,agent 在 `tools` 里列出才有。agent 的沙箱 profile 没有 `display` 时,case 加载失败。 + +- `Exec` 加 `user`:工具自己指定运行用户,覆盖 agent 的 `os_user`。这两个工具都以 `swarmdisplay` 运行 + `swarm-display computer|browser `。 +- `swarm-display` 通过 `/run/swarm-display` 里的 unix socket 把请求交给守护进程。X 服务器有 cookie 认证,浏览器由 + 守护进程通过 pipe 控制(Playwright 默认),不开 CDP 端口。所以非 root 的 agent 绕过这两个工具就碰不到屏幕和 + 浏览器。 + +**`computer`**:`screenshot`、`click`(`button`、`count`)、`mouse_move`、`drag`、`type`、`key`、`scroll`、`wait`、 +`cursor_position`。坐标是 `[x, y]` 屏幕像素。除了 `cursor_position`,每个动作之后都返回一张截图,文字里写屏幕尺寸。 + +**`browser`**:`navigate`、`back`、`forward`、`reload`、`snapshot`、`click`、`type`、`select`、`press`、`hover`、 +`scroll`、`wait`、`tabs`、`tab_new`、`tab_select`、`tab_close`、`screenshot`。 + +- 每个动作都返回 URL、标题,以及 Playwright `aria_snapshot(mode="ai")` 产生的页面快照,快照里每个元素带 + `[ref=eN]`。元素用 `ref` 指定,内部通过 `aria-ref=` 选择器定位。Playwright 1.64 发布后换成 `page.get_by_ref`。 +- `screenshot` 动作,或者任何动作加 `screenshot: true`,会额外附上视口的截图(提出人要求 browser 也能截图)。 + +#### 7. 截图怎么出沙箱:`Exec.collect` + +`ExecRequest` 加 `collect`:命令结束后,sandboxd 以这次调用的用户读出这些路径上的普通文件,读完就删掉,作为 blob 流回去; +`ExecHeader.collected` 列出每个文件的路径、大小、sha256,以及是否缺失、是否超过上限(8 MiB)。 + +以调用的用户读,是因为沙箱 drop 了所有 capability,root 读不了别的用户的 `0700` 目录;而且一次调用能收走的,本来就 +是它的用户能读到的。两个工具都收 `/run/swarm-display/out/screenshot.png`。截图不写进 key path,所以不产生文件变更事件,agent 也看不到。 +worker 在边界上检查每张截图:PNG 签名、IHDR 里的宽高、尺寸上限;不合格的截图变成工具错误。 + +### 三、图片进上下文 + +#### 8. 消息带图片引用,不带图片字节 + +- `ToolResult` 和 `ToolMessage` 加 `images`:`ImageRef(sha256, media_type, width, height)` 的元组。图片字节在 + blob store 里,按哈希引用。`messages` 行和事件里只有引用。 +- `after_tool_result` hook 可以改写或去掉图片;改写照常记成 `intervention`。 +- 只有工具结果带图片。用户消息、频道消息、系统提示不带。 + +#### 9. 每次请求最多带最近 3 张图 + +截图累积很快。`RequestOptions` 加 `max_images`,默认 3:一次请求只带上下文里最后 3 张图,更早的图换成文字 +`[image omitted]`。`before_model_request` 可以改这个值;它记在 `ModelEvent` 里,导出时按同样的规则展开 `input`, +所以 `.eval` 里展示的就是模型实际看到的内容。 + +#### 10. gateway 协议:图片放在 tool 消息里,发给后端时挪到 user 消息 + +- worker 到 gateway:`tool` 消息的 `content` 可以是内容块列表(`text`、`image_url` 带 `data:image/png;base64,` + URL)。这是 SwarmEval 自己子集的扩展,gateway 照常校验。 +- gateway 到后端:OpenAI Chat Completions 的 tool 消息只能放文本。所以 gateway 在做完 reasoning passback **之后**, + 把 tool 消息里的图片挪到紧跟在这一串 tool 消息后面的一条 user 消息里,每张图前面标上它属于哪次工具调用。放在 + passback 之后,是因为 `within_turn` 以"最后一条 user 消息"为界,挪出来的 user 消息不能把这个边界移动。 +- worker 组请求时从 blob store 取图片;同一个 run 里同一个哈希只取一次。 + +#### 11. 事件格式 10,不写兼容层 + +- `metadata.swarmeval.schema_version` 升到 10。新增:`ToolEvent` 的 `images`;`ModelEvent` 的 `max_images`; + `exec` 观测里的 `collected`。 +- 新字段都是可选的,所以旧 run 不需要额外代码就照常能读。提出人同意,需要大量兼容代码时可以不兼容;这里不需要。 +- `ToolEvent.result` 在 Postgres 里仍然是文本,读它的地方(detect、analysis、transcript)都不用改。 +- 导出 `.eval` 时,带图的 `ToolEvent.result` 和 `ChatMessageTool` 变成 `ContentText` 加 + `ContentImage("attachment://")`,图片作为 `data:` URI 放进 `EvalSample.attachments`,每张图只放一次。 + 这样 Inspect View 可以直接看到截图,`.eval` 不依赖 blob store。 + +### 四、fork + +#### 12. 用过显示工具之后,不能再 fork + +fork 只恢复 key path 里的文件,浏览器和桌面的内存状态恢复不了。所以,如果 fork 点之前有执行过的 `browser` 或 +`computer` 调用(`executed_arguments` 不为空),就拒绝这次 fork,错误信息里说明原因(提出人选定)。在第一次 +显示工具调用之前 fork 照常可以。 + +## Rejected + +- **浏览器放在 worker 里,或者单独的出口容器(方案 B)。** 要放宽隔离默认值,还会多一个事件来源。提出人选了 A。 +- **截图走 stdout。** stdout 会转成文本,截图又要再存一份 blob;而且工具自己的文字输出就没地方放了。 +- **用 `ReadFile`(docker copy API)读截图。** gVisor 下,copy API 看不到容器 rootfs 覆盖层里的写入。 +- **截图写进 key path。** 会产生文件变更事件,agent 也能读到、能改。 +- **按 ref 定位只用公开 API(`boxes=True` 加鼠标坐标)。** 元素在视口外时会点错。 +- **tool 结果的图片在 worker 里就拆成 user 消息。** 会移动 `within_turn` 的 passback 边界;而且事件里存的上下文 + 和模型实际看到的不一致。 +- **所有截图都发给模型。** 上下文会无限增长。 +- **只按 pid 子树排除显示栈的进程。** Chromium 的一些辅助进程会脱离父进程、被重新挂到 pid 1 上,这样就排除不掉。 + +## Open questions + +(无) + +## Plan + +1. proto 和 sandboxd:`display`、`collect`、按显示 uid 排除进程。Go 单元测试,以及用真实显示镜像跑的集成测试。 +2. 显示镜像和 `swarm-display`。 +3. Python:消息和记录、两个工具、sandbox 客户端、gateway 协议和后端转换、事件格式 10、导出、fork 拒绝、case 加载。 +4. 文档:sandboxd、agent-runtime、case-format、event-log、model-gateway、architecture,以及 CHANGELOG。 + +退出条件:`mise run check` 通过;在 docker 里用显示镜像跑通 `computer` 截图和点击、`browser` 打开沙箱内的页面 +并点击 ref;`.eval` 里的截图能解析出来。 diff --git a/swarmeval/core/loader.py b/swarmeval/core/loader.py index d29dd57..25ba379 100644 --- a/swarmeval/core/loader.py +++ b/swarmeval/core/loader.py @@ -35,6 +35,7 @@ SandboxPlan, check_canaries, check_crossing, + check_displays, plan_sandboxes, seed_files, ) @@ -261,6 +262,7 @@ def _variant( env_where = f"{env_path}" + (f" (variant {values})" if values else "") sandboxes = plan_sandboxes(case, env, env_path) check_canaries(env, sandboxes, env_where) + check_displays(case, env, sandboxes, where) seeds = seed_files(env, sandboxes, files.tree, env_where) scripts: dict[str, str] = {} for scorer in case.scorers: diff --git a/swarmeval/core/models.py b/swarmeval/core/models.py index 30f5938..dedba91 100644 --- a/swarmeval/core/models.py +++ b/swarmeval/core/models.py @@ -24,6 +24,7 @@ ) from swarmeval.detect.defs import DetectorDef +from swarmeval.runtime.display import DISPLAY_STATE, DISPLAY_TOOLS, DISPLAY_USER from swarmeval.runtime.extensions import ExtensionUse from swarmeval.runtime.tools import BUILTIN_TOOLS, SandboxTool @@ -41,7 +42,8 @@ DEFAULT_SLOT = "default" """The model slot of an agent that names none.""" -ENV_SCHEMA_VERSIONS = frozenset({1}) +ENV_SCHEMA_VERSIONS = frozenset({1, 2}) +"""`env.yaml` versions read. Version 2 adds a profile's `display`.""" Name = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_]*$", max_length=63)] """Ids of cases, agents, channels, sandboxes, profiles, and variant axes. They end up in @@ -334,9 +336,22 @@ def slots(self) -> tuple[str, ...]: def has_sandbox(self, agent: AgentDef) -> bool: return self.schema_version < NO_SANDBOX_VERSION or agent.sandbox != NO_SANDBOX + @model_validator(mode="after") + def _display_user_is_reserved(self) -> Self: + for agent in self.swarm.agents: + if agent.os_user == DISPLAY_USER: + raise ValueError( + f"agent `{agent.id}` sets `os_user: {DISPLAY_USER}`, the user that runs a " + "sandbox's display: its commands could reach the screen and the browser " + "around the `browser` and `computer` tools. Pick another user." + ) + return self + @model_validator(mode="after") def _sandboxless_agents_run_nothing(self) -> Self: - sandbox_tools = {t.name for t in BUILTIN_TOOLS if isinstance(t, SandboxTool)} + sandbox_tools = { + t.name for t in (*BUILTIN_TOOLS, *DISPLAY_TOOLS) if isinstance(t, SandboxTool) + } for agent in self.swarm.agents: if self.has_sandbox(agent): continue @@ -398,17 +413,47 @@ class FileCopy(Strict): """Where it lands. A directory's contents go under it.""" +class Display(Strict): + """A virtual screen with a browser on it (docs/case-format.md#display). Needs an image built + from `swarmeval/display`.""" + + width: Annotated[int, Field(ge=320, le=1920)] = 1024 + height: Annotated[int, Field(ge=240, le=1200)] = 768 + url: Annotated[str, Field(min_length=1, max_length=2048, pattern=r"^[^\x00\r\n]*$")] | None = ( + None + ) + """Page the browser opens first; `about:blank` when unset.""" + + class SandboxProfile(Strict): image: Annotated[str, Field(min_length=1)] fs: tuple[Mount, ...] = () limits: ResourceLimits = ResourceLimits() files: tuple[FileCopy, ...] = () + display: Display | None = None @model_validator(mode="after") def _unique_mounts(self) -> Self: _no_duplicates("mount path", [m.path for m in self.fs]) return self + @model_validator(mode="after") + def _display_state_outside_key_paths(self) -> Self: + if self.display is None: + return self + for m in self.fs: + p = m.path + if ( + p == "/" + or DISPLAY_STATE.startswith(p + "/") + or (p + "/").startswith(DISPLAY_STATE + "/") + ): + raise ValueError( + f"key path `{p}` overlaps `{DISPLAY_STATE}`, where the display keeps its " + "state out of the agent's reach. Mount a path outside it." + ) + return self + class SandboxInstance(Strict): profile: Name @@ -444,6 +489,17 @@ class EnvFile(Strict): version 5, one with `sandbox: none` gets none.""" canaries: tuple[CanaryDef, ...] = () + @model_validator(mode="after") + def _display_needs_version_2(self) -> Self: + if self.schema_version < 2: + for name, profile in self.sandbox_profiles.items(): + if profile.display is not None: + raise ValueError( + f"profile `{name}` has `display`, which `schema_version: 2` added. Set " + "`schema_version: 2`." + ) + return self + @model_validator(mode="after") def _unique_canaries(self) -> Self: _no_duplicates("canary", [c.id for c in self.canaries]) diff --git a/swarmeval/core/topology.py b/swarmeval/core/topology.py index 93b7787..c70518c 100644 --- a/swarmeval/core/topology.py +++ b/swarmeval/core/topology.py @@ -10,6 +10,7 @@ from swarmeval.core.errors import CaseError from swarmeval.core.models import NO_SANDBOX, NO_SANDBOX_VERSION, CaseFile, EnvFile +from swarmeval.runtime.display import DISPLAY_TOOL_NAMES Tree = Callable[[str, str], list[tuple[str, Path]]] """A case path and the field naming it → each file under it, relative to it, and its path.""" @@ -52,6 +53,26 @@ def check_crossing( ) +def check_displays( + case: CaseFile, env: EnvFile, sandboxes: Mapping[str, SandboxPlan], where: str +) -> None: + """`browser` and `computer` drive a sandbox's display, so the agent's profile needs one. An + agent with no sandbox cannot list them (`CaseFile`).""" + tools = {a.id: a.tools for a in case.swarm.agents} + for plan in sandboxes.values(): + if env.sandbox_profiles[plan.profile].display is not None: + continue + for agent_id in plan.agents: + wanted = [t for t in tools[agent_id] if t in DISPLAY_TOOL_NAMES] + if wanted: + raise CaseError( + f"{where}: agent `{agent_id}` lists {', '.join(f'`{t}`' for t in wanted)}, " + f"but its sandbox `{plan.id}` has profile `{plan.profile}`, which has no " + "`display`. Add `display: {}` to the profile and use an image built from " + "swarmeval/display." + ) + + def check_canaries(env: EnvFile, sandboxes: Mapping[str, SandboxPlan], where: str) -> None: for canary in env.canaries: plan = sandboxes.get(canary.sandbox) diff --git a/swarmeval/events/__init__.py b/swarmeval/events/__init__.py index 9dc123b..841fd09 100644 --- a/swarmeval/events/__init__.py +++ b/swarmeval/events/__init__.py @@ -10,6 +10,7 @@ RunHeader, StoredRun, assemble, + blob_key, export_key, export_run, load_run, @@ -54,6 +55,7 @@ "StoredRun", "VariantLogError", "assemble", + "blob_key", "events_key", "export_events", "export_key", diff --git a/swarmeval/events/convert.py b/swarmeval/events/convert.py index 972bdfe..cba6f89 100644 --- a/swarmeval/events/convert.py +++ b/swarmeval/events/convert.py @@ -8,6 +8,7 @@ import json import math import shlex +from collections.abc import Sequence from dataclasses import dataclass from typing import Any, Literal @@ -27,6 +28,7 @@ ChatMessageTool, ChatMessageUser, Content, + ContentImage, ContentReasoning, ContentText, GenerateConfig, @@ -42,12 +44,15 @@ from pydantic import JsonValue from swarmeval.runtime.messages import ( + IMAGE_OMITTED, AssistantMessage, ChatMessage, + ImageRef, SystemMessage, ToolCall, ToolMessage, UserMessage, + visible_images, ) from swarmeval.runtime.records import ( Exec, @@ -61,9 +66,14 @@ ToolCallRecord, ) -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 """Version of the `metadata.swarmeval` extension. Bump it when any field below changes shape. +10: tool events may carry `images` (`ImageRef`s: a blob hash, media type, and size) from the +`computer` and `browser` tools; model events carry `max_images`, how many of the context's +latest images the request sent; `exec` observations carry `collected`, the files sandboxd read +out of the sandbox after the call. Version 9 runs have none of these and read as before. + 9: a fork's `intervention` (`hook: fork`) may be `replace_model`, whose `after` names the model slot and the models before and after; such a fork's agents of that slot run the new model. A run's `task_args` name its models as `model.`. Version 8 runs have neither and read as @@ -245,6 +255,7 @@ def _model_event(record: ModelCallRecord) -> tuple[ModelEvent, dict[str, JsonVal # What the model produced, byte for byte: Inspect's tool calls hold parsed arguments. "raw_tool_arguments": {c.id: c.arguments for c in record.response.tool_calls}, "gateway": record.gateway.model_dump(mode="json"), + "max_images": options.max_images, } return event, extra @@ -273,6 +284,8 @@ def _tool_event(record: ToolCallRecord) -> tuple[ToolEvent, dict[str, JsonValue] extra["exec"] = _exec_observations(record.exec_result) if record.web is not None: extra["web"] = record.web.model_dump(mode="json", exclude={"body"}) + if result.images: + extra["images"] = [i.model_dump(mode="json") for i in result.images] return event, extra @@ -362,7 +375,24 @@ def from_inspect_assistant( ) -def to_inspect_message(message: ChatMessage) -> InspectMessage: +ATTACHMENT = "attachment://" +"""Inspect's reference into `EvalSample.attachments`. Images are keyed by their sha256, and the +export puts each one's bytes there once.""" + + +def image_content( + images: Sequence[ImageRef], shown: Sequence[bool] +) -> list[ContentText | ContentImage]: + """Images as Inspect content; one not `shown` to the model is the text it saw instead.""" + return [ + ContentImage(image=f"{ATTACHMENT}{i.sha256}") if show else ContentText(text=IMAGE_OMITTED) + for i, show in zip(images, shown, strict=True) + ] + + +def to_inspect_message(message: ChatMessage, shown: Sequence[bool] = ()) -> InspectMessage: + """`shown` says, per image of a tool message, whether the request carried it; images past + its end were carried.""" match message: case SystemMessage(): return ChatMessageSystem(content=message.content) @@ -372,6 +402,28 @@ def to_inspect_message(message: ChatMessage) -> InspectMessage: return to_inspect_assistant(message) case ToolMessage(): error = ToolCallError("unknown", message.content) if message.is_error else None - return ChatMessageTool( - content=message.content, tool_call_id=message.tool_call_id, error=error - ) + content: str | list[Content] = message.content + if message.images: + visible = [*shown, *([True] * (len(message.images) - len(shown)))] + content = [ + ContentText(text=message.content), + *image_content(message.images, visible), + ] + return ChatMessageTool(content=content, tool_call_id=message.tool_call_id, error=error) + + +def to_inspect_context( + messages: Sequence[ChatMessage], max_images: int | None +) -> list[InspectMessage]: + """A request's context as the model saw it: with `max_images`, only the latest images; `None` + (events before schema 10) shows every image.""" + if max_images is None: + return [to_inspect_message(m) for m in messages] + keep = visible_images(messages, max_images) + return [ + to_inspect_message( + m, + [(i, j) in keep for j in range(len(m.images))] if isinstance(m, ToolMessage) else (), + ) + for i, m in enumerate(messages) + ] diff --git a/swarmeval/events/export.py b/swarmeval/events/export.py index cbab283..bed7e74 100644 --- a/swarmeval/events/export.py +++ b/swarmeval/events/export.py @@ -5,14 +5,17 @@ """ import asyncio +import base64 import copy +import dataclasses +import hashlib import tempfile from collections import defaultdict -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from importlib.metadata import version from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, cast from inspect_ai.event import ( Event, @@ -22,6 +25,7 @@ ScoreEvent, SpanBeginEvent, SpanEndEvent, + ToolEvent, ) from inspect_ai.log import ( EvalConfig, @@ -36,7 +40,7 @@ EvalSpec, write_eval_log, ) -from inspect_ai.model import ModelUsage +from inspect_ai.model import ContentImage, ContentText, ModelUsage from inspect_ai.scorer import Score from pyarrow import NativeFile from pyarrow.fs import S3FileSystem @@ -46,13 +50,14 @@ from swarmeval.db import control_runs, events, messages, run_specs from swarmeval.events.chain import ChainRow, ChainStart, verify -from swarmeval.events.convert import SCHEMA_VERSION, to_inspect_message -from swarmeval.runtime.messages import ChatMessage +from swarmeval.events.convert import SCHEMA_VERSION, to_inspect_context +from swarmeval.runtime.messages import ChatMessage, ImageRef, ToolMessage from swarmeval.runtime.records import LifecycleRecord _EVENT = TypeAdapter[Event](Event) _MESSAGE = TypeAdapter[ChatMessage](ChatMessage) _LIFECYCLE = TypeAdapter[LifecycleRecord](LifecycleRecord) +_IMAGES = TypeAdapter[list[ImageRef]](list[ImageRef]) @dataclass(frozen=True) @@ -84,6 +89,8 @@ class StoredRun: forked_from: str | None = None chain_start: ChainStart | None = None """For a fork: where its chain links into its source's.""" + images: Mapping[str, bytes] = dataclasses.field(default_factory=dict[str, bytes]) + """Every image `image_hashes` names, by sha256. The export embeds them.""" @dataclass(frozen=True) @@ -128,6 +135,39 @@ def export_key(run_id: str) -> str: return f"runs/{run_id}/sample.eval" +def blob_key(sha256: str) -> str: + """Where the blob store keeps content with this hash (docs/event-log.md#large-objects).""" + return f"blobs/sha256/{sha256}" + + +def image_hashes(run: StoredRun) -> set[str]: + """Every image a run's contexts or tool events name.""" + hashes = { + i.sha256 + for context in run.messages.values() + for m in context + if isinstance(m, ToolMessage) + for i in m.images + } + for row in run.events: + ours = cast(dict[str, Any], row.payload)["metadata"]["swarmeval"] + hashes.update(i.sha256 for i in _IMAGES.validate_python(ours.get("images", []))) + return hashes + + +async def load_images(store: ObjectStore, hashes: Iterable[str]) -> dict[str, bytes]: + async def one(sha256: str) -> tuple[str, bytes]: + data = await asyncio.to_thread(store.get, blob_key(sha256)) + if hashlib.sha256(data).hexdigest() != sha256: + raise ValueError( + f"blob {blob_key(sha256)} does not hash to its name. The object store was " + "changed after upload; do not export from it." + ) + return sha256, data + + return dict(await asyncio.gather(*(one(h) for h in sorted(hashes)))) + + async def load_run(engine: AsyncEngine, run_id: str) -> StoredRun: async with engine.connect() as conn: workspace = ( @@ -185,6 +225,12 @@ def assemble(header: RunHeader, run: StoredRun) -> EvalLog: "after the run has started." ) verify(header.run_id, run.events, run.chain_start) + missing = image_hashes(run) - run.images.keys() + if missing: + raise ValueError( + f"run {header.run_id} names {len(missing)} image(s) the export was not given, such " + f"as {min(missing)}. Load them with `load_images`." + ) stored = [_restore(row, run) for row in run.events] sample_events = _with_agent_spans(stored, tuple(header.models)) first, last = stored[0].timestamp, stored[-1].timestamp @@ -204,6 +250,7 @@ def assemble(header: RunHeader, run: StoredRun) -> EvalLog: error=error, limit=_limit(stored), scores=scores or None, + attachments={sha256: _data_url(data) for sha256, data in run.images.items()}, metadata={ "swarmeval": { "schema_version": SCHEMA_VERSION, @@ -265,7 +312,9 @@ def write_eval(log: EvalLog, path: Path) -> None: async def export_run(engine: AsyncEngine, header: RunHeader, store: ObjectStore) -> str: """Writes the run's `.eval` to `store` and returns its key within the bucket.""" - log = assemble(header, await load_run(engine, header.run_id)) + run = await load_run(engine, header.run_id) + run = dataclasses.replace(run, images=await load_images(store, image_hashes(run))) + log = assemble(header, run) key = export_key(header.run_id) await asyncio.to_thread(_upload, log, store, key) return key @@ -289,10 +338,21 @@ def _restore(row: ChainRow, run: StoredRun) -> Event: ref = ours["input"] if ref["gen"] is not None: context = run.messages[(ours["agent_id"], ref["gen"])][: ref["len"]] - event.input = [to_inspect_message(m) for m in context] + event.input = to_inspect_context(context, ours.get("max_images")) + if isinstance(event, ToolEvent) and ours.get("images"): + # Inline, as Inspect writes tool results: it resolves `attachment://` in messages only. + images = _IMAGES.validate_python(ours["images"]) + event.result = [ + ContentText(text=str(event.result)), + *(ContentImage(image=_data_url(run.images[i.sha256])) for i in images), + ] return event +def _data_url(png: bytes) -> str: + return f"data:image/png;base64,{base64.b64encode(png).decode()}" + + def _agent_of(event: Event) -> str | None: assert event.metadata is not None, "every stored event carries metadata.swarmeval" return event.metadata["swarmeval"]["agent_id"] diff --git a/swarmeval/events/fork.py b/swarmeval/events/fork.py index 4a97436..9f8ac83 100644 --- a/swarmeval/events/fork.py +++ b/swarmeval/events/fork.py @@ -16,6 +16,7 @@ from swarmeval.db import canaries, checkpoints, events, messages, run_specs from swarmeval.events.seal import ChainHead +from swarmeval.runtime.display import DISPLAY_TOOL_NAMES from swarmeval.runtime.extensions import CanaryInfo, SandboxCanaryInfo from swarmeval.runtime.messages import ChatMessage from swarmeval.runtime.records import Checkpoint @@ -74,6 +75,29 @@ async def point_at(engine: AsyncEngine, run_id: str, seq: int, what: str = "") - "checkpoints, or the run was recorded before forks were possible. Fork at a " "later event of a `round_robin` or `event_driven` run." ) + display = ( + await conn.execute( + select(events.c.event_id, events.c.payload["function"].astext) + .where( + events.c.run_id == run_id, + events.c.seq <= row.seq, + events.c.type == "tool", + events.c.payload["function"].astext.in_(DISPLAY_TOOL_NAMES), + events.c.payload["metadata"]["swarmeval"]["executed_arguments"].astext.is_not( + None + ), + ) + .order_by(events.c.seq) + .limit(1) + ) + ).one_or_none() + if display is not None: + raise ForkPointError( + f"run `{run_id}` used `{display[1]}` (event `{display[0]}`) before the turn " + f"holding {what or f'seq {seq + 1}'}. A fork restores files but not a browser or " + "a screen, so it cannot go on from there. Fork at an event before the first " + "`browser` or `computer` call." + ) checkpoint = Checkpoint.model_validate(row.state) if checkpoint.spawned: raise ForkPointError( diff --git a/swarmeval/events/transcript.py b/swarmeval/events/transcript.py index bc61b7e..fa204a7 100644 --- a/swarmeval/events/transcript.py +++ b/swarmeval/events/transcript.py @@ -19,7 +19,7 @@ from swarmeval.db import events, messages from swarmeval.events.convert import from_inspect_assistant from swarmeval.events.fork import lineage -from swarmeval.runtime.messages import AssistantMessage, ChatMessage, RequestOptions +from swarmeval.runtime.messages import AssistantMessage, ChatMessage, ImageRef, RequestOptions from swarmeval.runtime.records import ( GatewayRecord, InterventionRecord, @@ -31,6 +31,7 @@ _EVENT = TypeAdapter[Event](Event) _TYPES = ("model", "tool", "swarmeval.intervention", "swarmeval.msg.send", "swarmeval.msg.deliver") _MESSAGE = TypeAdapter[ChatMessage](ChatMessage) +_IMAGES = TypeAdapter[list[ImageRef]](list[ImageRef]) @dataclass(frozen=True) @@ -123,14 +124,16 @@ async def load_transcript(engine: AsyncEngine, run_id: str) -> RunTranscript: case ModelEvent() as event: model_calls.append(_model_call(event_id, agent_id, event)) case ToolEvent() as event: + assert event.metadata is not None, "stored events carry metadata.swarmeval" + ours = event.metadata["swarmeval"] result = ToolResult( call_id=event.id, tool=event.function, content=str(event.result), is_error=bool(event.failed), + images=tuple(_IMAGES.validate_python(ours.get("images", []))), ) - assert event.metadata is not None, "stored events carry metadata.swarmeval" - executed = event.metadata["swarmeval"]["executed_arguments"] + executed = ours["executed_arguments"] tool_results.append( ToolOutcome(event_id, agent_id, parent_id, result, executed) ) @@ -179,6 +182,7 @@ def _model_call(event_id: str, agent_id: str | None, event: ModelEvent) -> Model top_p=config.top_p, max_output_tokens=config.max_tokens, seed=config.seed, + **({"max_images": ours["max_images"]} if "max_images" in ours else {}), ), response=from_inspect_assistant( event.output.choices[0].message, ours["raw_tool_arguments"] diff --git a/swarmeval/gateway/model/client.py b/swarmeval/gateway/model/client.py index f225813..1bbdce6 100644 --- a/swarmeval/gateway/model/client.py +++ b/swarmeval/gateway/model/client.py @@ -11,13 +11,14 @@ """ import asyncio +import base64 import hashlib import json import secrets -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from types import TracebackType -from typing import Literal, Self +from typing import Literal, Protocol, Self import grpc import httpx2 @@ -30,8 +31,12 @@ WireAssistantMessage, WireFunction, WireFunctionCall, + WireImagePart, + WireImageURL, WireMessage, + WirePart, WireSystemMessage, + WireTextPart, WireTool, WireToolCall, WireToolMessage, @@ -40,6 +45,7 @@ from swarmeval.proto.swarmeval.modelgw.v1 import recorder_pb2 as pb from swarmeval.proto.swarmeval.modelgw.v1.recorder_pb2_grpc import RecorderServiceStub from swarmeval.runtime.messages import ( + IMAGE_OMITTED, AssistantMessage, ChatMessage, ModelRequest, @@ -49,6 +55,7 @@ ToolMessage, Usage, UserMessage, + visible_images, ) from swarmeval.runtime.ports import AgentCaller, Caller, ExtensionCaller, RecordedResponse from swarmeval.runtime.records import ( @@ -65,6 +72,12 @@ """model-gateway's status for a call the model backend failed after retries.""" +class ImageSource(Protocol): + async def get(self, sha256: str) -> bytes: + """The image stored under `sha256`, checked against it.""" + ... + + class ModelGatewayError(Exception): """model-gateway failed a call, or broke the recording protocol. `status` is the HTTP status when the gateway answered with an error; 503 means the run is not attached.""" @@ -106,9 +119,11 @@ def __init__( owner_epoch: int, callers: Sequence[Caller], writer: RunWriter, + images: ImageSource, ) -> None: """`http` is based at the gateway's HTTP address and should have no read timeout: a - call lasts as long as the model takes.""" + call lasts as long as the model takes. `images` holds the images tools returned, which + requests carry inline.""" self._http = http self._stub = RecorderServiceStub(channel) self._run_id = run_id @@ -120,6 +135,9 @@ def __init__( self._stream: grpc.aio.StreamStreamCall[pb.AttachRequest, pb.AttachResponse] | None = None self._reader: asyncio.Task[None] | None = None self._broken: BaseException | None = None + self._images = images + self._sent_images: dict[str, bytes] = {} + """The last request's images. The next one mostly carries the same ones.""" async def __aenter__(self) -> Self: stream = self._stub.Attach() @@ -167,7 +185,8 @@ async def generate( name = caller_name(caller) self._calls += 1 call_id = f"call_{self._calls}" - body = to_wire(request).model_dump_json(exclude_none=True).encode() + wire = to_wire(request, await self._fetch_images(request)) + body = wire.model_dump_json(exclude_none=True).encode() pending = _Pending(caller, request, hashlib.sha256(body).hexdigest(), parent_id) self._pending[call_id] = pending headers = { @@ -200,6 +219,19 @@ async def generate( ) return RecordedResponse(response=response, event=event) + async def _fetch_images(self, request: ModelRequest) -> dict[str, bytes]: + wanted = { + m.images[j].sha256 + for i, j in visible_images(request.messages, request.options.max_images) + if isinstance(m := request.messages[i], ToolMessage) + } + fetched = { + sha256: self._sent_images.get(sha256) or await self._images.get(sha256) + for sha256 in sorted(wanted) + } + self._sent_images = fetched + return fetched + async def _read( self, stream: grpc.aio.StreamStreamCall[pb.AttachRequest, pb.AttachResponse] ) -> None: @@ -345,11 +377,36 @@ def _wire_message(message: ChatMessage) -> WireMessage: return WireToolMessage(tool_call_id=tool_call_id, content=content) -def to_wire(request: ModelRequest) -> ChatRequest: +def _with_images( + message: WireToolMessage, tool: ToolMessage, shown: Sequence[bool], images: Mapping[str, bytes] +) -> WireToolMessage: + parts: list[WirePart] = [WireTextPart(text=tool.content)] if tool.content else [] + for image, show in zip(tool.images, shown, strict=True): + if show: + data = base64.b64encode(images[image.sha256]).decode() + url = WireImageURL(url=f"data:{image.media_type};base64,{data}") + parts.append(WireImagePart(image_url=url)) + else: + parts.append(WireTextPart(text=IMAGE_OMITTED)) + return message.model_copy(update={"content": parts}) + + +def to_wire(request: ModelRequest, images: Mapping[str, bytes] | None = None) -> ChatRequest: + """`images` holds the bytes of every image the request shows, by sha256: the context's last + `options.max_images`; earlier ones become `IMAGE_OMITTED`.""" options = request.options + keep = visible_images(request.messages, options.max_images) + messages: list[WireMessage] = [] + for i, m in enumerate(request.messages): + wire = _wire_message(m) + if isinstance(m, ToolMessage) and m.images: + assert isinstance(wire, WireToolMessage) + shown = [(i, j) in keep for j in range(len(m.images))] + wire = _with_images(wire, m, shown, {} if images is None else images) + messages.append(wire) return ChatRequest( model=request.model, - messages=[_wire_message(m) for m in request.messages], + messages=messages, tools=[ WireTool( function=WireFunction( diff --git a/swarmeval/gateway/model/upstream.py b/swarmeval/gateway/model/upstream.py index 843a0da..2b1603d 100644 --- a/swarmeval/gateway/model/upstream.py +++ b/swarmeval/gateway/model/upstream.py @@ -7,7 +7,7 @@ import time from collections.abc import Mapping from dataclasses import dataclass -from typing import cast +from typing import Any, cast import httpx2 from openai import APIConnectionError, APIStatusError, AsyncOpenAI, omit @@ -80,6 +80,34 @@ def passback(messages: list[WireMessage], mode: ReasoningPassback) -> list[WireM return kept +TOOL_IMAGES_LABEL = "Images from tool call {call_id}:" + + +def move_tool_images(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Tool messages with text only, as Chat Completions requires, each run of them followed by a + user message holding their images, each image labelled with its tool call. Runs after + `passback`, so the added user messages do not move its turn boundary.""" + out: list[dict[str, Any]] = [] + images: list[dict[str, Any]] = [] + for message in messages: + if message["role"] != "tool" and images: + out.append({"role": "user", "content": images}) + images = [] + content = message.get("content") + if message["role"] == "tool" and isinstance(content, list): + parts = cast(list[dict[str, Any]], content) + texts = [p["text"] for p in parts if p["type"] == "text"] + pictures = [p for p in parts if p["type"] == "image_url"] + message = {**message, "content": "\n".join(texts)} + if pictures: + label = TOOL_IMAGES_LABEL.format(call_id=message["tool_call_id"]) + images += [{"type": "text", "text": label}, *pictures] + out.append(message) + if images: + out.append({"role": "user", "content": images}) + return out + + class Upstreams: """One SDK client per backend, created on first use.""" @@ -139,10 +167,12 @@ async def complete(self, request: ChatRequest) -> UpstreamResult: }.items() if v is not None } - messages = [ - m.model_dump(exclude_none=True) - for m in passback(request.messages, backend.reasoning_passback) - ] + messages = move_tool_images( + [ + m.model_dump(exclude_none=True) + for m in passback(request.messages, backend.reasoning_passback) + ] + ) tools = [t.model_dump() for t in request.tools or ()] counter = [0] token = _attempts.set(counter) diff --git a/swarmeval/gateway/model/wire.py b/swarmeval/gateway/model/wire.py index f1191f5..9518fa5 100644 --- a/swarmeval/gateway/model/wire.py +++ b/swarmeval/gateway/model/wire.py @@ -41,10 +41,31 @@ class WireAssistantMessage(Wire): tool_calls: list[WireToolCall] | None = None +class WireTextPart(Wire): + type: Literal["text"] = "text" + text: str + + +class WireImageURL(Wire): + url: Annotated[str, Field(pattern=r"^data:image/png;base64,[A-Za-z0-9+/]+={0,2}$")] + """A PNG inline as a `data:` URL; the gateway fetches nothing.""" + + +class WireImagePart(Wire): + type: Literal["image_url"] = "image_url" + image_url: WireImageURL + + +WirePart = Annotated[WireTextPart | WireImagePart, Field(discriminator="type")] + + class WireToolMessage(Wire): role: Literal["tool"] = "tool" tool_call_id: str - content: str + content: str | list[WirePart] + """Parts carry the images a tool returned. Chat Completions takes only text in a tool + message, so the gateway moves the images into a user message for the backend + (`upstream.move_tool_images`).""" WireMessage = Annotated[ diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py index aefd799..571b4f8 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.py @@ -25,7 +25,7 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2swarmeval/proto/swarmeval/sandbox/v1/sandbox.proto\x12\x14swarmeval.sandbox.v1\x1a\x1egoogle/protobuf/duration.proto\"7\n\x10\x43reateRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x13\n\x0bsandbox_ids\x18\x02 \x03(\t\"\x13\n\x11\x43reateRunResponse\";\n\x05Mount\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x11\n\tread_only\x18\x02 \x01(\x08\x12\x11\n\tprotected\x18\x03 \x01(\x08\"Q\n\tResources\x12\x0c\n\x04\x63pus\x18\x01 \x01(\x01\x12\x14\n\x0cmemory_bytes\x18\x02 \x01(\x03\x12\x0c\n\x04pids\x18\x03 \x01(\x03\x12\x12\n\ndisk_bytes\x18\x04 \x01(\x03\"\xfc\x02\n\x14\x43reateSandboxRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12+\n\x06mounts\x18\x04 \x03(\x0b\x32\x1b.swarmeval.sandbox.v1.Mount\x12\x32\n\tresources\x18\x05 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.Resources\x12-\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.SeedFile\x12\r\n\x05users\x18\x07 \x03(\t\x12@\n\x03\x65nv\x18\x08 \x03(\x0b\x32\x33.swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry\x12\x10\n\x08hostname\x18\t \x01(\t\x12\x12\n\nmachine_id\x18\n \x01(\t\x1a*\n\x08\x45nvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"7\n\x08SeedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\"(\n\x15\x43reateSandboxResponse\x12\x0f\n\x07runtime\x18\x01 \x01(\t\"\x97\x01\n\x0b\x45xecRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ll_id\x18\x03 \x01(\t\x12\x0c\n\x04\x61rgv\x18\x04 \x03(\t\x12\x0b\n\x03\x63wd\x18\x05 \x01(\t\x12\x0c\n\x04user\x18\x06 \x01(\t\x12*\n\x07timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\"{\n\x0c\x45xecResponse\x12\x32\n\x06header\x18\x01 \x01(\x0b\x32 .swarmeval.sandbox.v1.ExecHeaderH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"\xda\x02\n\nExecHeader\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x11\n\ttimed_out\x18\x02 \x01(\x08\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x06stdout\x18\x04 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12,\n\x06stderr\x18\x05 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12:\n\x12\x62\x61\x63kground_changes\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12/\n\x07\x63hanges\x18\x07 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12\x30\n\tprocesses\x18\x08 \x03(\x0b\x32\x1d.swarmeval.sandbox.v1.Process\"K\n\x06Output\x12\x0e\n\x06inline\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x13\n\x0b\x62lob_sha256\x18\x03 \x01(\t\x12\x0e\n\x06\x63\x61pped\x18\x04 \x01(\x08\"\xe1\x04\n\x08\x46sChange\x12\x0c\n\x04path\x18\x01 \x01(\t\x12-\n\x02op\x18\x02 \x01(\x0e\x32!.swarmeval.sandbox.v1.FsChange.Op\x12\x31\n\x04kind\x18\x03 \x01(\x0e\x32#.swarmeval.sandbox.v1.FsChange.Kind\x12\x0b\n\x03uid\x18\x04 \x01(\r\x12\x0c\n\x04mode\x18\x05 \x01(\r\x12\x0c\n\x04size\x18\x06 \x01(\x03\x12\x15\n\rbefore_sha256\x18\x07 \x01(\t\x12\x14\n\x0c\x61\x66ter_sha256\x18\x08 \x01(\t\x12\x11\n\tprotected\x18\t \x01(\x08\x12?\n\x0b\x61ttribution\x18\n \x01(\x0e\x32*.swarmeval.sandbox.v1.FsChange.Attribution\x12\x17\n\x0f\x63\x61ndidate_calls\x18\x0b \x03(\t\x12\x0f\n\x07\x63ontent\x18\x0c \x01(\x08\x12\x10\n\x08mtime_ns\x18\r \x01(\x03\"E\n\x02Op\x12\x12\n\x0eOP_UNSPECIFIED\x10\x00\x12\r\n\tOP_CREATE\x10\x01\x12\r\n\tOP_MODIFY\x10\x02\x12\r\n\tOP_DELETE\x10\x03\"[\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_FILE\x10\x01\x12\x0c\n\x08KIND_DIR\x10\x02\x12\x10\n\x0cKIND_SYMLINK\x10\x03\x12\x0e\n\nKIND_OTHER\x10\x04\"[\n\x0b\x41ttribution\x12\x1b\n\x17\x41TTRIBUTION_UNSPECIFIED\x10\x00\x12\x14\n\x10\x41TTRIBUTION_CALL\x10\x01\x12\x19\n\x15\x41TTRIBUTION_AMBIGUOUS\x10\x02\"C\n\x07Process\x12\x0b\n\x03pid\x18\x01 \x01(\x05\x12\x0c\n\x04ppid\x18\x02 \x01(\x05\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0f\n\x07\x63mdline\x18\x04 \x01(\t\"7\n\tBlobChunk\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04last\x18\x03 \x01(\x08\"\xab\x01\n\x13RestoreFilesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0e\n\x06remove\x18\x03 \x03(\t\x12.\n\x04\x64irs\x18\x04 \x03(\x0b\x32 .swarmeval.sandbox.v1.RestoreDir\x12\x30\n\x05\x66iles\x18\x05 \x03(\x0b\x32!.swarmeval.sandbox.v1.RestoreFile\"5\n\nRestoreDir\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04mode\x18\x02 \x01(\r\x12\x0b\n\x03uid\x18\x03 \x01(\r\"G\n\x0bRestoreFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\x12\x0b\n\x03uid\x18\x04 \x01(\r\"\'\n\x14RestoreFilesResponse\x12\x0f\n\x07unowned\x18\x01 \x03(\t\"V\n\x0fReadFileRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x11\n\tmax_bytes\x18\x04 \x01(\x03\"D\n\x10ReadFileResponse\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x11\n\ttruncated\x18\x03 \x01(\x08\"\"\n\x10\x46inalDiffRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x85\x01\n\x11\x46inalDiffResponse\x12\x37\n\x07\x63hanges\x18\x01 \x01(\x0b\x32$.swarmeval.sandbox.v1.SandboxChangesH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"U\n\x0eSandboxChanges\x12\x12\n\nsandbox_id\x18\x01 \x01(\t\x12/\n\x07\x63hanges\x18\x02 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\"#\n\x11\x44\x65stroyRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x14\n\x12\x44\x65stroyRunResponse2\xac\x05\n\x0eSandboxService\x12\\\n\tCreateRun\x12&.swarmeval.sandbox.v1.CreateRunRequest\x1a\'.swarmeval.sandbox.v1.CreateRunResponse\x12h\n\rCreateSandbox\x12*.swarmeval.sandbox.v1.CreateSandboxRequest\x1a+.swarmeval.sandbox.v1.CreateSandboxResponse\x12O\n\x04\x45xec\x12!.swarmeval.sandbox.v1.ExecRequest\x1a\".swarmeval.sandbox.v1.ExecResponse0\x01\x12Y\n\x08ReadFile\x12%.swarmeval.sandbox.v1.ReadFileRequest\x1a&.swarmeval.sandbox.v1.ReadFileResponse\x12\x65\n\x0cRestoreFiles\x12).swarmeval.sandbox.v1.RestoreFilesRequest\x1a*.swarmeval.sandbox.v1.RestoreFilesResponse\x12^\n\tFinalDiff\x12&.swarmeval.sandbox.v1.FinalDiffRequest\x1a\'.swarmeval.sandbox.v1.FinalDiffResponse0\x01\x12_\n\nDestroyRun\x12\'.swarmeval.sandbox.v1.DestroyRunRequest\x1a(.swarmeval.sandbox.v1.DestroyRunResponseBJZHgithub.com/jackeydou/DUNE/go/internal/gen/swarmeval/sandbox/v1;sandboxv1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2swarmeval/proto/swarmeval/sandbox/v1/sandbox.proto\x12\x14swarmeval.sandbox.v1\x1a\x1egoogle/protobuf/duration.proto\"7\n\x10\x43reateRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x13\n\x0bsandbox_ids\x18\x02 \x03(\t\"\x13\n\x11\x43reateRunResponse\";\n\x05Mount\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x11\n\tread_only\x18\x02 \x01(\x08\x12\x11\n\tprotected\x18\x03 \x01(\x08\"Q\n\tResources\x12\x0c\n\x04\x63pus\x18\x01 \x01(\x01\x12\x14\n\x0cmemory_bytes\x18\x02 \x01(\x03\x12\x0c\n\x04pids\x18\x03 \x01(\x03\x12\x12\n\ndisk_bytes\x18\x04 \x01(\x03\"\xac\x03\n\x14\x43reateSandboxRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12+\n\x06mounts\x18\x04 \x03(\x0b\x32\x1b.swarmeval.sandbox.v1.Mount\x12\x32\n\tresources\x18\x05 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.Resources\x12-\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.SeedFile\x12\r\n\x05users\x18\x07 \x03(\t\x12@\n\x03\x65nv\x18\x08 \x03(\x0b\x32\x33.swarmeval.sandbox.v1.CreateSandboxRequest.EnvEntry\x12\x10\n\x08hostname\x18\t \x01(\t\x12\x12\n\nmachine_id\x18\n \x01(\t\x12.\n\x07\x64isplay\x18\x0b \x01(\x0b\x32\x1d.swarmeval.sandbox.v1.Display\x1a*\n\x08\x45nvEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"5\n\x07\x44isplay\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x0b\n\x03url\x18\x03 \x01(\t\"7\n\x08SeedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\"(\n\x15\x43reateSandboxResponse\x12\x0f\n\x07runtime\x18\x01 \x01(\t\"\xa8\x01\n\x0b\x45xecRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ll_id\x18\x03 \x01(\t\x12\x0c\n\x04\x61rgv\x18\x04 \x03(\t\x12\x0b\n\x03\x63wd\x18\x05 \x01(\t\x12\x0c\n\x04user\x18\x06 \x01(\t\x12*\n\x07timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x0f\n\x07\x63ollect\x18\x08 \x03(\t\"{\n\x0c\x45xecResponse\x12\x32\n\x06header\x18\x01 \x01(\x0b\x32 .swarmeval.sandbox.v1.ExecHeaderH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"\x92\x03\n\nExecHeader\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x11\n\ttimed_out\x18\x02 \x01(\x08\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x06stdout\x18\x04 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12,\n\x06stderr\x18\x05 \x01(\x0b\x32\x1c.swarmeval.sandbox.v1.Output\x12:\n\x12\x62\x61\x63kground_changes\x18\x06 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12/\n\x07\x63hanges\x18\x07 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\x12\x30\n\tprocesses\x18\x08 \x03(\x0b\x32\x1d.swarmeval.sandbox.v1.Process\x12\x36\n\tcollected\x18\t \x03(\x0b\x32#.swarmeval.sandbox.v1.CollectedFile\"L\n\rCollectedFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07missing\x18\x02 \x01(\x08\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0e\n\x06sha256\x18\x04 \x01(\t\"K\n\x06Output\x12\x0e\n\x06inline\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x13\n\x0b\x62lob_sha256\x18\x03 \x01(\t\x12\x0e\n\x06\x63\x61pped\x18\x04 \x01(\x08\"\xe1\x04\n\x08\x46sChange\x12\x0c\n\x04path\x18\x01 \x01(\t\x12-\n\x02op\x18\x02 \x01(\x0e\x32!.swarmeval.sandbox.v1.FsChange.Op\x12\x31\n\x04kind\x18\x03 \x01(\x0e\x32#.swarmeval.sandbox.v1.FsChange.Kind\x12\x0b\n\x03uid\x18\x04 \x01(\r\x12\x0c\n\x04mode\x18\x05 \x01(\r\x12\x0c\n\x04size\x18\x06 \x01(\x03\x12\x15\n\rbefore_sha256\x18\x07 \x01(\t\x12\x14\n\x0c\x61\x66ter_sha256\x18\x08 \x01(\t\x12\x11\n\tprotected\x18\t \x01(\x08\x12?\n\x0b\x61ttribution\x18\n \x01(\x0e\x32*.swarmeval.sandbox.v1.FsChange.Attribution\x12\x17\n\x0f\x63\x61ndidate_calls\x18\x0b \x03(\t\x12\x0f\n\x07\x63ontent\x18\x0c \x01(\x08\x12\x10\n\x08mtime_ns\x18\r \x01(\x03\"E\n\x02Op\x12\x12\n\x0eOP_UNSPECIFIED\x10\x00\x12\r\n\tOP_CREATE\x10\x01\x12\r\n\tOP_MODIFY\x10\x02\x12\r\n\tOP_DELETE\x10\x03\"[\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_FILE\x10\x01\x12\x0c\n\x08KIND_DIR\x10\x02\x12\x10\n\x0cKIND_SYMLINK\x10\x03\x12\x0e\n\nKIND_OTHER\x10\x04\"[\n\x0b\x41ttribution\x12\x1b\n\x17\x41TTRIBUTION_UNSPECIFIED\x10\x00\x12\x14\n\x10\x41TTRIBUTION_CALL\x10\x01\x12\x19\n\x15\x41TTRIBUTION_AMBIGUOUS\x10\x02\"C\n\x07Process\x12\x0b\n\x03pid\x18\x01 \x01(\x05\x12\x0c\n\x04ppid\x18\x02 \x01(\x05\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0f\n\x07\x63mdline\x18\x04 \x01(\t\"7\n\tBlobChunk\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0c\n\x04last\x18\x03 \x01(\x08\"\xab\x01\n\x13RestoreFilesRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0e\n\x06remove\x18\x03 \x03(\t\x12.\n\x04\x64irs\x18\x04 \x03(\x0b\x32 .swarmeval.sandbox.v1.RestoreDir\x12\x30\n\x05\x66iles\x18\x05 \x03(\x0b\x32!.swarmeval.sandbox.v1.RestoreFile\"5\n\nRestoreDir\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04mode\x18\x02 \x01(\r\x12\x0b\n\x03uid\x18\x03 \x01(\r\"G\n\x0bRestoreFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x0c\n\x04mode\x18\x03 \x01(\r\x12\x0b\n\x03uid\x18\x04 \x01(\r\"\'\n\x14RestoreFilesResponse\x12\x0f\n\x07unowned\x18\x01 \x03(\t\"V\n\x0fReadFileRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x11\n\tmax_bytes\x18\x04 \x01(\x03\"D\n\x10ReadFileResponse\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04size\x18\x02 \x01(\x03\x12\x11\n\ttruncated\x18\x03 \x01(\x08\"\"\n\x10\x46inalDiffRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x85\x01\n\x11\x46inalDiffResponse\x12\x37\n\x07\x63hanges\x18\x01 \x01(\x0b\x32$.swarmeval.sandbox.v1.SandboxChangesH\x00\x12/\n\x04\x62lob\x18\x02 \x01(\x0b\x32\x1f.swarmeval.sandbox.v1.BlobChunkH\x00\x42\x06\n\x04item\"U\n\x0eSandboxChanges\x12\x12\n\nsandbox_id\x18\x01 \x01(\t\x12/\n\x07\x63hanges\x18\x02 \x03(\x0b\x32\x1e.swarmeval.sandbox.v1.FsChange\"#\n\x11\x44\x65stroyRunRequest\x12\x0e\n\x06run_id\x18\x01 \x01(\t\"\x14\n\x12\x44\x65stroyRunResponse2\xac\x05\n\x0eSandboxService\x12\\\n\tCreateRun\x12&.swarmeval.sandbox.v1.CreateRunRequest\x1a\'.swarmeval.sandbox.v1.CreateRunResponse\x12h\n\rCreateSandbox\x12*.swarmeval.sandbox.v1.CreateSandboxRequest\x1a+.swarmeval.sandbox.v1.CreateSandboxResponse\x12O\n\x04\x45xec\x12!.swarmeval.sandbox.v1.ExecRequest\x1a\".swarmeval.sandbox.v1.ExecResponse0\x01\x12Y\n\x08ReadFile\x12%.swarmeval.sandbox.v1.ReadFileRequest\x1a&.swarmeval.sandbox.v1.ReadFileResponse\x12\x65\n\x0cRestoreFiles\x12).swarmeval.sandbox.v1.RestoreFilesRequest\x1a*.swarmeval.sandbox.v1.RestoreFilesResponse\x12^\n\tFinalDiff\x12&.swarmeval.sandbox.v1.FinalDiffRequest\x1a\'.swarmeval.sandbox.v1.FinalDiffResponse0\x01\x12_\n\nDestroyRun\x12\'.swarmeval.sandbox.v1.DestroyRunRequest\x1a(.swarmeval.sandbox.v1.DestroyRunResponseBJZHgithub.com/jackeydou/DUNE/go/internal/gen/swarmeval/sandbox/v1;sandboxv1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -44,55 +44,59 @@ _globals['_RESOURCES']._serialized_start=247 _globals['_RESOURCES']._serialized_end=328 _globals['_CREATESANDBOXREQUEST']._serialized_start=331 - _globals['_CREATESANDBOXREQUEST']._serialized_end=711 - _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_start=669 - _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_end=711 - _globals['_SEEDFILE']._serialized_start=713 - _globals['_SEEDFILE']._serialized_end=768 - _globals['_CREATESANDBOXRESPONSE']._serialized_start=770 - _globals['_CREATESANDBOXRESPONSE']._serialized_end=810 - _globals['_EXECREQUEST']._serialized_start=813 - _globals['_EXECREQUEST']._serialized_end=964 - _globals['_EXECRESPONSE']._serialized_start=966 - _globals['_EXECRESPONSE']._serialized_end=1089 - _globals['_EXECHEADER']._serialized_start=1092 - _globals['_EXECHEADER']._serialized_end=1438 - _globals['_OUTPUT']._serialized_start=1440 - _globals['_OUTPUT']._serialized_end=1515 - _globals['_FSCHANGE']._serialized_start=1518 - _globals['_FSCHANGE']._serialized_end=2127 - _globals['_FSCHANGE_OP']._serialized_start=1872 - _globals['_FSCHANGE_OP']._serialized_end=1941 - _globals['_FSCHANGE_KIND']._serialized_start=1943 - _globals['_FSCHANGE_KIND']._serialized_end=2034 - _globals['_FSCHANGE_ATTRIBUTION']._serialized_start=2036 - _globals['_FSCHANGE_ATTRIBUTION']._serialized_end=2127 - _globals['_PROCESS']._serialized_start=2129 - _globals['_PROCESS']._serialized_end=2196 - _globals['_BLOBCHUNK']._serialized_start=2198 - _globals['_BLOBCHUNK']._serialized_end=2253 - _globals['_RESTOREFILESREQUEST']._serialized_start=2256 - _globals['_RESTOREFILESREQUEST']._serialized_end=2427 - _globals['_RESTOREDIR']._serialized_start=2429 - _globals['_RESTOREDIR']._serialized_end=2482 - _globals['_RESTOREFILE']._serialized_start=2484 - _globals['_RESTOREFILE']._serialized_end=2555 - _globals['_RESTOREFILESRESPONSE']._serialized_start=2557 - _globals['_RESTOREFILESRESPONSE']._serialized_end=2596 - _globals['_READFILEREQUEST']._serialized_start=2598 - _globals['_READFILEREQUEST']._serialized_end=2684 - _globals['_READFILERESPONSE']._serialized_start=2686 - _globals['_READFILERESPONSE']._serialized_end=2754 - _globals['_FINALDIFFREQUEST']._serialized_start=2756 - _globals['_FINALDIFFREQUEST']._serialized_end=2790 - _globals['_FINALDIFFRESPONSE']._serialized_start=2793 - _globals['_FINALDIFFRESPONSE']._serialized_end=2926 - _globals['_SANDBOXCHANGES']._serialized_start=2928 - _globals['_SANDBOXCHANGES']._serialized_end=3013 - _globals['_DESTROYRUNREQUEST']._serialized_start=3015 - _globals['_DESTROYRUNREQUEST']._serialized_end=3050 - _globals['_DESTROYRUNRESPONSE']._serialized_start=3052 - _globals['_DESTROYRUNRESPONSE']._serialized_end=3072 - _globals['_SANDBOXSERVICE']._serialized_start=3075 - _globals['_SANDBOXSERVICE']._serialized_end=3759 + _globals['_CREATESANDBOXREQUEST']._serialized_end=759 + _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_start=717 + _globals['_CREATESANDBOXREQUEST_ENVENTRY']._serialized_end=759 + _globals['_DISPLAY']._serialized_start=761 + _globals['_DISPLAY']._serialized_end=814 + _globals['_SEEDFILE']._serialized_start=816 + _globals['_SEEDFILE']._serialized_end=871 + _globals['_CREATESANDBOXRESPONSE']._serialized_start=873 + _globals['_CREATESANDBOXRESPONSE']._serialized_end=913 + _globals['_EXECREQUEST']._serialized_start=916 + _globals['_EXECREQUEST']._serialized_end=1084 + _globals['_EXECRESPONSE']._serialized_start=1086 + _globals['_EXECRESPONSE']._serialized_end=1209 + _globals['_EXECHEADER']._serialized_start=1212 + _globals['_EXECHEADER']._serialized_end=1614 + _globals['_COLLECTEDFILE']._serialized_start=1616 + _globals['_COLLECTEDFILE']._serialized_end=1692 + _globals['_OUTPUT']._serialized_start=1694 + _globals['_OUTPUT']._serialized_end=1769 + _globals['_FSCHANGE']._serialized_start=1772 + _globals['_FSCHANGE']._serialized_end=2381 + _globals['_FSCHANGE_OP']._serialized_start=2126 + _globals['_FSCHANGE_OP']._serialized_end=2195 + _globals['_FSCHANGE_KIND']._serialized_start=2197 + _globals['_FSCHANGE_KIND']._serialized_end=2288 + _globals['_FSCHANGE_ATTRIBUTION']._serialized_start=2290 + _globals['_FSCHANGE_ATTRIBUTION']._serialized_end=2381 + _globals['_PROCESS']._serialized_start=2383 + _globals['_PROCESS']._serialized_end=2450 + _globals['_BLOBCHUNK']._serialized_start=2452 + _globals['_BLOBCHUNK']._serialized_end=2507 + _globals['_RESTOREFILESREQUEST']._serialized_start=2510 + _globals['_RESTOREFILESREQUEST']._serialized_end=2681 + _globals['_RESTOREDIR']._serialized_start=2683 + _globals['_RESTOREDIR']._serialized_end=2736 + _globals['_RESTOREFILE']._serialized_start=2738 + _globals['_RESTOREFILE']._serialized_end=2809 + _globals['_RESTOREFILESRESPONSE']._serialized_start=2811 + _globals['_RESTOREFILESRESPONSE']._serialized_end=2850 + _globals['_READFILEREQUEST']._serialized_start=2852 + _globals['_READFILEREQUEST']._serialized_end=2938 + _globals['_READFILERESPONSE']._serialized_start=2940 + _globals['_READFILERESPONSE']._serialized_end=3008 + _globals['_FINALDIFFREQUEST']._serialized_start=3010 + _globals['_FINALDIFFREQUEST']._serialized_end=3044 + _globals['_FINALDIFFRESPONSE']._serialized_start=3047 + _globals['_FINALDIFFRESPONSE']._serialized_end=3180 + _globals['_SANDBOXCHANGES']._serialized_start=3182 + _globals['_SANDBOXCHANGES']._serialized_end=3267 + _globals['_DESTROYRUNREQUEST']._serialized_start=3269 + _globals['_DESTROYRUNREQUEST']._serialized_end=3304 + _globals['_DESTROYRUNRESPONSE']._serialized_start=3306 + _globals['_DESTROYRUNRESPONSE']._serialized_end=3326 + _globals['_SANDBOXSERVICE']._serialized_start=3329 + _globals['_SANDBOXSERVICE']._serialized_end=4013 # @@protoc_insertion_point(module_scope) diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi index 663ec85..32a1661 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2.pyi @@ -153,6 +153,7 @@ class CreateSandboxRequest(_message.Message): ENV_FIELD_NUMBER: _builtins.int HOSTNAME_FIELD_NUMBER: _builtins.int MACHINE_ID_FIELD_NUMBER: _builtins.int + DISPLAY_FIELD_NUMBER: _builtins.int run_id: _builtins.str sandbox_id: _builtins.str image: _builtins.str @@ -190,6 +191,12 @@ class CreateSandboxRequest(_message.Message): override the image's. Names match [A-Za-z_][A-Za-z0-9_]{0,127}; values hold no NUL. """ + @_builtins.property + def display(self) -> Global___Display: + """Starts the image's display stack (docs/services/sandboxd.md#display). Unset, the sandbox has + no display. + """ + def __init__( self, *, @@ -203,15 +210,47 @@ class CreateSandboxRequest(_message.Message): env: _abc.Mapping[_builtins.str, _builtins.str] | None = ..., hostname: _builtins.str = ..., machine_id: _builtins.str = ..., + display: Global___Display | None = ..., ) -> None: ... - _HasFieldArgType: _TypeAlias = _typing.Literal["resources", b"resources"] # noqa: Y015 + _HasFieldArgType: _TypeAlias = _typing.Literal["display", b"display", "resources", b"resources"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["env", b"env", "files", b"files", "hostname", b"hostname", "image", b"image", "machine_id", b"machine_id", "mounts", b"mounts", "resources", b"resources", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "users", b"users"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["display", b"display", "env", b"env", "files", b"files", "hostname", b"hostname", "image", b"image", "machine_id", b"machine_id", "mounts", b"mounts", "resources", b"resources", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "users", b"users"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... Global___CreateSandboxRequest: _TypeAlias = CreateSandboxRequest # noqa: Y015 +@_typing.final +class Display(_message.Message): + """A virtual screen with a browser on it, run by the image's `swarm-display` as the user + `swarmdisplay`. Its processes are never reported or attributed to a call. + """ + + DESCRIPTOR: _descriptor.Descriptor + + WIDTH_FIELD_NUMBER: _builtins.int + HEIGHT_FIELD_NUMBER: _builtins.int + URL_FIELD_NUMBER: _builtins.int + width: _builtins.int + """Pixels, 320 to 1920 wide and 240 to 1200 high.""" + height: _builtins.int + url: _builtins.str + """Page the browser opens first. Empty opens about:blank.""" + def __init__( + self, + *, + width: _builtins.int = ..., + height: _builtins.int = ..., + url: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["height", b"height", "url", b"url", "width", b"width"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___Display: _TypeAlias = Display # noqa: Y015 + @_typing.final class SeedFile(_message.Message): DESCRIPTOR: _descriptor.Descriptor @@ -270,6 +309,7 @@ class ExecRequest(_message.Message): CWD_FIELD_NUMBER: _builtins.int USER_FIELD_NUMBER: _builtins.int TIMEOUT_FIELD_NUMBER: _builtins.int + COLLECT_FIELD_NUMBER: _builtins.int run_id: _builtins.str sandbox_id: _builtins.str call_id: _builtins.str @@ -283,6 +323,13 @@ class ExecRequest(_message.Message): def timeout(self) -> _duration_pb2.Duration: """Required. On expiry the command and its descendants are killed.""" + @_builtins.property + def collect(self) -> _containers.RepeatedScalarFieldContainer[_builtins.str]: + """Absolute paths outside every key path. After the command, sandboxd reads each regular file + there as `user`, removes it, and returns it as a blob. For output a tool writes besides its + text, such as a screenshot. + """ + def __init__( self, *, @@ -293,10 +340,11 @@ class ExecRequest(_message.Message): cwd: _builtins.str = ..., user: _builtins.str = ..., timeout: _duration_pb2.Duration | None = ..., + collect: _abc.Iterable[_builtins.str] | None = ..., ) -> None: ... _HasFieldArgType: _TypeAlias = _typing.Literal["timeout", b"timeout"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["argv", b"argv", "call_id", b"call_id", "cwd", b"cwd", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "timeout", b"timeout", "user", b"user"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["argv", b"argv", "call_id", b"call_id", "collect", b"collect", "cwd", b"cwd", "run_id", b"run_id", "sandbox_id", b"sandbox_id", "timeout", b"timeout", "user", b"user"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... @@ -340,6 +388,7 @@ class ExecHeader(_message.Message): BACKGROUND_CHANGES_FIELD_NUMBER: _builtins.int CHANGES_FIELD_NUMBER: _builtins.int PROCESSES_FIELD_NUMBER: _builtins.int + COLLECTED_FIELD_NUMBER: _builtins.int exit_code: _builtins.int """-1 when timed out.""" timed_out: _builtins.bool @@ -363,6 +412,10 @@ class ExecHeader(_message.Message): def processes(self) -> _containers.RepeatedCompositeFieldContainer[Global___Process]: """Started while this call ran and still alive after it.""" + @_builtins.property + def collected(self) -> _containers.RepeatedCompositeFieldContainer[Global___CollectedFile]: + """One per `collect` path, in request order.""" + def __init__( self, *, @@ -374,15 +427,49 @@ class ExecHeader(_message.Message): background_changes: _abc.Iterable[Global___FsChange] | None = ..., changes: _abc.Iterable[Global___FsChange] | None = ..., processes: _abc.Iterable[Global___Process] | None = ..., + collected: _abc.Iterable[Global___CollectedFile] | None = ..., ) -> None: ... _HasFieldArgType: _TypeAlias = _typing.Literal["duration", b"duration", "stderr", b"stderr", "stdout", b"stdout"] # noqa: Y015 def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... - _ClearFieldArgType: _TypeAlias = _typing.Literal["background_changes", b"background_changes", "changes", b"changes", "duration", b"duration", "exit_code", b"exit_code", "processes", b"processes", "stderr", b"stderr", "stdout", b"stdout", "timed_out", b"timed_out"] # noqa: Y015 + _ClearFieldArgType: _TypeAlias = _typing.Literal["background_changes", b"background_changes", "changes", b"changes", "collected", b"collected", "duration", b"duration", "exit_code", b"exit_code", "processes", b"processes", "stderr", b"stderr", "stdout", b"stdout", "timed_out", b"timed_out"] # noqa: Y015 def ClearField(self, field_name: _ClearFieldArgType) -> None: ... def WhichOneof(self, oneof_group: _Never) -> None: ... Global___ExecHeader: _TypeAlias = ExecHeader # noqa: Y015 +@_typing.final +class CollectedFile(_message.Message): + DESCRIPTOR: _descriptor.Descriptor + + PATH_FIELD_NUMBER: _builtins.int + MISSING_FIELD_NUMBER: _builtins.int + SIZE_FIELD_NUMBER: _builtins.int + SHA256_FIELD_NUMBER: _builtins.int + path: _builtins.str + missing: _builtins.bool + """No regular file was there.""" + size: _builtins.int + """Bytes of the file. Over sandboxd's collect limit the file is removed but not sent, and + `sha256` is empty. + """ + sha256: _builtins.str + """The content follows as a blob with this hash.""" + def __init__( + self, + *, + path: _builtins.str = ..., + missing: _builtins.bool = ..., + size: _builtins.int = ..., + sha256: _builtins.str = ..., + ) -> None: ... + _HasFieldArgType: _TypeAlias = _Never # noqa: Y015 + def HasField(self, field_name: _HasFieldArgType) -> _builtins.bool: ... + _ClearFieldArgType: _TypeAlias = _typing.Literal["missing", b"missing", "path", b"path", "sha256", b"sha256", "size", b"size"] # noqa: Y015 + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + def WhichOneof(self, oneof_group: _Never) -> None: ... + +Global___CollectedFile: _TypeAlias = CollectedFile # noqa: Y015 + @_typing.final class Output(_message.Message): DESCRIPTOR: _descriptor.Descriptor diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py index 1159cc6..74e31d6 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.py @@ -96,7 +96,8 @@ def CreateRun(self, request, context): def CreateSandbox(self, request, context): """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi index ad6ba86..374ac77 100644 --- a/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi +++ b/swarmeval/proto/swarmeval/sandbox/v1/sandbox_pb2_grpc.pyi @@ -46,7 +46,8 @@ class SandboxServiceStub: """ CreateSandbox: _grpc.UnaryUnaryMultiCallable[_sandbox_pb2.CreateSandboxRequest, _sandbox_pb2.CreateSandboxResponse] """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ Exec: _grpc.UnaryStreamMultiCallable[_sandbox_pb2.ExecRequest, _sandbox_pb2.ExecResponse] """Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -85,7 +86,8 @@ class SandboxServiceAsyncStub(SandboxServiceStub): """ CreateSandbox: _aio.UnaryUnaryMultiCallable[_sandbox_pb2.CreateSandboxRequest, _sandbox_pb2.CreateSandboxResponse] # type: ignore[assignment] """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ Exec: _aio.UnaryStreamMultiCallable[_sandbox_pb2.ExecRequest, _sandbox_pb2.ExecResponse] # type: ignore[assignment] """Runs one tool call. The stream carries exactly one header first, then the blobs the @@ -133,7 +135,8 @@ class SandboxServiceServicer(metaclass=_abc_1.ABCMeta): context: _ServicerContext, ) -> _typing.Union[_sandbox_pb2.CreateSandboxResponse, _abc.Awaitable[_sandbox_pb2.CreateSandboxResponse]]: """Creates and starts one sandbox, on its network if CreateRun made one. Each top-level key path starts as a copy of the image's - content at that path, plus any seed files. Fails with ALREADY_EXISTS if the sandbox exists. + content at that path, plus any seed files. With `display` set, the image's display stack is + started before the first manifest. Fails with ALREADY_EXISTS if the sandbox exists. """ @_abc_1.abstractmethod diff --git a/swarmeval/runtime/display.py b/swarmeval/runtime/display.py new file mode 100644 index 0000000..d100b49 --- /dev/null +++ b/swarmeval/runtime/display.py @@ -0,0 +1,222 @@ +"""`computer` and `browser`: tools that drive a sandbox's display (docs/agent-runtime.md#tools). + +Both run `swarm-display` inside the sandbox as the display's own user, which only these tools +run as, so an agent's shell cannot reach the screen. An image an action produces is collected by +sandboxd from outside the key paths and reaches the agent as an `ImageRef`. +""" + +import json +from collections.abc import Callable +from typing import Annotated, Literal, Self + +from pydantic import BaseModel, Field, model_validator + +from swarmeval.runtime.messages import ImageRef, ToolCall +from swarmeval.runtime.records import Exec, ExecResult, ToolResult +from swarmeval.runtime.tools import SandboxTool, exec_output + +DISPLAY_USER = "swarmdisplay" +"""Runs the display stack and these tools. Display images have it (deploy/images/display).""" +DISPLAY_STATE = "/run/swarm-display" +"""The display's state directory, kept out of every key path.""" +DISPLAY_TOOL_NAMES = ("computer", "browser") +SCREENSHOT = f"{DISPLAY_STATE}/out/screenshot.png" +ACTION_TIMEOUT_S = 90.0 + +Point = Annotated[ + list[int], Field(min_length=2, max_length=2, description="[x, y] in screen pixels.") +] + + +def _require(args: BaseModel, action: str, needs: dict[str, tuple[str, ...]]) -> None: + missing = [f for f in needs.get(action, ()) if getattr(args, f) is None] + if missing: + names = ", ".join(f"`{f}`" for f in missing) + raise ValueError(f"action `{action}` needs {names}") + + +class ComputerArgs(BaseModel): + action: Literal[ + "screenshot", + "click", + "mouse_move", + "drag", + "type", + "key", + "scroll", + "wait", + "cursor_position", + ] = Field(description="What to do. Every action but `cursor_position` returns a screenshot.") + coordinate: Point | None = Field( + default=None, + description="Where to click or move to (`click`, `mouse_move`, `drag`'s end), or where " + "to scroll (`scroll`, optional).", + ) + start_coordinate: Point | None = Field(default=None, description="Where `drag` starts.") + button: Literal["left", "middle", "right"] = Field(default="left", description="For `click`.") + count: int = Field(default=1, ge=1, le=3, description="Clicks: 2 for a double click.") + text: str | None = Field( + default=None, + max_length=10_000, + description="For `type`, the text to type. For `key`, xdotool key names, combos joined " + "with `+` and several keys separated by spaces: `ctrl+l`, `Return`, `alt+Tab`.", + ) + direction: Literal["up", "down", "left", "right"] | None = Field( + default=None, description="For `scroll`." + ) + amount: int = Field(default=3, ge=1, le=20, description="Wheel clicks for `scroll`.") + seconds: float = Field(default=1.0, gt=0, le=10, description="For `wait`.") + + @model_validator(mode="after") + def _needs(self) -> Self: + _require( + self, + self.action, + { + "click": ("coordinate",), + "mouse_move": ("coordinate",), + "drag": ("start_coordinate", "coordinate"), + "type": ("text",), + "key": ("text",), + "scroll": ("direction",), + }, + ) + return self + + +class BrowserArgs(BaseModel): + action: Literal[ + "navigate", + "back", + "forward", + "reload", + "snapshot", + "screenshot", + "click", + "hover", + "type", + "select", + "press", + "scroll", + "wait", + "tabs", + "tab_new", + "tab_select", + "tab_close", + ] = Field(description="What to do. Every action returns the page snapshot afterwards.") + url: str | None = Field( + default=None, max_length=8192, description="For `navigate`, and optionally `tab_new`." + ) + ref: str | None = Field( + default=None, + pattern=r"^[A-Za-z0-9]{1,32}$", + description="The element to act on, as its `ref` in the latest snapshot, such as `e12` " + "(`click`, `hover`, `type`, `select`).", + ) + text: str | None = Field( + default=None, max_length=10_000, description="For `type`: replaces the field's text." + ) + submit: bool = Field(default=False, description="For `type`: press Enter afterwards.") + values: list[str] | None = Field( + default=None, description="For `select`: option values or labels." + ) + key: str | None = Field( + default=None, + max_length=100, + description="For `press`: a key or combo such as `Enter`, `Escape`, `Control+a`.", + ) + direction: Literal["up", "down", "left", "right"] = Field( + default="down", description="For `scroll`." + ) + amount: int = Field(default=3, ge=1, le=20, description="For `scroll`: screens' worth / 2.") + seconds: float = Field(default=1.0, gt=0, le=10, description="For `wait`.") + index: int | None = Field( + default=None, ge=1, description="Tab number, from 1 (`tab_select`, `tab_close`)." + ) + screenshot: bool = Field( + default=False, + description="Attach a screenshot of the page after the action. `screenshot` always does.", + ) + + @model_validator(mode="after") + def _needs(self) -> Self: + _require( + self, + self.action, + { + "navigate": ("url",), + "click": ("ref",), + "hover": ("ref",), + "type": ("ref", "text"), + "select": ("ref", "values"), + "press": ("key",), + "tab_select": ("index",), + }, + ) + return self + + +def _build(tool: str) -> Callable[[BaseModel], Exec]: + def build(args: BaseModel) -> Exec: + arguments = json.dumps(args.model_dump(exclude_none=True)) + return Exec( + argv=("swarm-display", tool, arguments), + timeout_s=ACTION_TIMEOUT_S, + user=DISPLAY_USER, + collect=(SCREENSHOT,), + ) + + return build + + +def display_output(call: ToolCall, result: ExecResult) -> ToolResult: + """The action's text, with its screenshot as an image when it took one.""" + text = exec_output(call, result) + images: list[ImageRef] = [] + problems: list[str] = [] + for c in result.collected: + if c.missing: + continue + if c.sha256 is None: + problems.append(f"[the screenshot was {c.size} bytes, over sandboxd's limit]") + elif c.png is None: + problems.append("[the display returned a screenshot that is not a PNG image]") + else: + images.append(ImageRef(sha256=c.sha256, width=c.png.width, height=c.png.height)) + content = text.content + "".join(f"\n{p}" for p in problems) + return text.model_copy( + update={ + "content": content, + "is_error": text.is_error or bool(problems), + "images": tuple(images), + } + ) + + +COMPUTER = SandboxTool( + name="computer", + description=( + "Use the mouse and keyboard on your sandbox's screen, a desktop showing a web browser. " + "Coordinates are screen pixels from the top left, as in the screenshots. Each action " + "returns a screenshot taken after it." + ), + args=ComputerArgs, + build=_build("computer"), + output=display_output, +) + +BROWSER = SandboxTool( + name="browser", + description=( + "Drive the web browser on your sandbox's screen. Every action returns the active tab's " + "URL, title, and an accessibility snapshot in which each element has a `[ref=eN]`; act " + "on elements by that ref. Refs change when the page does: use the latest snapshot. The " + "sandbox has no internet; pages are served from inside it." + ), + args=BrowserArgs, + build=_build("browser"), + output=display_output, +) + +DISPLAY_TOOLS = (COMPUTER, BROWSER) +"""Handed to `RunLoop` beside `BUILTIN_TOOLS`.""" diff --git a/swarmeval/runtime/execute.py b/swarmeval/runtime/execute.py index 66e722a..0b09553 100644 --- a/swarmeval/runtime/execute.py +++ b/swarmeval/runtime/execute.py @@ -92,4 +92,5 @@ async def execute( ) from err assert agent.sandbox_id is not None, "checked in _check_tools" result = await self._sandbox.exec(agent.sandbox_id, agent.os_user, command, call_id=call.id) - return Execution(exec_output(call, result), call.arguments, exec_result=result) + output = tool.output or exec_output + return Execution(output(call, result), call.arguments, exec_result=result) diff --git a/swarmeval/runtime/loop.py b/swarmeval/runtime/loop.py index 7b5557d..30902ed 100644 --- a/swarmeval/runtime/loop.py +++ b/swarmeval/runtime/loop.py @@ -577,7 +577,10 @@ async def _tool_step( admitted = await d.after_tool_result(agent.info, result, tool_event.event_id) txn = admitted.txn message = ToolMessage( - tool_call_id=call.id, content=admitted.value.content, is_error=admitted.value.is_error + tool_call_id=call.id, + content=admitted.value.content, + is_error=admitted.value.is_error, + images=admitted.value.images, ) self._admit(txn, agent, [(message, admitted.intervention_id or tool_event.event_id)]) await self._writer.commit(txn) diff --git a/swarmeval/runtime/messages.py b/swarmeval/runtime/messages.py index 7a15ac6..0fd75d1 100644 --- a/swarmeval/runtime/messages.py +++ b/swarmeval/runtime/messages.py @@ -6,6 +6,7 @@ is how the dispatcher detects an intervention. """ +from collections.abc import Sequence from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, JsonValue @@ -39,11 +40,22 @@ class AssistantMessage(Frozen): tool_calls: tuple[ToolCall, ...] = () +class ImageRef(Frozen): + """An image a tool returned. The bytes are in the blob store under `sha256`; messages and + events hold only the reference.""" + + sha256: str + media_type: Literal["image/png"] = "image/png" + width: int + height: int + + class ToolMessage(Frozen): role: Literal["tool"] = "tool" tool_call_id: str content: str is_error: bool = False + images: tuple[ImageRef, ...] = () ChatMessage = Annotated[ @@ -71,6 +83,24 @@ class RequestOptions(Frozen): top_p: float | None = None max_output_tokens: int | None = None seed: int | None = None + max_images: int = Field(default=3, ge=0) + """Images sent with the request: the context's last `max_images`; earlier ones are replaced + by `IMAGE_OMITTED` (`visible_images`).""" + + +IMAGE_OMITTED = "[image omitted]" + + +def visible_images(messages: Sequence[ChatMessage], max_images: int) -> set[tuple[int, int]]: + """`(message index, image index)` of the images a request with `max_images` carries: the + last ones in the context.""" + every = [ + (i, j) + for i, m in enumerate(messages) + if isinstance(m, ToolMessage) + for j in range(len(m.images)) + ] + return set(every[max(0, len(every) - max_images) :]) if max_images else set() class ModelRequest(Frozen): diff --git a/swarmeval/runtime/records.py b/swarmeval/runtime/records.py index 25ddb14..3735083 100644 --- a/swarmeval/runtime/records.py +++ b/swarmeval/runtime/records.py @@ -15,6 +15,7 @@ AssistantMessage, ChatMessage, Frozen, + ImageRef, RequestOptions, ToolCall, Usage, @@ -43,6 +44,7 @@ class ToolResult(Frozen): tool: str content: str is_error: bool = False + images: tuple[ImageRef, ...] = () class Exec(Frozen): @@ -51,6 +53,11 @@ class Exec(Frozen): argv: tuple[str, ...] cwd: str | None = None timeout_s: float = 60.0 + user: str | None = None + """Runs as this user instead of the calling agent's `os_user`, such as the display's.""" + collect: tuple[str, ...] = () + """Files outside the key paths that sandboxd reads, removes, and returns after the command + (docs/services/sandboxd.md#exec-diff-and-process-snapshot).""" class FsChange(Frozen): @@ -93,6 +100,23 @@ class Truncated(Frozen): """The kept output is itself cut at sandboxd's blob limit.""" +class PngInfo(Frozen): + width: int + height: int + + +class CollectedFile(Frozen): + path: str + missing: bool = False + """No regular file was there.""" + size: int = 0 + sha256: str | None = None + """The content is in the blob store under this hash. `None` when missing or over sandboxd's + collect limit.""" + png: PngInfo | None = None + """Set when the content is a PNG image, read from its header by sandboxd's client.""" + + class ExecResult(Frozen): exit_code: int stdout: str @@ -107,6 +131,8 @@ class ExecResult(Frozen): """Changed between the previous call on this sandbox and this one, by background processes. Always ambiguous.""" processes: tuple[ProcessInfo, ...] = () + collected: tuple[CollectedFile, ...] = () + """One per `Exec.collect` path, in order.""" class WebRequest(Frozen): diff --git a/swarmeval/runtime/tools.py b/swarmeval/runtime/tools.py index db6e583..9cdab8b 100644 --- a/swarmeval/runtime/tools.py +++ b/swarmeval/runtime/tools.py @@ -29,6 +29,8 @@ class SandboxTool: args: type[BaseModel] build: Callable[[Any], Exec] owner: str | None = None + output: Callable[[ToolCall, ExecResult], ToolResult] | None = None + """Turns sandboxd's result into what the agent sees; `exec_output` when unset.""" @dataclass(frozen=True) @@ -165,9 +167,10 @@ def _build_web_request(args: WebRequestArgs) -> WebRequest: ) BUILTIN_TOOLS: tuple[Tool, ...] = (SHELL, WEB_REQUEST) -"""Tools the caller hands to `RunLoop`. The loop adds the Message Bus's `send_message` itself.""" +"""Tools the caller hands to `RunLoop`, with `swarmeval.runtime.display.DISPLAY_TOOLS`. The loop +adds the Message Bus's `send_message` itself.""" -BUILTIN_TOOL_NAMES = ("shell", "send_message", "web_request") +BUILTIN_TOOL_NAMES = ("shell", "send_message", "web_request", "computer", "browser") """Every tool name the runtime provides. Extensions may not reuse them.""" diff --git a/swarmeval/sandbox/blobs.py b/swarmeval/sandbox/blobs.py index 790645e..ecb8351 100644 --- a/swarmeval/sandbox/blobs.py +++ b/swarmeval/sandbox/blobs.py @@ -9,11 +9,7 @@ from dataclasses import dataclass from typing import Protocol -from swarmeval.events import ObjectStore - - -def blob_key(sha256: str) -> str: - return f"blobs/sha256/{sha256}" +from swarmeval.events import ObjectStore, blob_key class BlobStore(Protocol): diff --git a/swarmeval/sandbox/client.py b/swarmeval/sandbox/client.py index b8159d4..c01aa19 100644 --- a/swarmeval/sandbox/client.py +++ b/swarmeval/sandbox/client.py @@ -7,6 +7,7 @@ """ import hashlib +import struct from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import timedelta @@ -18,7 +19,15 @@ from swarmeval.core.models import SandboxProfile from swarmeval.proto.swarmeval.sandbox.v1 import sandbox_pb2 as pb from swarmeval.proto.swarmeval.sandbox.v1.sandbox_pb2_grpc import SandboxServiceStub -from swarmeval.runtime.records import Exec, ExecResult, FsChange, ProcessInfo, Truncated +from swarmeval.runtime.records import ( + CollectedFile, + Exec, + ExecResult, + FsChange, + PngInfo, + ProcessInfo, + Truncated, +) from swarmeval.sandbox.blobs import BlobStore @@ -147,6 +156,9 @@ async def create( hostname=hostname, machine_id=machine_id, ) + if profile.display is not None: + d = profile.display + request.display.CopyFrom(pb.Display(width=d.width, height=d.height, url=d.url or "")) try: response = await self._stub.CreateSandbox(request) except grpc.aio.AioRpcError as err: @@ -162,8 +174,9 @@ async def exec( call_id=call_id, argv=command.argv, cwd=command.cwd or "", - user=os_user or "", + user=command.user or os_user or "", timeout=_duration(command.timeout_s), + collect=command.collect, ) where = f"sandbox `{sandbox_id}` call `{call_id}`" stream = self._stub.Exec(request, timeout=command.timeout_s + self._exec_margin_s) @@ -189,7 +202,8 @@ async def exec( f"sandboxd Exec for run `{self._run_id}` {where}: the stream ended without a " "header." ) - await self._store(blobs.finish()) + received = blobs.finish() + await self._store(received) stdout, stdout_truncated = _output(header.stdout) stderr, stderr_truncated = _output(header.stderr) return ExecResult( @@ -206,6 +220,16 @@ async def exec( ProcessInfo(pid=p.pid, ppid=p.ppid, user=_clean(p.user), cmdline=_clean(p.cmdline)) for p in header.processes ), + collected=tuple( + CollectedFile( + path=c.path, + missing=c.missing, + size=c.size, + sha256=c.sha256 or None, + png=png_info(received[c.sha256]) if c.sha256 else None, + ) + for c in header.collected + ), ) async def restore( @@ -354,6 +378,21 @@ def finish(self) -> dict[str, bytes]: return self._done +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +_PNG_MAX_SIDE = 8192 + + +def png_info(data: bytes) -> PngInfo | None: + """The size a PNG's header declares, or `None` for anything that is not a PNG of at most + 8192 pixels a side. The content comes from inside a sandbox.""" + if len(data) < 24 or not data.startswith(_PNG_SIGNATURE) or data[12:16] != b"IHDR": + return None + width, height = struct.unpack(">II", data[16:24]) + if not (0 < width <= _PNG_MAX_SIDE and 0 < height <= _PNG_MAX_SIDE): + return None + return PngInfo(width=width, height=height) + + def _duration(seconds: float) -> Duration: duration = Duration() duration.FromTimedelta(timedelta(seconds=seconds)) @@ -365,6 +404,7 @@ def _expected_blobs(header: pb.ExecHeader) -> list[str]: for change in (*header.changes, *header.background_changes): if change.content: hashes.append(change.after_sha256) + hashes.extend(c.sha256 for c in header.collected if c.sha256) return hashes diff --git a/swarmeval/worker/run.py b/swarmeval/worker/run.py index cfccb01..8b41d8c 100644 --- a/swarmeval/worker/run.py +++ b/swarmeval/worker/run.py @@ -34,6 +34,7 @@ ) from swarmeval.honeypot import PlacedCanary, place, place_sandboxes from swarmeval.runtime import RunConfigError, RunLoop, RunSpec +from swarmeval.runtime.display import DISPLAY_TOOLS from swarmeval.runtime.extensions import ( ExtensionError, ExtensionLoadError, @@ -53,7 +54,7 @@ from swarmeval.worker.fork import last_changes, lineage_views, load_fork, restore from swarmeval.worker.pause import CANCELLED, QueuePauser from swarmeval.worker.probes import IsolationError, ProbeSandbox, check_isolation -from swarmeval.worker.transcript import check_transcript +from swarmeval.worker.transcript import check_transcript, transcript_images log = logging.getLogger(__name__) @@ -232,6 +233,7 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: owner_epoch=run.owner_epoch, callers=callers, writer=writer, + images=blobs, ) ) web = await stack.enter_async_context(HttpWebClient(blobs)) @@ -242,7 +244,7 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: sandbox_executor=sandboxes, pauser=QueuePauser(deps.queue, run.run_id, run.owner_epoch, deps.cancel_poll_s), extensions=extensions, - tools=BUILTIN_TOOLS, + tools=(*BUILTIN_TOOLS, *DISPLAY_TOOLS), web_client=web, fork=fork, ) @@ -265,7 +267,9 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: writer=writer, prior=prior, ).run(committed) - await _check_transcript(deps.engine, spec, loop, writer, last_lifecycle(committed)) + await _check_transcript( + deps.engine, spec, loop, writer, blobs, last_lifecycle(committed) + ) except (ExtensionError, RunConfigError, ScoringError) as err: return Outcome("failed", str(err)) except IsolationError as err: @@ -280,15 +284,23 @@ async def execute(run: RunRow, deps: WorkerDeps) -> Outcome: async def _check_transcript( - engine: AsyncEngine, spec: RunSpec, loop: RunLoop, writer: RunWriter, end: str + engine: AsyncEngine, + spec: RunSpec, + loop: RunLoop, + writer: RunWriter, + blobs: S3BlobStore, + end: str, ) -> None: """Commits the transcript check as the run's last event before export, so the exports carry it. A mismatch is a finding about the run, not a failure of it. Its parent is the - last event a mismatch names, or else `end`, the loop's last lifecycle event.""" + last event a mismatch names, or else `end`, the loop's last lifecycle event. Every image + the contexts hold is read into memory for it, since requests carried them inline.""" + transcript = await load_transcript(engine, spec.run_id) check = check_transcript( - await load_transcript(engine, spec.run_id), + transcript, prompts={a.id: initial_context(a) for a in spec.agents}, tools=loop.tool_schemas(), + images={h: await blobs.get(h) for h in sorted(transcript_images(transcript))}, ) parent = check.event_ids[-1] if check.event_ids else end await writer.commit(Transaction(events=[EventDraft(record=check, parent_id=parent)])) diff --git a/swarmeval/worker/transcript.py b/swarmeval/worker/transcript.py index 987c542..28b3f6d 100644 --- a/swarmeval/worker/transcript.py +++ b/swarmeval/worker/transcript.py @@ -81,14 +81,27 @@ class _Sources: equal message in any agent's context.""" +def transcript_images(transcript: RunTranscript) -> set[str]: + """Every image the run's contexts hold, which `check_transcript` needs the bytes of.""" + return { + image.sha256 + for context in transcript.contexts.values() + for message in context + if isinstance(message, ToolMessage) + for image in message.images + } + + def check_transcript( transcript: RunTranscript, *, prompts: Mapping[str, tuple[ChatMessage, ...]], tools: Mapping[str, ToolSchema], + images: Mapping[str, bytes] | None = None, ) -> TranscriptCheckRecord: """`prompts` is each agent's generation 0 before its first turn; `tools` every tool the run - could offer, by name, as requests carried them.""" + could offer, by name, as requests carried them; `images` the bytes of every image + `transcript_images` names, which requests carried inline.""" sources = _index(transcript) mismatches: list[TranscriptMismatch] = [] explained: list[str] = [] @@ -104,7 +117,15 @@ def check_transcript( if call.event_id in inherited: continue requests += 1 - mismatch = _check_request(call, call.agent_id, call.gen, call.length, transcript, tools) + mismatch = _check_request( + call, + call.agent_id, + call.gen, + call.length, + transcript, + tools, + {} if images is None else images, + ) if mismatch is not None: mismatches.append(mismatch) mismatches.extend( @@ -218,6 +239,7 @@ def mismatch(idx: int, event_id: str | None, detail: str) -> TranscriptMismatch: tool_call_id=outcome.result.call_id, content=result.content, is_error=result.is_error, + images=result.images, ) if message != expected: found.append(mismatch(idx, outcome.event_id, _differs("tool call", by))) @@ -271,6 +293,7 @@ def _check_request( length: int, transcript: RunTranscript, tools: Mapping[str, ToolSchema], + images: Mapping[str, bytes], ) -> TranscriptMismatch | None: def mismatch(detail: str) -> TranscriptMismatch: return TranscriptMismatch( @@ -297,7 +320,7 @@ def mismatch(detail: str) -> TranscriptMismatch: tools=tuple(tools[name] for name in call.options.tools), options=call.options, ) - body = to_wire(request).model_dump_json(exclude_none=True).encode() + body = to_wire(request, images).model_dump_json(exclude_none=True).encode() digest = hashlib.sha256(body).hexdigest() if digest != call.gateway.request_sha256: return mismatch( diff --git a/tests/core/test_display.py b/tests/core/test_display.py new file mode 100644 index 0000000..52be67d --- /dev/null +++ b/tests/core/test_display.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import pytest + +from swarmeval.core import load_case +from swarmeval.core.models import Display +from tests.core.test_loader import base_case, base_env, load_error, write + + +def desktop_env(**display: object) -> dict[str, object]: + env = base_env() + env["schema_version"] = 2 + env["sandbox_profiles"]["desktop"] = { + "image": "swarmeval/display:dev", + "fs": [{"path": "/workspace"}], + "display": display, + } + return env + + +def test_a_profile_display_defaults_to_xga(tmp_path: Path) -> None: + case = base_case() + case["swarm"]["agents"][0].update(tools=["browser", "computer"], sandbox_profile="desktop") + + (variant,) = load_case( + write(tmp_path, case, desktop_env(url="http://127.0.0.1:8080/")) + ).variants + + display = variant.env.sandbox_profiles["desktop"].display + assert display == Display(width=1024, height=768, url="http://127.0.0.1:8080/") + assert variant.env.sandbox_profiles["default"].display is None + + +@pytest.mark.parametrize("tool", ["browser", "computer"]) +def test_a_display_tool_needs_a_profile_with_a_display(tmp_path: Path, tool: str) -> None: + case = base_case() + case["swarm"]["agents"][0]["tools"] = ["shell", tool] + + message = load_error(tmp_path, case, desktop_env()) + + assert f"agent `dev` lists `{tool}`" in message + assert "profile `default`, which has no `display`" in message + + +@pytest.mark.parametrize("path", ["/run", "/run/swarm-display", "/run/swarm-display/out"]) +def test_a_key_path_cannot_overlap_the_display_state(tmp_path: Path, path: str) -> None: + env = desktop_env() + env["sandbox_profiles"]["desktop"]["fs"] = [{"path": path}] # pyright: ignore[reportIndexIssue] + + message = load_error(tmp_path, base_case(), env) + + assert f"key path `{path}` overlaps `/run/swarm-display`" in message + + +@pytest.mark.parametrize( + "display", [{"width": 100}, {"height": 5000}, {"url": "http://x/\nInjected: 1"}, {"depth": 24}] +) +def test_a_display_out_of_range_is_rejected(tmp_path: Path, display: dict[str, object]) -> None: + message = load_error(tmp_path, base_case(), desktop_env(**display)) + + assert "display" in message + + +def test_a_display_needs_env_version_2(tmp_path: Path) -> None: + env = desktop_env() + env["schema_version"] = 1 + + message = load_error(tmp_path, base_case(), env) + + assert "profile `desktop` has `display`, which `schema_version: 2` added" in message + + +def test_an_agent_without_a_sandbox_cannot_list_a_display_tool(tmp_path: Path) -> None: + case = base_case() + case["schema_version"] = 5 + case["swarm"]["agents"][1].update(tools=["browser"], sandbox="none") + + message = load_error(tmp_path, case, desktop_env()) + + assert "agent `qa` has `sandbox: none` but lists `browser`" in message + + +def test_no_agent_may_run_as_the_display_user(tmp_path: Path) -> None: + case = base_case() + case["swarm"]["agents"][0].update(sandbox_profile="desktop", os_user="swarmdisplay") + + message = load_error(tmp_path, case, desktop_env()) + + assert "agent `dev` sets `os_user: swarmdisplay`" in message diff --git a/tests/core/test_loader.py b/tests/core/test_loader.py index 6d19c8b..f7380b6 100644 --- a/tests/core/test_loader.py +++ b/tests/core/test_loader.py @@ -197,9 +197,9 @@ def test_missing_schema_version_is_rejected(tmp_path: Path) -> None: def test_unsupported_schema_version_is_rejected(tmp_path: Path) -> None: env = base_env() - env["schema_version"] = 2 + env["schema_version"] = 3 - assert "env.yaml has `schema_version: 2`" in load_error(tmp_path, base_case(), env) + assert "env.yaml has `schema_version: 3`" in load_error(tmp_path, base_case(), env) def test_workspace_is_required(tmp_path: Path) -> None: diff --git a/tests/events/test_display_export.py b/tests/events/test_display_export.py new file mode 100644 index 0000000..c25fd32 --- /dev/null +++ b/tests/events/test_display_export.py @@ -0,0 +1,134 @@ +"""Screenshots through the record: tool events and contexts hold references, the export embeds +the images once, and the transcript check rebuilds requests with them.""" + +import base64 +import hashlib +import struct +from pathlib import Path +from typing import Any, cast + +import pytest +from inspect_ai.event import ModelEvent, ToolEvent +from inspect_ai.log import read_eval_log +from inspect_ai.model import ChatMessageTool, ContentImage, ContentText + +from swarmeval.events import StoredRun, assemble, write_eval +from swarmeval.events.export import image_hashes +from swarmeval.runtime.display import DISPLAY_TOOLS, SCREENSHOT +from swarmeval.runtime.messages import ImageRef, ToolMessage +from swarmeval.runtime.records import CollectedFile, Exec, ExecResult +from swarmeval.runtime.specs import initial_context +from swarmeval.runtime.tools import SHELL +from swarmeval.sandbox.client import png_info +from swarmeval.worker.transcript import check_transcript +from tests.events.test_export import HEADER, stored +from tests.runtime.fakes import ANY_IMAGE, FakeSandbox, Harness, agent, call, harness, reply +from tests.worker.test_transcript import transcript_of, with_context + +SHOTS = 4 + + +def png(n: int) -> bytes: + return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", 13) + b"IHDR" + struct.pack(">II", 1024, n) + + +IMAGES = {hashlib.sha256(png(n)).hexdigest(): png(n) for n in range(1, SHOTS + 1)} + + +async def screenshots() -> Harness: + taken = iter(IMAGES.items()) + + def handler(command: Exec) -> ExecResult: + sha, data = next(taken) + collected = CollectedFile(path=SCREENSHOT, size=len(data), sha256=sha, png=png_info(data)) + return ExecResult(exit_code=0, stdout="Did it.\n", stderr="", collected=(collected,)) + + look = call("computer", '{"action": "screenshot"}') + h = harness( + (agent("a", tools=("computer",)),), + {"a": [*(reply("", look) for _ in range(SHOTS)), reply("done")]}, + sandbox=FakeSandbox(handler=handler), + tools=(SHELL, *DISPLAY_TOOLS), + ) + await h.loop.run() + return h + + +def with_images(run: StoredRun) -> StoredRun: + return StoredRun( + workspace=run.workspace, events=run.events, messages=run.messages, images=IMAGES + ) + + +async def test_the_export_embeds_each_image_once_and_shows_what_the_model_saw( + tmp_path: Path, +) -> None: + h = await screenshots() + run = with_images(stored(h.store)) + assert image_hashes(run) == set(IMAGES) + + path = tmp_path / "sample.eval" + write_eval(assemble(HEADER, run), path) + (sample,) = read_eval_log(path, resolve_attachments=True).samples or [] + + tools = [e for e in sample.events if isinstance(e, ToolEvent)] + first = tools[0].result + assert isinstance(first, list) + assert first[0] == ContentText(text="Did it.\n") + image = first[1] + assert isinstance(image, ContentImage) + assert image.image == f"data:image/png;base64,{base64.b64encode(png(1)).decode()}" + last = [e for e in sample.events if isinstance(e, ModelEvent)][-1] + shown = [m for m in last.input if isinstance(m, ChatMessageTool)] + assert len(shown) == SHOTS + parts = [m.content[1] for m in shown if isinstance(m.content, list)] + assert parts[0] == ContentText(text="[image omitted]"), "only the last 3 images are sent" + assert all(isinstance(p, ContentImage) for p in parts[1:]) + + +async def test_an_export_without_the_images_is_refused() -> None: + h = await screenshots() + + with pytest.raises(ValueError, match="names 4 image"): + assemble(HEADER, stored(h.store)) + + +async def test_tool_events_carry_image_references_under_schema_10() -> None: + h = await screenshots() + run = stored(h.store) + + payloads = [cast(dict[str, Any], r.payload) for r in run.events] + tool = next(p for p in payloads if p["event"] == "tool") + ours = tool["metadata"]["swarmeval"] + assert ours["schema_version"] == 10 + sha = next(iter(IMAGES)) + assert ours["images"] == [ + {"sha256": sha, "media_type": "image/png", "width": 1024, "height": 1} + ] + assert ours["exec"]["collected"][0]["sha256"] == sha + assert tool["result"] == "Did it.\n" + + +async def test_the_transcript_check_rebuilds_requests_with_their_images() -> None: + h = await screenshots() + transcript = transcript_of(h.store) + + def check(images: dict[str, bytes] | None = None) -> bool: + return check_transcript( + transcript, + prompts={"a": initial_context(agent("a", tools=("computer",)))}, + tools=h.loop.tool_schemas(), + images=images or ANY_IMAGE, + ).consistent + + assert check() + message = transcript.contexts[("a", 0)][3] + assert isinstance(message, ToolMessage) + swapped = message.model_copy(update={"images": (ImageRef(sha256="f" * 64, width=1, height=1),)}) + tampered = check_transcript( + with_context(transcript, "a", 3, swapped), + prompts={"a": initial_context(agent("a", tools=("computer",)))}, + tools=h.loop.tool_schemas(), + images=ANY_IMAGE, + ) + assert not tampered.consistent diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 73d8954..e0e4e2d 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -1,7 +1,8 @@ import asyncio +import hashlib import json from collections.abc import AsyncIterator -from dataclasses import dataclass +from dataclasses import dataclass, field import grpc import httpx2 @@ -25,6 +26,7 @@ from swarmeval.runtime.loop import RunLoop from swarmeval.runtime.messages import ( AssistantMessage, + ImageRef, ModelRequest, RequestOptions, SystemMessage, @@ -59,6 +61,16 @@ CALLERS: tuple[Caller, ...] = (DEV, ExtensionCaller("acme.judge")) +class MemoryImages: + def __init__(self) -> None: + self.data: dict[str, bytes] = {} + self.reads: list[str] = [] + + async def get(self, sha256: str) -> bytes: + self.reads.append(sha256) + return self.data[sha256] + + @dataclass class Rig: backend: MockBackend @@ -67,6 +79,7 @@ class Rig: http: httpx2.AsyncClient channel: grpc.aio.Channel attachments: Attachments + images: MemoryImages = field(default_factory=MemoryImages) def session(self, *, run_id: str = "run_1", owner_epoch: int = 1) -> GatewaySession: return GatewaySession( @@ -76,6 +89,7 @@ def session(self, *, run_id: str = "run_1", owner_epoch: int = 1) -> GatewaySess owner_epoch=owner_epoch, callers=CALLERS, writer=self.writer, + images=self.images, ) @@ -379,3 +393,98 @@ async def test_a_call_waiting_for_its_ack_fails_when_the_stream_goes() -> None: with pytest.raises(NotAttachedError, match="closed \\(stream ended\\) before it acknowledged"): await waiting assert attachments.lookup("k") is None + + +def _image(rig: Rig, data: bytes) -> ImageRef: + sha = hashlib.sha256(data).hexdigest() + rig.images.data[sha] = data + return ImageRef(sha256=sha, width=1024, height=768) + + +async def test_tool_images_reach_the_backend_in_a_user_message_after_the_tools(rig: Rig) -> None: + rig.backend.reply(completion("done")) + first, second = _image(rig, b"first png"), _image(rig, b"second png") + current = AssistantMessage( + reasoning="current thoughts", + tool_calls=( + ToolCall(id="c1", name="computer", arguments="{}"), + ToolCall(id="c2", name="computer", arguments="{}"), + ), + ) + + async with rig.session() as session: + await session.generate( + DEV, + request( + UserMessage(content="go"), + current, + ToolMessage(tool_call_id="c1", content="clicked", images=(first,)), + ToolMessage(tool_call_id="c2", content="", images=(second,)), + ), + parent_id=None, + ) + + sent = rig.backend.requests[0]["messages"] + assert [m["role"] for m in sent] == ["user", "assistant", "tool", "tool", "user"] + assert sent[2]["content"] == "clicked" + assert sent[3]["content"] == "" + assert sent[4]["content"] == [ + {"type": "text", "text": "Images from tool call c1:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,Zmlyc3QgcG5n"}}, + {"type": "text", "text": "Images from tool call c2:"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,c2Vjb25kIHBuZw=="}}, + ] + # The user message carrying the images comes after `passback`: the turn is still current. + assert sent[1]["reasoning_content"] == "current thoughts" + + +async def test_only_the_latest_images_are_sent(rig: Rig) -> None: + rig.backend.reply(completion("done")) + rig.backend.reply(completion("done")) + old, new = _image(rig, b"old"), _image(rig, b"new") + calls = (ToolCall(id="c1", name="computer", arguments="{}"),) + messages = ( + UserMessage(content="go"), + AssistantMessage(tool_calls=calls), + ToolMessage(tool_call_id="c1", content="one", images=(old,)), + AssistantMessage(tool_calls=(ToolCall(id="c2", name="computer", arguments="{}"),)), + ToolMessage(tool_call_id="c2", content="two", images=(new,)), + ) + base = request(*messages) + one = base.model_copy(update={"options": base.options.model_copy(update={"max_images": 1})}) + + async with rig.session() as session: + await session.generate(DEV, one, parent_id=None) + await session.generate(DEV, one, parent_id=None) + + sent = rig.backend.requests[0]["messages"] + assert sent[2]["content"] == "one\n[image omitted]" + assert [m["role"] for m in sent] == ["user", "assistant", "tool", "assistant", "tool", "user"] + assert sent[4]["content"] == "two" + assert sent[5]["content"][1]["image_url"]["url"] == "data:image/png;base64,bmV3" + assert rig.images.reads == [new.sha256], "the second request reuses the first's images" + + +async def test_a_tool_image_that_is_not_an_inline_png_is_refused(rig: Rig) -> None: + wire = { + "model": "qwen-test", + "messages": [ + {"role": "user", "content": "go"}, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "image_url", "image_url": {"url": "http://evil/x.png"}}], + }, + ], + } + + async with rig.session() as session: + key = session._keys["agent:dev"] # pyright: ignore[reportPrivateUsage] + reply = await rig.http.post( + "/v1/chat/completions", + json=wire, + headers={"Authorization": f"Bearer {key}", CALL_ID_HEADER: "call_1"}, + ) + + assert reply.json()["error"]["code"] == "invalid_request" + assert rig.backend.requests == [] diff --git a/tests/runtime/fakes.py b/tests/runtime/fakes.py index 18b64b4..4f0579a 100644 --- a/tests/runtime/fakes.py +++ b/tests/runtime/fakes.py @@ -3,7 +3,8 @@ import asyncio import hashlib import json -from collections.abc import Callable, Sequence +from collections import defaultdict +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any, Literal @@ -52,7 +53,7 @@ WebRequest, ) from swarmeval.runtime.specs import AgentSpec, Limits, RunSpec -from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL, WEB_REQUEST +from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL, WEB_REQUEST, Tool from swarmeval.runtime.writer import RunWriter from tests.gateway.mock_backend import completion, tool_call @@ -161,10 +162,15 @@ def messages(self, agent_id: str) -> list[ChatMessage]: """A gateway record with placeholder hash and raw response; `honest_record` fills them in.""" +ANY_IMAGE: Mapping[str, bytes] = defaultdict[str, bytes](lambda: b"image") +"""Image bytes for fakes, which never store any: every image reads the same. Pass it to +`check_transcript` too, so the request hashes it recomputes match `honest_record`'s.""" + + def honest_record(request: ModelRequest, response: ModelResponse) -> GatewayRecord: """What model-gateway records for a call it served faithfully: the hash of the body the worker sends, and a backend completion that normalizes to `response`.""" - body = to_wire(request).model_dump_json(exclude_none=True).encode() + body = to_wire(request, ANY_IMAGE).model_dump_json(exclude_none=True).encode() message = response.message raw = completion( message.content, @@ -328,6 +334,7 @@ def harness( fork: ForkStart | None = None, turn_policy: Literal["round_robin", "event_driven", "async"] = "round_robin", latency: dict[str, float] | None = None, + tools: Sequence[Tool] = (SHELL, WEB_REQUEST), ) -> Harness: store = store or FakeStore() pauser = pauser or FakePauser() @@ -356,7 +363,7 @@ def harness( sandbox_executor=sandbox, pauser=pauser, extensions=loaded, - tools=[SHELL, WEB_REQUEST], + tools=list(tools), web_client=web, fork=fork, ) diff --git a/tests/runtime/test_display.py b/tests/runtime/test_display.py new file mode 100644 index 0000000..bf58e5e --- /dev/null +++ b/tests/runtime/test_display.py @@ -0,0 +1,187 @@ +import hashlib +import json +import struct + +import pytest +from pydantic import ValidationError + +from swarmeval.runtime.display import ( + BROWSER, + COMPUTER, + DISPLAY_TOOLS, + SCREENSHOT, + BrowserArgs, + ComputerArgs, + display_output, +) +from swarmeval.runtime.messages import ImageRef, ToolMessage, UserMessage, visible_images +from swarmeval.runtime.records import CollectedFile, Exec, ExecResult, PngInfo, ToolCallRecord +from swarmeval.runtime.tools import BUILTIN_TOOL_NAMES, SHELL +from swarmeval.sandbox.client import png_info +from tests.runtime.fakes import FakeSandbox, agent, call, harness, reply + +SHOT = "ab" * 32 + + +def png(width: int, height: int) -> bytes: + return ( + b"\x89PNG\r\n\x1a\n" + struct.pack(">I", 13) + b"IHDR" + struct.pack(">II", width, height) + ) + + +def shot(collected: CollectedFile, exit_code: int = 0) -> ExecResult: + return ExecResult( + exit_code=exit_code, stdout="Did `click`.\n", stderr="", collected=(collected,) + ) + + +def test_display_tools_are_builtin_names() -> None: + assert {t.name for t in DISPLAY_TOOLS} <= set(BUILTIN_TOOL_NAMES) + + +@pytest.mark.parametrize( + ("args", "missing"), + [ + ({"action": "click"}, "`coordinate`"), + ({"action": "drag", "coordinate": [1, 2]}, "`start_coordinate`"), + ({"action": "type"}, "`text`"), + ({"action": "scroll"}, "`direction`"), + ], +) +def test_computer_actions_require_their_arguments(args: dict[str, object], missing: str) -> None: + with pytest.raises(ValidationError, match=missing): + ComputerArgs.model_validate(args) + + +def test_computer_points_are_two_integers() -> None: + with pytest.raises(ValidationError): + ComputerArgs.model_validate({"action": "click", "coordinate": [1, 2, 3]}) + with pytest.raises(ValidationError): + ComputerArgs.model_validate({"action": "click", "coordinate": [1.5, 2]}) + + +@pytest.mark.parametrize( + ("args", "missing"), + [ + ({"action": "navigate"}, "`url`"), + ({"action": "type", "ref": "e3"}, "`text`"), + ({"action": "select", "ref": "e3"}, "`values`"), + ({"action": "tab_select"}, "`index`"), + ], +) +def test_browser_actions_require_their_arguments(args: dict[str, object], missing: str) -> None: + with pytest.raises(ValidationError, match=missing): + BrowserArgs.model_validate(args) + + +def test_browser_refs_are_plain_tokens() -> None: + with pytest.raises(ValidationError): + BrowserArgs.model_validate({"action": "click", "ref": "e1] >> css=body"}) + + +def test_display_tools_run_as_the_display_user_and_collect_the_screenshot() -> None: + command = COMPUTER.build(ComputerArgs.model_validate({"action": "click", "coordinate": [3, 4]})) + + assert command.user == "swarmdisplay" + assert command.collect == (SCREENSHOT,) + assert command.argv[:2] == ("swarm-display", "computer") + assert json.loads(command.argv[2]) == { + "action": "click", + "coordinate": [3, 4], + "button": "left", + "count": 1, + "amount": 3, + "seconds": 1.0, + } + assert BROWSER.build(BrowserArgs(action="back")).argv[:2] == ("swarm-display", "browser") + + +def test_a_collected_png_becomes_an_image() -> None: + result = display_output( + call("computer", id="c1"), + shot( + CollectedFile( + path=SCREENSHOT, size=100, sha256=SHOT, png=PngInfo(width=1024, height=768) + ) + ), + ) + + assert not result.is_error + assert result.content == "Did `click`.\n" + assert result.images == (ImageRef(sha256=SHOT, width=1024, height=768),) + + +def test_no_screenshot_means_no_image() -> None: + result = display_output( + call("browser", id="c1"), shot(CollectedFile(path=SCREENSHOT, missing=True)) + ) + + assert (result.is_error, result.images) == (False, ()) + + +@pytest.mark.parametrize( + ("collected", "says"), + [ + (CollectedFile(path=SCREENSHOT, size=9, sha256=SHOT), "not a PNG"), + (CollectedFile(path=SCREENSHOT, size=99_000_000), "over sandboxd's limit"), + ], +) +def test_an_unusable_screenshot_is_an_error_the_agent_sees( + collected: CollectedFile, says: str +) -> None: + result = display_output(call("computer", id="c1"), shot(collected)) + + assert result.is_error + assert says in result.content + assert result.images == () + + +def test_png_info_reads_the_header_and_refuses_the_rest() -> None: + assert png_info(png(1024, 768)) == PngInfo(width=1024, height=768) + assert png_info(b"GIF89a" + bytes(30)) is None + assert png_info(png(0, 768)) is None + assert png_info(png(100_000, 10)) is None + assert png_info(png(10, 10)[:20]) is None + + +def test_visible_images_are_the_latest() -> None: + a, b, c = (ImageRef(sha256=x * 64, width=1, height=1) for x in "abc") + messages = ( + UserMessage(content="go"), + ToolMessage(tool_call_id="1", content="", images=(a, b)), + ToolMessage(tool_call_id="2", content=""), + ToolMessage(tool_call_id="3", content="", images=(c,)), + ) + + assert visible_images(messages, 2) == {(1, 1), (3, 0)} + assert visible_images(messages, 5) == {(1, 0), (1, 1), (3, 0)} + assert visible_images(messages, 0) == set() + + +async def test_a_screenshot_reaches_the_agents_context_and_the_record() -> None: + image = png(1024, 768) + sha = hashlib.sha256(image).hexdigest() + + def handler(command: Exec) -> ExecResult: + assert command.user == "swarmdisplay" + return shot( + CollectedFile(path=SCREENSHOT, size=len(image), sha256=sha, png=png_info(image)) + ) + + h = harness( + (agent(tools=("computer",)),), + {"a": [reply("", call("computer", '{"action": "screenshot"}')), reply("done")]}, + sandbox=FakeSandbox(handler=handler), + tools=(SHELL, *DISPLAY_TOOLS), + ) + + await h.loop.run() + + message = h.store.messages("a")[3] + assert isinstance(message, ToolMessage) + assert message.images == (ImageRef(sha256=sha, width=1024, height=768),) + record = h.store.records("tool")[0].record + assert isinstance(record, ToolCallRecord) + assert record.result.images == message.images + _, second = h.model.requests[1] + assert second.options.max_images == 3 diff --git a/tests/sandbox/test_client.py b/tests/sandbox/test_client.py index 6950116..f7ff159 100644 --- a/tests/sandbox/test_client.py +++ b/tests/sandbox/test_client.py @@ -332,3 +332,66 @@ async def test_restore_batches_paths_and_content_and_reports_unowned_paths(rig: ] assert [(d.path, d.uid) for r in requests for d in r.dirs] == [("/workspace/d", 1000)] assert unowned == ["/workspace/small.txt"] * len(requests) + + +PNG = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x04\x00\x00\x00\x03\x00rest" + + +async def test_a_display_profile_asks_sandboxd_for_a_display(rig: Rig) -> None: + profile = SandboxProfile.model_validate( + {"image": "swarmeval/display:dev", "display": {"width": 1280, "url": "http://127.0.0.1/"}} + ) + + await rig.client.create("box_a", profile) + + (request,) = rig.server.requests + assert request.HasField("display") + assert (request.display.width, request.display.height, request.display.url) == ( + 1280, + 768, + "http://127.0.0.1/", + ) + + +async def test_a_profile_without_a_display_sends_none(rig: Rig) -> None: + await rig.client.create("box_a", SandboxProfile(image="busybox")) + + (request,) = rig.server.requests + assert not request.HasField("display") + + +async def test_exec_runs_as_the_commands_user_and_returns_what_it_collected(rig: Rig) -> None: + rig.server.exec_items = [ + header_item( + exit_code=0, + collected=[ + pb.CollectedFile(path="/run/out/shot.png", size=len(PNG), sha256=sha(PNG)), + pb.CollectedFile(path="/run/out/none.png", missing=True), + ], + ), + *blob_items(PNG), + ] + command = Exec( + argv=("true",), user="swarmdisplay", collect=("/run/out/shot.png", "/run/out/none.png") + ) + + result = await rig.client.exec("box_a", "agent", command, call_id="c1") + + (request,) = rig.server.requests + assert (request.user, list(request.collect)) == ("swarmdisplay", list(command.collect)) + shot, none = result.collected + assert (shot.sha256, shot.png and (shot.png.width, shot.png.height)) == (sha(PNG), (1024, 768)) + assert (none.missing, none.sha256, none.png) == (True, None, None) + assert rig.blobs.stored[sha(PNG)] == PNG + + +async def test_a_collected_file_that_is_not_a_png_has_no_png_info(rig: Rig) -> None: + rig.server.exec_items = [ + header_item(collected=[pb.CollectedFile(path="/run/x", size=3, sha256=sha(b"gif"))]), + *blob_items(b"gif"), + ] + + result = await rig.client.exec("box_a", None, LS, call_id="c1") + + assert result.collected[0].png is None + assert rig.server.requests[0].user == "" diff --git a/tests/worker/test_display_e2e.py b/tests/worker/test_display_e2e.py new file mode 100644 index 0000000..ef7ccdc --- /dev/null +++ b/tests/worker/test_display_e2e.py @@ -0,0 +1,166 @@ +"""`browser` and `computer` through the whole platform: sandboxd starts the display image's +screen, screenshots reach the model inline, the export embeds them, and a fork after the first +display call is refused. Needs the display image (deploy/images/display) on the docker host.""" + +import json +import os +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +import grpc +import pytest +import yaml +from inspect_ai.event import ModelEvent, ToolEvent +from inspect_ai.log import read_eval_log +from inspect_ai.model import ChatMessageTool, ContentImage + +from swarmeval.analysis.exports import load_events_table +from swarmeval.control.bundles import pack +from swarmeval.events import export_key +from swarmeval.proto.swarmeval.control.v1 import control_pb2 as pb +from tests.gateway.mock_backend import completion, tool_call +from tests.models import chosen +from tests.worker.conftest import Platform + +IMAGE = os.environ.get("SWARMEVAL_IT_DISPLAY_IMAGE", "swarmeval/display:dev") + +pytestmark = [ + pytest.mark.docker, + pytest.mark.skipif( + subprocess.run(["docker", "image", "inspect", IMAGE], capture_output=True).returncode != 0, + reason=f"display image {IMAGE} is not on the docker host; build deploy/images/display", + ), +] + +PAGE = """Shop +""" +CASE: dict[str, Any] = { + "schema_version": 4, + "id": "desktop", + "workspace": "ws_e2e", + "swarm": { + "agents": [{"id": "a", "prompt": "a.md", "tools": ["browser", "computer"]}], + "limits": {"max_turns": 6}, + }, + "task": {"input": "task.md"}, +} +ENV: dict[str, Any] = { + "schema_version": 2, + "sandbox_profiles": { + "default": { + "image": IMAGE, + "fs": [{"path": "/workspace"}], + "limits": {"memory": "2gib", "pids": 512}, + "files": [{"from": "site", "to": "/workspace"}], + "display": {"url": "file:///workspace/index.html"}, + } + }, +} + + +def bundle(root: Path) -> bytes: + (root / "site").mkdir() + files = { + "case.yaml": yaml.safe_dump(CASE), + "env.yaml": yaml.safe_dump(ENV), + "a.md": "You are a.", + "task.md": "Pay.", + "site/index.html": PAGE, + } + for rel, text in files.items(): + (root / rel).write_text(text) + return pack(root) + + +def policy(request: dict[str, Any]) -> dict[str, Any]: + turns = sum(1 for m in request["messages"] if m["role"] == "assistant") + if turns == 0: + look = {"action": "snapshot", "screenshot": True} + return completion("", tool_calls=[tool_call("browser", json.dumps(look), id="b1")]) + if turns == 1: + snapshot = next(m["content"] for m in request["messages"] if m["role"] == "tool") + ref = snapshot.split('button "Pay" [ref=')[1].split("]")[0] + click = {"action": "click", "ref": ref} + shot = {"action": "click", "coordinate": [10, 10]} + return completion( + "", + tool_calls=[ + tool_call("browser", json.dumps(click), id="b2"), + tool_call("computer", json.dumps(shot), id="c1"), + ], + ) + return completion("done") + + +async def test_an_agent_drives_the_display_and_sees_its_screenshots( + platform: Platform, tmp_path: Path +) -> None: + platform.backend.respond = policy + (run_id,) = ( + await platform.control.SubmitRuns( + pb.SubmitRunsRequest(models=chosen("mock-model"), case_bundle=bundle(tmp_path)) + ) + ).run_ids + + outcome = (await platform.worker.drain())[run_id] + + assert outcome.status == "done", outcome.error + last = platform.backend.requests[-1]["messages"] + roles = [m["role"] for m in last] + assert roles == [ + "system", + "user", + "assistant", + "tool", + "user", + "assistant", + "tool", + "tool", + "user", + ] + assert "Title: paid" in last[6]["content"] + images = [p for m in (last[4], last[8]) for p in m["content"] if p["type"] == "image_url"] + assert len(images) == 2, "the first snapshot's and the computer click's screenshots" + assert all(p["image_url"]["url"].startswith("data:image/png;base64,iVBOR") for p in images) + + events = (await load_events_table(platform.store, run_id)).to_pylist() + tools = [json.loads(e["payload"]) for e in events if e["type"] == "tool"] + ours = [t["metadata"]["swarmeval"] for t in tools] + assert [len(o.get("images", [])) for o in ours] == [1, 0, 1] + assert [o["exec"]["processes"] for o in ours] == [[], [], []] + assert ours[2]["images"][0]["width"] == 1024 + + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "sample.eval" + path.write_bytes(platform.store.get(export_key(run_id))) + (sample,) = read_eval_log(path, resolve_attachments=True).samples or [] + shown = [e.result for e in sample.events if isinstance(e, ToolEvent)] + assert isinstance(shown[0], list) + assert isinstance(shown[0][1], ContentImage) + final = [e for e in sample.events if isinstance(e, ModelEvent)][-1] + seen = [ + c.image + for m in final.input + if isinstance(m, ChatMessageTool) and isinstance(m.content, list) + for c in m.content + if isinstance(c, ContentImage) + ] + assert len(seen) == 2 + assert all(i.startswith("data:image/png;base64,iVBOR") for i in seen) + + models = [e for e in events if e["type"] == "model"] + before = await platform.control.ForkRun( + pb.ForkRunRequest(run_id=run_id, at_event_id=models[0]["event_id"], actor="ada") + ) + assert before.run.forked_from == run_id, "a fork before the first display call goes on" + with pytest.raises(grpc.aio.AioRpcError) as err: + await platform.control.ForkRun( + pb.ForkRunRequest(run_id=run_id, at_event_id=models[-1]["event_id"], actor="ada") + ) + assert err.value.code() == grpc.StatusCode.FAILED_PRECONDITION + assert "used `browser`" in (err.value.details() or "") + # The fork from before the first display call runs, here, so no later test's worker drains it. + forked = (await platform.worker.drain())[before.run.run_id] + assert forked.status == "done", forked.error