From 92c8fd2c024360541082a6373fdfef398f2ff608 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 11:41:25 +0800 Subject: [PATCH 01/23] Add isolated Rainbond WebArena slice lifecycle --- .env.example | 3 + deploy/webarena-rainbond/gitlab/Dockerfile | 7 + .../gitlab/entrypoint-wrapper.sh | 53 +++ deploy/webarena-rainbond/reddit/Dockerfile | 8 + .../reddit/entrypoint-wrapper.sh | 53 +++ .../shopping-admin/Dockerfile | 1 + deploy/webarena-rainbond/shopping/Dockerfile | 1 + .../public_webarena/README.md | 26 ++ .../rainbond-pool.example.json | 30 ++ .../public_webarena/rainbond_slices.py | 441 ++++++++++++++++++ .../public_webarena/runner.py | 6 +- .../public_webarena/webarena_environment.py | 110 ++++- scripts/manage_webarena_rainbond_pool.py | 51 ++ scripts/run_webarena_qualification.py | 75 ++- .../test_qualification_orchestrator.py | 34 +- .../public_webarena/test_rainbond_slices.py | 178 +++++++ .../test_webarena_environment.py | 97 ++++ 17 files changed, 1113 insertions(+), 61 deletions(-) create mode 100644 deploy/webarena-rainbond/gitlab/Dockerfile create mode 100755 deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh create mode 100644 deploy/webarena-rainbond/reddit/Dockerfile create mode 100755 deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh create mode 100644 deploy/webarena-rainbond/shopping-admin/Dockerfile create mode 100644 deploy/webarena-rainbond/shopping/Dockerfile create mode 100644 examples/trajectory_replay/public_webarena/rainbond-pool.example.json create mode 100644 harness/trajectory_replay/public_webarena/rainbond_slices.py create mode 100644 scripts/manage_webarena_rainbond_pool.py create mode 100644 tests/trajectory_replay/public_webarena/test_rainbond_slices.py diff --git a/.env.example b/.env.example index 6f47cf88e..e863fd18d 100644 --- a/.env.example +++ b/.env.example @@ -71,3 +71,6 @@ ZENROWS_API_KEY= SCRAPFLY_API_KEY= OXYLABS_USERNAME= OXYLABS_PASSWORD= +# Rainbond OpenAPI token used only by the disposable WebArena slice manager. +# Create it in Rainbond: Personal Center -> Access Token. +RAINBOND_API_TOKEN= diff --git a/deploy/webarena-rainbond/gitlab/Dockerfile b/deploy/webarena-rainbond/gitlab/Dockerfile new file mode 100644 index 000000000..12fde38a4 --- /dev/null +++ b/deploy/webarena-rainbond/gitlab/Dockerfile @@ -0,0 +1,7 @@ +FROM am1n3e/webarena-verified-gitlab@sha256:673ba94d757c5b3f4bb55751ad8b745a33bad2b987349322403cbc41035a92e0 + +COPY entrypoint-wrapper.sh /usr/local/bin/webarena-rainbond-entrypoint +RUN chmod 0755 /usr/local/bin/webarena-rainbond-entrypoint + +ENV WA_ORIGINAL_ENTRYPOINT=/opt/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/webarena-rainbond-entrypoint"] diff --git a/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh b/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh new file mode 100755 index 000000000..1d389af5f --- /dev/null +++ b/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +case "${WA_ENV_CTRL_EXTERNAL_SITE_URL:-}" in + http://*|https://*) ;; + *) + echo "WA_ENV_CTRL_EXTERNAL_SITE_URL must be an absolute HTTP URL" >&2 + exit 64 + ;; +esac + +"${WA_ORIGINAL_ENTRYPOINT}" "$@" & +child_pid=$! + +terminate() { + kill -TERM "${child_pid}" 2>/dev/null || true + wait "${child_pid}" || true +} +trap terminate INT TERM + +control_url="http://127.0.0.1:${WA_ENV_CTRL_PORT:-8877}/status" +attempt=0 +until curl --fail --silent --show-error "${control_url}" >/dev/null 2>&1; do + if ! kill -0 "${child_pid}" 2>/dev/null; then + wait "${child_pid}" + exit $? + fi + attempt=$((attempt + 1)) + if [ "${attempt}" -ge 360 ]; then + echo "env-ctrl did not become ready within 30 minutes" >&2 + terminate + exit 1 + fi + sleep 5 +done + +attempt=0 +until env-ctrl --verbose init --base-url "${WA_ENV_CTRL_EXTERNAL_SITE_URL}"; do + if ! kill -0 "${child_pid}" 2>/dev/null; then + wait "${child_pid}" + exit $? + fi + attempt=$((attempt + 1)) + if [ "${attempt}" -ge 60 ]; then + echo "env-ctrl init failed after 60 attempts" >&2 + terminate + exit 1 + fi + sleep 5 +done + +touch /tmp/rainbond-env-ctrl-init-ok +wait "${child_pid}" diff --git a/deploy/webarena-rainbond/reddit/Dockerfile b/deploy/webarena-rainbond/reddit/Dockerfile new file mode 100644 index 000000000..b8e50f52f --- /dev/null +++ b/deploy/webarena-rainbond/reddit/Dockerfile @@ -0,0 +1,8 @@ +FROM am1n3e/webarena-verified-reddit@sha256:0594908059a03e5f610005689440a95c05e776e734006197c7b1eba12734cb46 + +COPY entrypoint-wrapper.sh /usr/local/bin/webarena-rainbond-entrypoint +RUN chmod 0755 /usr/local/bin/webarena-rainbond-entrypoint + +ENV WA_ORIGINAL_ENTRYPOINT=/docker-entrypoint.sh +ENTRYPOINT ["/usr/local/bin/webarena-rainbond-entrypoint"] +CMD ["supervisord", "-n", "-j", "/supervisord.pid"] diff --git a/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh b/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh new file mode 100755 index 000000000..1d389af5f --- /dev/null +++ b/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +case "${WA_ENV_CTRL_EXTERNAL_SITE_URL:-}" in + http://*|https://*) ;; + *) + echo "WA_ENV_CTRL_EXTERNAL_SITE_URL must be an absolute HTTP URL" >&2 + exit 64 + ;; +esac + +"${WA_ORIGINAL_ENTRYPOINT}" "$@" & +child_pid=$! + +terminate() { + kill -TERM "${child_pid}" 2>/dev/null || true + wait "${child_pid}" || true +} +trap terminate INT TERM + +control_url="http://127.0.0.1:${WA_ENV_CTRL_PORT:-8877}/status" +attempt=0 +until curl --fail --silent --show-error "${control_url}" >/dev/null 2>&1; do + if ! kill -0 "${child_pid}" 2>/dev/null; then + wait "${child_pid}" + exit $? + fi + attempt=$((attempt + 1)) + if [ "${attempt}" -ge 360 ]; then + echo "env-ctrl did not become ready within 30 minutes" >&2 + terminate + exit 1 + fi + sleep 5 +done + +attempt=0 +until env-ctrl --verbose init --base-url "${WA_ENV_CTRL_EXTERNAL_SITE_URL}"; do + if ! kill -0 "${child_pid}" 2>/dev/null; then + wait "${child_pid}" + exit $? + fi + attempt=$((attempt + 1)) + if [ "${attempt}" -ge 60 ]; then + echo "env-ctrl init failed after 60 attempts" >&2 + terminate + exit 1 + fi + sleep 5 +done + +touch /tmp/rainbond-env-ctrl-init-ok +wait "${child_pid}" diff --git a/deploy/webarena-rainbond/shopping-admin/Dockerfile b/deploy/webarena-rainbond/shopping-admin/Dockerfile new file mode 100644 index 000000000..bf91f7434 --- /dev/null +++ b/deploy/webarena-rainbond/shopping-admin/Dockerfile @@ -0,0 +1 @@ +FROM am1n3e/webarena-verified-shopping_admin@sha256:d0531dd27ed98d0c459ff9e88118bf2ed8b660b0ed99c38837db46c065a5be13 diff --git a/deploy/webarena-rainbond/shopping/Dockerfile b/deploy/webarena-rainbond/shopping/Dockerfile new file mode 100644 index 000000000..dfe0030bd --- /dev/null +++ b/deploy/webarena-rainbond/shopping/Dockerfile @@ -0,0 +1 @@ +FROM am1n3e/webarena-verified-shopping@sha256:3e8cb9b945ea9b1c94ab26dba53e8d12dd0406abbf4bf686fd3bb2b6a5908feb diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index d39889440..be3799ef5 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -32,3 +32,29 @@ Measured attempts, screenshots, HAR, and reports stay outside Git. The repositor the suite, qualification reasons, evaluator rules, source bindings, and a sanitized publication summary. Use `verify_metrics.py` to audit raw measurements and `rebuild_report.py` to render from existing attempts; neither reruns browsers. + +## Disposable Rainbond environments + +For concurrent state-changing runs, use a complete Rainbond slice per worker instead of +sharing sites or relying on separate login accounts. A slice contains Shopping Admin, +Shopping, Reddit, and GitLab with independent writable container layers and unique +domains. Reset deletes and recreates the whole slice from the pinned baseline; destroy +removes the application. The read-only Map service may be shared. + +Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, +and provision the required concurrency: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config provision --pool-id --count --output +``` + +Pass each generated `slice-*/webarena.config.json` as a separate +`--webarena-config` argument to `scripts/run_webarena_qualification.py`. Overlapping +site batches run concurrently only when distinct slice configs are supplied. Afterwards: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config destroy --slice-dir /slice-1 \ + --slice-dir /slice-2 +``` diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json new file mode 100644 index 000000000..61cd7823b --- /dev/null +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -0,0 +1,30 @@ +{ + "schema": "lex_webarena_rainbond_pool/1", + "api_base_url": "https://rainbond.example.com", + "team_id": "replace-with-efficiency-team-id", + "team_name": "replace-with-efficiency-team-name", + "region_name": "rainbond", + "source_app_id": 192, + "source_services": { + "shopping_admin": "replace-with-baseline-service-id", + "shopping": "replace-with-baseline-service-id", + "reddit": "replace-with-baseline-service-id", + "gitlab": "replace-with-baseline-service-id" + }, + "source_component_names": { + "shopping_admin": "shopping-admin", + "shopping": "shopping", + "reddit": "reddit", + "gitlab": "gitlab" + }, + "app_prefix": "webarena-slice", + "domain_suffix": "dev.example.com", + "token_env": "RAINBOND_API_TOKEN", + "image_lock": "../../../../evaluation/browser-use-public/webarena-environment-images.json", + "readiness_timeout_seconds": 1800, + "shared_environments": { + "__MAP__": { + "urls": ["https://replace-with-shared-read-only-map"] + } + } +} diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py new file mode 100644 index 000000000..fe260abee --- /dev/null +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -0,0 +1,441 @@ +"""Provision disposable, fully isolated WebArena slices on Rainbond.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import quote + +SCHEMA = "lex_webarena_rainbond_pool/1" +RECEIPT_SCHEMA = "lex_webarena_rainbond_slice/1" +STATEFUL_SITES = ("shopping_admin", "shopping", "reddit", "gitlab") +SITE_PORTS = {"shopping_admin": 80, "shopping": 80, "reddit": 80, "gitlab": 8023} +_SAFE_ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") + + +def _canonical_sha256(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _image_digest(value: str) -> str: + marker = "@sha256:" + if marker not in value: + raise ValueError(f"container image is not digest pinned: {value!r}") + return "sha256:" + value.rsplit(marker, 1)[1] + + +def _atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + content = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +class RainbondAPI: + """Small OpenAPI client containing only the calls required by slice lifecycle.""" + + def __init__(self, base_url: str, token: str, *, timeout_seconds: float = 60) -> None: + self.base_url = base_url.rstrip("/") + self.token = token + self.timeout_seconds = timeout_seconds + + def request(self, method: str, path: str, body: Any | None = None) -> Any: + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request( + self.base_url + path, + data=data, + method=method, + headers={ + "Authorization": self.token, + "Accept": "application/json", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response: + payload = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:2000] + raise RuntimeError( + f"Rainbond OpenAPI {method} {path} failed with HTTP {exc.code}: {detail}" + ) from exc + if not payload: + return None + return json.loads(payload) + + +@dataclass(frozen=True) +class PoolConfig: + api_base_url: str + team_id: str + team_name: str + region_name: str + source_app_id: int + source_services: dict[str, str] + app_prefix: str + domain_suffix: str + token_env: str + image_lock: Path + source_runtime_images: dict[str, str] + shared_environments: dict[str, Any] + readiness_timeout_seconds: int + + @classmethod + def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: + config_path = Path(path).resolve() + raw = json.loads(config_path.read_text(encoding="utf-8")) + if raw.get("schema") != SCHEMA: + raise ValueError(f"Rainbond pool config schema must be {SCHEMA}") + source_services = raw.get("source_services") + if not isinstance(source_services, dict) or set(source_services) != set(STATEFUL_SITES): + raise ValueError(f"source_services must contain exactly {list(STATEFUL_SITES)}") + normalized_services = { + str(site): str(service_id) for site, service_id in source_services.items() + } + if any(not value for value in normalized_services.values()): + raise ValueError("source_services values must be non-empty service IDs") + prefix = str(raw.get("app_prefix") or "webarena-slice") + if not _SAFE_ID.fullmatch(prefix): + raise ValueError("app_prefix must be a lowercase DNS-safe identifier") + suffix = str(raw.get("domain_suffix") or "").strip().strip(".") + if not suffix or "/" in suffix or ":" in suffix: + raise ValueError("domain_suffix must be a DNS suffix without scheme or port") + lock_value = Path(str(raw.get("image_lock") or "")) + lock_path = lock_value if lock_value.is_absolute() else (config_path.parent / lock_value) + if not lock_path.is_file(): + raise ValueError("Rainbond pool image_lock is missing") + runtime_images_value = raw.get("source_runtime_images") or {} + if not isinstance(runtime_images_value, dict) or set(runtime_images_value) - set( + STATEFUL_SITES + ): + raise ValueError("source_runtime_images contains an unknown site") + runtime_images = {str(site): str(image) for site, image in runtime_images_value.items()} + for image in runtime_images.values(): + _image_digest(image) + value = cls( + api_base_url=str(raw.get("api_base_url") or "").rstrip("/"), + team_id=str(raw.get("team_id") or ""), + team_name=str(raw.get("team_name") or ""), + region_name=str(raw.get("region_name") or ""), + source_app_id=int(raw.get("source_app_id") or 0), + source_services=normalized_services, + app_prefix=prefix, + domain_suffix=suffix, + token_env=str(raw.get("token_env") or "RAINBOND_API_TOKEN"), + image_lock=lock_path.resolve(), + source_runtime_images=runtime_images, + shared_environments=dict(raw.get("shared_environments") or {}), + readiness_timeout_seconds=int(raw.get("readiness_timeout_seconds") or 1800), + ) + if ( + not all((value.api_base_url, value.team_id, value.team_name, value.region_name)) + or value.source_app_id < 1 + ): + raise ValueError("Rainbond pool API, team, region, and source app are required") + return value, raw, config_path + + +class RainbondSlicePool: + """Create and replace isolated four-site applications from one immutable baseline.""" + + def __init__( + self, + config_path: str | Path, + *, + api: RainbondAPI | None = None, + sleep: Any = time.sleep, + ) -> None: + self.config, self.raw_config, self.config_path = PoolConfig.load(config_path) + self.config_sha256 = _canonical_sha256(self.raw_config) + token = os.environ.get(self.config.token_env, "") + if api is None and not token: + raise ValueError( + f"Rainbond API token environment variable is missing: {self.config.token_env}" + ) + self.api = api or RainbondAPI(self.config.api_base_url, token) + self.sleep = sleep + self.image_lock = json.loads(self.config.image_lock.read_text(encoding="utf-8")) + + def _apps_path(self) -> str: + config = self.config + return f"/openapi/v1/teams/{quote(config.team_id)}/regions/{quote(config.region_name)}/apps" + + def _app_path(self, app_id: int) -> str: + return f"{self._apps_path()}/{app_id}" + + def _domain(self, pool_id: str, slot: int, site: str, app_id: int) -> str: + label = ( + f"{self.config.app_prefix}-{pool_id}-{slot}-{app_id}-{site.replace('_', '-')}" + ) + if len(label) > 63: + label = f"{label[:50].rstrip('-')}-{hashlib.sha256(label.encode()).hexdigest()[:10]}" + return f"{label}.{self.config.domain_suffix}" + + @staticmethod + def _validate_pool_id(pool_id: str) -> str: + normalized = pool_id.strip().lower() + if not _SAFE_ID.fullmatch(normalized): + raise ValueError("pool_id must be a lowercase DNS-safe identifier") + return normalized + + def _create_app(self, pool_id: str, slot: int) -> int: + name = f"{self.config.app_prefix}-{pool_id}-{slot}" + note = f"managed-by=browser-eval;pool={pool_id};slot={slot}" + result = self.api.request("POST", self._apps_path(), {"app_name": name, "app_note": note}) + app_id = int((result or {}).get("ID") or 0) + if app_id < 1: + raise RuntimeError("Rainbond create app response has no ID") + return app_id + + def _copy_components(self, app_id: int) -> dict[str, dict[str, Any]]: + config = self.config + result = self.api.request( + "POST", + f"{self._app_path(config.source_app_id)}/copy", + { + "services": [ + {"service_id": config.source_services[site]} for site in STATEFUL_SITES + ], + "target_team_name": config.team_name, + "target_region_name": config.region_name, + "target_app_id": app_id, + }, + ) + rows = list((result or {}).get("services") or []) + by_source_name = { + site: str(self.raw_config.get("source_component_names", {}).get(site) or site) + for site in STATEFUL_SITES + } + mapped: dict[str, dict[str, Any]] = {} + for site, component_name in by_source_name.items(): + matches = [row for row in rows if str(row.get("service_cname")) == component_name] + if len(matches) != 1: + raise RuntimeError( + f"Rainbond copy returned {len(matches)} components named {component_name!r}" + ) + mapped[site] = matches[0] + return mapped + + def _configure_component( + self, + app_id: int, + *, + pool_id: str, + slot: int, + site: str, + component: dict[str, Any], + ) -> dict[str, Any]: + service_id = str(component.get("service_id") or "") + if not service_id: + raise RuntimeError(f"Rainbond copied component has no service_id: {site}") + domain = self._domain(pool_id, slot, site, app_id) + site_url = f"http://{domain}" + magento = site in {"shopping_admin", "shopping"} + self.api.request( + "POST", + f"{self._app_path(app_id)}/domains", + { + "protocol": "http", + "http": { + "service_id": service_id, + "container_port": SITE_PORTS[site], + "certificate_id": 0, + "domain_name": domain, + "domain_cookie": "", + "domain_header": "", + "domain_path": "/", + "the_weight": 100, + "whether_open": True, + "auto_ssl": False, + "auto_ssl_config": "", + "rule_extensions": [], + "configuration": { + "proxy_body_size": 100 if magento else 0, + "proxy_buffer_numbers": 8 if magento else 4, + "proxy_buffer_size": 64 if magento else 4, + "proxy_buffering": "on" if magento else "off", + "proxy_connect_timeout": 75, + "proxy_read_timeout": 600, + "proxy_send_timeout": 600, + "set_headers": [], + }, + }, + }, + ) + envs = [ + { + "note": "public origin for WebArena initialization", + "name": "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "value": site_url, + "is_change": True, + "scope": "inner", + } + ] + if magento: + envs.append( + { + "note": "allow Magento initialization on loaded nodes", + "name": "WA_ENV_CTRL_CMD_TIMEOUT_SEC", + "value": "600", + "is_change": True, + "scope": "inner", + } + ) + if site == "gitlab": + envs.append( + { + "note": "preserve the image's internal 8023 listener", + "name": "WA_ENV_CTRL_SKIP_RECONFIGURE", + "value": "true", + "is_change": True, + "scope": "inner", + } + ) + self.api.request( + "PUT", + f"{self._app_path(app_id)}/services/{quote(service_id)}/envs", + {"envs": envs}, + ) + expected_image = str(self.image_lock["images"][site]["pinned"]) + expected_runtime_image = self.config.source_runtime_images.get(site, expected_image) + actual_image = str(component.get("image") or "") + if actual_image and _image_digest(actual_image) != _image_digest(expected_runtime_image): + raise RuntimeError( + f"Rainbond copied {site} runtime image mismatch: " + f"{actual_image!r} != {expected_runtime_image!r}" + ) + return { + "service_id": service_id, + "service_cname": component.get("service_cname"), + "image": expected_image, + "runtime_image": actual_image or expected_image, + "site_url": site_url, + } + + def _wait_site(self, url: str) -> None: + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_error = "not checked" + while time.monotonic() < deadline: + try: + request = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(request, timeout=20) as response: + if 200 <= response.status < 500: + return + last_error = f"HTTP {response.status}" + except (OSError, urllib.error.URLError) as exc: + last_error = str(exc) + self.sleep(5) + raise TimeoutError(f"Rainbond WebArena site did not become ready: {url}: {last_error}") + + def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: + pool_id = self._validate_pool_id(pool_id) + if slot < 1: + raise ValueError("slot must be positive") + app_id = self._create_app(pool_id, slot) + try: + copied = self._copy_components(app_id) + components = { + site: self._configure_component( + app_id, + pool_id=pool_id, + slot=slot, + site=site, + component=copied[site], + ) + for site in STATEFUL_SITES + } + self.api.request( + "POST", + f"{self._app_path(app_id)}/operations", + { + "action": "deploy", + "service_ids": [row["service_id"] for row in components.values()], + }, + ) + for component in components.values(): + self._wait_site(str(component["site_url"])) + except BaseException: + self.api.request("DELETE", f"{self._app_path(app_id)}?force=1") + raise + receipt = { + "schema": RECEIPT_SCHEMA, + "pool_config_sha256": self.config_sha256, + "pool_id": pool_id, + "slot": slot, + "generation": uuid.uuid4().hex, + "app_id": app_id, + "components": components, + "created_at_unix": int(time.time()), + } + _atomic_json(receipt_path, receipt) + return receipt + + def destroy(self, receipt_path: Path, *, missing_ok: bool = True) -> None: + if not receipt_path.exists(): + if missing_ok: + return + raise FileNotFoundError(receipt_path) + receipt = self.validate_receipt(receipt_path) + self.api.request("DELETE", f"{self._app_path(int(receipt['app_id']))}?force=1") + receipt_path.unlink() + + def reset(self, receipt_path: Path) -> dict[str, Any]: + previous = self.validate_receipt(receipt_path) + pool_id = str(previous["pool_id"]) + slot = int(previous["slot"]) + self.destroy(receipt_path, missing_ok=False) + return self.provision(pool_id, slot, receipt_path) + + def validate_receipt(self, receipt_path: Path) -> dict[str, Any]: + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if ( + receipt.get("schema") != RECEIPT_SCHEMA + or receipt.get("pool_config_sha256") != self.config_sha256 + or set(receipt.get("components") or {}) != set(STATEFUL_SITES) + or int(receipt.get("app_id") or 0) < 1 + ): + raise ValueError("Rainbond slice receipt is invalid or belongs to another pool") + return receipt + + def runtime_config(self, receipt: dict[str, Any]) -> dict[str, Any]: + environments = dict(self.config.shared_environments) + for site, component in receipt["components"].items(): + environments[f"__{site.upper()}__"] = {"urls": [component["site_url"]]} + return { + "image_lock": str(self.config.image_lock), + "environments": environments, + "rainbond_slice": { + "pool_config": str(self.config_path), + "receipt": "slice-receipt.json", + }, + } + + +def provision_pool(config_path: Path, output_dir: Path, pool_id: str, count: int) -> list[Path]: + if count < 1: + raise ValueError("count must be positive") + pool = RainbondSlicePool(config_path) + created: list[Path] = [] + try: + for slot in range(1, count + 1): + slot_dir = output_dir / f"slice-{slot}" + receipt_path = slot_dir / "slice-receipt.json" + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + created.append(slot_dir) + except BaseException: + for slot_dir in reversed(created): + pool.destroy(slot_dir / "slice-receipt.json") + raise + return created diff --git a/harness/trajectory_replay/public_webarena/runner.py b/harness/trajectory_replay/public_webarena/runner.py index b9dc2357e..3e34cb03e 100644 --- a/harness/trajectory_replay/public_webarena/runner.py +++ b/harness/trajectory_replay/public_webarena/runner.py @@ -946,17 +946,16 @@ def run_evaluation( for task_index, declared_task in enumerate(runnable_tasks): task = _materialize_task(declared_task, fixture_origin) - if (task.environment or {}).get("kind") == "webarena_verified": - assert webarena_environment is not None - task = webarena_environment.materialize_task_actions(task) if trajectory_authoring_run: task = _bounded_authoring_task(task) + base_task = task engine_order = ( engines if len(engines) == 1 or task_index % 2 == 0 else tuple(reversed(engines)) ) for engine in engine_order: initialized_sites = initialized_sites_by_engine[engine] for attempt in range(1, max_retries + 2): + task = base_task artifact_dir = ( output / "tasks" / task.task_id / engine / f"attempt-{attempt}" ) @@ -1003,6 +1002,7 @@ def run_evaluation( authoring_initialized_sites.update(task_sites) if "mutate" not in task.tags: initialized_sites.update(task_sites) + task = webarena_environment.materialize_task_actions(base_task) prepare_budget_ms = _remaining_task_budget_ms( attempt_deadline, phase="environment prepare" ) diff --git a/harness/trajectory_replay/public_webarena/webarena_environment.py b/harness/trajectory_replay/public_webarena/webarena_environment.py index 9701ac20b..d32ffa293 100644 --- a/harness/trajectory_replay/public_webarena/webarena_environment.py +++ b/harness/trajectory_replay/public_webarena/webarena_environment.py @@ -22,6 +22,7 @@ from webarena_verified.environments.container.manager import ContainerManager from webarena_verified.types.task import WebArenaSite +from .rainbond_slices import RainbondSlicePool from .suite import BrowserTask STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) @@ -46,9 +47,8 @@ def wall_clock_timeout(seconds: float, *, operation: str = "environment reset"): """Bound one blocking environment phase, including Docker SDK calls, on POSIX.""" if seconds <= 0: raise ValueError("wall-clock timeout must be positive") - if ( - threading.current_thread() is not threading.main_thread() - or not hasattr(signal, "setitimer") + if threading.current_thread() is not threading.main_thread() or not hasattr( + signal, "setitimer" ): raise RuntimeError("WebArena reset wall-clock timeout requires the POSIX main thread") previous_timer = signal.getitimer(signal.ITIMER_REAL) @@ -132,9 +132,7 @@ def container_run( # services become healthy. The image default of 180 seconds is # too short on a busy qualification host and can leave the baked # CMU origin active even though env-ctrl /status is healthy. - environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str( - MAGENTO_INIT_TIMEOUT_SECONDS - ) + environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str(MAGENTO_INIT_TIMEOUT_SECONDS) # GitLab's image entrypoint does not run env-ctrl init. Its fallback # reconfiguration also rewrites the internal nginx listener to the # public host port, breaking Docker's host-port -> 8023 mapping. Keep @@ -151,8 +149,7 @@ def container_run( for host_port, container_port in port_mappings.items() }, volumes={ - source: {"bind": target, "mode": "rw"} - for source, target in volume_mappings.items() + source: {"bind": target, "mode": "rw"} for source, target in volume_mappings.items() }, environment=environment, ) @@ -195,6 +192,29 @@ def __init__(self, config_path: str | Path) -> None: if self.image_lock.get("schema") != "lex_webarena_environment_images/1": raise ValueError("WebArena image lock schema is invalid") self.remote_receipts = self.config.get("remote_runtime_receipts") or {} + slice_value = self.config.get("rainbond_slice") + self.rainbond_pool: RainbondSlicePool | None = None + self.rainbond_receipt_path: Path | None = None + if slice_value is not None: + if not isinstance(slice_value, dict): + raise ValueError("rainbond_slice must be an object") + pool_raw = str(slice_value.get("pool_config") or "").strip() + receipt_raw = str(slice_value.get("receipt") or "").strip() + if not pool_raw or not receipt_raw: + raise ValueError("rainbond_slice requires pool_config and receipt") + pool_value = Path(pool_raw) + receipt_value = Path(receipt_raw) + pool_path = ( + pool_value + if pool_value.is_absolute() + else (self.config_path.parent / pool_value).resolve() + ) + self.rainbond_receipt_path = ( + receipt_value + if receipt_value.is_absolute() + else (self.config_path.parent / receipt_value).resolve() + ) + self.rainbond_pool = RainbondSlicePool(pool_path) def site_docker_client(self, default_client: Any) -> tuple[Any, bool]: host = self.config.get("site_docker_host") @@ -216,9 +236,34 @@ def verify_runtime(self, docker_client: Any, tasks: tuple[BrowserTask, ...]) -> {str(site) for task in tasks for site in (task.environment or {}).get("sites") or []} ) rows: dict[str, Any] = {} + slice_receipt = ( + self.rainbond_pool.validate_receipt(self.rainbond_receipt_path) + if self.rainbond_pool is not None and self.rainbond_receipt_path is not None + else None + ) for site in sites: locked = self.image_lock["images"][site] expected = str(locked["pinned"]) + if slice_receipt is not None and site in STATEFUL_SITES: + component = (slice_receipt.get("components") or {}).get(site) or {} + if component.get("image") != expected: + raise RuntimeError(f"Rainbond slice image mismatch for {site}") + site_url = str(component.get("site_url") or "") + configured_urls = self._site_config(site).get("urls") or [] + if configured_urls != [site_url]: + raise RuntimeError(f"Rainbond slice URL mismatch for {site}") + self._wait_site_ready(site_url, timeout_seconds=60) + rows[site] = { + "mode": "remote_rainbond_isolated_slice", + "app_id": slice_receipt["app_id"], + "generation": slice_receipt["generation"], + "service_id": component["service_id"], + "pinned_image": expected, + "receipt_sha256": hashlib.sha256( + self.rainbond_receipt_path.read_bytes() + ).hexdigest(), + } + continue receipt_value = self.remote_receipts.get(site) if receipt_value is not None: if site in STATEFUL_SITES: @@ -413,9 +458,35 @@ def _reset_impl( self, task: BrowserTask, docker_client: Any, *, timeout_seconds: float ) -> dict[str, Any]: sites = list((task.environment or {}).get("sites") or []) + if self.rainbond_pool is not None and any(site in STATEFUL_SITES for site in sites): + assert self.rainbond_receipt_path is not None + receipt = self.rainbond_pool.reset(self.rainbond_receipt_path) + for site, component in receipt["components"].items(): + token = f"__{site.upper()}__" + self.environments[token] = {"urls": [component["site_url"]]} + for site in sites: + site_config = self._site_config(site) + site_urls = site_config.get("urls") or [] + index = int(site_config.get("active_url_idx") or 0) + if not site_urls or index < 0 or index >= len(site_urls): + raise ValueError(f"WebArena environment has no active URL for site {site!r}") + self._wait_site_ready(str(site_urls[index]), timeout_seconds=timeout_seconds) + return { + "mode": "remote_rainbond_isolated_slice", + "reset": "application_deleted_and_recreated_from_pinned_baseline", + "app_id": receipt["app_id"], + "generation": receipt["generation"], + "sites": { + site: { + "service_id": receipt["components"][site]["service_id"], + "site_url": receipt["components"][site]["site_url"], + } + for site in sites + if site in receipt["components"] + }, + } receipts: dict[str, Any] = {} for site in sites: - url = self._control_url(site) if site in self.remote_receipts: if site in STATEFUL_SITES: raise ValueError(f"remote reset is unsupported for stateful site {site}") @@ -430,6 +501,7 @@ def _reset_impl( "reset": "site_health_check_only", } continue + url = self._control_url(site) if site in STATEFUL_SITES: locked = self.image_lock["images"][site] upstream = get_container_config(site=WebArenaSite(site)) @@ -441,12 +513,8 @@ def _reset_impl( "host_env_ctrl_port": control_port, } ) - defer_removal = bool( - self.config.get("defer_stateful_container_removal", False) - ) - backend = _DockerClientBackend( - docker_client, defer_removal=defer_removal - ) + defer_removal = bool(self.config.get("defer_stateful_container_removal", False)) + backend = _DockerClientBackend(docker_client, defer_removal=defer_removal) manager = ContainerManager( site=WebArenaSite(site), config=config, @@ -490,9 +558,7 @@ def _reset_impl( # WA_ENV_CTRL_EXTERNAL_SITE_URL, so its supervisor attempt # exits before mutating state. Supply the URL explicitly # after that no-op attempt has exited. - self._wait_container_initialization( - container, timeout_seconds=timeout_seconds - ) + self._wait_container_initialization(container, timeout_seconds=timeout_seconds) self._configure_external_url(container, started.url, site=site) self._wait_ready(url, timeout_seconds=timeout_seconds) receipts[site]["external_url_configuration"] = { @@ -502,9 +568,7 @@ def _reset_impl( else: # Shopping and Shopping Admin run their one env-ctrl init # from the official image entrypoint. - self._wait_container_initialization( - container, timeout_seconds=timeout_seconds - ) + self._wait_container_initialization(container, timeout_seconds=timeout_seconds) self._verify_entrypoint_initialization(container) receipts[site]["external_url_configuration"] = { "external_url": started.url, @@ -655,9 +719,7 @@ def _wait_site_ready(url: str, *, timeout_seconds: float) -> None: time.sleep(0.5) raise RuntimeError(f"WebArena site did not become ready: {last_error}") - def prepare( - self, task: BrowserTask, page: Any, *, timeout_seconds: float = 300 - ) -> Any: + def prepare(self, task: BrowserTask, page: Any, *, timeout_seconds: float = 300) -> Any: with wall_clock_timeout(timeout_seconds, operation="environment prepare"): return self._prepare_impl(task, page) diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py new file mode 100644 index 000000000..2aa4197e5 --- /dev/null +++ b/scripts/manage_webarena_rainbond_pool.py @@ -0,0 +1,51 @@ +"""Create, reset, and destroy disposable Rainbond WebArena environment slices.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from harness.trajectory_replay.public_webarena.rainbond_slices import ( + RainbondSlicePool, + provision_pool, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, required=True) + subparsers = parser.add_subparsers(dest="command", required=True) + provision = subparsers.add_parser("provision") + provision.add_argument("--pool-id", required=True) + provision.add_argument("--count", type=int, required=True) + provision.add_argument("--output", type=Path, required=True) + reset = subparsers.add_parser("reset") + reset.add_argument("--slice-dir", type=Path, required=True) + destroy = subparsers.add_parser("destroy") + destroy.add_argument("--slice-dir", type=Path, action="append", required=True) + args = parser.parse_args() + + if args.command == "provision": + paths = provision_pool( + args.config.resolve(), args.output.resolve(), args.pool_id, args.count + ) + print(json.dumps([str(path / "webarena.config.json") for path in paths])) + return + + pool = RainbondSlicePool(args.config.resolve()) + if args.command == "reset": + slot_dir = args.slice_dir.resolve() + receipt = pool.reset(slot_dir / "slice-receipt.json") + config = pool.runtime_config(receipt) + target = slot_dir / "webarena.config.json" + target.write_text(json.dumps(config, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(target) + return + + for slot_dir in args.slice_dir: + pool.destroy(slot_dir.resolve() / "slice-receipt.json") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_webarena_qualification.py b/scripts/run_webarena_qualification.py index dbd30ff2d..4efcf0a89 100644 --- a/scripts/run_webarena_qualification.py +++ b/scripts/run_webarena_qualification.py @@ -62,11 +62,17 @@ def qualification_shards(scope_path: Path, suite_path: Path) -> tuple[dict[str, def qualification_waves( - shards: tuple[dict[str, Any], ...], scope: dict[str, Any], *, max_workers: int + shards: tuple[dict[str, Any], ...], + scope: dict[str, Any], + *, + max_workers: int, + isolated_instances: int = 1, ) -> tuple[tuple[tuple[dict[str, Any], ...], ...], ...]: """Group complete site batches into deterministic, conflict-free parallel waves.""" if max_workers < 1: raise ValueError("max_workers must be positive") + if isolated_instances < 1: + raise ValueError("isolated_instances must be positive") sites_by_batch = { str(batch["id"]): frozenset(str(site) for site in batch.get("sites") or []) for batch in scope.get("qualification_batches") or [] @@ -84,7 +90,9 @@ def qualification_waves( selected: list[str] = [] for batch_id in remaining: sites = sites_by_batch[batch_id] - if len(selected) < max_workers and not occupied.intersection(sites): + capacity = min(max_workers, isolated_instances) + isolated = isolated_instances > 1 + if len(selected) < capacity and (isolated or not occupied.intersection(sites)): selected.append(batch_id) occupied.update(sites) waves.append(tuple(tuple(shards_by_batch[batch_id]) for batch_id in selected)) @@ -108,10 +116,9 @@ def validate_completed_shard( raise ValueError(f"{shard['id']}: incomplete qualification output; missing {missing}") manifest = json.loads(required["manifest.json"].read_text(encoding="utf-8")) provenance = manifest.get("artifact_provenance") or {} - if ( - provenance.get("attempts_sha256") != _sha256(required["attempts.jsonl"]) - or provenance.get("summary_sha256") != _sha256(required["summary.json"]) - ): + if provenance.get("attempts_sha256") != _sha256(required["attempts.jsonl"]) or provenance.get( + "summary_sha256" + ) != _sha256(required["summary.json"]): raise ValueError(f"{shard['id']}: qualification artifact hash mismatch") protocol = manifest.get("protocol") or {} selection = protocol.get("task_selection") or {} @@ -212,11 +219,7 @@ def build_scoped_repair_queue( "architecture": scope.get("architecture"), "sites": row.get("sites") or [], "incompatible_sites": row.get("incompatible_sites") or [], - **( - {"kind": row["exclusion_kind"]} - if row.get("exclusion_kind") - else {} - ), + **({"kind": row["exclusion_kind"]} if row.get("exclusion_kind") else {}), **({"evidence_id": row["evidence_id"]} if row.get("evidence_id") else {}), }, }, @@ -246,7 +249,16 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--suite", type=Path, required=True) parser.add_argument("--native-scope", type=Path, required=True) - parser.add_argument("--webarena-config", type=Path, required=True) + parser.add_argument( + "--webarena-config", + type=Path, + action="append", + required=True, + help=( + "runtime config; repeat with independently provisioned Rainbond slice configs " + "to run overlapping site batches concurrently" + ), + ) parser.add_argument( "--env-file", type=Path, @@ -278,10 +290,13 @@ def main() -> None: args = parser.parse_args() suite_path = args.suite.resolve() scope_path = args.native_scope.resolve() - config_path = args.webarena_config.resolve() + config_paths = tuple(path.resolve() for path in args.webarena_config) evidence_root = args.evidence_root.resolve() - if not config_path.is_file(): - raise ValueError("WebArena runtime config is missing") + missing_configs = [str(path) for path in config_paths if not path.is_file()] + if missing_configs: + raise ValueError(f"WebArena runtime configs are missing: {missing_configs}") + if len(set(config_paths)) != len(config_paths): + raise ValueError("WebArena runtime configs must be distinct isolated instances") all_shards = qualification_shards(scope_path, suite_path) scope = json.loads(scope_path.read_text(encoding="utf-8")) selected_batches = set(args.batch_ids or []) @@ -296,14 +311,15 @@ def main() -> None: ) suite_sha256 = _sha256(suite_path) scope_sha256 = _sha256(scope_path) - waves = qualification_waves(selected_shards, scope, max_workers=args.max_site_workers) + waves = qualification_waves( + selected_shards, + scope, + max_workers=args.max_site_workers, + isolated_instances=len(config_paths), + ) - def run_shard(shard: dict[str, Any], qualification_pass_index: int) -> Path: - output = ( - evidence_root - / f"pass-{qualification_pass_index}" - / str(shard["id"]) - ) + def run_shard(shard: dict[str, Any], qualification_pass_index: int, config_path: Path) -> Path: + output = evidence_root / f"pass-{qualification_pass_index}" / str(shard["id"]) if output.exists(): try: validate_completed_shard( @@ -401,21 +417,30 @@ def run_shard(shard: dict[str, Any], qualification_pass_index: int) -> Path: def run_batch( batch: tuple[dict[str, Any], ...], + config_path: Path, qualification_pass_index: int = qualification_pass_index, ) -> list[tuple[tuple[int, str], Path]]: return [ ( (qualification_pass_index, str(shard["id"])), - run_shard(shard, qualification_pass_index), + run_shard(shard, qualification_pass_index, config_path), ) for shard in batch ] + assigned = tuple(zip(wave, config_paths, strict=False)) + if len(assigned) != len(wave): + raise RuntimeError("parallel wave exceeds isolated WebArena slice count") if args.dry_run or len(wave) == 1: - completed = [run_batch(batch) for batch in wave] + completed = [run_batch(batch, config) for batch, config in assigned] else: with ThreadPoolExecutor(max_workers=len(wave)) as executor: - completed = list(executor.map(run_batch, wave)) + completed = list( + executor.map( + lambda item: run_batch(item[0], item[1]), + assigned, + ) + ) for batch_rows in completed: attempts_by_id.update(batch_rows) attempts_paths = [ diff --git a/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py b/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py index 592fb7ec8..b2958ac98 100644 --- a/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py +++ b/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py @@ -33,9 +33,7 @@ def test_canonical_qualification_plan_is_an_exact_eighteen_shard_partition(): def test_parallel_qualification_waves_never_share_a_site(): - scope = json.loads( - Path("evaluation/browser-use-public/webarena-amd64-scope.json").read_text() - ) + scope = json.loads(Path("evaluation/browser-use-public/webarena-amd64-scope.json").read_text()) shards = qualification_shards( Path("evaluation/browser-use-public/webarena-amd64-scope.json"), Path("evaluation/browser-use-public/webarena-human-suite.json"), @@ -46,9 +44,9 @@ def test_parallel_qualification_waves_never_share_a_site(): assert len(waves) > 1 assert all(len(wave) <= 3 for wave in waves) - assert { - shard["id"] for wave in waves for batch in wave for shard in batch - } == {shard["id"] for shard in shards} + assert {shard["id"] for wave in waves for batch in wave for shard in batch} == { + shard["id"] for shard in shards + } for wave in waves: occupied = set() for batch in wave: @@ -62,6 +60,26 @@ def test_parallel_qualification_rejects_non_positive_worker_count(): qualification_waves((), {"qualification_batches": []}, max_workers=0) +def test_independent_slices_allow_overlapping_sites_in_the_same_wave(): + shards = ( + {"id": "a-0", "batch_id": "a"}, + {"id": "b-0", "batch_id": "b"}, + ) + scope = { + "qualification_batches": [ + {"id": "a", "sites": ["reddit"]}, + {"id": "b", "sites": ["reddit"]}, + ] + } + + shared = qualification_waves(shards, scope, max_workers=2) + isolated = qualification_waves(shards, scope, max_workers=2, isolated_instances=2) + + assert len(shared) == 2 + assert len(isolated) == 1 + assert [batch[0]["id"] for batch in isolated[0]] == ["a-0", "b-0"] + + def test_completed_shard_validation_binds_manifest_hashes_scope_and_attempts(tmp_path): output = tmp_path / "shopping-0-of-1" output.mkdir() @@ -316,9 +334,7 @@ def test_scoped_queue_covers_measured_and_architecture_excluded_tasks(tmp_path): ], } - queue = build_scoped_repair_queue( - suite, [attempts], scope, scope_sha256="f" * 64 - ) + queue = build_scoped_repair_queue(suite, [attempts], scope, scope_sha256="f" * 64) assert [row["task_id"] for row in queue["tasks"]] == ["included", "excluded"] assert queue["tasks"][1]["adjudication"] == { diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py new file mode 100644 index 000000000..1306650a3 --- /dev/null +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from harness.trajectory_replay.public_webarena.rainbond_slices import ( + RECEIPT_SCHEMA, + STATEFUL_SITES, + RainbondSlicePool, + provision_pool, +) + +PINNED = { + site: f"example/{site}@sha256:{index:064x}" + for index, site in enumerate(STATEFUL_SITES, start=1) +} + + +class FakeAPI: + def __init__(self, *, fail_domain_for: str | None = None) -> None: + self.calls = [] + self.next_app_id = 200 + self.fail_domain_for = fail_domain_for + + def request(self, method, path, body=None): + self.calls.append((method, path, body)) + if method == "POST" and path.endswith("/apps"): + app_id = self.next_app_id + self.next_app_id += 1 + return {"ID": app_id} + if method == "POST" and path.endswith("/192/copy"): + return { + "services": [ + { + "service_id": f"service-{site}-{self.next_app_id}", + "service_cname": site, + "image": PINNED[site], + } + for site in STATEFUL_SITES + ] + } + if method == "POST" and path.endswith("/domains"): + service_id = body["http"]["service_id"] + if self.fail_domain_for and self.fail_domain_for in service_id: + raise RuntimeError("injected domain failure") + return {} + + +def _pool_config(tmp_path: Path) -> Path: + lock = tmp_path / "images.json" + lock.write_text( + json.dumps( + { + "schema": "lex_webarena_environment_images/1", + "images": {site: {"pinned": image} for site, image in PINNED.items()}, + } + ), + encoding="utf-8", + ) + config = tmp_path / "pool.json" + config.write_text( + json.dumps( + { + "schema": "lex_webarena_rainbond_pool/1", + "api_base_url": "https://rainbond.example", + "team_id": "team-id", + "team_name": "efficiency", + "region_name": "rainbond", + "source_app_id": 192, + "source_services": {site: f"source-{site}" for site in STATEFUL_SITES}, + "app_prefix": "webarena-slice", + "domain_suffix": "dev.example.test", + "token_env": "TEST_RAINBOND_TOKEN", + "image_lock": "images.json", + "readiness_timeout_seconds": 1, + } + ), + encoding="utf-8", + ) + return config + + +def _pool(tmp_path: Path, api: FakeAPI) -> RainbondSlicePool: + pool = RainbondSlicePool(_pool_config(tmp_path), api=api, sleep=lambda _: None) + pool._wait_site = lambda _url: None + return pool + + +def test_provision_creates_one_complete_isolated_app_without_serializing_token(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-1" / "slice-receipt.json" + + receipt = pool.provision("run-a", 1, receipt_path) + + assert receipt["schema"] == RECEIPT_SCHEMA + assert set(receipt["components"]) == set(STATEFUL_SITES) + assert len({row["site_url"] for row in receipt["components"].values()}) == 4 + assert all("run-a-1" in row["site_url"] for row in receipt["components"].values()) + assert "TEST_RAINBOND_TOKEN" not in receipt_path.read_text() + assert not any("secret-token" in json.dumps(call) for call in api.calls) + deploy = next(call for call in api.calls if call[1].endswith("/operations")) + assert set(deploy[2]["service_ids"]) == { + row["service_id"] for row in receipt["components"].values() + } + domains = [call[2]["http"] for call in api.calls if call[1].endswith("/domains")] + admin = next(row for row in domains if "shopping-admin" in row["domain_name"]) + reddit = next(row for row in domains if "reddit" in row["domain_name"]) + assert admin["configuration"]["proxy_buffer_size"] == 64 + assert admin["configuration"]["proxy_buffer_numbers"] == 8 + assert reddit["configuration"]["proxy_buffer_size"] == 4 + + +def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-b", 2, receipt_path) + + second = pool.reset(receipt_path) + + assert second["app_id"] != first["app_id"] + assert second["generation"] != first["generation"] + assert {row["site_url"] for row in second["components"].values()}.isdisjoint( + {row["site_url"] for row in first["components"].values()} + ) + assert all( + f"-{second['app_id']}-" in row["site_url"] + for row in second["components"].values() + ) + assert any( + method == "DELETE" and f"/{first['app_id']}?force=1" in path + for method, path, _body in api.calls + ) + + +def test_partial_provision_failure_removes_the_new_app_and_leaves_no_receipt(tmp_path): + api = FakeAPI(fail_domain_for="reddit") + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-receipt.json" + + with pytest.raises(RuntimeError, match="injected"): + pool.provision("run-c", 1, receipt_path) + + assert not receipt_path.exists() + assert any(method == "DELETE" and "/200?force=1" in path for method, path, _ in api.calls) + + +def test_provision_pool_rolls_back_already_created_slots(tmp_path, monkeypatch): + destroyed = [] + + class Pool: + def __init__(self, _config): + pass + + def provision(self, _pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("second slot failed") + receipt_path.parent.mkdir(parents=True) + receipt_path.write_text("{}") + return {"slot": slot} + + def runtime_config(self, _receipt): + return {"ok": True} + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + + with pytest.raises(RuntimeError, match="second slot"): + provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) + + assert destroyed == [tmp_path / "out" / "slice-1" / "slice-receipt.json"] diff --git a/tests/trajectory_replay/public_webarena/test_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_webarena_environment.py index e5cc510df..24328cd07 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_environment.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_environment.py @@ -130,6 +130,103 @@ def test_runtime_verification_rejects_mutable_or_wrong_image(tmp_path): environment.verify_runtime(_Docker([]), (_task(),)) +def test_rainbond_slice_verification_and_reset_replace_the_complete_application( + tmp_path, monkeypatch +): + pinned = "am1n3e/webarena-verified-shopping_admin@sha256:" + "d" * 64 + lock = tmp_path / "images.json" + lock.write_text( + json.dumps( + { + "schema": "lex_webarena_environment_images/1", + "images": {"shopping_admin": {"pinned": pinned}}, + } + ), + encoding="utf-8", + ) + receipt_path = tmp_path / "slice-receipt.json" + first = { + "app_id": 301, + "generation": "first", + "components": { + "shopping_admin": { + "service_id": "old-service", + "image": pinned, + "site_url": "https://admin-slice.example.test", + } + }, + } + receipt_path.write_text(json.dumps(first), encoding="utf-8") + resets = [] + + class Pool: + def __init__(self, path): + assert path == tmp_path / "pool.json" + + def validate_receipt(self, path): + assert path == receipt_path + return json.loads(path.read_text()) + + def reset(self, path): + resets.append(path) + return { + "app_id": 302, + "generation": "second", + "components": { + "shopping_admin": { + "service_id": "new-service", + "image": pinned, + "site_url": "https://admin-slice-302.example.test", + } + }, + } + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.webarena_environment.RainbondSlicePool", + Pool, + ) + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "image_lock": "images.json", + "environments": { + "__SHOPPING_ADMIN__": {"urls": ["https://admin-slice.example.test"]} + }, + "rainbond_slice": { + "pool_config": "pool.json", + "receipt": "slice-receipt.json", + }, + } + ), + encoding="utf-8", + ) + environment = WebArenaEnvironment(config) + monkeypatch.setattr(environment, "_wait_site_ready", lambda *_args, **_kwargs: None) + + runtime = environment.verify_runtime(object(), (_task(),)) + reset = environment.reset(_task(), object()) + + assert runtime["sites"]["shopping_admin"]["mode"] == ("remote_rainbond_isolated_slice") + assert runtime["sites"]["shopping_admin"]["app_id"] == 301 + assert reset == { + "mode": "remote_rainbond_isolated_slice", + "reset": "application_deleted_and_recreated_from_pinned_baseline", + "app_id": 302, + "generation": "second", + "sites": { + "shopping_admin": { + "service_id": "new-service", + "site_url": "https://admin-slice-302.example.test", + } + }, + } + assert environment.environments["__SHOPPING_ADMIN__"]["urls"] == [ + "https://admin-slice-302.example.test" + ] + assert resets == [receipt_path] + + def test_site_docker_client_defaults_or_uses_validated_unix_socket(tmp_path, monkeypatch): environment, _ = _environment(tmp_path) default_client = object() From 78861b17bcd24c9c309fb7ebdebdedf0c2b13420 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 12:03:41 +0800 Subject: [PATCH 02/23] Add warm-pool scaling and internal site initialization --- deploy/webarena-rainbond/gitlab/Dockerfile | 7 - .../gitlab/entrypoint-wrapper.sh | 53 ---- deploy/webarena-rainbond/initializer.sh | 31 +++ deploy/webarena-rainbond/reddit/Dockerfile | 8 - .../reddit/entrypoint-wrapper.sh | 53 ---- .../shopping-admin/Dockerfile | 1 - deploy/webarena-rainbond/shopping/Dockerfile | 1 - .../public_webarena/README.md | 22 +- .../rainbond-pool.example.json | 4 + .../public_webarena/rainbond_slices.py | 231 +++++++++++++++--- scripts/manage_webarena_rainbond_pool.py | 17 ++ .../public_webarena/test_rainbond_slices.py | 82 ++++++- 12 files changed, 355 insertions(+), 155 deletions(-) delete mode 100644 deploy/webarena-rainbond/gitlab/Dockerfile delete mode 100755 deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh create mode 100755 deploy/webarena-rainbond/initializer.sh delete mode 100644 deploy/webarena-rainbond/reddit/Dockerfile delete mode 100755 deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh delete mode 100644 deploy/webarena-rainbond/shopping-admin/Dockerfile delete mode 100644 deploy/webarena-rainbond/shopping/Dockerfile diff --git a/deploy/webarena-rainbond/gitlab/Dockerfile b/deploy/webarena-rainbond/gitlab/Dockerfile deleted file mode 100644 index 12fde38a4..000000000 --- a/deploy/webarena-rainbond/gitlab/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM am1n3e/webarena-verified-gitlab@sha256:673ba94d757c5b3f4bb55751ad8b745a33bad2b987349322403cbc41035a92e0 - -COPY entrypoint-wrapper.sh /usr/local/bin/webarena-rainbond-entrypoint -RUN chmod 0755 /usr/local/bin/webarena-rainbond-entrypoint - -ENV WA_ORIGINAL_ENTRYPOINT=/opt/entrypoint.sh -ENTRYPOINT ["/usr/local/bin/webarena-rainbond-entrypoint"] diff --git a/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh b/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh deleted file mode 100755 index 1d389af5f..000000000 --- a/deploy/webarena-rainbond/gitlab/entrypoint-wrapper.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -set -eu - -case "${WA_ENV_CTRL_EXTERNAL_SITE_URL:-}" in - http://*|https://*) ;; - *) - echo "WA_ENV_CTRL_EXTERNAL_SITE_URL must be an absolute HTTP URL" >&2 - exit 64 - ;; -esac - -"${WA_ORIGINAL_ENTRYPOINT}" "$@" & -child_pid=$! - -terminate() { - kill -TERM "${child_pid}" 2>/dev/null || true - wait "${child_pid}" || true -} -trap terminate INT TERM - -control_url="http://127.0.0.1:${WA_ENV_CTRL_PORT:-8877}/status" -attempt=0 -until curl --fail --silent --show-error "${control_url}" >/dev/null 2>&1; do - if ! kill -0 "${child_pid}" 2>/dev/null; then - wait "${child_pid}" - exit $? - fi - attempt=$((attempt + 1)) - if [ "${attempt}" -ge 360 ]; then - echo "env-ctrl did not become ready within 30 minutes" >&2 - terminate - exit 1 - fi - sleep 5 -done - -attempt=0 -until env-ctrl --verbose init --base-url "${WA_ENV_CTRL_EXTERNAL_SITE_URL}"; do - if ! kill -0 "${child_pid}" 2>/dev/null; then - wait "${child_pid}" - exit $? - fi - attempt=$((attempt + 1)) - if [ "${attempt}" -ge 60 ]; then - echo "env-ctrl init failed after 60 attempts" >&2 - terminate - exit 1 - fi - sleep 5 -done - -touch /tmp/rainbond-env-ctrl-init-ok -wait "${child_pid}" diff --git a/deploy/webarena-rainbond/initializer.sh b/deploy/webarena-rainbond/initializer.sh new file mode 100755 index 000000000..b637c27cc --- /dev/null +++ b/deploy/webarena-rainbond/initializer.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +wait_ctrl() { + url="$1" + attempts=0 + until wget -qO- "$url/status" >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge "${ENV_CTRL_MAX_ATTEMPTS:-360}" ]; then + echo "env-ctrl timeout: $url" >&2 + exit 1 + fi + sleep "${ENV_CTRL_RETRY_SECONDS:-5}" + done +} + +init_ctrl() { + url="$1" + response="$(wget -qO- --post-data='' "$url/init")" + echo "$response" + echo "$response" | grep -Eq '"success"[[:space:]]*:[[:space:]]*true' +} + +wait_ctrl "$REDDIT_CTRL_URL" +wait_ctrl "$GITLAB_CTRL_URL" +init_ctrl "$REDDIT_CTRL_URL" +init_ctrl "$GITLAB_CTRL_URL" + +mkdir -p /www +printf 'ready\n' >/www/index.html +exec httpd -f -p 8080 -h /www diff --git a/deploy/webarena-rainbond/reddit/Dockerfile b/deploy/webarena-rainbond/reddit/Dockerfile deleted file mode 100644 index b8e50f52f..000000000 --- a/deploy/webarena-rainbond/reddit/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM am1n3e/webarena-verified-reddit@sha256:0594908059a03e5f610005689440a95c05e776e734006197c7b1eba12734cb46 - -COPY entrypoint-wrapper.sh /usr/local/bin/webarena-rainbond-entrypoint -RUN chmod 0755 /usr/local/bin/webarena-rainbond-entrypoint - -ENV WA_ORIGINAL_ENTRYPOINT=/docker-entrypoint.sh -ENTRYPOINT ["/usr/local/bin/webarena-rainbond-entrypoint"] -CMD ["supervisord", "-n", "-j", "/supervisord.pid"] diff --git a/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh b/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh deleted file mode 100755 index 1d389af5f..000000000 --- a/deploy/webarena-rainbond/reddit/entrypoint-wrapper.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -set -eu - -case "${WA_ENV_CTRL_EXTERNAL_SITE_URL:-}" in - http://*|https://*) ;; - *) - echo "WA_ENV_CTRL_EXTERNAL_SITE_URL must be an absolute HTTP URL" >&2 - exit 64 - ;; -esac - -"${WA_ORIGINAL_ENTRYPOINT}" "$@" & -child_pid=$! - -terminate() { - kill -TERM "${child_pid}" 2>/dev/null || true - wait "${child_pid}" || true -} -trap terminate INT TERM - -control_url="http://127.0.0.1:${WA_ENV_CTRL_PORT:-8877}/status" -attempt=0 -until curl --fail --silent --show-error "${control_url}" >/dev/null 2>&1; do - if ! kill -0 "${child_pid}" 2>/dev/null; then - wait "${child_pid}" - exit $? - fi - attempt=$((attempt + 1)) - if [ "${attempt}" -ge 360 ]; then - echo "env-ctrl did not become ready within 30 minutes" >&2 - terminate - exit 1 - fi - sleep 5 -done - -attempt=0 -until env-ctrl --verbose init --base-url "${WA_ENV_CTRL_EXTERNAL_SITE_URL}"; do - if ! kill -0 "${child_pid}" 2>/dev/null; then - wait "${child_pid}" - exit $? - fi - attempt=$((attempt + 1)) - if [ "${attempt}" -ge 60 ]; then - echo "env-ctrl init failed after 60 attempts" >&2 - terminate - exit 1 - fi - sleep 5 -done - -touch /tmp/rainbond-env-ctrl-init-ok -wait "${child_pid}" diff --git a/deploy/webarena-rainbond/shopping-admin/Dockerfile b/deploy/webarena-rainbond/shopping-admin/Dockerfile deleted file mode 100644 index bf91f7434..000000000 --- a/deploy/webarena-rainbond/shopping-admin/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM am1n3e/webarena-verified-shopping_admin@sha256:d0531dd27ed98d0c459ff9e88118bf2ed8b660b0ed99c38837db46c065a5be13 diff --git a/deploy/webarena-rainbond/shopping/Dockerfile b/deploy/webarena-rainbond/shopping/Dockerfile deleted file mode 100644 index dfe0030bd..000000000 --- a/deploy/webarena-rainbond/shopping/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM am1n3e/webarena-verified-shopping@sha256:3e8cb9b945ea9b1c94ab26dba53e8d12dd0406abbf4bf686fd3bb2b6a5908feb diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index be3799ef5..583d6a3c1 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -41,8 +41,24 @@ Shopping, Reddit, and GitLab with independent writable container layers and uniq domains. Reset deletes and recreates the whole slice from the pinned baseline; destroy removes the application. The read-only Map service may be shared. +The baseline also contains a small digest-pinned initializer component. It calls the +Reddit and GitLab `env-ctrl` endpoints only over Rainbond's internal service network, +then exposes a harmless readiness page. The unauthenticated control endpoints are never +published through the gateway. Its command is the base64-encoded content of +`deploy/webarena-rainbond/initializer.sh`, executed by the pinned Alpine image; port +8080 is internal-only and is the component readiness probe. + Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, -and provision the required concurrency: +and reconcile the warm pool to the required concurrency. Keep at least one ready slice; +additional slices are created in bounded parallel and removed when demand falls: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config scale --pool-id --count \ + --minimum-ready 1 --output +``` + +For a one-shot pool that will be destroyed in full, use: ```text uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ @@ -58,3 +74,7 @@ uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ --config destroy --slice-dir /slice-1 \ --slice-dir /slice-2 ``` + +Each receipt records create, copy, configure, deploy-request, readiness, and total cold-start +durations. Use those measurements to size the warm pool; task admission should allocate a +ready slice rather than waiting for a cold start. diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json index 61cd7823b..f184cdee2 100644 --- a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -11,6 +11,9 @@ "reddit": "replace-with-baseline-service-id", "gitlab": "replace-with-baseline-service-id" }, + "initializer_service": "replace-with-slice-initializer-service-id", + "initializer_image": "alpine@sha256:replace-with-pinned-multiarch-digest", + "initializer_component_name": "slice-initializer", "source_component_names": { "shopping_admin": "shopping-admin", "shopping": "shopping", @@ -22,6 +25,7 @@ "token_env": "RAINBOND_API_TOKEN", "image_lock": "../../../../evaluation/browser-use-public/webarena-environment-images.json", "readiness_timeout_seconds": 1800, + "max_parallel_provisions": 2, "shared_environments": { "__MAP__": { "urls": ["https://replace-with-shared-read-only-map"] diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index fe260abee..83d7716df 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -2,6 +2,7 @@ from __future__ import annotations +import fcntl import hashlib import json import os @@ -10,6 +11,8 @@ import urllib.error import urllib.request import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any @@ -34,6 +37,13 @@ def _image_digest(value: str) -> str: return "sha256:" + value.rsplit(marker, 1)[1] +def _validate_pool_id(pool_id: str) -> str: + normalized = pool_id.strip().lower() + if not _SAFE_ID.fullmatch(normalized): + raise ValueError("pool_id must be a lowercase DNS-safe identifier") + return normalized + + def _atomic_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) content = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" @@ -83,13 +93,15 @@ class PoolConfig: region_name: str source_app_id: int source_services: dict[str, str] + initializer_service: str + initializer_image: str app_prefix: str domain_suffix: str token_env: str image_lock: Path - source_runtime_images: dict[str, str] shared_environments: dict[str, Any] readiness_timeout_seconds: int + max_parallel_provisions: int @classmethod def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: @@ -115,14 +127,11 @@ def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: lock_path = lock_value if lock_value.is_absolute() else (config_path.parent / lock_value) if not lock_path.is_file(): raise ValueError("Rainbond pool image_lock is missing") - runtime_images_value = raw.get("source_runtime_images") or {} - if not isinstance(runtime_images_value, dict) or set(runtime_images_value) - set( - STATEFUL_SITES - ): - raise ValueError("source_runtime_images contains an unknown site") - runtime_images = {str(site): str(image) for site, image in runtime_images_value.items()} - for image in runtime_images.values(): - _image_digest(image) + initializer_service = str(raw.get("initializer_service") or "") + initializer_image = str(raw.get("initializer_image") or "") + if not initializer_service or not initializer_image: + raise ValueError("initializer_service and initializer_image are required") + _image_digest(initializer_image) value = cls( api_base_url=str(raw.get("api_base_url") or "").rstrip("/"), team_id=str(raw.get("team_id") or ""), @@ -130,17 +139,20 @@ def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: region_name=str(raw.get("region_name") or ""), source_app_id=int(raw.get("source_app_id") or 0), source_services=normalized_services, + initializer_service=initializer_service, + initializer_image=initializer_image, app_prefix=prefix, domain_suffix=suffix, token_env=str(raw.get("token_env") or "RAINBOND_API_TOKEN"), image_lock=lock_path.resolve(), - source_runtime_images=runtime_images, shared_environments=dict(raw.get("shared_environments") or {}), readiness_timeout_seconds=int(raw.get("readiness_timeout_seconds") or 1800), + max_parallel_provisions=int(raw.get("max_parallel_provisions") or 2), ) if ( not all((value.api_base_url, value.team_id, value.team_name, value.region_name)) or value.source_app_id < 1 + or value.max_parallel_provisions < 1 ): raise ValueError("Rainbond pool API, team, region, and source app are required") return value, raw, config_path @@ -184,10 +196,7 @@ def _domain(self, pool_id: str, slot: int, site: str, app_id: int) -> str: @staticmethod def _validate_pool_id(pool_id: str) -> str: - normalized = pool_id.strip().lower() - if not _SAFE_ID.fullmatch(normalized): - raise ValueError("pool_id must be a lowercase DNS-safe identifier") - return normalized + return _validate_pool_id(pool_id) def _create_app(self, pool_id: str, slot: int) -> int: name = f"{self.config.app_prefix}-{pool_id}-{slot}" @@ -198,7 +207,7 @@ def _create_app(self, pool_id: str, slot: int) -> int: raise RuntimeError("Rainbond create app response has no ID") return app_id - def _copy_components(self, app_id: int) -> dict[str, dict[str, Any]]: + def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]: config = self.config result = self.api.request( "POST", @@ -206,7 +215,8 @@ def _copy_components(self, app_id: int) -> dict[str, dict[str, Any]]: { "services": [ {"service_id": config.source_services[site]} for site in STATEFUL_SITES - ], + ] + + [{"service_id": config.initializer_service}], "target_team_name": config.team_name, "target_region_name": config.region_name, "target_app_id": app_id, @@ -225,7 +235,18 @@ def _copy_components(self, app_id: int) -> dict[str, dict[str, Any]]: f"Rainbond copy returned {len(matches)} components named {component_name!r}" ) mapped[site] = matches[0] - return mapped + initializer_name = str( + self.raw_config.get("initializer_component_name") or "slice-initializer" + ) + initializer_matches = [ + row for row in rows if str(row.get("service_cname")) == initializer_name + ] + if len(initializer_matches) != 1: + raise RuntimeError( + f"Rainbond copy returned {len(initializer_matches)} components named " + f"{initializer_name!r}" + ) + return mapped, initializer_matches[0] def _configure_component( self, @@ -308,12 +329,10 @@ def _configure_component( {"envs": envs}, ) expected_image = str(self.image_lock["images"][site]["pinned"]) - expected_runtime_image = self.config.source_runtime_images.get(site, expected_image) actual_image = str(component.get("image") or "") - if actual_image and _image_digest(actual_image) != _image_digest(expected_runtime_image): + if actual_image and _image_digest(actual_image) != _image_digest(expected_image): raise RuntimeError( - f"Rainbond copied {site} runtime image mismatch: " - f"{actual_image!r} != {expected_runtime_image!r}" + f"Rainbond copied {site} image mismatch: {actual_image!r} != {expected_image!r}" ) return { "service_id": service_id, @@ -323,6 +342,62 @@ def _configure_component( "site_url": site_url, } + def _configure_initializer( + self, + app_id: int, + *, + initializer: dict[str, Any], + copied_sites: dict[str, dict[str, Any]], + ) -> dict[str, Any]: + service_id = str(initializer.get("service_id") or "") + actual_image = str(initializer.get("image") or "") + if not service_id: + raise RuntimeError("Rainbond copied initializer has no service_id") + if actual_image and _image_digest(actual_image) != _image_digest( + self.config.initializer_image + ): + raise RuntimeError("Rainbond copied initializer image mismatch") + aliases = { + site: str(copied_sites[site].get("service_alias") or "") + for site in ("reddit", "gitlab") + } + if not all(aliases.values()): + raise RuntimeError("Rainbond copy response has no internal service aliases") + self.api.request( + "PUT", + f"{self._app_path(app_id)}/services/{quote(service_id)}/envs", + { + "envs": [ + { + "name": f"{site.upper()}_CTRL_URL", + "value": f"http://{alias}:8877", + "is_change": True, + "scope": "inner", + } + for site, alias in aliases.items() + ] + }, + ) + return {"service_id": service_id} + + def _wait_component_running(self, app_id: int, service_id: str) -> None: + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_status = "not found" + while time.monotonic() < deadline: + response = self.api.request("GET", f"{self._app_path(app_id)}/services") + rows = response if isinstance(response, list) else (response or {}).get("services", []) + match = next( + (row for row in rows if str(row.get("service_id")) == service_id), None + ) + if match is not None: + last_status = str(match.get("status") or "unknown") + if last_status == "running": + return + self.sleep(5) + raise TimeoutError( + f"Rainbond initializer did not become ready: {service_id}: {last_status}" + ) + def _wait_site(self, url: str) -> None: deadline = time.monotonic() + self.config.readiness_timeout_seconds last_error = "not checked" @@ -339,12 +414,19 @@ def _wait_site(self, url: str) -> None: raise TimeoutError(f"Rainbond WebArena site did not become ready: {url}: {last_error}") def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: + started = time.monotonic() + phase_started = started + timings: dict[str, float] = {} pool_id = self._validate_pool_id(pool_id) if slot < 1: raise ValueError("slot must be positive") app_id = self._create_app(pool_id, slot) + timings["create_app_seconds"] = round(time.monotonic() - phase_started, 3) try: - copied = self._copy_components(app_id) + phase_started = time.monotonic() + copied, copied_initializer = self._copy_components(app_id) + timings["copy_components_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() components = { site: self._configure_component( app_id, @@ -355,16 +437,28 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An ) for site in STATEFUL_SITES } + initializer = self._configure_initializer( + app_id, + initializer=copied_initializer, + copied_sites=copied, + ) + timings["configure_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() self.api.request( "POST", f"{self._app_path(app_id)}/operations", { "action": "deploy", - "service_ids": [row["service_id"] for row in components.values()], + "service_ids": [row["service_id"] for row in components.values()] + + [initializer["service_id"]], }, ) + timings["deploy_request_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() + self._wait_component_running(app_id, str(initializer["service_id"])) for component in components.values(): self._wait_site(str(component["site_url"])) + timings["readiness_seconds"] = round(time.monotonic() - phase_started, 3) except BaseException: self.api.request("DELETE", f"{self._app_path(app_id)}?force=1") raise @@ -376,6 +470,8 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An "generation": uuid.uuid4().hex, "app_id": app_id, "components": components, + "initializer": initializer, + "timings": {**timings, "total_seconds": round(time.monotonic() - started, 3)}, "created_at_unix": int(time.time()), } _atomic_json(receipt_path, receipt) @@ -427,15 +523,90 @@ def provision_pool(config_path: Path, output_dir: Path, pool_id: str, count: int raise ValueError("count must be positive") pool = RainbondSlicePool(config_path) created: list[Path] = [] + + def create_slot(slot: int) -> Path: + slot_dir = output_dir / f"slice-{slot}" + receipt_path = slot_dir / "slice-receipt.json" + if receipt_path.exists(): + raise FileExistsError(f"Rainbond slice already exists: {receipt_path}") + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + return slot_dir + try: - for slot in range(1, count + 1): - slot_dir = output_dir / f"slice-{slot}" - receipt_path = slot_dir / "slice-receipt.json" - receipt = pool.provision(pool_id, slot, receipt_path) - _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) - created.append(slot_dir) + workers = min(count, getattr(getattr(pool, "config", None), "max_parallel_provisions", 2)) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = {executor.submit(create_slot, slot): slot for slot in range(1, count + 1)} + first_error: BaseException | None = None + for future in as_completed(futures): + try: + created.append(future.result()) + except BaseException as exc: + first_error = first_error or exc + if first_error is not None: + raise first_error except BaseException: for slot_dir in reversed(created): pool.destroy(slot_dir / "slice-receipt.json") raise - return created + return sorted(created) + + +@contextmanager +def _pool_lock(output_dir: Path) -> Any: + output_dir.mkdir(parents=True, exist_ok=True) + lock_path = output_dir / ".pool.lock" + with lock_path.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def scale_pool( + config_path: Path, + output_dir: Path, + pool_id: str, + count: int, + *, + minimum_ready: int = 1, +) -> list[Path]: + """Reconcile a persistent warm pool, never scaling below minimum_ready.""" + if minimum_ready < 1 or count < minimum_ready: + raise ValueError("count must be at least minimum_ready, and minimum_ready must be positive") + with _pool_lock(output_dir): + pool = RainbondSlicePool(config_path) + pool_id = _validate_pool_id(pool_id) + existing: dict[int, Path] = {} + for receipt_path in output_dir.glob("slice-*/slice-receipt.json"): + receipt = pool.validate_receipt(receipt_path) + if str(receipt.get("pool_id")) != pool_id: + raise ValueError(f"slice belongs to another pool: {receipt_path}") + existing[int(receipt["slot"])] = receipt_path.parent + missing_slots = [slot for slot in range(1, count + 1) if slot not in existing] + if missing_slots: + workers = min(len(missing_slots), pool.config.max_parallel_provisions) + + def create_slot(slot: int) -> tuple[int, Path]: + slot_dir = output_dir / f"slice-{slot}" + receipt = pool.provision(pool_id, slot, slot_dir / "slice-receipt.json") + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + return slot, slot_dir + + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(create_slot, slot) for slot in missing_slots] + for future in as_completed(futures): + slot, slot_dir = future.result() + existing[slot] = slot_dir + for slot in sorted((value for value in existing if value > count), reverse=True): + slot_dir = existing.pop(slot) + pool.destroy(slot_dir / "slice-receipt.json") + config_file = slot_dir / "webarena.config.json" + if config_file.exists(): + config_file.unlink() + try: + slot_dir.rmdir() + except OSError: + pass + return [existing[slot] for slot in sorted(existing)] diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py index 2aa4197e5..59812c753 100644 --- a/scripts/manage_webarena_rainbond_pool.py +++ b/scripts/manage_webarena_rainbond_pool.py @@ -9,6 +9,7 @@ from harness.trajectory_replay.public_webarena.rainbond_slices import ( RainbondSlicePool, provision_pool, + scale_pool, ) @@ -20,6 +21,11 @@ def main() -> None: provision.add_argument("--pool-id", required=True) provision.add_argument("--count", type=int, required=True) provision.add_argument("--output", type=Path, required=True) + scale = subparsers.add_parser("scale") + scale.add_argument("--pool-id", required=True) + scale.add_argument("--count", type=int, required=True) + scale.add_argument("--minimum-ready", type=int, default=1) + scale.add_argument("--output", type=Path, required=True) reset = subparsers.add_parser("reset") reset.add_argument("--slice-dir", type=Path, required=True) destroy = subparsers.add_parser("destroy") @@ -33,6 +39,17 @@ def main() -> None: print(json.dumps([str(path / "webarena.config.json") for path in paths])) return + if args.command == "scale": + paths = scale_pool( + args.config.resolve(), + args.output.resolve(), + args.pool_id, + args.count, + minimum_ready=args.minimum_ready, + ) + print(json.dumps([str(path / "webarena.config.json") for path in paths])) + return + pool = RainbondSlicePool(args.config.resolve()) if args.command == "reset": slot_dir = args.slice_dir.resolve() diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 1306650a3..ed9969393 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -10,12 +10,14 @@ STATEFUL_SITES, RainbondSlicePool, provision_pool, + scale_pool, ) PINNED = { site: f"example/{site}@sha256:{index:064x}" for index, site in enumerate(STATEFUL_SITES, start=1) } +INITIALIZER_IMAGE = "alpine@sha256:" + "a" * 64 class FakeAPI: @@ -35,16 +37,32 @@ def request(self, method, path, body=None): "services": [ { "service_id": f"service-{site}-{self.next_app_id}", + "service_alias": f"alias-{site}-{self.next_app_id}", "service_cname": site, "image": PINNED[site], } for site in STATEFUL_SITES ] + + [ + { + "service_id": f"service-initializer-{self.next_app_id}", + "service_alias": f"alias-initializer-{self.next_app_id}", + "service_cname": "slice-initializer", + "image": INITIALIZER_IMAGE, + } + ] } if method == "POST" and path.endswith("/domains"): service_id = body["http"]["service_id"] if self.fail_domain_for and self.fail_domain_for in service_id: raise RuntimeError("injected domain failure") + if method == "GET" and path.endswith("/services"): + return [ + { + "service_id": f"service-initializer-{self.next_app_id}", + "status": "running", + } + ] return {} @@ -70,6 +88,8 @@ def _pool_config(tmp_path: Path) -> Path: "region_name": "rainbond", "source_app_id": 192, "source_services": {site: f"source-{site}" for site in STATEFUL_SITES}, + "initializer_service": "source-initializer", + "initializer_image": INITIALIZER_IMAGE, "app_prefix": "webarena-slice", "domain_suffix": "dev.example.test", "token_env": "TEST_RAINBOND_TOKEN", @@ -98,19 +118,29 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t assert receipt["schema"] == RECEIPT_SCHEMA assert set(receipt["components"]) == set(STATEFUL_SITES) assert len({row["site_url"] for row in receipt["components"].values()}) == 4 + assert receipt["initializer"]["service_id"] == "service-initializer-201" assert all("run-a-1" in row["site_url"] for row in receipt["components"].values()) assert "TEST_RAINBOND_TOKEN" not in receipt_path.read_text() assert not any("secret-token" in json.dumps(call) for call in api.calls) deploy = next(call for call in api.calls if call[1].endswith("/operations")) assert set(deploy[2]["service_ids"]) == { row["service_id"] for row in receipt["components"].values() - } + } | {receipt["initializer"]["service_id"]} domains = [call[2]["http"] for call in api.calls if call[1].endswith("/domains")] admin = next(row for row in domains if "shopping-admin" in row["domain_name"]) reddit = next(row for row in domains if "reddit" in row["domain_name"]) assert admin["configuration"]["proxy_buffer_size"] == 64 assert admin["configuration"]["proxy_buffer_numbers"] == 8 assert reddit["configuration"]["proxy_buffer_size"] == 4 + initializer_envs = next( + call[2]["envs"] + for call in api.calls + if call[0] == "PUT" and "service-initializer" in call[1] + ) + assert {row["name"]: row["value"] for row in initializer_envs} == { + "REDDIT_CTRL_URL": "http://alias-reddit-201:8877", + "GITLAB_CTRL_URL": "http://alias-gitlab-201:8877", + } def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): @@ -176,3 +206,53 @@ def destroy(self, receipt_path): provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) assert destroyed == [tmp_path / "out" / "slice-1" / "slice-receipt.json"] + + +def test_scale_pool_keeps_one_warm_slice_and_removes_only_excess(tmp_path, monkeypatch): + created = [] + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, pool_id, slot, receipt_path): + created.append(slot) + receipt = { + "schema": RECEIPT_SCHEMA, + "pool_id": pool_id, + "slot": slot, + "app_id": 100 + slot, + "components": {site: {} for site in STATEFUL_SITES}, + } + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, _receipt): + return {"ok": True} + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + receipt_path.unlink() + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + output = tmp_path / "pool" + + assert len(scale_pool(tmp_path / "config.json", output, "warm", 3)) == 3 + assert len(scale_pool(tmp_path / "config.json", output, "warm", 1)) == 1 + + assert sorted(created) == [1, 2, 3] + assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2"] + with pytest.raises(ValueError, match="minimum_ready"): + scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=1) From bdc62c69009e5dfd78afb165b462e1ef86eeeaaa Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 12:09:53 +0800 Subject: [PATCH 03/23] Handle Rainbond copied component names --- .../public_webarena/rainbond_slices.py | 16 ++++++++++++++-- .../public_webarena/test_rainbond_slices.py | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 83d7716df..c70c6db7a 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -229,7 +229,13 @@ def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict } mapped: dict[str, dict[str, Any]] = {} for site, component_name in by_source_name.items(): - matches = [row for row in rows if str(row.get("service_cname")) == component_name] + copied_name = re.compile(rf"^{re.escape(component_name)}-copy(?:-\d+)?$") + matches = [ + row + for row in rows + if str(row.get("service_cname")) == component_name + or copied_name.fullmatch(str(row.get("service_cname"))) + ] if len(matches) != 1: raise RuntimeError( f"Rainbond copy returned {len(matches)} components named {component_name!r}" @@ -238,8 +244,14 @@ def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict initializer_name = str( self.raw_config.get("initializer_component_name") or "slice-initializer" ) + copied_initializer_name = re.compile( + rf"^{re.escape(initializer_name)}-copy(?:-\d+)?$" + ) initializer_matches = [ - row for row in rows if str(row.get("service_cname")) == initializer_name + row + for row in rows + if str(row.get("service_cname")) == initializer_name + or copied_initializer_name.fullmatch(str(row.get("service_cname"))) ] if len(initializer_matches) != 1: raise RuntimeError( diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index ed9969393..e57f08a77 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -38,7 +38,7 @@ def request(self, method, path, body=None): { "service_id": f"service-{site}-{self.next_app_id}", "service_alias": f"alias-{site}-{self.next_app_id}", - "service_cname": site, + "service_cname": f"{site}-copy", "image": PINNED[site], } for site in STATEFUL_SITES @@ -47,7 +47,7 @@ def request(self, method, path, body=None): { "service_id": f"service-initializer-{self.next_app_id}", "service_alias": f"alias-initializer-{self.next_app_id}", - "service_cname": "slice-initializer", + "service_cname": "slice-initializer-copy", "image": INITIALIZER_IMAGE, } ] From 7f0735290465493d1ffb05dbf25da2ef8a3f295d Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 12:18:48 +0800 Subject: [PATCH 04/23] Resolve copied Rainbond internal services --- .../public_webarena/rainbond_slices.py | 68 +++++++++++++++++-- .../public_webarena/test_rainbond_slices.py | 39 ++++++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index c70c6db7a..a95052fd2 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -358,6 +358,8 @@ def _configure_initializer( self, app_id: int, *, + pool_id: str, + slot: int, initializer: dict[str, Any], copied_sites: dict[str, dict[str, Any]], ) -> dict[str, Any]: @@ -370,11 +372,9 @@ def _configure_initializer( ): raise RuntimeError("Rainbond copied initializer image mismatch") aliases = { - site: str(copied_sites[site].get("service_alias") or "") + site: self._internal_service_host(app_id, copied_sites[site], 8877) for site in ("reddit", "gitlab") } - if not all(aliases.values()): - raise RuntimeError("Rainbond copy response has no internal service aliases") self.api.request( "PUT", f"{self._app_path(app_id)}/services/{quote(service_id)}/envs", @@ -390,7 +390,64 @@ def _configure_initializer( ] }, ) - return {"service_id": service_id} + readiness_url = f"http://{self._domain(pool_id, slot, 'initializer', app_id)}" + self.api.request( + "POST", + f"{self._app_path(app_id)}/domains", + { + "protocol": "http", + "http": { + "service_id": service_id, + "container_port": 8080, + "certificate_id": 0, + "domain_name": readiness_url.removeprefix("http://"), + "domain_cookie": "", + "domain_header": "", + "domain_path": "/", + "the_weight": 100, + "whether_open": True, + "auto_ssl": False, + "auto_ssl_config": "", + "rule_extensions": [], + "configuration": { + "proxy_body_size": 0, + "proxy_buffer_numbers": 4, + "proxy_buffer_size": 4, + "proxy_buffering": "off", + "proxy_connect_timeout": 75, + "proxy_read_timeout": 600, + "proxy_send_timeout": 600, + "set_headers": [], + }, + }, + }, + ) + return {"service_id": service_id, "readiness_url": readiness_url} + + def _internal_service_host( + self, app_id: int, component: dict[str, Any], port: int + ) -> str: + service_id = str(component.get("service_id") or "") + response = self.api.request( + "GET", + f"{self._app_path(app_id)}/services/{quote(service_id)}/ports", + ) + rows = response if isinstance(response, list) else (response or {}).get("ports", []) + match = next( + ( + row + for row in rows + if int(row.get("container_port") or 0) == port + and bool(row.get("is_inner_service")) + ), + None, + ) + host = str((match or {}).get("k8s_service_name") or "") + if not host: + raise RuntimeError( + f"Rainbond copied component {service_id} has no internal service on port {port}" + ) + return host def _wait_component_running(self, app_id: int, service_id: str) -> None: deadline = time.monotonic() + self.config.readiness_timeout_seconds @@ -451,6 +508,8 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An } initializer = self._configure_initializer( app_id, + pool_id=pool_id, + slot=slot, initializer=copied_initializer, copied_sites=copied, ) @@ -468,6 +527,7 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An timings["deploy_request_seconds"] = round(time.monotonic() - phase_started, 3) phase_started = time.monotonic() self._wait_component_running(app_id, str(initializer["service_id"])) + self._wait_site(str(initializer["readiness_url"])) for component in components.values(): self._wait_site(str(component["site_url"])) timings["readiness_seconds"] = round(time.monotonic() - phase_started, 3) diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index e57f08a77..2fc63928e 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -56,6 +56,17 @@ def request(self, method, path, body=None): service_id = body["http"]["service_id"] if self.fail_domain_for and self.fail_domain_for in service_id: raise RuntimeError("injected domain failure") + if method == "GET" and path.endswith("/ports"): + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + return { + "ports": [ + { + "container_port": 8877, + "is_inner_service": True, + "k8s_service_name": f"internal-{service_id}", + } + ] + } if method == "GET" and path.endswith("/services"): return [ { @@ -127,22 +138,46 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t row["service_id"] for row in receipt["components"].values() } | {receipt["initializer"]["service_id"]} domains = [call[2]["http"] for call in api.calls if call[1].endswith("/domains")] + assert len(domains) == 5 admin = next(row for row in domains if "shopping-admin" in row["domain_name"]) reddit = next(row for row in domains if "reddit" in row["domain_name"]) assert admin["configuration"]["proxy_buffer_size"] == 64 assert admin["configuration"]["proxy_buffer_numbers"] == 8 assert reddit["configuration"]["proxy_buffer_size"] == 4 + initializer_domain = next( + row for row in domains if row["service_id"] == "service-initializer-201" + ) + assert initializer_domain["container_port"] == 8080 + assert receipt["initializer"]["readiness_url"].endswith( + initializer_domain["domain_name"] + ) initializer_envs = next( call[2]["envs"] for call in api.calls if call[0] == "PUT" and "service-initializer" in call[1] ) assert {row["name"]: row["value"] for row in initializer_envs} == { - "REDDIT_CTRL_URL": "http://alias-reddit-201:8877", - "GITLAB_CTRL_URL": "http://alias-gitlab-201:8877", + "REDDIT_CTRL_URL": "http://internal-service-reddit-201:8877", + "GITLAB_CTRL_URL": "http://internal-service-gitlab-201:8877", } +def test_provision_rejects_copied_site_without_internal_control_port(tmp_path): + class MissingControlPortAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "GET" and path.endswith("/ports"): + return {"ports": []} + return super().request(method, path, body) + + api = MissingControlPortAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="no internal service on port 8877"): + pool.provision("run-a", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): api = FakeAPI() pool = _pool(tmp_path, api) From 19bbeedde2e6df64001345c1d802ba095f942438 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 12:28:52 +0800 Subject: [PATCH 05/23] Add exclusive WebArena slice leases --- .../public_webarena/README.md | 35 +++- .../public_webarena/rainbond_slices.py | 105 +++++++++++- scripts/manage_webarena_rainbond_pool.py | 40 ++++- .../public_webarena/test_rainbond_slices.py | 160 ++++++++++++++++++ 4 files changed, 320 insertions(+), 20 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 583d6a3c1..c2e8e0e12 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -43,21 +43,37 @@ removes the application. The read-only Map service may be shared. The baseline also contains a small digest-pinned initializer component. It calls the Reddit and GitLab `env-ctrl` endpoints only over Rainbond's internal service network, -then exposes a harmless readiness page. The unauthenticated control endpoints are never -published through the gateway. Its command is the base64-encoded content of +then exposes a harmless readiness page on its unique slice domain. The unauthenticated +control endpoints themselves are never published through the gateway. Its command is the base64-encoded content of `deploy/webarena-rainbond/initializer.sh`, executed by the pinned Alpine image; port -8080 is internal-only and is the component readiness probe. +8080 serves only the final readiness marker. Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, -and reconcile the warm pool to the required concurrency. Keep at least one ready slice; -additional slices are created in bounded parallel and removed when demand falls: +and reconcile the warm pool to the active task count plus at least one idle slice. +Additional slices are created in bounded parallel and removed when demand falls: ```text uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ - --config scale --pool-id --count \ + --config scale --pool-id --active \ --minimum-ready 1 --output ``` +Admission is an atomic lease, so two schedulers cannot assign the same writable sites: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config acquire --output --owner +``` + +Use the returned config for exactly one task. Releasing a lease creates and verifies a +clean replacement application before deleting the dirty generation and making the slot +available again. If replacement fails, the lease remains held and cannot be reused: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config release --output --lease-id +``` + For a one-shot pool that will be destroyed in full, use: ```text @@ -65,9 +81,10 @@ uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ --config provision --pool-id --count --output ``` -Pass each generated `slice-*/webarena.config.json` as a separate -`--webarena-config` argument to `scripts/run_webarena_qualification.py`. Overlapping -site batches run concurrently only when distinct slice configs are supplied. Afterwards: +Pass each leased `webarena.config.json` as a separate `--webarena-config` argument to +`scripts/run_webarena_qualification.py`. Overlapping site batches run concurrently only +when distinct slice configs are supplied. Afterwards, release each reusable lease. For a +one-shot pool, destroy the slices instead: ```text uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index a95052fd2..21e131b6d 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -20,6 +20,7 @@ SCHEMA = "lex_webarena_rainbond_pool/1" RECEIPT_SCHEMA = "lex_webarena_rainbond_slice/1" +LEASE_SCHEMA = "lex_webarena_rainbond_slice_lease/1" STATEFUL_SITES = ("shopping_admin", "shopping", "reddit", "gitlab") SITE_PORTS = {"shopping_admin": 80, "shopping": 80, "reddit": 80, "gitlab": 8023} _SAFE_ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") @@ -562,8 +563,20 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: previous = self.validate_receipt(receipt_path) pool_id = str(previous["pool_id"]) slot = int(previous["slot"]) - self.destroy(receipt_path, missing_ok=False) - return self.provision(pool_id, slot, receipt_path) + replacement_path = receipt_path.with_name("slice-receipt.replacement.json") + if replacement_path.exists(): + raise RuntimeError(f"unfinished Rainbond slice replacement: {replacement_path}") + replacement = self.provision(pool_id, slot, replacement_path) + try: + self.api.request("DELETE", f"{self._app_path(int(previous['app_id']))}?force=1") + except BaseException: + self.api.request( + "DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1" + ) + replacement_path.unlink(missing_ok=True) + raise + replacement_path.replace(receipt_path) + return replacement def validate_receipt(self, receipt_path: Path) -> dict[str, Any]: receipt = json.loads(receipt_path.read_text(encoding="utf-8")) @@ -671,7 +684,17 @@ def create_slot(slot: int) -> tuple[int, Path]: for future in as_completed(futures): slot, slot_dir = future.result() existing[slot] = slot_dir - for slot in sorted((value for value in existing if value > count), reverse=True): + excess_slots = sorted((value for value in existing if value > count), reverse=True) + leased_excess = [ + existing[slot] for slot in excess_slots + if (existing[slot] / "slice-lease.json").exists() + ] + if leased_excess: + raise RuntimeError( + "cannot remove leased Rainbond slices: " + + ", ".join(str(path) for path in leased_excess) + ) + for slot in excess_slots: slot_dir = existing.pop(slot) pool.destroy(slot_dir / "slice-receipt.json") config_file = slot_dir / "webarena.config.json" @@ -682,3 +705,79 @@ def create_slot(slot: int) -> tuple[int, Path]: except OSError: pass return [existing[slot] for slot in sorted(existing)] + + +def reconcile_pool_capacity( + config_path: Path, + output_dir: Path, + pool_id: str, + active_count: int, + *, + minimum_ready: int = 1, +) -> list[Path]: + """Keep one or more idle slices in addition to current task demand.""" + if active_count < 0: + raise ValueError("active_count must not be negative") + return scale_pool( + config_path, + output_dir, + pool_id, + active_count + minimum_ready, + minimum_ready=minimum_ready, + ) + + +def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: + """Atomically grant one ready slice to one task owner.""" + owner = owner.strip() + if not owner: + raise ValueError("lease owner is required") + with _pool_lock(output_dir): + for slot_dir in sorted(output_dir.glob("slice-*")): + receipt_path = slot_dir / "slice-receipt.json" + config_path = slot_dir / "webarena.config.json" + lease_path = slot_dir / "slice-lease.json" + if not receipt_path.is_file() or not config_path.is_file() or lease_path.exists(): + continue + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if receipt.get("schema") != RECEIPT_SCHEMA: + raise ValueError(f"invalid Rainbond slice receipt: {receipt_path}") + lease = { + "schema": LEASE_SCHEMA, + "lease_id": uuid.uuid4().hex, + "owner": owner, + "slot": int(receipt["slot"]), + "app_id": int(receipt["app_id"]), + "generation": str(receipt["generation"]), + "config": str(config_path.resolve()), + "acquired_at_unix": int(time.time()), + } + _atomic_json(lease_path, lease) + return lease + raise RuntimeError("no unleased Rainbond WebArena slice is ready") + + +def release_slice( + config_path: Path, output_dir: Path, lease_id: str +) -> dict[str, Any]: + """Replace a dirty slice before returning it to the ready pool.""" + with _pool_lock(output_dir): + matches: list[tuple[Path, dict[str, Any]]] = [] + for lease_path in output_dir.glob("slice-*/slice-lease.json"): + lease = json.loads(lease_path.read_text(encoding="utf-8")) + if lease.get("schema") == LEASE_SCHEMA and lease.get("lease_id") == lease_id: + matches.append((lease_path, lease)) + if len(matches) != 1: + raise ValueError("lease_id does not identify exactly one active slice") + lease_path, lease = matches[0] + slot_dir = lease_path.parent + pool = RainbondSlicePool(config_path) + receipt = pool.reset(slot_dir / "slice-receipt.json") + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + lease_path.unlink() + return { + "lease_id": lease_id, + "released_app_id": int(lease["app_id"]), + "ready_app_id": int(receipt["app_id"]), + "config": str((slot_dir / "webarena.config.json").resolve()), + } diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py index 59812c753..b4d956eb3 100644 --- a/scripts/manage_webarena_rainbond_pool.py +++ b/scripts/manage_webarena_rainbond_pool.py @@ -8,7 +8,10 @@ from harness.trajectory_replay.public_webarena.rainbond_slices import ( RainbondSlicePool, + acquire_slice, provision_pool, + reconcile_pool_capacity, + release_slice, scale_pool, ) @@ -23,13 +26,21 @@ def main() -> None: provision.add_argument("--output", type=Path, required=True) scale = subparsers.add_parser("scale") scale.add_argument("--pool-id", required=True) - scale.add_argument("--count", type=int, required=True) + scale_group = scale.add_mutually_exclusive_group(required=True) + scale_group.add_argument("--count", type=int) + scale_group.add_argument("--active", type=int) scale.add_argument("--minimum-ready", type=int, default=1) scale.add_argument("--output", type=Path, required=True) reset = subparsers.add_parser("reset") reset.add_argument("--slice-dir", type=Path, required=True) destroy = subparsers.add_parser("destroy") destroy.add_argument("--slice-dir", type=Path, action="append", required=True) + acquire = subparsers.add_parser("acquire") + acquire.add_argument("--output", type=Path, required=True) + acquire.add_argument("--owner", required=True) + release = subparsers.add_parser("release") + release.add_argument("--output", type=Path, required=True) + release.add_argument("--lease-id", required=True) args = parser.parse_args() if args.command == "provision": @@ -40,16 +51,29 @@ def main() -> None: return if args.command == "scale": - paths = scale_pool( - args.config.resolve(), - args.output.resolve(), - args.pool_id, - args.count, - minimum_ready=args.minimum_ready, - ) + if args.active is not None: + paths = reconcile_pool_capacity( + args.config.resolve(), args.output.resolve(), args.pool_id, args.active, + minimum_ready=args.minimum_ready, + ) + else: + paths = scale_pool( + args.config.resolve(), args.output.resolve(), args.pool_id, args.count, + minimum_ready=args.minimum_ready, + ) print(json.dumps([str(path / "webarena.config.json") for path in paths])) return + if args.command == "acquire": + print(json.dumps(acquire_slice(args.output.resolve(), args.owner), sort_keys=True)) + return + + if args.command == "release": + print(json.dumps(release_slice( + args.config.resolve(), args.output.resolve(), args.lease_id + ), sort_keys=True)) + return + pool = RainbondSlicePool(args.config.resolve()) if args.command == "reset": slot_dir = args.slice_dir.resolve() diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 2fc63928e..cc12e878c 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -6,10 +6,14 @@ import pytest from harness.trajectory_replay.public_webarena.rainbond_slices import ( + LEASE_SCHEMA, RECEIPT_SCHEMA, STATEFUL_SITES, RainbondSlicePool, + acquire_slice, provision_pool, + reconcile_pool_capacity, + release_slice, scale_pool, ) @@ -199,6 +203,18 @@ def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): method == "DELETE" and f"/{first['app_id']}?force=1" in path for method, path, _body in api.calls ) + create_replacement = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "POST" and path.endswith("/apps") and index > 0 + ) + delete_old = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "DELETE" and f"/{first['app_id']}?force=1" in path + ) + assert create_replacement < delete_old + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() def test_partial_provision_failure_removes_the_new_app_and_leaves_no_receipt(tmp_path): @@ -291,3 +307,147 @@ def destroy(self, receipt_path): assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2"] with pytest.raises(ValueError, match="minimum_ready"): scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=1) + + +def test_capacity_keeps_idle_slice_in_addition_to_active_demand(tmp_path, monkeypatch): + calls = [] + + def fake_scale(config, output, pool_id, count, *, minimum_ready): + calls.append((config, output, pool_id, count, minimum_ready)) + return [] + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.scale_pool", fake_scale + ) + + reconcile_pool_capacity( + tmp_path / "pool.json", tmp_path / "runtime", "warm", 3, minimum_ready=1 + ) + + assert calls[0][3:] == (4, 1) + + +def test_scale_down_is_atomic_when_an_excess_slice_is_leased(tmp_path, monkeypatch): + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + output = tmp_path / "runtime" + for slot in (1, 2, 3): + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text( + json.dumps({"pool_id": "warm", "slot": slot}) + ) + (output / "slice-2" / "slice-lease.json").write_text("{}") + + with pytest.raises(RuntimeError, match="leased Rainbond slices"): + scale_pool(tmp_path / "pool.json", output, "warm", 1) + + assert destroyed == [] + + +def _ready_slot(output: Path, slot: int, app_id: int) -> Path: + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text( + json.dumps( + { + "schema": RECEIPT_SCHEMA, + "slot": slot, + "app_id": app_id, + "generation": f"generation-{slot}", + } + ) + ) + (slot_dir / "webarena.config.json").write_text("{}") + return slot_dir + + +def test_acquire_grants_distinct_slices_and_exhausts_pool(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + + first = acquire_slice(output, "task-a") + second = acquire_slice(output, "task-b") + + assert first["schema"] == LEASE_SCHEMA + assert first["slot"] != second["slot"] + with pytest.raises(RuntimeError, match="no unleased"): + acquire_slice(output, "task-c") + + +def test_release_replaces_dirty_generation_before_unlocking_slice(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + + class Pool: + def __init__(self, _config): + pass + + def reset(self, _receipt_path): + assert (slot_dir / "slice-lease.json").exists() + return { + "schema": RECEIPT_SCHEMA, + "slot": 1, + "app_id": 201, + "generation": "generation-new", + } + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + + assert result["released_app_id"] == 101 + assert result["ready_app_id"] == 201 + assert not (slot_dir / "slice-lease.json").exists() + assert json.loads((slot_dir / "webarena.config.json").read_text()) == {"app_id": 201} + + +def test_failed_release_keeps_lease_so_dirty_slice_cannot_be_reassigned( + tmp_path, monkeypatch +): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + + class Pool: + def __init__(self, _config): + pass + + def reset(self, _receipt_path): + raise RuntimeError("replacement failed") + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + + with pytest.raises(RuntimeError, match="replacement failed"): + release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + + assert (slot_dir / "slice-lease.json").exists() + with pytest.raises(RuntimeError, match="no unleased"): + acquire_slice(output, "task-b") From fea1d471cace3796ad777c65983256eff9192846 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 12:51:59 +0800 Subject: [PATCH 06/23] Document Magento gateway header fix --- .../trajectory_replay/public_webarena/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index c2e8e0e12..563e1fbe8 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -48,6 +48,20 @@ control endpoints themselves are never published through the gateway. Its comman `deploy/webarena-rainbond/initializer.sh`, executed by the pinned Alpine image; port 8080 serves only the final readiness marker. +Both Magento baseline components must also carry this image start command before they +are copied. It removes Magento's report-only CSP response header at the inner Nginx +boundary; the header exceeds the upstream-header limit of the APISIX gateway used by +this Rainbond installation and otherwise produces a gateway 502 even while the Pod and +port are healthy: + +```text +sh -c "sed -i '/fastcgi_pass[[:space:]]*fastcgi_backend;/a\\ fastcgi_hide_header Content-Security-Policy-Report-Only;' /var/www/magento2/nginx.conf.sample && exec /usr/bin/python3 /usr/bin/supervisord -n -j /supervisord.pid" +``` + +The command changes only the report-only response header. It preserves the image's +entrypoint initialization and supervisor process, and application readiness still +requires a successful request through each public slice domain. + Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, and reconcile the warm pool to the active task count plus at least one idle slice. Additional slices are created in bounded parallel and removed when demand falls: From ce9dfe4109d37fc36285308c3e7801119cb80abf Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 13:07:30 +0800 Subject: [PATCH 07/23] Use mounted Rainbond runtime configuration --- .../public_webarena/README.md | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 563e1fbe8..60776694b 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -44,23 +44,25 @@ removes the application. The read-only Map service may be shared. The baseline also contains a small digest-pinned initializer component. It calls the Reddit and GitLab `env-ctrl` endpoints only over Rainbond's internal service network, then exposes a harmless readiness page on its unique slice domain. The unauthenticated -control endpoints themselves are never published through the gateway. Its command is the base64-encoded content of -`deploy/webarena-rainbond/initializer.sh`, executed by the pinned Alpine image; port -8080 serves only the final readiness marker. +control endpoints themselves are never published through the gateway. +`deploy/webarena-rainbond/initializer.sh` is mounted as a Rainbond `config-file` at +`/opt/webarena/initializer.sh` and run as `/bin/sh /opt/webarena/initializer.sh` by +the pinned Alpine image; port 8080 serves only the final readiness marker. Keeping the +script in a mounted file avoids Rainbond's whitespace-only image-command parsing. -Both Magento baseline components must also carry this image start command before they -are copied. It removes Magento's report-only CSP response header at the inner Nginx -boundary; the header exceeds the upstream-header limit of the APISIX gateway used by -this Rainbond installation and otherwise produces a gateway 502 even while the Pod and -port are healthy: +Both Magento baseline components must also mount a Rainbond `config-file` named +`wa-hide-csp-*` at `/etc/nginx/conf.d/00-webarena-hide-csp.conf` before they are copied. +Its complete content is: ```text -sh -c "sed -i '/fastcgi_pass[[:space:]]*fastcgi_backend;/a\\ fastcgi_hide_header Content-Security-Policy-Report-Only;' /var/www/magento2/nginx.conf.sample && exec /usr/bin/python3 /usr/bin/supervisord -n -j /supervisord.pid" +fastcgi_hide_header Content-Security-Policy-Report-Only; ``` -The command changes only the report-only response header. It preserves the image's -entrypoint initialization and supervisor process, and application readiness still -requires a successful request through each public slice domain. +The Nginx configuration changes only the report-only response header. That header +exceeds the upstream-header limit of the APISIX gateway used by this Rainbond +installation and otherwise produces a gateway 502 even while the Pod and port are +healthy. The images retain their default startup commands, and application readiness +still requires a successful request through each public slice domain. Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, and reconcile the warm pool to the active task count plus at least one idle slice. From 7e1c2a3167e960910d16faf038a59d872b678318 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 13:36:05 +0800 Subject: [PATCH 08/23] Handle Rainbond list response variants --- .../public_webarena/rainbond_slices.py | 20 ++++++++++++++++--- .../public_webarena/test_rainbond_slices.py | 19 ++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 21e131b6d..c077ffbd6 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -187,6 +187,20 @@ def _apps_path(self) -> str: def _app_path(self, app_id: int) -> str: return f"{self._apps_path()}/{app_id}" + @staticmethod + def _response_rows(response: Any, *keys: str) -> list[dict[str, Any]]: + if isinstance(response, list): + return [row for row in response if isinstance(row, dict)] + if not isinstance(response, dict): + return [] + for key in (*keys, "items"): + value = response.get(key) + if isinstance(value, list): + return [row for row in value if isinstance(row, dict)] + if isinstance(value, dict) and isinstance(value.get("items"), list): + return [row for row in value["items"] if isinstance(row, dict)] + return [] + def _domain(self, pool_id: str, slot: int, site: str, app_id: int) -> str: label = ( f"{self.config.app_prefix}-{pool_id}-{slot}-{app_id}-{site.replace('_', '-')}" @@ -223,7 +237,7 @@ def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict "target_app_id": app_id, }, ) - rows = list((result or {}).get("services") or []) + rows = self._response_rows(result, "services") by_source_name = { site: str(self.raw_config.get("source_component_names", {}).get(site) or site) for site in STATEFUL_SITES @@ -433,7 +447,7 @@ def _internal_service_host( "GET", f"{self._app_path(app_id)}/services/{quote(service_id)}/ports", ) - rows = response if isinstance(response, list) else (response or {}).get("ports", []) + rows = self._response_rows(response, "ports") match = next( ( row @@ -455,7 +469,7 @@ def _wait_component_running(self, app_id: int, service_id: str) -> None: last_status = "not found" while time.monotonic() < deadline: response = self.api.request("GET", f"{self._app_path(app_id)}/services") - rows = response if isinstance(response, list) else (response or {}).get("services", []) + rows = self._response_rows(response, "services") match = next( (row for row in rows if str(row.get("service_id")) == service_id), None ) diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index cc12e878c..bdabf9f03 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -166,6 +166,25 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t } +def test_provision_accepts_console_items_response_shapes(tmp_path): + class ItemsAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + return {"items": response["services"]} + if method == "GET" and path.endswith("/ports"): + return {"ports": {"items": response["ports"]}} + if method == "GET" and path.endswith("/services"): + return {"items": response} + return response + + receipt = _pool(tmp_path, ItemsAPI()).provision( + "run-items", 1, tmp_path / "slice-receipt.json" + ) + + assert set(receipt["components"]) == set(STATEFUL_SITES) + + def test_provision_rejects_copied_site_without_internal_control_port(tmp_path): class MissingControlPortAPI(FakeAPI): def request(self, method, path, body=None): From c9991464f4391f42080b0d819834b8b5b1dea37e Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 13:38:08 +0800 Subject: [PATCH 09/23] Document WebArena pool capacity requirements --- .../trajectory_replay/public_webarena/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 60776694b..759465ed2 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -111,3 +111,18 @@ uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ Each receipt records create, copy, configure, deploy-request, readiness, and total cold-start durations. Use those measurements to size the warm pool; task admission should allocate a ready slice rather than waiting for a cold start. + +The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus +100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable +resources, not cluster totals. A pool serving `N` active tasks with one idle warm slice +therefore needs at least `(N + 1) * 10.1` cores and `(N + 1) * 20.125 GiB`, with additional +headroom for Rainbond's rolling-update surge Pods and unrelated workloads. If that headroom +is unavailable, replacements remain pending instead of weakening isolation or silently +sharing a site. + +Rainbond remains the runtime data plane and this repository's lease controller remains the +source of truth for lifecycle and reset. Kestra is optional above it for queues, schedules, +timeouts, retries, and guaranteed cleanup; it is not required to allocate a slice and must +not become a runtime dependency. Argo Workflows is only justified when the deployment moves +to direct Kubernetes-native orchestration at substantially larger scale, because using it +for the current pool would duplicate Rainbond application lifecycle management. From c04a604682c72afd21ab91e99e98fd3009f5e445 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 13:45:29 +0800 Subject: [PATCH 10/23] Require GitLab shared-memory mount --- examples/trajectory_replay/public_webarena/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 759465ed2..6bb09ad1a 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -64,6 +64,13 @@ installation and otherwise produces a gateway 502 even while the Pod and port ar healthy. The images retain their default startup commands, and application readiness still requires a successful request through each public slice domain. +The GitLab baseline must mount an exclusive 2 GiB Rainbond `memoryfs` volume named +`gitlab-shm` at `/dev/shm`. This preserves the upstream Compose `shm_size: 2gb` +runtime contract; a generic persistent volume or the container runtime's small default +shared-memory mount is not equivalent. The four seeded site images otherwise keep their +data in the slice's disposable writable layers. Do not mount empty volumes over their +preloaded database directories. + Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, and reconcile the warm pool to the active task count plus at least one idle slice. Additional slices are created in bounded parallel and removed when demand falls: From f583cd6c2e08e6ff1d189536d53d5cf1a87e5d23 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 14:47:41 +0800 Subject: [PATCH 11/23] fix rainbond slice runtime wiring --- deploy/webarena-rainbond/initializer.sh | 40 ++++++- .../public_webarena/README.md | 34 ++++-- .../public_webarena/rainbond_slices.py | 101 ++++++++++++++---- .../public_webarena/test_rainbond_slices.py | 60 +++++++---- 4 files changed, 185 insertions(+), 50 deletions(-) diff --git a/deploy/webarena-rainbond/initializer.sh b/deploy/webarena-rainbond/initializer.sh index b637c27cc..59630003a 100755 --- a/deploy/webarena-rainbond/initializer.sh +++ b/deploy/webarena-rainbond/initializer.sh @@ -16,9 +16,31 @@ wait_ctrl() { init_ctrl() { url="$1" - response="$(wget -qO- --post-data='' "$url/init")" - echo "$response" - echo "$response" | grep -Eq '"success"[[:space:]]*:[[:space:]]*true' + attempts=0 + while :; do + set +e + response="$(wget -qO- --post-data='' "$url/init" 2>&1)" + result=$? + set -e + if echo "$response" | grep -Eq '"success"[[:space:]]*:[[:space:]]*true'; then + echo "$response" + return + fi + # env-ctrl can restart its managed process before flushing the HTTP response. + # BusyBox wget reports that successful reset path as an empty/broken response. + if [ "$result" -ne 0 ] && echo "$response" | grep -Eq 'error getting response|Connection reset'; then + wait_ctrl "$url" + echo "env-ctrl init completed after control endpoint restart: $url" + return + fi + attempts=$((attempts + 1)) + if [ "$attempts" -ge "${ENV_CTRL_MAX_ATTEMPTS:-360}" ]; then + echo "env-ctrl init timeout: $url: $response" >&2 + exit 1 + fi + echo "env-ctrl init retry: $url" >&2 + sleep "${ENV_CTRL_RETRY_SECONDS:-5}" + done } wait_ctrl "$REDDIT_CTRL_URL" @@ -28,4 +50,14 @@ init_ctrl "$GITLAB_CTRL_URL" mkdir -p /www printf 'ready\n' >/www/index.html -exec httpd -f -p 8080 -h /www +if busybox --list | grep -qx httpd; then + exec busybox httpd -f -p 8080 -h /www +fi +if busybox --list | grep -qx nc; then + while :; do + printf 'HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\nready\n' | + busybox nc -l -p 8080 + done +fi +echo "initializer image has neither the httpd nor nc BusyBox applet" >&2 +exit 1 diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 6bb09ad1a..52f4995bc 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -49,6 +49,10 @@ control endpoints themselves are never published through the gateway. `/opt/webarena/initializer.sh` and run as `/bin/sh /opt/webarena/initializer.sh` by the pinned Alpine image; port 8080 serves only the final readiness marker. Keeping the script in a mounted file avoids Rainbond's whitespace-only image-command parsing. +The initializer depends on the copied Reddit and GitLab components so Rainbond makes +their internal Service names resolvable. Keep only the site-specific exported variables +`REDDIT_CTRL_URL` and `GITLAB_CTRL_URL`; remove generic duplicated `CTRL_HOST` and +`CTRL_PORT` exports before creating both dependency relations. Both Magento baseline components must also mount a Rainbond `config-file` named `wa-hide-csp-*` at `/etc/nginx/conf.d/00-webarena-hide-csp.conf` before they are copied. @@ -64,12 +68,30 @@ installation and otherwise produces a gateway 502 even while the Pod and port ar healthy. The images retain their default startup commands, and application readiness still requires a successful request through each public slice domain. -The GitLab baseline must mount an exclusive 2 GiB Rainbond `memoryfs` volume named -`gitlab-shm` at `/dev/shm`. This preserves the upstream Compose `shm_size: 2gb` -runtime contract; a generic persistent volume or the container runtime's small default -shared-memory mount is not equivalent. The four seeded site images otherwise keep their -data in the slice's disposable writable layers. Do not mount empty volumes over their -preloaded database directories. +The GitLab baseline must reproduce the upstream Compose `shm_size: 2gb` contract with +native Kubernetes attributes, not a Rainbond `memoryfs` storage volume. In the GitLab +component's **Other settings → Kubernetes attributes**, set `volumes` to: + +```yaml +- name: gitlab-shm + emptyDir: + medium: Memory + sizeLimit: 2Gi +``` + +and set `volumeMounts` to: + +```yaml +- name: gitlab-shm + mountPath: /dev/shm +``` + +Rainbond attribute values omit the top-level `volumes:` and `volumeMounts:` keys. +Verify inside a running GitLab container that `/dev/shm` is a 2 GiB `tmpfs`; a +Rainbond `memoryfs` volume is backed by the cluster filesystem in this deployment and +is not equivalent. The four seeded site images otherwise keep their data in the +slice's disposable writable layers. Do not mount empty volumes over their preloaded +database directories. Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, and reconcile the warm pool to the active task count plus at least one idle slice. diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index c077ffbd6..e320200bb 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -202,9 +202,7 @@ def _response_rows(response: Any, *keys: str) -> list[dict[str, Any]]: return [] def _domain(self, pool_id: str, slot: int, site: str, app_id: int) -> str: - label = ( - f"{self.config.app_prefix}-{pool_id}-{slot}-{app_id}-{site.replace('_', '-')}" - ) + label = f"{self.config.app_prefix}-{pool_id}-{slot}-{app_id}-{site.replace('_', '-')}" if len(label) > 63: label = f"{label[:50].rstrip('-')}-{hashlib.sha256(label.encode()).hexdigest()[:10]}" return f"{label}.{self.config.domain_suffix}" @@ -259,9 +257,7 @@ def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict initializer_name = str( self.raw_config.get("initializer_component_name") or "slice-initializer" ) - copied_initializer_name = re.compile( - rf"^{re.escape(initializer_name)}-copy(?:-\d+)?$" - ) + copied_initializer_name = re.compile(rf"^{re.escape(initializer_name)}-copy(?:-\d+)?$") initializer_matches = [ row for row in rows @@ -287,6 +283,13 @@ def _configure_component( service_id = str(component.get("service_id") or "") if not service_id: raise RuntimeError(f"Rainbond copied component has no service_id: {site}") + self._coalesce_internal_ports( + app_id, + service_id, + site_port=SITE_PORTS[site], + control_port=8877, + control_alias=f"{site.upper()}_CTRL", + ) domain = self._domain(pool_id, slot, site, app_id) site_url = f"http://{domain}" magento = site in {"shopping_admin", "shopping"} @@ -369,6 +372,56 @@ def _configure_component( "site_url": site_url, } + def _coalesce_internal_ports( + self, + app_id: int, + service_id: str, + *, + site_port: int, + control_port: int, + control_alias: str, + ) -> None: + path = f"{self._app_path(app_id)}/services/{quote(service_id)}/ports" + + def inner_ports() -> dict[int, dict[str, Any]]: + response = self.api.request("GET", path) + return { + int(row.get("container_port") or 0): row + for row in self._response_rows(response, "ports") + if bool(row.get("is_inner_service")) + } + + ports = inner_ports() + site_host = str(ports.get(site_port, {}).get("k8s_service_name") or "") + control_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") + if not site_host or not control_host: + raise RuntimeError( + f"Rainbond copied component {service_id} must expose internal ports " + f"{site_port} and {control_port}" + ) + if site_host == control_host: + return + control_path = f"{path}/{control_port}" + self.api.request("PUT", control_path, {"action": "close_inner"}) + self.api.request("DELETE", control_path) + self.api.request( + "POST", + path, + { + "port": control_port, + "protocol": str(ports[control_port].get("protocol") or "http"), + "port_alias": control_alias, + "is_inner_service": True, + }, + ) + ports = inner_ports() + updated_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") + if updated_host != site_host: + raise RuntimeError( + f"Rainbond did not attach control port {control_port} to internal " + f"service {site_host!r}: got {updated_host!r}" + ) + def _configure_initializer( self, app_id: int, @@ -403,6 +456,20 @@ def _configure_initializer( } for site, alias in aliases.items() ] + + [ + { + "name": "ENV_CTRL_MAX_ATTEMPTS", + "value": "360", + "is_change": True, + "scope": "inner", + }, + { + "name": "ENV_CTRL_RETRY_SECONDS", + "value": "5", + "is_change": True, + "scope": "inner", + }, + ] }, ) readiness_url = f"http://{self._domain(pool_id, slot, 'initializer', app_id)}" @@ -439,9 +506,7 @@ def _configure_initializer( ) return {"service_id": service_id, "readiness_url": readiness_url} - def _internal_service_host( - self, app_id: int, component: dict[str, Any], port: int - ) -> str: + def _internal_service_host(self, app_id: int, component: dict[str, Any], port: int) -> str: service_id = str(component.get("service_id") or "") response = self.api.request( "GET", @@ -452,8 +517,7 @@ def _internal_service_host( ( row for row in rows - if int(row.get("container_port") or 0) == port - and bool(row.get("is_inner_service")) + if int(row.get("container_port") or 0) == port and bool(row.get("is_inner_service")) ), None, ) @@ -470,9 +534,7 @@ def _wait_component_running(self, app_id: int, service_id: str) -> None: while time.monotonic() < deadline: response = self.api.request("GET", f"{self._app_path(app_id)}/services") rows = self._response_rows(response, "services") - match = next( - (row for row in rows if str(row.get("service_id")) == service_id), None - ) + match = next((row for row in rows if str(row.get("service_id")) == service_id), None) if match is not None: last_status = str(match.get("status") or "unknown") if last_status == "running": @@ -584,9 +646,7 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: try: self.api.request("DELETE", f"{self._app_path(int(previous['app_id']))}?force=1") except BaseException: - self.api.request( - "DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1" - ) + self.api.request("DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1") replacement_path.unlink(missing_ok=True) raise replacement_path.replace(receipt_path) @@ -700,7 +760,8 @@ def create_slot(slot: int) -> tuple[int, Path]: existing[slot] = slot_dir excess_slots = sorted((value for value in existing if value > count), reverse=True) leased_excess = [ - existing[slot] for slot in excess_slots + existing[slot] + for slot in excess_slots if (existing[slot] / "slice-lease.json").exists() ] if leased_excess: @@ -771,9 +832,7 @@ def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: raise RuntimeError("no unleased Rainbond WebArena slice is ready") -def release_slice( - config_path: Path, output_dir: Path, lease_id: str -) -> dict[str, Any]: +def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[str, Any]: """Replace a dirty slice before returning it to the ready pool.""" with _pool_lock(output_dir): matches: list[tuple[Path, dict[str, Any]]] = [] diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index bdabf9f03..e33bb7ec7 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -29,6 +29,7 @@ def __init__(self, *, fail_domain_for: str | None = None) -> None: self.calls = [] self.next_app_id = 200 self.fail_domain_for = fail_domain_for + self.port_hosts = {} def request(self, method, path, body=None): self.calls.append((method, path, body)) @@ -62,15 +63,34 @@ def request(self, method, path, body=None): raise RuntimeError("injected domain failure") if method == "GET" and path.endswith("/ports"): service_id = path.split("/services/", 1)[1].split("/", 1)[0] + site = next(site for site in STATEFUL_SITES if site in service_id) + site_host = f"internal-{service_id}" return { "ports": [ + { + "container_port": 8023 if site == "gitlab" else 80, + "is_inner_service": True, + "k8s_service_name": site_host, + }, { "container_port": 8877, "is_inner_service": True, - "k8s_service_name": f"internal-{service_id}", - } + "k8s_service_name": self.port_hosts.get( + service_id, f"control-{service_id}" + ), + }, ] } + if method == "PUT" and "/ports/8877" in path: + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + self.port_hosts.pop(service_id, None) + return {} + if method == "DELETE" and "/ports/8877" in path: + return {} + if method == "POST" and path.endswith("/ports"): + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + self.port_hosts[service_id] = f"internal-{service_id}" + return {} if method == "GET" and path.endswith("/services"): return [ { @@ -152,9 +172,7 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t row for row in domains if row["service_id"] == "service-initializer-201" ) assert initializer_domain["container_port"] == 8080 - assert receipt["initializer"]["readiness_url"].endswith( - initializer_domain["domain_name"] - ) + assert receipt["initializer"]["readiness_url"].endswith(initializer_domain["domain_name"]) initializer_envs = next( call[2]["envs"] for call in api.calls @@ -163,6 +181,19 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t assert {row["name"]: row["value"] for row in initializer_envs} == { "REDDIT_CTRL_URL": "http://internal-service-reddit-201:8877", "GITLAB_CTRL_URL": "http://internal-service-gitlab-201:8877", + "ENV_CTRL_MAX_ATTEMPTS": "360", + "ENV_CTRL_RETRY_SECONDS": "5", + } + port_mutations = [ + call for call in api.calls if "/ports" in call[1] and call[0] != "GET" + ] + assert [call[0] for call in port_mutations] == ["PUT", "DELETE", "POST"] * 4 + recreated = [call for call in port_mutations if call[0] == "POST"] + assert {call[2]["port_alias"] for call in recreated} == { + "SHOPPING_ADMIN_CTRL", + "SHOPPING_CTRL", + "REDDIT_CTRL", + "GITLAB_CTRL", } @@ -178,9 +209,7 @@ def request(self, method, path, body=None): return {"items": response} return response - receipt = _pool(tmp_path, ItemsAPI()).provision( - "run-items", 1, tmp_path / "slice-receipt.json" - ) + receipt = _pool(tmp_path, ItemsAPI()).provision("run-items", 1, tmp_path / "slice-receipt.json") assert set(receipt["components"]) == set(STATEFUL_SITES) @@ -195,7 +224,7 @@ def request(self, method, path, body=None): api = MissingControlPortAPI() pool = _pool(tmp_path, api) - with pytest.raises(RuntimeError, match="no internal service on port 8877"): + with pytest.raises(RuntimeError, match="must expose internal ports 80 and 8877"): pool.provision("run-a", 1, tmp_path / "slice-receipt.json") assert any(method == "DELETE" for method, _path, _body in api.calls) @@ -214,10 +243,7 @@ def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): assert {row["site_url"] for row in second["components"].values()}.isdisjoint( {row["site_url"] for row in first["components"].values()} ) - assert all( - f"-{second['app_id']}-" in row["site_url"] - for row in second["components"].values() - ) + assert all(f"-{second['app_id']}-" in row["site_url"] for row in second["components"].values()) assert any( method == "DELETE" and f"/{first['app_id']}?force=1" in path for method, path, _body in api.calls @@ -371,9 +397,7 @@ def destroy(self, receipt_path): for slot in (1, 2, 3): slot_dir = output / f"slice-{slot}" slot_dir.mkdir(parents=True) - (slot_dir / "slice-receipt.json").write_text( - json.dumps({"pool_id": "warm", "slot": slot}) - ) + (slot_dir / "slice-receipt.json").write_text(json.dumps({"pool_id": "warm", "slot": slot})) (output / "slice-2" / "slice-lease.json").write_text("{}") with pytest.raises(RuntimeError, match="leased Rainbond slices"): @@ -446,9 +470,7 @@ def runtime_config(self, receipt): assert json.loads((slot_dir / "webarena.config.json").read_text()) == {"app_id": 201} -def test_failed_release_keeps_lease_so_dirty_slice_cannot_be_reassigned( - tmp_path, monkeypatch -): +def test_failed_release_keeps_lease_so_dirty_slice_cannot_be_reassigned(tmp_path, monkeypatch): output = tmp_path / "runtime" slot_dir = _ready_slot(output, 1, 101) lease = acquire_slice(output, "task-a") From df945bdeb28d962f864d11116ee009d09e7331a0 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 15:27:22 +0800 Subject: [PATCH 12/23] Make WebArena resets capacity safe --- .../public_webarena/README.md | 7 ++ .../rainbond-pool.example.json | 1 + .../public_webarena/rainbond_slices.py | 62 +++++++++- .../public_webarena/test_rainbond_slices.py | 113 ++++++++++++++---- 4 files changed, 160 insertions(+), 23 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 52f4995bc..cec4f1a6b 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -119,6 +119,13 @@ uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ --config release --output --lease-id ``` +Set `reset_strategy` to `stop_before_replace` when the cluster has capacity for only one +slice. The controller keeps the lease held, stops the dirty application to release its +CPU and memory, creates and verifies the replacement, and only then deletes the stopped +generation. If provisioning fails, it restarts and verifies the previous generation while +keeping the lease held. Use `replace_before_destroy` only when the cluster has enough +surge capacity for two complete slices during every reset. + For a one-shot pool that will be destroyed in full, use: ```text diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json index f184cdee2..c95ff57e1 100644 --- a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -26,6 +26,7 @@ "image_lock": "../../../../evaluation/browser-use-public/webarena-environment-images.json", "readiness_timeout_seconds": 1800, "max_parallel_provisions": 2, + "reset_strategy": "stop_before_replace", "shared_environments": { "__MAP__": { "urls": ["https://replace-with-shared-read-only-map"] diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index e320200bb..90c3a675d 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -103,6 +103,7 @@ class PoolConfig: shared_environments: dict[str, Any] readiness_timeout_seconds: int max_parallel_provisions: int + reset_strategy: str @classmethod def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: @@ -149,6 +150,7 @@ def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: shared_environments=dict(raw.get("shared_environments") or {}), readiness_timeout_seconds=int(raw.get("readiness_timeout_seconds") or 1800), max_parallel_provisions=int(raw.get("max_parallel_provisions") or 2), + reset_strategy=str(raw.get("reset_strategy") or "replace_before_destroy"), ) if ( not all((value.api_base_url, value.team_id, value.team_name, value.region_name)) @@ -156,6 +158,8 @@ def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: or value.max_parallel_provisions < 1 ): raise ValueError("Rainbond pool API, team, region, and source app are required") + if value.reset_strategy not in {"replace_before_destroy", "stop_before_replace"}: + raise ValueError("reset_strategy must be replace_before_destroy or stop_before_replace") return value, raw, config_path @@ -559,6 +563,51 @@ def _wait_site(self, url: str) -> None: self.sleep(5) raise TimeoutError(f"Rainbond WebArena site did not become ready: {url}: {last_error}") + def _operate_existing_app(self, receipt: dict[str, Any], action: str) -> None: + service_ids = [ + str(component["service_id"]) for component in (receipt.get("components") or {}).values() + ] + service_ids.append(str(receipt["initializer"]["service_id"])) + self.api.request( + "POST", + f"{self._app_path(int(receipt['app_id']))}/operations", + {"action": action, "service_ids": service_ids}, + ) + + def _wait_app_stopped(self, receipt: dict[str, Any]) -> None: + app_id = int(receipt["app_id"]) + expected = { + str(component["service_id"]) for component in (receipt.get("components") or {}).values() + } + expected.add(str(receipt["initializer"]["service_id"])) + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_statuses: dict[str, str] = {} + while time.monotonic() < deadline: + response = self.api.request("GET", f"{self._app_path(app_id)}/services") + rows = self._response_rows(response, "services") + last_statuses = { + str(row.get("service_id")): str(row.get("status") or "unknown") + for row in rows + if str(row.get("service_id")) in expected + } + if set(last_statuses) == expected and all( + status in {"closed", "undeploy"} for status in last_statuses.values() + ): + return + self.sleep(5) + raise TimeoutError( + f"Rainbond app did not stop before replacement: {app_id}: {last_statuses}" + ) + + def _restore_previous_app(self, receipt: dict[str, Any]) -> None: + self._operate_existing_app(receipt, "start") + self._wait_component_running( + int(receipt["app_id"]), str(receipt["initializer"]["service_id"]) + ) + self._wait_site(str(receipt["initializer"]["readiness_url"])) + for component in receipt["components"].values(): + self._wait_site(str(component["site_url"])) + def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: started = time.monotonic() phase_started = started @@ -642,12 +691,23 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: replacement_path = receipt_path.with_name("slice-receipt.replacement.json") if replacement_path.exists(): raise RuntimeError(f"unfinished Rainbond slice replacement: {replacement_path}") - replacement = self.provision(pool_id, slot, replacement_path) + stopped_previous = self.config.reset_strategy == "stop_before_replace" + if stopped_previous: + self._operate_existing_app(previous, "stop") + self._wait_app_stopped(previous) + try: + replacement = self.provision(pool_id, slot, replacement_path) + except BaseException: + if stopped_previous: + self._restore_previous_app(previous) + raise try: self.api.request("DELETE", f"{self._app_path(int(previous['app_id']))}?force=1") except BaseException: self.api.request("DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1") replacement_path.unlink(missing_ok=True) + if stopped_previous: + self._restore_previous_app(previous) raise replacement_path.replace(receipt_path) return replacement diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index e33bb7ec7..689bbeabd 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -101,7 +101,7 @@ def request(self, method, path, body=None): return {} -def _pool_config(tmp_path: Path) -> Path: +def _pool_config(tmp_path: Path, *, reset_strategy: str | None = None) -> Path: lock = tmp_path / "images.json" lock.write_text( json.dumps( @@ -113,25 +113,26 @@ def _pool_config(tmp_path: Path) -> Path: encoding="utf-8", ) config = tmp_path / "pool.json" + value = { + "schema": "lex_webarena_rainbond_pool/1", + "api_base_url": "https://rainbond.example", + "team_id": "team-id", + "team_name": "efficiency", + "region_name": "rainbond", + "source_app_id": 192, + "source_services": {site: f"source-{site}" for site in STATEFUL_SITES}, + "initializer_service": "source-initializer", + "initializer_image": INITIALIZER_IMAGE, + "app_prefix": "webarena-slice", + "domain_suffix": "dev.example.test", + "token_env": "TEST_RAINBOND_TOKEN", + "image_lock": "images.json", + "readiness_timeout_seconds": 1, + } + if reset_strategy is not None: + value["reset_strategy"] = reset_strategy config.write_text( - json.dumps( - { - "schema": "lex_webarena_rainbond_pool/1", - "api_base_url": "https://rainbond.example", - "team_id": "team-id", - "team_name": "efficiency", - "region_name": "rainbond", - "source_app_id": 192, - "source_services": {site: f"source-{site}" for site in STATEFUL_SITES}, - "initializer_service": "source-initializer", - "initializer_image": INITIALIZER_IMAGE, - "app_prefix": "webarena-slice", - "domain_suffix": "dev.example.test", - "token_env": "TEST_RAINBOND_TOKEN", - "image_lock": "images.json", - "readiness_timeout_seconds": 1, - } - ), + json.dumps(value), encoding="utf-8", ) return config @@ -184,9 +185,7 @@ def test_provision_creates_one_complete_isolated_app_without_serializing_token(t "ENV_CTRL_MAX_ATTEMPTS": "360", "ENV_CTRL_RETRY_SECONDS": "5", } - port_mutations = [ - call for call in api.calls if "/ports" in call[1] and call[0] != "GET" - ] + port_mutations = [call for call in api.calls if "/ports" in call[1] and call[0] != "GET"] assert [call[0] for call in port_mutations] == ["PUT", "DELETE", "POST"] * 4 recreated = [call for call in port_mutations if call[0] == "POST"] assert {call[2]["port_alias"] for call in recreated} == { @@ -262,6 +261,76 @@ def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): assert not receipt_path.with_name("slice-receipt.replacement.json").exists() +def test_capacity_safe_reset_stops_old_app_before_creating_replacement(tmp_path): + api = FakeAPI() + pool = RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="stop_before_replace"), + api=api, + sleep=lambda _: None, + ) + pool._wait_site = lambda _url: None + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-capacity", 1, receipt_path) + + second = pool.reset(receipt_path) + + stop_index = next( + index + for index, (method, path, body) in enumerate(api.calls) + if method == "POST" + and path.endswith(f"/{first['app_id']}/operations") + and body["action"] == "stop" + ) + replacement_index = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "POST" and path.endswith("/apps") and index > stop_index + ) + delete_index = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "DELETE" and path.endswith(f"/{first['app_id']}?force=1") + ) + assert stop_index < replacement_index < delete_index + assert second["app_id"] != first["app_id"] + + +def test_capacity_safe_reset_restores_old_app_and_keeps_receipt_on_failure(tmp_path): + api = FakeAPI() + pool = RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="stop_before_replace"), + api=api, + sleep=lambda _: None, + ) + pool._wait_site = lambda _url: None + pool._wait_app_stopped = lambda _receipt: None + pool._wait_component_running = lambda _app_id, _service_id: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-recovery", 1, receipt_path) + api.fail_domain_for = "reddit" + + with pytest.raises(RuntimeError, match="injected domain failure"): + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + operations = [ + body["action"] + for method, path, body in api.calls + if method == "POST" and path.endswith(f"/{first['app_id']}/operations") + ] + assert operations[-2:] == ["stop", "start"] + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() + + +def test_pool_config_rejects_unknown_reset_strategy(tmp_path): + with pytest.raises(ValueError, match="reset_strategy"): + RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="unsafe"), + api=FakeAPI(), + ) + + def test_partial_provision_failure_removes_the_new_app_and_leaves_no_receipt(tmp_path): api = FakeAPI(fail_domain_for="reddit") pool = _pool(tmp_path, api) From dec078618be3dff3c836d2f4bf06f2bc3887b193 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 15:32:08 +0800 Subject: [PATCH 13/23] Harden Rainbond pool failure cleanup --- .../public_webarena/rainbond_slices.py | 138 ++++++++++++--- scripts/run_webarena_qualification.py | 1 + .../public_webarena/test_rainbond_slices.py | 158 ++++++++++++++++++ 3 files changed, 270 insertions(+), 27 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 90c3a675d..002ad2f38 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -7,6 +7,7 @@ import json import os import re +import threading import time import urllib.error import urllib.request @@ -24,6 +25,8 @@ STATEFUL_SITES = ("shopping_admin", "shopping", "reddit", "gitlab") SITE_PORTS = {"shopping_admin": 80, "shopping": 80, "reddit": 80, "gitlab": 8023} _SAFE_ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") +_LOCAL_POOL_LOCKS: dict[Path, threading.Lock] = {} +_LOCAL_POOL_LOCKS_GUARD = threading.Lock() def _canonical_sha256(value: Any) -> str: @@ -53,6 +56,16 @@ def _atomic_json(path: Path, value: Any) -> None: temporary.replace(path) +def _attempt_cleanup(original: BaseException, description: str, action: Any) -> bool: + """Run cleanup without hiding the failure that made cleanup necessary.""" + try: + action() + except BaseException as cleanup_error: + original.add_note(f"cleanup failed ({description}): {cleanup_error}") + return False + return True + + class RainbondAPI: """Small OpenAPI client containing only the calls required by slice lifecycle.""" @@ -364,7 +377,9 @@ def _configure_component( ) expected_image = str(self.image_lock["images"][site]["pinned"]) actual_image = str(component.get("image") or "") - if actual_image and _image_digest(actual_image) != _image_digest(expected_image): + if not actual_image: + raise RuntimeError(f"Rainbond copied {site} component has no image in response") + if _image_digest(actual_image) != _image_digest(expected_image): raise RuntimeError( f"Rainbond copied {site} image mismatch: {actual_image!r} != {expected_image!r}" ) @@ -657,8 +672,12 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An for component in components.values(): self._wait_site(str(component["site_url"])) timings["readiness_seconds"] = round(time.monotonic() - phase_started, 3) - except BaseException: - self.api.request("DELETE", f"{self._app_path(app_id)}?force=1") + except BaseException as exc: + _attempt_cleanup( + exc, + f"delete incomplete Rainbond app {app_id}", + lambda: self.api.request("DELETE", f"{self._app_path(app_id)}?force=1"), + ) raise receipt = { "schema": RECEIPT_SCHEMA, @@ -697,17 +716,36 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: self._wait_app_stopped(previous) try: replacement = self.provision(pool_id, slot, replacement_path) - except BaseException: + except BaseException as exc: if stopped_previous: - self._restore_previous_app(previous) + _attempt_cleanup( + exc, + f"restart previous Rainbond app {previous['app_id']}", + lambda: self._restore_previous_app(previous), + ) raise try: self.api.request("DELETE", f"{self._app_path(int(previous['app_id']))}?force=1") - except BaseException: - self.api.request("DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1") - replacement_path.unlink(missing_ok=True) - if stopped_previous: - self._restore_previous_app(previous) + except BaseException as exc: + replacement_deleted = _attempt_cleanup( + exc, + f"delete replacement Rainbond app {replacement['app_id']}", + lambda: self.api.request( + "DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1" + ), + ) + if replacement_deleted: + _attempt_cleanup( + exc, + f"remove replacement receipt {replacement_path}", + lambda: replacement_path.unlink(missing_ok=True), + ) + if stopped_previous and replacement_deleted: + _attempt_cleanup( + exc, + f"restart previous Rainbond app {previous['app_id']}", + lambda: self._restore_previous_app(previous), + ) raise replacement_path.replace(receipt_path) return replacement @@ -748,8 +786,17 @@ def create_slot(slot: int) -> Path: receipt_path = slot_dir / "slice-receipt.json" if receipt_path.exists(): raise FileExistsError(f"Rainbond slice already exists: {receipt_path}") - receipt = pool.provision(pool_id, slot, receipt_path) - _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + try: + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + except BaseException as exc: + if receipt_path.exists(): + _attempt_cleanup( + exc, + f"destroy incomplete slot {slot}", + lambda: pool.destroy(receipt_path), + ) + raise return slot_dir try: @@ -764,23 +811,31 @@ def create_slot(slot: int) -> Path: first_error = first_error or exc if first_error is not None: raise first_error - except BaseException: + except BaseException as exc: for slot_dir in reversed(created): - pool.destroy(slot_dir / "slice-receipt.json") + _attempt_cleanup( + exc, + f"destroy rolled-back slot {slot_dir.name}", + lambda slot_dir=slot_dir: pool.destroy(slot_dir / "slice-receipt.json"), + ) raise return sorted(created) @contextmanager def _pool_lock(output_dir: Path) -> Any: - output_dir.mkdir(parents=True, exist_ok=True) - lock_path = output_dir / ".pool.lock" - with lock_path.open("a+", encoding="utf-8") as handle: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX) - try: - yield - finally: - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + output_dir = output_dir.resolve() + with _LOCAL_POOL_LOCKS_GUARD: + local_lock = _LOCAL_POOL_LOCKS.setdefault(output_dir, threading.Lock()) + with local_lock: + output_dir.mkdir(parents=True, exist_ok=True) + lock_path = output_dir / ".pool.lock" + with lock_path.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) def scale_pool( @@ -809,15 +864,39 @@ def scale_pool( def create_slot(slot: int) -> tuple[int, Path]: slot_dir = output_dir / f"slice-{slot}" - receipt = pool.provision(pool_id, slot, slot_dir / "slice-receipt.json") - _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + receipt_path = slot_dir / "slice-receipt.json" + try: + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + except BaseException as exc: + if receipt_path.exists(): + _attempt_cleanup( + exc, + f"destroy incomplete slot {slot}", + lambda: pool.destroy(receipt_path), + ) + raise return slot, slot_dir + created: dict[int, Path] = {} + first_error: BaseException | None = None with ThreadPoolExecutor(max_workers=workers) as executor: futures = [executor.submit(create_slot, slot) for slot in missing_slots] for future in as_completed(futures): - slot, slot_dir = future.result() - existing[slot] = slot_dir + try: + slot, slot_dir = future.result() + created[slot] = slot_dir + except BaseException as exc: + first_error = first_error or exc + if first_error is not None: + for slot, slot_dir in sorted(created.items(), reverse=True): + _attempt_cleanup( + first_error, + f"destroy rolled-back slot {slot}", + lambda slot_dir=slot_dir: pool.destroy(slot_dir / "slice-receipt.json"), + ) + raise first_error + existing.update(created) excess_slots = sorted((value for value in existing if value > count), reverse=True) leased_excess = [ existing[slot] @@ -868,7 +947,12 @@ def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: if not owner: raise ValueError("lease owner is required") with _pool_lock(output_dir): - for slot_dir in sorted(output_dir.glob("slice-*")): + + def slot_sort_key(path: Path) -> tuple[int, int | str]: + suffix = path.name.removeprefix("slice-") + return (0, int(suffix)) if suffix.isdigit() else (1, path.name) + + for slot_dir in sorted(output_dir.glob("slice-*"), key=slot_sort_key): receipt_path = slot_dir / "slice-receipt.json" config_path = slot_dir / "webarena.config.json" lease_path = slot_dir / "slice-lease.json" diff --git a/scripts/run_webarena_qualification.py b/scripts/run_webarena_qualification.py index 4efcf0a89..cb64448da 100644 --- a/scripts/run_webarena_qualification.py +++ b/scripts/run_webarena_qualification.py @@ -428,6 +428,7 @@ def run_batch( for shard in batch ] + # Extra ready slices are allowed; only a shortage is an error. assigned = tuple(zip(wave, config_paths, strict=False)) if len(assigned) != len(wave): raise RuntimeError("parallel wave exceeds isolated WebArena slice count") diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 689bbeabd..0254fed24 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -229,6 +230,39 @@ def request(self, method, path, body=None): assert any(method == "DELETE" for method, _path, _body in api.calls) +def test_provision_rejects_copied_site_without_image_identity(tmp_path): + class MissingImageAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + response["services"][0].pop("image") + return response + + api = MissingImageAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="has no image in response"): + pool.provision("run-image", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + +def test_provision_cleanup_does_not_hide_original_failure(tmp_path): + class CleanupFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "DELETE" and path.endswith("?force=1"): + raise RuntimeError("cleanup delete failed") + return super().request(method, path, body) + + api = CleanupFailureAPI(fail_domain_for="reddit") + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="injected domain failure") as caught: + pool.provision("run-cleanup", 1, tmp_path / "slice-receipt.json") + + assert any("cleanup delete failed" in note for note in caught.value.__notes__) + + def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): api = FakeAPI() pool = _pool(tmp_path, api) @@ -323,6 +357,28 @@ def test_capacity_safe_reset_restores_old_app_and_keeps_receipt_on_failure(tmp_p assert not receipt_path.with_name("slice-receipt.replacement.json").exists() +def test_reset_cleanup_does_not_hide_original_delete_failure(tmp_path): + class DeleteFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "DELETE" and path.endswith("/200?force=1"): + raise RuntimeError("old app delete failed") + if method == "DELETE" and path.endswith("/201?force=1"): + raise RuntimeError("replacement cleanup failed") + return super().request(method, path, body) + + api = DeleteFailureAPI() + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-delete", 1, receipt_path) + + with pytest.raises(RuntimeError, match="old app delete failed") as caught: + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + assert receipt_path.with_name("slice-receipt.replacement.json").exists() + assert any("replacement cleanup failed" in note for note in caught.value.__notes__) + + def test_pool_config_rejects_unknown_reset_strategy(tmp_path): with pytest.raises(ValueError, match="reset_strategy"): RainbondSlicePool( @@ -373,6 +429,39 @@ def destroy(self, receipt_path): assert destroyed == [tmp_path / "out" / "slice-1" / "slice-receipt.json"] +def test_provision_pool_preserves_failure_when_rollback_cleanup_fails(tmp_path, monkeypatch): + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, _pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("provision failed") + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text("{}") + return {"slot": slot} + + def runtime_config(self, _receipt): + return {"ok": True} + + def destroy(self, _receipt_path): + raise RuntimeError("rollback failed") + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + + with pytest.raises(RuntimeError, match="provision failed") as caught: + provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) + + assert any("rollback failed" in note for note in caught.value.__notes__) + + def test_scale_pool_keeps_one_warm_slice_and_removes_only_excess(tmp_path, monkeypatch): created = [] destroyed = [] @@ -423,6 +512,48 @@ def destroy(self, receipt_path): scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=1) +def test_scale_pool_rolls_back_every_new_slot_after_partial_failure(tmp_path, monkeypatch): + destroyed = [] + + class Config: + max_parallel_provisions = 3 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("slot 2 failed") + receipt = {"pool_id": pool_id, "slot": slot} + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, _receipt): + return {"ok": True} + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path.parent.name) + receipt_path.unlink() + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + output = tmp_path / "runtime" + + with pytest.raises(RuntimeError, match="slot 2 failed"): + scale_pool(tmp_path / "pool.json", output, "warm", 3) + + assert sorted(destroyed) == ["slice-1", "slice-3"] + assert not list(output.glob("slice-*/slice-receipt.json")) + + def test_capacity_keeps_idle_slice_in_addition_to_active_demand(tmp_path, monkeypatch): calls = [] @@ -506,6 +637,33 @@ def test_acquire_grants_distinct_slices_and_exhausts_pool(tmp_path): acquire_slice(output, "task-c") +def test_acquire_uses_numeric_slot_order(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 10, 110) + _ready_slot(output, 2, 102) + + lease = acquire_slice(output, "task-a") + + assert lease["slot"] == 2 + + +def test_concurrent_thread_acquires_never_share_a_slice(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + + with ThreadPoolExecutor(max_workers=2) as executor: + leases = list( + executor.map( + lambda owner: acquire_slice(output, owner), + ("task-a", "task-b"), + ) + ) + + assert {lease["slot"] for lease in leases} == {1, 2} + assert len({lease["lease_id"] for lease in leases}) == 2 + + def test_release_replaces_dirty_generation_before_unlocking_slice(tmp_path, monkeypatch): output = tmp_path / "runtime" slot_dir = _ready_slot(output, 1, 101) From b99424ec8d14fe0688e02bc997b136def3ba4f75 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 15:40:58 +0800 Subject: [PATCH 14/23] Complete Rainbond pool cleanup coverage --- .../public_webarena/rainbond_slices.py | 23 ++++++-- scripts/manage_webarena_rainbond_pool.py | 30 ++++++++-- .../public_webarena/test_rainbond_slices.py | 56 +++++++++++++++++++ 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 002ad2f38..0a3820bb4 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -454,9 +454,9 @@ def _configure_initializer( actual_image = str(initializer.get("image") or "") if not service_id: raise RuntimeError("Rainbond copied initializer has no service_id") - if actual_image and _image_digest(actual_image) != _image_digest( - self.config.initializer_image - ): + if not actual_image: + raise RuntimeError("Rainbond copied initializer component has no image in response") + if _image_digest(actual_image) != _image_digest(self.config.initializer_image): raise RuntimeError("Rainbond copied initializer image mismatch") aliases = { site: self._internal_service_host(app_id, copied_sites[site], 8877) @@ -908,9 +908,20 @@ def create_slot(slot: int) -> tuple[int, Path]: "cannot remove leased Rainbond slices: " + ", ".join(str(path) for path in leased_excess) ) + first_destroy_error: BaseException | None = None for slot in excess_slots: - slot_dir = existing.pop(slot) - pool.destroy(slot_dir / "slice-receipt.json") + slot_dir = existing[slot] + try: + pool.destroy(slot_dir / "slice-receipt.json") + except BaseException as exc: + if first_destroy_error is None: + first_destroy_error = exc + else: + first_destroy_error.add_note( + f"additional scale-down failure for slot {slot}: {exc}" + ) + continue + existing.pop(slot) config_file = slot_dir / "webarena.config.json" if config_file.exists(): config_file.unlink() @@ -918,6 +929,8 @@ def create_slot(slot: int) -> tuple[int, Path]: slot_dir.rmdir() except OSError: pass + if first_destroy_error is not None: + raise first_destroy_error return [existing[slot] for slot in sorted(existing)] diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py index b4d956eb3..4def6b495 100644 --- a/scripts/manage_webarena_rainbond_pool.py +++ b/scripts/manage_webarena_rainbond_pool.py @@ -53,12 +53,18 @@ def main() -> None: if args.command == "scale": if args.active is not None: paths = reconcile_pool_capacity( - args.config.resolve(), args.output.resolve(), args.pool_id, args.active, + args.config.resolve(), + args.output.resolve(), + args.pool_id, + args.active, minimum_ready=args.minimum_ready, ) else: paths = scale_pool( - args.config.resolve(), args.output.resolve(), args.pool_id, args.count, + args.config.resolve(), + args.output.resolve(), + args.pool_id, + args.count, minimum_ready=args.minimum_ready, ) print(json.dumps([str(path / "webarena.config.json") for path in paths])) @@ -69,9 +75,12 @@ def main() -> None: return if args.command == "release": - print(json.dumps(release_slice( - args.config.resolve(), args.output.resolve(), args.lease_id - ), sort_keys=True)) + print( + json.dumps( + release_slice(args.config.resolve(), args.output.resolve(), args.lease_id), + sort_keys=True, + ) + ) return pool = RainbondSlicePool(args.config.resolve()) @@ -84,8 +93,17 @@ def main() -> None: print(target) return + first_error: BaseException | None = None for slot_dir in args.slice_dir: - pool.destroy(slot_dir.resolve() / "slice-receipt.json") + try: + pool.destroy(slot_dir.resolve() / "slice-receipt.json") + except BaseException as exc: + if first_error is None: + first_error = exc + else: + first_error.add_note(f"additional destroy failure for {slot_dir}: {exc}") + if first_error is not None: + raise first_error if __name__ == "__main__": diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 0254fed24..8ccd53e0a 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -247,6 +247,23 @@ def request(self, method, path, body=None): assert any(method == "DELETE" for method, _path, _body in api.calls) +def test_provision_rejects_initializer_without_image_identity(tmp_path): + class MissingInitializerImageAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + response["services"][-1].pop("image") + return response + + api = MissingInitializerImageAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="initializer component has no image"): + pool.provision("run-initializer-image", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + def test_provision_cleanup_does_not_hide_original_failure(tmp_path): class CleanupFailureAPI(FakeAPI): def request(self, method, path, body=None): @@ -606,6 +623,45 @@ def destroy(self, receipt_path): assert destroyed == [] +def test_scale_down_continues_after_one_destroy_failure(tmp_path, monkeypatch): + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + slot = int(receipt_path.parent.name.removeprefix("slice-")) + destroyed.append(slot) + if slot == 3: + raise RuntimeError("slot 3 delete failed") + receipt_path.unlink() + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + output = tmp_path / "runtime" + for slot in (1, 2, 3): + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text(json.dumps({"pool_id": "warm", "slot": slot})) + + with pytest.raises(RuntimeError, match="slot 3 delete failed"): + scale_pool(tmp_path / "pool.json", output, "warm", 1) + + assert destroyed == [3, 2] + assert (output / "slice-3" / "slice-receipt.json").exists() + assert not (output / "slice-2" / "slice-receipt.json").exists() + + def _ready_slot(output: Path, slot: int, app_id: int) -> Path: slot_dir = output / f"slice-{slot}" slot_dir.mkdir(parents=True) From 780f9d7c7231c3812ec83e99a104bfb9a36586e8 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 15:47:50 +0800 Subject: [PATCH 15/23] Persist Rainbond reset runtime state --- .../public_webarena/rainbond_slices.py | 14 ++++++++++- .../public_webarena/webarena_environment.py | 13 ++++++++--- .../test_webarena_environment.py | 23 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 0a3820bb4..4c02c7037 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -387,7 +387,7 @@ def _configure_component( "service_id": service_id, "service_cname": component.get("service_cname"), "image": expected_image, - "runtime_image": actual_image or expected_image, + "runtime_image": actual_image, "site_url": site_url, } @@ -774,6 +774,18 @@ def runtime_config(self, receipt: dict[str, Any]) -> dict[str, Any]: }, } + def persist_runtime_config( + self, + receipt: dict[str, Any], + path: Path, + *, + base_config: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Atomically persist receipt-authoritative URLs for crash recovery.""" + value = {**(base_config or {}), **self.runtime_config(receipt)} + _atomic_json(path, value) + return value + def provision_pool(config_path: Path, output_dir: Path, pool_id: str, count: int) -> list[Path]: if count < 1: diff --git a/harness/trajectory_replay/public_webarena/webarena_environment.py b/harness/trajectory_replay/public_webarena/webarena_environment.py index d32ffa293..dbc9a86b0 100644 --- a/harness/trajectory_replay/public_webarena/webarena_environment.py +++ b/harness/trajectory_replay/public_webarena/webarena_environment.py @@ -215,6 +215,10 @@ def __init__(self, config_path: str | Path) -> None: else (self.config_path.parent / receipt_value).resolve() ) self.rainbond_pool = RainbondSlicePool(pool_path) + current_receipt = self.rainbond_pool.validate_receipt(self.rainbond_receipt_path) + current_runtime = self.rainbond_pool.runtime_config(current_receipt) + self.config = {**self.config, **current_runtime} + self.environments = self.config["environments"] def site_docker_client(self, default_client: Any) -> tuple[Any, bool]: host = self.config.get("site_docker_host") @@ -461,9 +465,12 @@ def _reset_impl( if self.rainbond_pool is not None and any(site in STATEFUL_SITES for site in sites): assert self.rainbond_receipt_path is not None receipt = self.rainbond_pool.reset(self.rainbond_receipt_path) - for site, component in receipt["components"].items(): - token = f"__{site.upper()}__" - self.environments[token] = {"urls": [component["site_url"]]} + self.config = self.rainbond_pool.persist_runtime_config( + receipt, + self.config_path, + base_config=self.config, + ) + self.environments = self.config["environments"] for site in sites: site_config = self._site_config(site) site_urls = site_config.get("urls") or [] diff --git a/tests/trajectory_replay/public_webarena/test_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_webarena_environment.py index 24328cd07..e14dcd63d 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_environment.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_environment.py @@ -167,6 +167,25 @@ def validate_receipt(self, path): assert path == receipt_path return json.loads(path.read_text()) + def runtime_config(self, receipt): + return { + "image_lock": str(lock), + "environments": { + "__SHOPPING_ADMIN__": { + "urls": [receipt["components"]["shopping_admin"]["site_url"]] + } + }, + "rainbond_slice": { + "pool_config": str(tmp_path / "pool.json"), + "receipt": "slice-receipt.json", + }, + } + + def persist_runtime_config(self, receipt, path, *, base_config): + value = {**base_config, **self.runtime_config(receipt)} + path.write_text(json.dumps(value), encoding="utf-8") + return value + def reset(self, path): resets.append(path) return { @@ -224,6 +243,10 @@ def reset(self, path): assert environment.environments["__SHOPPING_ADMIN__"]["urls"] == [ "https://admin-slice-302.example.test" ] + persisted = json.loads(config.read_text()) + assert persisted["environments"]["__SHOPPING_ADMIN__"]["urls"] == [ + "https://admin-slice-302.example.test" + ] assert resets == [receipt_path] From e69b68b81b04bf245fc9e61ea2d347e9063da70b Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 16:07:47 +0800 Subject: [PATCH 16/23] Reclaim failed Rainbond slices --- .../public_webarena/README.md | 35 ++++++------- .../rainbond-pool.example.json | 2 +- .../public_webarena/rainbond_slices.py | 26 ++++------ scripts/manage_webarena_rainbond_pool.py | 3 +- tests/test_manage_webarena_rainbond_pool.py | 49 +++++++++++++++++++ .../public_webarena/test_rainbond_slices.py | 27 ++++++++-- 6 files changed, 101 insertions(+), 41 deletions(-) create mode 100644 tests/test_manage_webarena_rainbond_pool.py diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index cec4f1a6b..66f3d6066 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -112,19 +112,20 @@ uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ Use the returned config for exactly one task. Releasing a lease creates and verifies a clean replacement application before deleting the dirty generation and making the slot -available again. If replacement fails, the lease remains held and cannot be reused: +available again. If replacement fails, the lease remains held, the dirty generation is +stopped, and the slot cannot be reused: ```text uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ --config release --output --lease-id ``` -Set `reset_strategy` to `stop_before_replace` when the cluster has capacity for only one -slice. The controller keeps the lease held, stops the dirty application to release its -CPU and memory, creates and verifies the replacement, and only then deletes the stopped -generation. If provisioning fails, it restarts and verifies the previous generation while -keeping the lease held. Use `replace_before_destroy` only when the cluster has enough -surge capacity for two complete slices during every reset. +The default `replace_before_destroy` strategy allows the dirty and replacement generations +to overlap while the replacement becomes ready. This preserves available capacity during +normal concurrent operation without imposing a fixed pool size. `stop_before_replace` is +an operator-selected fallback for a temporarily constrained cluster: it stops the dirty +generation before creating its replacement. Either way, a failed reset leaves the lease +held and the dirty generation stopped rather than returning an invalid slot to the pool. For a one-shot pool that will be destroyed in full, use: @@ -150,15 +151,11 @@ ready slice rather than waiting for a cold start. The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus 100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable -resources, not cluster totals. A pool serving `N` active tasks with one idle warm slice -therefore needs at least `(N + 1) * 10.1` cores and `(N + 1) * 20.125 GiB`, with additional -headroom for Rainbond's rolling-update surge Pods and unrelated workloads. If that headroom -is unavailable, replacements remain pending instead of weakening isolation or silently -sharing a site. - -Rainbond remains the runtime data plane and this repository's lease controller remains the -source of truth for lifecycle and reset. Kestra is optional above it for queues, schedules, -timeouts, retries, and guaranteed cleanup; it is not required to allocate a slice and must -not become a runtime dependency. Argo Workflows is only justified when the deployment moves -to direct Kubernetes-native orchestration at substantially larger scale, because using it -for the current pool would duplicate Rainbond application lifecycle management. +resources, not cluster totals. Reconcile the pool to current active demand plus the chosen +idle reserve; the controller destroys excess unleased slices when demand falls. Rainbond +may also need temporary headroom while a replacement generation starts and for rolling +updates. When that headroom is unavailable, use `stop_before_replace` for that deployment +rather than weakening isolation or sharing writable sites. + +Rainbond is the runtime data plane and this repository's lease controller is the source of +truth for allocation, reset, quarantine, and cleanup. diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json index c95ff57e1..c9f3f87c5 100644 --- a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -26,7 +26,7 @@ "image_lock": "../../../../evaluation/browser-use-public/webarena-environment-images.json", "readiness_timeout_seconds": 1800, "max_parallel_provisions": 2, - "reset_strategy": "stop_before_replace", + "reset_strategy": "replace_before_destroy", "shared_environments": { "__MAP__": { "urls": ["https://replace-with-shared-read-only-map"] diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index 4c02c7037..e1d0a8b0f 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -614,14 +614,9 @@ def _wait_app_stopped(self, receipt: dict[str, Any]) -> None: f"Rainbond app did not stop before replacement: {app_id}: {last_statuses}" ) - def _restore_previous_app(self, receipt: dict[str, Any]) -> None: - self._operate_existing_app(receipt, "start") - self._wait_component_running( - int(receipt["app_id"]), str(receipt["initializer"]["service_id"]) - ) - self._wait_site(str(receipt["initializer"]["readiness_url"])) - for component in receipt["components"].values(): - self._wait_site(str(component["site_url"])) + def _stop_existing_app(self, receipt: dict[str, Any]) -> None: + self._operate_existing_app(receipt, "stop") + self._wait_app_stopped(receipt) def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: started = time.monotonic() @@ -712,16 +707,15 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: raise RuntimeError(f"unfinished Rainbond slice replacement: {replacement_path}") stopped_previous = self.config.reset_strategy == "stop_before_replace" if stopped_previous: - self._operate_existing_app(previous, "stop") - self._wait_app_stopped(previous) + self._stop_existing_app(previous) try: replacement = self.provision(pool_id, slot, replacement_path) except BaseException as exc: - if stopped_previous: + if not stopped_previous: _attempt_cleanup( exc, - f"restart previous Rainbond app {previous['app_id']}", - lambda: self._restore_previous_app(previous), + f"stop failed Rainbond slice {previous['app_id']}", + lambda: self._stop_existing_app(previous), ) raise try: @@ -740,11 +734,11 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: f"remove replacement receipt {replacement_path}", lambda: replacement_path.unlink(missing_ok=True), ) - if stopped_previous and replacement_deleted: + if not stopped_previous: _attempt_cleanup( exc, - f"restart previous Rainbond app {previous['app_id']}", - lambda: self._restore_previous_app(previous), + f"stop failed Rainbond slice {previous['app_id']}", + lambda: self._stop_existing_app(previous), ) raise replacement_path.replace(receipt_path) diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py index 4def6b495..80671ad8d 100644 --- a/scripts/manage_webarena_rainbond_pool.py +++ b/scripts/manage_webarena_rainbond_pool.py @@ -87,9 +87,8 @@ def main() -> None: if args.command == "reset": slot_dir = args.slice_dir.resolve() receipt = pool.reset(slot_dir / "slice-receipt.json") - config = pool.runtime_config(receipt) target = slot_dir / "webarena.config.json" - target.write_text(json.dumps(config, indent=2, sort_keys=True) + "\n", encoding="utf-8") + pool.persist_runtime_config(receipt, target) print(target) return diff --git a/tests/test_manage_webarena_rainbond_pool.py b/tests/test_manage_webarena_rainbond_pool.py new file mode 100644 index 000000000..19d659281 --- /dev/null +++ b/tests/test_manage_webarena_rainbond_pool.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "manage_webarena_rainbond_pool.py" +SPEC = importlib.util.spec_from_file_location("manage_webarena_rainbond_pool", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +manage_webarena_rainbond_pool = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(manage_webarena_rainbond_pool) + + +def test_reset_persists_runtime_config_atomically(tmp_path, monkeypatch, capsys): + config_path = tmp_path / "pool.json" + slot_dir = tmp_path / "slice-1" + slot_dir.mkdir() + calls = [] + + class Pool: + def __init__(self, config): + assert config == config_path.resolve() + + def reset(self, receipt_path): + assert receipt_path == slot_dir.resolve() / "slice-receipt.json" + return {"app_id": 201} + + def persist_runtime_config(self, receipt, target): + calls.append((receipt, target)) + + monkeypatch.setattr(manage_webarena_rainbond_pool, "RainbondSlicePool", Pool) + monkeypatch.setattr( + sys, + "argv", + [ + "manage_webarena_rainbond_pool.py", + "--config", + str(config_path), + "reset", + "--slice-dir", + str(slot_dir), + ], + ) + + manage_webarena_rainbond_pool.main() + + target = slot_dir.resolve() / "webarena.config.json" + assert calls == [({"app_id": 201}, target)] + assert capsys.readouterr().out.strip() == str(target) diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 8ccd53e0a..2aab28c51 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -347,7 +347,7 @@ def test_capacity_safe_reset_stops_old_app_before_creating_replacement(tmp_path) assert second["app_id"] != first["app_id"] -def test_capacity_safe_reset_restores_old_app_and_keeps_receipt_on_failure(tmp_path): +def test_stop_before_replace_keeps_old_app_stopped_and_receipt_on_failure(tmp_path): api = FakeAPI() pool = RainbondSlicePool( _pool_config(tmp_path, reset_strategy="stop_before_replace"), @@ -356,7 +356,6 @@ def test_capacity_safe_reset_restores_old_app_and_keeps_receipt_on_failure(tmp_p ) pool._wait_site = lambda _url: None pool._wait_app_stopped = lambda _receipt: None - pool._wait_component_running = lambda _app_id, _service_id: None receipt_path = tmp_path / "slice-receipt.json" first = pool.provision("run-recovery", 1, receipt_path) api.fail_domain_for = "reddit" @@ -370,7 +369,28 @@ def test_capacity_safe_reset_restores_old_app_and_keeps_receipt_on_failure(tmp_p for method, path, body in api.calls if method == "POST" and path.endswith(f"/{first['app_id']}/operations") ] - assert operations[-2:] == ["stop", "start"] + assert operations[-1:] == ["stop"] + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() + + +def test_default_reset_stops_dirty_app_and_keeps_receipt_on_failure(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-quarantine", 1, receipt_path) + api.fail_domain_for = "reddit" + + with pytest.raises(RuntimeError, match="injected domain failure"): + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + operations = [ + body["action"] + for method, path, body in api.calls + if method == "POST" and path.endswith(f"/{first['app_id']}/operations") + ] + assert operations[-1:] == ["stop"] assert not receipt_path.with_name("slice-receipt.replacement.json").exists() @@ -385,6 +405,7 @@ def request(self, method, path, body=None): api = DeleteFailureAPI() pool = _pool(tmp_path, api) + pool._wait_app_stopped = lambda _receipt: None receipt_path = tmp_path / "slice-receipt.json" first = pool.provision("run-delete", 1, receipt_path) From 0acb965ef5cfce3ef7c9e45f185c4786aef01ee1 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 16 Sep 2026 21:23:29 +0800 Subject: [PATCH 17/23] Add live Rainbond slice smoke test --- .../public_webarena/README.md | 16 ++++ pyproject.toml | 2 +- .../test_rainbond_live_smoke.py | 86 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 66f3d6066..05c31f2a5 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -149,6 +149,22 @@ Each receipt records create, copy, configure, deploy-request, readiness, and tot durations. Use those measurements to size the warm pool; task admission should allocate a ready slice rather than waiting for a cold start. +After a slice becomes ready, exercise all four public origins concurrently: + +```text +WEBARENA_RAINBOND_SMOKE_URLS='{ + "shopping_admin":"http://", + "shopping":"http://", + "reddit":"http://", + "gitlab":"http://" +}' uv run --frozen python -m pytest \ + tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py \ + -m real_smoke -q -s +``` + +The smoke test makes eight simultaneous page loads per site, checks that each origin +returns its expected application, and rejects a per-site P95 above ten seconds. + The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus 100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable resources, not cluster totals. Reconcile the pool to current active demand plus the chosen diff --git a/pyproject.toml b/pyproject.toml index 933fc65fe..7ff25847a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ oss-eval = [ testpaths = ["tests"] addopts = ["--import-mode=importlib"] markers = [ - "real_smoke: starts real browser containers and accesses a public website", + "real_smoke: accesses real containers or live deployment endpoints", ] [tool.ruff] diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py new file mode 100644 index 000000000..1e7f23b32 --- /dev/null +++ b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py @@ -0,0 +1,86 @@ +"""Live HTTP smoke test for one isolated Rainbond WebArena slice. + +Set WEBARENA_RAINBOND_SMOKE_URLS to a JSON object containing the four site +URLs, then run this file with the real_smoke marker. The test sends requests +to every site concurrently so a green result covers the shared gateway path +as well as each site's basic page rendering. +""" + +from __future__ import annotations + +import json +import os +import statistics +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest + +SITES = { + "shopping_admin": "Home Page", + "shopping": "<title>One Stop Market", + "reddit": "<title>Postmill", + "gitlab": "<title>Sign in · GitLab", +} +REQUESTS_PER_SITE = 8 +REQUEST_TIMEOUT_SECONDS = 30 +P95_LIMIT_SECONDS = 10 + + +def _configured_urls() -> dict[str, str]: + raw = os.environ.get("WEBARENA_RAINBOND_SMOKE_URLS", "") + if not raw: + pytest.skip("WEBARENA_RAINBOND_SMOKE_URLS is not set") + value = json.loads(raw) + if not isinstance(value, dict) or set(value) != set(SITES): + raise AssertionError( + "WEBARENA_RAINBOND_SMOKE_URLS must contain exactly: " + + ", ".join(sorted(SITES)) + ) + urls = {site: str(url).rstrip("/") + "/" for site, url in value.items()} + if any(not url.startswith(("http://", "https://")) for url in urls.values()): + raise AssertionError("all Rainbond smoke URLs must use HTTP or HTTPS") + return urls + + +def _request(site: str, url: str) -> tuple[str, float, int]: + started = time.monotonic() + request = urllib.request.Request( + url, + headers={"User-Agent": "browser-eval-rainbond-smoke/1"}, + ) + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + body = response.read() + status = response.status + elapsed = time.monotonic() - started + assert status == 200, f"{site} returned HTTP {status}" + assert SITES[site].encode() in body, f"{site} returned the wrong page" + return site, elapsed, len(body) + + +@pytest.mark.real_smoke +def test_isolated_rainbond_slice_handles_concurrent_page_loads(): + urls = _configured_urls() + jobs = [ + (site, urls[site]) + for _ in range(REQUESTS_PER_SITE) + for site in sorted(SITES) + ] + results: list[tuple[str, float, int]] = [] + with ThreadPoolExecutor(max_workers=len(jobs)) as executor: + futures = [executor.submit(_request, site, url) for site, url in jobs] + for future in as_completed(futures): + results.append(future.result()) + + assert len(results) == len(SITES) * REQUESTS_PER_SITE + for site in sorted(SITES): + durations = sorted(elapsed for name, elapsed, _size in results if name == site) + sizes = [size for name, _elapsed, size in results if name == site] + p95 = durations[min(len(durations) - 1, int(len(durations) * 0.95))] + assert p95 < P95_LIMIT_SECONDS, f"{site} p95 was {p95:.3f}s" + assert min(sizes) > 1_000, f"{site} returned an unexpectedly small page" + print( + f"{site}: requests={len(durations)} " + f"median={statistics.median(durations):.3f}s p95={p95:.3f}s" + ) From 1db11615bc16fe9e4de0a145a979ce547f7e2441 Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 10:51:51 +0800 Subject: [PATCH 18/23] Guard Rainbond cleanup and verify concurrent slices --- .../public_webarena/README.md | 20 ++++- .../public_webarena/rainbond_slices.py | 55 +++++++++--- .../test_rainbond_live_smoke.py | 73 ++++++++++++---- .../public_webarena/test_rainbond_slices.py | 85 ++++++++++++++++++- 4 files changed, 202 insertions(+), 31 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index 05c31f2a5..dc680ca49 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -39,7 +39,8 @@ For concurrent state-changing runs, use a complete Rainbond slice per worker ins sharing sites or relying on separate login accounts. A slice contains Shopping Admin, Shopping, Reddit, and GitLab with independent writable container layers and unique domains. Reset deletes and recreates the whole slice from the pinned baseline; destroy -removes the application. The read-only Map service may be shared. +removes the application. Read-only Map and Wikipedia services may be shared when +separately deployed; they are different services, not substitutes for each other. The baseline also contains a small digest-pinned initializer component. It calls the Reddit and GitLab `env-ctrl` endpoints only over Rainbond's internal service network, @@ -164,6 +165,23 @@ WEBARENA_RAINBOND_SMOKE_URLS='{ The smoke test makes eight simultaneous page loads per site, checks that each origin returns its expected application, and rejects a per-site P95 above ten seconds. +To verify multiple slices in the same concurrency window, supply a JSON array of +four-site objects instead. Duplicate mutable-site URLs are rejected. Set +`WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL` to a shared Kiwix origin to add eight +simultaneous requests for its May 2022 Boston Celtics article. A green HTTP test +does not validate login, mutations, or database isolation; check those separately +before assigning state-changing tasks. + +Keep the stopped baseline application used by `source_app_id`, `source_services`, +and `initializer_service`: it is required for new slices and resets even though it +uses no running compute. Remove superseded slice generations only after their +replacement receipt is authoritative and no lease references the old app. Never +delete a shared Wikipedia data volume as part of a mutable slice reset. +Deletion first stops the app's services and waits for their stopped status. If stopping +fails, the app record and receipt remain available for recovery. After deletion, also +inspect Kubernetes-native workloads and storage: a missing Rainbond app record alone +does not prove that the platform removed every underlying resource. Never remove a +PVC without verifying its ownership and that no retained workload mounts it. The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus 100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index e1d0a8b0f..dfc08e0a9 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -433,13 +433,19 @@ def inner_ports() -> dict[int, dict[str, Any]]: "is_inner_service": True, }, ) - ports = inner_ports() - updated_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") - if updated_host != site_host: - raise RuntimeError( - f"Rainbond did not attach control port {control_port} to internal " - f"service {site_host!r}: got {updated_host!r}" - ) + # The write may finish before the read model reflects the recreated port. + # Retry only observation: repeating DELETE/POST could disrupt the service. + for attempt in range(3): + ports = inner_ports() + updated_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") + if updated_host == site_host: + return + if attempt < 2: + self.sleep(attempt + 1) + raise RuntimeError( + f"Rainbond did not attach control port {control_port} to internal " + f"service {site_host!r}: got {updated_host!r}" + ) def _configure_initializer( self, @@ -618,6 +624,31 @@ def _stop_existing_app(self, receipt: dict[str, Any]) -> None: self._operate_existing_app(receipt, "stop") self._wait_app_stopped(receipt) + def _delete_app(self, app_id: int) -> None: + """Stop runtime services before removing their control-plane app record.""" + path = self._app_path(app_id) + rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + service_ids = {str(row["service_id"]) for row in rows if row.get("service_id")} + if service_ids: + self.api.request( + "POST", f"{path}/operations", + {"action": "stop", "service_ids": sorted(service_ids)}, + ) + deadline = time.monotonic() + self.config.readiness_timeout_seconds + while True: + rows = self._response_rows( + self.api.request("GET", f"{path}/services"), "services" + ) + statuses = { + str(row.get("service_id")): row.get("status") for row in rows + } + if all(statuses.get(sid) in {"closed", "undeploy"} for sid in service_ids): + break + if time.monotonic() >= deadline: + raise TimeoutError(f"Rainbond app did not stop before deletion: {app_id}") + self.sleep(5) + self.api.request("DELETE", f"{path}?force=1") + def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: started = time.monotonic() phase_started = started @@ -671,7 +702,7 @@ def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, An _attempt_cleanup( exc, f"delete incomplete Rainbond app {app_id}", - lambda: self.api.request("DELETE", f"{self._app_path(app_id)}?force=1"), + lambda: self._delete_app(app_id), ) raise receipt = { @@ -695,7 +726,7 @@ def destroy(self, receipt_path: Path, *, missing_ok: bool = True) -> None: return raise FileNotFoundError(receipt_path) receipt = self.validate_receipt(receipt_path) - self.api.request("DELETE", f"{self._app_path(int(receipt['app_id']))}?force=1") + self._delete_app(int(receipt["app_id"])) receipt_path.unlink() def reset(self, receipt_path: Path) -> dict[str, Any]: @@ -719,14 +750,12 @@ def reset(self, receipt_path: Path) -> dict[str, Any]: ) raise try: - self.api.request("DELETE", f"{self._app_path(int(previous['app_id']))}?force=1") + self._delete_app(int(previous["app_id"])) except BaseException as exc: replacement_deleted = _attempt_cleanup( exc, f"delete replacement Rainbond app {replacement['app_id']}", - lambda: self.api.request( - "DELETE", f"{self._app_path(int(replacement['app_id']))}?force=1" - ), + lambda: self._delete_app(int(replacement["app_id"])), ) if replacement_deleted: _attempt_cleanup( diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py index 1e7f23b32..1e9bc1d19 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py @@ -1,7 +1,7 @@ """Live HTTP smoke test for one isolated Rainbond WebArena slice. -Set WEBARENA_RAINBOND_SMOKE_URLS to a JSON object containing the four site -URLs, then run this file with the real_smoke marker. The test sends requests +Set WEBARENA_RAINBOND_SMOKE_URLS to one four-site URL object or a list of +such objects to exercise multiple slices together. The test sends requests to every site concurrently so a green result covers the shared gateway path as well as each site's basic page rendering. """ @@ -24,6 +24,7 @@ "gitlab": "<title>Sign in · GitLab", } REQUESTS_PER_SITE = 8 +WIKIPEDIA_ARTICLE = "/content/wikipedia_en_all_maxi_2022-05/A/Boston_Celtics" REQUEST_TIMEOUT_SECONDS = 30 P95_LIMIT_SECONDS = 10 @@ -33,14 +34,21 @@ def _configured_urls() -> dict[str, str]: if not raw: pytest.skip("WEBARENA_RAINBOND_SMOKE_URLS is not set") value = json.loads(raw) - if not isinstance(value, dict) or set(value) != set(SITES): - raise AssertionError( - "WEBARENA_RAINBOND_SMOKE_URLS must contain exactly: " - + ", ".join(sorted(SITES)) - ) - urls = {site: str(url).rstrip("/") + "/" for site, url in value.items()} + slices = value if isinstance(value, list) else [value] + assert slices, "at least one slice is required" + urls = {} + for index, sites in enumerate(slices, start=1): + if not isinstance(sites, dict) or set(sites) != set(SITES): + raise AssertionError("each slice must contain exactly: " + ", ".join(sorted(SITES))) + for site, url in sites.items(): + urls[f"slice-{index}/{site}"] = str(url).rstrip("/") + "/" + assert len(set(urls.values())) == len(urls), "mutable sites must have distinct URLs" if any(not url.startswith(("http://", "https://")) for url in urls.values()): raise AssertionError("all Rainbond smoke URLs must use HTTP or HTTPS") + wikipedia = os.environ.get("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", "").rstrip("/") + if wikipedia: + assert wikipedia.startswith(("http://", "https://")), "Wikipedia must use HTTP or HTTPS" + urls["shared/wikipedia"] = wikipedia + WIKIPEDIA_ARTICLE return urls @@ -50,12 +58,19 @@ def _request(site: str, url: str) -> tuple[str, float, int]: url, headers={"User-Agent": "browser-eval-rainbond-smoke/1"}, ) - with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: - body = response.read() - status = response.status + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + body = response.read() + status = response.status + except Exception as exc: + raise AssertionError(f"{site} request failed: {type(exc).__name__}: {exc}") from exc elapsed = time.monotonic() - started assert status == 200, f"{site} returned HTTP {status}" - assert SITES[site].encode() in body, f"{site} returned the wrong page" + expected = ( + "<title>Boston Celtics" if site == "shared/wikipedia" + else SITES[site.rsplit("/", 1)[-1]] + ) + assert expected.encode() in body, f"{site} returned the wrong page" return site, elapsed, len(body) @@ -65,16 +80,21 @@ def test_isolated_rainbond_slice_handles_concurrent_page_loads(): jobs = [ (site, urls[site]) for _ in range(REQUESTS_PER_SITE) - for site in sorted(SITES) + for site in sorted(urls) ] results: list[tuple[str, float, int]] = [] + failures = [] with ThreadPoolExecutor(max_workers=len(jobs)) as executor: futures = [executor.submit(_request, site, url) for site, url in jobs] for future in as_completed(futures): - results.append(future.result()) + try: + results.append(future.result()) + except Exception as exc: + failures.append(str(exc)) - assert len(results) == len(SITES) * REQUESTS_PER_SITE - for site in sorted(SITES): + assert not failures, "\n".join(failures) + assert len(results) == len(urls) * REQUESTS_PER_SITE + for site in sorted(urls): durations = sorted(elapsed for name, elapsed, _size in results if name == site) sizes = [size for name, _elapsed, size in results if name == site] p95 = durations[min(len(durations) - 1, int(len(durations) * 0.95))] @@ -84,3 +104,24 @@ def test_isolated_rainbond_slice_handles_concurrent_page_loads(): f"{site}: requests={len(durations)} " f"median={statistics.median(durations):.3f}s p95={p95:.3f}s" ) + + +def test_smoke_configuration_accepts_multiple_isolated_slices(monkeypatch): + monkeypatch.delenv("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", raising=False) + slices = [{site: f"http://slot-{i}-{site}.example" for site in SITES} for i in (1, 2)] + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps(slices)) + assert len(_configured_urls()) == 8 + + +def test_smoke_configuration_includes_one_shared_wikipedia(monkeypatch): + sites = {site: f"http://{site}.example" for site in SITES} + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps(sites)) + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", "https://wiki.example/") + assert _configured_urls()["shared/wikipedia"] == "https://wiki.example" + WIKIPEDIA_ARTICLE + + +def test_smoke_configuration_rejects_shared_mutable_sites(monkeypatch): + sites = {site: f"http://{site}.example" for site in SITES} + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps([sites, sites])) + with pytest.raises(AssertionError, match="distinct URLs"): + _configured_urls() diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 2aab28c51..5ad74a72d 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -31,9 +31,12 @@ def __init__(self, *, fail_domain_for: str | None = None) -> None: self.next_app_id = 200 self.fail_domain_for = fail_domain_for self.port_hosts = {} + self.stopped_apps = set() def request(self, method, path, body=None): self.calls.append((method, path, body)) + if method == "POST" and path.endswith("/operations") and body["action"] == "stop": + self.stopped_apps.add(path.removesuffix("/operations")) if method == "POST" and path.endswith("/apps"): app_id = self.next_app_id self.next_app_id += 1 @@ -96,7 +99,10 @@ def request(self, method, path, body=None): return [ { "service_id": f"service-initializer-{self.next_app_id}", - "status": "running", + "status": ( + "closed" if path.removesuffix("/services") in self.stopped_apps + else "running" + ), } ] return {} @@ -145,6 +151,83 @@ def _pool(tmp_path: Path, api: FakeAPI) -> RainbondSlicePool: return pool +def test_app_deletion_waits_for_service_stop(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + pool._delete_app(200) + methods = [method for method, _, _ in api.calls] + assert methods == ["GET", "POST", "GET", "DELETE"] + assert api.calls[1][2]["action"] == "stop" + + +def test_app_deletion_keeps_record_when_stop_fails(tmp_path): + class StopFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "POST" and path.endswith("/operations"): + raise RuntimeError("stop failed") + return super().request(method, path, body) + + api = StopFailureAPI() + pool = _pool(tmp_path, api) + with pytest.raises(RuntimeError, match="stop failed"): + pool._delete_app(200) + assert not any(method == "DELETE" for method, _, _ in api.calls) + + +def test_app_deletion_does_not_trust_a_successful_stop_ack(tmp_path, monkeypatch): + class StillRunningAPI(FakeAPI): + def request(self, method, path, body=None): + result = super().request(method, path, body) + if method == "GET" and path.endswith("/services"): + result[0]["status"] = "running" + return result + + api = StillRunningAPI() + pool = _pool(tmp_path, api) + ticks = iter([0, 100_000]) + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.time.monotonic", + lambda: next(ticks), + ) + with pytest.raises(TimeoutError, match="did not stop before deletion"): + pool._delete_app(200) + assert not any(method == "DELETE" for method, _, _ in api.calls) + + +@pytest.mark.parametrize("stale_reads", [2, 4]) +def test_control_port_convergence_is_bounded_and_does_not_repeat_writes(tmp_path, stale_reads): + class EventuallyConsistentAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "GET" and path.endswith("/ports") and self.port_hosts: + if self.stale_reads: + self.stale_reads -= 1 + response["ports"][1]["k8s_service_name"] = "stale-control" + return response + + api = EventuallyConsistentAPI() + api.stale_reads = stale_reads + pool = _pool(tmp_path, api) + waits = [] + pool.sleep = waits.append + + def configure(): + pool._coalesce_internal_ports( + 200, "service-shopping", site_port=80, control_port=8877, + control_alias="SHOPPING_CTRL", + ) + + if stale_reads == 2: + configure() + else: + with pytest.raises(RuntimeError, match="did not attach control port"): + configure() + assert waits == [1, 2] + assert sum(method == "POST" for method, _, _ in api.calls) == 1 + assert sum(method == "DELETE" for method, _, _ in api.calls) == 1 + assert sum(method == "GET" for method, _, _ in api.calls) == 4 + + def test_provision_creates_one_complete_isolated_app_without_serializing_token(tmp_path): api = FakeAPI() pool = _pool(tmp_path, api) From 75b8fc20c4e3c4961ac3aac5becde30682d11076 Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 11:21:05 +0800 Subject: [PATCH 19/23] Make slice release retry idempotent --- .../public_webarena/rainbond_slices.py | 21 +++++++- .../public_webarena/test_rainbond_slices.py | 54 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index dfc08e0a9..bf37dabfc 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -1037,8 +1037,25 @@ def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[st lease_path, lease = matches[0] slot_dir = lease_path.parent pool = RainbondSlicePool(config_path) - receipt = pool.reset(slot_dir / "slice-receipt.json") - _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + receipt_path = slot_dir / "slice-receipt.json" + runtime_config_path = slot_dir / "webarena.config.json" + current_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if int(current_receipt["app_id"]) != int(lease["app_id"]): + current_receipt = pool.validate_receipt(receipt_path) + if ( + runtime_config_path.is_file() + and json.loads(runtime_config_path.read_text(encoding="utf-8")) + == pool.runtime_config(current_receipt) + ): + lease_path.unlink() + return { + "lease_id": lease_id, + "released_app_id": int(lease["app_id"]), + "ready_app_id": int(current_receipt["app_id"]), + "config": str(runtime_config_path.resolve()), + } + receipt = pool.reset(receipt_path) + _atomic_json(runtime_config_path, pool.runtime_config(receipt)) lease_path.unlink() return { "lease_id": lease_id, diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 5ad74a72d..d35562b60 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -879,3 +879,57 @@ def reset(self, _receipt_path): assert (slot_dir / "slice-lease.json").exists() with pytest.raises(RuntimeError, match="no unleased"): acquire_slice(output, "task-b") + + +def test_release_retry_only_unlinks_after_reset_was_persisted(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + lease_path = slot_dir / "slice-lease.json" + reset_calls = [] + + class Pool: + def __init__(self, _config): + pass + + def reset(self, receipt_path): + reset_calls.append(receipt_path) + receipt = { + "schema": RECEIPT_SCHEMA, + "slot": 1, + "app_id": 201, + "generation": "generation-new", + } + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool + ) + original_unlink = Path.unlink + failures = 0 + + def fail_first_lease_unlink(path, *args, **kwargs): + nonlocal failures + if path == lease_path and failures == 0: + failures += 1 + raise OSError("transient unlink failure") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_first_lease_unlink) + with pytest.raises(OSError, match="transient unlink failure"): + release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert lease_path.exists() + assert len(reset_calls) == 1 + + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert result["released_app_id"] == 101 + assert result["ready_app_id"] == 201 + assert not lease_path.exists() + assert len(reset_calls) == 1 From 4707864710fbcbca16016785bd98741ce0d4911f Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 11:46:20 +0800 Subject: [PATCH 20/23] Handle natural pool ordering and idempotent teardown --- .../public_webarena/rainbond_slices.py | 20 ++++++++++++------- .../public_webarena/test_rainbond_slices.py | 11 ++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/trajectory_replay/public_webarena/rainbond_slices.py index bf37dabfc..46006f678 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/trajectory_replay/public_webarena/rainbond_slices.py @@ -48,6 +48,11 @@ def _validate_pool_id(pool_id: str) -> str: return normalized +def _slot_sort_key(path: Path) -> tuple[int, int | str]: + suffix = path.name.removeprefix("slice-") + return (0, int(suffix)) if suffix.isdigit() else (1, path.name) + + def _atomic_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) content = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" @@ -627,7 +632,12 @@ def _stop_existing_app(self, receipt: dict[str, Any]) -> None: def _delete_app(self, app_id: int) -> None: """Stop runtime services before removing their control-plane app record.""" path = self._app_path(app_id) - rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + try: + rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + except RuntimeError as exc: + if isinstance(exc.__cause__, urllib.error.HTTPError) and exc.__cause__.code == 404: + return + raise service_ids = {str(row["service_id"]) for row in rows if row.get("service_id")} if service_ids: self.api.request( @@ -854,7 +864,7 @@ def create_slot(slot: int) -> Path: lambda slot_dir=slot_dir: pool.destroy(slot_dir / "slice-receipt.json"), ) raise - return sorted(created) + return sorted(created, key=_slot_sort_key) @contextmanager @@ -996,11 +1006,7 @@ def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: raise ValueError("lease owner is required") with _pool_lock(output_dir): - def slot_sort_key(path: Path) -> tuple[int, int | str]: - suffix = path.name.removeprefix("slice-") - return (0, int(suffix)) if suffix.isdigit() else (1, path.name) - - for slot_dir in sorted(output_dir.glob("slice-*"), key=slot_sort_key): + for slot_dir in sorted(output_dir.glob("slice-*"), key=_slot_sort_key): receipt_path = slot_dir / "slice-receipt.json" config_path = slot_dir / "webarena.config.json" lease_path = slot_dir / "slice-lease.json" diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index d35562b60..d4d491140 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import urllib.error from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -160,6 +161,16 @@ def test_app_deletion_waits_for_service_stop(tmp_path): assert api.calls[1][2]["action"] == "stop" +def test_app_deletion_is_idempotent_after_control_plane_record_is_gone(tmp_path): + class MissingAppAPI(FakeAPI): + def request(self, method, path, body=None): + error = urllib.error.HTTPError(path, 404, "Not Found", None, None) + raise RuntimeError("Rainbond OpenAPI returned HTTP 404") from error + + pool = _pool(tmp_path, MissingAppAPI()) + pool._delete_app(200) + + def test_app_deletion_keeps_record_when_stop_fails(tmp_path): class StopFailureAPI(FakeAPI): def request(self, method, path, body=None): From 0c485e52b3e46e65f23cf420a5a395961249cf45 Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 12:50:42 +0800 Subject: [PATCH 21/23] Consolidate WebArena environment lifecycle in browser-eval --- Makefile | 1 + deploy/webarena/LICENSE | 21 ++ deploy/webarena/README.md | 27 +++ deploy/webarena/runtime.example.json | 49 +++++ deploy/webarena/site-delivery-contract.json | 137 ++++++++++++ .../webarena-fixed-trajectory-reproduction.md | 42 ++-- .../webarena-env-contract.json | 19 -- .../rainbond-pool.example.json | 8 +- harness/environments/__init__.py | 1 + harness/environments/webarena/__init__.py | 1 + harness/environments/webarena/backend.py | 115 ++++++++++ harness/environments/webarena/cli.py | 88 ++++++++ harness/environments/webarena/config.py | 80 +++++++ harness/environments/webarena/manager.py | 204 ++++++++++++++++++ harness/environments/webarena/map_archives.py | 58 +++++ .../webarena/rainbond.py} | 25 ++- .../public_webarena/webarena_environment.py | 171 +-------------- pyproject.toml | 2 + scripts/manage_webarena_environment.py | 165 +------------- scripts/manage_webarena_rainbond_pool.py | 2 +- scripts/verify_webarena_env_contract.py | 158 ++++++-------- tests/environments/test_webarena_backend.py | 92 ++++++++ tests/environments/test_webarena_cli.py | 52 +++++ .../test_webarena_environment_config.py | 65 ++++++ .../test_webarena_environment_manager.py | 198 +++++++++++++++++ .../test_webarena_map_archives.py | 78 +++++++ .../test_manage_webarena_environment.py | 69 ------ .../public_webarena/test_rainbond_slices.py | 90 +++++--- .../test_webarena_env_contract.py | 196 ++++++----------- .../test_webarena_environment.py | 10 +- 30 files changed, 1508 insertions(+), 716 deletions(-) create mode 100644 deploy/webarena/LICENSE create mode 100644 deploy/webarena/README.md create mode 100644 deploy/webarena/runtime.example.json create mode 100644 deploy/webarena/site-delivery-contract.json delete mode 100644 evaluation/browser-use-public/webarena-env-contract.json create mode 100644 harness/environments/__init__.py create mode 100644 harness/environments/webarena/__init__.py create mode 100644 harness/environments/webarena/backend.py create mode 100644 harness/environments/webarena/cli.py create mode 100644 harness/environments/webarena/config.py create mode 100644 harness/environments/webarena/manager.py create mode 100644 harness/environments/webarena/map_archives.py rename harness/{trajectory_replay/public_webarena/rainbond_slices.py => environments/webarena/rainbond.py} (98%) create mode 100644 tests/environments/test_webarena_backend.py create mode 100644 tests/environments/test_webarena_cli.py create mode 100644 tests/environments/test_webarena_environment_config.py create mode 100644 tests/environments/test_webarena_environment_manager.py create mode 100644 tests/environments/test_webarena_map_archives.py delete mode 100644 tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py diff --git a/Makefile b/Makefile index 035bcc17e..ec53dd4a0 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ naming-check: runtime-check test-python: runtime-check $(TEST_PY) -m unittest discover -s tests -v $(TEST_PY) -m pytest tests/browser_use -q + $(TEST_PY) -m pytest tests/environments tests/test_manage_webarena_rainbond_pool.py -q $(TEST_PY) -m pytest tests/trajectory_replay -q test-ui: diff --git a/deploy/webarena/LICENSE b/deploy/webarena/LICENSE new file mode 100644 index 000000000..cfb4b00ca --- /dev/null +++ b/deploy/webarena/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LexMount + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deploy/webarena/README.md b/deploy/webarena/README.md new file mode 100644 index 000000000..23f762698 --- /dev/null +++ b/deploy/webarena/README.md @@ -0,0 +1,27 @@ +# WebArena 环境 + +Docker 和 Rainbond 环境由 `harness/environments/webarena/` 维护。任务和评分器来自 +固定版本的 `webarena-verified` 依赖。镜像身份只有一份： +`evaluation/browser-use-public/webarena-environment-images.json`。 + +`runtime.example.json` 提供本地 Docker 的主机和端口模板；`site-delivery-contract.json` +描述镜像内端口、初始化方式与只读 Map 数据挂载。Map 为可选环境，契约的存在不会触发下载或部署。 +有状态站点从镜像内的初始数据库重建，不挂载跨槽位共享的可写数据库目录。 +GitLab 在 Rainbond 中需要独立的 2 GiB `/dev/shm` 内存挂载。 + +复制运行配置到 Git 外并填写 `hostname`，将 `image_lock` 改为镜像锁绝对路径： + +```sh +uv run --frozen python scripts/manage_webarena_environment.py verify --config <配置> --site reddit +uv run --frozen python scripts/manage_webarena_environment.py start --config <配置> --site reddit --receipt <回执> +uv run --frozen python scripts/manage_webarena_environment.py reset --config <配置> --site reddit --receipt <回执> +uv run --frozen python scripts/verify_webarena_env_contract.py +``` + +Rainbond 的独立槽位、租约、并发访问检查和资源回收见 +[运行示例](../../examples/trajectory_replay/public_webarena/README.md)。 +运行配置、地址、凭据和回执保存在 Git 外；代码和配置模板可公开提交。 + +Docker 管理代码和 Map 校验契约源自 +[`lexmount/webarena-env`](https://github.com/lexmount/webarena-env/tree/daecdf8b46e71ccc70d7cc2fdaf92a83af899624)， +保留其 [MIT 许可证](LICENSE)。 diff --git a/deploy/webarena/runtime.example.json b/deploy/webarena/runtime.example.json new file mode 100644 index 000000000..d3d437da9 --- /dev/null +++ b/deploy/webarena/runtime.example.json @@ -0,0 +1,49 @@ +{ + "schema": "lexmount_webarena_runtime/1", + "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "hostname": "${HOST}", + "sites": { + "shopping_admin": { + "port": 7780, + "control_port": 7781 + }, + "shopping": { + "port": 7770, + "control_port": 7771 + }, + "gitlab": { + "port": 8023, + "control_port": 8024 + }, + "reddit": { + "port": 9999, + "control_port": 9998 + }, + "map": { + "port": 3000, + "control_port": 3001 + }, + "wikipedia": { + "port": 8888, + "control_port": 8889 + } + }, + "credentials_env": { + "shopping_admin": { + "username": "WEBARENA_SHOPPING_ADMIN_USERNAME", + "password": "WEBARENA_SHOPPING_ADMIN_PASSWORD" + }, + "shopping": { + "username": "WEBARENA_SHOPPING_USERNAME", + "password": "WEBARENA_SHOPPING_PASSWORD" + }, + "reddit": { + "username": "WEBARENA_REDDIT_USERNAME", + "password": "WEBARENA_REDDIT_PASSWORD" + }, + "gitlab": { + "username": "WEBARENA_GITLAB_USERNAME", + "password": "WEBARENA_GITLAB_PASSWORD" + } + } +} diff --git a/deploy/webarena/site-delivery-contract.json b/deploy/webarena/site-delivery-contract.json new file mode 100644 index 000000000..4461c18d7 --- /dev/null +++ b/deploy/webarena/site-delivery-contract.json @@ -0,0 +1,137 @@ +{ + "schema": "lexmount_webarena_site_delivery/1", + "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "webarena_verified_version": "1.2.3", + "components": { + "shopping_admin": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/", + "baseline": "fresh_container_writable_layer", + "init": "official_entrypoint", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_CMD_TIMEOUT_SEC" + ], + "required_env_values": { + "WA_ENV_CTRL_CMD_TIMEOUT_SEC": "600" + }, + "mounts": [] + }, + "shopping": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/customer/account/login", + "baseline": "fresh_container_writable_layer", + "init": "official_entrypoint", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_CMD_TIMEOUT_SEC" + ], + "required_env_values": { + "WA_ENV_CTRL_CMD_TIMEOUT_SEC": "600" + }, + "mounts": [] + }, + "reddit": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/login", + "baseline": "fresh_container_writable_layer", + "init": "official_explicit_env_ctrl", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL" + ], + "required_env_values": {}, + "mounts": [] + }, + "gitlab": { + "architecture": "amd64", + "service_port": 8023, + "control_port": 8877, + "health_path": "/users/sign_in", + "baseline": "fresh_container_writable_layer", + "init": "official_explicit_env_ctrl", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_SKIP_RECONFIGURE" + ], + "required_env_values": { + "WA_ENV_CTRL_SKIP_RECONFIGURE": "true" + }, + "mounts": [] + }, + "map": { + "architecture": "amd64", + "service_port": 8080, + "control_port": 8877, + "health_path": "/", + "baseline": "verified_read_only_dataset", + "init": "official_dataset_setup", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL" + ], + "required_env_values": {}, + "mounts": [ + { + "volume": "webarena_verified_map_tile_db", + "path": "/data/database" + }, + { + "volume": "webarena_verified_map_routing_car", + "path": "/data/routing/car" + }, + { + "volume": "webarena_verified_map_routing_bike", + "path": "/data/routing/bike" + }, + { + "volume": "webarena_verified_map_routing_foot", + "path": "/data/routing/foot" + }, + { + "volume": "webarena_verified_map_nominatim_db", + "path": "/data/nominatim/postgres" + }, + { + "volume": "webarena_verified_map_nominatim_flatnode", + "path": "/data/nominatim/flatnode" + }, + { + "volume": "webarena_verified_map_website_db", + "path": "/var/lib/postgresql/14/main" + }, + { + "volume": "webarena_verified_map_tiles", + "path": "/data/tiles" + }, + { + "volume": "webarena_verified_map_style", + "path": "/data/style" + } + ], + "data_urls": [ + "https://webarena-map-server-data.s3.amazonaws.com/osm_tile_server.tar", + "https://webarena-map-server-data.s3.amazonaws.com/nominatim_volumes.tar", + "https://webarena-map-server-data.s3.amazonaws.com/osrm_routing.tar" + ], + "data_archive_sha256": { + "osm_tile_server.tar": "2d802a915af14a13bcf393e9abd23d0de003eb043137736a54876410e05c4df2", + "nominatim_volumes.tar": "888ad53f754948f660796b6fcb440294a3dec647e141d606b1c7cb2616695baa", + "osrm_routing.tar": "99a4d10aae831fb3410b7d4743be03d9e02691283ec5105952839b84b9204806" + }, + "data_archive_bytes": { + "osm_tile_server.tar": 41280327680, + "nominatim_volumes.tar": 124774901760, + "osrm_routing.tar": 21278935040 + }, + "data_archive_digest_basis": "5090_downloaded_bytes_not_upstream_signature" + } + }, + "excluded_on_amd64": [ + "wikipedia" + ] +} diff --git a/docs/webarena-fixed-trajectory-reproduction.md b/docs/webarena-fixed-trajectory-reproduction.md index 58bc33531..9071a500a 100644 --- a/docs/webarena-fixed-trajectory-reproduction.md +++ b/docs/webarena-fixed-trajectory-reproduction.md @@ -1,17 +1,19 @@ -# WebArena 固定轨迹回放：双仓库复现契约 +# WebArena 固定轨迹回放：环境与复现 这是一套 RPA 式固定轨迹回放，不是 Agent browser-use。正式运行不调用 Agent、模型或 LLM judge；两个浏览器重放相同的 WebArena 公开人工轨迹，再由固定 evaluator 验收最终 回答、服务状态和 HAR。 -## 两个仓库各自负责什么 +## 代码与依赖 -- `lexmount/webarena-env#2`：固定 WebArena-Verified 1.2.3、站点镜像 digest、原生架构、 - 端口、启动/重置和机器可读回执。 -- 本仓库 PR：固定公开轨迹清单、离线修复记录、资格门禁、Chromium/Moli 镜像、动作 +- `harness/environments/webarena/`：Docker 和 Rainbond 站点生命周期、初始化与机器可读回执； + `deploy/webarena/` 保存部署配置和数据挂载契约。 +- `harness/trajectory_replay/public_webarena/`：公开轨迹清单、离线修复记录、资格门禁、Chromium/Moli 镜像、动作 重放、HAR、evaluator、一次失败重试、首次/最终成功统计、证据审计和独立报告发布。 -两者共同足以在新机器上重新执行同一协议，但不会仅凭 Git 还原旧运行的逐字节证据： +WebArena-Verified 1.2.3 作为独立依赖提供任务和评分器；全部官方镜像只在 +`evaluation/browser-use-public/webarena-environment-images.json` 中锁定。 +本仓库提供在新机器上重新执行协议所需的代码，但不会仅凭 Git 还原旧运行的逐字节证据： 原始截图、HAR、attempts、manifest 和 summary 按仓库规则保存在 Git 外。Git 中的 `evaluation/browser-use-public/current-report.json` 保存当前公开结果与外部证据哈希， 用于证明报告绑定，而不是替代一次新运行。 @@ -28,14 +30,16 @@ LLM judge；两个浏览器重放相同的 WebArena 公开人工轨迹，再由 ## 从空机器复现 -1. 检出并固定两个 PR 的合并提交；两个仓库都执行 `uv sync --frozen` 和各自的检查。 - 再在本仓库运行 - `uv run --frozen python scripts/verify_webarena_env_contract.py --environment-repo <webarena-env>`， - 确认上游版本、六个站点、镜像 digest 和架构完全一致。 -2. 在 `webarena-env` 复制 `config/runtime.example.json`，填写浏览器容器可访问的主机名， - 运行 `webarena-env verify/start` 并保留回执。Map 的三份 tar 须先用 - `uv run --frozen python -m webarena_env.map_archives --data-dir <下载目录>` 核对 SHA-256， - 再按官方 setup 初始化九个数据卷。账号只通过环境变量提供。 +1. 检出并固定本仓库提交，执行 `make setup` 和 `make ci`。运行 + `uv run --frozen python scripts/verify_webarena_env_contract.py`，检查部署和评测 + 配置使用同一份镜像锁以及一致的端口、凭据变量名。 +2. 复制 `deploy/webarena/runtime.example.json` 到 Git 外，填写可访问的 `hostname`， + 将 `image_lock` 设置为本仓库镜像锁的绝对路径。使用 + `uv run --frozen python scripts/manage_webarena_environment.py verify --config <配置> --site reddit` + 校验环境，将 `verify` 换为 `start` 或 `reset` 启动或重建站点，通过 `--receipt <路径>` 保存回执。 + 仅在需要 Map 的任务范围内准备地图数据；`setup --config <配置> --site map --data-dir <目录>` + 会先核对三份归档的 SHA-256，再运行官方初始化。当前 Rainbond 部署不包含 Map。 + 账号只通过环境变量提供。 3. 在本仓库复制 `examples/trajectory_replay/public_webarena/webarena.config.example.json`，使用同一组端口、 镜像 digest、网站地址与凭据变量名；运行环境 smoke，确认 CDP、登录、HAR 与站点 origin。 示例默认共用本机 Docker daemon，不假定存在某个用户的 rootless socket。若使用两条 @@ -55,9 +59,11 @@ uv run --frozen python -m harness.trajectory_replay.public_webarena.cli \ 5. 运行 `examples/trajectory_replay/public_webarena/verify_metrics.py` 和报告打包器。任何缺失回答、动作、终止 原因、评分，或进入浏览器阶段后缺失截图、最终页面、HAR，都按失败处理。 -正式运行前必须核对两个仓库的 WebArena-Verified 版本和六个站点镜像 digest 完全一致。 +正式运行前必须核对 WebArena-Verified 版本和任务涉及的站点镜像 digest。 这项检查不允许用 mutable tag、跨架构模拟或手工修改数据库替代。 -当前正式执行器按 Docker 容器身份验证并重建有状态站点，不能直接用 Rainbond 中的 -远程有状态组件替代。Rainbond 的镜像部署和站点连通性可以先单独验证；要发布与本协议 -等价的成绩，仍需受控重建与初态回执接入评测执行器。 +执行器支持 Docker 容器重建和带回执的 Rainbond 独立槽位重建。每个并发执行者必须 +独占一整个有状态槽位；只读 Wikipedia 可共享。Rainbond 使用方法见 +[环境示例](../examples/trajectory_replay/public_webarena/README.md)。HTTP 并发测试只证明 +网页可访问；正式运行仍须生成与本次重置、镜像、初态和就绪检查绑定的回执。 +共享 Kiwix 的 amd64 部署与官方 arm64 镜像身份不同，不能直接沿用官方镜像的验证回执。 diff --git a/evaluation/browser-use-public/webarena-env-contract.json b/evaluation/browser-use-public/webarena-env-contract.json deleted file mode 100644 index d451df0e3..000000000 --- a/evaluation/browser-use-public/webarena-env-contract.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "schema": "lexmount_webarena_cross_repo_contract/1", - "environment_repository": "https://github.com/lexmount/webarena-env", - "environment_pull_request": "https://github.com/lexmount/webarena-env/pull/2", - "webarena_verified_version": "1.2.3", - "environment_lock": "config/images.lock.json", - "environment_runtime_template": "config/runtime.example.json", - "environment_site_delivery_contract": "config/site-delivery-contract.json", - "evaluation_lock": "evaluation/browser-use-public/webarena-environment-images.json", - "evaluation_runtime_template": "examples/trajectory_replay/public_webarena/webarena.config.example.json", - "required_sites": [ - "shopping_admin", - "shopping", - "reddit", - "gitlab", - "wikipedia", - "map" - ] -} diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json index c9f3f87c5..266ca7890 100644 --- a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -23,13 +23,9 @@ "app_prefix": "webarena-slice", "domain_suffix": "dev.example.com", "token_env": "RAINBOND_API_TOKEN", - "image_lock": "../../../../evaluation/browser-use-public/webarena-environment-images.json", + "image_lock": "../../../evaluation/browser-use-public/webarena-environment-images.json", "readiness_timeout_seconds": 1800, "max_parallel_provisions": 2, "reset_strategy": "replace_before_destroy", - "shared_environments": { - "__MAP__": { - "urls": ["https://replace-with-shared-read-only-map"] - } - } + "shared_environments": {} } diff --git a/harness/environments/__init__.py b/harness/environments/__init__.py new file mode 100644 index 000000000..ddd02400b --- /dev/null +++ b/harness/environments/__init__.py @@ -0,0 +1 @@ +"""Reproducible site environments used by browser evaluations.""" diff --git a/harness/environments/webarena/__init__.py b/harness/environments/webarena/__init__.py new file mode 100644 index 000000000..715229d20 --- /dev/null +++ b/harness/environments/webarena/__init__.py @@ -0,0 +1 @@ +"""WebArena Docker and Rainbond environment lifecycle.""" diff --git a/harness/environments/webarena/backend.py b/harness/environments/webarena/backend.py new file mode 100644 index 000000000..d0308b496 --- /dev/null +++ b/harness/environments/webarena/backend.py @@ -0,0 +1,115 @@ +"""Docker backend with WebArena initialization safeguards.""" + +from __future__ import annotations + +import socket +from typing import Any + +import docker + +MAGENTO_INIT_TIMEOUT_SECONDS = 600 + + +class DockerBackend: + """Bind WebArena's container manager to an explicit Docker SDK client.""" + + def __init__(self, client: Any, *, defer_removal: bool = False) -> None: + self.client = client + self.defer_removal = defer_removal + self.retired_containers: list[str] = [] + + def _container(self, name: str) -> Any | None: + try: + return self.client.containers.get(name) + except docker.errors.NotFound: + return None + + def container_exists(self, *, name: str) -> bool: + return self._container(name) is not None + + def container_running(self, *, name: str) -> bool: + container = self._container(name) + if container is None: + return False + container.reload() + return container.status == "running" + + def container_remove(self, *, name: str) -> None: + container = self._container(name) + if container is not None: + if not self.defer_removal: + container.remove(force=True) + return + # Removing WebArena's tens-of-gigabytes writable layer can block a + # rootless daemon for hours. Stopping releases the fixed ports; + # renaming preserves the old layer for deferred garbage collection. + # The next container still starts from the immutable pinned image, + # and these official images have no shared writable mounts. + container.reload() + if container.status == "running": + container.stop(timeout=10) + retired_name = f"{name}__retired__{container.id[:12]}" + container.rename(retired_name) + self.retired_containers.append(retired_name) + + def container_run( + self, + *, + name: str, + image: str, + port_mappings: dict[int, int], + volume_mappings: dict[str, str], + env_vars: dict[str, str] | None = None, + ) -> None: + environment = dict(env_vars or {}) + if self.defer_removal and volume_mappings: + raise RuntimeError( + "deferred WebArena container removal requires no shared writable mounts" + ) + if name in { + "webarena_verified_shopping", + "webarena_verified_shopping_admin", + }: + # Magento's official init performs cache/database work after the + # services become healthy. The image default of 180 seconds is + # too short on a busy qualification host and can leave the baked + # CMU origin active even though env-ctrl /status is healthy. + environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str(MAGENTO_INIT_TIMEOUT_SECONDS) + # GitLab's image entrypoint does not run env-ctrl init. Its fallback + # reconfiguration also rewrites the internal nginx listener to the + # public host port, breaking Docker's host-port -> 8023 mapping. Keep + # the baked internal listener here; WebArenaEnvironment performs one + # explicit official init after the service is healthy. + if name == "webarena_verified_gitlab": + environment["WA_ENV_CTRL_SKIP_RECONFIGURE"] = "true" + self.client.containers.run( + image, + detach=True, + name=name, + ports={ + f"{container_port}/tcp": host_port + for host_port, container_port in port_mappings.items() + }, + volumes={ + source: {"bind": target, "mode": "rw"} for source, target in volume_mappings.items() + }, + environment=environment, + ) + + def get_container_ports(self, *, name: str) -> dict[int, int]: + container = self._container(name) + if container is None: + return {} + container.reload() + result: dict[int, int] = {} + ports = container.attrs.get("NetworkSettings", {}).get("Ports") or {} + for key, bindings in ports.items(): + if bindings: + result[int(str(key).split("/", 1)[0])] = int(bindings[0]["HostPort"]) + return result + + def find_free_port(self) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as value: + value.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + value.bind(("", 0)) + return int(value.getsockname()[1]) diff --git a/harness/environments/webarena/cli.py b/harness/environments/webarena/cli.py new file mode 100644 index 000000000..99c43cc60 --- /dev/null +++ b/harness/environments/webarena/cli.py @@ -0,0 +1,88 @@ +"""Command-line interface for the pinned WebArena environment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from webarena_verified.environments.setup import setup_init +from webarena_verified.types.task import WebArenaSite + +from .config import SITES, RuntimeConfig +from .manager import EnvironmentManager +from .map_archives import verify_archives + + +def _write_receipt(path: Path | None, payload: dict[str, Any]) -> None: + rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + if path is not None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + print(rendered, end="") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=("setup", "verify", "start", "reset", "stop")) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--site", choices=SITES, action="append") + parser.add_argument("--receipt", type=Path) + parser.add_argument("--timeout", type=int, default=900) + parser.add_argument("--data-dir") + parser.add_argument("--defer-removal", action="store_true") + args = parser.parse_args() + if args.timeout < 1: + parser.error("--timeout must be positive") + config = RuntimeConfig.load(args.config) + names = tuple(args.site or config.sites) + if args.command == "setup" and ( + not args.data_dir or any(name not in {"map", "wikipedia"} for name in names) + ): + parser.error("setup requires --site map|wikipedia and --data-dir") + manager = ( + EnvironmentManager(config, defer_removal=args.defer_removal) + if args.command != "setup" + else None + ) + rows: list[dict[str, Any]] = [] + for name in names: + try: + if args.command == "setup": + archives = verify_archives(Path(args.data_dir)) if name == "map" else [] + result = setup_init(site=WebArenaSite(name), data_dir=Path(args.data_dir)) + rows.append({"site": name, "result": result, "archives": archives}) + elif args.command == "verify": + rows.append(manager.verify(name)) + elif args.command == "stop": + rows.append(manager.stop(name)) + else: + rows.append( + manager.start( + name, + reset=args.command == "reset", + timeout=args.timeout, + data_dir=args.data_dir, + ) + ) + except Exception: + _write_receipt( + args.receipt, + { + "schema": "lexmount_webarena_receipt/1", + "command": args.command, + "status": "partial_failure", + "failed_site": name, + "sites": rows, + }, + ) + raise + _write_receipt( + args.receipt, + {"schema": "lexmount_webarena_receipt/1", "command": args.command, "sites": rows}, + ) + + +if __name__ == "__main__": + main() diff --git a/harness/environments/webarena/config.py b/harness/environments/webarena/config.py new file mode 100644 index 000000000..df1f75000 --- /dev/null +++ b/harness/environments/webarena/config.py @@ -0,0 +1,80 @@ +"""Validated image lock and host-local runtime configuration.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +SITES = ("shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia") +ARCH_ALIASES = {"aarch64": "arm64", "x86_64": "amd64"} + + +@dataclass(frozen=True) +class SiteConfig: + name: str + image: str + architecture: str + port: int + control_port: int + + +@dataclass(frozen=True) +class RuntimeConfig: + path: Path + hostname: str + sites: dict[str, SiteConfig] + + @classmethod + def load(cls, path: str | Path) -> RuntimeConfig: + config_path = Path(path).resolve() + payload = json.loads(config_path.read_text(encoding="utf-8")) + if payload.get("schema") != "lexmount_webarena_runtime/1": + raise ValueError("invalid WebArena runtime schema") + hostname = str(payload.get("hostname") or "").strip() + if not hostname or "${" in hostname: + raise ValueError("runtime hostname must replace the example placeholder") + lock_path = Path(str(payload.get("image_lock") or "")) + if not lock_path.is_absolute(): + lock_path = config_path.parent / lock_path + lock = json.loads(lock_path.read_text(encoding="utf-8")) + if lock.get("schema") != "lex_webarena_environment_images/1": + raise ValueError("invalid WebArena image lock schema") + if lock.get("webarena_verified_version") != "1.2.3": + raise ValueError("image lock does not match webarena-verified 1.2.3") + runtime_sites = payload.get("sites") or {} + locked_sites = lock.get("images") or {} + sites: dict[str, SiteConfig] = {} + used_ports: set[int] = set() + for name in SITES: + runtime = runtime_sites.get(name) + image = locked_sites.get(name) + if not isinstance(runtime, dict) or not isinstance(image, dict): + raise ValueError(f"runtime or image lock is missing site {name}") + port = int(runtime.get("port") or 0) + control_port = int(runtime.get("control_port") or 0) + if not 1 <= port <= 65535 or not 1 <= control_port <= 65535: + raise ValueError(f"site {name} ports must be between 1 and 65535") + if port == control_port or {port, control_port} & used_ports: + raise ValueError(f"site {name} ports must be unique") + used_ports.update((port, control_port)) + pinned = str(image.get("pinned") or "") + if "@sha256:" not in pinned: + raise ValueError(f"site {name} image must be pinned by digest") + architecture = normalized_architecture(image.get("architecture")) + if architecture not in {"amd64", "arm64"}: + raise ValueError(f"site {name} architecture must be explicitly amd64 or arm64") + sites[name] = SiteConfig( + name=name, + image=pinned, + architecture=architecture, + port=port, + control_port=control_port, + ) + return cls(path=config_path, hostname=hostname, sites=sites) + + +def normalized_architecture(value: Any) -> str: + architecture = str(value or "") + return ARCH_ALIASES.get(architecture, architecture) diff --git a/harness/environments/webarena/manager.py b/harness/environments/webarena/manager.py new file mode 100644 index 000000000..65de486ea --- /dev/null +++ b/harness/environments/webarena/manager.py @@ -0,0 +1,204 @@ +"""Create and verify WebArena-Verified containers from immutable inputs.""" + +from __future__ import annotations + +import json +import re +import time +import urllib.error +import urllib.request +from typing import Any + +import docker +from webarena_verified.environments.container.config import get_container_config +from webarena_verified.environments.container.manager import ContainerManager +from webarena_verified.types.task import WebArenaSite + +from .backend import DockerBackend +from .config import RuntimeConfig, SiteConfig, normalized_architecture + +STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) + + +class EnvironmentManager: + def __init__( + self, + config: RuntimeConfig, + client: Any | None = None, + *, + defer_removal: bool = False, + ) -> None: + self.config = config + self.client = client or docker.from_env(timeout=900) + self.backend = DockerBackend(self.client, defer_removal=defer_removal) + + def _require_native_architecture(self, site: SiteConfig) -> str: + daemon_arch = normalized_architecture(self.client.info().get("Architecture")) + if daemon_arch != site.architecture: + raise RuntimeError( + f"{site.name} requires native {site.architecture}; Docker daemon is {daemon_arch}" + ) + return daemon_arch + + def _manager(self, site: SiteConfig) -> ContainerManager: + self._require_native_architecture(site) + upstream = get_container_config(site=WebArenaSite(site.name)) + locked = upstream.model_copy( + update={ + "docker_img": site.image, + "host_port": site.port, + "host_env_ctrl_port": site.control_port, + } + ) + return ContainerManager( + site=WebArenaSite(site.name), + config=locked, + backend=self.backend, + hostname=self.config.hostname, + ) + + def verify(self, name: str, *, require_running: bool = False) -> dict[str, Any]: + site = self.config.sites[name] + manager = self._manager(site) + daemon_arch = site.architecture + result: dict[str, Any] = { + "site": name, + "daemon_architecture": daemon_arch, + "required_architecture": site.architecture, + "requested_image": site.image, + "container_name": manager.container_name, + } + try: + container = self.client.containers.get(result["container_name"]) + except docker.errors.NotFound as exc: + if require_running: + raise RuntimeError( + f"WebArena container is missing: {result['container_name']}" + ) from exc + result["status"] = "not_created" + return result + container.reload() + image = container.image + if image is None: + raise RuntimeError(f"{name} container has no image metadata") + repo_digests = sorted(image.attrs.get("RepoDigests") or []) + if site.image not in repo_digests: + raise RuntimeError(f"{name} image mismatch: expected {site.image}, got {repo_digests}") + if require_running and container.status != "running": + raise RuntimeError(f"WebArena container is not running: {result['container_name']}") + upstream = get_container_config(site=WebArenaSite(name)) + expected_ports = { + int(upstream.container_port): site.port, + int(upstream.env_ctrl_port): site.control_port, + } + observed_ports = self.backend.get_container_ports(name=str(result["container_name"])) + if observed_ports != expected_ports: + raise RuntimeError( + f"{name} port mapping mismatch: expected {expected_ports}, got {observed_ports}" + ) + result.update( + { + "status": container.status, + "container_id": container.id, + "image_id": image.id, + "repo_digests": repo_digests, + "port_mappings": observed_ports, + "site_url": f"http://{self.config.hostname}:{site.port}", + "control_url": f"http://{self.config.hostname}:{site.control_port}", + } + ) + return result + + def start( + self, + name: str, + *, + reset: bool, + timeout: int, + data_dir: str | None = None, + ) -> dict[str, Any]: + site = self.config.sites[name] + manager = self._manager(site) + if not reset and self.backend.container_running(name=manager.container_name): + return {"command": "start", **self.verify(name, require_running=True)} + started = manager.start( + port=site.port, + env_ctrl_port=site.control_port, + data_dir=data_dir, + wait=False, + ) + container = self.client.containers.get(manager.container_name) + self._wait_control(started.env_ctrl_url, timeout=timeout) + if name in {"gitlab", "reddit"}: + self._wait_entrypoint_init(container, timeout=timeout) + self._explicit_init(container, started.url, site=name) + self._wait_control(started.env_ctrl_url, timeout=timeout) + elif name in {"shopping", "shopping_admin"}: + self._wait_entrypoint_init(container, timeout=timeout) + self._verify_entrypoint_init(container) + receipt = self.verify(name, require_running=True) + receipt.update( + { + "command": "reset" if reset else "start", + "site_url": started.url, + "control_url": started.env_ctrl_url, + "retired_containers": list(self.backend.retired_containers), + "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + ) + return receipt + + def stop(self, name: str) -> dict[str, Any]: + manager = self._manager(self.config.sites[name]) + manager.stop() + return {"command": "stop", "site": name, "container_name": manager.container_name} + + @staticmethod + def _wait_control(url: str, *, timeout: int) -> None: + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url.rstrip("/") + "/status", timeout=5) as response: + payload = json.load(response) + if response.status == 200 and payload.get("success") is True: + return + except (json.JSONDecodeError, OSError, urllib.error.URLError) as exc: + last_error = exc + time.sleep(0.5) + raise RuntimeError(f"WebArena control endpoint did not become ready: {last_error}") + + @staticmethod + def _wait_entrypoint_init(container: Any, *, timeout: int) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + processes = container.top(ps_args="-eo pid,args").get("Processes") or [] + active = any( + "environment_control.cli init" in " ".join(map(str, row)) + or re.search(r"\benv-ctrl\b.*\binit\b", " ".join(map(str, row))) + for row in processes + ) + if not active: + return + time.sleep(0.5) + raise RuntimeError("WebArena entrypoint initialization did not finish") + + @staticmethod + def _explicit_init(container: Any, url: str, *, site: str) -> None: + result = container.exec_run( + ["env-ctrl", "--verbose", "init", "--base-url", url], demux=False + ) + output = result.output.decode("utf-8", errors="replace") + try: + payload = json.loads(output) + except json.JSONDecodeError as exc: + raise RuntimeError(f"WebArena {site} init returned invalid JSON") from exc + if result.exit_code != 0 or payload.get("success") is not True: + raise RuntimeError(f"WebArena {site} init failed: {payload.get('message')}") + + @staticmethod + def _verify_entrypoint_init(container: Any) -> None: + result = container.exec_run(["sh", "-lc", "tail -n 1 /tmp/env-ctrl-init.log"], demux=False) + output = result.output.decode("utf-8", errors="replace").strip() + if result.exit_code != 0 or output != "[OK]": + raise RuntimeError("WebArena entrypoint initialization failed") diff --git a/harness/environments/webarena/map_archives.py b/harness/environments/webarena/map_archives.py new file mode 100644 index 000000000..03963361c --- /dev/null +++ b/harness/environments/webarena/map_archives.py @@ -0,0 +1,58 @@ +"""Check the exact Map archives before initializing persistent data volumes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from urllib.parse import urlparse + +DEFAULT_CONTRACT = ( + Path(__file__).resolve().parents[3] / "deploy/webarena/site-delivery-contract.json" +) + + +def verify_archives( + data_dir: Path, contract_path: Path = DEFAULT_CONTRACT +) -> list[dict[str, str | int]]: + """Check each Map archive against the contract size and SHA-256, failing on drift.""" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + map_site = contract["components"]["map"] + expected = map_site["data_archive_sha256"] + expected_bytes = map_site["data_archive_bytes"] + names = [Path(urlparse(url).path).name for url in map_site["data_urls"]] + if ( + len(names) != len(set(names)) + or set(names) != set(expected) + or set(names) != set(expected_bytes) + ): + raise ValueError("Map archive checksum inventory differs from upstream URLs") + rows: list[dict[str, str | int]] = [] + for name in names: + archive = data_dir / name + observed_bytes = archive.stat().st_size + if observed_bytes != expected_bytes[name]: + raise ValueError(f"Map archive size mismatch: {name}") + digest = hashlib.sha256() + with archive.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + if observed != expected[name]: + raise ValueError(f"Map archive checksum mismatch: {name}") + rows.append({"archive": name, "sha256": observed, "bytes": observed_bytes}) + return rows + + +def main() -> None: + """Run the archive verification CLI before initializing persistent Map volumes.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--contract", type=Path, default=DEFAULT_CONTRACT) + args = parser.parse_args() + print(json.dumps({"status": "pass", "archives": verify_archives(args.data_dir, args.contract)})) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/public_webarena/rainbond_slices.py b/harness/environments/webarena/rainbond.py similarity index 98% rename from harness/trajectory_replay/public_webarena/rainbond_slices.py rename to harness/environments/webarena/rainbond.py index 46006f678..49609c58e 100644 --- a/harness/trajectory_replay/public_webarena/rainbond_slices.py +++ b/harness/environments/webarena/rainbond.py @@ -641,17 +641,14 @@ def _delete_app(self, app_id: int) -> None: service_ids = {str(row["service_id"]) for row in rows if row.get("service_id")} if service_ids: self.api.request( - "POST", f"{path}/operations", + "POST", + f"{path}/operations", {"action": "stop", "service_ids": sorted(service_ids)}, ) deadline = time.monotonic() + self.config.readiness_timeout_seconds while True: - rows = self._response_rows( - self.api.request("GET", f"{path}/services"), "services" - ) - statuses = { - str(row.get("service_id")): row.get("status") for row in rows - } + rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + statuses = {str(row.get("service_id")): row.get("status") for row in rows} if all(statuses.get(sid) in {"closed", "undeploy"} for sid in service_ids): break if time.monotonic() >= deadline: @@ -1005,7 +1002,6 @@ def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: if not owner: raise ValueError("lease owner is required") with _pool_lock(output_dir): - for slot_dir in sorted(output_dir.glob("slice-*"), key=_slot_sort_key): receipt_path = slot_dir / "slice-receipt.json" config_path = slot_dir / "webarena.config.json" @@ -1042,17 +1038,20 @@ def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[st raise ValueError("lease_id does not identify exactly one active slice") lease_path, lease = matches[0] slot_dir = lease_path.parent + # Keep the lease while resetting so acquire/scale skip this slot. The slot + # lock serializes competing releases without blocking unrelated ready slots. + with _pool_lock(slot_dir): + if not lease_path.is_file() or json.loads(lease_path.read_text(encoding="utf-8")) != lease: + raise ValueError("lease changed before release") pool = RainbondSlicePool(config_path) receipt_path = slot_dir / "slice-receipt.json" runtime_config_path = slot_dir / "webarena.config.json" current_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) if int(current_receipt["app_id"]) != int(lease["app_id"]): current_receipt = pool.validate_receipt(receipt_path) - if ( - runtime_config_path.is_file() - and json.loads(runtime_config_path.read_text(encoding="utf-8")) - == pool.runtime_config(current_receipt) - ): + if runtime_config_path.is_file() and json.loads( + runtime_config_path.read_text(encoding="utf-8") + ) == pool.runtime_config(current_receipt): lease_path.unlink() return { "lease_id": lease_id, diff --git a/harness/trajectory_replay/public_webarena/webarena_environment.py b/harness/trajectory_replay/public_webarena/webarena_environment.py index dbc9a86b0..4cdb0aff5 100644 --- a/harness/trajectory_replay/public_webarena/webarena_environment.py +++ b/harness/trajectory_replay/public_webarena/webarena_environment.py @@ -7,7 +7,6 @@ import os import re import signal -import socket import threading import time import urllib.error @@ -22,11 +21,13 @@ from webarena_verified.environments.container.manager import ContainerManager from webarena_verified.types.task import WebArenaSite -from .rainbond_slices import RainbondSlicePool +from harness.environments.webarena.backend import MAGENTO_INIT_TIMEOUT_SECONDS, DockerBackend +from harness.environments.webarena.manager import EnvironmentManager +from harness.environments.webarena.rainbond import RainbondSlicePool + from .suite import BrowserTask STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) -MAGENTO_INIT_TIMEOUT_SECONDS = 600 LEGACY_PORT_SITES = { 3000: "map", 4399: "reddit", @@ -68,111 +69,6 @@ def expire(_signum: int, _frame: Any) -> None: signal.signal(signal.SIGALRM, previous_handler) -class _DockerClientBackend: - """Bind WebArena's container manager to an explicit Docker SDK client.""" - - def __init__(self, client: Any, *, defer_removal: bool = False) -> None: - self.client = client - self.defer_removal = defer_removal - self.retired_containers: list[str] = [] - - def _container(self, name: str) -> Any | None: - try: - return self.client.containers.get(name) - except docker.errors.NotFound: - return None - - def container_exists(self, *, name: str) -> bool: - return self._container(name) is not None - - def container_running(self, *, name: str) -> bool: - container = self._container(name) - if container is None: - return False - container.reload() - return container.status == "running" - - def container_remove(self, *, name: str) -> None: - container = self._container(name) - if container is not None: - if not self.defer_removal: - container.remove(force=True) - return - # Removing WebArena's tens-of-gigabytes writable layer can block a - # rootless daemon for hours. Stopping releases the fixed ports; - # renaming preserves the old layer for deferred garbage collection. - # The next container still starts from the immutable pinned image, - # and these official images have no shared writable mounts. - container.reload() - if container.status == "running": - container.stop(timeout=10) - retired_name = f"{name}__retired__{container.id[:12]}" - container.rename(retired_name) - self.retired_containers.append(retired_name) - - def container_run( - self, - *, - name: str, - image: str, - port_mappings: dict[int, int], - volume_mappings: dict[str, str], - env_vars: dict[str, str] | None = None, - ) -> None: - environment = dict(env_vars or {}) - if self.defer_removal and volume_mappings: - raise RuntimeError( - "deferred WebArena container removal requires no shared writable mounts" - ) - if name in { - "webarena_verified_shopping", - "webarena_verified_shopping_admin", - }: - # Magento's official init performs cache/database work after the - # services become healthy. The image default of 180 seconds is - # too short on a busy qualification host and can leave the baked - # CMU origin active even though env-ctrl /status is healthy. - environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str(MAGENTO_INIT_TIMEOUT_SECONDS) - # GitLab's image entrypoint does not run env-ctrl init. Its fallback - # reconfiguration also rewrites the internal nginx listener to the - # public host port, breaking Docker's host-port -> 8023 mapping. Keep - # the baked internal listener here; WebArenaEnvironment performs one - # explicit official init after the service is healthy. - if name == "webarena_verified_gitlab": - environment["WA_ENV_CTRL_SKIP_RECONFIGURE"] = "true" - self.client.containers.run( - image, - detach=True, - name=name, - ports={ - f"{container_port}/tcp": host_port - for host_port, container_port in port_mappings.items() - }, - volumes={ - source: {"bind": target, "mode": "rw"} for source, target in volume_mappings.items() - }, - environment=environment, - ) - - def get_container_ports(self, *, name: str) -> dict[int, int]: - container = self._container(name) - if container is None: - return {} - container.reload() - result: dict[int, int] = {} - ports = container.attrs.get("NetworkSettings", {}).get("Ports") or {} - for key, bindings in ports.items(): - if bindings: - result[int(str(key).split("/", 1)[0])] = int(bindings[0]["HostPort"]) - return result - - def find_free_port(self) -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as value: - value.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - value.bind(("", 0)) - return int(value.getsockname()[1]) - - class WebArenaEnvironment: def __init__(self, config_path: str | Path) -> None: self.config_path = Path(config_path) @@ -521,7 +417,7 @@ def _reset_impl( } ) defer_removal = bool(self.config.get("defer_stateful_container_removal", False)) - backend = _DockerClientBackend(docker_client, defer_removal=defer_removal) + backend = DockerBackend(docker_client, defer_removal=defer_removal) manager = ContainerManager( site=WebArenaSite(site), config=config, @@ -609,68 +505,19 @@ def _control_url(self, site: str) -> str: @staticmethod def _wait_ready(url: str, *, timeout_seconds: float) -> None: - deadline = time.monotonic() + timeout_seconds - last_error: Exception | None = None - while time.monotonic() < deadline: - try: - with urllib.request.urlopen(url.rstrip("/") + "/status", timeout=5) as response: - payload = json.load(response) - if response.status == 200 and payload.get("success") is True: - return - except (OSError, urllib.error.URLError) as exc: - last_error = exc - time.sleep(0.5) - raise RuntimeError(f"WebArena environment did not become ready: {last_error}") + EnvironmentManager._wait_control(url, timeout=timeout_seconds) @staticmethod def _wait_container_initialization(container: Any, *, timeout_seconds: float) -> None: - """Wait for the image entrypoint's one-time env-ctrl init to exit.""" - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - # Docker's remote ``top`` endpoint requires the PID column to be - # present even though we only inspect the command text. - process_table = container.top(ps_args="-eo pid,args") - processes = process_table.get("Processes") or [] - active = any( - "environment_control.cli init" in " ".join(map(str, row)) - or re.search(r"\benv-ctrl\b.*\binit\b", " ".join(map(str, row))) - for row in processes - ) - if not active: - return - time.sleep(0.5) - raise RuntimeError("WebArena entrypoint initialization did not finish") + EnvironmentManager._wait_entrypoint_init(container, timeout=timeout_seconds) @staticmethod def _configure_external_url(container: Any, external_url: str, *, site: str) -> None: - """Run a site's official initializer with an explicit public URL.""" - result = container.exec_run( - ["env-ctrl", "--verbose", "init", "--base-url", external_url], - demux=False, - ) - output = result.output - if isinstance(output, bytes): - output = output.decode("utf-8", errors="replace") - try: - payload = json.loads(str(output or "")) - except json.JSONDecodeError as exc: - raise RuntimeError(f"WebArena {site} init returned invalid JSON") from exc - if result.exit_code != 0 or payload.get("success") is not True: - message = str(payload.get("message") or "official initializer failed") - raise RuntimeError(f"WebArena {site} init failed: {message}") + EnvironmentManager._explicit_init(container, external_url, site=site) @staticmethod def _verify_entrypoint_initialization(container: Any) -> None: - """Fail immediately if the official supervisor init did not finish cleanly.""" - result = container.exec_run( - ["sh", "-lc", "tail -n 1 /tmp/env-ctrl-init.log"], - demux=False, - ) - output = result.output - if isinstance(output, bytes): - output = output.decode("utf-8", errors="replace") - if result.exit_code != 0 or str(output or "").strip() != "[OK]": - raise RuntimeError("WebArena entrypoint initialization failed") + EnvironmentManager._verify_entrypoint_init(container) @staticmethod def _wait_site_ready(url: str, *, timeout_seconds: float) -> None: diff --git a/pyproject.toml b/pyproject.toml index 7ff25847a..9a97f3b2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ markers = [ [tool.ruff] line-length = 100 include = [ + "harness/environments/**/*.py", + "tests/environments/**/*.py", "harness/browser_use/**/*.py", "harness/trajectory_replay/**/*.py", "tests/browser_use/**/*.py", diff --git a/scripts/manage_webarena_environment.py b/scripts/manage_webarena_environment.py index 890acebcf..cf2f39b79 100644 --- a/scripts/manage_webarena_environment.py +++ b/scripts/manage_webarena_environment.py @@ -1,167 +1,6 @@ -"""Start pinned WebArena-Verified sites and record their concrete runtime identity.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import time -from pathlib import Path - -import docker -from webarena_verified.environments.container.config import get_container_config -from webarena_verified.environments.container.manager import ContainerManager -from webarena_verified.environments.setup import setup_init -from webarena_verified.types.task import WebArenaSite - -from harness.trajectory_replay.public_webarena.webarena_environment import ( - _DockerClientBackend, -) - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _locked_image(lock_path: Path, site: str) -> dict: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - if payload.get("schema") != "lex_webarena_environment_images/1": - raise ValueError("invalid WebArena environment image lock") - return dict(payload["images"][site]) - - -def _manager( - site_name: str, - lock_path: Path, - *, - allow_emulation: bool = False, - hostname: str = "localhost", - port: int | None = None, - control_port: int | None = None, -) -> ContainerManager: - site = WebArenaSite(site_name) - locked = _locked_image(lock_path, site_name) - daemon_arch = str(docker.from_env().info().get("Architecture") or "") - aliases = {"aarch64": "arm64", "x86_64": "amd64"} - daemon_arch = aliases.get(daemon_arch, daemon_arch) - image_arch = str(locked.get("architecture") or "") - if not allow_emulation and image_arch and image_arch != daemon_arch: - raise RuntimeError( - f"{site_name} image requires native {image_arch}; Docker daemon is {daemon_arch}. " - "Cross-architecture execution is disabled because it can change behavior or crash." - ) - updates: dict[str, object] = {"docker_img": str(locked["pinned"])} - if port is not None: - updates["host_port"] = port - if control_port is not None: - updates["host_env_ctrl_port"] = control_port - config = get_container_config(site=site).model_copy(update=updates) - return ContainerManager( - site=site, - config=config, - backend=_DockerClientBackend(docker.from_env()), - hostname=hostname, - ) - - -def _write_receipt(path: Path, payload: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - - -def main() -> None: - def port_number(value: str) -> int: - port = int(value) - if not 1 <= port <= 65535: - raise argparse.ArgumentTypeError("port must be between 1 and 65535") - return port - - parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("setup", "start", "status", "stop")) - parser.add_argument( - "--site", - required=True, - choices=("shopping_admin", "shopping", "reddit", "gitlab", "wikipedia", "map"), - ) - parser.add_argument( - "--image-lock", - type=Path, - default=Path("evaluation/browser-use-public/webarena-environment-images.json"), - ) - parser.add_argument("--data-dir", type=Path) - parser.add_argument("--receipt", type=Path) - parser.add_argument("--timeout", type=int, default=900) - parser.add_argument("--port", type=port_number) - parser.add_argument("--control-port", type=port_number) - parser.add_argument("--allow-emulation", action="store_true") - parser.add_argument("--hostname", default="localhost") - args = parser.parse_args() - site = WebArenaSite(args.site) - if (args.port is None) != (args.control_port is None): - parser.error("--port and --control-port must be provided together") - if args.port is not None and args.port == args.control_port: - parser.error("--port and --control-port must differ") - - if args.command == "setup": - if args.site not in {"wikipedia", "map"} or args.data_dir is None: - parser.error("setup requires --site wikipedia|map and --data-dir") - result = setup_init(site=site, data_dir=args.data_dir) - files = [ - { - "name": path.name, - "bytes": path.stat().st_size, - "sha256": _sha256(path), - } - for path in sorted(args.data_dir.iterdir()) - if path.is_file() - ] - receipt = {"command": "setup", "site": args.site, "result": result, "files": files} - else: - manager = _manager( - args.site, - args.image_lock, - allow_emulation=args.allow_emulation or args.command != "start", - hostname=args.hostname, - port=args.port, - control_port=args.control_port, - ) - if args.command == "start": - config = manager.config - started = manager.start( - port=int(config.host_port), - env_ctrl_port=int(config.host_env_ctrl_port), - timeout=args.timeout, - data_dir=args.data_dir, - ) - container = docker.from_env().containers.get(manager.container_name) - container.reload() - daemon_architecture = docker.from_env().info().get("Architecture") - receipt = { - "command": "start", - "site": args.site, - "requested_image": config.docker_img, - "container_name": manager.container_name, - "container_id": container.id, - "image_id": container.image.id, - "repo_digests": sorted(container.image.attrs.get("RepoDigests") or []), - "daemon_architecture": daemon_architecture, - "site_url": started.url, - "control_url": started.env_ctrl_url, - "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - } - elif args.command == "status": - receipt = manager.status().model_dump(mode="json") - else: - manager.stop() - receipt = {"command": "stop", "site": args.site} - if args.receipt: - _write_receipt(args.receipt, receipt) - print(json.dumps(receipt, indent=2, ensure_ascii=False)) +"""Manage the repository's pinned WebArena Docker environments.""" +from harness.environments.webarena.cli import main if __name__ == "__main__": main() diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py index 80671ad8d..1a990641d 100644 --- a/scripts/manage_webarena_rainbond_pool.py +++ b/scripts/manage_webarena_rainbond_pool.py @@ -6,7 +6,7 @@ import json from pathlib import Path -from harness.trajectory_replay.public_webarena.rainbond_slices import ( +from harness.environments.webarena.rainbond import ( RainbondSlicePool, acquire_slice, provision_pool, diff --git a/scripts/verify_webarena_env_contract.py b/scripts/verify_webarena_env_contract.py index 3a7fadf7c..5d53f2ad5 100644 --- a/scripts/verify_webarena_env_contract.py +++ b/scripts/verify_webarena_env_contract.py @@ -1,119 +1,85 @@ -"""Verify that webarena-env and browser-eval bind the same site images.""" +"""Verify the repository's WebArena deployment and evaluation configuration.""" from __future__ import annotations -import argparse import json import re from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] -DEFAULT_CONTRACT = ROOT / "evaluation/browser-use-public/webarena-env-contract.json" -def verify(environment_repo: Path, contract_path: Path = DEFAULT_CONTRACT) -> dict: - contract = json.loads(contract_path.read_text(encoding="utf-8")) - if contract.get("schema") != "lexmount_webarena_cross_repo_contract/1": - raise ValueError("invalid cross-repository contract schema") - env_lock = json.loads( - (environment_repo.resolve() / contract["environment_lock"]).read_text(encoding="utf-8") +def verify(root: Path = ROOT) -> dict: + lock_path = root / "evaluation/browser-use-public/webarena-environment-images.json" + lock = json.loads(lock_path.read_text(encoding="utf-8")) + runtime_path = root / "deploy/webarena/runtime.example.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + evaluation = json.loads( + ( + root / "examples/trajectory_replay/public_webarena/webarena.config.example.json" + ).read_text(encoding="utf-8") ) - eval_lock = json.loads( - (ROOT / contract["evaluation_lock"]).read_text(encoding="utf-8") - ) - expected_version = contract["webarena_verified_version"] - if env_lock.get("webarena_verified_version") != expected_version: - raise ValueError("webarena-env package version binding differs") - if eval_lock.get("webarena_verified_version") != expected_version: - raise ValueError("browser-eval package version binding differs") - sites = contract["required_sites"] - if set(env_lock.get("images") or {}) != set(sites): - raise ValueError("webarena-env image inventory differs") - if set(eval_lock.get("images") or {}) != set(sites): - raise ValueError("browser-eval image inventory differs") - mismatches = [] - for site in sites: - left = env_lock["images"][site] - right = eval_lock["images"][site] - for key in ("pinned", "architecture", "os"): - if left.get(key) != right.get(key): - mismatches.append(f"{site}.{key}") - if mismatches: - raise ValueError("cross-repository image binding differs: " + ", ".join(mismatches)) - env_runtime = json.loads( - (environment_repo.resolve() / contract["environment_runtime_template"]).read_text( - encoding="utf-8" - ) - ) - eval_runtime = json.loads( - (ROOT / contract["evaluation_runtime_template"]).read_text(encoding="utf-8") - ) - for site in sites: - token = f"__{site.upper()}__" - site_row = (env_runtime.get("sites") or {}).get(site) or {} - eval_row = (eval_runtime.get("environments") or {}).get(token) or {} - urls = eval_row.get("urls") or [] - control_url = (eval_runtime.get("control_urls") or {}).get(site) - if len(urls) != 1 or not control_url: - raise ValueError(f"cross-repository runtime template is missing {site} URLs") - if urlparse(str(urls[0])).port != site_row.get("port"): - raise ValueError(f"cross-repository runtime site port differs: {site}") - if urlparse(str(control_url)).port != site_row.get("control_port"): - raise ValueError(f"cross-repository runtime control port differs: {site}") - env_credentials = (env_runtime.get("credentials_env") or {}).get(site) - if env_credentials is not None and env_credentials != eval_row.get("credentials_env"): - raise ValueError(f"cross-repository credential variable names differ: {site}") - delivery_path = environment_repo.resolve() / contract["environment_site_delivery_contract"] - if not delivery_path.is_file(): - raise ValueError(f"environment repo is missing site delivery contract: {delivery_path}") + delivery_path = root / "deploy/webarena/site-delivery-contract.json" delivery = json.loads(delivery_path.read_text(encoding="utf-8")) - if delivery.get("webarena_verified_version") != expected_version: - raise ValueError("cross-repository site delivery package version differs") - image_lock = delivery.get("image_lock") - if not isinstance(image_lock, str) or not image_lock: - raise ValueError("cross-repository site delivery image_lock is missing") - if (delivery_path.parent / image_lock).resolve() != ( - environment_repo.resolve() / contract["environment_lock"] - ).resolve(): - raise ValueError("cross-repository site delivery image lock path differs") - components = delivery.get("components") or {} - if set(components) != set(sites) - {"wikipedia"}: - raise ValueError("cross-repository site delivery inventory differs") - for site, component in components.items(): - if component.get("image") != env_lock["images"][site]["pinned"]: - raise ValueError(f"cross-repository site delivery image differs: {site}") - map_site = components["map"] - archive_urls = map_site.get("data_urls") or [] - archive_digests = map_site.get("data_archive_sha256") or {} - archive_bytes = map_site.get("data_archive_bytes") or {} - archive_names = [Path(urlparse(url).path).name for url in archive_urls] + if lock.get("schema") != "lex_webarena_environment_images/1": + raise ValueError("invalid image lock schema") if ( - len(archive_names) != 3 - or len(set(archive_names)) != 3 - or set(archive_names) != set(archive_digests) - or set(archive_names) != set(archive_bytes) - or any(not re.fullmatch(r"[a-f0-9]{64}", str(value)) for value in archive_digests.values()) - or any(not isinstance(value, int) or value <= 0 for value in archive_bytes.values()) + lock.get("webarena_verified_version") != "1.2.3" + or delivery.get("webarena_verified_version") != "1.2.3" ): - raise ValueError("cross-repository Map archive checksum contract is incomplete") + raise ValueError("WebArena package version differs") + sites = {"shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia"} + if set(lock.get("images") or {}) != sites or set(runtime.get("sites") or {}) != sites: + raise ValueError("site inventory differs") + for path, config in ((runtime_path, runtime), (delivery_path, delivery)): + if ( + not config.get("image_lock") + or (path.parent / config["image_lock"]).resolve() != lock_path.resolve() + ): + raise ValueError("deployment must reference the canonical image lock") + if set(delivery.get("components") or {}) != sites - {"wikipedia"}: + raise ValueError("site delivery inventory differs") + for site in sites: + image = lock["images"][site] + if not re.fullmatch(r".+@sha256:[a-f0-9]{64}", image.get("pinned", "")): + raise ValueError(f"image must have a valid digest: {site}") + if image.get("architecture") not in {"amd64", "arm64"} or image.get("os") != "linux": + raise ValueError(f"image platform is invalid: {site}") + row = runtime["sites"][site] + eval_row = evaluation["environments"][f"__{site.upper()}__"] + if len(eval_row["urls"]) != 1 or urlparse(eval_row["urls"][0]).port != row["port"]: + raise ValueError(f"site port differs: {site}") + if urlparse(evaluation["control_urls"][site]).port != row["control_port"]: + raise ValueError(f"control port differs: {site}") + if site in runtime["credentials_env"] and runtime["credentials_env"][site] != eval_row.get( + "credentials_env" + ): + raise ValueError(f"credential variable names differ: {site}") + component = delivery["components"].get(site) + if component is not None: + if "image" in component or component.get("architecture") != image["architecture"]: + raise ValueError(f"site delivery must use the canonical image identity: {site}") + map_site = delivery["components"]["map"] + names = [Path(urlparse(url).path).name for url in map_site["data_urls"]] + digests = map_site["data_archive_sha256"] + sizes = map_site["data_archive_bytes"] + if ( + len(names) != 3 + or len(set(names)) != 3 + or set(names) != set(digests) + or set(names) != set(sizes) + or any(not re.fullmatch(r"[a-f0-9]{64}", str(value)) for value in digests.values()) + or any(type(value) is not int or value <= 0 for value in sizes.values()) + ): + raise ValueError("Map archive checksum contract is incomplete") return { "status": "pass", - "webarena_verified_version": expected_version, "sites": len(sites), + "image_lock": "canonical", "runtime_templates": "matching", - "map_archive_checksums": "complete", - "environment_pull_request": contract["environment_pull_request"], } -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--environment-repo", type=Path, required=True) - parser.add_argument("--contract", type=Path, default=DEFAULT_CONTRACT) - args = parser.parse_args() - print(json.dumps(verify(args.environment_repo, args.contract), ensure_ascii=False)) - - if __name__ == "__main__": - main() + print(json.dumps(verify(), ensure_ascii=False)) diff --git a/tests/environments/test_webarena_backend.py b/tests/environments/test_webarena_backend.py new file mode 100644 index 000000000..5bef570d7 --- /dev/null +++ b/tests/environments/test_webarena_backend.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import docker +import pytest +from webarena_verified.environments.container.config import get_container_config +from webarena_verified.environments.container.manager import ContainerManager +from webarena_verified.types.task import WebArenaSite + +from harness.environments.webarena.backend import DockerBackend + + +def test_container_run_translates_upstream_host_to_container_port_mapping() -> None: + calls = [] + containers = SimpleNamespace(run=lambda *args, **kwargs: calls.append((args, kwargs))) + backend = DockerBackend(SimpleNamespace(containers=containers)) + + backend.container_run( + name="webarena_verified_map", + image="example/map@sha256:" + "a" * 64, + port_mappings={23000: 3000, 23001: 5555}, + volume_mappings={}, + ) + + assert calls[0][1]["ports"] == {"3000/tcp": 23000, "5555/tcp": 23001} + + +@pytest.mark.parametrize( + ("site", "expected_variable", "expected_value"), + [ + (WebArenaSite.SHOPPING, "WA_ENV_CTRL_CMD_TIMEOUT_SEC", "600"), + (WebArenaSite.SHOPPING_ADMIN, "WA_ENV_CTRL_CMD_TIMEOUT_SEC", "600"), + (WebArenaSite.GITLAB, "WA_ENV_CTRL_SKIP_RECONFIGURE", "true"), + ], +) +def test_pinned_upstream_start_uses_expected_name_ports_and_environment( + site: WebArenaSite, expected_variable: str, expected_value: str +) -> None: + calls = [] + + class Containers: + def get(self, _name: str): + raise docker.errors.NotFound("missing") + + def run(self, *args, **kwargs): + calls.append((args, kwargs)) + + backend = DockerBackend(SimpleNamespace(containers=Containers())) + manager = ContainerManager(site=site, backend=backend) + config = get_container_config(site=site) + + manager.start(port=23000, env_ctrl_port=23001, wait=False) + + assert manager.container_name == f"webarena_verified_{site.value}" + assert calls[0][1]["name"] == manager.container_name + assert calls[0][1]["ports"] == { + f"{config.container_port}/tcp": 23000, + f"{config.env_ctrl_port}/tcp": 23001, + } + assert calls[0][1]["environment"][expected_variable] == expected_value + + +def test_deferred_removal_stops_and_retains_the_named_container() -> None: + actions = [] + + class Container: + id = "1234567890abcdef" + status = "running" + + def reload(self): + actions.append("reload") + + def stop(self, *, timeout: int): + actions.append(("stop", timeout)) + + def rename(self, name: str): + actions.append(("rename", name)) + + def remove(self, *, force: bool): + actions.append(("remove", force)) + + backend = DockerBackend( + SimpleNamespace(containers=SimpleNamespace(get=lambda _name: Container())), + defer_removal=True, + ) + + backend.container_remove(name="webarena_verified_shopping") + + retired = "webarena_verified_shopping__retired__1234567890ab" + assert actions == ["reload", ("stop", 10), ("rename", retired)] + assert backend.retired_containers == [retired] diff --git a/tests/environments/test_webarena_cli.py b/tests/environments/test_webarena_cli.py new file mode 100644 index 000000000..686950122 --- /dev/null +++ b/tests/environments/test_webarena_cli.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import json +import sys +from types import SimpleNamespace + +import pytest + +from harness.environments.webarena.cli import main + + +def test_batch_failure_retains_completed_site_receipts(monkeypatch, tmp_path) -> None: + receipt = tmp_path / "batch.json" + + class Manager: + def verify(self, name: str): + if name == "reddit": + raise RuntimeError("second site unavailable") + return {"site": name, "status": "running"} + + monkeypatch.setattr( + "harness.environments.webarena.cli.RuntimeConfig.load", + lambda _path: SimpleNamespace(sites={"map": object(), "reddit": object()}), + ) + monkeypatch.setattr( + "harness.environments.webarena.cli.EnvironmentManager", + lambda *_args, **_kwargs: Manager(), + ) + monkeypatch.setattr( + sys, + "argv", + [ + "webarena-env", + "verify", + "--config", + str(tmp_path / "config.json"), + "--site", + "map", + "--site", + "reddit", + "--receipt", + str(receipt), + ], + ) + + with pytest.raises(RuntimeError, match="second site unavailable"): + main() + + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["status"] == "partial_failure" + assert payload["failed_site"] == "reddit" + assert payload["sites"] == [{"site": "map", "status": "running"}] diff --git a/tests/environments/test_webarena_environment_config.py b/tests/environments/test_webarena_environment_config.py new file mode 100644 index 000000000..8719a6f32 --- /dev/null +++ b/tests/environments/test_webarena_environment_config.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from harness.environments.webarena.config import RuntimeConfig, normalized_architecture + +ROOT = Path(__file__).resolve().parents[2] + + +def _runtime(tmp_path: Path, *, hostname: str = "127.0.0.1") -> Path: + lock = json.loads( + (ROOT / "evaluation/browser-use-public/webarena-environment-images.json").read_text( + encoding="utf-8" + ) + ) + (tmp_path / "images.lock.json").write_text(json.dumps(lock), encoding="utf-8") + runtime = json.loads( + (ROOT / "deploy/webarena/runtime.example.json").read_text(encoding="utf-8") + ) + runtime["hostname"] = hostname + runtime["image_lock"] = "images.lock.json" + path = tmp_path / "runtime.json" + path.write_text(json.dumps(runtime), encoding="utf-8") + return path + + +def test_runtime_loads_every_digest_pinned_site(tmp_path: Path) -> None: + config = RuntimeConfig.load(_runtime(tmp_path)) + + assert set(config.sites) == { + "shopping_admin", + "shopping", + "reddit", + "gitlab", + "map", + "wikipedia", + } + assert config.sites["gitlab"].architecture == "amd64" + assert config.sites["wikipedia"].architecture == "arm64" + assert all("@sha256:" in site.image for site in config.sites.values()) + + +def test_example_hostname_must_be_replaced() -> None: + with pytest.raises(ValueError, match="replace the example placeholder"): + RuntimeConfig.load(ROOT / "deploy/webarena/runtime.example.json") + + +def test_architecture_aliases_cover_intel_and_amd_x86_hosts() -> None: + assert normalized_architecture("x86_64") == "amd64" + assert normalized_architecture("amd64") == "amd64" + assert normalized_architecture("aarch64") == "arm64" + + +def test_missing_locked_architecture_fails_before_docker_use(tmp_path: Path) -> None: + path = _runtime(tmp_path) + lock_path = tmp_path / "images.lock.json" + lock = json.loads(lock_path.read_text(encoding="utf-8")) + lock["images"]["map"]["architecture"] = "" + lock_path.write_text(json.dumps(lock), encoding="utf-8") + + with pytest.raises(ValueError, match="explicitly amd64 or arm64"): + RuntimeConfig.load(path) diff --git a/tests/environments/test_webarena_environment_manager.py b/tests/environments/test_webarena_environment_manager.py new file mode 100644 index 000000000..c61625b61 --- /dev/null +++ b/tests/environments/test_webarena_environment_manager.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from harness.environments.webarena.config import RuntimeConfig, SiteConfig +from harness.environments.webarena.manager import EnvironmentManager + + +class MissingContainers: + def get(self, _name: str): + import docker + + raise docker.errors.NotFound("missing") + + +def _config() -> RuntimeConfig: + return RuntimeConfig( + path=Path(__file__), + hostname="127.0.0.1", + sites={ + "map": SiteConfig( + name="map", + image="example/map@sha256:" + "a" * 64, + architecture="amd64", + port=23000, + control_port=23001, + ), + "wikipedia": SiteConfig( + name="wikipedia", + image="example/wiki@sha256:" + "b" * 64, + architecture="arm64", + port=18888, + control_port=18889, + ), + }, + ) + + +@pytest.mark.parametrize(("site", "internal_port"), [("shopping_admin", 80), ("gitlab", 8023)]) +def test_manager_overrides_host_ports_without_changing_internal_ports(site, internal_port): + config = _config() + config.sites[site] = SiteConfig( + name=site, + image="example/site@sha256:" + "a" * 64, + architecture="amd64", + port=17780, + control_port=17781, + ) + manager = EnvironmentManager(config, SimpleNamespace(info=lambda: {"Architecture": "x86_64"})) + upstream = manager._manager(config.sites[site]) + assert upstream.config.docker_img == config.sites[site].image + assert upstream.config.host_port == 17780 + assert upstream.config.host_env_ctrl_port == 17781 + assert upstream.config.container_port == internal_port + + +def test_verify_accepts_intel_x86_64_for_amd64_site() -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + + receipt = manager.verify("map") + + assert receipt["daemon_architecture"] == "amd64" + assert receipt["status"] == "not_created" + + +def test_verify_rejects_arm_image_on_x86_without_emulation() -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + + with pytest.raises(RuntimeError, match="requires native arm64"): + manager.verify("wikipedia") + + +def test_wait_control_retries_transient_invalid_json(monkeypatch) -> None: + class Response(BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + responses = iter((Response(b"not-json"), Response(b'{"success": true}'))) + monkeypatch.setattr( + "harness.environments.webarena.manager.urllib.request.urlopen", + lambda *_args, **_kwargs: next(responses), + ) + monkeypatch.setattr("harness.environments.webarena.manager.time.sleep", lambda _seconds: None) + + EnvironmentManager._wait_control("http://127.0.0.1:23001", timeout=1) + + +def test_verify_uses_upstream_manager_container_name(monkeypatch) -> None: + requested = [] + + class Containers(MissingContainers): + def get(self, name: str): + requested.append(name) + return super().get(name) + + client = SimpleNamespace(info=lambda: {"Architecture": "x86_64"}, containers=Containers()) + manager = EnvironmentManager(_config(), client) + monkeypatch.setattr( + manager, "_manager", lambda _site: SimpleNamespace(container_name="upstream-map") + ) + + receipt = manager.verify("map") + + assert requested == ["upstream-map"] + assert receipt["container_name"] == "upstream-map" + + +@pytest.mark.parametrize( + ("site_name", "expected_steps"), + [ + ("gitlab", ["control", "entrypoint", "explicit", "control"]), + ("reddit", ["control", "entrypoint", "explicit", "control"]), + ("shopping", ["control", "entrypoint", "verified"]), + ("shopping_admin", ["control", "entrypoint", "verified"]), + ], +) +def test_start_runs_only_the_declared_site_initialization( + monkeypatch, site_name: str, expected_steps: list[str] +) -> None: + config = _config() + config.sites[site_name] = SiteConfig( + name=site_name, + image="example/site@sha256:" + "c" * 64, + architecture="amd64", + port=23000, + control_port=23001, + ) + container = SimpleNamespace() + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, + containers=SimpleNamespace(get=lambda _name: container), + ) + manager = EnvironmentManager(config, client) + steps = [] + upstream = SimpleNamespace( + container_name=f"webarena_verified_{site_name}", + start=lambda **_kwargs: SimpleNamespace( + url="http://127.0.0.1:23000", + env_ctrl_url="http://127.0.0.1:23001", + ), + ) + monkeypatch.setattr(manager, "_manager", lambda _site: upstream) + monkeypatch.setattr(manager, "verify", lambda *_args, **_kwargs: {"status": "running"}) + monkeypatch.setattr(manager, "_wait_control", lambda *_args, **_kwargs: steps.append("control")) + monkeypatch.setattr( + manager, + "_wait_entrypoint_init", + lambda *_args, **_kwargs: steps.append("entrypoint"), + ) + monkeypatch.setattr( + manager, "_explicit_init", lambda *_args, **_kwargs: steps.append("explicit") + ) + monkeypatch.setattr( + manager, + "_verify_entrypoint_init", + lambda *_args, **_kwargs: steps.append("verified"), + ) + + receipt = manager.start(site_name, reset=True, timeout=1) + + assert steps == expected_steps + assert receipt["command"] == "reset" + assert receipt["status"] == "running" + + +def test_stop_delegates_to_pinned_upstream_manager(monkeypatch) -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + calls = [] + monkeypatch.setattr( + manager, + "_manager", + lambda _site: SimpleNamespace( + container_name="upstream-map", stop=lambda: calls.append("stop") + ), + ) + + receipt = manager.stop("map") + + assert calls == ["stop"] + assert receipt == {"command": "stop", "site": "map", "container_name": "upstream-map"} diff --git a/tests/environments/test_webarena_map_archives.py b/tests/environments/test_webarena_map_archives.py new file mode 100644 index 000000000..8a0518c3e --- /dev/null +++ b/tests/environments/test_webarena_map_archives.py @@ -0,0 +1,78 @@ +"""Map data volumes may only be initialized from locked archive bytes.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from harness.environments.webarena.map_archives import DEFAULT_CONTRACT, verify_archives + + +def _contract(tmp_path: Path, *, wrong_digest: bool = False) -> Path: + archives = {"osm_tile_server.tar": b"tile", "nominatim_volumes.tar": b"geo"} + for name, data in archives.items(): + (tmp_path / name).write_bytes(data) + checksums = {name: hashlib.sha256(data).hexdigest() for name, data in archives.items()} + if wrong_digest: + checksums["nominatim_volumes.tar"] = "0" * 64 + contract = { + "components": { + "map": { + "data_urls": [f"https://example.invalid/{name}" for name in archives], + "data_archive_sha256": checksums, + "data_archive_bytes": {name: len(data) for name, data in archives.items()}, + } + } + } + path = tmp_path / "contract.json" + path.write_text(json.dumps(contract), encoding="utf-8") + return path + + +def test_map_archive_verifier_accepts_exact_local_bytes(tmp_path: Path) -> None: + rows = verify_archives(tmp_path, _contract(tmp_path)) + assert [row["archive"] for row in rows] == [ + "osm_tile_server.tar", + "nominatim_volumes.tar", + ] + + +def test_map_archive_verifier_uses_canonical_contract_outside_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _contract(tmp_path) + monkeypatch.chdir(tmp_path) + assert DEFAULT_CONTRACT.is_absolute() and DEFAULT_CONTRACT.is_file() + with pytest.raises(ValueError, match="size mismatch: osm_tile_server.tar"): + verify_archives(tmp_path) + + +def test_map_archive_verifier_rejects_wrong_bytes(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="nominatim_volumes.tar"): + verify_archives(tmp_path, _contract(tmp_path, wrong_digest=True)) + + +def test_map_archive_verifier_rejects_missing_file(tmp_path: Path) -> None: + contract = _contract(tmp_path) + (tmp_path / "osm_tile_server.tar").unlink() + with pytest.raises(FileNotFoundError): + verify_archives(tmp_path, contract) + + +def test_map_archive_verifier_rejects_truncated_file_before_hashing(tmp_path: Path) -> None: + contract = _contract(tmp_path) + (tmp_path / "osm_tile_server.tar").write_bytes(b"bad") + with pytest.raises(ValueError, match="size mismatch: osm_tile_server.tar"): + verify_archives(tmp_path, contract) + + +def test_map_archive_verifier_rejects_inventory_drift(tmp_path: Path) -> None: + contract = _contract(tmp_path) + payload = json.loads(contract.read_text(encoding="utf-8")) + payload["components"]["map"]["data_urls"].append("https://example.invalid/other.tar") + contract.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="inventory"): + verify_archives(tmp_path, contract) diff --git a/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py deleted file mode 100644 index cb7410c6e..000000000 --- a/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import json -from types import SimpleNamespace - -from scripts.manage_webarena_environment import _manager - - -def test_manager_applies_explicit_host_ports(tmp_path, monkeypatch): - pinned = "am1n3e/webarena-verified-shopping_admin@sha256:" + "a" * 64 - lock = tmp_path / "images.json" - lock.write_text( - json.dumps( - { - "schema": "lex_webarena_environment_images/1", - "images": { - "shopping_admin": { - "pinned": pinned, - "architecture": "amd64", - } - }, - } - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "scripts.manage_webarena_environment.docker.from_env", - lambda: SimpleNamespace(info=lambda: {"Architecture": "x86_64"}), - ) - - manager = _manager( - "shopping_admin", - lock, - port=17780, - control_port=17781, - ) - - assert manager.config.docker_img == pinned - assert manager.config.host_port == 17780 - assert manager.config.host_env_ctrl_port == 17781 - assert manager.config.container_port == 80 - - -def test_manager_keeps_gitlab_internal_health_port(tmp_path, monkeypatch): - pinned = "am1n3e/webarena-verified-gitlab@sha256:" + "b" * 64 - lock = tmp_path / "images.json" - lock.write_text( - json.dumps( - { - "schema": "lex_webarena_environment_images/1", - "images": { - "gitlab": { - "pinned": pinned, - "architecture": "amd64", - } - }, - } - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "scripts.manage_webarena_environment.docker.from_env", - lambda: SimpleNamespace(info=lambda: {"Architecture": "amd64"}), - ) - - manager = _manager("gitlab", lock, port=18023, control_port=18024) - - assert manager.config.host_port == 18023 - assert manager.config.container_port == 8023 diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index d4d491140..3bbc09d14 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -1,13 +1,14 @@ from __future__ import annotations import json +import threading import urllib.error from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest -from harness.trajectory_replay.public_webarena.rainbond_slices import ( +from harness.environments.webarena.rainbond import ( LEASE_SCHEMA, RECEIPT_SCHEMA, STATEFUL_SITES, @@ -26,6 +27,43 @@ INITIALIZER_IMAGE = "alpine@sha256:" + "a" * 64 +def test_slow_release_does_not_block_acquisition_of_another_slot(tmp_path, monkeypatch): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + lease = acquire_slice(output, "task-a") + resetting = threading.Event() + finish_reset = threading.Event() + + class Pool: + def __init__(self, _config): + pass + + def reset(self, receipt_path): + resetting.set() + assert finish_reset.wait(5) + receipt = json.loads(receipt_path.read_text()) + receipt["app_id"] = 201 + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + with ThreadPoolExecutor(max_workers=2) as executor: + releasing = executor.submit( + release_slice, tmp_path / "pool.json", output, lease["lease_id"] + ) + try: + assert resetting.wait(2) + acquiring = executor.submit(acquire_slice, output, "task-b") + assert acquiring.result(timeout=2)["app_id"] == 102 + finally: + finish_reset.set() + assert releasing.result(timeout=2)["ready_app_id"] == 201 + + class FakeAPI: def __init__(self, *, fail_domain_for: str | None = None) -> None: self.calls = [] @@ -101,7 +139,8 @@ def request(self, method, path, body=None): { "service_id": f"service-initializer-{self.next_app_id}", "status": ( - "closed" if path.removesuffix("/services") in self.stopped_apps + "closed" + if path.removesuffix("/services") in self.stopped_apps else "running" ), } @@ -197,7 +236,7 @@ def request(self, method, path, body=None): pool = _pool(tmp_path, api) ticks = iter([0, 100_000]) monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.time.monotonic", + "harness.environments.webarena.rainbond.time.monotonic", lambda: next(ticks), ) with pytest.raises(TimeoutError, match="did not stop before deletion"): @@ -224,7 +263,10 @@ def request(self, method, path, body=None): def configure(): pool._coalesce_internal_ports( - 200, "service-shopping", site_port=80, control_port=8877, + 200, + "service-shopping", + site_port=80, + control_port=8877, control_alias="SHOPPING_CTRL", ) @@ -551,9 +593,7 @@ def runtime_config(self, _receipt): def destroy(self, receipt_path): destroyed.append(receipt_path) - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) with pytest.raises(RuntimeError, match="second slot"): provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) @@ -584,9 +624,7 @@ def runtime_config(self, _receipt): def destroy(self, _receipt_path): raise RuntimeError("rollback failed") - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) with pytest.raises(RuntimeError, match="provision failed") as caught: provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) @@ -630,9 +668,7 @@ def destroy(self, receipt_path): destroyed.append(receipt_path) receipt_path.unlink() - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) output = tmp_path / "pool" assert len(scale_pool(tmp_path / "config.json", output, "warm", 3)) == 3 @@ -674,9 +710,7 @@ def destroy(self, receipt_path): destroyed.append(receipt_path.parent.name) receipt_path.unlink() - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) output = tmp_path / "runtime" with pytest.raises(RuntimeError, match="slot 2 failed"): @@ -693,9 +727,7 @@ def fake_scale(config, output, pool_id, count, *, minimum_ready): calls.append((config, output, pool_id, count, minimum_ready)) return [] - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.scale_pool", fake_scale - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.scale_pool", fake_scale) reconcile_pool_capacity( tmp_path / "pool.json", tmp_path / "runtime", "warm", 3, minimum_ready=1 @@ -722,9 +754,7 @@ def validate_receipt(self, receipt_path): def destroy(self, receipt_path): destroyed.append(receipt_path) - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) output = tmp_path / "runtime" for slot in (1, 2, 3): slot_dir = output / f"slice-{slot}" @@ -760,9 +790,7 @@ def destroy(self, receipt_path): raise RuntimeError("slot 3 delete failed") receipt_path.unlink() - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) output = tmp_path / "runtime" for slot in (1, 2, 3): slot_dir = output / f"slice-{slot}" @@ -856,9 +884,7 @@ def reset(self, _receipt_path): def runtime_config(self, receipt): return {"app_id": receipt["app_id"]} - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) @@ -880,9 +906,7 @@ def __init__(self, _config): def reset(self, _receipt_path): raise RuntimeError("replacement failed") - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) with pytest.raises(RuntimeError, match="replacement failed"): release_slice(tmp_path / "pool.json", output, lease["lease_id"]) @@ -920,9 +944,7 @@ def validate_receipt(self, receipt_path): def runtime_config(self, receipt): return {"app_id": receipt["app_id"]} - monkeypatch.setattr( - "harness.trajectory_replay.public_webarena.rainbond_slices.RainbondSlicePool", Pool - ) + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) original_unlink = Path.unlink failures = 0 diff --git a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py index 56e266af8..cc1963b2e 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py @@ -1,136 +1,72 @@ from __future__ import annotations import json -from pathlib import Path +import shutil import pytest -from scripts.verify_webarena_env_contract import verify - - -def _write_environment_repo(root: Path, evaluation_lock: Path) -> Path: - payload = json.loads(evaluation_lock.read_text(encoding="utf-8")) - compact = { - "schema": "lexmount_webarena_images/1", - "webarena_verified_version": payload["webarena_verified_version"], - "images": { - site: {key: row[key] for key in ("pinned", "architecture", "os")} - for site, row in payload["images"].items() - }, - } - path = root / "config/images.lock.json" - path.parent.mkdir(parents=True) - path.write_text(json.dumps(compact), encoding="utf-8") - runtime = json.loads( - Path("examples/trajectory_replay/public_webarena/webarena.config.example.json").read_text(encoding="utf-8") - ) - from urllib.parse import urlparse - - template = { - "sites": { - site: { - "port": urlparse(runtime["environments"][f"__{site.upper()}__"]["urls"][0]).port, - "control_port": urlparse(runtime["control_urls"][site]).port, - } - for site in compact["images"] - }, - "credentials_env": { - site: runtime["environments"][f"__{site.upper()}__"]["credentials_env"] - for site in ("shopping_admin", "shopping", "reddit", "gitlab") - }, - } - (root / "config/runtime.example.json").write_text(json.dumps(template), encoding="utf-8") - archive_names = ("osm_tile_server.tar", "nominatim_volumes.tar", "osrm_routing.tar") - delivery = { - "webarena_verified_version": compact["webarena_verified_version"], - "image_lock": "images.lock.json", - "components": { - site: {"image": row["pinned"]} - for site, row in compact["images"].items() - if site != "wikipedia" - } - } - delivery["components"]["map"].update( - { - "data_urls": [f"https://example.invalid/{name}" for name in archive_names], - "data_archive_sha256": {name: "0" * 64 for name in archive_names}, - "data_archive_bytes": {name: 1 for name in archive_names}, - } - ) - (root / "config/site-delivery-contract.json").write_text( - json.dumps(delivery), encoding="utf-8" - ) - return path - - -def test_cross_repo_contract_accepts_matching_images(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - assert verify(tmp_path)["status"] == "pass" - - -def test_cross_repo_contract_rejects_one_changed_digest(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - path = _write_environment_repo(tmp_path, evaluation_lock) - payload = json.loads(path.read_text(encoding="utf-8")) - payload["images"]["map"]["pinned"] = "example.invalid/map@sha256:" + "0" * 64 - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="map.pinned"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_port_and_credential_drift(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/runtime.example.json" - payload = json.loads(path.read_text(encoding="utf-8")) - payload["sites"]["map"]["port"] += 1 - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="site port differs: map"): - verify(tmp_path) - payload["sites"]["map"]["port"] -= 1 - payload["credentials_env"]["shopping_admin"]["username"] = "WRONG_USERNAME" - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="credential variable names differ: shopping_admin"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_missing_map_digest(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - del payload["components"]["map"]["data_archive_sha256"]["osrm_routing.tar"] - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="Map archive checksum contract"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_site_delivery_lock_path_drift(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - payload["image_lock"] = "../images.lock.json" - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="image lock path differs"): - verify(tmp_path) - - -def test_cross_repo_contract_reports_missing_site_delivery_contract(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - (tmp_path / "config/site-delivery-contract.json").unlink() - with pytest.raises(ValueError, match="missing site delivery contract"): - verify(tmp_path) - - -def test_cross_repo_contract_reports_missing_image_lock_field(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - del payload["image_lock"] - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="image_lock is missing"): - verify(tmp_path) +from scripts.verify_webarena_env_contract import ROOT, verify + + +def _checkout(tmp_path): + for name in ( + "evaluation/browser-use-public/webarena-environment-images.json", + "examples/trajectory_replay/public_webarena/webarena.config.example.json", + "deploy/webarena/runtime.example.json", + "deploy/webarena/site-delivery-contract.json", + ): + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, target) + return tmp_path + + +def test_repository_contract_uses_one_image_lock(): + assert verify()["status"] == "pass" + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("image_lock", "other.json", "canonical image lock"), + ("webarena_verified_version", "0.0.0", "package version"), + ], +) +def test_delivery_rejects_wrong_binding(tmp_path, field, value, message): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + data[field] = value + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match=message): + verify(root) + + +def test_deployment_rejects_port_drift(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/runtime.example.json" + data = json.loads(path.read_text()) + data["sites"]["reddit"]["port"] += 1 + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="site port differs: reddit"): + verify(root) + + +def test_deployment_rejects_duplicate_image_identity(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + data["components"]["reddit"]["image"] = "reddit:latest" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="canonical image identity"): + verify(root) + + +def test_map_contract_rejects_missing_checksum(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + del data["components"]["map"]["data_archive_sha256"]["osrm_routing.tar"] + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="Map archive checksum"): + verify(root) diff --git a/tests/trajectory_replay/public_webarena/test_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_webarena_environment.py index e14dcd63d..873c18e6d 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_environment.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_environment.py @@ -6,10 +6,10 @@ import pytest +from harness.environments.webarena.backend import DockerBackend from harness.trajectory_replay.public_webarena.suite import BrowserTask from harness.trajectory_replay.public_webarena.webarena_environment import ( WebArenaEnvironment, - _DockerClientBackend, wall_clock_timeout, ) @@ -526,7 +526,7 @@ class Containers: def run(self, *args, **kwargs): calls.append((args, kwargs)) - backend = _DockerClientBackend(SimpleNamespace(containers=Containers())) + backend = DockerBackend(SimpleNamespace(containers=Containers())) backend.container_run( name=f"webarena_verified_{site}", @@ -552,7 +552,7 @@ class Containers: def run(self, *args, **kwargs): calls.append((args, kwargs)) - backend = _DockerClientBackend(SimpleNamespace(containers=Containers())) + backend = DockerBackend(SimpleNamespace(containers=Containers())) backend.container_run( name="webarena_verified_gitlab", @@ -570,7 +570,7 @@ def run(self, *args, **kwargs): def test_stateful_backend_can_defer_large_container_removal(): container = _Container([]) - backend = _DockerClientBackend( + backend = DockerBackend( SimpleNamespace(containers=_Containers(container)), defer_removal=True ) @@ -582,7 +582,7 @@ def test_stateful_backend_can_defer_large_container_removal(): def test_deferred_removal_rejects_shared_writable_mounts(): - backend = _DockerClientBackend( + backend = DockerBackend( SimpleNamespace(containers=SimpleNamespace(run=lambda *_args, **_kwargs: None)), defer_removal=True, ) From fa1343f5dc6b988f0a4ffd62c98fbac454e944a1 Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 13:03:03 +0800 Subject: [PATCH 22/23] Distinguish release cleanup from task preparation resets --- .../public_webarena/README.md | 6 +- harness/environments/webarena/rainbond.py | 34 ++++++---- .../public_webarena/test_rainbond_slices.py | 65 +++++++++++++++++-- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index dc680ca49..e4d48e588 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -186,7 +186,11 @@ PVC without verifying its ownership and that no retained workload mounts it. The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus 100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable resources, not cluster totals. Reconcile the pool to current active demand plus the chosen -idle reserve; the controller destroys excess unleased slices when demand falls. Rainbond +idle reserve; the controller destroys excess unleased slices when demand falls. +Use `scale --count 0 --minimum-ready 0` to remove all unleased slots when no +idle reserve is needed. Active leases remain protected. Release progress is +persisted before reset so a retry cannot confuse task preparation with cleanup +or unnecessarily repeat a completed replacement after a local write failure. Rainbond may also need temporary headroom while a replacement generation starts and for rolling updates. When that headroom is unavailable, use `stop_before_replace` for that deployment rather than weakening isolation or sharing writable sites. diff --git a/harness/environments/webarena/rainbond.py b/harness/environments/webarena/rainbond.py index 49609c58e..a20fd11fe 100644 --- a/harness/environments/webarena/rainbond.py +++ b/harness/environments/webarena/rainbond.py @@ -889,8 +889,10 @@ def scale_pool( minimum_ready: int = 1, ) -> list[Path]: """Reconcile a persistent warm pool, never scaling below minimum_ready.""" - if minimum_ready < 1 or count < minimum_ready: - raise ValueError("count must be at least minimum_ready, and minimum_ready must be positive") + if minimum_ready < 0 or count < minimum_ready: + raise ValueError( + "count must be at least minimum_ready, and minimum_ready must not be negative" + ) with _pool_lock(output_dir): pool = RainbondSlicePool(config_path) pool_id = _validate_pool_id(pool_id) @@ -1047,18 +1049,24 @@ def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[st receipt_path = slot_dir / "slice-receipt.json" runtime_config_path = slot_dir / "webarena.config.json" current_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - if int(current_receipt["app_id"]) != int(lease["app_id"]): + release_from_app_id = lease.get("release_from_app_id") + if release_from_app_id is not None and int(current_receipt["app_id"]) != int( + release_from_app_id + ): current_receipt = pool.validate_receipt(receipt_path) - if runtime_config_path.is_file() and json.loads( - runtime_config_path.read_text(encoding="utf-8") - ) == pool.runtime_config(current_receipt): - lease_path.unlink() - return { - "lease_id": lease_id, - "released_app_id": int(lease["app_id"]), - "ready_app_id": int(current_receipt["app_id"]), - "config": str(runtime_config_path.resolve()), - } + _atomic_json(runtime_config_path, pool.runtime_config(current_receipt)) + lease_path.unlink() + return { + "lease_id": lease_id, + "released_app_id": int(lease["app_id"]), + "ready_app_id": int(current_receipt["app_id"]), + "config": str(runtime_config_path.resolve()), + } + if release_from_app_id is None: + # Task preparation also replaces instances. Only this release's + # replacement proves the task's mutations have been removed. + lease = {**lease, "release_from_app_id": int(current_receipt["app_id"])} + _atomic_json(lease_path, lease) receipt = pool.reset(receipt_path) _atomic_json(runtime_config_path, pool.runtime_config(receipt)) lease_path.unlink() diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py index 3bbc09d14..a74888f9d 100644 --- a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -678,6 +678,8 @@ def destroy(self, receipt_path): assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2"] with pytest.raises(ValueError, match="minimum_ready"): scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=1) + assert scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=0) == [] + assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2", "slice-1"] def test_scale_pool_rolls_back_every_new_slot_after_partial_failure(tmp_path, monkeypatch): @@ -736,7 +738,8 @@ def fake_scale(config, output, pool_id, count, *, minimum_ready): assert calls[0][3:] == (4, 1) -def test_scale_down_is_atomic_when_an_excess_slice_is_leased(tmp_path, monkeypatch): +@pytest.mark.parametrize("count", [0, 1]) +def test_scale_down_is_atomic_when_an_excess_slice_is_leased(tmp_path, monkeypatch, count): destroyed = [] class Config: @@ -763,7 +766,7 @@ def destroy(self, receipt_path): (output / "slice-2" / "slice-lease.json").write_text("{}") with pytest.raises(RuntimeError, match="leased Rainbond slices"): - scale_pool(tmp_path / "pool.json", output, "warm", 1) + scale_pool(tmp_path / "pool.json", output, "warm", count, minimum_ready=0) assert destroyed == [] @@ -916,7 +919,8 @@ def reset(self, _receipt_path): acquire_slice(output, "task-b") -def test_release_retry_only_unlinks_after_reset_was_persisted(tmp_path, monkeypatch): +@pytest.mark.parametrize("failure_point", ["unlink", "config"]) +def test_release_retry_only_unlinks_after_reset_was_persisted(tmp_path, monkeypatch, failure_point): output = tmp_path / "runtime" slot_dir = _ready_slot(output, 1, 101) lease = acquire_slice(output, "task-a") @@ -946,17 +950,30 @@ def runtime_config(self, receipt): monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) original_unlink = Path.unlink + original_replace = Path.replace failures = 0 def fail_first_lease_unlink(path, *args, **kwargs): nonlocal failures - if path == lease_path and failures == 0: + if failure_point == "unlink" and path == lease_path and failures == 0: failures += 1 raise OSError("transient unlink failure") return original_unlink(path, *args, **kwargs) + def fail_first_config_replace(path, target): + nonlocal failures + if ( + failure_point == "config" + and Path(target) == slot_dir / "webarena.config.json" + and failures == 0 + ): + failures += 1 + raise OSError("transient config failure") + return original_replace(path, target) + monkeypatch.setattr(Path, "unlink", fail_first_lease_unlink) - with pytest.raises(OSError, match="transient unlink failure"): + monkeypatch.setattr(Path, "replace", fail_first_config_replace) + with pytest.raises(OSError, match="transient .* failure"): release_slice(tmp_path / "pool.json", output, lease["lease_id"]) assert lease_path.exists() assert len(reset_calls) == 1 @@ -966,3 +983,41 @@ def fail_first_lease_unlink(path, *args, **kwargs): assert result["ready_app_id"] == 201 assert not lease_path.exists() assert len(reset_calls) == 1 + + +def test_release_does_not_mistake_task_preparation_for_cleanup(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + receipt_path = slot_dir / "slice-receipt.json" + receipt = json.loads(receipt_path.read_text()) + receipt["app_id"] = 201 + receipt_path.write_text(json.dumps(receipt)) + (slot_dir / "webarena.config.json").write_text(json.dumps({"app_id": 201})) + reset_calls = [] + + class Pool: + def __init__(self, _config): + pass + + def validate_receipt(self, path): + return json.loads(path.read_text()) + + def runtime_config(self, current): + return {"app_id": current["app_id"]} + + def reset(self, path): + reset_calls.append(path) + assert ( + json.loads((slot_dir / "slice-lease.json").read_text())["release_from_app_id"] + == 201 + ) + clean = {**receipt, "app_id": 301} + path.write_text(json.dumps(clean)) + return clean + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert result["ready_app_id"] == 301 + assert reset_calls == [receipt_path] + assert not (slot_dir / "slice-lease.json").exists() From 1b423c2f4c2f762f5a340effc7999289443e7d9e Mon Sep 17 00:00:00 2001 From: kevin <xxx> Date: Thu, 17 Sep 2026 13:04:30 +0800 Subject: [PATCH 23/23] Validate every WebArena template against the canonical image lock --- .../public_webarena/webarena.config.example.json | 2 +- scripts/verify_webarena_env_contract.py | 16 +++++++++++----- .../test_webarena_env_contract.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/examples/trajectory_replay/public_webarena/webarena.config.example.json b/examples/trajectory_replay/public_webarena/webarena.config.example.json index 4ad8efa7d..281dbad8d 100644 --- a/examples/trajectory_replay/public_webarena/webarena.config.example.json +++ b/examples/trajectory_replay/public_webarena/webarena.config.example.json @@ -1,5 +1,5 @@ { - "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "image_lock": "../../../evaluation/browser-use-public/webarena-environment-images.json", "docker_site_hostname": "host.docker.internal", "environments": { "__SHOPPING_ADMIN__": { diff --git a/scripts/verify_webarena_env_contract.py b/scripts/verify_webarena_env_contract.py index 5d53f2ad5..de718725f 100644 --- a/scripts/verify_webarena_env_contract.py +++ b/scripts/verify_webarena_env_contract.py @@ -15,11 +15,12 @@ def verify(root: Path = ROOT) -> dict: lock = json.loads(lock_path.read_text(encoding="utf-8")) runtime_path = root / "deploy/webarena/runtime.example.json" runtime = json.loads(runtime_path.read_text(encoding="utf-8")) - evaluation = json.loads( - ( - root / "examples/trajectory_replay/public_webarena/webarena.config.example.json" - ).read_text(encoding="utf-8") + evaluation_path = ( + root / "examples/trajectory_replay/public_webarena/webarena.config.example.json" ) + evaluation = json.loads(evaluation_path.read_text(encoding="utf-8")) + pool_path = root / "examples/trajectory_replay/public_webarena/rainbond-pool.example.json" + pool = json.loads(pool_path.read_text(encoding="utf-8")) delivery_path = root / "deploy/webarena/site-delivery-contract.json" delivery = json.loads(delivery_path.read_text(encoding="utf-8")) if lock.get("schema") != "lex_webarena_environment_images/1": @@ -32,7 +33,12 @@ def verify(root: Path = ROOT) -> dict: sites = {"shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia"} if set(lock.get("images") or {}) != sites or set(runtime.get("sites") or {}) != sites: raise ValueError("site inventory differs") - for path, config in ((runtime_path, runtime), (delivery_path, delivery)): + for path, config in ( + (runtime_path, runtime), + (delivery_path, delivery), + (evaluation_path, evaluation), + (pool_path, pool), + ): if ( not config.get("image_lock") or (path.parent / config["image_lock"]).resolve() != lock_path.resolve() diff --git a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py index cc1963b2e..04c2d2947 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py @@ -12,6 +12,7 @@ def _checkout(tmp_path): for name in ( "evaluation/browser-use-public/webarena-environment-images.json", "examples/trajectory_replay/public_webarena/webarena.config.example.json", + "examples/trajectory_replay/public_webarena/rainbond-pool.example.json", "deploy/webarena/runtime.example.json", "deploy/webarena/site-delivery-contract.json", ): @@ -25,6 +26,17 @@ def test_repository_contract_uses_one_image_lock(): assert verify()["status"] == "pass" +@pytest.mark.parametrize("template", ["webarena.config", "rainbond-pool"]) +def test_evaluation_templates_reject_wrong_relative_lock_path(tmp_path, template): + root = _checkout(tmp_path) + path = root / f"examples/trajectory_replay/public_webarena/{template}.example.json" + data = json.loads(path.read_text()) + data["image_lock"] = "../../evaluation/browser-use-public/webarena-environment-images.json" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="canonical image lock"): + verify(root) + + @pytest.mark.parametrize( ("field", "value", "message"), [