diff --git a/.env.example b/.env.example index 6f47cf88e..e863fd18d 100644 --- a/.env.example +++ b/.env.example @@ -71,3 +71,6 @@ ZENROWS_API_KEY= SCRAPFLY_API_KEY= OXYLABS_USERNAME= OXYLABS_PASSWORD= +# Rainbond OpenAPI token used only by the disposable WebArena slice manager. +# Create it in Rainbond: Personal Center -> Access Token. +RAINBOND_API_TOKEN= diff --git a/Makefile b/Makefile index 035bcc17e..ec53dd4a0 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ naming-check: runtime-check test-python: runtime-check $(TEST_PY) -m unittest discover -s tests -v $(TEST_PY) -m pytest tests/browser_use -q + $(TEST_PY) -m pytest tests/environments tests/test_manage_webarena_rainbond_pool.py -q $(TEST_PY) -m pytest tests/trajectory_replay -q test-ui: diff --git a/deploy/webarena-rainbond/initializer.sh b/deploy/webarena-rainbond/initializer.sh new file mode 100755 index 000000000..59630003a --- /dev/null +++ b/deploy/webarena-rainbond/initializer.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -eu + +wait_ctrl() { + url="$1" + attempts=0 + until wget -qO- "$url/status" >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge "${ENV_CTRL_MAX_ATTEMPTS:-360}" ]; then + echo "env-ctrl timeout: $url" >&2 + exit 1 + fi + sleep "${ENV_CTRL_RETRY_SECONDS:-5}" + done +} + +init_ctrl() { + url="$1" + attempts=0 + while :; do + set +e + response="$(wget -qO- --post-data='' "$url/init" 2>&1)" + result=$? + set -e + if echo "$response" | grep -Eq '"success"[[:space:]]*:[[:space:]]*true'; then + echo "$response" + return + fi + # env-ctrl can restart its managed process before flushing the HTTP response. + # BusyBox wget reports that successful reset path as an empty/broken response. + if [ "$result" -ne 0 ] && echo "$response" | grep -Eq 'error getting response|Connection reset'; then + wait_ctrl "$url" + echo "env-ctrl init completed after control endpoint restart: $url" + return + fi + attempts=$((attempts + 1)) + if [ "$attempts" -ge "${ENV_CTRL_MAX_ATTEMPTS:-360}" ]; then + echo "env-ctrl init timeout: $url: $response" >&2 + exit 1 + fi + echo "env-ctrl init retry: $url" >&2 + sleep "${ENV_CTRL_RETRY_SECONDS:-5}" + done +} + +wait_ctrl "$REDDIT_CTRL_URL" +wait_ctrl "$GITLAB_CTRL_URL" +init_ctrl "$REDDIT_CTRL_URL" +init_ctrl "$GITLAB_CTRL_URL" + +mkdir -p /www +printf 'ready\n' >/www/index.html +if busybox --list | grep -qx httpd; then + exec busybox httpd -f -p 8080 -h /www +fi +if busybox --list | grep -qx nc; then + while :; do + printf 'HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\nready\n' | + busybox nc -l -p 8080 + done +fi +echo "initializer image has neither the httpd nor nc BusyBox applet" >&2 +exit 1 diff --git a/deploy/webarena/LICENSE b/deploy/webarena/LICENSE new file mode 100644 index 000000000..cfb4b00ca --- /dev/null +++ b/deploy/webarena/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LexMount + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deploy/webarena/README.md b/deploy/webarena/README.md new file mode 100644 index 000000000..23f762698 --- /dev/null +++ b/deploy/webarena/README.md @@ -0,0 +1,27 @@ +# WebArena 环境 + +Docker 和 Rainbond 环境由 `harness/environments/webarena/` 维护。任务和评分器来自 +固定版本的 `webarena-verified` 依赖。镜像身份只有一份： +`evaluation/browser-use-public/webarena-environment-images.json`。 + +`runtime.example.json` 提供本地 Docker 的主机和端口模板；`site-delivery-contract.json` +描述镜像内端口、初始化方式与只读 Map 数据挂载。Map 为可选环境，契约的存在不会触发下载或部署。 +有状态站点从镜像内的初始数据库重建，不挂载跨槽位共享的可写数据库目录。 +GitLab 在 Rainbond 中需要独立的 2 GiB `/dev/shm` 内存挂载。 + +复制运行配置到 Git 外并填写 `hostname`，将 `image_lock` 改为镜像锁绝对路径： + +```sh +uv run --frozen python scripts/manage_webarena_environment.py verify --config <配置> --site reddit +uv run --frozen python scripts/manage_webarena_environment.py start --config <配置> --site reddit --receipt <回执> +uv run --frozen python scripts/manage_webarena_environment.py reset --config <配置> --site reddit --receipt <回执> +uv run --frozen python scripts/verify_webarena_env_contract.py +``` + +Rainbond 的独立槽位、租约、并发访问检查和资源回收见 +[运行示例](../../examples/trajectory_replay/public_webarena/README.md)。 +运行配置、地址、凭据和回执保存在 Git 外；代码和配置模板可公开提交。 + +Docker 管理代码和 Map 校验契约源自 +[`lexmount/webarena-env`](https://github.com/lexmount/webarena-env/tree/daecdf8b46e71ccc70d7cc2fdaf92a83af899624)， +保留其 [MIT 许可证](LICENSE)。 diff --git a/deploy/webarena/runtime.example.json b/deploy/webarena/runtime.example.json new file mode 100644 index 000000000..d3d437da9 --- /dev/null +++ b/deploy/webarena/runtime.example.json @@ -0,0 +1,49 @@ +{ + "schema": "lexmount_webarena_runtime/1", + "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "hostname": "${HOST}", + "sites": { + "shopping_admin": { + "port": 7780, + "control_port": 7781 + }, + "shopping": { + "port": 7770, + "control_port": 7771 + }, + "gitlab": { + "port": 8023, + "control_port": 8024 + }, + "reddit": { + "port": 9999, + "control_port": 9998 + }, + "map": { + "port": 3000, + "control_port": 3001 + }, + "wikipedia": { + "port": 8888, + "control_port": 8889 + } + }, + "credentials_env": { + "shopping_admin": { + "username": "WEBARENA_SHOPPING_ADMIN_USERNAME", + "password": "WEBARENA_SHOPPING_ADMIN_PASSWORD" + }, + "shopping": { + "username": "WEBARENA_SHOPPING_USERNAME", + "password": "WEBARENA_SHOPPING_PASSWORD" + }, + "reddit": { + "username": "WEBARENA_REDDIT_USERNAME", + "password": "WEBARENA_REDDIT_PASSWORD" + }, + "gitlab": { + "username": "WEBARENA_GITLAB_USERNAME", + "password": "WEBARENA_GITLAB_PASSWORD" + } + } +} diff --git a/deploy/webarena/site-delivery-contract.json b/deploy/webarena/site-delivery-contract.json new file mode 100644 index 000000000..4461c18d7 --- /dev/null +++ b/deploy/webarena/site-delivery-contract.json @@ -0,0 +1,137 @@ +{ + "schema": "lexmount_webarena_site_delivery/1", + "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "webarena_verified_version": "1.2.3", + "components": { + "shopping_admin": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/", + "baseline": "fresh_container_writable_layer", + "init": "official_entrypoint", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_CMD_TIMEOUT_SEC" + ], + "required_env_values": { + "WA_ENV_CTRL_CMD_TIMEOUT_SEC": "600" + }, + "mounts": [] + }, + "shopping": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/customer/account/login", + "baseline": "fresh_container_writable_layer", + "init": "official_entrypoint", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_CMD_TIMEOUT_SEC" + ], + "required_env_values": { + "WA_ENV_CTRL_CMD_TIMEOUT_SEC": "600" + }, + "mounts": [] + }, + "reddit": { + "architecture": "amd64", + "service_port": 80, + "control_port": 8877, + "health_path": "/login", + "baseline": "fresh_container_writable_layer", + "init": "official_explicit_env_ctrl", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL" + ], + "required_env_values": {}, + "mounts": [] + }, + "gitlab": { + "architecture": "amd64", + "service_port": 8023, + "control_port": 8877, + "health_path": "/users/sign_in", + "baseline": "fresh_container_writable_layer", + "init": "official_explicit_env_ctrl", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "WA_ENV_CTRL_SKIP_RECONFIGURE" + ], + "required_env_values": { + "WA_ENV_CTRL_SKIP_RECONFIGURE": "true" + }, + "mounts": [] + }, + "map": { + "architecture": "amd64", + "service_port": 8080, + "control_port": 8877, + "health_path": "/", + "baseline": "verified_read_only_dataset", + "init": "official_dataset_setup", + "required_env_names": [ + "WA_ENV_CTRL_EXTERNAL_SITE_URL" + ], + "required_env_values": {}, + "mounts": [ + { + "volume": "webarena_verified_map_tile_db", + "path": "/data/database" + }, + { + "volume": "webarena_verified_map_routing_car", + "path": "/data/routing/car" + }, + { + "volume": "webarena_verified_map_routing_bike", + "path": "/data/routing/bike" + }, + { + "volume": "webarena_verified_map_routing_foot", + "path": "/data/routing/foot" + }, + { + "volume": "webarena_verified_map_nominatim_db", + "path": "/data/nominatim/postgres" + }, + { + "volume": "webarena_verified_map_nominatim_flatnode", + "path": "/data/nominatim/flatnode" + }, + { + "volume": "webarena_verified_map_website_db", + "path": "/var/lib/postgresql/14/main" + }, + { + "volume": "webarena_verified_map_tiles", + "path": "/data/tiles" + }, + { + "volume": "webarena_verified_map_style", + "path": "/data/style" + } + ], + "data_urls": [ + "https://webarena-map-server-data.s3.amazonaws.com/osm_tile_server.tar", + "https://webarena-map-server-data.s3.amazonaws.com/nominatim_volumes.tar", + "https://webarena-map-server-data.s3.amazonaws.com/osrm_routing.tar" + ], + "data_archive_sha256": { + "osm_tile_server.tar": "2d802a915af14a13bcf393e9abd23d0de003eb043137736a54876410e05c4df2", + "nominatim_volumes.tar": "888ad53f754948f660796b6fcb440294a3dec647e141d606b1c7cb2616695baa", + "osrm_routing.tar": "99a4d10aae831fb3410b7d4743be03d9e02691283ec5105952839b84b9204806" + }, + "data_archive_bytes": { + "osm_tile_server.tar": 41280327680, + "nominatim_volumes.tar": 124774901760, + "osrm_routing.tar": 21278935040 + }, + "data_archive_digest_basis": "5090_downloaded_bytes_not_upstream_signature" + } + }, + "excluded_on_amd64": [ + "wikipedia" + ] +} diff --git a/docs/webarena-fixed-trajectory-reproduction.md b/docs/webarena-fixed-trajectory-reproduction.md index 58bc33531..9071a500a 100644 --- a/docs/webarena-fixed-trajectory-reproduction.md +++ b/docs/webarena-fixed-trajectory-reproduction.md @@ -1,17 +1,19 @@ -# WebArena 固定轨迹回放：双仓库复现契约 +# WebArena 固定轨迹回放：环境与复现 这是一套 RPA 式固定轨迹回放，不是 Agent browser-use。正式运行不调用 Agent、模型或 LLM judge；两个浏览器重放相同的 WebArena 公开人工轨迹，再由固定 evaluator 验收最终 回答、服务状态和 HAR。 -## 两个仓库各自负责什么 +## 代码与依赖 -- `lexmount/webarena-env#2`：固定 WebArena-Verified 1.2.3、站点镜像 digest、原生架构、 - 端口、启动/重置和机器可读回执。 -- 本仓库 PR：固定公开轨迹清单、离线修复记录、资格门禁、Chromium/Moli 镜像、动作 +- `harness/environments/webarena/`：Docker 和 Rainbond 站点生命周期、初始化与机器可读回执； + `deploy/webarena/` 保存部署配置和数据挂载契约。 +- `harness/trajectory_replay/public_webarena/`：公开轨迹清单、离线修复记录、资格门禁、Chromium/Moli 镜像、动作 重放、HAR、evaluator、一次失败重试、首次/最终成功统计、证据审计和独立报告发布。 -两者共同足以在新机器上重新执行同一协议，但不会仅凭 Git 还原旧运行的逐字节证据： +WebArena-Verified 1.2.3 作为独立依赖提供任务和评分器；全部官方镜像只在 +`evaluation/browser-use-public/webarena-environment-images.json` 中锁定。 +本仓库提供在新机器上重新执行协议所需的代码，但不会仅凭 Git 还原旧运行的逐字节证据： 原始截图、HAR、attempts、manifest 和 summary 按仓库规则保存在 Git 外。Git 中的 `evaluation/browser-use-public/current-report.json` 保存当前公开结果与外部证据哈希， 用于证明报告绑定，而不是替代一次新运行。 @@ -28,14 +30,16 @@ LLM judge；两个浏览器重放相同的 WebArena 公开人工轨迹，再由 ## 从空机器复现 -1. 检出并固定两个 PR 的合并提交；两个仓库都执行 `uv sync --frozen` 和各自的检查。 - 再在本仓库运行 - `uv run --frozen python scripts/verify_webarena_env_contract.py --environment-repo `， - 确认上游版本、六个站点、镜像 digest 和架构完全一致。 -2. 在 `webarena-env` 复制 `config/runtime.example.json`，填写浏览器容器可访问的主机名， - 运行 `webarena-env verify/start` 并保留回执。Map 的三份 tar 须先用 - `uv run --frozen python -m webarena_env.map_archives --data-dir <下载目录>` 核对 SHA-256， - 再按官方 setup 初始化九个数据卷。账号只通过环境变量提供。 +1. 检出并固定本仓库提交，执行 `make setup` 和 `make ci`。运行 + `uv run --frozen python scripts/verify_webarena_env_contract.py`，检查部署和评测 + 配置使用同一份镜像锁以及一致的端口、凭据变量名。 +2. 复制 `deploy/webarena/runtime.example.json` 到 Git 外，填写可访问的 `hostname`， + 将 `image_lock` 设置为本仓库镜像锁的绝对路径。使用 + `uv run --frozen python scripts/manage_webarena_environment.py verify --config <配置> --site reddit` + 校验环境，将 `verify` 换为 `start` 或 `reset` 启动或重建站点，通过 `--receipt <路径>` 保存回执。 + 仅在需要 Map 的任务范围内准备地图数据；`setup --config <配置> --site map --data-dir <目录>` + 会先核对三份归档的 SHA-256，再运行官方初始化。当前 Rainbond 部署不包含 Map。 + 账号只通过环境变量提供。 3. 在本仓库复制 `examples/trajectory_replay/public_webarena/webarena.config.example.json`，使用同一组端口、 镜像 digest、网站地址与凭据变量名；运行环境 smoke，确认 CDP、登录、HAR 与站点 origin。 示例默认共用本机 Docker daemon，不假定存在某个用户的 rootless socket。若使用两条 @@ -55,9 +59,11 @@ uv run --frozen python -m harness.trajectory_replay.public_webarena.cli \ 5. 运行 `examples/trajectory_replay/public_webarena/verify_metrics.py` 和报告打包器。任何缺失回答、动作、终止 原因、评分，或进入浏览器阶段后缺失截图、最终页面、HAR，都按失败处理。 -正式运行前必须核对两个仓库的 WebArena-Verified 版本和六个站点镜像 digest 完全一致。 +正式运行前必须核对 WebArena-Verified 版本和任务涉及的站点镜像 digest。 这项检查不允许用 mutable tag、跨架构模拟或手工修改数据库替代。 -当前正式执行器按 Docker 容器身份验证并重建有状态站点，不能直接用 Rainbond 中的 -远程有状态组件替代。Rainbond 的镜像部署和站点连通性可以先单独验证；要发布与本协议 -等价的成绩，仍需受控重建与初态回执接入评测执行器。 +执行器支持 Docker 容器重建和带回执的 Rainbond 独立槽位重建。每个并发执行者必须 +独占一整个有状态槽位；只读 Wikipedia 可共享。Rainbond 使用方法见 +[环境示例](../examples/trajectory_replay/public_webarena/README.md)。HTTP 并发测试只证明 +网页可访问；正式运行仍须生成与本次重置、镜像、初态和就绪检查绑定的回执。 +共享 Kiwix 的 amd64 部署与官方 arm64 镜像身份不同，不能直接沿用官方镜像的验证回执。 diff --git a/evaluation/browser-use-public/webarena-env-contract.json b/evaluation/browser-use-public/webarena-env-contract.json deleted file mode 100644 index d451df0e3..000000000 --- a/evaluation/browser-use-public/webarena-env-contract.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "schema": "lexmount_webarena_cross_repo_contract/1", - "environment_repository": "https://github.com/lexmount/webarena-env", - "environment_pull_request": "https://github.com/lexmount/webarena-env/pull/2", - "webarena_verified_version": "1.2.3", - "environment_lock": "config/images.lock.json", - "environment_runtime_template": "config/runtime.example.json", - "environment_site_delivery_contract": "config/site-delivery-contract.json", - "evaluation_lock": "evaluation/browser-use-public/webarena-environment-images.json", - "evaluation_runtime_template": "examples/trajectory_replay/public_webarena/webarena.config.example.json", - "required_sites": [ - "shopping_admin", - "shopping", - "reddit", - "gitlab", - "wikipedia", - "map" - ] -} diff --git a/examples/trajectory_replay/public_webarena/README.md b/examples/trajectory_replay/public_webarena/README.md index d39889440..e4d48e588 100644 --- a/examples/trajectory_replay/public_webarena/README.md +++ b/examples/trajectory_replay/public_webarena/README.md @@ -32,3 +32,168 @@ Measured attempts, screenshots, HAR, and reports stay outside Git. The repositor the suite, qualification reasons, evaluator rules, source bindings, and a sanitized publication summary. Use `verify_metrics.py` to audit raw measurements and `rebuild_report.py` to render from existing attempts; neither reruns browsers. + +## Disposable Rainbond environments + +For concurrent state-changing runs, use a complete Rainbond slice per worker instead of +sharing sites or relying on separate login accounts. A slice contains Shopping Admin, +Shopping, Reddit, and GitLab with independent writable container layers and unique +domains. Reset deletes and recreates the whole slice from the pinned baseline; destroy +removes the application. Read-only Map and Wikipedia services may be shared when +separately deployed; they are different services, not substitutes for each other. + +The baseline also contains a small digest-pinned initializer component. It calls the +Reddit and GitLab `env-ctrl` endpoints only over Rainbond's internal service network, +then exposes a harmless readiness page on its unique slice domain. The unauthenticated +control endpoints themselves are never published through the gateway. +`deploy/webarena-rainbond/initializer.sh` is mounted as a Rainbond `config-file` at +`/opt/webarena/initializer.sh` and run as `/bin/sh /opt/webarena/initializer.sh` by +the pinned Alpine image; port 8080 serves only the final readiness marker. Keeping the +script in a mounted file avoids Rainbond's whitespace-only image-command parsing. +The initializer depends on the copied Reddit and GitLab components so Rainbond makes +their internal Service names resolvable. Keep only the site-specific exported variables +`REDDIT_CTRL_URL` and `GITLAB_CTRL_URL`; remove generic duplicated `CTRL_HOST` and +`CTRL_PORT` exports before creating both dependency relations. + +Both Magento baseline components must also mount a Rainbond `config-file` named +`wa-hide-csp-*` at `/etc/nginx/conf.d/00-webarena-hide-csp.conf` before they are copied. +Its complete content is: + +```text +fastcgi_hide_header Content-Security-Policy-Report-Only; +``` + +The Nginx configuration changes only the report-only response header. That header +exceeds the upstream-header limit of the APISIX gateway used by this Rainbond +installation and otherwise produces a gateway 502 even while the Pod and port are +healthy. The images retain their default startup commands, and application readiness +still requires a successful request through each public slice domain. + +The GitLab baseline must reproduce the upstream Compose `shm_size: 2gb` contract with +native Kubernetes attributes, not a Rainbond `memoryfs` storage volume. In the GitLab +component's **Other settings → Kubernetes attributes**, set `volumes` to: + +```yaml +- name: gitlab-shm + emptyDir: + medium: Memory + sizeLimit: 2Gi +``` + +and set `volumeMounts` to: + +```yaml +- name: gitlab-shm + mountPath: /dev/shm +``` + +Rainbond attribute values omit the top-level `volumes:` and `volumeMounts:` keys. +Verify inside a running GitLab container that `/dev/shm` is a 2 GiB `tmpfs`; a +Rainbond `memoryfs` volume is backed by the cluster filesystem in this deployment and +is not equivalent. The four seeded site images otherwise keep their data in the +slice's disposable writable layers. Do not mount empty volumes over their preloaded +database directories. + +Copy `rainbond-pool.example.json`, set `RAINBOND_API_TOKEN` in the process environment, +and reconcile the warm pool to the active task count plus at least one idle slice. +Additional slices are created in bounded parallel and removed when demand falls: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config scale --pool-id --active \ + --minimum-ready 1 --output +``` + +Admission is an atomic lease, so two schedulers cannot assign the same writable sites: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config acquire --output --owner +``` + +Use the returned config for exactly one task. Releasing a lease creates and verifies a +clean replacement application before deleting the dirty generation and making the slot +available again. If replacement fails, the lease remains held, the dirty generation is +stopped, and the slot cannot be reused: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config release --output --lease-id +``` + +The default `replace_before_destroy` strategy allows the dirty and replacement generations +to overlap while the replacement becomes ready. This preserves available capacity during +normal concurrent operation without imposing a fixed pool size. `stop_before_replace` is +an operator-selected fallback for a temporarily constrained cluster: it stops the dirty +generation before creating its replacement. Either way, a failed reset leaves the lease +held and the dirty generation stopped rather than returning an invalid slot to the pool. + +For a one-shot pool that will be destroyed in full, use: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config provision --pool-id --count --output +``` + +Pass each leased `webarena.config.json` as a separate `--webarena-config` argument to +`scripts/run_webarena_qualification.py`. Overlapping site batches run concurrently only +when distinct slice configs are supplied. Afterwards, release each reusable lease. For a +one-shot pool, destroy the slices instead: + +```text +uv run --frozen python scripts/manage_webarena_rainbond_pool.py \ + --config destroy --slice-dir /slice-1 \ + --slice-dir /slice-2 +``` + +Each receipt records create, copy, configure, deploy-request, readiness, and total cold-start +durations. Use those measurements to size the warm pool; task admission should allocate a +ready slice rather than waiting for a cold start. + +After a slice becomes ready, exercise all four public origins concurrently: + +```text +WEBARENA_RAINBOND_SMOKE_URLS='{ + "shopping_admin":"http://", + "shopping":"http://", + "reddit":"http://", + "gitlab":"http://" +}' uv run --frozen python -m pytest \ + tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py \ + -m real_smoke -q -s +``` + +The smoke test makes eight simultaneous page loads per site, checks that each origin +returns its expected application, and rejects a per-site P95 above ten seconds. +To verify multiple slices in the same concurrency window, supply a JSON array of +four-site objects instead. Duplicate mutable-site URLs are rejected. Set +`WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL` to a shared Kiwix origin to add eight +simultaneous requests for its May 2022 Boston Celtics article. A green HTTP test +does not validate login, mutations, or database isolation; check those separately +before assigning state-changing tasks. + +Keep the stopped baseline application used by `source_app_id`, `source_services`, +and `initializer_service`: it is required for new slices and resets even though it +uses no running compute. Remove superseded slice generations only after their +replacement receipt is authoritative and no lease references the old app. Never +delete a shared Wikipedia data volume as part of a mutable slice reset. +Deletion first stops the app's services and waits for their stopped status. If stopping +fails, the app record and receipt remain available for recovery. After deletion, also +inspect Kubernetes-native workloads and storage: a missing Rainbond app record alone +does not prove that the platform removed every underlying resource. Never remove a +PVC without verifying its ownership and that no retained workload mounts it. + +The baseline resource requests total 10 CPU cores and 20 GiB of memory per slice, plus +100 millicores and 128 MiB for the initializer. Capacity planning must use schedulable +resources, not cluster totals. Reconcile the pool to current active demand plus the chosen +idle reserve; the controller destroys excess unleased slices when demand falls. +Use `scale --count 0 --minimum-ready 0` to remove all unleased slots when no +idle reserve is needed. Active leases remain protected. Release progress is +persisted before reset so a retry cannot confuse task preparation with cleanup +or unnecessarily repeat a completed replacement after a local write failure. Rainbond +may also need temporary headroom while a replacement generation starts and for rolling +updates. When that headroom is unavailable, use `stop_before_replace` for that deployment +rather than weakening isolation or sharing writable sites. + +Rainbond is the runtime data plane and this repository's lease controller is the source of +truth for allocation, reset, quarantine, and cleanup. diff --git a/examples/trajectory_replay/public_webarena/rainbond-pool.example.json b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json new file mode 100644 index 000000000..266ca7890 --- /dev/null +++ b/examples/trajectory_replay/public_webarena/rainbond-pool.example.json @@ -0,0 +1,31 @@ +{ + "schema": "lex_webarena_rainbond_pool/1", + "api_base_url": "https://rainbond.example.com", + "team_id": "replace-with-efficiency-team-id", + "team_name": "replace-with-efficiency-team-name", + "region_name": "rainbond", + "source_app_id": 192, + "source_services": { + "shopping_admin": "replace-with-baseline-service-id", + "shopping": "replace-with-baseline-service-id", + "reddit": "replace-with-baseline-service-id", + "gitlab": "replace-with-baseline-service-id" + }, + "initializer_service": "replace-with-slice-initializer-service-id", + "initializer_image": "alpine@sha256:replace-with-pinned-multiarch-digest", + "initializer_component_name": "slice-initializer", + "source_component_names": { + "shopping_admin": "shopping-admin", + "shopping": "shopping", + "reddit": "reddit", + "gitlab": "gitlab" + }, + "app_prefix": "webarena-slice", + "domain_suffix": "dev.example.com", + "token_env": "RAINBOND_API_TOKEN", + "image_lock": "../../../evaluation/browser-use-public/webarena-environment-images.json", + "readiness_timeout_seconds": 1800, + "max_parallel_provisions": 2, + "reset_strategy": "replace_before_destroy", + "shared_environments": {} +} diff --git a/examples/trajectory_replay/public_webarena/webarena.config.example.json b/examples/trajectory_replay/public_webarena/webarena.config.example.json index 4ad8efa7d..281dbad8d 100644 --- a/examples/trajectory_replay/public_webarena/webarena.config.example.json +++ b/examples/trajectory_replay/public_webarena/webarena.config.example.json @@ -1,5 +1,5 @@ { - "image_lock": "../../evaluation/browser-use-public/webarena-environment-images.json", + "image_lock": "../../../evaluation/browser-use-public/webarena-environment-images.json", "docker_site_hostname": "host.docker.internal", "environments": { "__SHOPPING_ADMIN__": { diff --git a/harness/environments/__init__.py b/harness/environments/__init__.py new file mode 100644 index 000000000..ddd02400b --- /dev/null +++ b/harness/environments/__init__.py @@ -0,0 +1 @@ +"""Reproducible site environments used by browser evaluations.""" diff --git a/harness/environments/webarena/__init__.py b/harness/environments/webarena/__init__.py new file mode 100644 index 000000000..715229d20 --- /dev/null +++ b/harness/environments/webarena/__init__.py @@ -0,0 +1 @@ +"""WebArena Docker and Rainbond environment lifecycle.""" diff --git a/harness/environments/webarena/backend.py b/harness/environments/webarena/backend.py new file mode 100644 index 000000000..d0308b496 --- /dev/null +++ b/harness/environments/webarena/backend.py @@ -0,0 +1,115 @@ +"""Docker backend with WebArena initialization safeguards.""" + +from __future__ import annotations + +import socket +from typing import Any + +import docker + +MAGENTO_INIT_TIMEOUT_SECONDS = 600 + + +class DockerBackend: + """Bind WebArena's container manager to an explicit Docker SDK client.""" + + def __init__(self, client: Any, *, defer_removal: bool = False) -> None: + self.client = client + self.defer_removal = defer_removal + self.retired_containers: list[str] = [] + + def _container(self, name: str) -> Any | None: + try: + return self.client.containers.get(name) + except docker.errors.NotFound: + return None + + def container_exists(self, *, name: str) -> bool: + return self._container(name) is not None + + def container_running(self, *, name: str) -> bool: + container = self._container(name) + if container is None: + return False + container.reload() + return container.status == "running" + + def container_remove(self, *, name: str) -> None: + container = self._container(name) + if container is not None: + if not self.defer_removal: + container.remove(force=True) + return + # Removing WebArena's tens-of-gigabytes writable layer can block a + # rootless daemon for hours. Stopping releases the fixed ports; + # renaming preserves the old layer for deferred garbage collection. + # The next container still starts from the immutable pinned image, + # and these official images have no shared writable mounts. + container.reload() + if container.status == "running": + container.stop(timeout=10) + retired_name = f"{name}__retired__{container.id[:12]}" + container.rename(retired_name) + self.retired_containers.append(retired_name) + + def container_run( + self, + *, + name: str, + image: str, + port_mappings: dict[int, int], + volume_mappings: dict[str, str], + env_vars: dict[str, str] | None = None, + ) -> None: + environment = dict(env_vars or {}) + if self.defer_removal and volume_mappings: + raise RuntimeError( + "deferred WebArena container removal requires no shared writable mounts" + ) + if name in { + "webarena_verified_shopping", + "webarena_verified_shopping_admin", + }: + # Magento's official init performs cache/database work after the + # services become healthy. The image default of 180 seconds is + # too short on a busy qualification host and can leave the baked + # CMU origin active even though env-ctrl /status is healthy. + environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str(MAGENTO_INIT_TIMEOUT_SECONDS) + # GitLab's image entrypoint does not run env-ctrl init. Its fallback + # reconfiguration also rewrites the internal nginx listener to the + # public host port, breaking Docker's host-port -> 8023 mapping. Keep + # the baked internal listener here; WebArenaEnvironment performs one + # explicit official init after the service is healthy. + if name == "webarena_verified_gitlab": + environment["WA_ENV_CTRL_SKIP_RECONFIGURE"] = "true" + self.client.containers.run( + image, + detach=True, + name=name, + ports={ + f"{container_port}/tcp": host_port + for host_port, container_port in port_mappings.items() + }, + volumes={ + source: {"bind": target, "mode": "rw"} for source, target in volume_mappings.items() + }, + environment=environment, + ) + + def get_container_ports(self, *, name: str) -> dict[int, int]: + container = self._container(name) + if container is None: + return {} + container.reload() + result: dict[int, int] = {} + ports = container.attrs.get("NetworkSettings", {}).get("Ports") or {} + for key, bindings in ports.items(): + if bindings: + result[int(str(key).split("/", 1)[0])] = int(bindings[0]["HostPort"]) + return result + + def find_free_port(self) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as value: + value.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + value.bind(("", 0)) + return int(value.getsockname()[1]) diff --git a/harness/environments/webarena/cli.py b/harness/environments/webarena/cli.py new file mode 100644 index 000000000..99c43cc60 --- /dev/null +++ b/harness/environments/webarena/cli.py @@ -0,0 +1,88 @@ +"""Command-line interface for the pinned WebArena environment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from webarena_verified.environments.setup import setup_init +from webarena_verified.types.task import WebArenaSite + +from .config import SITES, RuntimeConfig +from .manager import EnvironmentManager +from .map_archives import verify_archives + + +def _write_receipt(path: Path | None, payload: dict[str, Any]) -> None: + rendered = json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + if path is not None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + print(rendered, end="") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=("setup", "verify", "start", "reset", "stop")) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--site", choices=SITES, action="append") + parser.add_argument("--receipt", type=Path) + parser.add_argument("--timeout", type=int, default=900) + parser.add_argument("--data-dir") + parser.add_argument("--defer-removal", action="store_true") + args = parser.parse_args() + if args.timeout < 1: + parser.error("--timeout must be positive") + config = RuntimeConfig.load(args.config) + names = tuple(args.site or config.sites) + if args.command == "setup" and ( + not args.data_dir or any(name not in {"map", "wikipedia"} for name in names) + ): + parser.error("setup requires --site map|wikipedia and --data-dir") + manager = ( + EnvironmentManager(config, defer_removal=args.defer_removal) + if args.command != "setup" + else None + ) + rows: list[dict[str, Any]] = [] + for name in names: + try: + if args.command == "setup": + archives = verify_archives(Path(args.data_dir)) if name == "map" else [] + result = setup_init(site=WebArenaSite(name), data_dir=Path(args.data_dir)) + rows.append({"site": name, "result": result, "archives": archives}) + elif args.command == "verify": + rows.append(manager.verify(name)) + elif args.command == "stop": + rows.append(manager.stop(name)) + else: + rows.append( + manager.start( + name, + reset=args.command == "reset", + timeout=args.timeout, + data_dir=args.data_dir, + ) + ) + except Exception: + _write_receipt( + args.receipt, + { + "schema": "lexmount_webarena_receipt/1", + "command": args.command, + "status": "partial_failure", + "failed_site": name, + "sites": rows, + }, + ) + raise + _write_receipt( + args.receipt, + {"schema": "lexmount_webarena_receipt/1", "command": args.command, "sites": rows}, + ) + + +if __name__ == "__main__": + main() diff --git a/harness/environments/webarena/config.py b/harness/environments/webarena/config.py new file mode 100644 index 000000000..df1f75000 --- /dev/null +++ b/harness/environments/webarena/config.py @@ -0,0 +1,80 @@ +"""Validated image lock and host-local runtime configuration.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +SITES = ("shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia") +ARCH_ALIASES = {"aarch64": "arm64", "x86_64": "amd64"} + + +@dataclass(frozen=True) +class SiteConfig: + name: str + image: str + architecture: str + port: int + control_port: int + + +@dataclass(frozen=True) +class RuntimeConfig: + path: Path + hostname: str + sites: dict[str, SiteConfig] + + @classmethod + def load(cls, path: str | Path) -> RuntimeConfig: + config_path = Path(path).resolve() + payload = json.loads(config_path.read_text(encoding="utf-8")) + if payload.get("schema") != "lexmount_webarena_runtime/1": + raise ValueError("invalid WebArena runtime schema") + hostname = str(payload.get("hostname") or "").strip() + if not hostname or "${" in hostname: + raise ValueError("runtime hostname must replace the example placeholder") + lock_path = Path(str(payload.get("image_lock") or "")) + if not lock_path.is_absolute(): + lock_path = config_path.parent / lock_path + lock = json.loads(lock_path.read_text(encoding="utf-8")) + if lock.get("schema") != "lex_webarena_environment_images/1": + raise ValueError("invalid WebArena image lock schema") + if lock.get("webarena_verified_version") != "1.2.3": + raise ValueError("image lock does not match webarena-verified 1.2.3") + runtime_sites = payload.get("sites") or {} + locked_sites = lock.get("images") or {} + sites: dict[str, SiteConfig] = {} + used_ports: set[int] = set() + for name in SITES: + runtime = runtime_sites.get(name) + image = locked_sites.get(name) + if not isinstance(runtime, dict) or not isinstance(image, dict): + raise ValueError(f"runtime or image lock is missing site {name}") + port = int(runtime.get("port") or 0) + control_port = int(runtime.get("control_port") or 0) + if not 1 <= port <= 65535 or not 1 <= control_port <= 65535: + raise ValueError(f"site {name} ports must be between 1 and 65535") + if port == control_port or {port, control_port} & used_ports: + raise ValueError(f"site {name} ports must be unique") + used_ports.update((port, control_port)) + pinned = str(image.get("pinned") or "") + if "@sha256:" not in pinned: + raise ValueError(f"site {name} image must be pinned by digest") + architecture = normalized_architecture(image.get("architecture")) + if architecture not in {"amd64", "arm64"}: + raise ValueError(f"site {name} architecture must be explicitly amd64 or arm64") + sites[name] = SiteConfig( + name=name, + image=pinned, + architecture=architecture, + port=port, + control_port=control_port, + ) + return cls(path=config_path, hostname=hostname, sites=sites) + + +def normalized_architecture(value: Any) -> str: + architecture = str(value or "") + return ARCH_ALIASES.get(architecture, architecture) diff --git a/harness/environments/webarena/manager.py b/harness/environments/webarena/manager.py new file mode 100644 index 000000000..65de486ea --- /dev/null +++ b/harness/environments/webarena/manager.py @@ -0,0 +1,204 @@ +"""Create and verify WebArena-Verified containers from immutable inputs.""" + +from __future__ import annotations + +import json +import re +import time +import urllib.error +import urllib.request +from typing import Any + +import docker +from webarena_verified.environments.container.config import get_container_config +from webarena_verified.environments.container.manager import ContainerManager +from webarena_verified.types.task import WebArenaSite + +from .backend import DockerBackend +from .config import RuntimeConfig, SiteConfig, normalized_architecture + +STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) + + +class EnvironmentManager: + def __init__( + self, + config: RuntimeConfig, + client: Any | None = None, + *, + defer_removal: bool = False, + ) -> None: + self.config = config + self.client = client or docker.from_env(timeout=900) + self.backend = DockerBackend(self.client, defer_removal=defer_removal) + + def _require_native_architecture(self, site: SiteConfig) -> str: + daemon_arch = normalized_architecture(self.client.info().get("Architecture")) + if daemon_arch != site.architecture: + raise RuntimeError( + f"{site.name} requires native {site.architecture}; Docker daemon is {daemon_arch}" + ) + return daemon_arch + + def _manager(self, site: SiteConfig) -> ContainerManager: + self._require_native_architecture(site) + upstream = get_container_config(site=WebArenaSite(site.name)) + locked = upstream.model_copy( + update={ + "docker_img": site.image, + "host_port": site.port, + "host_env_ctrl_port": site.control_port, + } + ) + return ContainerManager( + site=WebArenaSite(site.name), + config=locked, + backend=self.backend, + hostname=self.config.hostname, + ) + + def verify(self, name: str, *, require_running: bool = False) -> dict[str, Any]: + site = self.config.sites[name] + manager = self._manager(site) + daemon_arch = site.architecture + result: dict[str, Any] = { + "site": name, + "daemon_architecture": daemon_arch, + "required_architecture": site.architecture, + "requested_image": site.image, + "container_name": manager.container_name, + } + try: + container = self.client.containers.get(result["container_name"]) + except docker.errors.NotFound as exc: + if require_running: + raise RuntimeError( + f"WebArena container is missing: {result['container_name']}" + ) from exc + result["status"] = "not_created" + return result + container.reload() + image = container.image + if image is None: + raise RuntimeError(f"{name} container has no image metadata") + repo_digests = sorted(image.attrs.get("RepoDigests") or []) + if site.image not in repo_digests: + raise RuntimeError(f"{name} image mismatch: expected {site.image}, got {repo_digests}") + if require_running and container.status != "running": + raise RuntimeError(f"WebArena container is not running: {result['container_name']}") + upstream = get_container_config(site=WebArenaSite(name)) + expected_ports = { + int(upstream.container_port): site.port, + int(upstream.env_ctrl_port): site.control_port, + } + observed_ports = self.backend.get_container_ports(name=str(result["container_name"])) + if observed_ports != expected_ports: + raise RuntimeError( + f"{name} port mapping mismatch: expected {expected_ports}, got {observed_ports}" + ) + result.update( + { + "status": container.status, + "container_id": container.id, + "image_id": image.id, + "repo_digests": repo_digests, + "port_mappings": observed_ports, + "site_url": f"http://{self.config.hostname}:{site.port}", + "control_url": f"http://{self.config.hostname}:{site.control_port}", + } + ) + return result + + def start( + self, + name: str, + *, + reset: bool, + timeout: int, + data_dir: str | None = None, + ) -> dict[str, Any]: + site = self.config.sites[name] + manager = self._manager(site) + if not reset and self.backend.container_running(name=manager.container_name): + return {"command": "start", **self.verify(name, require_running=True)} + started = manager.start( + port=site.port, + env_ctrl_port=site.control_port, + data_dir=data_dir, + wait=False, + ) + container = self.client.containers.get(manager.container_name) + self._wait_control(started.env_ctrl_url, timeout=timeout) + if name in {"gitlab", "reddit"}: + self._wait_entrypoint_init(container, timeout=timeout) + self._explicit_init(container, started.url, site=name) + self._wait_control(started.env_ctrl_url, timeout=timeout) + elif name in {"shopping", "shopping_admin"}: + self._wait_entrypoint_init(container, timeout=timeout) + self._verify_entrypoint_init(container) + receipt = self.verify(name, require_running=True) + receipt.update( + { + "command": "reset" if reset else "start", + "site_url": started.url, + "control_url": started.env_ctrl_url, + "retired_containers": list(self.backend.retired_containers), + "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + ) + return receipt + + def stop(self, name: str) -> dict[str, Any]: + manager = self._manager(self.config.sites[name]) + manager.stop() + return {"command": "stop", "site": name, "container_name": manager.container_name} + + @staticmethod + def _wait_control(url: str, *, timeout: int) -> None: + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url.rstrip("/") + "/status", timeout=5) as response: + payload = json.load(response) + if response.status == 200 and payload.get("success") is True: + return + except (json.JSONDecodeError, OSError, urllib.error.URLError) as exc: + last_error = exc + time.sleep(0.5) + raise RuntimeError(f"WebArena control endpoint did not become ready: {last_error}") + + @staticmethod + def _wait_entrypoint_init(container: Any, *, timeout: int) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + processes = container.top(ps_args="-eo pid,args").get("Processes") or [] + active = any( + "environment_control.cli init" in " ".join(map(str, row)) + or re.search(r"\benv-ctrl\b.*\binit\b", " ".join(map(str, row))) + for row in processes + ) + if not active: + return + time.sleep(0.5) + raise RuntimeError("WebArena entrypoint initialization did not finish") + + @staticmethod + def _explicit_init(container: Any, url: str, *, site: str) -> None: + result = container.exec_run( + ["env-ctrl", "--verbose", "init", "--base-url", url], demux=False + ) + output = result.output.decode("utf-8", errors="replace") + try: + payload = json.loads(output) + except json.JSONDecodeError as exc: + raise RuntimeError(f"WebArena {site} init returned invalid JSON") from exc + if result.exit_code != 0 or payload.get("success") is not True: + raise RuntimeError(f"WebArena {site} init failed: {payload.get('message')}") + + @staticmethod + def _verify_entrypoint_init(container: Any) -> None: + result = container.exec_run(["sh", "-lc", "tail -n 1 /tmp/env-ctrl-init.log"], demux=False) + output = result.output.decode("utf-8", errors="replace").strip() + if result.exit_code != 0 or output != "[OK]": + raise RuntimeError("WebArena entrypoint initialization failed") diff --git a/harness/environments/webarena/map_archives.py b/harness/environments/webarena/map_archives.py new file mode 100644 index 000000000..03963361c --- /dev/null +++ b/harness/environments/webarena/map_archives.py @@ -0,0 +1,58 @@ +"""Check the exact Map archives before initializing persistent data volumes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from urllib.parse import urlparse + +DEFAULT_CONTRACT = ( + Path(__file__).resolve().parents[3] / "deploy/webarena/site-delivery-contract.json" +) + + +def verify_archives( + data_dir: Path, contract_path: Path = DEFAULT_CONTRACT +) -> list[dict[str, str | int]]: + """Check each Map archive against the contract size and SHA-256, failing on drift.""" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + map_site = contract["components"]["map"] + expected = map_site["data_archive_sha256"] + expected_bytes = map_site["data_archive_bytes"] + names = [Path(urlparse(url).path).name for url in map_site["data_urls"]] + if ( + len(names) != len(set(names)) + or set(names) != set(expected) + or set(names) != set(expected_bytes) + ): + raise ValueError("Map archive checksum inventory differs from upstream URLs") + rows: list[dict[str, str | int]] = [] + for name in names: + archive = data_dir / name + observed_bytes = archive.stat().st_size + if observed_bytes != expected_bytes[name]: + raise ValueError(f"Map archive size mismatch: {name}") + digest = hashlib.sha256() + with archive.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): + digest.update(chunk) + observed = digest.hexdigest() + if observed != expected[name]: + raise ValueError(f"Map archive checksum mismatch: {name}") + rows.append({"archive": name, "sha256": observed, "bytes": observed_bytes}) + return rows + + +def main() -> None: + """Run the archive verification CLI before initializing persistent Map volumes.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--contract", type=Path, default=DEFAULT_CONTRACT) + args = parser.parse_args() + print(json.dumps({"status": "pass", "archives": verify_archives(args.data_dir, args.contract)})) + + +if __name__ == "__main__": + main() diff --git a/harness/environments/webarena/rainbond.py b/harness/environments/webarena/rainbond.py new file mode 100644 index 000000000..a20fd11fe --- /dev/null +++ b/harness/environments/webarena/rainbond.py @@ -0,0 +1,1078 @@ +"""Provision disposable, fully isolated WebArena slices on Rainbond.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import threading +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import quote + +SCHEMA = "lex_webarena_rainbond_pool/1" +RECEIPT_SCHEMA = "lex_webarena_rainbond_slice/1" +LEASE_SCHEMA = "lex_webarena_rainbond_slice_lease/1" +STATEFUL_SITES = ("shopping_admin", "shopping", "reddit", "gitlab") +SITE_PORTS = {"shopping_admin": 80, "shopping": 80, "reddit": 80, "gitlab": 8023} +_SAFE_ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") +_LOCAL_POOL_LOCKS: dict[Path, threading.Lock] = {} +_LOCAL_POOL_LOCKS_GUARD = threading.Lock() + + +def _canonical_sha256(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _image_digest(value: str) -> str: + marker = "@sha256:" + if marker not in value: + raise ValueError(f"container image is not digest pinned: {value!r}") + return "sha256:" + value.rsplit(marker, 1)[1] + + +def _validate_pool_id(pool_id: str) -> str: + normalized = pool_id.strip().lower() + if not _SAFE_ID.fullmatch(normalized): + raise ValueError("pool_id must be a lowercase DNS-safe identifier") + return normalized + + +def _slot_sort_key(path: Path) -> tuple[int, int | str]: + suffix = path.name.removeprefix("slice-") + return (0, int(suffix)) if suffix.isdigit() else (1, path.name) + + +def _atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + content = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +def _attempt_cleanup(original: BaseException, description: str, action: Any) -> bool: + """Run cleanup without hiding the failure that made cleanup necessary.""" + try: + action() + except BaseException as cleanup_error: + original.add_note(f"cleanup failed ({description}): {cleanup_error}") + return False + return True + + +class RainbondAPI: + """Small OpenAPI client containing only the calls required by slice lifecycle.""" + + def __init__(self, base_url: str, token: str, *, timeout_seconds: float = 60) -> None: + self.base_url = base_url.rstrip("/") + self.token = token + self.timeout_seconds = timeout_seconds + + def request(self, method: str, path: str, body: Any | None = None) -> Any: + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request( + self.base_url + path, + data=data, + method=method, + headers={ + "Authorization": self.token, + "Accept": "application/json", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response: + payload = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:2000] + raise RuntimeError( + f"Rainbond OpenAPI {method} {path} failed with HTTP {exc.code}: {detail}" + ) from exc + if not payload: + return None + return json.loads(payload) + + +@dataclass(frozen=True) +class PoolConfig: + api_base_url: str + team_id: str + team_name: str + region_name: str + source_app_id: int + source_services: dict[str, str] + initializer_service: str + initializer_image: str + app_prefix: str + domain_suffix: str + token_env: str + image_lock: Path + shared_environments: dict[str, Any] + readiness_timeout_seconds: int + max_parallel_provisions: int + reset_strategy: str + + @classmethod + def load(cls, path: str | Path) -> tuple[PoolConfig, dict[str, Any], Path]: + config_path = Path(path).resolve() + raw = json.loads(config_path.read_text(encoding="utf-8")) + if raw.get("schema") != SCHEMA: + raise ValueError(f"Rainbond pool config schema must be {SCHEMA}") + source_services = raw.get("source_services") + if not isinstance(source_services, dict) or set(source_services) != set(STATEFUL_SITES): + raise ValueError(f"source_services must contain exactly {list(STATEFUL_SITES)}") + normalized_services = { + str(site): str(service_id) for site, service_id in source_services.items() + } + if any(not value for value in normalized_services.values()): + raise ValueError("source_services values must be non-empty service IDs") + prefix = str(raw.get("app_prefix") or "webarena-slice") + if not _SAFE_ID.fullmatch(prefix): + raise ValueError("app_prefix must be a lowercase DNS-safe identifier") + suffix = str(raw.get("domain_suffix") or "").strip().strip(".") + if not suffix or "/" in suffix or ":" in suffix: + raise ValueError("domain_suffix must be a DNS suffix without scheme or port") + lock_value = Path(str(raw.get("image_lock") or "")) + lock_path = lock_value if lock_value.is_absolute() else (config_path.parent / lock_value) + if not lock_path.is_file(): + raise ValueError("Rainbond pool image_lock is missing") + initializer_service = str(raw.get("initializer_service") or "") + initializer_image = str(raw.get("initializer_image") or "") + if not initializer_service or not initializer_image: + raise ValueError("initializer_service and initializer_image are required") + _image_digest(initializer_image) + value = cls( + api_base_url=str(raw.get("api_base_url") or "").rstrip("/"), + team_id=str(raw.get("team_id") or ""), + team_name=str(raw.get("team_name") or ""), + region_name=str(raw.get("region_name") or ""), + source_app_id=int(raw.get("source_app_id") or 0), + source_services=normalized_services, + initializer_service=initializer_service, + initializer_image=initializer_image, + app_prefix=prefix, + domain_suffix=suffix, + token_env=str(raw.get("token_env") or "RAINBOND_API_TOKEN"), + image_lock=lock_path.resolve(), + shared_environments=dict(raw.get("shared_environments") or {}), + readiness_timeout_seconds=int(raw.get("readiness_timeout_seconds") or 1800), + max_parallel_provisions=int(raw.get("max_parallel_provisions") or 2), + reset_strategy=str(raw.get("reset_strategy") or "replace_before_destroy"), + ) + if ( + not all((value.api_base_url, value.team_id, value.team_name, value.region_name)) + or value.source_app_id < 1 + or value.max_parallel_provisions < 1 + ): + raise ValueError("Rainbond pool API, team, region, and source app are required") + if value.reset_strategy not in {"replace_before_destroy", "stop_before_replace"}: + raise ValueError("reset_strategy must be replace_before_destroy or stop_before_replace") + return value, raw, config_path + + +class RainbondSlicePool: + """Create and replace isolated four-site applications from one immutable baseline.""" + + def __init__( + self, + config_path: str | Path, + *, + api: RainbondAPI | None = None, + sleep: Any = time.sleep, + ) -> None: + self.config, self.raw_config, self.config_path = PoolConfig.load(config_path) + self.config_sha256 = _canonical_sha256(self.raw_config) + token = os.environ.get(self.config.token_env, "") + if api is None and not token: + raise ValueError( + f"Rainbond API token environment variable is missing: {self.config.token_env}" + ) + self.api = api or RainbondAPI(self.config.api_base_url, token) + self.sleep = sleep + self.image_lock = json.loads(self.config.image_lock.read_text(encoding="utf-8")) + + def _apps_path(self) -> str: + config = self.config + return f"/openapi/v1/teams/{quote(config.team_id)}/regions/{quote(config.region_name)}/apps" + + def _app_path(self, app_id: int) -> str: + return f"{self._apps_path()}/{app_id}" + + @staticmethod + def _response_rows(response: Any, *keys: str) -> list[dict[str, Any]]: + if isinstance(response, list): + return [row for row in response if isinstance(row, dict)] + if not isinstance(response, dict): + return [] + for key in (*keys, "items"): + value = response.get(key) + if isinstance(value, list): + return [row for row in value if isinstance(row, dict)] + if isinstance(value, dict) and isinstance(value.get("items"), list): + return [row for row in value["items"] if isinstance(row, dict)] + return [] + + def _domain(self, pool_id: str, slot: int, site: str, app_id: int) -> str: + label = f"{self.config.app_prefix}-{pool_id}-{slot}-{app_id}-{site.replace('_', '-')}" + if len(label) > 63: + label = f"{label[:50].rstrip('-')}-{hashlib.sha256(label.encode()).hexdigest()[:10]}" + return f"{label}.{self.config.domain_suffix}" + + @staticmethod + def _validate_pool_id(pool_id: str) -> str: + return _validate_pool_id(pool_id) + + def _create_app(self, pool_id: str, slot: int) -> int: + name = f"{self.config.app_prefix}-{pool_id}-{slot}" + note = f"managed-by=browser-eval;pool={pool_id};slot={slot}" + result = self.api.request("POST", self._apps_path(), {"app_name": name, "app_note": note}) + app_id = int((result or {}).get("ID") or 0) + if app_id < 1: + raise RuntimeError("Rainbond create app response has no ID") + return app_id + + def _copy_components(self, app_id: int) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]: + config = self.config + result = self.api.request( + "POST", + f"{self._app_path(config.source_app_id)}/copy", + { + "services": [ + {"service_id": config.source_services[site]} for site in STATEFUL_SITES + ] + + [{"service_id": config.initializer_service}], + "target_team_name": config.team_name, + "target_region_name": config.region_name, + "target_app_id": app_id, + }, + ) + rows = self._response_rows(result, "services") + by_source_name = { + site: str(self.raw_config.get("source_component_names", {}).get(site) or site) + for site in STATEFUL_SITES + } + mapped: dict[str, dict[str, Any]] = {} + for site, component_name in by_source_name.items(): + copied_name = re.compile(rf"^{re.escape(component_name)}-copy(?:-\d+)?$") + matches = [ + row + for row in rows + if str(row.get("service_cname")) == component_name + or copied_name.fullmatch(str(row.get("service_cname"))) + ] + if len(matches) != 1: + raise RuntimeError( + f"Rainbond copy returned {len(matches)} components named {component_name!r}" + ) + mapped[site] = matches[0] + initializer_name = str( + self.raw_config.get("initializer_component_name") or "slice-initializer" + ) + copied_initializer_name = re.compile(rf"^{re.escape(initializer_name)}-copy(?:-\d+)?$") + initializer_matches = [ + row + for row in rows + if str(row.get("service_cname")) == initializer_name + or copied_initializer_name.fullmatch(str(row.get("service_cname"))) + ] + if len(initializer_matches) != 1: + raise RuntimeError( + f"Rainbond copy returned {len(initializer_matches)} components named " + f"{initializer_name!r}" + ) + return mapped, initializer_matches[0] + + def _configure_component( + self, + app_id: int, + *, + pool_id: str, + slot: int, + site: str, + component: dict[str, Any], + ) -> dict[str, Any]: + service_id = str(component.get("service_id") or "") + if not service_id: + raise RuntimeError(f"Rainbond copied component has no service_id: {site}") + self._coalesce_internal_ports( + app_id, + service_id, + site_port=SITE_PORTS[site], + control_port=8877, + control_alias=f"{site.upper()}_CTRL", + ) + domain = self._domain(pool_id, slot, site, app_id) + site_url = f"http://{domain}" + magento = site in {"shopping_admin", "shopping"} + self.api.request( + "POST", + f"{self._app_path(app_id)}/domains", + { + "protocol": "http", + "http": { + "service_id": service_id, + "container_port": SITE_PORTS[site], + "certificate_id": 0, + "domain_name": domain, + "domain_cookie": "", + "domain_header": "", + "domain_path": "/", + "the_weight": 100, + "whether_open": True, + "auto_ssl": False, + "auto_ssl_config": "", + "rule_extensions": [], + "configuration": { + "proxy_body_size": 100 if magento else 0, + "proxy_buffer_numbers": 8 if magento else 4, + "proxy_buffer_size": 64 if magento else 4, + "proxy_buffering": "on" if magento else "off", + "proxy_connect_timeout": 75, + "proxy_read_timeout": 600, + "proxy_send_timeout": 600, + "set_headers": [], + }, + }, + }, + ) + envs = [ + { + "note": "public origin for WebArena initialization", + "name": "WA_ENV_CTRL_EXTERNAL_SITE_URL", + "value": site_url, + "is_change": True, + "scope": "inner", + } + ] + if magento: + envs.append( + { + "note": "allow Magento initialization on loaded nodes", + "name": "WA_ENV_CTRL_CMD_TIMEOUT_SEC", + "value": "600", + "is_change": True, + "scope": "inner", + } + ) + if site == "gitlab": + envs.append( + { + "note": "preserve the image's internal 8023 listener", + "name": "WA_ENV_CTRL_SKIP_RECONFIGURE", + "value": "true", + "is_change": True, + "scope": "inner", + } + ) + self.api.request( + "PUT", + f"{self._app_path(app_id)}/services/{quote(service_id)}/envs", + {"envs": envs}, + ) + expected_image = str(self.image_lock["images"][site]["pinned"]) + actual_image = str(component.get("image") or "") + if not actual_image: + raise RuntimeError(f"Rainbond copied {site} component has no image in response") + if _image_digest(actual_image) != _image_digest(expected_image): + raise RuntimeError( + f"Rainbond copied {site} image mismatch: {actual_image!r} != {expected_image!r}" + ) + return { + "service_id": service_id, + "service_cname": component.get("service_cname"), + "image": expected_image, + "runtime_image": actual_image, + "site_url": site_url, + } + + def _coalesce_internal_ports( + self, + app_id: int, + service_id: str, + *, + site_port: int, + control_port: int, + control_alias: str, + ) -> None: + path = f"{self._app_path(app_id)}/services/{quote(service_id)}/ports" + + def inner_ports() -> dict[int, dict[str, Any]]: + response = self.api.request("GET", path) + return { + int(row.get("container_port") or 0): row + for row in self._response_rows(response, "ports") + if bool(row.get("is_inner_service")) + } + + ports = inner_ports() + site_host = str(ports.get(site_port, {}).get("k8s_service_name") or "") + control_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") + if not site_host or not control_host: + raise RuntimeError( + f"Rainbond copied component {service_id} must expose internal ports " + f"{site_port} and {control_port}" + ) + if site_host == control_host: + return + control_path = f"{path}/{control_port}" + self.api.request("PUT", control_path, {"action": "close_inner"}) + self.api.request("DELETE", control_path) + self.api.request( + "POST", + path, + { + "port": control_port, + "protocol": str(ports[control_port].get("protocol") or "http"), + "port_alias": control_alias, + "is_inner_service": True, + }, + ) + # The write may finish before the read model reflects the recreated port. + # Retry only observation: repeating DELETE/POST could disrupt the service. + for attempt in range(3): + ports = inner_ports() + updated_host = str(ports.get(control_port, {}).get("k8s_service_name") or "") + if updated_host == site_host: + return + if attempt < 2: + self.sleep(attempt + 1) + raise RuntimeError( + f"Rainbond did not attach control port {control_port} to internal " + f"service {site_host!r}: got {updated_host!r}" + ) + + def _configure_initializer( + self, + app_id: int, + *, + pool_id: str, + slot: int, + initializer: dict[str, Any], + copied_sites: dict[str, dict[str, Any]], + ) -> dict[str, Any]: + service_id = str(initializer.get("service_id") or "") + actual_image = str(initializer.get("image") or "") + if not service_id: + raise RuntimeError("Rainbond copied initializer has no service_id") + if not actual_image: + raise RuntimeError("Rainbond copied initializer component has no image in response") + if _image_digest(actual_image) != _image_digest(self.config.initializer_image): + raise RuntimeError("Rainbond copied initializer image mismatch") + aliases = { + site: self._internal_service_host(app_id, copied_sites[site], 8877) + for site in ("reddit", "gitlab") + } + self.api.request( + "PUT", + f"{self._app_path(app_id)}/services/{quote(service_id)}/envs", + { + "envs": [ + { + "name": f"{site.upper()}_CTRL_URL", + "value": f"http://{alias}:8877", + "is_change": True, + "scope": "inner", + } + for site, alias in aliases.items() + ] + + [ + { + "name": "ENV_CTRL_MAX_ATTEMPTS", + "value": "360", + "is_change": True, + "scope": "inner", + }, + { + "name": "ENV_CTRL_RETRY_SECONDS", + "value": "5", + "is_change": True, + "scope": "inner", + }, + ] + }, + ) + readiness_url = f"http://{self._domain(pool_id, slot, 'initializer', app_id)}" + self.api.request( + "POST", + f"{self._app_path(app_id)}/domains", + { + "protocol": "http", + "http": { + "service_id": service_id, + "container_port": 8080, + "certificate_id": 0, + "domain_name": readiness_url.removeprefix("http://"), + "domain_cookie": "", + "domain_header": "", + "domain_path": "/", + "the_weight": 100, + "whether_open": True, + "auto_ssl": False, + "auto_ssl_config": "", + "rule_extensions": [], + "configuration": { + "proxy_body_size": 0, + "proxy_buffer_numbers": 4, + "proxy_buffer_size": 4, + "proxy_buffering": "off", + "proxy_connect_timeout": 75, + "proxy_read_timeout": 600, + "proxy_send_timeout": 600, + "set_headers": [], + }, + }, + }, + ) + return {"service_id": service_id, "readiness_url": readiness_url} + + def _internal_service_host(self, app_id: int, component: dict[str, Any], port: int) -> str: + service_id = str(component.get("service_id") or "") + response = self.api.request( + "GET", + f"{self._app_path(app_id)}/services/{quote(service_id)}/ports", + ) + rows = self._response_rows(response, "ports") + match = next( + ( + row + for row in rows + if int(row.get("container_port") or 0) == port and bool(row.get("is_inner_service")) + ), + None, + ) + host = str((match or {}).get("k8s_service_name") or "") + if not host: + raise RuntimeError( + f"Rainbond copied component {service_id} has no internal service on port {port}" + ) + return host + + def _wait_component_running(self, app_id: int, service_id: str) -> None: + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_status = "not found" + while time.monotonic() < deadline: + response = self.api.request("GET", f"{self._app_path(app_id)}/services") + rows = self._response_rows(response, "services") + match = next((row for row in rows if str(row.get("service_id")) == service_id), None) + if match is not None: + last_status = str(match.get("status") or "unknown") + if last_status == "running": + return + self.sleep(5) + raise TimeoutError( + f"Rainbond initializer did not become ready: {service_id}: {last_status}" + ) + + def _wait_site(self, url: str) -> None: + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_error = "not checked" + while time.monotonic() < deadline: + try: + request = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(request, timeout=20) as response: + if 200 <= response.status < 500: + return + last_error = f"HTTP {response.status}" + except (OSError, urllib.error.URLError) as exc: + last_error = str(exc) + self.sleep(5) + raise TimeoutError(f"Rainbond WebArena site did not become ready: {url}: {last_error}") + + def _operate_existing_app(self, receipt: dict[str, Any], action: str) -> None: + service_ids = [ + str(component["service_id"]) for component in (receipt.get("components") or {}).values() + ] + service_ids.append(str(receipt["initializer"]["service_id"])) + self.api.request( + "POST", + f"{self._app_path(int(receipt['app_id']))}/operations", + {"action": action, "service_ids": service_ids}, + ) + + def _wait_app_stopped(self, receipt: dict[str, Any]) -> None: + app_id = int(receipt["app_id"]) + expected = { + str(component["service_id"]) for component in (receipt.get("components") or {}).values() + } + expected.add(str(receipt["initializer"]["service_id"])) + deadline = time.monotonic() + self.config.readiness_timeout_seconds + last_statuses: dict[str, str] = {} + while time.monotonic() < deadline: + response = self.api.request("GET", f"{self._app_path(app_id)}/services") + rows = self._response_rows(response, "services") + last_statuses = { + str(row.get("service_id")): str(row.get("status") or "unknown") + for row in rows + if str(row.get("service_id")) in expected + } + if set(last_statuses) == expected and all( + status in {"closed", "undeploy"} for status in last_statuses.values() + ): + return + self.sleep(5) + raise TimeoutError( + f"Rainbond app did not stop before replacement: {app_id}: {last_statuses}" + ) + + def _stop_existing_app(self, receipt: dict[str, Any]) -> None: + self._operate_existing_app(receipt, "stop") + self._wait_app_stopped(receipt) + + def _delete_app(self, app_id: int) -> None: + """Stop runtime services before removing their control-plane app record.""" + path = self._app_path(app_id) + try: + rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + except RuntimeError as exc: + if isinstance(exc.__cause__, urllib.error.HTTPError) and exc.__cause__.code == 404: + return + raise + service_ids = {str(row["service_id"]) for row in rows if row.get("service_id")} + if service_ids: + self.api.request( + "POST", + f"{path}/operations", + {"action": "stop", "service_ids": sorted(service_ids)}, + ) + deadline = time.monotonic() + self.config.readiness_timeout_seconds + while True: + rows = self._response_rows(self.api.request("GET", f"{path}/services"), "services") + statuses = {str(row.get("service_id")): row.get("status") for row in rows} + if all(statuses.get(sid) in {"closed", "undeploy"} for sid in service_ids): + break + if time.monotonic() >= deadline: + raise TimeoutError(f"Rainbond app did not stop before deletion: {app_id}") + self.sleep(5) + self.api.request("DELETE", f"{path}?force=1") + + def provision(self, pool_id: str, slot: int, receipt_path: Path) -> dict[str, Any]: + started = time.monotonic() + phase_started = started + timings: dict[str, float] = {} + pool_id = self._validate_pool_id(pool_id) + if slot < 1: + raise ValueError("slot must be positive") + app_id = self._create_app(pool_id, slot) + timings["create_app_seconds"] = round(time.monotonic() - phase_started, 3) + try: + phase_started = time.monotonic() + copied, copied_initializer = self._copy_components(app_id) + timings["copy_components_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() + components = { + site: self._configure_component( + app_id, + pool_id=pool_id, + slot=slot, + site=site, + component=copied[site], + ) + for site in STATEFUL_SITES + } + initializer = self._configure_initializer( + app_id, + pool_id=pool_id, + slot=slot, + initializer=copied_initializer, + copied_sites=copied, + ) + timings["configure_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() + self.api.request( + "POST", + f"{self._app_path(app_id)}/operations", + { + "action": "deploy", + "service_ids": [row["service_id"] for row in components.values()] + + [initializer["service_id"]], + }, + ) + timings["deploy_request_seconds"] = round(time.monotonic() - phase_started, 3) + phase_started = time.monotonic() + self._wait_component_running(app_id, str(initializer["service_id"])) + self._wait_site(str(initializer["readiness_url"])) + for component in components.values(): + self._wait_site(str(component["site_url"])) + timings["readiness_seconds"] = round(time.monotonic() - phase_started, 3) + except BaseException as exc: + _attempt_cleanup( + exc, + f"delete incomplete Rainbond app {app_id}", + lambda: self._delete_app(app_id), + ) + raise + receipt = { + "schema": RECEIPT_SCHEMA, + "pool_config_sha256": self.config_sha256, + "pool_id": pool_id, + "slot": slot, + "generation": uuid.uuid4().hex, + "app_id": app_id, + "components": components, + "initializer": initializer, + "timings": {**timings, "total_seconds": round(time.monotonic() - started, 3)}, + "created_at_unix": int(time.time()), + } + _atomic_json(receipt_path, receipt) + return receipt + + def destroy(self, receipt_path: Path, *, missing_ok: bool = True) -> None: + if not receipt_path.exists(): + if missing_ok: + return + raise FileNotFoundError(receipt_path) + receipt = self.validate_receipt(receipt_path) + self._delete_app(int(receipt["app_id"])) + receipt_path.unlink() + + def reset(self, receipt_path: Path) -> dict[str, Any]: + previous = self.validate_receipt(receipt_path) + pool_id = str(previous["pool_id"]) + slot = int(previous["slot"]) + replacement_path = receipt_path.with_name("slice-receipt.replacement.json") + if replacement_path.exists(): + raise RuntimeError(f"unfinished Rainbond slice replacement: {replacement_path}") + stopped_previous = self.config.reset_strategy == "stop_before_replace" + if stopped_previous: + self._stop_existing_app(previous) + try: + replacement = self.provision(pool_id, slot, replacement_path) + except BaseException as exc: + if not stopped_previous: + _attempt_cleanup( + exc, + f"stop failed Rainbond slice {previous['app_id']}", + lambda: self._stop_existing_app(previous), + ) + raise + try: + self._delete_app(int(previous["app_id"])) + except BaseException as exc: + replacement_deleted = _attempt_cleanup( + exc, + f"delete replacement Rainbond app {replacement['app_id']}", + lambda: self._delete_app(int(replacement["app_id"])), + ) + if replacement_deleted: + _attempt_cleanup( + exc, + f"remove replacement receipt {replacement_path}", + lambda: replacement_path.unlink(missing_ok=True), + ) + if not stopped_previous: + _attempt_cleanup( + exc, + f"stop failed Rainbond slice {previous['app_id']}", + lambda: self._stop_existing_app(previous), + ) + raise + replacement_path.replace(receipt_path) + return replacement + + def validate_receipt(self, receipt_path: Path) -> dict[str, Any]: + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if ( + receipt.get("schema") != RECEIPT_SCHEMA + or receipt.get("pool_config_sha256") != self.config_sha256 + or set(receipt.get("components") or {}) != set(STATEFUL_SITES) + or int(receipt.get("app_id") or 0) < 1 + ): + raise ValueError("Rainbond slice receipt is invalid or belongs to another pool") + return receipt + + def runtime_config(self, receipt: dict[str, Any]) -> dict[str, Any]: + environments = dict(self.config.shared_environments) + for site, component in receipt["components"].items(): + environments[f"__{site.upper()}__"] = {"urls": [component["site_url"]]} + return { + "image_lock": str(self.config.image_lock), + "environments": environments, + "rainbond_slice": { + "pool_config": str(self.config_path), + "receipt": "slice-receipt.json", + }, + } + + def persist_runtime_config( + self, + receipt: dict[str, Any], + path: Path, + *, + base_config: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Atomically persist receipt-authoritative URLs for crash recovery.""" + value = {**(base_config or {}), **self.runtime_config(receipt)} + _atomic_json(path, value) + return value + + +def provision_pool(config_path: Path, output_dir: Path, pool_id: str, count: int) -> list[Path]: + if count < 1: + raise ValueError("count must be positive") + pool = RainbondSlicePool(config_path) + created: list[Path] = [] + + def create_slot(slot: int) -> Path: + slot_dir = output_dir / f"slice-{slot}" + receipt_path = slot_dir / "slice-receipt.json" + if receipt_path.exists(): + raise FileExistsError(f"Rainbond slice already exists: {receipt_path}") + try: + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + except BaseException as exc: + if receipt_path.exists(): + _attempt_cleanup( + exc, + f"destroy incomplete slot {slot}", + lambda: pool.destroy(receipt_path), + ) + raise + return slot_dir + + try: + workers = min(count, getattr(getattr(pool, "config", None), "max_parallel_provisions", 2)) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = {executor.submit(create_slot, slot): slot for slot in range(1, count + 1)} + first_error: BaseException | None = None + for future in as_completed(futures): + try: + created.append(future.result()) + except BaseException as exc: + first_error = first_error or exc + if first_error is not None: + raise first_error + except BaseException as exc: + for slot_dir in reversed(created): + _attempt_cleanup( + exc, + f"destroy rolled-back slot {slot_dir.name}", + lambda slot_dir=slot_dir: pool.destroy(slot_dir / "slice-receipt.json"), + ) + raise + return sorted(created, key=_slot_sort_key) + + +@contextmanager +def _pool_lock(output_dir: Path) -> Any: + output_dir = output_dir.resolve() + with _LOCAL_POOL_LOCKS_GUARD: + local_lock = _LOCAL_POOL_LOCKS.setdefault(output_dir, threading.Lock()) + with local_lock: + output_dir.mkdir(parents=True, exist_ok=True) + lock_path = output_dir / ".pool.lock" + with lock_path.open("a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def scale_pool( + config_path: Path, + output_dir: Path, + pool_id: str, + count: int, + *, + minimum_ready: int = 1, +) -> list[Path]: + """Reconcile a persistent warm pool, never scaling below minimum_ready.""" + if minimum_ready < 0 or count < minimum_ready: + raise ValueError( + "count must be at least minimum_ready, and minimum_ready must not be negative" + ) + with _pool_lock(output_dir): + pool = RainbondSlicePool(config_path) + pool_id = _validate_pool_id(pool_id) + existing: dict[int, Path] = {} + for receipt_path in output_dir.glob("slice-*/slice-receipt.json"): + receipt = pool.validate_receipt(receipt_path) + if str(receipt.get("pool_id")) != pool_id: + raise ValueError(f"slice belongs to another pool: {receipt_path}") + existing[int(receipt["slot"])] = receipt_path.parent + missing_slots = [slot for slot in range(1, count + 1) if slot not in existing] + if missing_slots: + workers = min(len(missing_slots), pool.config.max_parallel_provisions) + + def create_slot(slot: int) -> tuple[int, Path]: + slot_dir = output_dir / f"slice-{slot}" + receipt_path = slot_dir / "slice-receipt.json" + try: + receipt = pool.provision(pool_id, slot, receipt_path) + _atomic_json(slot_dir / "webarena.config.json", pool.runtime_config(receipt)) + except BaseException as exc: + if receipt_path.exists(): + _attempt_cleanup( + exc, + f"destroy incomplete slot {slot}", + lambda: pool.destroy(receipt_path), + ) + raise + return slot, slot_dir + + created: dict[int, Path] = {} + first_error: BaseException | None = None + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(create_slot, slot) for slot in missing_slots] + for future in as_completed(futures): + try: + slot, slot_dir = future.result() + created[slot] = slot_dir + except BaseException as exc: + first_error = first_error or exc + if first_error is not None: + for slot, slot_dir in sorted(created.items(), reverse=True): + _attempt_cleanup( + first_error, + f"destroy rolled-back slot {slot}", + lambda slot_dir=slot_dir: pool.destroy(slot_dir / "slice-receipt.json"), + ) + raise first_error + existing.update(created) + excess_slots = sorted((value for value in existing if value > count), reverse=True) + leased_excess = [ + existing[slot] + for slot in excess_slots + if (existing[slot] / "slice-lease.json").exists() + ] + if leased_excess: + raise RuntimeError( + "cannot remove leased Rainbond slices: " + + ", ".join(str(path) for path in leased_excess) + ) + first_destroy_error: BaseException | None = None + for slot in excess_slots: + slot_dir = existing[slot] + try: + pool.destroy(slot_dir / "slice-receipt.json") + except BaseException as exc: + if first_destroy_error is None: + first_destroy_error = exc + else: + first_destroy_error.add_note( + f"additional scale-down failure for slot {slot}: {exc}" + ) + continue + existing.pop(slot) + config_file = slot_dir / "webarena.config.json" + if config_file.exists(): + config_file.unlink() + try: + slot_dir.rmdir() + except OSError: + pass + if first_destroy_error is not None: + raise first_destroy_error + return [existing[slot] for slot in sorted(existing)] + + +def reconcile_pool_capacity( + config_path: Path, + output_dir: Path, + pool_id: str, + active_count: int, + *, + minimum_ready: int = 1, +) -> list[Path]: + """Keep one or more idle slices in addition to current task demand.""" + if active_count < 0: + raise ValueError("active_count must not be negative") + return scale_pool( + config_path, + output_dir, + pool_id, + active_count + minimum_ready, + minimum_ready=minimum_ready, + ) + + +def acquire_slice(output_dir: Path, owner: str) -> dict[str, Any]: + """Atomically grant one ready slice to one task owner.""" + owner = owner.strip() + if not owner: + raise ValueError("lease owner is required") + with _pool_lock(output_dir): + for slot_dir in sorted(output_dir.glob("slice-*"), key=_slot_sort_key): + receipt_path = slot_dir / "slice-receipt.json" + config_path = slot_dir / "webarena.config.json" + lease_path = slot_dir / "slice-lease.json" + if not receipt_path.is_file() or not config_path.is_file() or lease_path.exists(): + continue + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if receipt.get("schema") != RECEIPT_SCHEMA: + raise ValueError(f"invalid Rainbond slice receipt: {receipt_path}") + lease = { + "schema": LEASE_SCHEMA, + "lease_id": uuid.uuid4().hex, + "owner": owner, + "slot": int(receipt["slot"]), + "app_id": int(receipt["app_id"]), + "generation": str(receipt["generation"]), + "config": str(config_path.resolve()), + "acquired_at_unix": int(time.time()), + } + _atomic_json(lease_path, lease) + return lease + raise RuntimeError("no unleased Rainbond WebArena slice is ready") + + +def release_slice(config_path: Path, output_dir: Path, lease_id: str) -> dict[str, Any]: + """Replace a dirty slice before returning it to the ready pool.""" + with _pool_lock(output_dir): + matches: list[tuple[Path, dict[str, Any]]] = [] + for lease_path in output_dir.glob("slice-*/slice-lease.json"): + lease = json.loads(lease_path.read_text(encoding="utf-8")) + if lease.get("schema") == LEASE_SCHEMA and lease.get("lease_id") == lease_id: + matches.append((lease_path, lease)) + if len(matches) != 1: + raise ValueError("lease_id does not identify exactly one active slice") + lease_path, lease = matches[0] + slot_dir = lease_path.parent + # Keep the lease while resetting so acquire/scale skip this slot. The slot + # lock serializes competing releases without blocking unrelated ready slots. + with _pool_lock(slot_dir): + if not lease_path.is_file() or json.loads(lease_path.read_text(encoding="utf-8")) != lease: + raise ValueError("lease changed before release") + pool = RainbondSlicePool(config_path) + receipt_path = slot_dir / "slice-receipt.json" + runtime_config_path = slot_dir / "webarena.config.json" + current_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + release_from_app_id = lease.get("release_from_app_id") + if release_from_app_id is not None and int(current_receipt["app_id"]) != int( + release_from_app_id + ): + current_receipt = pool.validate_receipt(receipt_path) + _atomic_json(runtime_config_path, pool.runtime_config(current_receipt)) + lease_path.unlink() + return { + "lease_id": lease_id, + "released_app_id": int(lease["app_id"]), + "ready_app_id": int(current_receipt["app_id"]), + "config": str(runtime_config_path.resolve()), + } + if release_from_app_id is None: + # Task preparation also replaces instances. Only this release's + # replacement proves the task's mutations have been removed. + lease = {**lease, "release_from_app_id": int(current_receipt["app_id"])} + _atomic_json(lease_path, lease) + receipt = pool.reset(receipt_path) + _atomic_json(runtime_config_path, pool.runtime_config(receipt)) + lease_path.unlink() + return { + "lease_id": lease_id, + "released_app_id": int(lease["app_id"]), + "ready_app_id": int(receipt["app_id"]), + "config": str((slot_dir / "webarena.config.json").resolve()), + } diff --git a/harness/trajectory_replay/public_webarena/runner.py b/harness/trajectory_replay/public_webarena/runner.py index b9dc2357e..3e34cb03e 100644 --- a/harness/trajectory_replay/public_webarena/runner.py +++ b/harness/trajectory_replay/public_webarena/runner.py @@ -946,17 +946,16 @@ def run_evaluation( for task_index, declared_task in enumerate(runnable_tasks): task = _materialize_task(declared_task, fixture_origin) - if (task.environment or {}).get("kind") == "webarena_verified": - assert webarena_environment is not None - task = webarena_environment.materialize_task_actions(task) if trajectory_authoring_run: task = _bounded_authoring_task(task) + base_task = task engine_order = ( engines if len(engines) == 1 or task_index % 2 == 0 else tuple(reversed(engines)) ) for engine in engine_order: initialized_sites = initialized_sites_by_engine[engine] for attempt in range(1, max_retries + 2): + task = base_task artifact_dir = ( output / "tasks" / task.task_id / engine / f"attempt-{attempt}" ) @@ -1003,6 +1002,7 @@ def run_evaluation( authoring_initialized_sites.update(task_sites) if "mutate" not in task.tags: initialized_sites.update(task_sites) + task = webarena_environment.materialize_task_actions(base_task) prepare_budget_ms = _remaining_task_budget_ms( attempt_deadline, phase="environment prepare" ) diff --git a/harness/trajectory_replay/public_webarena/webarena_environment.py b/harness/trajectory_replay/public_webarena/webarena_environment.py index 9701ac20b..4cdb0aff5 100644 --- a/harness/trajectory_replay/public_webarena/webarena_environment.py +++ b/harness/trajectory_replay/public_webarena/webarena_environment.py @@ -7,7 +7,6 @@ import os import re import signal -import socket import threading import time import urllib.error @@ -22,10 +21,13 @@ from webarena_verified.environments.container.manager import ContainerManager from webarena_verified.types.task import WebArenaSite +from harness.environments.webarena.backend import MAGENTO_INIT_TIMEOUT_SECONDS, DockerBackend +from harness.environments.webarena.manager import EnvironmentManager +from harness.environments.webarena.rainbond import RainbondSlicePool + from .suite import BrowserTask STATEFUL_SITES = frozenset({"shopping_admin", "shopping", "reddit", "gitlab"}) -MAGENTO_INIT_TIMEOUT_SECONDS = 600 LEGACY_PORT_SITES = { 3000: "map", 4399: "reddit", @@ -46,9 +48,8 @@ def wall_clock_timeout(seconds: float, *, operation: str = "environment reset"): """Bound one blocking environment phase, including Docker SDK calls, on POSIX.""" if seconds <= 0: raise ValueError("wall-clock timeout must be positive") - if ( - threading.current_thread() is not threading.main_thread() - or not hasattr(signal, "setitimer") + if threading.current_thread() is not threading.main_thread() or not hasattr( + signal, "setitimer" ): raise RuntimeError("WebArena reset wall-clock timeout requires the POSIX main thread") previous_timer = signal.getitimer(signal.ITIMER_REAL) @@ -68,114 +69,6 @@ def expire(_signum: int, _frame: Any) -> None: signal.signal(signal.SIGALRM, previous_handler) -class _DockerClientBackend: - """Bind WebArena's container manager to an explicit Docker SDK client.""" - - def __init__(self, client: Any, *, defer_removal: bool = False) -> None: - self.client = client - self.defer_removal = defer_removal - self.retired_containers: list[str] = [] - - def _container(self, name: str) -> Any | None: - try: - return self.client.containers.get(name) - except docker.errors.NotFound: - return None - - def container_exists(self, *, name: str) -> bool: - return self._container(name) is not None - - def container_running(self, *, name: str) -> bool: - container = self._container(name) - if container is None: - return False - container.reload() - return container.status == "running" - - def container_remove(self, *, name: str) -> None: - container = self._container(name) - if container is not None: - if not self.defer_removal: - container.remove(force=True) - return - # Removing WebArena's tens-of-gigabytes writable layer can block a - # rootless daemon for hours. Stopping releases the fixed ports; - # renaming preserves the old layer for deferred garbage collection. - # The next container still starts from the immutable pinned image, - # and these official images have no shared writable mounts. - container.reload() - if container.status == "running": - container.stop(timeout=10) - retired_name = f"{name}__retired__{container.id[:12]}" - container.rename(retired_name) - self.retired_containers.append(retired_name) - - def container_run( - self, - *, - name: str, - image: str, - port_mappings: dict[int, int], - volume_mappings: dict[str, str], - env_vars: dict[str, str] | None = None, - ) -> None: - environment = dict(env_vars or {}) - if self.defer_removal and volume_mappings: - raise RuntimeError( - "deferred WebArena container removal requires no shared writable mounts" - ) - if name in { - "webarena_verified_shopping", - "webarena_verified_shopping_admin", - }: - # Magento's official init performs cache/database work after the - # services become healthy. The image default of 180 seconds is - # too short on a busy qualification host and can leave the baked - # CMU origin active even though env-ctrl /status is healthy. - environment["WA_ENV_CTRL_CMD_TIMEOUT_SEC"] = str( - MAGENTO_INIT_TIMEOUT_SECONDS - ) - # GitLab's image entrypoint does not run env-ctrl init. Its fallback - # reconfiguration also rewrites the internal nginx listener to the - # public host port, breaking Docker's host-port -> 8023 mapping. Keep - # the baked internal listener here; WebArenaEnvironment performs one - # explicit official init after the service is healthy. - if name == "webarena_verified_gitlab": - environment["WA_ENV_CTRL_SKIP_RECONFIGURE"] = "true" - self.client.containers.run( - image, - detach=True, - name=name, - ports={ - f"{container_port}/tcp": host_port - for host_port, container_port in port_mappings.items() - }, - volumes={ - source: {"bind": target, "mode": "rw"} - for source, target in volume_mappings.items() - }, - environment=environment, - ) - - def get_container_ports(self, *, name: str) -> dict[int, int]: - container = self._container(name) - if container is None: - return {} - container.reload() - result: dict[int, int] = {} - ports = container.attrs.get("NetworkSettings", {}).get("Ports") or {} - for key, bindings in ports.items(): - if bindings: - result[int(str(key).split("/", 1)[0])] = int(bindings[0]["HostPort"]) - return result - - def find_free_port(self) -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as value: - value.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - value.bind(("", 0)) - return int(value.getsockname()[1]) - - class WebArenaEnvironment: def __init__(self, config_path: str | Path) -> None: self.config_path = Path(config_path) @@ -195,6 +88,33 @@ def __init__(self, config_path: str | Path) -> None: if self.image_lock.get("schema") != "lex_webarena_environment_images/1": raise ValueError("WebArena image lock schema is invalid") self.remote_receipts = self.config.get("remote_runtime_receipts") or {} + slice_value = self.config.get("rainbond_slice") + self.rainbond_pool: RainbondSlicePool | None = None + self.rainbond_receipt_path: Path | None = None + if slice_value is not None: + if not isinstance(slice_value, dict): + raise ValueError("rainbond_slice must be an object") + pool_raw = str(slice_value.get("pool_config") or "").strip() + receipt_raw = str(slice_value.get("receipt") or "").strip() + if not pool_raw or not receipt_raw: + raise ValueError("rainbond_slice requires pool_config and receipt") + pool_value = Path(pool_raw) + receipt_value = Path(receipt_raw) + pool_path = ( + pool_value + if pool_value.is_absolute() + else (self.config_path.parent / pool_value).resolve() + ) + self.rainbond_receipt_path = ( + receipt_value + if receipt_value.is_absolute() + else (self.config_path.parent / receipt_value).resolve() + ) + self.rainbond_pool = RainbondSlicePool(pool_path) + current_receipt = self.rainbond_pool.validate_receipt(self.rainbond_receipt_path) + current_runtime = self.rainbond_pool.runtime_config(current_receipt) + self.config = {**self.config, **current_runtime} + self.environments = self.config["environments"] def site_docker_client(self, default_client: Any) -> tuple[Any, bool]: host = self.config.get("site_docker_host") @@ -216,9 +136,34 @@ def verify_runtime(self, docker_client: Any, tasks: tuple[BrowserTask, ...]) -> {str(site) for task in tasks for site in (task.environment or {}).get("sites") or []} ) rows: dict[str, Any] = {} + slice_receipt = ( + self.rainbond_pool.validate_receipt(self.rainbond_receipt_path) + if self.rainbond_pool is not None and self.rainbond_receipt_path is not None + else None + ) for site in sites: locked = self.image_lock["images"][site] expected = str(locked["pinned"]) + if slice_receipt is not None and site in STATEFUL_SITES: + component = (slice_receipt.get("components") or {}).get(site) or {} + if component.get("image") != expected: + raise RuntimeError(f"Rainbond slice image mismatch for {site}") + site_url = str(component.get("site_url") or "") + configured_urls = self._site_config(site).get("urls") or [] + if configured_urls != [site_url]: + raise RuntimeError(f"Rainbond slice URL mismatch for {site}") + self._wait_site_ready(site_url, timeout_seconds=60) + rows[site] = { + "mode": "remote_rainbond_isolated_slice", + "app_id": slice_receipt["app_id"], + "generation": slice_receipt["generation"], + "service_id": component["service_id"], + "pinned_image": expected, + "receipt_sha256": hashlib.sha256( + self.rainbond_receipt_path.read_bytes() + ).hexdigest(), + } + continue receipt_value = self.remote_receipts.get(site) if receipt_value is not None: if site in STATEFUL_SITES: @@ -413,9 +358,38 @@ def _reset_impl( self, task: BrowserTask, docker_client: Any, *, timeout_seconds: float ) -> dict[str, Any]: sites = list((task.environment or {}).get("sites") or []) + if self.rainbond_pool is not None and any(site in STATEFUL_SITES for site in sites): + assert self.rainbond_receipt_path is not None + receipt = self.rainbond_pool.reset(self.rainbond_receipt_path) + self.config = self.rainbond_pool.persist_runtime_config( + receipt, + self.config_path, + base_config=self.config, + ) + self.environments = self.config["environments"] + for site in sites: + site_config = self._site_config(site) + site_urls = site_config.get("urls") or [] + index = int(site_config.get("active_url_idx") or 0) + if not site_urls or index < 0 or index >= len(site_urls): + raise ValueError(f"WebArena environment has no active URL for site {site!r}") + self._wait_site_ready(str(site_urls[index]), timeout_seconds=timeout_seconds) + return { + "mode": "remote_rainbond_isolated_slice", + "reset": "application_deleted_and_recreated_from_pinned_baseline", + "app_id": receipt["app_id"], + "generation": receipt["generation"], + "sites": { + site: { + "service_id": receipt["components"][site]["service_id"], + "site_url": receipt["components"][site]["site_url"], + } + for site in sites + if site in receipt["components"] + }, + } receipts: dict[str, Any] = {} for site in sites: - url = self._control_url(site) if site in self.remote_receipts: if site in STATEFUL_SITES: raise ValueError(f"remote reset is unsupported for stateful site {site}") @@ -430,6 +404,7 @@ def _reset_impl( "reset": "site_health_check_only", } continue + url = self._control_url(site) if site in STATEFUL_SITES: locked = self.image_lock["images"][site] upstream = get_container_config(site=WebArenaSite(site)) @@ -441,12 +416,8 @@ def _reset_impl( "host_env_ctrl_port": control_port, } ) - defer_removal = bool( - self.config.get("defer_stateful_container_removal", False) - ) - backend = _DockerClientBackend( - docker_client, defer_removal=defer_removal - ) + defer_removal = bool(self.config.get("defer_stateful_container_removal", False)) + backend = DockerBackend(docker_client, defer_removal=defer_removal) manager = ContainerManager( site=WebArenaSite(site), config=config, @@ -490,9 +461,7 @@ def _reset_impl( # WA_ENV_CTRL_EXTERNAL_SITE_URL, so its supervisor attempt # exits before mutating state. Supply the URL explicitly # after that no-op attempt has exited. - self._wait_container_initialization( - container, timeout_seconds=timeout_seconds - ) + self._wait_container_initialization(container, timeout_seconds=timeout_seconds) self._configure_external_url(container, started.url, site=site) self._wait_ready(url, timeout_seconds=timeout_seconds) receipts[site]["external_url_configuration"] = { @@ -502,9 +471,7 @@ def _reset_impl( else: # Shopping and Shopping Admin run their one env-ctrl init # from the official image entrypoint. - self._wait_container_initialization( - container, timeout_seconds=timeout_seconds - ) + self._wait_container_initialization(container, timeout_seconds=timeout_seconds) self._verify_entrypoint_initialization(container) receipts[site]["external_url_configuration"] = { "external_url": started.url, @@ -538,68 +505,19 @@ def _control_url(self, site: str) -> str: @staticmethod def _wait_ready(url: str, *, timeout_seconds: float) -> None: - deadline = time.monotonic() + timeout_seconds - last_error: Exception | None = None - while time.monotonic() < deadline: - try: - with urllib.request.urlopen(url.rstrip("/") + "/status", timeout=5) as response: - payload = json.load(response) - if response.status == 200 and payload.get("success") is True: - return - except (OSError, urllib.error.URLError) as exc: - last_error = exc - time.sleep(0.5) - raise RuntimeError(f"WebArena environment did not become ready: {last_error}") + EnvironmentManager._wait_control(url, timeout=timeout_seconds) @staticmethod def _wait_container_initialization(container: Any, *, timeout_seconds: float) -> None: - """Wait for the image entrypoint's one-time env-ctrl init to exit.""" - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - # Docker's remote ``top`` endpoint requires the PID column to be - # present even though we only inspect the command text. - process_table = container.top(ps_args="-eo pid,args") - processes = process_table.get("Processes") or [] - active = any( - "environment_control.cli init" in " ".join(map(str, row)) - or re.search(r"\benv-ctrl\b.*\binit\b", " ".join(map(str, row))) - for row in processes - ) - if not active: - return - time.sleep(0.5) - raise RuntimeError("WebArena entrypoint initialization did not finish") + EnvironmentManager._wait_entrypoint_init(container, timeout=timeout_seconds) @staticmethod def _configure_external_url(container: Any, external_url: str, *, site: str) -> None: - """Run a site's official initializer with an explicit public URL.""" - result = container.exec_run( - ["env-ctrl", "--verbose", "init", "--base-url", external_url], - demux=False, - ) - output = result.output - if isinstance(output, bytes): - output = output.decode("utf-8", errors="replace") - try: - payload = json.loads(str(output or "")) - except json.JSONDecodeError as exc: - raise RuntimeError(f"WebArena {site} init returned invalid JSON") from exc - if result.exit_code != 0 or payload.get("success") is not True: - message = str(payload.get("message") or "official initializer failed") - raise RuntimeError(f"WebArena {site} init failed: {message}") + EnvironmentManager._explicit_init(container, external_url, site=site) @staticmethod def _verify_entrypoint_initialization(container: Any) -> None: - """Fail immediately if the official supervisor init did not finish cleanly.""" - result = container.exec_run( - ["sh", "-lc", "tail -n 1 /tmp/env-ctrl-init.log"], - demux=False, - ) - output = result.output - if isinstance(output, bytes): - output = output.decode("utf-8", errors="replace") - if result.exit_code != 0 or str(output or "").strip() != "[OK]": - raise RuntimeError("WebArena entrypoint initialization failed") + EnvironmentManager._verify_entrypoint_init(container) @staticmethod def _wait_site_ready(url: str, *, timeout_seconds: float) -> None: @@ -655,9 +573,7 @@ def _wait_site_ready(url: str, *, timeout_seconds: float) -> None: time.sleep(0.5) raise RuntimeError(f"WebArena site did not become ready: {last_error}") - def prepare( - self, task: BrowserTask, page: Any, *, timeout_seconds: float = 300 - ) -> Any: + def prepare(self, task: BrowserTask, page: Any, *, timeout_seconds: float = 300) -> Any: with wall_clock_timeout(timeout_seconds, operation="environment prepare"): return self._prepare_impl(task, page) diff --git a/pyproject.toml b/pyproject.toml index 933fc65fe..9a97f3b2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,14 @@ oss-eval = [ testpaths = ["tests"] addopts = ["--import-mode=importlib"] markers = [ - "real_smoke: starts real browser containers and accesses a public website", + "real_smoke: accesses real containers or live deployment endpoints", ] [tool.ruff] line-length = 100 include = [ + "harness/environments/**/*.py", + "tests/environments/**/*.py", "harness/browser_use/**/*.py", "harness/trajectory_replay/**/*.py", "tests/browser_use/**/*.py", diff --git a/scripts/manage_webarena_environment.py b/scripts/manage_webarena_environment.py index 890acebcf..cf2f39b79 100644 --- a/scripts/manage_webarena_environment.py +++ b/scripts/manage_webarena_environment.py @@ -1,167 +1,6 @@ -"""Start pinned WebArena-Verified sites and record their concrete runtime identity.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import time -from pathlib import Path - -import docker -from webarena_verified.environments.container.config import get_container_config -from webarena_verified.environments.container.manager import ContainerManager -from webarena_verified.environments.setup import setup_init -from webarena_verified.types.task import WebArenaSite - -from harness.trajectory_replay.public_webarena.webarena_environment import ( - _DockerClientBackend, -) - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _locked_image(lock_path: Path, site: str) -> dict: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - if payload.get("schema") != "lex_webarena_environment_images/1": - raise ValueError("invalid WebArena environment image lock") - return dict(payload["images"][site]) - - -def _manager( - site_name: str, - lock_path: Path, - *, - allow_emulation: bool = False, - hostname: str = "localhost", - port: int | None = None, - control_port: int | None = None, -) -> ContainerManager: - site = WebArenaSite(site_name) - locked = _locked_image(lock_path, site_name) - daemon_arch = str(docker.from_env().info().get("Architecture") or "") - aliases = {"aarch64": "arm64", "x86_64": "amd64"} - daemon_arch = aliases.get(daemon_arch, daemon_arch) - image_arch = str(locked.get("architecture") or "") - if not allow_emulation and image_arch and image_arch != daemon_arch: - raise RuntimeError( - f"{site_name} image requires native {image_arch}; Docker daemon is {daemon_arch}. " - "Cross-architecture execution is disabled because it can change behavior or crash." - ) - updates: dict[str, object] = {"docker_img": str(locked["pinned"])} - if port is not None: - updates["host_port"] = port - if control_port is not None: - updates["host_env_ctrl_port"] = control_port - config = get_container_config(site=site).model_copy(update=updates) - return ContainerManager( - site=site, - config=config, - backend=_DockerClientBackend(docker.from_env()), - hostname=hostname, - ) - - -def _write_receipt(path: Path, payload: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - - -def main() -> None: - def port_number(value: str) -> int: - port = int(value) - if not 1 <= port <= 65535: - raise argparse.ArgumentTypeError("port must be between 1 and 65535") - return port - - parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("setup", "start", "status", "stop")) - parser.add_argument( - "--site", - required=True, - choices=("shopping_admin", "shopping", "reddit", "gitlab", "wikipedia", "map"), - ) - parser.add_argument( - "--image-lock", - type=Path, - default=Path("evaluation/browser-use-public/webarena-environment-images.json"), - ) - parser.add_argument("--data-dir", type=Path) - parser.add_argument("--receipt", type=Path) - parser.add_argument("--timeout", type=int, default=900) - parser.add_argument("--port", type=port_number) - parser.add_argument("--control-port", type=port_number) - parser.add_argument("--allow-emulation", action="store_true") - parser.add_argument("--hostname", default="localhost") - args = parser.parse_args() - site = WebArenaSite(args.site) - if (args.port is None) != (args.control_port is None): - parser.error("--port and --control-port must be provided together") - if args.port is not None and args.port == args.control_port: - parser.error("--port and --control-port must differ") - - if args.command == "setup": - if args.site not in {"wikipedia", "map"} or args.data_dir is None: - parser.error("setup requires --site wikipedia|map and --data-dir") - result = setup_init(site=site, data_dir=args.data_dir) - files = [ - { - "name": path.name, - "bytes": path.stat().st_size, - "sha256": _sha256(path), - } - for path in sorted(args.data_dir.iterdir()) - if path.is_file() - ] - receipt = {"command": "setup", "site": args.site, "result": result, "files": files} - else: - manager = _manager( - args.site, - args.image_lock, - allow_emulation=args.allow_emulation or args.command != "start", - hostname=args.hostname, - port=args.port, - control_port=args.control_port, - ) - if args.command == "start": - config = manager.config - started = manager.start( - port=int(config.host_port), - env_ctrl_port=int(config.host_env_ctrl_port), - timeout=args.timeout, - data_dir=args.data_dir, - ) - container = docker.from_env().containers.get(manager.container_name) - container.reload() - daemon_architecture = docker.from_env().info().get("Architecture") - receipt = { - "command": "start", - "site": args.site, - "requested_image": config.docker_img, - "container_name": manager.container_name, - "container_id": container.id, - "image_id": container.image.id, - "repo_digests": sorted(container.image.attrs.get("RepoDigests") or []), - "daemon_architecture": daemon_architecture, - "site_url": started.url, - "control_url": started.env_ctrl_url, - "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - } - elif args.command == "status": - receipt = manager.status().model_dump(mode="json") - else: - manager.stop() - receipt = {"command": "stop", "site": args.site} - if args.receipt: - _write_receipt(args.receipt, receipt) - print(json.dumps(receipt, indent=2, ensure_ascii=False)) +"""Manage the repository's pinned WebArena Docker environments.""" +from harness.environments.webarena.cli import main if __name__ == "__main__": main() diff --git a/scripts/manage_webarena_rainbond_pool.py b/scripts/manage_webarena_rainbond_pool.py new file mode 100644 index 000000000..1a990641d --- /dev/null +++ b/scripts/manage_webarena_rainbond_pool.py @@ -0,0 +1,109 @@ +"""Create, reset, and destroy disposable Rainbond WebArena environment slices.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from harness.environments.webarena.rainbond import ( + RainbondSlicePool, + acquire_slice, + provision_pool, + reconcile_pool_capacity, + release_slice, + scale_pool, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, required=True) + subparsers = parser.add_subparsers(dest="command", required=True) + provision = subparsers.add_parser("provision") + provision.add_argument("--pool-id", required=True) + provision.add_argument("--count", type=int, required=True) + provision.add_argument("--output", type=Path, required=True) + scale = subparsers.add_parser("scale") + scale.add_argument("--pool-id", required=True) + scale_group = scale.add_mutually_exclusive_group(required=True) + scale_group.add_argument("--count", type=int) + scale_group.add_argument("--active", type=int) + scale.add_argument("--minimum-ready", type=int, default=1) + scale.add_argument("--output", type=Path, required=True) + reset = subparsers.add_parser("reset") + reset.add_argument("--slice-dir", type=Path, required=True) + destroy = subparsers.add_parser("destroy") + destroy.add_argument("--slice-dir", type=Path, action="append", required=True) + acquire = subparsers.add_parser("acquire") + acquire.add_argument("--output", type=Path, required=True) + acquire.add_argument("--owner", required=True) + release = subparsers.add_parser("release") + release.add_argument("--output", type=Path, required=True) + release.add_argument("--lease-id", required=True) + args = parser.parse_args() + + if args.command == "provision": + paths = provision_pool( + args.config.resolve(), args.output.resolve(), args.pool_id, args.count + ) + print(json.dumps([str(path / "webarena.config.json") for path in paths])) + return + + if args.command == "scale": + if args.active is not None: + paths = reconcile_pool_capacity( + args.config.resolve(), + args.output.resolve(), + args.pool_id, + args.active, + minimum_ready=args.minimum_ready, + ) + else: + paths = scale_pool( + args.config.resolve(), + args.output.resolve(), + args.pool_id, + args.count, + minimum_ready=args.minimum_ready, + ) + print(json.dumps([str(path / "webarena.config.json") for path in paths])) + return + + if args.command == "acquire": + print(json.dumps(acquire_slice(args.output.resolve(), args.owner), sort_keys=True)) + return + + if args.command == "release": + print( + json.dumps( + release_slice(args.config.resolve(), args.output.resolve(), args.lease_id), + sort_keys=True, + ) + ) + return + + pool = RainbondSlicePool(args.config.resolve()) + if args.command == "reset": + slot_dir = args.slice_dir.resolve() + receipt = pool.reset(slot_dir / "slice-receipt.json") + target = slot_dir / "webarena.config.json" + pool.persist_runtime_config(receipt, target) + print(target) + return + + first_error: BaseException | None = None + for slot_dir in args.slice_dir: + try: + pool.destroy(slot_dir.resolve() / "slice-receipt.json") + except BaseException as exc: + if first_error is None: + first_error = exc + else: + first_error.add_note(f"additional destroy failure for {slot_dir}: {exc}") + if first_error is not None: + raise first_error + + +if __name__ == "__main__": + main() diff --git a/scripts/run_webarena_qualification.py b/scripts/run_webarena_qualification.py index dbd30ff2d..cb64448da 100644 --- a/scripts/run_webarena_qualification.py +++ b/scripts/run_webarena_qualification.py @@ -62,11 +62,17 @@ def qualification_shards(scope_path: Path, suite_path: Path) -> tuple[dict[str, def qualification_waves( - shards: tuple[dict[str, Any], ...], scope: dict[str, Any], *, max_workers: int + shards: tuple[dict[str, Any], ...], + scope: dict[str, Any], + *, + max_workers: int, + isolated_instances: int = 1, ) -> tuple[tuple[tuple[dict[str, Any], ...], ...], ...]: """Group complete site batches into deterministic, conflict-free parallel waves.""" if max_workers < 1: raise ValueError("max_workers must be positive") + if isolated_instances < 1: + raise ValueError("isolated_instances must be positive") sites_by_batch = { str(batch["id"]): frozenset(str(site) for site in batch.get("sites") or []) for batch in scope.get("qualification_batches") or [] @@ -84,7 +90,9 @@ def qualification_waves( selected: list[str] = [] for batch_id in remaining: sites = sites_by_batch[batch_id] - if len(selected) < max_workers and not occupied.intersection(sites): + capacity = min(max_workers, isolated_instances) + isolated = isolated_instances > 1 + if len(selected) < capacity and (isolated or not occupied.intersection(sites)): selected.append(batch_id) occupied.update(sites) waves.append(tuple(tuple(shards_by_batch[batch_id]) for batch_id in selected)) @@ -108,10 +116,9 @@ def validate_completed_shard( raise ValueError(f"{shard['id']}: incomplete qualification output; missing {missing}") manifest = json.loads(required["manifest.json"].read_text(encoding="utf-8")) provenance = manifest.get("artifact_provenance") or {} - if ( - provenance.get("attempts_sha256") != _sha256(required["attempts.jsonl"]) - or provenance.get("summary_sha256") != _sha256(required["summary.json"]) - ): + if provenance.get("attempts_sha256") != _sha256(required["attempts.jsonl"]) or provenance.get( + "summary_sha256" + ) != _sha256(required["summary.json"]): raise ValueError(f"{shard['id']}: qualification artifact hash mismatch") protocol = manifest.get("protocol") or {} selection = protocol.get("task_selection") or {} @@ -212,11 +219,7 @@ def build_scoped_repair_queue( "architecture": scope.get("architecture"), "sites": row.get("sites") or [], "incompatible_sites": row.get("incompatible_sites") or [], - **( - {"kind": row["exclusion_kind"]} - if row.get("exclusion_kind") - else {} - ), + **({"kind": row["exclusion_kind"]} if row.get("exclusion_kind") else {}), **({"evidence_id": row["evidence_id"]} if row.get("evidence_id") else {}), }, }, @@ -246,7 +249,16 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--suite", type=Path, required=True) parser.add_argument("--native-scope", type=Path, required=True) - parser.add_argument("--webarena-config", type=Path, required=True) + parser.add_argument( + "--webarena-config", + type=Path, + action="append", + required=True, + help=( + "runtime config; repeat with independently provisioned Rainbond slice configs " + "to run overlapping site batches concurrently" + ), + ) parser.add_argument( "--env-file", type=Path, @@ -278,10 +290,13 @@ def main() -> None: args = parser.parse_args() suite_path = args.suite.resolve() scope_path = args.native_scope.resolve() - config_path = args.webarena_config.resolve() + config_paths = tuple(path.resolve() for path in args.webarena_config) evidence_root = args.evidence_root.resolve() - if not config_path.is_file(): - raise ValueError("WebArena runtime config is missing") + missing_configs = [str(path) for path in config_paths if not path.is_file()] + if missing_configs: + raise ValueError(f"WebArena runtime configs are missing: {missing_configs}") + if len(set(config_paths)) != len(config_paths): + raise ValueError("WebArena runtime configs must be distinct isolated instances") all_shards = qualification_shards(scope_path, suite_path) scope = json.loads(scope_path.read_text(encoding="utf-8")) selected_batches = set(args.batch_ids or []) @@ -296,14 +311,15 @@ def main() -> None: ) suite_sha256 = _sha256(suite_path) scope_sha256 = _sha256(scope_path) - waves = qualification_waves(selected_shards, scope, max_workers=args.max_site_workers) + waves = qualification_waves( + selected_shards, + scope, + max_workers=args.max_site_workers, + isolated_instances=len(config_paths), + ) - def run_shard(shard: dict[str, Any], qualification_pass_index: int) -> Path: - output = ( - evidence_root - / f"pass-{qualification_pass_index}" - / str(shard["id"]) - ) + def run_shard(shard: dict[str, Any], qualification_pass_index: int, config_path: Path) -> Path: + output = evidence_root / f"pass-{qualification_pass_index}" / str(shard["id"]) if output.exists(): try: validate_completed_shard( @@ -401,21 +417,31 @@ def run_shard(shard: dict[str, Any], qualification_pass_index: int) -> Path: def run_batch( batch: tuple[dict[str, Any], ...], + config_path: Path, qualification_pass_index: int = qualification_pass_index, ) -> list[tuple[tuple[int, str], Path]]: return [ ( (qualification_pass_index, str(shard["id"])), - run_shard(shard, qualification_pass_index), + run_shard(shard, qualification_pass_index, config_path), ) for shard in batch ] + # Extra ready slices are allowed; only a shortage is an error. + assigned = tuple(zip(wave, config_paths, strict=False)) + if len(assigned) != len(wave): + raise RuntimeError("parallel wave exceeds isolated WebArena slice count") if args.dry_run or len(wave) == 1: - completed = [run_batch(batch) for batch in wave] + completed = [run_batch(batch, config) for batch, config in assigned] else: with ThreadPoolExecutor(max_workers=len(wave)) as executor: - completed = list(executor.map(run_batch, wave)) + completed = list( + executor.map( + lambda item: run_batch(item[0], item[1]), + assigned, + ) + ) for batch_rows in completed: attempts_by_id.update(batch_rows) attempts_paths = [ diff --git a/scripts/verify_webarena_env_contract.py b/scripts/verify_webarena_env_contract.py index 3a7fadf7c..de718725f 100644 --- a/scripts/verify_webarena_env_contract.py +++ b/scripts/verify_webarena_env_contract.py @@ -1,119 +1,91 @@ -"""Verify that webarena-env and browser-eval bind the same site images.""" +"""Verify the repository's WebArena deployment and evaluation configuration.""" from __future__ import annotations -import argparse import json import re from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] -DEFAULT_CONTRACT = ROOT / "evaluation/browser-use-public/webarena-env-contract.json" -def verify(environment_repo: Path, contract_path: Path = DEFAULT_CONTRACT) -> dict: - contract = json.loads(contract_path.read_text(encoding="utf-8")) - if contract.get("schema") != "lexmount_webarena_cross_repo_contract/1": - raise ValueError("invalid cross-repository contract schema") - env_lock = json.loads( - (environment_repo.resolve() / contract["environment_lock"]).read_text(encoding="utf-8") +def verify(root: Path = ROOT) -> dict: + lock_path = root / "evaluation/browser-use-public/webarena-environment-images.json" + lock = json.loads(lock_path.read_text(encoding="utf-8")) + runtime_path = root / "deploy/webarena/runtime.example.json" + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + evaluation_path = ( + root / "examples/trajectory_replay/public_webarena/webarena.config.example.json" ) - eval_lock = json.loads( - (ROOT / contract["evaluation_lock"]).read_text(encoding="utf-8") - ) - expected_version = contract["webarena_verified_version"] - if env_lock.get("webarena_verified_version") != expected_version: - raise ValueError("webarena-env package version binding differs") - if eval_lock.get("webarena_verified_version") != expected_version: - raise ValueError("browser-eval package version binding differs") - sites = contract["required_sites"] - if set(env_lock.get("images") or {}) != set(sites): - raise ValueError("webarena-env image inventory differs") - if set(eval_lock.get("images") or {}) != set(sites): - raise ValueError("browser-eval image inventory differs") - mismatches = [] - for site in sites: - left = env_lock["images"][site] - right = eval_lock["images"][site] - for key in ("pinned", "architecture", "os"): - if left.get(key) != right.get(key): - mismatches.append(f"{site}.{key}") - if mismatches: - raise ValueError("cross-repository image binding differs: " + ", ".join(mismatches)) - env_runtime = json.loads( - (environment_repo.resolve() / contract["environment_runtime_template"]).read_text( - encoding="utf-8" - ) - ) - eval_runtime = json.loads( - (ROOT / contract["evaluation_runtime_template"]).read_text(encoding="utf-8") - ) - for site in sites: - token = f"__{site.upper()}__" - site_row = (env_runtime.get("sites") or {}).get(site) or {} - eval_row = (eval_runtime.get("environments") or {}).get(token) or {} - urls = eval_row.get("urls") or [] - control_url = (eval_runtime.get("control_urls") or {}).get(site) - if len(urls) != 1 or not control_url: - raise ValueError(f"cross-repository runtime template is missing {site} URLs") - if urlparse(str(urls[0])).port != site_row.get("port"): - raise ValueError(f"cross-repository runtime site port differs: {site}") - if urlparse(str(control_url)).port != site_row.get("control_port"): - raise ValueError(f"cross-repository runtime control port differs: {site}") - env_credentials = (env_runtime.get("credentials_env") or {}).get(site) - if env_credentials is not None and env_credentials != eval_row.get("credentials_env"): - raise ValueError(f"cross-repository credential variable names differ: {site}") - delivery_path = environment_repo.resolve() / contract["environment_site_delivery_contract"] - if not delivery_path.is_file(): - raise ValueError(f"environment repo is missing site delivery contract: {delivery_path}") + evaluation = json.loads(evaluation_path.read_text(encoding="utf-8")) + pool_path = root / "examples/trajectory_replay/public_webarena/rainbond-pool.example.json" + pool = json.loads(pool_path.read_text(encoding="utf-8")) + delivery_path = root / "deploy/webarena/site-delivery-contract.json" delivery = json.loads(delivery_path.read_text(encoding="utf-8")) - if delivery.get("webarena_verified_version") != expected_version: - raise ValueError("cross-repository site delivery package version differs") - image_lock = delivery.get("image_lock") - if not isinstance(image_lock, str) or not image_lock: - raise ValueError("cross-repository site delivery image_lock is missing") - if (delivery_path.parent / image_lock).resolve() != ( - environment_repo.resolve() / contract["environment_lock"] - ).resolve(): - raise ValueError("cross-repository site delivery image lock path differs") - components = delivery.get("components") or {} - if set(components) != set(sites) - {"wikipedia"}: - raise ValueError("cross-repository site delivery inventory differs") - for site, component in components.items(): - if component.get("image") != env_lock["images"][site]["pinned"]: - raise ValueError(f"cross-repository site delivery image differs: {site}") - map_site = components["map"] - archive_urls = map_site.get("data_urls") or [] - archive_digests = map_site.get("data_archive_sha256") or {} - archive_bytes = map_site.get("data_archive_bytes") or {} - archive_names = [Path(urlparse(url).path).name for url in archive_urls] + if lock.get("schema") != "lex_webarena_environment_images/1": + raise ValueError("invalid image lock schema") if ( - len(archive_names) != 3 - or len(set(archive_names)) != 3 - or set(archive_names) != set(archive_digests) - or set(archive_names) != set(archive_bytes) - or any(not re.fullmatch(r"[a-f0-9]{64}", str(value)) for value in archive_digests.values()) - or any(not isinstance(value, int) or value <= 0 for value in archive_bytes.values()) + lock.get("webarena_verified_version") != "1.2.3" + or delivery.get("webarena_verified_version") != "1.2.3" + ): + raise ValueError("WebArena package version differs") + sites = {"shopping_admin", "shopping", "reddit", "gitlab", "map", "wikipedia"} + if set(lock.get("images") or {}) != sites or set(runtime.get("sites") or {}) != sites: + raise ValueError("site inventory differs") + for path, config in ( + (runtime_path, runtime), + (delivery_path, delivery), + (evaluation_path, evaluation), + (pool_path, pool), ): - raise ValueError("cross-repository Map archive checksum contract is incomplete") + if ( + not config.get("image_lock") + or (path.parent / config["image_lock"]).resolve() != lock_path.resolve() + ): + raise ValueError("deployment must reference the canonical image lock") + if set(delivery.get("components") or {}) != sites - {"wikipedia"}: + raise ValueError("site delivery inventory differs") + for site in sites: + image = lock["images"][site] + if not re.fullmatch(r".+@sha256:[a-f0-9]{64}", image.get("pinned", "")): + raise ValueError(f"image must have a valid digest: {site}") + if image.get("architecture") not in {"amd64", "arm64"} or image.get("os") != "linux": + raise ValueError(f"image platform is invalid: {site}") + row = runtime["sites"][site] + eval_row = evaluation["environments"][f"__{site.upper()}__"] + if len(eval_row["urls"]) != 1 or urlparse(eval_row["urls"][0]).port != row["port"]: + raise ValueError(f"site port differs: {site}") + if urlparse(evaluation["control_urls"][site]).port != row["control_port"]: + raise ValueError(f"control port differs: {site}") + if site in runtime["credentials_env"] and runtime["credentials_env"][site] != eval_row.get( + "credentials_env" + ): + raise ValueError(f"credential variable names differ: {site}") + component = delivery["components"].get(site) + if component is not None: + if "image" in component or component.get("architecture") != image["architecture"]: + raise ValueError(f"site delivery must use the canonical image identity: {site}") + map_site = delivery["components"]["map"] + names = [Path(urlparse(url).path).name for url in map_site["data_urls"]] + digests = map_site["data_archive_sha256"] + sizes = map_site["data_archive_bytes"] + if ( + len(names) != 3 + or len(set(names)) != 3 + or set(names) != set(digests) + or set(names) != set(sizes) + or any(not re.fullmatch(r"[a-f0-9]{64}", str(value)) for value in digests.values()) + or any(type(value) is not int or value <= 0 for value in sizes.values()) + ): + raise ValueError("Map archive checksum contract is incomplete") return { "status": "pass", - "webarena_verified_version": expected_version, "sites": len(sites), + "image_lock": "canonical", "runtime_templates": "matching", - "map_archive_checksums": "complete", - "environment_pull_request": contract["environment_pull_request"], } -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--environment-repo", type=Path, required=True) - parser.add_argument("--contract", type=Path, default=DEFAULT_CONTRACT) - args = parser.parse_args() - print(json.dumps(verify(args.environment_repo, args.contract), ensure_ascii=False)) - - if __name__ == "__main__": - main() + print(json.dumps(verify(), ensure_ascii=False)) diff --git a/tests/environments/test_webarena_backend.py b/tests/environments/test_webarena_backend.py new file mode 100644 index 000000000..5bef570d7 --- /dev/null +++ b/tests/environments/test_webarena_backend.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import docker +import pytest +from webarena_verified.environments.container.config import get_container_config +from webarena_verified.environments.container.manager import ContainerManager +from webarena_verified.types.task import WebArenaSite + +from harness.environments.webarena.backend import DockerBackend + + +def test_container_run_translates_upstream_host_to_container_port_mapping() -> None: + calls = [] + containers = SimpleNamespace(run=lambda *args, **kwargs: calls.append((args, kwargs))) + backend = DockerBackend(SimpleNamespace(containers=containers)) + + backend.container_run( + name="webarena_verified_map", + image="example/map@sha256:" + "a" * 64, + port_mappings={23000: 3000, 23001: 5555}, + volume_mappings={}, + ) + + assert calls[0][1]["ports"] == {"3000/tcp": 23000, "5555/tcp": 23001} + + +@pytest.mark.parametrize( + ("site", "expected_variable", "expected_value"), + [ + (WebArenaSite.SHOPPING, "WA_ENV_CTRL_CMD_TIMEOUT_SEC", "600"), + (WebArenaSite.SHOPPING_ADMIN, "WA_ENV_CTRL_CMD_TIMEOUT_SEC", "600"), + (WebArenaSite.GITLAB, "WA_ENV_CTRL_SKIP_RECONFIGURE", "true"), + ], +) +def test_pinned_upstream_start_uses_expected_name_ports_and_environment( + site: WebArenaSite, expected_variable: str, expected_value: str +) -> None: + calls = [] + + class Containers: + def get(self, _name: str): + raise docker.errors.NotFound("missing") + + def run(self, *args, **kwargs): + calls.append((args, kwargs)) + + backend = DockerBackend(SimpleNamespace(containers=Containers())) + manager = ContainerManager(site=site, backend=backend) + config = get_container_config(site=site) + + manager.start(port=23000, env_ctrl_port=23001, wait=False) + + assert manager.container_name == f"webarena_verified_{site.value}" + assert calls[0][1]["name"] == manager.container_name + assert calls[0][1]["ports"] == { + f"{config.container_port}/tcp": 23000, + f"{config.env_ctrl_port}/tcp": 23001, + } + assert calls[0][1]["environment"][expected_variable] == expected_value + + +def test_deferred_removal_stops_and_retains_the_named_container() -> None: + actions = [] + + class Container: + id = "1234567890abcdef" + status = "running" + + def reload(self): + actions.append("reload") + + def stop(self, *, timeout: int): + actions.append(("stop", timeout)) + + def rename(self, name: str): + actions.append(("rename", name)) + + def remove(self, *, force: bool): + actions.append(("remove", force)) + + backend = DockerBackend( + SimpleNamespace(containers=SimpleNamespace(get=lambda _name: Container())), + defer_removal=True, + ) + + backend.container_remove(name="webarena_verified_shopping") + + retired = "webarena_verified_shopping__retired__1234567890ab" + assert actions == ["reload", ("stop", 10), ("rename", retired)] + assert backend.retired_containers == [retired] diff --git a/tests/environments/test_webarena_cli.py b/tests/environments/test_webarena_cli.py new file mode 100644 index 000000000..686950122 --- /dev/null +++ b/tests/environments/test_webarena_cli.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import json +import sys +from types import SimpleNamespace + +import pytest + +from harness.environments.webarena.cli import main + + +def test_batch_failure_retains_completed_site_receipts(monkeypatch, tmp_path) -> None: + receipt = tmp_path / "batch.json" + + class Manager: + def verify(self, name: str): + if name == "reddit": + raise RuntimeError("second site unavailable") + return {"site": name, "status": "running"} + + monkeypatch.setattr( + "harness.environments.webarena.cli.RuntimeConfig.load", + lambda _path: SimpleNamespace(sites={"map": object(), "reddit": object()}), + ) + monkeypatch.setattr( + "harness.environments.webarena.cli.EnvironmentManager", + lambda *_args, **_kwargs: Manager(), + ) + monkeypatch.setattr( + sys, + "argv", + [ + "webarena-env", + "verify", + "--config", + str(tmp_path / "config.json"), + "--site", + "map", + "--site", + "reddit", + "--receipt", + str(receipt), + ], + ) + + with pytest.raises(RuntimeError, match="second site unavailable"): + main() + + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["status"] == "partial_failure" + assert payload["failed_site"] == "reddit" + assert payload["sites"] == [{"site": "map", "status": "running"}] diff --git a/tests/environments/test_webarena_environment_config.py b/tests/environments/test_webarena_environment_config.py new file mode 100644 index 000000000..8719a6f32 --- /dev/null +++ b/tests/environments/test_webarena_environment_config.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from harness.environments.webarena.config import RuntimeConfig, normalized_architecture + +ROOT = Path(__file__).resolve().parents[2] + + +def _runtime(tmp_path: Path, *, hostname: str = "127.0.0.1") -> Path: + lock = json.loads( + (ROOT / "evaluation/browser-use-public/webarena-environment-images.json").read_text( + encoding="utf-8" + ) + ) + (tmp_path / "images.lock.json").write_text(json.dumps(lock), encoding="utf-8") + runtime = json.loads( + (ROOT / "deploy/webarena/runtime.example.json").read_text(encoding="utf-8") + ) + runtime["hostname"] = hostname + runtime["image_lock"] = "images.lock.json" + path = tmp_path / "runtime.json" + path.write_text(json.dumps(runtime), encoding="utf-8") + return path + + +def test_runtime_loads_every_digest_pinned_site(tmp_path: Path) -> None: + config = RuntimeConfig.load(_runtime(tmp_path)) + + assert set(config.sites) == { + "shopping_admin", + "shopping", + "reddit", + "gitlab", + "map", + "wikipedia", + } + assert config.sites["gitlab"].architecture == "amd64" + assert config.sites["wikipedia"].architecture == "arm64" + assert all("@sha256:" in site.image for site in config.sites.values()) + + +def test_example_hostname_must_be_replaced() -> None: + with pytest.raises(ValueError, match="replace the example placeholder"): + RuntimeConfig.load(ROOT / "deploy/webarena/runtime.example.json") + + +def test_architecture_aliases_cover_intel_and_amd_x86_hosts() -> None: + assert normalized_architecture("x86_64") == "amd64" + assert normalized_architecture("amd64") == "amd64" + assert normalized_architecture("aarch64") == "arm64" + + +def test_missing_locked_architecture_fails_before_docker_use(tmp_path: Path) -> None: + path = _runtime(tmp_path) + lock_path = tmp_path / "images.lock.json" + lock = json.loads(lock_path.read_text(encoding="utf-8")) + lock["images"]["map"]["architecture"] = "" + lock_path.write_text(json.dumps(lock), encoding="utf-8") + + with pytest.raises(ValueError, match="explicitly amd64 or arm64"): + RuntimeConfig.load(path) diff --git a/tests/environments/test_webarena_environment_manager.py b/tests/environments/test_webarena_environment_manager.py new file mode 100644 index 000000000..c61625b61 --- /dev/null +++ b/tests/environments/test_webarena_environment_manager.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from harness.environments.webarena.config import RuntimeConfig, SiteConfig +from harness.environments.webarena.manager import EnvironmentManager + + +class MissingContainers: + def get(self, _name: str): + import docker + + raise docker.errors.NotFound("missing") + + +def _config() -> RuntimeConfig: + return RuntimeConfig( + path=Path(__file__), + hostname="127.0.0.1", + sites={ + "map": SiteConfig( + name="map", + image="example/map@sha256:" + "a" * 64, + architecture="amd64", + port=23000, + control_port=23001, + ), + "wikipedia": SiteConfig( + name="wikipedia", + image="example/wiki@sha256:" + "b" * 64, + architecture="arm64", + port=18888, + control_port=18889, + ), + }, + ) + + +@pytest.mark.parametrize(("site", "internal_port"), [("shopping_admin", 80), ("gitlab", 8023)]) +def test_manager_overrides_host_ports_without_changing_internal_ports(site, internal_port): + config = _config() + config.sites[site] = SiteConfig( + name=site, + image="example/site@sha256:" + "a" * 64, + architecture="amd64", + port=17780, + control_port=17781, + ) + manager = EnvironmentManager(config, SimpleNamespace(info=lambda: {"Architecture": "x86_64"})) + upstream = manager._manager(config.sites[site]) + assert upstream.config.docker_img == config.sites[site].image + assert upstream.config.host_port == 17780 + assert upstream.config.host_env_ctrl_port == 17781 + assert upstream.config.container_port == internal_port + + +def test_verify_accepts_intel_x86_64_for_amd64_site() -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + + receipt = manager.verify("map") + + assert receipt["daemon_architecture"] == "amd64" + assert receipt["status"] == "not_created" + + +def test_verify_rejects_arm_image_on_x86_without_emulation() -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + + with pytest.raises(RuntimeError, match="requires native arm64"): + manager.verify("wikipedia") + + +def test_wait_control_retries_transient_invalid_json(monkeypatch) -> None: + class Response(BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + responses = iter((Response(b"not-json"), Response(b'{"success": true}'))) + monkeypatch.setattr( + "harness.environments.webarena.manager.urllib.request.urlopen", + lambda *_args, **_kwargs: next(responses), + ) + monkeypatch.setattr("harness.environments.webarena.manager.time.sleep", lambda _seconds: None) + + EnvironmentManager._wait_control("http://127.0.0.1:23001", timeout=1) + + +def test_verify_uses_upstream_manager_container_name(monkeypatch) -> None: + requested = [] + + class Containers(MissingContainers): + def get(self, name: str): + requested.append(name) + return super().get(name) + + client = SimpleNamespace(info=lambda: {"Architecture": "x86_64"}, containers=Containers()) + manager = EnvironmentManager(_config(), client) + monkeypatch.setattr( + manager, "_manager", lambda _site: SimpleNamespace(container_name="upstream-map") + ) + + receipt = manager.verify("map") + + assert requested == ["upstream-map"] + assert receipt["container_name"] == "upstream-map" + + +@pytest.mark.parametrize( + ("site_name", "expected_steps"), + [ + ("gitlab", ["control", "entrypoint", "explicit", "control"]), + ("reddit", ["control", "entrypoint", "explicit", "control"]), + ("shopping", ["control", "entrypoint", "verified"]), + ("shopping_admin", ["control", "entrypoint", "verified"]), + ], +) +def test_start_runs_only_the_declared_site_initialization( + monkeypatch, site_name: str, expected_steps: list[str] +) -> None: + config = _config() + config.sites[site_name] = SiteConfig( + name=site_name, + image="example/site@sha256:" + "c" * 64, + architecture="amd64", + port=23000, + control_port=23001, + ) + container = SimpleNamespace() + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, + containers=SimpleNamespace(get=lambda _name: container), + ) + manager = EnvironmentManager(config, client) + steps = [] + upstream = SimpleNamespace( + container_name=f"webarena_verified_{site_name}", + start=lambda **_kwargs: SimpleNamespace( + url="http://127.0.0.1:23000", + env_ctrl_url="http://127.0.0.1:23001", + ), + ) + monkeypatch.setattr(manager, "_manager", lambda _site: upstream) + monkeypatch.setattr(manager, "verify", lambda *_args, **_kwargs: {"status": "running"}) + monkeypatch.setattr(manager, "_wait_control", lambda *_args, **_kwargs: steps.append("control")) + monkeypatch.setattr( + manager, + "_wait_entrypoint_init", + lambda *_args, **_kwargs: steps.append("entrypoint"), + ) + monkeypatch.setattr( + manager, "_explicit_init", lambda *_args, **_kwargs: steps.append("explicit") + ) + monkeypatch.setattr( + manager, + "_verify_entrypoint_init", + lambda *_args, **_kwargs: steps.append("verified"), + ) + + receipt = manager.start(site_name, reset=True, timeout=1) + + assert steps == expected_steps + assert receipt["command"] == "reset" + assert receipt["status"] == "running" + + +def test_stop_delegates_to_pinned_upstream_manager(monkeypatch) -> None: + client = SimpleNamespace( + info=lambda: {"Architecture": "x86_64"}, containers=MissingContainers() + ) + manager = EnvironmentManager(_config(), client) + calls = [] + monkeypatch.setattr( + manager, + "_manager", + lambda _site: SimpleNamespace( + container_name="upstream-map", stop=lambda: calls.append("stop") + ), + ) + + receipt = manager.stop("map") + + assert calls == ["stop"] + assert receipt == {"command": "stop", "site": "map", "container_name": "upstream-map"} diff --git a/tests/environments/test_webarena_map_archives.py b/tests/environments/test_webarena_map_archives.py new file mode 100644 index 000000000..8a0518c3e --- /dev/null +++ b/tests/environments/test_webarena_map_archives.py @@ -0,0 +1,78 @@ +"""Map data volumes may only be initialized from locked archive bytes.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from harness.environments.webarena.map_archives import DEFAULT_CONTRACT, verify_archives + + +def _contract(tmp_path: Path, *, wrong_digest: bool = False) -> Path: + archives = {"osm_tile_server.tar": b"tile", "nominatim_volumes.tar": b"geo"} + for name, data in archives.items(): + (tmp_path / name).write_bytes(data) + checksums = {name: hashlib.sha256(data).hexdigest() for name, data in archives.items()} + if wrong_digest: + checksums["nominatim_volumes.tar"] = "0" * 64 + contract = { + "components": { + "map": { + "data_urls": [f"https://example.invalid/{name}" for name in archives], + "data_archive_sha256": checksums, + "data_archive_bytes": {name: len(data) for name, data in archives.items()}, + } + } + } + path = tmp_path / "contract.json" + path.write_text(json.dumps(contract), encoding="utf-8") + return path + + +def test_map_archive_verifier_accepts_exact_local_bytes(tmp_path: Path) -> None: + rows = verify_archives(tmp_path, _contract(tmp_path)) + assert [row["archive"] for row in rows] == [ + "osm_tile_server.tar", + "nominatim_volumes.tar", + ] + + +def test_map_archive_verifier_uses_canonical_contract_outside_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _contract(tmp_path) + monkeypatch.chdir(tmp_path) + assert DEFAULT_CONTRACT.is_absolute() and DEFAULT_CONTRACT.is_file() + with pytest.raises(ValueError, match="size mismatch: osm_tile_server.tar"): + verify_archives(tmp_path) + + +def test_map_archive_verifier_rejects_wrong_bytes(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="nominatim_volumes.tar"): + verify_archives(tmp_path, _contract(tmp_path, wrong_digest=True)) + + +def test_map_archive_verifier_rejects_missing_file(tmp_path: Path) -> None: + contract = _contract(tmp_path) + (tmp_path / "osm_tile_server.tar").unlink() + with pytest.raises(FileNotFoundError): + verify_archives(tmp_path, contract) + + +def test_map_archive_verifier_rejects_truncated_file_before_hashing(tmp_path: Path) -> None: + contract = _contract(tmp_path) + (tmp_path / "osm_tile_server.tar").write_bytes(b"bad") + with pytest.raises(ValueError, match="size mismatch: osm_tile_server.tar"): + verify_archives(tmp_path, contract) + + +def test_map_archive_verifier_rejects_inventory_drift(tmp_path: Path) -> None: + contract = _contract(tmp_path) + payload = json.loads(contract.read_text(encoding="utf-8")) + payload["components"]["map"]["data_urls"].append("https://example.invalid/other.tar") + contract.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="inventory"): + verify_archives(tmp_path, contract) diff --git a/tests/test_manage_webarena_rainbond_pool.py b/tests/test_manage_webarena_rainbond_pool.py new file mode 100644 index 000000000..19d659281 --- /dev/null +++ b/tests/test_manage_webarena_rainbond_pool.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "manage_webarena_rainbond_pool.py" +SPEC = importlib.util.spec_from_file_location("manage_webarena_rainbond_pool", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +manage_webarena_rainbond_pool = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(manage_webarena_rainbond_pool) + + +def test_reset_persists_runtime_config_atomically(tmp_path, monkeypatch, capsys): + config_path = tmp_path / "pool.json" + slot_dir = tmp_path / "slice-1" + slot_dir.mkdir() + calls = [] + + class Pool: + def __init__(self, config): + assert config == config_path.resolve() + + def reset(self, receipt_path): + assert receipt_path == slot_dir.resolve() / "slice-receipt.json" + return {"app_id": 201} + + def persist_runtime_config(self, receipt, target): + calls.append((receipt, target)) + + monkeypatch.setattr(manage_webarena_rainbond_pool, "RainbondSlicePool", Pool) + monkeypatch.setattr( + sys, + "argv", + [ + "manage_webarena_rainbond_pool.py", + "--config", + str(config_path), + "reset", + "--slice-dir", + str(slot_dir), + ], + ) + + manage_webarena_rainbond_pool.main() + + target = slot_dir.resolve() / "webarena.config.json" + assert calls == [({"app_id": 201}, target)] + assert capsys.readouterr().out.strip() == str(target) diff --git a/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py deleted file mode 100644 index cb7410c6e..000000000 --- a/tests/trajectory_replay/public_webarena/test_manage_webarena_environment.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import json -from types import SimpleNamespace - -from scripts.manage_webarena_environment import _manager - - -def test_manager_applies_explicit_host_ports(tmp_path, monkeypatch): - pinned = "am1n3e/webarena-verified-shopping_admin@sha256:" + "a" * 64 - lock = tmp_path / "images.json" - lock.write_text( - json.dumps( - { - "schema": "lex_webarena_environment_images/1", - "images": { - "shopping_admin": { - "pinned": pinned, - "architecture": "amd64", - } - }, - } - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "scripts.manage_webarena_environment.docker.from_env", - lambda: SimpleNamespace(info=lambda: {"Architecture": "x86_64"}), - ) - - manager = _manager( - "shopping_admin", - lock, - port=17780, - control_port=17781, - ) - - assert manager.config.docker_img == pinned - assert manager.config.host_port == 17780 - assert manager.config.host_env_ctrl_port == 17781 - assert manager.config.container_port == 80 - - -def test_manager_keeps_gitlab_internal_health_port(tmp_path, monkeypatch): - pinned = "am1n3e/webarena-verified-gitlab@sha256:" + "b" * 64 - lock = tmp_path / "images.json" - lock.write_text( - json.dumps( - { - "schema": "lex_webarena_environment_images/1", - "images": { - "gitlab": { - "pinned": pinned, - "architecture": "amd64", - } - }, - } - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "scripts.manage_webarena_environment.docker.from_env", - lambda: SimpleNamespace(info=lambda: {"Architecture": "amd64"}), - ) - - manager = _manager("gitlab", lock, port=18023, control_port=18024) - - assert manager.config.host_port == 18023 - assert manager.config.container_port == 8023 diff --git a/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py b/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py index 592fb7ec8..b2958ac98 100644 --- a/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py +++ b/tests/trajectory_replay/public_webarena/test_qualification_orchestrator.py @@ -33,9 +33,7 @@ def test_canonical_qualification_plan_is_an_exact_eighteen_shard_partition(): def test_parallel_qualification_waves_never_share_a_site(): - scope = json.loads( - Path("evaluation/browser-use-public/webarena-amd64-scope.json").read_text() - ) + scope = json.loads(Path("evaluation/browser-use-public/webarena-amd64-scope.json").read_text()) shards = qualification_shards( Path("evaluation/browser-use-public/webarena-amd64-scope.json"), Path("evaluation/browser-use-public/webarena-human-suite.json"), @@ -46,9 +44,9 @@ def test_parallel_qualification_waves_never_share_a_site(): assert len(waves) > 1 assert all(len(wave) <= 3 for wave in waves) - assert { - shard["id"] for wave in waves for batch in wave for shard in batch - } == {shard["id"] for shard in shards} + assert {shard["id"] for wave in waves for batch in wave for shard in batch} == { + shard["id"] for shard in shards + } for wave in waves: occupied = set() for batch in wave: @@ -62,6 +60,26 @@ def test_parallel_qualification_rejects_non_positive_worker_count(): qualification_waves((), {"qualification_batches": []}, max_workers=0) +def test_independent_slices_allow_overlapping_sites_in_the_same_wave(): + shards = ( + {"id": "a-0", "batch_id": "a"}, + {"id": "b-0", "batch_id": "b"}, + ) + scope = { + "qualification_batches": [ + {"id": "a", "sites": ["reddit"]}, + {"id": "b", "sites": ["reddit"]}, + ] + } + + shared = qualification_waves(shards, scope, max_workers=2) + isolated = qualification_waves(shards, scope, max_workers=2, isolated_instances=2) + + assert len(shared) == 2 + assert len(isolated) == 1 + assert [batch[0]["id"] for batch in isolated[0]] == ["a-0", "b-0"] + + def test_completed_shard_validation_binds_manifest_hashes_scope_and_attempts(tmp_path): output = tmp_path / "shopping-0-of-1" output.mkdir() @@ -316,9 +334,7 @@ def test_scoped_queue_covers_measured_and_architecture_excluded_tasks(tmp_path): ], } - queue = build_scoped_repair_queue( - suite, [attempts], scope, scope_sha256="f" * 64 - ) + queue = build_scoped_repair_queue(suite, [attempts], scope, scope_sha256="f" * 64) assert [row["task_id"] for row in queue["tasks"]] == ["included", "excluded"] assert queue["tasks"][1]["adjudication"] == { diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py new file mode 100644 index 000000000..1e9bc1d19 --- /dev/null +++ b/tests/trajectory_replay/public_webarena/test_rainbond_live_smoke.py @@ -0,0 +1,127 @@ +"""Live HTTP smoke test for one isolated Rainbond WebArena slice. + +Set WEBARENA_RAINBOND_SMOKE_URLS to one four-site URL object or a list of +such objects to exercise multiple slices together. The test sends requests +to every site concurrently so a green result covers the shared gateway path +as well as each site's basic page rendering. +""" + +from __future__ import annotations + +import json +import os +import statistics +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest + +SITES = { + "shopping_admin": "Home Page", + "shopping": "<title>One Stop Market", + "reddit": "<title>Postmill", + "gitlab": "<title>Sign in · GitLab", +} +REQUESTS_PER_SITE = 8 +WIKIPEDIA_ARTICLE = "/content/wikipedia_en_all_maxi_2022-05/A/Boston_Celtics" +REQUEST_TIMEOUT_SECONDS = 30 +P95_LIMIT_SECONDS = 10 + + +def _configured_urls() -> dict[str, str]: + raw = os.environ.get("WEBARENA_RAINBOND_SMOKE_URLS", "") + if not raw: + pytest.skip("WEBARENA_RAINBOND_SMOKE_URLS is not set") + value = json.loads(raw) + slices = value if isinstance(value, list) else [value] + assert slices, "at least one slice is required" + urls = {} + for index, sites in enumerate(slices, start=1): + if not isinstance(sites, dict) or set(sites) != set(SITES): + raise AssertionError("each slice must contain exactly: " + ", ".join(sorted(SITES))) + for site, url in sites.items(): + urls[f"slice-{index}/{site}"] = str(url).rstrip("/") + "/" + assert len(set(urls.values())) == len(urls), "mutable sites must have distinct URLs" + if any(not url.startswith(("http://", "https://")) for url in urls.values()): + raise AssertionError("all Rainbond smoke URLs must use HTTP or HTTPS") + wikipedia = os.environ.get("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", "").rstrip("/") + if wikipedia: + assert wikipedia.startswith(("http://", "https://")), "Wikipedia must use HTTP or HTTPS" + urls["shared/wikipedia"] = wikipedia + WIKIPEDIA_ARTICLE + return urls + + +def _request(site: str, url: str) -> tuple[str, float, int]: + started = time.monotonic() + request = urllib.request.Request( + url, + headers={"User-Agent": "browser-eval-rainbond-smoke/1"}, + ) + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + body = response.read() + status = response.status + except Exception as exc: + raise AssertionError(f"{site} request failed: {type(exc).__name__}: {exc}") from exc + elapsed = time.monotonic() - started + assert status == 200, f"{site} returned HTTP {status}" + expected = ( + "<title>Boston Celtics" if site == "shared/wikipedia" + else SITES[site.rsplit("/", 1)[-1]] + ) + assert expected.encode() in body, f"{site} returned the wrong page" + return site, elapsed, len(body) + + +@pytest.mark.real_smoke +def test_isolated_rainbond_slice_handles_concurrent_page_loads(): + urls = _configured_urls() + jobs = [ + (site, urls[site]) + for _ in range(REQUESTS_PER_SITE) + for site in sorted(urls) + ] + results: list[tuple[str, float, int]] = [] + failures = [] + with ThreadPoolExecutor(max_workers=len(jobs)) as executor: + futures = [executor.submit(_request, site, url) for site, url in jobs] + for future in as_completed(futures): + try: + results.append(future.result()) + except Exception as exc: + failures.append(str(exc)) + + assert not failures, "\n".join(failures) + assert len(results) == len(urls) * REQUESTS_PER_SITE + for site in sorted(urls): + durations = sorted(elapsed for name, elapsed, _size in results if name == site) + sizes = [size for name, _elapsed, size in results if name == site] + p95 = durations[min(len(durations) - 1, int(len(durations) * 0.95))] + assert p95 < P95_LIMIT_SECONDS, f"{site} p95 was {p95:.3f}s" + assert min(sizes) > 1_000, f"{site} returned an unexpectedly small page" + print( + f"{site}: requests={len(durations)} " + f"median={statistics.median(durations):.3f}s p95={p95:.3f}s" + ) + + +def test_smoke_configuration_accepts_multiple_isolated_slices(monkeypatch): + monkeypatch.delenv("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", raising=False) + slices = [{site: f"http://slot-{i}-{site}.example" for site in SITES} for i in (1, 2)] + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps(slices)) + assert len(_configured_urls()) == 8 + + +def test_smoke_configuration_includes_one_shared_wikipedia(monkeypatch): + sites = {site: f"http://{site}.example" for site in SITES} + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps(sites)) + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_WIKIPEDIA_URL", "https://wiki.example/") + assert _configured_urls()["shared/wikipedia"] == "https://wiki.example" + WIKIPEDIA_ARTICLE + + +def test_smoke_configuration_rejects_shared_mutable_sites(monkeypatch): + sites = {site: f"http://{site}.example" for site in SITES} + monkeypatch.setenv("WEBARENA_RAINBOND_SMOKE_URLS", json.dumps([sites, sites])) + with pytest.raises(AssertionError, match="distinct URLs"): + _configured_urls() diff --git a/tests/trajectory_replay/public_webarena/test_rainbond_slices.py b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py new file mode 100644 index 000000000..a74888f9d --- /dev/null +++ b/tests/trajectory_replay/public_webarena/test_rainbond_slices.py @@ -0,0 +1,1023 @@ +from __future__ import annotations + +import json +import threading +import urllib.error +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from harness.environments.webarena.rainbond import ( + LEASE_SCHEMA, + RECEIPT_SCHEMA, + STATEFUL_SITES, + RainbondSlicePool, + acquire_slice, + provision_pool, + reconcile_pool_capacity, + release_slice, + scale_pool, +) + +PINNED = { + site: f"example/{site}@sha256:{index:064x}" + for index, site in enumerate(STATEFUL_SITES, start=1) +} +INITIALIZER_IMAGE = "alpine@sha256:" + "a" * 64 + + +def test_slow_release_does_not_block_acquisition_of_another_slot(tmp_path, monkeypatch): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + lease = acquire_slice(output, "task-a") + resetting = threading.Event() + finish_reset = threading.Event() + + class Pool: + def __init__(self, _config): + pass + + def reset(self, receipt_path): + resetting.set() + assert finish_reset.wait(5) + receipt = json.loads(receipt_path.read_text()) + receipt["app_id"] = 201 + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + with ThreadPoolExecutor(max_workers=2) as executor: + releasing = executor.submit( + release_slice, tmp_path / "pool.json", output, lease["lease_id"] + ) + try: + assert resetting.wait(2) + acquiring = executor.submit(acquire_slice, output, "task-b") + assert acquiring.result(timeout=2)["app_id"] == 102 + finally: + finish_reset.set() + assert releasing.result(timeout=2)["ready_app_id"] == 201 + + +class FakeAPI: + def __init__(self, *, fail_domain_for: str | None = None) -> None: + self.calls = [] + self.next_app_id = 200 + self.fail_domain_for = fail_domain_for + self.port_hosts = {} + self.stopped_apps = set() + + def request(self, method, path, body=None): + self.calls.append((method, path, body)) + if method == "POST" and path.endswith("/operations") and body["action"] == "stop": + self.stopped_apps.add(path.removesuffix("/operations")) + if method == "POST" and path.endswith("/apps"): + app_id = self.next_app_id + self.next_app_id += 1 + return {"ID": app_id} + if method == "POST" and path.endswith("/192/copy"): + return { + "services": [ + { + "service_id": f"service-{site}-{self.next_app_id}", + "service_alias": f"alias-{site}-{self.next_app_id}", + "service_cname": f"{site}-copy", + "image": PINNED[site], + } + for site in STATEFUL_SITES + ] + + [ + { + "service_id": f"service-initializer-{self.next_app_id}", + "service_alias": f"alias-initializer-{self.next_app_id}", + "service_cname": "slice-initializer-copy", + "image": INITIALIZER_IMAGE, + } + ] + } + if method == "POST" and path.endswith("/domains"): + service_id = body["http"]["service_id"] + if self.fail_domain_for and self.fail_domain_for in service_id: + raise RuntimeError("injected domain failure") + if method == "GET" and path.endswith("/ports"): + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + site = next(site for site in STATEFUL_SITES if site in service_id) + site_host = f"internal-{service_id}" + return { + "ports": [ + { + "container_port": 8023 if site == "gitlab" else 80, + "is_inner_service": True, + "k8s_service_name": site_host, + }, + { + "container_port": 8877, + "is_inner_service": True, + "k8s_service_name": self.port_hosts.get( + service_id, f"control-{service_id}" + ), + }, + ] + } + if method == "PUT" and "/ports/8877" in path: + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + self.port_hosts.pop(service_id, None) + return {} + if method == "DELETE" and "/ports/8877" in path: + return {} + if method == "POST" and path.endswith("/ports"): + service_id = path.split("/services/", 1)[1].split("/", 1)[0] + self.port_hosts[service_id] = f"internal-{service_id}" + return {} + if method == "GET" and path.endswith("/services"): + return [ + { + "service_id": f"service-initializer-{self.next_app_id}", + "status": ( + "closed" + if path.removesuffix("/services") in self.stopped_apps + else "running" + ), + } + ] + return {} + + +def _pool_config(tmp_path: Path, *, reset_strategy: str | None = None) -> Path: + lock = tmp_path / "images.json" + lock.write_text( + json.dumps( + { + "schema": "lex_webarena_environment_images/1", + "images": {site: {"pinned": image} for site, image in PINNED.items()}, + } + ), + encoding="utf-8", + ) + config = tmp_path / "pool.json" + value = { + "schema": "lex_webarena_rainbond_pool/1", + "api_base_url": "https://rainbond.example", + "team_id": "team-id", + "team_name": "efficiency", + "region_name": "rainbond", + "source_app_id": 192, + "source_services": {site: f"source-{site}" for site in STATEFUL_SITES}, + "initializer_service": "source-initializer", + "initializer_image": INITIALIZER_IMAGE, + "app_prefix": "webarena-slice", + "domain_suffix": "dev.example.test", + "token_env": "TEST_RAINBOND_TOKEN", + "image_lock": "images.json", + "readiness_timeout_seconds": 1, + } + if reset_strategy is not None: + value["reset_strategy"] = reset_strategy + config.write_text( + json.dumps(value), + encoding="utf-8", + ) + return config + + +def _pool(tmp_path: Path, api: FakeAPI) -> RainbondSlicePool: + pool = RainbondSlicePool(_pool_config(tmp_path), api=api, sleep=lambda _: None) + pool._wait_site = lambda _url: None + return pool + + +def test_app_deletion_waits_for_service_stop(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + pool._delete_app(200) + methods = [method for method, _, _ in api.calls] + assert methods == ["GET", "POST", "GET", "DELETE"] + assert api.calls[1][2]["action"] == "stop" + + +def test_app_deletion_is_idempotent_after_control_plane_record_is_gone(tmp_path): + class MissingAppAPI(FakeAPI): + def request(self, method, path, body=None): + error = urllib.error.HTTPError(path, 404, "Not Found", None, None) + raise RuntimeError("Rainbond OpenAPI returned HTTP 404") from error + + pool = _pool(tmp_path, MissingAppAPI()) + pool._delete_app(200) + + +def test_app_deletion_keeps_record_when_stop_fails(tmp_path): + class StopFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "POST" and path.endswith("/operations"): + raise RuntimeError("stop failed") + return super().request(method, path, body) + + api = StopFailureAPI() + pool = _pool(tmp_path, api) + with pytest.raises(RuntimeError, match="stop failed"): + pool._delete_app(200) + assert not any(method == "DELETE" for method, _, _ in api.calls) + + +def test_app_deletion_does_not_trust_a_successful_stop_ack(tmp_path, monkeypatch): + class StillRunningAPI(FakeAPI): + def request(self, method, path, body=None): + result = super().request(method, path, body) + if method == "GET" and path.endswith("/services"): + result[0]["status"] = "running" + return result + + api = StillRunningAPI() + pool = _pool(tmp_path, api) + ticks = iter([0, 100_000]) + monkeypatch.setattr( + "harness.environments.webarena.rainbond.time.monotonic", + lambda: next(ticks), + ) + with pytest.raises(TimeoutError, match="did not stop before deletion"): + pool._delete_app(200) + assert not any(method == "DELETE" for method, _, _ in api.calls) + + +@pytest.mark.parametrize("stale_reads", [2, 4]) +def test_control_port_convergence_is_bounded_and_does_not_repeat_writes(tmp_path, stale_reads): + class EventuallyConsistentAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "GET" and path.endswith("/ports") and self.port_hosts: + if self.stale_reads: + self.stale_reads -= 1 + response["ports"][1]["k8s_service_name"] = "stale-control" + return response + + api = EventuallyConsistentAPI() + api.stale_reads = stale_reads + pool = _pool(tmp_path, api) + waits = [] + pool.sleep = waits.append + + def configure(): + pool._coalesce_internal_ports( + 200, + "service-shopping", + site_port=80, + control_port=8877, + control_alias="SHOPPING_CTRL", + ) + + if stale_reads == 2: + configure() + else: + with pytest.raises(RuntimeError, match="did not attach control port"): + configure() + assert waits == [1, 2] + assert sum(method == "POST" for method, _, _ in api.calls) == 1 + assert sum(method == "DELETE" for method, _, _ in api.calls) == 1 + assert sum(method == "GET" for method, _, _ in api.calls) == 4 + + +def test_provision_creates_one_complete_isolated_app_without_serializing_token(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-1" / "slice-receipt.json" + + receipt = pool.provision("run-a", 1, receipt_path) + + assert receipt["schema"] == RECEIPT_SCHEMA + assert set(receipt["components"]) == set(STATEFUL_SITES) + assert len({row["site_url"] for row in receipt["components"].values()}) == 4 + assert receipt["initializer"]["service_id"] == "service-initializer-201" + assert all("run-a-1" in row["site_url"] for row in receipt["components"].values()) + assert "TEST_RAINBOND_TOKEN" not in receipt_path.read_text() + assert not any("secret-token" in json.dumps(call) for call in api.calls) + deploy = next(call for call in api.calls if call[1].endswith("/operations")) + assert set(deploy[2]["service_ids"]) == { + row["service_id"] for row in receipt["components"].values() + } | {receipt["initializer"]["service_id"]} + domains = [call[2]["http"] for call in api.calls if call[1].endswith("/domains")] + assert len(domains) == 5 + admin = next(row for row in domains if "shopping-admin" in row["domain_name"]) + reddit = next(row for row in domains if "reddit" in row["domain_name"]) + assert admin["configuration"]["proxy_buffer_size"] == 64 + assert admin["configuration"]["proxy_buffer_numbers"] == 8 + assert reddit["configuration"]["proxy_buffer_size"] == 4 + initializer_domain = next( + row for row in domains if row["service_id"] == "service-initializer-201" + ) + assert initializer_domain["container_port"] == 8080 + assert receipt["initializer"]["readiness_url"].endswith(initializer_domain["domain_name"]) + initializer_envs = next( + call[2]["envs"] + for call in api.calls + if call[0] == "PUT" and "service-initializer" in call[1] + ) + assert {row["name"]: row["value"] for row in initializer_envs} == { + "REDDIT_CTRL_URL": "http://internal-service-reddit-201:8877", + "GITLAB_CTRL_URL": "http://internal-service-gitlab-201:8877", + "ENV_CTRL_MAX_ATTEMPTS": "360", + "ENV_CTRL_RETRY_SECONDS": "5", + } + port_mutations = [call for call in api.calls if "/ports" in call[1] and call[0] != "GET"] + assert [call[0] for call in port_mutations] == ["PUT", "DELETE", "POST"] * 4 + recreated = [call for call in port_mutations if call[0] == "POST"] + assert {call[2]["port_alias"] for call in recreated} == { + "SHOPPING_ADMIN_CTRL", + "SHOPPING_CTRL", + "REDDIT_CTRL", + "GITLAB_CTRL", + } + + +def test_provision_accepts_console_items_response_shapes(tmp_path): + class ItemsAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + return {"items": response["services"]} + if method == "GET" and path.endswith("/ports"): + return {"ports": {"items": response["ports"]}} + if method == "GET" and path.endswith("/services"): + return {"items": response} + return response + + receipt = _pool(tmp_path, ItemsAPI()).provision("run-items", 1, tmp_path / "slice-receipt.json") + + assert set(receipt["components"]) == set(STATEFUL_SITES) + + +def test_provision_rejects_copied_site_without_internal_control_port(tmp_path): + class MissingControlPortAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "GET" and path.endswith("/ports"): + return {"ports": []} + return super().request(method, path, body) + + api = MissingControlPortAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="must expose internal ports 80 and 8877"): + pool.provision("run-a", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + +def test_provision_rejects_copied_site_without_image_identity(tmp_path): + class MissingImageAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + response["services"][0].pop("image") + return response + + api = MissingImageAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="has no image in response"): + pool.provision("run-image", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + +def test_provision_rejects_initializer_without_image_identity(tmp_path): + class MissingInitializerImageAPI(FakeAPI): + def request(self, method, path, body=None): + response = super().request(method, path, body) + if method == "POST" and path.endswith("/192/copy"): + response["services"][-1].pop("image") + return response + + api = MissingInitializerImageAPI() + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="initializer component has no image"): + pool.provision("run-initializer-image", 1, tmp_path / "slice-receipt.json") + + assert any(method == "DELETE" for method, _path, _body in api.calls) + + +def test_provision_cleanup_does_not_hide_original_failure(tmp_path): + class CleanupFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "DELETE" and path.endswith("?force=1"): + raise RuntimeError("cleanup delete failed") + return super().request(method, path, body) + + api = CleanupFailureAPI(fail_domain_for="reddit") + pool = _pool(tmp_path, api) + + with pytest.raises(RuntimeError, match="injected domain failure") as caught: + pool.provision("run-cleanup", 1, tmp_path / "slice-receipt.json") + + assert any("cleanup delete failed" in note for note in caught.value.__notes__) + + +def test_reset_deletes_the_whole_app_and_recreates_a_new_generation(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-b", 2, receipt_path) + + second = pool.reset(receipt_path) + + assert second["app_id"] != first["app_id"] + assert second["generation"] != first["generation"] + assert {row["site_url"] for row in second["components"].values()}.isdisjoint( + {row["site_url"] for row in first["components"].values()} + ) + assert all(f"-{second['app_id']}-" in row["site_url"] for row in second["components"].values()) + assert any( + method == "DELETE" and f"/{first['app_id']}?force=1" in path + for method, path, _body in api.calls + ) + create_replacement = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "POST" and path.endswith("/apps") and index > 0 + ) + delete_old = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "DELETE" and f"/{first['app_id']}?force=1" in path + ) + assert create_replacement < delete_old + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() + + +def test_capacity_safe_reset_stops_old_app_before_creating_replacement(tmp_path): + api = FakeAPI() + pool = RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="stop_before_replace"), + api=api, + sleep=lambda _: None, + ) + pool._wait_site = lambda _url: None + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-capacity", 1, receipt_path) + + second = pool.reset(receipt_path) + + stop_index = next( + index + for index, (method, path, body) in enumerate(api.calls) + if method == "POST" + and path.endswith(f"/{first['app_id']}/operations") + and body["action"] == "stop" + ) + replacement_index = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "POST" and path.endswith("/apps") and index > stop_index + ) + delete_index = next( + index + for index, (method, path, _body) in enumerate(api.calls) + if method == "DELETE" and path.endswith(f"/{first['app_id']}?force=1") + ) + assert stop_index < replacement_index < delete_index + assert second["app_id"] != first["app_id"] + + +def test_stop_before_replace_keeps_old_app_stopped_and_receipt_on_failure(tmp_path): + api = FakeAPI() + pool = RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="stop_before_replace"), + api=api, + sleep=lambda _: None, + ) + pool._wait_site = lambda _url: None + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-recovery", 1, receipt_path) + api.fail_domain_for = "reddit" + + with pytest.raises(RuntimeError, match="injected domain failure"): + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + operations = [ + body["action"] + for method, path, body in api.calls + if method == "POST" and path.endswith(f"/{first['app_id']}/operations") + ] + assert operations[-1:] == ["stop"] + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() + + +def test_default_reset_stops_dirty_app_and_keeps_receipt_on_failure(tmp_path): + api = FakeAPI() + pool = _pool(tmp_path, api) + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-quarantine", 1, receipt_path) + api.fail_domain_for = "reddit" + + with pytest.raises(RuntimeError, match="injected domain failure"): + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + operations = [ + body["action"] + for method, path, body in api.calls + if method == "POST" and path.endswith(f"/{first['app_id']}/operations") + ] + assert operations[-1:] == ["stop"] + assert not receipt_path.with_name("slice-receipt.replacement.json").exists() + + +def test_reset_cleanup_does_not_hide_original_delete_failure(tmp_path): + class DeleteFailureAPI(FakeAPI): + def request(self, method, path, body=None): + if method == "DELETE" and path.endswith("/200?force=1"): + raise RuntimeError("old app delete failed") + if method == "DELETE" and path.endswith("/201?force=1"): + raise RuntimeError("replacement cleanup failed") + return super().request(method, path, body) + + api = DeleteFailureAPI() + pool = _pool(tmp_path, api) + pool._wait_app_stopped = lambda _receipt: None + receipt_path = tmp_path / "slice-receipt.json" + first = pool.provision("run-delete", 1, receipt_path) + + with pytest.raises(RuntimeError, match="old app delete failed") as caught: + pool.reset(receipt_path) + + assert json.loads(receipt_path.read_text())["generation"] == first["generation"] + assert receipt_path.with_name("slice-receipt.replacement.json").exists() + assert any("replacement cleanup failed" in note for note in caught.value.__notes__) + + +def test_pool_config_rejects_unknown_reset_strategy(tmp_path): + with pytest.raises(ValueError, match="reset_strategy"): + RainbondSlicePool( + _pool_config(tmp_path, reset_strategy="unsafe"), + api=FakeAPI(), + ) + + +def test_partial_provision_failure_removes_the_new_app_and_leaves_no_receipt(tmp_path): + api = FakeAPI(fail_domain_for="reddit") + pool = _pool(tmp_path, api) + receipt_path = tmp_path / "slice-receipt.json" + + with pytest.raises(RuntimeError, match="injected"): + pool.provision("run-c", 1, receipt_path) + + assert not receipt_path.exists() + assert any(method == "DELETE" and "/200?force=1" in path for method, path, _ in api.calls) + + +def test_provision_pool_rolls_back_already_created_slots(tmp_path, monkeypatch): + destroyed = [] + + class Pool: + def __init__(self, _config): + pass + + def provision(self, _pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("second slot failed") + receipt_path.parent.mkdir(parents=True) + receipt_path.write_text("{}") + return {"slot": slot} + + def runtime_config(self, _receipt): + return {"ok": True} + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + + with pytest.raises(RuntimeError, match="second slot"): + provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) + + assert destroyed == [tmp_path / "out" / "slice-1" / "slice-receipt.json"] + + +def test_provision_pool_preserves_failure_when_rollback_cleanup_fails(tmp_path, monkeypatch): + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, _pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("provision failed") + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text("{}") + return {"slot": slot} + + def runtime_config(self, _receipt): + return {"ok": True} + + def destroy(self, _receipt_path): + raise RuntimeError("rollback failed") + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + + with pytest.raises(RuntimeError, match="provision failed") as caught: + provision_pool(tmp_path / "pool.json", tmp_path / "out", "run-d", 2) + + assert any("rollback failed" in note for note in caught.value.__notes__) + + +def test_scale_pool_keeps_one_warm_slice_and_removes_only_excess(tmp_path, monkeypatch): + created = [] + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, pool_id, slot, receipt_path): + created.append(slot) + receipt = { + "schema": RECEIPT_SCHEMA, + "pool_id": pool_id, + "slot": slot, + "app_id": 100 + slot, + "components": {site: {} for site in STATEFUL_SITES}, + } + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, _receipt): + return {"ok": True} + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + receipt_path.unlink() + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + output = tmp_path / "pool" + + assert len(scale_pool(tmp_path / "config.json", output, "warm", 3)) == 3 + assert len(scale_pool(tmp_path / "config.json", output, "warm", 1)) == 1 + + assert sorted(created) == [1, 2, 3] + assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2"] + with pytest.raises(ValueError, match="minimum_ready"): + scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=1) + assert scale_pool(tmp_path / "config.json", output, "warm", 0, minimum_ready=0) == [] + assert [path.parent.name for path in destroyed] == ["slice-3", "slice-2", "slice-1"] + + +def test_scale_pool_rolls_back_every_new_slot_after_partial_failure(tmp_path, monkeypatch): + destroyed = [] + + class Config: + max_parallel_provisions = 3 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def provision(self, pool_id, slot, receipt_path): + if slot == 2: + raise RuntimeError("slot 2 failed") + receipt = {"pool_id": pool_id, "slot": slot} + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def runtime_config(self, _receipt): + return {"ok": True} + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path.parent.name) + receipt_path.unlink() + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + output = tmp_path / "runtime" + + with pytest.raises(RuntimeError, match="slot 2 failed"): + scale_pool(tmp_path / "pool.json", output, "warm", 3) + + assert sorted(destroyed) == ["slice-1", "slice-3"] + assert not list(output.glob("slice-*/slice-receipt.json")) + + +def test_capacity_keeps_idle_slice_in_addition_to_active_demand(tmp_path, monkeypatch): + calls = [] + + def fake_scale(config, output, pool_id, count, *, minimum_ready): + calls.append((config, output, pool_id, count, minimum_ready)) + return [] + + monkeypatch.setattr("harness.environments.webarena.rainbond.scale_pool", fake_scale) + + reconcile_pool_capacity( + tmp_path / "pool.json", tmp_path / "runtime", "warm", 3, minimum_ready=1 + ) + + assert calls[0][3:] == (4, 1) + + +@pytest.mark.parametrize("count", [0, 1]) +def test_scale_down_is_atomic_when_an_excess_slice_is_leased(tmp_path, monkeypatch, count): + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + destroyed.append(receipt_path) + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + output = tmp_path / "runtime" + for slot in (1, 2, 3): + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text(json.dumps({"pool_id": "warm", "slot": slot})) + (output / "slice-2" / "slice-lease.json").write_text("{}") + + with pytest.raises(RuntimeError, match="leased Rainbond slices"): + scale_pool(tmp_path / "pool.json", output, "warm", count, minimum_ready=0) + + assert destroyed == [] + + +def test_scale_down_continues_after_one_destroy_failure(tmp_path, monkeypatch): + destroyed = [] + + class Config: + max_parallel_provisions = 2 + + class Pool: + config = Config() + + def __init__(self, _config): + pass + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def destroy(self, receipt_path): + slot = int(receipt_path.parent.name.removeprefix("slice-")) + destroyed.append(slot) + if slot == 3: + raise RuntimeError("slot 3 delete failed") + receipt_path.unlink() + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + output = tmp_path / "runtime" + for slot in (1, 2, 3): + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text(json.dumps({"pool_id": "warm", "slot": slot})) + + with pytest.raises(RuntimeError, match="slot 3 delete failed"): + scale_pool(tmp_path / "pool.json", output, "warm", 1) + + assert destroyed == [3, 2] + assert (output / "slice-3" / "slice-receipt.json").exists() + assert not (output / "slice-2" / "slice-receipt.json").exists() + + +def _ready_slot(output: Path, slot: int, app_id: int) -> Path: + slot_dir = output / f"slice-{slot}" + slot_dir.mkdir(parents=True) + (slot_dir / "slice-receipt.json").write_text( + json.dumps( + { + "schema": RECEIPT_SCHEMA, + "slot": slot, + "app_id": app_id, + "generation": f"generation-{slot}", + } + ) + ) + (slot_dir / "webarena.config.json").write_text("{}") + return slot_dir + + +def test_acquire_grants_distinct_slices_and_exhausts_pool(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + + first = acquire_slice(output, "task-a") + second = acquire_slice(output, "task-b") + + assert first["schema"] == LEASE_SCHEMA + assert first["slot"] != second["slot"] + with pytest.raises(RuntimeError, match="no unleased"): + acquire_slice(output, "task-c") + + +def test_acquire_uses_numeric_slot_order(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 10, 110) + _ready_slot(output, 2, 102) + + lease = acquire_slice(output, "task-a") + + assert lease["slot"] == 2 + + +def test_concurrent_thread_acquires_never_share_a_slice(tmp_path): + output = tmp_path / "runtime" + _ready_slot(output, 1, 101) + _ready_slot(output, 2, 102) + + with ThreadPoolExecutor(max_workers=2) as executor: + leases = list( + executor.map( + lambda owner: acquire_slice(output, owner), + ("task-a", "task-b"), + ) + ) + + assert {lease["slot"] for lease in leases} == {1, 2} + assert len({lease["lease_id"] for lease in leases}) == 2 + + +def test_release_replaces_dirty_generation_before_unlocking_slice(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + + class Pool: + def __init__(self, _config): + pass + + def reset(self, _receipt_path): + assert (slot_dir / "slice-lease.json").exists() + return { + "schema": RECEIPT_SCHEMA, + "slot": 1, + "app_id": 201, + "generation": "generation-new", + } + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + + assert result["released_app_id"] == 101 + assert result["ready_app_id"] == 201 + assert not (slot_dir / "slice-lease.json").exists() + assert json.loads((slot_dir / "webarena.config.json").read_text()) == {"app_id": 201} + + +def test_failed_release_keeps_lease_so_dirty_slice_cannot_be_reassigned(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + + class Pool: + def __init__(self, _config): + pass + + def reset(self, _receipt_path): + raise RuntimeError("replacement failed") + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + + with pytest.raises(RuntimeError, match="replacement failed"): + release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + + assert (slot_dir / "slice-lease.json").exists() + with pytest.raises(RuntimeError, match="no unleased"): + acquire_slice(output, "task-b") + + +@pytest.mark.parametrize("failure_point", ["unlink", "config"]) +def test_release_retry_only_unlinks_after_reset_was_persisted(tmp_path, monkeypatch, failure_point): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + lease_path = slot_dir / "slice-lease.json" + reset_calls = [] + + class Pool: + def __init__(self, _config): + pass + + def reset(self, receipt_path): + reset_calls.append(receipt_path) + receipt = { + "schema": RECEIPT_SCHEMA, + "slot": 1, + "app_id": 201, + "generation": "generation-new", + } + receipt_path.write_text(json.dumps(receipt)) + return receipt + + def validate_receipt(self, receipt_path): + return json.loads(receipt_path.read_text()) + + def runtime_config(self, receipt): + return {"app_id": receipt["app_id"]} + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + original_unlink = Path.unlink + original_replace = Path.replace + failures = 0 + + def fail_first_lease_unlink(path, *args, **kwargs): + nonlocal failures + if failure_point == "unlink" and path == lease_path and failures == 0: + failures += 1 + raise OSError("transient unlink failure") + return original_unlink(path, *args, **kwargs) + + def fail_first_config_replace(path, target): + nonlocal failures + if ( + failure_point == "config" + and Path(target) == slot_dir / "webarena.config.json" + and failures == 0 + ): + failures += 1 + raise OSError("transient config failure") + return original_replace(path, target) + + monkeypatch.setattr(Path, "unlink", fail_first_lease_unlink) + monkeypatch.setattr(Path, "replace", fail_first_config_replace) + with pytest.raises(OSError, match="transient .* failure"): + release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert lease_path.exists() + assert len(reset_calls) == 1 + + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert result["released_app_id"] == 101 + assert result["ready_app_id"] == 201 + assert not lease_path.exists() + assert len(reset_calls) == 1 + + +def test_release_does_not_mistake_task_preparation_for_cleanup(tmp_path, monkeypatch): + output = tmp_path / "runtime" + slot_dir = _ready_slot(output, 1, 101) + lease = acquire_slice(output, "task-a") + receipt_path = slot_dir / "slice-receipt.json" + receipt = json.loads(receipt_path.read_text()) + receipt["app_id"] = 201 + receipt_path.write_text(json.dumps(receipt)) + (slot_dir / "webarena.config.json").write_text(json.dumps({"app_id": 201})) + reset_calls = [] + + class Pool: + def __init__(self, _config): + pass + + def validate_receipt(self, path): + return json.loads(path.read_text()) + + def runtime_config(self, current): + return {"app_id": current["app_id"]} + + def reset(self, path): + reset_calls.append(path) + assert ( + json.loads((slot_dir / "slice-lease.json").read_text())["release_from_app_id"] + == 201 + ) + clean = {**receipt, "app_id": 301} + path.write_text(json.dumps(clean)) + return clean + + monkeypatch.setattr("harness.environments.webarena.rainbond.RainbondSlicePool", Pool) + result = release_slice(tmp_path / "pool.json", output, lease["lease_id"]) + assert result["ready_app_id"] == 301 + assert reset_calls == [receipt_path] + assert not (slot_dir / "slice-lease.json").exists() diff --git a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py index 56e266af8..04c2d2947 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_env_contract.py @@ -1,136 +1,84 @@ from __future__ import annotations import json -from pathlib import Path +import shutil import pytest -from scripts.verify_webarena_env_contract import verify - - -def _write_environment_repo(root: Path, evaluation_lock: Path) -> Path: - payload = json.loads(evaluation_lock.read_text(encoding="utf-8")) - compact = { - "schema": "lexmount_webarena_images/1", - "webarena_verified_version": payload["webarena_verified_version"], - "images": { - site: {key: row[key] for key in ("pinned", "architecture", "os")} - for site, row in payload["images"].items() - }, - } - path = root / "config/images.lock.json" - path.parent.mkdir(parents=True) - path.write_text(json.dumps(compact), encoding="utf-8") - runtime = json.loads( - Path("examples/trajectory_replay/public_webarena/webarena.config.example.json").read_text(encoding="utf-8") - ) - from urllib.parse import urlparse - - template = { - "sites": { - site: { - "port": urlparse(runtime["environments"][f"__{site.upper()}__"]["urls"][0]).port, - "control_port": urlparse(runtime["control_urls"][site]).port, - } - for site in compact["images"] - }, - "credentials_env": { - site: runtime["environments"][f"__{site.upper()}__"]["credentials_env"] - for site in ("shopping_admin", "shopping", "reddit", "gitlab") - }, - } - (root / "config/runtime.example.json").write_text(json.dumps(template), encoding="utf-8") - archive_names = ("osm_tile_server.tar", "nominatim_volumes.tar", "osrm_routing.tar") - delivery = { - "webarena_verified_version": compact["webarena_verified_version"], - "image_lock": "images.lock.json", - "components": { - site: {"image": row["pinned"]} - for site, row in compact["images"].items() - if site != "wikipedia" - } - } - delivery["components"]["map"].update( - { - "data_urls": [f"https://example.invalid/{name}" for name in archive_names], - "data_archive_sha256": {name: "0" * 64 for name in archive_names}, - "data_archive_bytes": {name: 1 for name in archive_names}, - } - ) - (root / "config/site-delivery-contract.json").write_text( - json.dumps(delivery), encoding="utf-8" - ) - return path - - -def test_cross_repo_contract_accepts_matching_images(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - assert verify(tmp_path)["status"] == "pass" - - -def test_cross_repo_contract_rejects_one_changed_digest(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - path = _write_environment_repo(tmp_path, evaluation_lock) - payload = json.loads(path.read_text(encoding="utf-8")) - payload["images"]["map"]["pinned"] = "example.invalid/map@sha256:" + "0" * 64 - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="map.pinned"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_port_and_credential_drift(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/runtime.example.json" - payload = json.loads(path.read_text(encoding="utf-8")) - payload["sites"]["map"]["port"] += 1 - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="site port differs: map"): - verify(tmp_path) - payload["sites"]["map"]["port"] -= 1 - payload["credentials_env"]["shopping_admin"]["username"] = "WRONG_USERNAME" - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="credential variable names differ: shopping_admin"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_missing_map_digest(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - del payload["components"]["map"]["data_archive_sha256"]["osrm_routing.tar"] - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="Map archive checksum contract"): - verify(tmp_path) - - -def test_cross_repo_contract_rejects_site_delivery_lock_path_drift(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - payload["image_lock"] = "../images.lock.json" - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="image lock path differs"): - verify(tmp_path) - - -def test_cross_repo_contract_reports_missing_site_delivery_contract(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - (tmp_path / "config/site-delivery-contract.json").unlink() - with pytest.raises(ValueError, match="missing site delivery contract"): - verify(tmp_path) - - -def test_cross_repo_contract_reports_missing_image_lock_field(tmp_path): - evaluation_lock = Path("evaluation/browser-use-public/webarena-environment-images.json") - _write_environment_repo(tmp_path, evaluation_lock) - path = tmp_path / "config/site-delivery-contract.json" - payload = json.loads(path.read_text(encoding="utf-8")) - del payload["image_lock"] - path.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="image_lock is missing"): - verify(tmp_path) +from scripts.verify_webarena_env_contract import ROOT, verify + + +def _checkout(tmp_path): + for name in ( + "evaluation/browser-use-public/webarena-environment-images.json", + "examples/trajectory_replay/public_webarena/webarena.config.example.json", + "examples/trajectory_replay/public_webarena/rainbond-pool.example.json", + "deploy/webarena/runtime.example.json", + "deploy/webarena/site-delivery-contract.json", + ): + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, target) + return tmp_path + + +def test_repository_contract_uses_one_image_lock(): + assert verify()["status"] == "pass" + + +@pytest.mark.parametrize("template", ["webarena.config", "rainbond-pool"]) +def test_evaluation_templates_reject_wrong_relative_lock_path(tmp_path, template): + root = _checkout(tmp_path) + path = root / f"examples/trajectory_replay/public_webarena/{template}.example.json" + data = json.loads(path.read_text()) + data["image_lock"] = "../../evaluation/browser-use-public/webarena-environment-images.json" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="canonical image lock"): + verify(root) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("image_lock", "other.json", "canonical image lock"), + ("webarena_verified_version", "0.0.0", "package version"), + ], +) +def test_delivery_rejects_wrong_binding(tmp_path, field, value, message): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + data[field] = value + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match=message): + verify(root) + + +def test_deployment_rejects_port_drift(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/runtime.example.json" + data = json.loads(path.read_text()) + data["sites"]["reddit"]["port"] += 1 + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="site port differs: reddit"): + verify(root) + + +def test_deployment_rejects_duplicate_image_identity(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + data["components"]["reddit"]["image"] = "reddit:latest" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="canonical image identity"): + verify(root) + + +def test_map_contract_rejects_missing_checksum(tmp_path): + root = _checkout(tmp_path) + path = root / "deploy/webarena/site-delivery-contract.json" + data = json.loads(path.read_text()) + del data["components"]["map"]["data_archive_sha256"]["osrm_routing.tar"] + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="Map archive checksum"): + verify(root) diff --git a/tests/trajectory_replay/public_webarena/test_webarena_environment.py b/tests/trajectory_replay/public_webarena/test_webarena_environment.py index e5cc510df..873c18e6d 100644 --- a/tests/trajectory_replay/public_webarena/test_webarena_environment.py +++ b/tests/trajectory_replay/public_webarena/test_webarena_environment.py @@ -6,10 +6,10 @@ import pytest +from harness.environments.webarena.backend import DockerBackend from harness.trajectory_replay.public_webarena.suite import BrowserTask from harness.trajectory_replay.public_webarena.webarena_environment import ( WebArenaEnvironment, - _DockerClientBackend, wall_clock_timeout, ) @@ -130,6 +130,126 @@ def test_runtime_verification_rejects_mutable_or_wrong_image(tmp_path): environment.verify_runtime(_Docker([]), (_task(),)) +def test_rainbond_slice_verification_and_reset_replace_the_complete_application( + tmp_path, monkeypatch +): + pinned = "am1n3e/webarena-verified-shopping_admin@sha256:" + "d" * 64 + lock = tmp_path / "images.json" + lock.write_text( + json.dumps( + { + "schema": "lex_webarena_environment_images/1", + "images": {"shopping_admin": {"pinned": pinned}}, + } + ), + encoding="utf-8", + ) + receipt_path = tmp_path / "slice-receipt.json" + first = { + "app_id": 301, + "generation": "first", + "components": { + "shopping_admin": { + "service_id": "old-service", + "image": pinned, + "site_url": "https://admin-slice.example.test", + } + }, + } + receipt_path.write_text(json.dumps(first), encoding="utf-8") + resets = [] + + class Pool: + def __init__(self, path): + assert path == tmp_path / "pool.json" + + def validate_receipt(self, path): + assert path == receipt_path + return json.loads(path.read_text()) + + def runtime_config(self, receipt): + return { + "image_lock": str(lock), + "environments": { + "__SHOPPING_ADMIN__": { + "urls": [receipt["components"]["shopping_admin"]["site_url"]] + } + }, + "rainbond_slice": { + "pool_config": str(tmp_path / "pool.json"), + "receipt": "slice-receipt.json", + }, + } + + def persist_runtime_config(self, receipt, path, *, base_config): + value = {**base_config, **self.runtime_config(receipt)} + path.write_text(json.dumps(value), encoding="utf-8") + return value + + def reset(self, path): + resets.append(path) + return { + "app_id": 302, + "generation": "second", + "components": { + "shopping_admin": { + "service_id": "new-service", + "image": pinned, + "site_url": "https://admin-slice-302.example.test", + } + }, + } + + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.webarena_environment.RainbondSlicePool", + Pool, + ) + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "image_lock": "images.json", + "environments": { + "__SHOPPING_ADMIN__": {"urls": ["https://admin-slice.example.test"]} + }, + "rainbond_slice": { + "pool_config": "pool.json", + "receipt": "slice-receipt.json", + }, + } + ), + encoding="utf-8", + ) + environment = WebArenaEnvironment(config) + monkeypatch.setattr(environment, "_wait_site_ready", lambda *_args, **_kwargs: None) + + runtime = environment.verify_runtime(object(), (_task(),)) + reset = environment.reset(_task(), object()) + + assert runtime["sites"]["shopping_admin"]["mode"] == ("remote_rainbond_isolated_slice") + assert runtime["sites"]["shopping_admin"]["app_id"] == 301 + assert reset == { + "mode": "remote_rainbond_isolated_slice", + "reset": "application_deleted_and_recreated_from_pinned_baseline", + "app_id": 302, + "generation": "second", + "sites": { + "shopping_admin": { + "service_id": "new-service", + "site_url": "https://admin-slice-302.example.test", + } + }, + } + assert environment.environments["__SHOPPING_ADMIN__"]["urls"] == [ + "https://admin-slice-302.example.test" + ] + persisted = json.loads(config.read_text()) + assert persisted["environments"]["__SHOPPING_ADMIN__"]["urls"] == [ + "https://admin-slice-302.example.test" + ] + assert resets == [receipt_path] + + def test_site_docker_client_defaults_or_uses_validated_unix_socket(tmp_path, monkeypatch): environment, _ = _environment(tmp_path) default_client = object() @@ -406,7 +526,7 @@ class Containers: def run(self, *args, **kwargs): calls.append((args, kwargs)) - backend = _DockerClientBackend(SimpleNamespace(containers=Containers())) + backend = DockerBackend(SimpleNamespace(containers=Containers())) backend.container_run( name=f"webarena_verified_{site}", @@ -432,7 +552,7 @@ class Containers: def run(self, *args, **kwargs): calls.append((args, kwargs)) - backend = _DockerClientBackend(SimpleNamespace(containers=Containers())) + backend = DockerBackend(SimpleNamespace(containers=Containers())) backend.container_run( name="webarena_verified_gitlab", @@ -450,7 +570,7 @@ def run(self, *args, **kwargs): def test_stateful_backend_can_defer_large_container_removal(): container = _Container([]) - backend = _DockerClientBackend( + backend = DockerBackend( SimpleNamespace(containers=_Containers(container)), defer_removal=True ) @@ -462,7 +582,7 @@ def test_stateful_backend_can_defer_large_container_removal(): def test_deferred_removal_rejects_shared_writable_mounts(): - backend = _DockerClientBackend( + backend = DockerBackend( SimpleNamespace(containers=SimpleNamespace(run=lambda *_args, **_kwargs: None)), defer_removal=True, )