diff --git a/.agents/skills/benchmark-editorial/SKILL.md b/.agents/skills/benchmark-editorial/SKILL.md index aa7cbb074..e9e6b2e55 100644 --- a/.agents/skills/benchmark-editorial/SKILL.md +++ b/.agents/skills/benchmark-editorial/SKILL.md @@ -7,7 +7,7 @@ description: Write or revise standalone articles interpreting this project's ben Produce a reader-facing article whose conclusions, examples and background remain understandable without opening the source report. The benchmark supplies evidence; the article explains what the results mean and why they matter. -Keep this skill and its dependencies in the repository. Read [editorial-standard.md](references/editorial-standard.md) before writing or revising, and [html-delivery.md](references/html-delivery.md) for HTML. Use the complete [Humanizer Chinese](../humanizer-chinese/SKILL.md) for Chinese prose and [Humanizer](../humanizer/SKILL.md) for English. Apply them in embedded/file mode: only finished prose enters the article. +Keep this skill and its dependencies in the repository. Read [editorial-standard.md](references/editorial-standard.md) before writing or revising, and [html-delivery.md](references/html-delivery.md) for HTML. Run the complete [Humanizer](../humanizer/SKILL.md) and then [Humanizer Chinese](../humanizer-chinese/SKILL.md) over Chinese prose; run Humanizer over English prose. Apply them in embedded/file mode: only finished prose enters the article. ## Choose the work being requested @@ -27,9 +27,11 @@ Choose an argument that the evidence supports, including exceptions that change Develop the reader's understanding through concrete results, necessary background and inspectable examples. Explain unfamiliar concepts through their purpose and operation; a sentence excluding them from the benchmark is not an explanation. Use charts and specimens to carry data so prose can explain the result instead of repeating it. There is no fixed chapter count, case count, headline formula or required hero image. +For report interpretation and summary pages, use a presentation-ready reading sequence: one finding per section, a data-bearing visual or evidence specimen as the main element, and short prose that explains it. Remove images that merely fill space. Keep full inventories and raw evidence on their functional report pages. When several methods fail for materially different reasons, give each method its own chapter with the complete reviewed distribution, mechanism, representative evidence, and cross-method distinction. + When revising, check the whole article for the same underlying problem, then make the smallest coherent repair. Review any removed material for lost explanations, evidence, limitations or transitions. Broader restructuring is justified when the argument or organization is defective, not simply because a paragraph received criticism. If a needed fact is absent, investigate it or narrow the claim; do not replace it with confidence or invented precision. -Run the language-appropriate Humanizer pass on headings, prose and captions. This is a prose edit, not a substitute for evidence or editorial judgment. Preserve technical meaning, natural language conventions and original measured text. Do not manufacture casual language or personal experience. +Run both Humanizer and Humanizer Chinese on Chinese headings, prose and captions; run Humanizer on English. This is a prose edit, not a substitute for evidence or editorial judgment. Preserve technical meaning, natural language conventions and original measured text. Do not manufacture casual language or personal experience. ## Review and delivery diff --git a/.agents/skills/benchmark-editorial/references/html-delivery.md b/.agents/skills/benchmark-editorial/references/html-delivery.md index fcb2a1bc0..7b59a57a5 100644 --- a/.agents/skills/benchmark-editorial/references/html-delivery.md +++ b/.agents/skills/benchmark-editorial/references/html-delivery.md @@ -2,6 +2,8 @@ The dashboard reading page and standalone HTML are two presentations of the same authored article. Use the current report renderer and preserve its integrity, safety and localization contracts. +Treat report home and interpretation pages as reusable presentation material. Each section should communicate one conclusion through a chart, meaningful diagram, image, or inspectable evidence specimen, with concise supporting text and exact values available to copy. Do not use decorative background art or repeat the same dense table across summary, interpretation, and analysis. Result analysis owns filterable inventories and detailed distributions; result pages own logs and complete evidence. + Generate Chinese by default. English prose, illustrations, rendering and language-specific checks run only on an explicit manual request. An absent English edition gets a not-generated notice and a return link, not stale translated pages. Preserve translation sources without distributing unused English assets. Apply cross-language parity checks only when both editions are generated; always test the notice in a Chinese-only release. ## Reading layout diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md index a9e57288a..e148fa62e 100644 --- a/.agents/skills/benchmark-report/SKILL.md +++ b/.agents/skills/benchmark-report/SKILL.md @@ -27,6 +27,7 @@ Repository reports default to Chinese-only output and checks. English generation 12. Present one authoritative logical result per method and task. The frozen protocol chooses the score-bearing execution; a renderer must never silently replace that rule. Keep every physical call in the evidence chain retain accumulated execution cost separately from the declared primary latency metric. State whether primary latency measures the final call or the complete user-visible operation; retries never increase the logical-result denominator. Do not make execution ordinal labels part of the public comparison unless retry behavior is the declared subject. 13. Apply the track-independent presentation contract in [report and release](references/report-and-release.md) to every report, including a new track. Wire semantic checks into its release command; prose instructions alone are not a release gate. 14. In this repository, publish a finished report to the Artifact Site host and site bound by that track’s current profile, following that host’s current `for-agents.md`. Never embed a default deployment hostname or borrow a target from another report. Build and validate before the external write, bind the returned slug, share URL, and version in the current profile, use optimistic locking for later updates, and verify the unauthenticated share page. Use repository publish targets when available; do not reconstruct a target from prose or an earlier receipt. Export the whole deployed tree only when the specific report publisher requires it or when diagnosing a release failure. +15. Make summary and interpretation routes presentation-ready. Give each section one finding and one primary visual such as a chart, mechanism diagram, evidence specimen, or meaningful image, with short supporting prose and exact values in accessible text. Do not add decorative hero art. Keep sortable inventories, logs, and full evidence on their functional pages. For Chinese report-authored prose, run both the repository Humanizer and Humanizer Chinese after the evidence pass; preserve measured language and quoted evidence. ## Stop conditions diff --git a/.agents/skills/benchmark-report/references/report-and-release.md b/.agents/skills/benchmark-report/references/report-and-release.md index 56377ce48..02af547d5 100644 --- a/.agents/skills/benchmark-report/references/report-and-release.md +++ b/.agents/skills/benchmark-report/references/report-and-release.md @@ -8,6 +8,12 @@ For the current trajectory report, `make trajectory-replay-report EVIDENCE=/abso ## Reader-facing structure +Make the home and interpretation routes presentation-ready. Each section should make one finding legible through a data-bearing chart, mechanism diagram, meaningful image, or inspectable specimen, followed by short explanatory prose. Keep exact values in accessible text. Decoration is not a visual explanation: remove images that carry no data, mechanism, topology, or evidence. Dense sortable inventories and raw logs remain on the analysis and evidence routes that own them. + +When multiple methods have material failures, give each one a separate interpretation chapter containing its full reviewed subtype distribution, a plain-language explanation of the dominant mechanisms, representative result evidence, and the difference from the other methods. The home page should show only a compact leading-error signature; result analysis owns complete stage and case distributions. + +After the factual pass, run both repository Humanizer and Humanizer Chinese over Chinese report-authored headings, prose, and captions. Run Humanizer over English. These passes may improve language only; they must preserve numbers, scope, technical meaning, quotations, and uncertainty. + Organize around reader questions, not storage tables. A useful report usually makes these answers easy to find: - What was compared, on which tasks and versions? diff --git a/.env.example b/.env.example index 4b5d17e03..284902ac5 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,10 @@ OBSCURA_BINARY=obscura # Pin the exact executable used by the cohort; do not leave this blank or rely # on Playwright's mutable bundled-browser fallback. CHROMIUM_BINARY=/Applications/Google Chrome.app/Contents/MacOS/Google Chrome -KITESURF_CDP_ENDPOINT=wss://kitesurf.cloudflare.app/devtools/browser +# Cloudflare Browser Run constructs the account-scoped Kitesurf CDP endpoint +# from these values. Do not persist an Authorization header or token in evidence. +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_API_TOKEN= # Browser Trajectory Replay diagnostics and autonomous Browser Use evaluation OPENAI_API_KEY= @@ -54,8 +57,6 @@ TAVILY_API_KEY= EXA_API_KEY= BROWSERBASE_API_KEY= BROWSERBASE_PROJECT_ID= -CLOUDFLARE_API_TOKEN= -CLOUDFLARE_ACCOUNT_ID= # Query-to-search-results and model-bound retrieval BOCHA_API_KEY= diff --git a/AGENTS.md b/AGENTS.md index 641eac0f1..dfc9b8f4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,15 @@ # Report maintenance +## Change lifecycle and repository footprint + +- Extend the current owner before adding a parallel implementation. A second browser, cohort, dataset size, provider, language or release must select behavior through data, a registry or parameters in the existing pipeline. It must not create a browser-specific runner, copied renderer, numbered implementation or compatibility alias. +- Keep command-line files as thin entry points. Parsing, validation, execution, aggregation and publication contracts belong in the track's shared `harness/` module so every entry point uses the same rules. +- Git owns reusable code, compact dataset definitions, acceptance contracts, provenance annotations and path/hash publication bindings. Measured attempts, generated reports, report prose and illustrations, review payloads, diagnostics, temporary queues and run-local manifests live under the configured data home or the pinned Artifact Site release. +- Keep one supported current implementation of each stage. Remove superseded generators, repair scripts, migrations and their exclusive tests in the same change once no supported entry point calls them. Retain a historical revision in Git only when a current versioned authority names it and reproduction requires its bytes. +- Treat a large diff as a design signal. Before adding files, identify the existing owner, the smallest reusable extension and the data that can stay outside Git. File or line counts alone are not release gates; unexplained duplicate stages, private evidence and unreferenced historical versions are. +- Finish structural changes with a repository inventory: search for references to removed entry points, list the supported entry points, verify that current profiles bind every external artifact, and run the behavior and release gates that exercise those public boundaries. +- Do not preserve a changed command under its old name when its inputs or meaning change. Restore the old contract or introduce the new behavior under the existing shared abstraction and migrate every caller atomically. + - Keep report-specific prose, illustrations and review payloads outside Git, alongside frozen evidence. Commit compact path/hash bindings instead. The replay build reads `research/trajectory-replay-content.json` from the `--content` directory (default: the data home's `report-assets/trajectory-replay/`); restore missing content from the pinned Artifact Site version, not an unversioned latest report. Do not silently fall back to a repository copy. Compact task definitions, acceptance contracts and provenance annotations remain versioned inputs. - Remove retired report-specific generators and their exclusive tests when no supported entry point calls them. Preserve published reports and historical evidence. Keep behavioral tests for supported build, package and publication boundaries; do not replace them with assertions about private helper structure. - WCXB builds read profile-bound files from `/report-assets/` and preserve the same relative paths in the public package; restore missing files from the profile's pinned Artifact Site version, never from an unversioned latest report. Do not silently fall back to a repository copy. @@ -9,6 +19,9 @@ These requirements apply to **all** report tracks; browser-specific rules below only add evidence types. - Follow the shared presentation contract in `.agents/skills/benchmark-report/references/report-and-release.md`. Metric labels, units, populations, unique directional highlights, one-summary result details, evidence dialogs and repairable error drilldown are common requirements, not Browser Use features. +- Make report home and interpretation pages presentation-ready: each section states one finding and lets a chart, diagram, image, or inspectable specimen carry the main information; prose explains the evidence briefly. Keep exact values available as text. Remove decorative imagery that does not encode data, mechanism, topology, or evidence. Preserve dense tables and logs on the result-analysis and evidence pages that own them instead of turning audit material into decoration. +- When several compared browsers or methods have material failures, give each one a dedicated interpretation chapter with its complete reviewed failure distribution, plain-language mechanism, representative evidence, and the difference from the other methods. Put only a compact leading-error summary on the home page; keep full stage and case inventories in result analysis. +- Run both repository Humanizer and Humanizer Chinese over finished Chinese report prose, headings, and captions after the factual pass. Run Humanizer over English prose. Treat these as language edits: they must preserve measurements, technical meaning, quoted evidence, and uncertainty. - Highlight the unique column maximum/minimum from raw values; tied or unavailable values stay neutral. A scoped numerical extreme is not a full-benchmark ranking. Do not remove highlights merely because successful-page populations differ; show the counts and scope. - Mean/P50/P95 share one explicitly stated population and parallel labels. Retain retry execution cost separately when the current contract selects final-call latency; one logical result remains one sample. - For a before/after build comparison, runs on different dates cannot supply the authoritative speed difference. Predeclare the complete task scope, alternate call order within each task, run serially on a quiet host, bind both binary hashes and all physical calls, and select a round before seeing results. If an earlier timing is corrected, retain it privately, supersede every affected logical timing, and make the report table and detail agree. Release checks must reject an absent or partial timing receipt. diff --git a/Makefile b/Makefile index 742a7600c..27009778b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PY := $(UV) run --frozen python TEST_PY := $(UV) run --frozen --all-extras python WEBMAINBENCH_SAMPLE ?= evaluation/public-webmainbench-200.json -.PHONY: setup import-env runtime-check naming-check test test-python test-ui fixture-smoke ci report report-check popular-sites-report popular-sites-report-check wcxb-release-check resource-image public-benchmark-report public-benchmark-sample webmainbench-run webmainbench-score webmainbench-review webmainbench-assemble browser-use-review browser-use-report browser-use-report-check browser-use-visual-check browser-use-release-check trajectory-replay-report trajectory-replay-report-check trajectory-replay-visual-check trajectory-replay-release-check trajectory-replay-package trajectory-replay-publish public-webarena-report public-webarena-report-audit public-webarena-report-release +.PHONY: setup import-env runtime-check naming-check test test-python test-ui fixture-smoke ci report report-check popular-sites-report popular-sites-report-check wcxb-release-check resource-image public-benchmark-report public-benchmark-sample webmainbench-run webmainbench-score webmainbench-review webmainbench-assemble browser-use-review browser-use-report browser-use-report-check browser-use-visual-check browser-use-release-check trajectory-replay-report trajectory-replay-report-check trajectory-replay-visual-check trajectory-replay-release-check trajectory-replay-package trajectory-replay-publish webarena-catalog-report webarena-catalog-release-check public-webarena-report public-webarena-report-audit public-webarena-report-release setup: $(UV) sync --frozen --all-extras @@ -100,6 +100,13 @@ trajectory-replay-package: runtime-check trajectory-replay-publish: runtime-check $(PY) scripts/publish_webarena_replay_artifact_site.py +webarena-catalog-report: runtime-check + @test -n "$(EVIDENCE_ROOT)" -a -n "$(LEDGER)" -a -n "$(SCOPE)" -a -n "$(CATALOG)" -a -n "$(SUITE)" -a -n "$(SITE_MAP)" -a -n "$(OUTPUT)" -a -n "$(ARCHIVE)" || (echo 'EVIDENCE_ROOT, LEDGER, SCOPE, CATALOG, SUITE, SITE_MAP, OUTPUT and ARCHIVE are required' >&2; exit 2) + $(PY) scripts/build_webarena_catalog_report.py --evidence-root "$(EVIDENCE_ROOT)" --ledger "$(LEDGER)" --scope "$(SCOPE)" --catalog "$(CATALOG)" --portable-suite "$(SUITE)" --site-map "$(SITE_MAP)" --output "$(OUTPUT)" --archive "$(ARCHIVE)" + +webarena-catalog-release-check: runtime-check + $(PY) scripts/check_webarena_catalog_release.py $(if $(DATA_HOME),--data-home "$(DATA_HOME)",) + public-webarena-report: runtime-check $(PY) scripts/release_webarena_trajectory_report.py $(if $(FORMAL_ROOT),--formal-root "$(FORMAL_ROOT)",) diff --git a/deploy/webarena/README.md b/deploy/webarena/README.md index ecae9d717..3122b885b 100644 --- a/deploy/webarena/README.md +++ b/deploy/webarena/README.md @@ -59,7 +59,8 @@ Nominatim 的 `flatnode.file` 导入／增量更新缓存。评测使用固定 镜像自带的 Rails OSM 对象库没有节点、道路或关系记录；搜索结果来自已恢复的 Nominatim 库，直接点击原本会得到 404。对象详情入口从同一份冻结数据按 OSM 类型和 ID 查询，展示名称、地址、坐标及可用的附加标签，保留原来的 `/node/`、 -`/way/`、`/relation/` 地址。它只展示 Nominatim 收录的对象；缺失对象仍返回 404， +`/way/`、`/relation/` 地址。Nominatim 不索引的评测所需路线 relation 只接受绑定到 +固定 Wikidata revision、OSM ID 和来源哈希的精确记录；其它缺失对象仍返回 404， 不会合成不存在的内容或声称提供完整 OSM 版本历史和成员关系。 三个原始归档约 174.5 GiB。初始化逐包下载、校验 SHA-256、稀疏解包并删除归档， diff --git a/docs/architecture.md b/docs/architecture.md index 08d1b8d8f..1d4646b75 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,3 +11,12 @@ The persistent evidence bundle's `execution.json` associates selected attempts w Formal pages cover rankings, datasets, methodology, results, provider descriptions and interface examples, environment, reproduction, and account setup. Source coverage and access restrictions are factual limitations, not a user-assistance workflow. The report application owns full rules, interfaces and auditable records. Its tables retain local metric definitions and link to the matching methodology example. The interpretation article owns the argument, mechanism experiments, counterexamples and decision implications; it repeats only enough context to stand alone. The homepage provides quantitative overview and article access, not a second prose summary. Provider integration details belong with their interface examples, not a separate page repeating task boundaries. + +## Browser trajectory replay ownership + +- `harness/trajectory_replay/` owns reusable action execution, deployment rebinding, official evaluation, evidence validation and report contracts. Files in `scripts/` only expose those stages as commands. +- A replay dataset is browser-neutral. Participant profiles bind the browser or engine, binary, backend, deployment, accepted catalog, ledger and publication target. Adding Moli or another browser reuses the same runner and creates participant evidence outside Git. +- The repository keeps the current compact dataset and correction provenance. Physical attempts, intermediate catalogs, repair queues and report bundles remain in `BROWSER_EVAL_DATA_HOME`; `current-report.json` binds the accepted external artifacts by relative path and hash. +- Canonical task definitions, evaluator semantics and corrections that apply to every runner belong in `lexmount/webarena-verified` on protected branch `lex-main`. `browser-eval` pins an immutable commit from that branch; release checks reject the ServiceNow repository, PyPI implementation or an unapproved side branch. +- `browser-eval` owns browser participants, deployment binding, trajectory qualification and replay, BrowserTool actions, reset/readiness receipts, evidence collection, retry ledgers, replay-only postconditions, reports and publication. Environment-specific references stay here with provenance instead of changing the canonical benchmark. +- A temporary evaluator compatibility patch may remain in `browser-eval` only while a proven generic fix is pending in the fork. It needs paired behavioral controls and must be removed after the pinned `lex-main` commit contains the fix. diff --git a/evaluation/trajectory-replay/webarena-verified-full/README.md b/evaluation/trajectory-replay/webarena-verified-full/README.md new file mode 100644 index 000000000..442c9dc3a --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/README.md @@ -0,0 +1,19 @@ +# WebArena-Verified full replay ownership + +This directory owns the compact, versioned inputs and release bindings for the +812-task Browser Trajectory Replay split. + +- `dataset.json`, `scope.json`, `environment.json`, and the action override + files define the reusable benchmark inputs. +- `current-report.json` binds the Moli-qualified canonical catalog. It is the + acceptance authority used before comparing browsers. +- `current-comparison.json` binds the current multi-browser results, reviewed + failures, report package, and Artifact Site version by relative path and + SHA-256. The measured files remain under `BROWSER_EVAL_DATA_HOME` and are not + committed to Git. +- `../webarena-verified/current-run.json` remains the independent 90-task split. + +The evaluator and canonical WebArena task fixes live in the pinned +`lexmount/webarena-verified` fork. Replay execution, browser adapters, evidence +aggregation, review, report generation, and publication live in this +`browser-eval` repository. diff --git a/evaluation/trajectory-replay/webarena-verified-full/action-overrides-audit.json b/evaluation/trajectory-replay/webarena-verified-full/action-overrides-audit.json new file mode 100644 index 000000000..8df2958b7 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/action-overrides-audit.json @@ -0,0 +1,134 @@ +{ + "schema": "lex_webarena_replay_action_override_audit/1", + "catalog_sha256": "7a4d780f35eec75c9fac75666dc75f25b9fb33aeabe754bda55857ed286702e0", + "portable_suite_sha256": "7bbcf099ce13d9b0bce9652a32f24df8aef489dcaf3dac2702daed33decd062e", + "evaluate_action_count": 804, + "function_body_findings": [ + { + "task_id": 28, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "8bde53ce9d3e142a8666c0454be9a7cadd89990b1d967f09c13fa858aff9de1a" + }, + { + "task_id": 284, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "81d1f17b0f4324eae2288a91e72e1ec5944454aa669b4ccf6ffd5cd41d2d7227" + }, + { + "task_id": 464, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "cde2b999aee833627b12ef6308d2863576b3113f121840fb465cefbb28b12298" + }, + { + "task_id": 464, + "mode": "direct", + "action_index": 5, + "precondition_action_sha256": "34b3abca537d968a7863b498dbd1209eab3b681a94ee58f47d5beb3379c12f4d" + }, + { + "task_id": 470, + "mode": "direct", + "action_index": 2, + "precondition_action_sha256": "5d5fb246799936030abcc1e6df87b0c5dd63225d79ad09efa50f86309d4ce022" + }, + { + "task_id": 470, + "mode": "direct", + "action_index": 4, + "precondition_action_sha256": "cb0ec091ca536361e3010c0fd963c9a7a0db6282d066eebfe4c7e5439d4838e7" + }, + { + "task_id": 475, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "a1d771740b706d502e67b1b65b5f00336d15d53dfabdbdb01cb96df765ba9b2d" + }, + { + "task_id": 475, + "mode": "direct", + "action_index": 4, + "precondition_action_sha256": "7380204d149036b699e85e0390f614ac8e0af042520d0636558b561b92cbe2bd" + }, + { + "task_id": 506, + "mode": "direct", + "action_index": 10, + "precondition_action_sha256": "323eee9e3108f1c6338dc06089e409c059ac5b5e03d28b8b1c58dfcc0faa901f" + }, + { + "task_id": 528, + "mode": "direct", + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca" + }, + { + "task_id": 531, + "mode": "direct", + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca" + }, + { + "task_id": 532, + "mode": "direct", + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca" + }, + { + "task_id": 576, + "mode": "direct", + "action_index": 4, + "precondition_action_sha256": "79a89255b630ffef60066fcea63381e419fc8bf33699c7cb30e1b5418c2eb6be" + }, + { + "task_id": 576, + "mode": "direct", + "action_index": 6, + "precondition_action_sha256": "5ec0f3b20f6c43a1c447cf739b92e06a7df2e27eecb93b81c03cf32a839110a5" + }, + { + "task_id": 576, + "mode": "direct", + "action_index": 7, + "precondition_action_sha256": "32b836cd6899d40670d28e285cd72adb324d575db525f9cff9fd5cef09b118fe" + }, + { + "task_id": 576, + "mode": "direct", + "action_index": 9, + "precondition_action_sha256": "1b77cbf6041bdcb42c4b6f90831610a3cca4df22127432c8dfa561b46bc1dcb6" + }, + { + "task_id": 676, + "mode": "direct", + "action_index": 2, + "precondition_action_sha256": "7ce56844a684a94f42b04667b2c9b3ac0c8ff9488beb0c710e2aee11868f4d6a" + }, + { + "task_id": 676, + "mode": "direct", + "action_index": 4, + "precondition_action_sha256": "54eaf68c87c52061c70107527f0718e375ec4c142e61df61fee19c1b6099b353" + }, + { + "task_id": 725, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "e800c4f5df09041fbdedf98fc2e6d627541ab5ea05e84e93e9df0870ef90d16a" + }, + { + "task_id": 727, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "ef40323ab40edba820095dbb7e50c4fe63f4af071c4fee3e67c1074a20192e03" + }, + { + "task_id": 728, + "mode": "direct", + "action_index": 3, + "precondition_action_sha256": "2b0eb9ceee5ccd8fb44b1567973933b1375a84ca0a696c2a0d56f5b6a5181b1b" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/action-overrides.json b/evaluation/trajectory-replay/webarena-verified-full/action-overrides.json new file mode 100644 index 000000000..51ce55527 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/action-overrides.json @@ -0,0 +1,409 @@ +{ + "schema": "lex_webarena_replay_action_overrides/1", + "amendments": [ + { + "task_id": 3, + "action_index": 2, + "precondition_action_sha256": "c620510a7bbdfbf321606582dd492d1d1c8d9ebc4eb62a12e89e51100325f7a8", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 5, + "action_index": 2, + "precondition_action_sha256": "fcd1769f363ee1b81ef8b1c8c527aa3a3f98c941f1e07ddebecf99dc0ec85a28", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 6, + "action_index": 2, + "precondition_action_sha256": "c620510a7bbdfbf321606582dd492d1d1c8d9ebc4eb62a12e89e51100325f7a8", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 28, + "action_index": 3, + "precondition_action_sha256": "8bde53ce9d3e142a8666c0454be9a7cadd89990b1d967f09c13fa858aff9de1a", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 108, + "action_index": 7, + "precondition_action_sha256": "f92c92c01a99a141b96ee6d742e82704e9e71a2b48674827316d8607405ebeef", + "set": { + "force": true, + "selector": "button[data-action=\"grid-filter-expand\"]", + "wait_for_enabled": true + } + }, + { + "task_id": 108, + "action_index": 9, + "precondition_action_sha256": "da89ecce172f5e67d58f46e208be3aca529c64044d55947e52847e3c45988237", + "set": { + "force": true, + "selector": "input[name=\"created_at[from]\"]" + } + }, + { + "task_id": 108, + "action_index": 10, + "precondition_action_sha256": "bb75d4df58ca2c6906d9557515a9712afc8357e8a39787e715c0979b8a34c706", + "set": { + "force": true, + "selector": "input[name=\"created_at[to]\"]" + } + }, + { + "task_id": 108, + "action_index": 11, + "precondition_action_sha256": "ed2a04b267f1549acc1561c5babdcdb3f97ecca0e90bf70ce0dd4af511f05cdd", + "set": { + "force": true, + "selector": "select[name=\"status\"]" + } + }, + { + "task_id": 265, + "action_index": 6, + "precondition_action_sha256": "cb8d4d63d9e4ac4c4af190c414a567a465fa4ba1abf8ecf5c5ff77f4338269db", + "set": { + "script": "async()=>{const state=JSON.parse(window.name);const geocode=async q=>{const r=await fetch('/nominatim/search?format=json&addressdetails=1&limit=10&q='+encodeURIComponent(q));if(!r.ok)throw Error('geocode '+r.status);const rows=await r.json();if(!rows.length)throw Error('not found '+q);return rows};const park=(await geocode('Acadia National Park')).find(x=>x.osm_type==='relation');if(!park)throw Error('park relation missing');const r=await fetch('/osrm/routed-car/route/v1/driving/'+state.source.lon+','+state.source.lat+';'+state.park.lon+','+state.park.lat+'?overview=false'),x=await r.json();if(x.code!=='Ok'||!x.routes?.length)throw Error('no route');return [{relation_id:park.osm_id,distance:x.routes[0].distance+'m'}];}" + } + }, + { + "task_id": 266, + "action_index": 6, + "precondition_action_sha256": "cb8d4d63d9e4ac4c4af190c414a567a465fa4ba1abf8ecf5c5ff77f4338269db", + "set": { + "script": "async()=>{const state=JSON.parse(window.name);const geocode=async q=>{const r=await fetch('/nominatim/search?format=json&addressdetails=1&limit=10&q='+encodeURIComponent(q));if(!r.ok)throw Error('geocode '+r.status);const rows=await r.json();if(!rows.length)throw Error('not found '+q);return rows};const park=(await geocode('Acadia National Park')).find(x=>x.osm_type==='relation');if(!park)throw Error('park relation missing');const r=await fetch('/osrm/routed-car/route/v1/driving/'+state.source.lon+','+state.source.lat+';'+state.park.lon+','+state.park.lat+'?overview=false'),x=await r.json();if(x.code!=='Ok'||!x.routes?.length)throw Error('no route');return [{relation_id:park.osm_id,distance:x.routes[0].distance+'m'}];}" + } + }, + { + "task_id": 267, + "action_index": 6, + "precondition_action_sha256": "719319dc83757f9ebe281a8214b11973e824ae250a230941ba1e28bcddc86f04", + "set": { + "script": "async()=>{const state=JSON.parse(window.name);const geocode=async q=>{const r=await fetch('/nominatim/search?format=json&addressdetails=1&limit=10&q='+encodeURIComponent(q));if(!r.ok)throw Error('geocode '+r.status);const rows=await r.json();if(!rows.length)throw Error('not found '+q);return rows};const park=(await geocode('Acadia National Park')).find(x=>x.osm_type==='relation');if(!park)throw Error('park relation missing');const r=await fetch('/osrm/routed-car/route/v1/driving/'+state.source.lon+','+state.source.lat+';'+state.park.lon+','+state.park.lat+'?overview=false'),x=await r.json();if(x.code!=='Ok'||!x.routes?.length)throw Error('no route');return [{relation_id:park.osm_id,duration:x.routes[0].duration+'s'}];}" + } + }, + { + "task_id": 268, + "action_index": 6, + "precondition_action_sha256": "96e385477d8f1c513ab9e9e998b004674284b36d49258ca6eab2c476101e61aa", + "set": { + "script": "async()=>{const state=JSON.parse(window.name);const geocode=async q=>{const r=await fetch('/nominatim/search?format=json&addressdetails=1&limit=10&q='+encodeURIComponent(q));if(!r.ok)throw Error('geocode '+r.status);const rows=await r.json();if(!rows.length)throw Error('not found '+q);return rows};const park=(await geocode('Acadia National Park')).find(x=>x.osm_type==='relation');if(!park)throw Error('park relation missing');document.cookie='_osm_directions_engine=fossgis_osrm_bicycle; path=/';const r=await fetch('/osrm/routed-bike/route/v1/driving/'+state.source.lon+','+state.source.lat+';'+state.park.lon+','+state.park.lat+'?overview=false'),x=await r.json();if(x.code!=='Ok'||!x.routes?.length)throw Error('no route');return [{relation_id:park.osm_id,duration:x.routes[0].duration+'s'}];}" + } + }, + { + "task_id": 284, + "action_index": 3, + "precondition_action_sha256": "81d1f17b0f4324eae2288a91e72e1ec5944454aa669b4ccf6ffd5cd41d2d7227", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 458, + "action_index": 2, + "precondition_action_sha256": "9d7fe3c9e8d76df53e3e7188f81d80dc1ee73007a4a3dc73c08d937b3837fb4f", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 459, + "action_index": 2, + "precondition_action_sha256": "715b2b82a4d7d85e4680bda37cba159038ef0c02a1d95e60b853aa00559bb223", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 462, + "action_index": 2, + "precondition_action_sha256": "2cda73aa5baa10c3e96a48a5428a17dc12a466affec03f7071029828dca9cb9d", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 463, + "action_index": 2, + "precondition_action_sha256": "4af715ca18653a078156a3a26f989c6fcbff360b72c22405d9fd28c8c9b427b9", + "set": { + "wait_for_selector_hidden": ".loading-mask" + } + }, + { + "task_id": 464, + "action_index": 3, + "precondition_action_sha256": "cde2b999aee833627b12ef6308d2863576b3113f121840fb465cefbb28b12298", + "set": { + "script_mode": "function_body", + "script": "const deadline=Date.now()+45000;let title,viewModel;\nwhile(Date.now() .fieldset-wrapper-title');\n if(title&&typeof requirejs==='function'&&requirejs.defined('ko')){\n viewModel=require('ko').contextFor(title)?.$data;\n if(typeof viewModel?.toggleOpened==='function')break;\n }\n await new Promise(r=>setTimeout(r,100));\n}\nif(!title||typeof viewModel?.toggleOpened!=='function')throw new Error('Content accordion binding not ready');\nviewModel.toggleOpened();\nwhile(Date.now()setTimeout(r,100));\n}\nthrow new Error('Content accordion did not open');" + } + }, + { + "task_id": 464, + "action_index": 5, + "precondition_action_sha256": "34b3abca537d968a7863b498dbd1209eab3b681a94ee58f47d5beb3379c12f4d", + "set": { + "script_mode": "function_body", + "script": "const deadline=Date.now()+45000;let field,source;\nwhile(Date.now()el.name==='short_description'||el.name==='product[short_description]');\n const form=typeof requirejs==='function'&&requirejs.defined('uiRegistry')?require('uiRegistry').get('product_form.product_form'):null;\n source=form?.source;\n if(fields.length===1&&typeof source?.set==='function'){field=fields[0];break;}\n if(fields.length>1)throw new Error('multiple live short_description fields');\n await new Promise(r=>setTimeout(r,100));\n}\nif(!field||typeof source?.set!=='function')throw new Error('short_description field or Magento data source not ready');\nconst value='

1 customer(s) love it!

';\nfield.value=value;field.dispatchEvent(new Event('input',{bubbles:true}));field.dispatchEvent(new Event('change',{bubbles:true}));\nsource.set('data.product.short_description',value);\nif(source.get('data.product.short_description')!==value)throw new Error('short_description data source mismatch');\nreturn value;" + } + }, + { + "task_id": 464, + "action_index": 7, + "precondition_action_sha256": "063e4cf5b6b97b63e4583192fe888ca55f52099222911996d0fc88149fdc5a81", + "set": { + "selector": "div.message-success", + "wait_state": "attached" + } + }, + { + "task_id": 470, + "action_index": 2, + "precondition_action_sha256": "5d5fb246799936030abcc1e6df87b0c5dd63225d79ad09efa50f86309d4ce022", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 470, + "action_index": 4, + "precondition_action_sha256": "cb0ec091ca536361e3010c0fd963c9a7a0db6282d066eebfe4c7e5439d4838e7", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 475, + "action_index": 3, + "precondition_action_sha256": "a1d771740b706d502e67b1b65b5f00336d15d53dfabdbdb01cb96df765ba9b2d", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 475, + "action_index": 4, + "precondition_action_sha256": "7380204d149036b699e85e0390f614ac8e0af042520d0636558b561b92cbe2bd", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 506, + "action_index": 6, + "precondition_action_sha256": "94eb127ffa488f0647fe3039f82221411a80d382139eed75f92901a2bbf6e6a2", + "set": { + "wait_for_enabled": true + } + }, + { + "task_id": 506, + "action_index": 10, + "precondition_action_sha256": "323eee9e3108f1c6338dc06089e409c059ac5b5e03d28b8b1c58dfcc0faa901f", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 528, + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 531, + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 532, + "action_index": 2, + "precondition_action_sha256": "e03ef9d946b19f0cb2d4a41add4ffd49f8a706ce7f0b2eb8520b77db09958eca", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 547, + "action_index": 3, + "precondition_action_sha256": "dcaf408ccdc6d18c52884ec3c645305be542dbae9ca6b109d4bebbdabd1e4f70", + "set": { + "script": "(async()=>{const CFG={\"sizes\":[\"S\"],\"colors\":[\"Brown\"],\"newSize\":null,\"price\":\"59\",\"qty\":\"100\"};const wait=async(fn,label,ms=55000)=>{const end=Date.now()+ms;while(Date.now()setTimeout(r,100));}throw Error(label)};const reg=require('uiRegistry');const modalName='product_form.product_form.configurableModal';\nconst modal=reg.get(modalName);\nif(!(modal?.state?.())){const button=await wait(()=>[...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Edit Configurations'&&!e.disabled),'wizard opener not ready');button.click();}\nawait wait(()=>reg.get('variation-steps-wizard')?.wizard?.index===0&®.get('variation-steps-wizard_step1'),'wizard did not open');\nreturn true;})()" + } + }, + { + "task_id": 547, + "action_index": 5, + "precondition_action_sha256": "9047fb906326481eefb147ca82168692517336f7a5e821e36c2835dd71c7c3d1", + "set": { + "script": "(async()=>{const CFG={\"sizes\":[\"S\"],\"colors\":[\"Brown\"],\"newSize\":null,\"price\":\"59\",\"qty\":\"100\"};const wait=async(fn,label,ms=55000)=>{const end=Date.now()+ms;while(Date.now()setTimeout(r,100));}throw Error(label)};const reg=require('uiRegistry');const root=await wait(()=>reg.get('variation-steps-wizard')?.wizard&®.get('variation-steps-wizard'),'wizard root missing');\nconst s1=await wait(()=>reg.get('variation-steps-wizard_step1'),'step1 missing');\nawait wait(()=>s1.multiselect?.()?.rows?.()?.length,'attribute rows not loaded');\nif(!s1.stepInitialized)s1.doSelectSavedAttributes();\nawait wait(()=>{const ids=s1.multiselect().selected();const ls=ids.map(id=>s1.attributesLabels[id]).sort();return ls.join(',')==='Color,Size'},'Color/Size not selected');\nroot.next();\nconst s2=await wait(()=>root.wizard.index===1&®.get('variation-steps-wizard_step2'),'step2 not active');\nawait wait(()=>{const a=s2.attributes?.();return a?.length===2&&a.every(x=>x.options?.()?.length)},'step2 models not loaded');\nreturn true;})()" + } + }, + { + "task_id": 547, + "action_index": 7, + "precondition_action_sha256": "c8a9fb184fa1129a4bfb3b0bb6c75bb419442435f87f2d13b8f412e06a9d29c2", + "set": { + "script": "(async()=>{const CFG={\"sizes\":[\"S\"],\"colors\":[\"Brown\"],\"newSize\":null,\"price\":\"59\",\"qty\":\"100\"};const wait=async(fn,label,ms=55000)=>{const end=Date.now()+ms;while(Date.now()setTimeout(r,100));}throw Error(label)};const reg=require('uiRegistry');const root=reg.get('variation-steps-wizard'),s2=reg.get('variation-steps-wizard_step2');\nconst attrs=s2.attributes(),byLabel=l=>attrs.find(a=>a.label===l),size=byLabel('Size'),color=byLabel('Color');\nif(!size||!color)throw Error('Size/Color model missing');\nlet created=null;\nif(CFG.newSize){created=size.options().find(o=>o.label===CFG.newSize);if(!created){s2.createOption.call(size);created=size.options().find(o=>o.is_new&&o.label==='');if(!created)throw Error('new size model missing');created.label=CFG.newSize;s2.saveOption.call(size,created);}}\nconst choose=(a,wanted)=>{const opts=a.options().filter(o=>wanted.includes(o.label));if(opts.length!==wanted.length)throw Error(a.label+' options missing');a.chosenOptions(opts.map(o=>o.id));};\nchoose(size,CFG.sizes);choose(color,CFG.colors);\nconst labels=a=>a.chosenOptions().map(id=>a.options().find(o=>o.id===id)?.label).filter(Boolean).sort();\nif(labels(size).join('|')!==[...CFG.sizes].sort().join('|')||labels(color).join('|')!==[...CFG.colors].sort().join('|'))throw Error('selection mismatch');\nroot.next();await wait(()=>root.wizard.index===2,'step2 force rejected');\nif(created)await wait(()=>{if(s2.notificationMessage?.error)throw Error(s2.notificationMessage.text);return created.is_new===false&&created.value&&created.value!==0},'new option save did not finish');\nreturn true;})()" + } + }, + { + "task_id": 547, + "action_index": 9, + "precondition_action_sha256": "473f9ae63cbbac8db7c63e2b1f6963f6930e371e8e5c2be3d7c7595e648f097e", + "set": { + "script": "(()=>true)()" + } + }, + { + "task_id": 547, + "action_index": 11, + "precondition_action_sha256": "67c3284605e6a0958e4b885293f63fb2ef59589827d7132756c21f474288ae36", + "set": { + "script": "(async()=>{const CFG={\"sizes\":[\"S\"],\"colors\":[\"Brown\"],\"newSize\":null,\"price\":\"59\",\"qty\":\"100\"};const wait=async(fn,label,ms=55000)=>{const end=Date.now()+ms;while(Date.now()setTimeout(r,100));}throw Error(label)};const reg=require('uiRegistry');const root=reg.get('variation-steps-wizard'),s3=await wait(()=>root.wizard.index===2&®.get('variation-steps-wizard_step3'),'step3 missing');\nconst sec=await wait(()=>s3.sections?.()?.price&&s3.sections?.()?.quantity&&s3.sections(),'bulk sections missing');\nsec.price.type('single');sec.price.value(CFG.price);sec.quantity.type('single');sec.quantity.value(CFG.qty);\nroot.next();\nconst s4=await wait(()=>root.wizard.index===3&®.get('variation-steps-wizard_step4'),'bulk validation/transition failed');\nconst expected=CFG.sizes.length*CFG.colors.length;\nawait wait(()=>s4.variations?.length===expected&&s4.variations.every(v=>v.options.every(o=>o.value&&o.value!==0)),'summary variations incomplete');\nreturn true;})()" + } + }, + { + "task_id": 547, + "action_index": 13, + "precondition_action_sha256": "d9b26056fae8668fbedc2b89c697b7e12f2b293726554e2ed0a1eda44576c311", + "set": { + "script": "(async()=>{const CFG={\"sizes\":[\"S\"],\"colors\":[\"Brown\"],\"newSize\":null,\"price\":\"59\",\"qty\":\"100\"};const wait=async(fn,label,ms=55000)=>{const end=Date.now()+ms;while(Date.now()setTimeout(r,100));}throw Error(label)};const reg=require('uiRegistry');const root=reg.get('variation-steps-wizard'),s4=reg.get('variation-steps-wizard_step4');\nif(root.wizard.index!==3||!s4?.variations?.length)throw Error('summary not ready');\nroot.next();\nconst vc=reg.get('product_form.product_form.configurableVariations'),modal=reg.get('product_form.product_form.configurableModal');\nconst expected=CFG.sizes.length*CFG.colors.length;\nconst arrayValue=v=>typeof v==='function'?v():v;\nawait wait(()=>{const vals=arrayValue(vc?.value),matrix=arrayValue(vc?.productMatrix),save=document.querySelector('#save-button');return modal?.state?.()===false&&Array.isArray(vals)&&vals.length===expected&&Array.isArray(matrix)&&matrix.length===expected&&vals.every(v=>v.options?.every(o=>o.value&&o.value!==0))&&save&&!save.disabled},'generated matrix/save gate not ready');\nreturn {generated:expected};})()" + } + }, + { + "task_id": 547, + "action_index": 15, + "precondition_action_sha256": "e9578be134d951b653b4ba467531939e8b30593f1964755f42d53fcfbac864a7", + "set": { + "script": "(async()=>{const end=Date.now()+55000;while(Date.now()setTimeout(r,100));}throw Error('parent Save button not ready');})()" + } + }, + { + "task_id": 576, + "action_index": 4, + "precondition_action_sha256": "79a89255b630ffef60066fcea63381e419fc8bf33699c7cb30e1b5418c2eb6be", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 576, + "action_index": 6, + "precondition_action_sha256": "5ec0f3b20f6c43a1c447cf739b92e06a7df2e27eecb93b81c03cf32a839110a5", + "set": { + "script_mode": "function_body", + "script": "const deadline=Date.now()+30000;while(Date.now()e.children.length===0&&e.textContent.trim()==='abisubramanya27');\n const option=text?.closest('[role=option],button[role=menuitem],li,button')||text?.parentElement;\n if(option){option.click();return true;}\n await new Promise(r=>setTimeout(r,100));\n}\nthrow new Error('abisubramanya27 suggestion not ready');" + } + }, + { + "task_id": 576, + "action_index": 7, + "precondition_action_sha256": "32b836cd6899d40670d28e285cd72adb324d575db525f9cff9fd5cef09b118fe", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 576, + "action_index": 9, + "precondition_action_sha256": "1b77cbf6041bdcb42c4b6f90831610a3cca4df22127432c8dfa561b46bc1dcb6", + "set": { + "script_mode": "function_body", + "script": "const deadline=Date.now()+30000;while(Date.now()e.children.length===0&&e.textContent.trim()==='lahwaacz');\n const option=text?.closest('[role=option],button[role=menuitem],li,button')||text?.parentElement;\n if(option){option.click();return true;}\n await new Promise(r=>setTimeout(r,100));\n}\nthrow new Error('lahwaacz suggestion not ready');" + } + }, + { + "task_id": 585, + "action_index": 6, + "precondition_action_sha256": "4d43b4e95011546d307551f31a8f406e3d0500b53c26024db11428a4d6617052", + "set": { + "force": true + } + }, + { + "task_id": 585, + "action_index": 7, + "precondition_action_sha256": "4ac81f333fa6faf387587ffac02e4102ae13ef0f02981865f78450c87d95af80", + "set": { + "force": true + } + }, + { + "task_id": 585, + "action_index": 8, + "precondition_action_sha256": "c8c18cc3c609171b5b9f7dc02d0289fceffa76054df204e0ad30469a18429585", + "set": { + "force": true + } + }, + { + "task_id": 585, + "action_index": 9, + "precondition_action_sha256": "ccdab5942d0a9d856bd36c59b9760194ae05fecf7f4cadde868f1ae929f445ab", + "set": { + "force": true, + "selector": "button:has-text(\"Submit Review\")" + } + }, + { + "task_id": 676, + "action_index": 2, + "precondition_action_sha256": "7ce56844a684a94f42b04667b2c9b3ac0c8ff9488beb0c710e2aee11868f4d6a", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 676, + "action_index": 4, + "precondition_action_sha256": "54eaf68c87c52061c70107527f0718e375ec4c142e61df61fee19c1b6099b353", + "set": { + "script_mode": "function_body", + "script": "if(typeof requirejs!=='function'||!requirejs.defined('uiRegistry'))throw new Error('uiRegistry not ready');\nconst filters=[],providers=[];\nrequire('uiRegistry').get(item=>{\n if(item.component==='Magento_Ui/js/grid/filters/filters'&&item.ns==='sales_order_grid')filters.push(item);\n if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid')providers.push(item);\n});\nif(filters.length!==1||providers.length!==1)throw new Error('expected one sales-order filter/provider pair');\nconst applied={placeholder:true,status:'fraud'};\nfilters[0].setData(applied,false);\nfilters[0].set('applied',applied);\nproviders[0].params={...providers[0].params,search:'',keywordUpdated:false,filters:applied};\nproviders[0].reload();\nreturn {namespace:'sales_order_grid',status:'fraud',component:filters[0].name};" + } + }, + { + "task_id": 725, + "action_index": 3, + "precondition_action_sha256": "e800c4f5df09041fbdedf98fc2e6d627541ab5ea05e84e93e9df0870ef90d16a", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 727, + "action_index": 3, + "precondition_action_sha256": "ef40323ab40edba820095dbb7e50c4fe63f4af071c4fee3e67c1074a20192e03", + "set": { + "script_mode": "function_body" + } + }, + { + "task_id": 728, + "action_index": 3, + "precondition_action_sha256": "2b0eb9ceee5ccd8fb44b1567973933b1375a84ca0a696c2a0d56f5b6a5181b1b", + "set": { + "script_mode": "function_body" + } + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/current-comparison.json b/evaluation/trajectory-replay/webarena-verified-full/current-comparison.json new file mode 100644 index 000000000..3641c8b22 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/current-comparison.json @@ -0,0 +1,100 @@ +{ + "schema": "lex_webarena_replay_comparison_release/1", + "benchmark": "WebArena-Verified", + "split": "full-812", + "evidence_root": "trajectory-replay/full-812-five-browsers-six-methods-20260929", + "execution_backend": "playwright", + "resource_collection_status": "not_recorded", + "task_count": 812, + "logical_results": 4872, + "physical_executions": 6554, + "participants": [ + { + "id": "chromium", + "engine": "chromium", + "label": "Chromium 153", + "pass": 812, + "fail": 0 + }, + { + "id": "moli-pr731-pr853", + "engine": "moli", + "label": "Moli · #731 + #853", + "pass": 812, + "fail": 0 + }, + { + "id": "moli-v1.1.11", + "engine": "moli", + "label": "Moli v1.1.11", + "pass": 624, + "fail": 188 + }, + { + "id": "lightpanda-nightly-9813", + "engine": "lightpanda", + "label": "Lightpanda nightly 9813", + "pass": 330, + "fail": 482 + }, + { + "id": "obscura-0.2.3", + "engine": "obscura", + "label": "Obscura 0.2.3", + "pass": 386, + "fail": 426 + }, + { + "id": "kitesurf-cloudflare", + "engine": "kitesurf", + "label": "Kitesurf · Cloudflare Browser Run", + "pass": 441, + "fail": 371 + } + ], + "artifacts": { + "profile": { + "path": "profile.json", + "sha256": "d9fe439d7800bba3e8b62577631bc682b53b933a27d8244aafc0db0330faa47f" + }, + "dataset": { + "path": "dataset.json", + "sha256": "ce40a54d399c3aa33487621e8d7e05aab293fc2547edf0786d5fdbd699216e74" + }, + "results": { + "path": "results.json", + "sha256": "6074435e6a16976fb77c2bac48ea8a580e11d6d70b779b6fef69a9b082f82b62" + }, + "failure_review": { + "path": "failure-review.json", + "sha256": "27a15133cf2910c3a1ebfecc684a6ada851a63aed7604328857e7286b7d08ff2" + }, + "execution_metadata_revision": { + "path": "execution-metadata-revision.json", + "sha256": "95e19532a4d5e44399fc06656273303a6fc70c20e4b404ba74fdc81d29892411" + }, + "report_manifest": { + "path": "report/report-manifest.json", + "sha256": "6fb52c5c78e87863a40b8ba8189e860740e3cb63b5780670fc85b3acaac56ad8" + }, + "publication_archive": { + "path": "artifact-site-package.zip", + "sha256": "db50211d6f03c91d25b57911637ed19f52b2a5bbb3fb568db9dc0ec31d5f1fd9" + }, + "publication_attestation": { + "path": "artifact-site-attestation.json", + "sha256": "9fddc99a0ace95addde143730dea77564377aaf3bd1c8cab2bdb627623d75b56" + }, + "publication_receipt": { + "path": "artifact-site-receipt.json", + "sha256": "a9e5fdf7fab0ce544957576eb06d6a48fd0689730ef1bf4ddaac22779cd5581a" + } + }, + "publication": { + "service": "artifact-site", + "host": "https://artifact-site.lexmount.cn", + "slug": "EFLrNgujgsVc", + "version_id": "ver_c0f0693e9c08401bb4d2b88393234e55", + "share_url": "https://artifact-site.lexmount.cn/v/4G8YKfQreobGLwg4fo-UJGohezNqgAEm" + } +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/current-report.json b/evaluation/trajectory-replay/webarena-verified-full/current-report.json new file mode 100644 index 000000000..8db04cca8 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/current-report.json @@ -0,0 +1,44 @@ +{ + "schema": "lex_webarena_verified_full_report_profile/1", + "benchmark": "WebArena-Verified", + "split": "full-812", + "browser": "moli", + "dataset_path": "evaluation/trajectory-replay/webarena-verified-full/dataset.json", + "dataset_sha256": "74fd5a77dc652216628e2a4ff7b0b249045f10b8cc8a21dc710d3d05a591dde1", + "scope_path": "evaluation/trajectory-replay/webarena-verified-full/scope.json", + "scope_sha256": "a209bc38d06e51dab3b32269be88ff5e9c30b8ccfebbac39ef56e20da4b994df", + "evidence_root": "trajectory-replay/full-812-playwright-moli-20260926", + "accepted_catalog": { + "path": "accepted-catalog-final.json", + "sha256": "7a4d780f35eec75c9fac75666dc75f25b9fb33aeabe754bda55857ed286702e0" + }, + "accepted_portable_suite": { + "path": "accepted-suite-final.json", + "sha256": "7bbcf099ce13d9b0bce9652a32f24df8aef489dcaf3dac2702daed33decd062e" + }, + "acceptance_ledger_path": "acceptance-final-v20.json", + "acceptance_ledger_sha256": "83d4ff9ca2291d48647d453f85cf3c0c7d433731548ca694506f377831b0a46c", + "accepted_counts": { + "total": 812, + "pass": 812, + "unresolved": 0, + "physical_attempts": 1024, + "excluded_duplicate_attempts": 0 + }, + "report_manifest_path": "report-moli-final-v1/manifest.json", + "report_manifest_sha256": "b0213c2f31d419a8aa7ad6e5a11b8576727ab9aa06458177fb0d3bbe85472a5a", + "report_archive_path": "report-moli-final-v1.zip", + "report_archive_sha256": "d43188f4c9c54b9d353acdac481eb680178814393f53a4f699206c5604da955a", + "generated_languages": [ + "zh" + ], + "publication": { + "service": "artifact-site", + "host": "https://artifact-site.lexmount.cn", + "title": "WebArena-Verified 全量轨迹重放 · Moli", + "slug": "EFLrNgujgsVc", + "version_id": "ver_e65cd20b5a784e2d84381bf00e3df02e", + "share_url": "https://artifact-site.lexmount.cn/v/4G8YKfQreobGLwg4fo-UJGohezNqgAEm" + }, + "execution_backend": "playwright" +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/dataset.json b/evaluation/trajectory-replay/webarena-verified-full/dataset.json new file mode 100644 index 000000000..3bb637472 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/dataset.json @@ -0,0 +1,20 @@ +{ + "schema": "lex_webarena_verified_full_dataset/1", + "benchmark": "WebArena-Verified", + "evaluator_source": { + "repository": "https://github.com/lexmount/webarena-verified.git", + "branch": "lex-main", + "revision": "da7b992eb10de78000f11a7f34fd4590c171fbd6" + }, + "official_dataset_sha256": "b87ff766f3533934d1bee4ac3a91d06c4939a52bb7341df61b7556496faa3d8b", + "task_count": 812, + "task_id_range": [0, 811], + "execution_order": "ascending_official_task_id", + "success_policy": "one verified official-evaluator pass per case", + "retry_policy": "never repeat a passed case; repair failures after the initial sweep", + "independent_sample_split": { + "path": "evaluation/trajectory-replay/webarena-verified/selection.json", + "sha256": "06d8d999164c7c2de6cb21d15ec12f5260bfc2a58ca81b098a8c7779d4d2d91a", + "task_count": 90 + } +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/environment.json b/evaluation/trajectory-replay/webarena-verified-full/environment.json new file mode 100644 index 000000000..76ce06959 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/environment.json @@ -0,0 +1,27 @@ +{ + "schema": "lex_webarena_replay_environment/1", + "platform": "Rainbond", + "region": "rainbond", + "team": "ye7izl3p", + "host_overrides": [ + { + "pattern": "*.dev.lexmount.net", + "address": "10.2.62.142" + } + ], + "deployment": { + "acceptance_app_id": 202, + "source_app_id": 195, + "shared_map_app_id": 201, + "shared_wikipedia_app_id": 198, + "site_reset_between_tasks": false + }, + "images": { + "gitlab": "sha256:673ba94d757c5b3f4bb55751ad8b745a33bad2b987349322403cbc41035a92e0", + "reddit": "sha256:0594908059a03e5f610005689440a95c05e776e734006197c7b1eba12734cb46", + "shopping": "sha256:3e8cb9b945ea9b1c94ab26dba53e8d12dd0406abbf4bf686fd3bb2b6a5908feb", + "shopping_admin": "sha256:d0531dd27ed98d0c459ff9e88118bf2ed8b660b0ed99c38837db46c065a5be13", + "map": "sha256:c769aa312c979d68e696dace7049d58eeeb016ce608f0f8779a181bf3714206f", + "wikipedia_runtime": "sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce" + } +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/evaluator-transition.json b/evaluation/trajectory-replay/webarena-verified-full/evaluator-transition.json new file mode 100644 index 000000000..3f87a065a --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/evaluator-transition.json @@ -0,0 +1,43 @@ +{ + "schema": "lex_webarena_evaluator_transition/1", + "repository": "https://github.com/lexmount/webarena-verified.git", + "from_revision": "1925befb545c6433f7a7a5a77c473ce9e74eb468", + "compatible_from_revisions": [ + "1925befb545c6433f7a7a5a77c473ce9e74eb468", + "37960b08e5d6529ee5a7464f8927d12a5ca0f9e8", + "4c3b14e5cda91889f5ee883c8661bccd1d96c1af" + ], + "compatible_sources": [ + { + "kind": "revision", + "identity": "1925befb545c6433f7a7a5a77c473ce9e74eb468", + "evaluator_checksum": "0d463b93fda4f17b836456ac8c8e63525a597333b63118d2d195bc9ea32ee5a9", + "data_checksum": "08c9f0d80d3b32779ab37ebab1c50183cf545b6443266a55b05445bff166dbb8" + }, + { + "kind": "revision", + "identity": "37960b08e5d6529ee5a7464f8927d12a5ca0f9e8", + "evaluator_checksum": "7f6b1efd6ae2f88643f623a8ab4b1aeb107c0fa912b59f5ffd753de20337a226", + "data_checksum": "08c9f0d80d3b32779ab37ebab1c50183cf545b6443266a55b05445bff166dbb8" + }, + { + "kind": "revision", + "identity": "4c3b14e5cda91889f5ee883c8661bccd1d96c1af", + "evaluator_checksum": "7f6b1efd6ae2f88643f623a8ab4b1aeb107c0fa912b59f5ffd753de20337a226", + "data_checksum": "b87ff766f3533934d1bee4ac3a91d06c4939a52bb7341df61b7556496faa3d8b" + }, + { + "kind": "evaluator_fixes_sha256", + "identity": "58b451900c239ed6fb9a9a99633409bc56190676b9f2273ed45c472e8eb1ff08", + "evaluator_checksum": "a1093ea956105bb31fee7801856cbe7a299710f0824f4490de55cdef335383d8", + "data_checksum": "7ab0ef7e61af98efc5e16ba2b4303ddc0b8f5b66b5c88c58cb76e89e9235b57a" + } + ], + "from_evaluator_checksum": "0d463b93fda4f17b836456ac8c8e63525a597333b63118d2d195bc9ea32ee5a9", + "from_data_checksum": "08c9f0d80d3b32779ab37ebab1c50183cf545b6443266a55b05445bff166dbb8", + "to_revision": "da7b992eb10de78000f11a7f34fd4590c171fbd6", + "to_evaluator_checksum": "c24ae27d56ee3c2f7e12b5db737e06fb456e537b17f2bc9368970f895be4711d", + "to_data_checksum": "b87ff766f3533934d1bee4ac3a91d06c4939a52bb7341df61b7556496faa3d8b", + "fork_diff_sha256": "3dc522227643183713d48846a636125e8a84ab3c53188e4de7ed65e837fcbda6", + "compatibility_rule": "Every previously passing attempt must be re-evaluated from its hash-bound actual response and HAR and must still pass under the new evaluator. Browser actions are not repeated." +} diff --git a/evaluation/trajectory-replay/webarena-verified-full/scope.json b/evaluation/trajectory-replay/webarena-verified-full/scope.json new file mode 100644 index 000000000..64c4f9a20 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified-full/scope.json @@ -0,0 +1,829 @@ +{ + "schema": "lex_webarena_verified_full_scope/1", + "benchmark": "WebArena-Verified", + "official_dataset_sha256": "b87ff766f3533934d1bee4ac3a91d06c4939a52bb7341df61b7556496faa3d8b", + "task_count": 812, + "task_ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 516, + 517, + 518, + 519, + 520, + 521, + 522, + 523, + 524, + 525, + 526, + 527, + 528, + 529, + 530, + 531, + 532, + 533, + 534, + 535, + 536, + 537, + 538, + 539, + 540, + 541, + 542, + 543, + 544, + 545, + 546, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 556, + 557, + 558, + 559, + 560, + 561, + 562, + 563, + 564, + 565, + 566, + 567, + 568, + 569, + 570, + 571, + 572, + 573, + 574, + 575, + 576, + 577, + 578, + 579, + 580, + 581, + 582, + 583, + 584, + 585, + 586, + 587, + 588, + 589, + 590, + 591, + 592, + 593, + 594, + 595, + 596, + 597, + 598, + 599, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615, + 616, + 617, + 618, + 619, + 620, + 621, + 622, + 623, + 624, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 657, + 658, + 659, + 660, + 661, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 669, + 670, + 671, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 680, + 681, + 682, + 683, + 684, + 685, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 694, + 695, + 696, + 697, + 698, + 699, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 708, + 709, + 710, + 711, + 712, + 713, + 714, + 715, + 716, + 717, + 718, + 719, + 720, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 728, + 729, + 730, + 731, + 732, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 742, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 754, + 755, + 756, + 757, + 758, + 759, + 760, + 761, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 774, + 775, + 776, + 777, + 778, + 779, + 780, + 781, + 782, + 783, + 784, + 785, + 786, + 787, + 788, + 789, + 790, + 791, + 792, + 793, + 794, + 795, + 796, + 797, + 798, + 799, + 800, + 801, + 802, + 803, + 804, + 805, + 806, + 807, + 808, + 809, + 810, + 811 + ], + "execution_order": "official_dataset_order", + "browser": "chromium", + "success_policy": "one verified official-evaluator pass per case", + "retry_policy": "never repeat a passed case; repair failures after the initial sweep", + "independent_sample_split": { + "path": "evaluation/trajectory-replay/webarena-verified/selection.json", + "sha256": "06d8d999164c7c2de6cb21d15ec12f5260bfc2a58ca81b098a8c7779d4d2d91a", + "task_count": 90 + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-280.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-280.json new file mode 100644 index 000000000..40d7a4e3b --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-280.json @@ -0,0 +1,45 @@ +{ + "task_id": 280, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/catalogsearch/result/?q=Anker&product_list_limit=36" + }, + { + "action": "evaluate", + "timeout": 200000, + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(!location.pathname.startsWith('/catalogsearch/result')||new URL(location.href).searchParams.get('q')!==\"Anker\")throw Error('wrong current catalog scope');\nconst rows=[],pages=[],seen=new Set();let total=null;\nfor(let page=1;page<=100;page++){\n const query='query {products(filter:{name:{match:'+JSON.stringify(\"Anker\")+'}},pageSize:100,currentPage:'+page+`){total_count page_info{current_page total_pages} items{sku name url_key price_range{minimum_price{final_price{value currency}}}}}}`;\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog response unavailable');\n const payload=await response.json(),data=payload.data?.products;\n if(payload.errors?.length||!data||!Number.isSafeInteger(data.total_count)||data.total_count<0||!Array.isArray(data.items))throw Error('invalid catalog response');\n if(total===null)total=data.total_count;\n const count=Math.max(1,Math.ceil(total/100));\n if(total!==data.total_count||count>100||data.page_info.current_page!==page||data.page_info.total_pages!==Math.ceil(total/100)||data.items.length!==Math.min(100,Math.max(0,total-(page-1)*100)))throw Error('incomplete or changed catalog');\n for(const row of data.items){\n const price=row.price_range?.minimum_price?.final_price;\n if(typeof row.sku!=='string'||!row.sku||seen.has(row.sku)||typeof row.name!=='string'||!row.name||typeof row.url_key!=='string'||!row.url_key||row.url_key.includes('/')||price?.currency!=='USD'||typeof price.value!=='number'||!Number.isFinite(price.value)||price.value<0)throw Error('catalog identity or price invalid');\n seen.add(row.sku);rows.push({sku:row.sku,name:row.name,url:new URL('/'+row.url_key+'.html',location.origin).href,price_cents:Math.round(price.value*100),currency:price.currency});\n }\n pages.push({page,rows:data.items.length,total,ids:data.items.map(r=>r.sku)});if(page===count)break;\n}\nif(rows.length!==total)throw Error('catalog population incomplete');\nconst qualified=rows.filter(r=>/anker/i.test(r.name)&&/charger/i.test(r.name)).map(r=>({...r,qualification_evidence:{rule:\"name contains both Anker and charger, matching the official task's catalog-name qualification\",matched_name:r.name}}));\nif(!qualified.length||new Set(qualified.map(r=>r.name)).size!==qualified.length)throw Error('qualified catalog identity absent or duplicated');\nconst prices=qualified.map(r=>r.price_cents);\nreturn {scope:true,complete:true,total_count:total,returned_items:rows.length,pages,qualified_products:qualified,answer:{names:qualified.map(r=>r.name),min:Math.min(...prices)/100,max:Math.max(...prices)/100}};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 280, + "canonical_calls_sha256": "e8c06d05e08447cc88534eb74a89956254a5ee159626de258c414512a0a588e7", + "sources": [ + { + "name": "observation", + "step": 2, + "action_sha256": "0e9d25a8b161a9dc256818d5d0f2841d66d57c272e6f07ba32364e335a878c01", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-282.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-282.json new file mode 100644 index 000000000..dc6eb442d --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-282.json @@ -0,0 +1,45 @@ +{ + "task_id": 282, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/catalogsearch/result/?q=Nike&product_list_limit=36" + }, + { + "action": "evaluate", + "timeout": 200000, + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(!location.pathname.startsWith('/catalogsearch/result')||new URL(location.href).searchParams.get('q')!==\"Nike\")throw Error('wrong current catalog scope');\nconst rows=[],pages=[],seen=new Set();let total=null;\nfor(let page=1;page<=100;page++){\n const query='query {products(filter:{name:{match:'+JSON.stringify(\"Nike\")+'}},pageSize:100,currentPage:'+page+`){total_count page_info{current_page total_pages} items{sku name url_key price_range{minimum_price{final_price{value currency}}}}}}`;\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog response unavailable');\n const payload=await response.json(),data=payload.data?.products;\n if(payload.errors?.length||!data||!Number.isSafeInteger(data.total_count)||data.total_count<0||!Array.isArray(data.items))throw Error('invalid catalog response');\n if(total===null)total=data.total_count;\n const count=Math.max(1,Math.ceil(total/100));\n if(total!==data.total_count||count>100||data.page_info.current_page!==page||data.page_info.total_pages!==Math.ceil(total/100)||data.items.length!==Math.min(100,Math.max(0,total-(page-1)*100)))throw Error('incomplete or changed catalog');\n for(const row of data.items){\n const price=row.price_range?.minimum_price?.final_price;\n if(typeof row.sku!=='string'||!row.sku||seen.has(row.sku)||typeof row.name!=='string'||!row.name||typeof row.url_key!=='string'||!row.url_key||row.url_key.includes('/')||price?.currency!=='USD'||typeof price.value!=='number'||!Number.isFinite(price.value)||price.value<0)throw Error('catalog identity or price invalid');\n seen.add(row.sku);rows.push({sku:row.sku,name:row.name,url:new URL('/'+row.url_key+'.html',location.origin).href,price_cents:Math.round(price.value*100),currency:price.currency});\n }\n pages.push({page,rows:data.items.length,total,ids:data.items.map(r=>r.sku)});if(page===count)break;\n}\nif(rows.length!==total)throw Error('catalog population incomplete');\nconst qualified=rows.filter(r=>/^nike\\b/i.test(r.name)&&/\\bslides?\\b/i.test(r.name)).map(r=>({...r,qualification_evidence:{rule:\"name begins with Nike and explicitly identifies a slide or slides\",matched_name:r.name}}));\nif(!qualified.length||new Set(qualified.map(r=>r.name)).size!==qualified.length)throw Error('qualified catalog identity absent or duplicated');\nconst prices=qualified.map(r=>r.price_cents);\nreturn {scope:true,complete:true,total_count:total,returned_items:rows.length,pages,qualified_products:qualified,answer:{names:qualified.map(r=>r.name),min:Math.min(...prices)/100,max:Math.max(...prices)/100}};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 282, + "canonical_calls_sha256": "df6bdfcba113c601868a052ad247754cacabf2d5b86d2c838994ce6d31cb75f4", + "sources": [ + { + "name": "observation", + "step": 2, + "action_sha256": "0e1133f823668f648c4ca692e4c13a29cb5a3f4741bef2accf3e4715c0ef04f1", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-post-510.json b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-post-510.json new file mode 100644 index 000000000..42c659824 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-post-510.json @@ -0,0 +1,270 @@ +{ + "schema": "lex_webarena_replay_task_corrections/1", + "scope": "Browser Trajectory Replay standard-trajectory validation only", + "historical_browser_use_scores_immutable": true, + "upstream": { + "benchmark": "WebArena-Verified", + "package_version": "1.2.3", + "dataset_sha256": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30" + }, + "corrections": [ + { + "task_id": 225, + "official_revision": 2, + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + }, + { + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." + }, + { + "op": "replace", + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + }, + { + "op": "replace", + "path": "/instantiation_dict", + "value": { + "brand_text": "Sephora", + "product_text": "Brush" + } + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-complete-bluetooth-catalog-20260918", + "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + } + ], + "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "reference_receipt": { + "path": "research/trajectory-replay-task279-revised-reference.json", + "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + }, + "regression_ids": [ + "tr-279-complete-thirteen-products", + "tr-279-reject-omitted-product", + "tr-279-reject-wired-product" + ] + }, + { + "task_id": 448, + "official_revision": 2, + "correction_id": "wv-448-final-profile-save-success-variants-20260920", + "rationale": "Evaluate the last POST to the profile endpoint, then require the intended website and either a 302 redirect or an explicit 200 JSON success response. An earlier correct save cannot excuse a later incorrect save.", + "root_cause": "Disabling last_event_only made any earlier matching save sufficient, even when a later successful save replaced the website with a wrong value. Select the final endpoint/method event before checking its value; preserve the two valid response representations.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request", + "tr-448-reject-correct-then-wrong", + "tr-448-accept-wrong-then-correct" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__GITLAB__/-/profile" + }, + { + "path": "/eval/1/expected/post_data/user[website_url]", + "equals": "https://egg.tart.com" + }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, + { + "path": "/eval/1/last_event_only", + "equals": true + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/last_event_only", + "value": true + } + ], + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] + } + ] + }, + { + "task_id": 510, + "official_revision": 2, + "correction_id": "wv-510-restore-cart-quantity-contract-20260920", + "rationale": "Retain the declared product-selection revision but restore the upstream cart-totals check for quantity and name. A single DOM product row does not prove a single unit; all three frozen task-510 HARs contain the totals response required for zero-call rescoring.", + "root_cause": "Removing the totals evaluator dropped items_qty=1. The replacement trajectory checked only the number of product rows, so one row containing two units was accepted.", + "regression_ids": ["tr-510-quantity-one", "tr-510-reject-quantity-two", "tr-510-reject-missing-totals", "tr-510-reject-wrong-product"], + "preconditions": [ + { + "path": "/intent", + "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + }, + { + "path": "/eval/1/expected/post_data/product", + "equals": "75640" + }, + { + "path": "/eval/2/expected/response_content/items_qty", + "equals": 1 + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." + }, + { + "op": "add", + "path": "/instantiation_dict/selection_policy", + "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" + }, + { + "op": "add", + "path": "/instantiation_dict/target_product_id", + "value": "75640" + } + ] + }, + { + "task_id": 528, + "official_revision": 2, + "correction_id": "wv-528-dom-state-without-submit-20260911", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 547, + "official_revision": 2, + "correction_id": "wv-547-existing-brown-option-20260912", + "rationale": "Brown already exists as global color option 51 in the seeded fixture. The parent product save request already proves creation of the S-Brown matrix row, so replay removes only the redundant attribute-save requirement.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93" + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ] + }, + { + "task_id": 699, + "official_revision": 3, + "correction_id": "wv-699-all-customer-groups-20260918", + "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", + "preconditions": [ + { + "path": "/eval/1/expected/post_data/customer_group_ids[0]", + "equals": "1" + }, + { + "path": "/eval/1/post_data_schema", + "equals": null + } + ], + "operations": [ + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[1]", + "value": "2" + }, + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[2]", + "value": "3" + } + ] + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/task-corrections.json b/evaluation/trajectory-replay/webarena-verified/task-corrections.json index 42c659824..b9c99bc89 100644 --- a/evaluation/trajectory-replay/webarena-verified/task-corrections.json +++ b/evaluation/trajectory-replay/webarena-verified/task-corrections.json @@ -4,194 +4,1201 @@ "historical_browser_use_scores_immutable": true, "upstream": { "benchmark": "WebArena-Verified", - "package_version": "1.2.3", - "dataset_sha256": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30" + "repository": "https://github.com/lexmount/webarena-verified.git", + "branch": "lex-main", + "revision": "da7b992eb10de78000f11a7f34fd4590c171fbd6", + "dataset_sha256": "b87ff766f3533934d1bee4ac3a91d06c4939a52bb7341df61b7556496faa3d8b" }, "corrections": [ + { + "task_id": 9, + "official_revision": 2, + "correction_id": "wv-9-map-live-reference-20260923", + "rationale": "Apply the stated 30 km driving threshold to the current deployed OSRM routes; no international airport qualifies.", + "root_cause": "The frozen answer includes an airport whose current deployed OSRM driving route exceeds the task threshold.", + "reference_receipt": { + "path": "research/trajectory-replay-task9-map-live-reference.json", + "sha256": "6b8cd7d01288a6af017958f17ee50dd26897d10888a7da1974f3a60d6b613131" + }, + "regression_ids": [ + "tr-9-accept-live-map-reference", + "tr-9-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data", + "equals": [ + { + "name": "Pittsburgh International Airport", + "state": "Pennsylvania", + "postcode": "15231" + } + ] + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data", + "value": [] + } + ] + }, + { + "task_id": 18, + "official_revision": 2, + "correction_id": "wv-18-map-live-reference-20260923", + "rationale": "Use the current deployed OSRM driving and walking durations for the requested endpoints.", + "root_cause": "The frozen two-minute and sixteen-minute durations do not correspond to the current driving and walking routes between the named endpoints.", + "reference_receipt": { + "path": "research/trajectory-replay-task18-map-live-reference.json", + "sha256": "db697ab146f801d429b0e4b5f052bb651510a2b091203c54bb8dbb7552aae5b9" + }, + "regression_ids": [ + "tr-18-accept-live-map-reference", + "tr-18-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data", + "equals": [ + { + "mode": "driving", + "duration": "2min" + }, + { + "mode": "walking", + "duration": "16min" + } + ] + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data", + "value": [ + { + "mode": "driving", + "duration": "876.7s" + }, + { + "mode": "walking", + "duration": "6431.6s" + } + ] + } + ] + }, + { + "task_id": 32, + "official_revision": 2, + "correction_id": "wv-32-map-live-reference-20260923", + "rationale": "Use the current shortest walking route distance to the locally owned supermarket.", + "root_cause": "The live route changed by more than the evaluator tolerance while the selected hotel and supermarket remain the task-qualified pair.", + "reference_receipt": { + "path": "research/trajectory-replay-task32-map-live-reference.json", + "sha256": "f22129008f04f05ad1393e9d8dd9bc489fcffd2d00add74d4065697b52e7895e" + }, + "regression_ids": [ + "tr-32-accept-live-map-reference", + "tr-32-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/distance", + "equals": "2km" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/distance", + "value": "1946.9m" + } + ] + }, + { + "task_id": 34, + "official_revision": 2, + "correction_id": "wv-34-map-live-reference-20260923", + "rationale": "Align the response schema with the keys required by the intent and use the current shortest walking duration.", + "root_cause": "The evaluator schema names unrelated keys, bypassing duration normalization, and the live route duration has changed.", + "reference_receipt": { + "path": "research/trajectory-replay-task34-map-live-reference.json", + "sha256": "868cbff5ae45cb0d0bb71583f44846d594b0668e0a93a15328953a9b31ef996a" + }, + "regression_ids": [ + "tr-34-accept-live-map-reference", + "tr-34-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/results_schema/items/properties", + "equals": { + "hotel": { + "type": "string", + "format": "location-name" + }, + "information": { + "type": "string", + "format": "duration" + } + } + }, + { + "path": "/eval/0/expected/retrieved_data/0/travel_time", + "equals": "3h 30min" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/results_schema/items/properties", + "value": { + "hotel_name": { + "type": "string", + "format": "location-name" + }, + "travel_time": { + "type": "string", + "format": "duration" + } + } + }, + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/travel_time", + "value": "9892.3s" + } + ] + }, + { + "task_id": 35, + "official_revision": 2, + "correction_id": "wv-35-map-live-reference-20260923", + "rationale": "Align the response schema with the hotel_name and travel_time keys required by the intent so duration normalization is applied.", + "root_cause": "The evaluator schema names unrelated keys, so it compares duration text literally instead of applying the declared duration format.", + "reference_receipt": { + "path": "research/trajectory-replay-task35-map-live-reference.json", + "sha256": "7d75eef4e0ec8b896f8efb62ae484e885c9cf0849c00698854464a99fe06fcfe" + }, + "regression_ids": [ + "tr-35-accept-live-map-reference", + "tr-35-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/results_schema/items/properties", + "equals": { + "hotel": { + "type": "string", + "format": "location-name" + }, + "information": { + "type": "string", + "format": "duration" + } + } + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/results_schema/items/properties", + "value": { + "hotel_name": { + "type": "string", + "format": "location-name" + }, + "travel_time": { + "type": "string", + "format": "duration" + } + } + } + ] + }, + { + "task_id": 61, + "official_revision": 2, + "correction_id": "wv-61-map-live-reference-20260923", + "rationale": "Select the spatially closest current Nominatim restaurant result; the official answer is currently classified as a cafe.", + "root_cause": "The frozen answer is a cafe in current map data, while the task asks for the closest restaurant.", + "reference_receipt": { + "path": "research/trajectory-replay-task61-map-live-reference.json", + "sha256": "5b986082d88f1fa263f74e4ad946057c7687c062b8590a22133202e4a5e82d56" + }, + "regression_ids": [ + "tr-61-accept-live-map-reference", + "tr-61-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0", + "equals": "La Prima Espresso" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0", + "value": "Hunan Express" + } + ] + }, + { + "task_id": 90, + "official_revision": 3, + "correction_id": "wv-90-map-live-reference-20260923", + "rationale": "Use the OpenStreetMap relation for New York State linked from the frozen state page, rather than the City of New York relation.", + "root_cause": "The frozen answer uses the City of New York relation where the task asks for the bordering state relation.", + "reference_receipt": { + "path": "research/trajectory-replay-task90-map-live-reference.json", + "sha256": "f37ebc5f10bb714f06e2b857d2ed28b92124082d5e10492ae4a0a13cb3486aa2" + }, + "regression_ids": [ + "tr-90-accept-live-map-reference", + "tr-90-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/2", + "equals": 175905 + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/2", + "value": 61320 + } + ] + }, + { + "task_id": 179, + "official_revision": 2, + "correction_id": "wv-179-live-authored-issue-state-20260923", + "rationale": "Use the current state of the authenticated user's latest authored issue whose title contains feature.", + "root_cause": "The frozen answer says the latest matching authored issue is closed, while the deployed GitLab API identifies issue 83747 as the latest match and reports it open.", + "reference_receipt": { + "path": "research/trajectory-replay-task179-live-issue-reference.json", + "sha256": "f3860c3b1188917970c28f199cec828571f66e0c3a6255247339cac091329440" + }, + "regression_ids": [ + "tr-179-accept-live-authored-issue-state", + "tr-179-reject-opposite-issue-state" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data", + "equals": [ + true + ] + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data", + "value": [ + false + ] + } + ] + }, { "task_id": 225, "official_revision": 2, - "correction_id": "wv-225-explicit-title-scope-20260918", - "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", - "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + }, + { + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." + }, + { + "op": "replace", + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + }, + { + "op": "replace", + "path": "/instantiation_dict", + "value": { + "brand_text": "Sephora", + "product_text": "Brush" + } + } + ] + }, + { + "task_id": 252, + "official_revision": 2, + "correction_id": "wv-252-map-live-reference-20260923", + "rationale": "Use the current Nominatim object geometry coordinates returned for Tokyo Japanese Food Store.", + "root_cause": "The frozen centroid differs from the current object geometry by slightly more than the coordinate evaluator tolerance.", + "reference_receipt": { + "path": "research/trajectory-replay-task252-map-live-reference.json", + "sha256": "922192487ce820dae28b24a9e5998239a47da12e02ebf759468278e6d0ebc9d7" + }, + "regression_ids": [ + "tr-252-accept-live-map-reference", + "tr-252-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/latitude", + "equals": "40.45761" + }, + { + "path": "/eval/0/expected/retrieved_data/0/longitude", + "equals": "-79.92934" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/latitude", + "value": "40.45760025" + }, + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/longitude", + "value": "-79.92932994166013" + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-complete-bluetooth-catalog-20260918", + "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + } + ], + "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "reference_receipt": { + "path": "research/trajectory-replay-task279-revised-reference.json", + "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + }, + "regression_ids": [ + "tr-279-complete-thirteen-products", + "tr-279-reject-omitted-product", + "tr-279-reject-wired-product" + ] + }, + { + "task_id": 280, + "official_revision": 2, + "correction_id": "wv-280-complete-live-catalog-20260923", + "rationale": "The frozen reference contains 12 title-matched Anker charger products. The deployed catalog's complete 28-item Anker population now contains 17 names meeting the same title rule, so the frozen list omits five current products.", + "preconditions": [ + { + "path": "/eval/0/expected", + "equals": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [ + "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More", + "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)", + "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)", + "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply", + "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)", + "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)", + "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More", + "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy", + "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More", + "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)", + "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More", + "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)" + ], + "min": 8.99, + "max": 59.99 + } + ], + "error_details": null + } + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected", + "value": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [ + "Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter)", + "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)", + "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)", + "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)", + "Anker Outlet Extender with Stepless Dimming Night Light,Surge Protector, 331 Outlet Extender with 6 AC Outlets and 2 USB Ports, 18W USB Wall Charger Plug for College Dorm Room, Home Office and Kitchen", + "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply", + "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)", + "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy", + "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More", + "USB C Cable 60W 10ft, Anker Powerline III USB-C to USB-C Cable 2.0, USB C Charger Cable for MacBook Pro 2020, iPad Pro 2020, Switch, Samsung Galaxy S20 Plus S9 S8 Plus, Pixel, and More (White)", + "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/ 13 Pro/ 13 Pro Max/ 12, Galaxy S21, Note 20, iPad Pro, Pixel, and More", + "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)", + "USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included)", + "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More", + "Anker iPhone Charger Cable, Powerline II Lightning Cable (10ft), Durable Cable, MFi Certified for iPhone X / 8/8 Plus /7/7 Plus / 6/6 Plus / 5s (White), iPad 8, and More", + "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)", + "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More" + ], + "min": 8.99, + "max": 59.99 + } + ], + "error_details": null + } + } + ], + "root_cause": "The frozen reference contains 12 title-matched Anker charger products. The deployed catalog's complete 28-item Anker population now contains 17 names meeting the same title rule, so the frozen list omits five current products.", + "reference_receipt": { + "path": "research/trajectory-replay-task280-revised-reference.json", + "sha256": "347955c046cfc451c74f214f4fe494e07625949c13ede11bd9dfca50efab28e1" + }, + "regression_ids": [ + "tr-280-accept-complete-seventeen-products", + "tr-280-reject-frozen-omitted-products", + "tr-280-reject-non-charger-anker-product" + ] + }, + { + "task_id": 282, + "official_revision": 2, + "correction_id": "wv-282-complete-live-catalog-20260923", + "rationale": "The frozen reference includes Nike womens Benassi Just Do It, which is absent from the deployed catalog's complete 148-item Nike population. The current population instead includes Nike Owaysis Mens Slide, changing both membership and the minimum price.", + "preconditions": [ + { + "path": "/eval/0/expected", + "equals": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [ + "Nike Men's Air Max Camden Slide Sandal", + "Nike Men's Benassi JDI Fanny Pack Slides", + "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)", + "Nike Offcourt Slide Mens Bq4639-002 Size 12", + "Nike Jordan Men's Break Slide Red AR6374-602", + "Nike Victori One Slide Mens Style : Dd9559-300", + "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)", + "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)", + "Nike womens Benassi Just Do It" + ], + "min": 27.6, + "max": 90.65 + } + ], + "error_details": null + } + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected", + "value": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [ + "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)", + "Nike Men's Air Max Camden Slide Sandal", + "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)", + "Nike Jordan Men's Break Slide Red AR6374-602", + "Nike Men's Benassi JDI Fanny Pack Slides", + "Nike Offcourt Slide Mens Bq4639-002 Size 12", + "Nike Victori One Slide Mens Style : Dd9559-300", + "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)", + "Nike Owaysis Mens Slide Casual Sandal Ct5545-002 Size 6 Black/Black-Black" + ], + "min": 36.86, + "max": 90.65 + } + ], + "error_details": null + } + } + ], + "root_cause": "The frozen reference includes Nike womens Benassi Just Do It, which is absent from the deployed catalog's complete 148-item Nike population. The current population instead includes Nike Owaysis Mens Slide, changing both membership and the minimum price.", + "reference_receipt": { + "path": "research/trajectory-replay-task282-revised-reference.json", + "sha256": "495fd15c19ce76be4a839f83ee15d8817c388867c236e8484c0fbcd941d46c7e" + }, + "regression_ids": [ + "tr-282-accept-current-nine-slides", + "tr-282-reject-retired-benassi-product", + "tr-282-reject-non-slide-nike-sandal" + ] + }, + { + "task_id": 363, + "official_revision": 2, + "correction_id": "wv-363-map-live-reference-20260923", + "rationale": "Use the current deployed OSRM walking distance for the requested endpoints.", + "root_cause": "The deployed walking route distance changed beyond the strict two-percent distance tolerance.", + "reference_receipt": { + "path": "research/trajectory-replay-task363-map-live-reference.json", + "sha256": "43da0e1c5b4e965dfdf822181adc5c8e3cf1069f6f239eda5ddf51eebdc801f1" + }, + "regression_ids": [ + "tr-363-accept-live-map-reference", + "tr-363-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0", + "equals": "748m" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0", + "value": "778.3m" + } + ] + }, + { + "task_id": 367, + "official_revision": 2, + "correction_id": "wv-367-map-live-reference-20260923", + "rationale": "Use the current deployed OSRM walking distance to the closest current CVS result.", + "root_cause": "The deployed walking route distance changed beyond the strict two-percent distance tolerance.", + "reference_receipt": { + "path": "research/trajectory-replay-task367-map-live-reference.json", + "sha256": "4d0975533f2f64f189b764477a77546dfa5dbae3d5a8a3f7cb3275e8efcd7033" + }, + "regression_ids": [ + "tr-367-accept-live-map-reference", + "tr-367-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0", + "equals": "1.4km" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0", + "value": "1454.8m" + } + ] + }, + { + "task_id": 383, + "official_revision": 2, + "correction_id": "wv-383-map-live-reference-20260923", + "rationale": "Use the current Nominatim supermarket set whose deployed OSRM driving durations are at most 15 minutes.", + "root_cause": "The frozen supermarket membership no longer matches the current map search and 15-minute OSRM threshold.", + "reference_receipt": { + "path": "research/trajectory-replay-task383-map-live-reference.json", + "sha256": "8a92c1a6067168dc91673778d37ebeed86487b71f9dea7a21d1e92849048054a" + }, + "regression_ids": [ + "tr-383-accept-live-map-reference", + "tr-383-reject-nearby-wrong-map-reference" + ], + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/supermarkets", + "equals": [ + "Giant Eagle", + "ALDI" + ] + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/supermarkets", + "value": [ + "Shop & Save", + "ALDI", + "Walmart Supercenter", + "Gordon Food Service", + "Magnet International Market" + ] + } + ] + }, + { + "task_id": 429, + "official_revision": 2, + "correction_id": "wv-429-live-college-way-identities-20260923", + "rationale": "The deployed Nominatim records identify Washington & Jefferson College and Chatham University as OSM ways. Use those object kinds and let each evaluator inspect the complete trace so both separate-tab navigations can be recognized.", + "root_cause": "The upstream URLs label numeric way IDs as relations, and the navigation-only projection retains just the final popup navigation in the captured multi-tab HAR. The relation routes are absent; the same IDs under /way resolve to the required colleges. Matching the exact GET 200 page requests in the complete HAR preserves the two-page requirement.", + "reference_receipt": { + "path": "research/trajectory-replay-task429-map-live-reference.json", + "sha256": "705e21e4b86f447d85c471eeff56267940e2e64771f1a4fdf97102f6f5883c77" + }, + "regression_ids": [ + "tr-429-accept-live-way-identities", + "tr-429-reject-relation-kind-for-way-identities" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__MAP__/relation/583390395" + }, + { + "path": "/eval/2/expected/url", + "equals": "__MAP__/relation/172206707" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/url", + "value": "__MAP__/way/583390395" + }, + { + "op": "replace", + "path": "/eval/2/expected/url", + "value": "__MAP__/way/172206707" + }, + { + "op": "add", + "path": "/eval/1/last_event_only", + "value": false + }, + { + "op": "add", + "path": "/eval/2/last_event_only", + "value": false + }, + { + "op": "replace", + "path": "/eval/1/navigation_only", + "value": false + }, + { + "op": "replace", + "path": "/eval/2/navigation_only", + "value": false + } + ], + "replay_assertions": [ + { + "kind": "network_event_selection", + "policy": "match_expected_url_method_and_query_before_recency", + "target": "both exact college detail GET requests in the multi-tab HAR" + } + ] + }, + { + "task_id": 430, + "official_revision": 2, + "correction_id": "wv-430-resolve-college-map-placeholder-20260923", + "rationale": "Resolve the unfinished map placeholder to the deployed Washington & Jefferson College way, the Pennsylvania filming location outside Pittsburgh.", + "root_cause": "The upstream evaluator shipped a literal TASK_430_MAP_RELATION_ID placeholder. The deployed map index resolves Washington & Jefferson College to way 583390395, and that detail route is live.", + "reference_receipt": { + "path": "research/trajectory-replay-task430-map-live-reference.json", + "sha256": "3eb03e15488451581a98706bf3779b23660cf78f922a41cc9053955538ececaf" + }, + "regression_ids": [ + "tr-430-accept-live-washington-jefferson-way", + "tr-430-reject-unresolved-placeholder" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__MAP__/TASK_430_MAP_RELATION_ID" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/url", + "value": "__MAP__/way/583390395" + } + ] + }, + { + "task_id": 448, + "official_revision": 2, + "correction_id": "wv-448-final-profile-save-success-variants-20260920", + "rationale": "Evaluate the last POST to the profile endpoint, then require the intended website and either a 302 redirect or an explicit 200 JSON success response. An earlier correct save cannot excuse a later incorrect save.", + "root_cause": "Disabling last_event_only made any earlier matching save sufficient, even when a later successful save replaced the website with a wrong value. Select the final endpoint/method event before checking its value; preserve the two valid response representations.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request", + "tr-448-reject-correct-then-wrong", + "tr-448-accept-wrong-then-correct" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__GITLAB__/-/profile" + }, + { + "path": "/eval/1/expected/post_data/user[website_url]", + "equals": "https://egg.tart.com" + }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, + { + "path": "/eval/1/last_event_only", + "equals": true + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/last_event_only", + "value": true + } + ], + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] + } + ] + }, + { + "task_id": 510, + "official_revision": 2, + "correction_id": "wv-510-restore-cart-quantity-contract-20260920", + "rationale": "Retain the declared product-selection revision but restore the upstream cart-totals check for quantity and name. A single DOM product row does not prove a single unit; all three frozen task-510 HARs contain the totals response required for zero-call rescoring.", + "root_cause": "Removing the totals evaluator dropped items_qty=1. The replacement trajectory checked only the number of product rows, so one row containing two units was accepted.", + "regression_ids": [ + "tr-510-quantity-one", + "tr-510-reject-quantity-two", + "tr-510-reject-missing-totals", + "tr-510-reject-wrong-product" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + }, + { + "path": "/eval/1/expected/post_data/product", + "equals": "75640" + }, + { + "path": "/eval/2/expected/response_content/items_qty", + "equals": 1 + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." + }, + { + "op": "add", + "path": "/instantiation_dict/selection_policy", + "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" + }, + { + "op": "add", + "path": "/instantiation_dict/target_product_id", + "value": "75640" + } + ] + }, + { + "task_id": 528, + "official_revision": 2, + "correction_id": "wv-528-dom-state-without-submit-20260911", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 529, + "official_revision": 2, + "correction_id": "wv-529-dom-state-without-submit-20260923", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000157. Amount: $122.05. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 530, + "official_revision": 2, + "correction_id": "wv-530-dom-state-without-submit-20260923", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000161. Amount: $68.88. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 599, + "official_revision": 2, + "correction_id": "wv-599-reddit-machinelearning-slug-20260923", + "rationale": "Use the deployed case-sensitive MachineLearning forum slug while preserving the official newest-post identity and subscription-state requirement.", + "root_cause": "The official contract combines the human-readable forum label machine learning with a URL path, but the deployed forum route is /f/MachineLearning; the spaced path is absent and cannot emit the required subscription event.", "reference_receipt": { - "path": "research/trajectory-replay-task225-revised-reference.json", - "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + "path": "research/trajectory-replay-task599-reddit-forum-reference.json", + "sha256": "84e403f1b2fdce393049c423edfcfdfce7d520e38f93c5a1630beb0bee2faaeb" }, "regression_ids": [ - "tr-225-complete-two-products-no-low-reviews", - "tr-225-reject-incomplete-product-scope", - "tr-225-reject-new-low-review" + "tr-599-accept-machinelearning-subscription", + "tr-599-reject-human-label-url" ], "preconditions": [ { - "path": "/intent", - "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + "path": "/eval/1/expected/url", + "equals": [ + "__REDDIT__/f/machine learning/subscribe.json", + "__REDDIT__/f/machine learning/unsubscribe.json" + ] }, { - "path": "/eval/0/expected/status", - "equals": "NOT_FOUND_ERROR" + "path": "/eval/1/expected/headers/referer", + "equals": "__REDDIT__/f/machine learning/120613/d-yan-lecun-s-recent-recommendations" } ], "operations": [ { "op": "replace", - "path": "/intent", - "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." - }, - { - "op": "replace", - "path": "/intent_template", - "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + "path": "/eval/1/expected/url", + "value": [ + "__REDDIT__/f/MachineLearning/subscribe.json", + "__REDDIT__/f/MachineLearning/unsubscribe.json" + ] }, { "op": "replace", - "path": "/instantiation_dict", - "value": { - "brand_text": "Sephora", - "product_text": "Brush" - } + "path": "/eval/1/expected/headers/referer", + "value": "__REDDIT__/f/MachineLearning/120613/d-yan-lecun-s-recent-recommendations" } ] }, { - "task_id": 279, + "task_id": 682, "official_revision": 2, - "correction_id": "wv-279-complete-bluetooth-catalog-20260918", - "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "correction_id": "wv-682-live-chatgpt-commit-count-20260923", + "rationale": "Use the sole deployed GitLab project matching chatGPT and its complete commit pagination when constructing the forum post.", + "root_cause": "The frozen contract expects 193 commits, while the deployed GitLab search returns only convexegg/chatgpt and that repository contains one commit.", + "reference_receipt": { + "path": "research/trajectory-replay-task682-live-repository-reference.json", + "sha256": "1fbe9dec8774394af8efd3dc8f78a4a6cdc8dfddefe8623ca10c4a8db9e75b8b" + }, + "regression_ids": [ + "tr-682-accept-live-repository-post", + "tr-682-reject-frozen-repository-post" + ], "preconditions": [ { - "path": "/eval/0/expected/retrieved_data/0/names/9", - "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + "path": "/eval/1/expected/post_data/submission[body]", + "equals": "193 commits already!" } ], "operations": [ { "op": "replace", - "path": "/eval/0/expected/retrieved_data/0/names/9", - "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + "path": "/eval/1/expected/post_data/submission[body]", + "value": "1 commits already!" } - ], - "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + ] + }, + { + "task_id": 683, + "official_revision": 2, + "correction_id": "wv-683-live-metaseq-project-path-20260923", + "rationale": "Use the sole deployed GitLab project matching metaseq while preserving the complete observed commit count.", + "root_cause": "The frozen contract names facebook/metaseq, while the deployed GitLab search returns exactly one match at root/metaseq; its complete three-page history still contains 272 commits.", "reference_receipt": { - "path": "research/trajectory-replay-task279-revised-reference.json", - "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + "path": "research/trajectory-replay-task683-live-repository-reference.json", + "sha256": "026d055ba37e6f70a04198aac8a3a6d2a1956be43be8015a18bba6c2dbaf20de" }, "regression_ids": [ - "tr-279-complete-thirteen-products", - "tr-279-reject-omitted-product", - "tr-279-reject-wired-product" + "tr-683-accept-live-repository-post", + "tr-683-reject-frozen-repository-post" + ], + "preconditions": [ + { + "path": "/eval/1/expected/post_data/submission[url]", + "equals": "__GITLAB__/facebook/metaseq" + }, + { + "path": "/eval/1/expected/post_data/submission[title]", + "equals": "facebook/metaseq" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/post_data/submission[url]", + "value": "__GITLAB__/root/metaseq" + }, + { + "op": "replace", + "path": "/eval/1/expected/post_data/submission[title]", + "value": "root/metaseq" + } ] }, { - "task_id": 448, + "task_id": 689, "official_revision": 2, - "correction_id": "wv-448-final-profile-save-success-variants-20260920", - "rationale": "Evaluate the last POST to the profile endpoint, then require the intended website and either a 302 redirect or an explicit 200 JSON success response. An earlier correct save cannot excuse a later incorrect save.", - "root_cause": "Disabling last_event_only made any earlier matching save sufficient, even when a later successful save replaced the website with a wrong value. Select the final endpoint/method event before checking its value; preserve the two valid response representations.", + "correction_id": "wv-689-dom-state-without-submit-20260923", + "rationale": "The task requires a filled contact form kept ready for review, so the synthetic dummy_bin POST cannot be positive evidence. Final DOM values plus the retained prohibition on the real submit request express the contract.", + "root_cause": "The upstream contract requires an artificial POST while also prohibiting the real contact-form POST, contradicting the instruction to leave the form ready for review.", "regression_ids": [ - "tr-448-302-success", - "tr-448-200-json-success", - "tr-448-200-json-failure", - "tr-448-wrong-website", - "tr-448-canceled-request", - "tr-448-reject-correct-then-wrong", - "tr-448-accept-wrong-then-correct" + "tr-689-accept-filled-unsubmitted-form", + "tr-689-reject-wrong-coupon-reason", + "tr-689-reject-real-contact-submit" ], "preconditions": [ { "path": "/eval/1/expected/url", - "equals": "__GITLAB__/-/profile" - }, - { - "path": "/eval/1/expected/post_data/user[website_url]", - "equals": "https://egg.tart.com" + "equals": "^http://.*/dummy_bin$" }, { - "path": "/eval/1/expected/response_status", - "equals": 302 + "path": "/eval/1/expected/post_data/comment", + "equals": "I am a loyal customer, give me a coupon" }, { - "path": "/eval/1/last_event_only", + "path": "/eval/2/should_not_exist", "equals": true } ], "operations": [ { - "op": "replace", - "path": "/eval/1/last_event_only", - "value": true + "op": "remove", + "path": "/eval/1" } ], - "network_event_alternatives": [ + "replay_assertions": [ { - "evaluator_index": 1, - "operations": [ - { - "op": "replace", - "path": "/eval/1/expected/response_status", - "value": 200 - }, - { - "op": "replace", - "path": "/eval/1/expected/response_content", - "value": { - "message": "Profile was successfully updated" - } - } - ] + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I am a loyal customer, give me a coupon" + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } } ] }, { - "task_id": 510, + "task_id": 690, "official_revision": 2, - "correction_id": "wv-510-restore-cart-quantity-contract-20260920", - "rationale": "Retain the declared product-selection revision but restore the upstream cart-totals check for quantity and name. A single DOM product row does not prove a single unit; all three frozen task-510 HARs contain the totals response required for zero-call rescoring.", - "root_cause": "Removing the totals evaluator dropped items_qty=1. The replacement trajectory checked only the number of product rows, so one row containing two units was accepted.", - "regression_ids": ["tr-510-quantity-one", "tr-510-reject-quantity-two", "tr-510-reject-missing-totals", "tr-510-reject-wrong-product"], + "correction_id": "wv-690-dom-state-without-submit-20260923", + "rationale": "The task requires a filled contact form kept ready for review, so the synthetic dummy_bin POST cannot be positive evidence. Final DOM values plus the retained prohibition on the real submit request express the contract.", + "root_cause": "The upstream contract requires an artificial POST while also prohibiting the real contact-form POST, contradicting the instruction to leave the form ready for review.", + "regression_ids": [ + "tr-690-accept-filled-unsubmitted-form", + "tr-690-reject-wrong-coupon-reason", + "tr-690-reject-real-contact-submit" + ], "preconditions": [ { - "path": "/intent", - "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" }, { - "path": "/eval/1/expected/post_data/product", - "equals": "75640" + "path": "/eval/1/expected/post_data/comment", + "equals": "they promised me a coupon last time" }, { - "path": "/eval/2/expected/response_content/items_qty", - "equals": 1 + "path": "/eval/2/should_not_exist", + "equals": true } ], "operations": [ { - "op": "replace", - "path": "/intent", - "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." - }, - { - "op": "add", - "path": "/instantiation_dict/selection_policy", - "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" - }, + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ { - "op": "add", - "path": "/instantiation_dict/target_product_id", - "value": "75640" + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "they promised me a coupon last time" + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } } ] }, { - "task_id": 528, + "task_id": 691, "official_revision": 2, - "correction_id": "wv-528-dom-state-without-submit-20260911", - "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "correction_id": "wv-691-dom-state-without-submit-20260923", + "rationale": "The task requires a filled contact form kept ready for review, so the synthetic dummy_bin POST cannot be positive evidence. Final DOM values plus the retained prohibition on the real submit request express the contract.", + "root_cause": "The upstream contract requires an artificial POST while also prohibiting the real contact-form POST, contradicting the instruction to leave the form ready for review.", + "regression_ids": [ + "tr-691-accept-filled-unsubmitted-form", + "tr-691-reject-wrong-coupon-reason", + "tr-691-reject-real-contact-submit" + ], "preconditions": [ { "path": "/eval/1/expected/url", "equals": "^http://.*/dummy_bin$" }, + { + "path": "/eval/1/expected/post_data/comment", + "equals": "I plan to make a bulk purchase" + }, { "path": "/eval/2/should_not_exist", "equals": true @@ -211,7 +1218,7 @@ "fields": { "input[name='name']": "Emma Lopez", "input[name='email']": "emma.lopez@gmail.com", - "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + "textarea[name='comment']": "I plan to make a bulk purchase" }, "forbid_network_event": { "url": "__SHOPPING__/contact/index/post", @@ -221,14 +1228,28 @@ ] }, { - "task_id": 547, + "task_id": 692, "official_revision": 2, - "correction_id": "wv-547-existing-brown-option-20260912", - "rationale": "Brown already exists as global color option 51 in the seeded fixture. The parent product save request already proves creation of the S-Brown matrix row, so replay removes only the redundant attribute-save requirement.", + "correction_id": "wv-692-dom-state-without-submit-20260923", + "rationale": "The task requires a filled contact form kept ready for review, so the synthetic dummy_bin POST cannot be positive evidence. Final DOM values plus the retained prohibition on the real submit request express the contract.", + "root_cause": "The upstream contract requires an artificial POST while also prohibiting the real contact-form POST, contradicting the instruction to leave the form ready for review.", + "regression_ids": [ + "tr-692-accept-filled-unsubmitted-form", + "tr-692-reject-wrong-coupon-reason", + "tr-692-reject-real-contact-submit" + ], "preconditions": [ { "path": "/eval/1/expected/url", - "equals": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93" + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/1/expected/post_data/comment", + "equals": "I am a student" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true } ], "operations": [ @@ -236,33 +1257,69 @@ "op": "remove", "path": "/eval/1" } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I am a student" + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } ] }, { - "task_id": 699, - "official_revision": 3, - "correction_id": "wv-699-all-customer-groups-20260918", - "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", + "task_id": 693, + "official_revision": 2, + "correction_id": "wv-693-dom-state-without-submit-20260923", + "rationale": "The task requires a filled contact form kept ready for review, so the synthetic dummy_bin POST cannot be positive evidence. Final DOM values plus the retained prohibition on the real submit request express the contract.", + "root_cause": "The upstream contract requires an artificial POST while also prohibiting the real contact-form POST, contradicting the instruction to leave the form ready for review.", + "regression_ids": [ + "tr-693-accept-filled-unsubmitted-form", + "tr-693-reject-wrong-coupon-reason", + "tr-693-reject-real-contact-submit" + ], "preconditions": [ { - "path": "/eval/1/expected/post_data/customer_group_ids[0]", - "equals": "1" + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/1/expected/post_data/comment", + "equals": "my refund is supposed to be replaced by a coupon" }, { - "path": "/eval/1/post_data_schema", - "equals": null + "path": "/eval/2/should_not_exist", + "equals": true } ], "operations": [ { - "op": "add", - "path": "/eval/1/expected/post_data/customer_group_ids[1]", - "value": "2" - }, + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ { - "op": "add", - "path": "/eval/1/expected/post_data/customer_group_ids[2]", - "value": "3" + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "my refund is supposed to be replaced by a coupon" + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } } ] } diff --git a/harness/browser_use/har.py b/harness/browser_use/har.py index 5b3b11c67..eb45f0f79 100644 --- a/harness/browser_use/har.py +++ b/harness/browser_use/har.py @@ -5,7 +5,9 @@ import base64 import json import time +from collections.abc import Callable from datetime import UTC, datetime +from http.cookies import SimpleCookie from pathlib import Path from typing import Any @@ -37,7 +39,37 @@ def _har_request(request: dict[str, Any]) -> dict[str, Any]: return result -def _har_response(response: dict[str, Any], body: dict[str, Any] | None) -> dict[str, Any]: +def _response_cookies( + base_headers: dict[str, Any] | None, extra_headers: dict[str, Any] | None +) -> list[dict[str, str]]: + """Expose only the Magento flash cookie needed by official evaluators.""" + cookies: dict[str, str] = {} + for headers in (base_headers, extra_headers): + raw = next( + ( + str(value) + for name, value in (headers or {}).items() + if str(name).lower() == "set-cookie" + ), + "", + ) + if not raw: + continue + parsed = SimpleCookie() + try: + parsed.load(raw) + except Exception: + continue + if "mage-messages" in parsed: + cookies["mage-messages"] = parsed["mage-messages"].value + return [{"name": name, "value": value} for name, value in cookies.items()] + + +def _har_response( + response: dict[str, Any], + body: dict[str, Any] | None, + extra_headers: dict[str, Any] | None = None, +) -> dict[str, Any]: headers = response.get("headers", {}) mime_type = str(response.get("mimeType") or "") content: dict[str, Any] = {"mimeType": mime_type, "text": ""} @@ -47,7 +79,7 @@ def _har_response(response: dict[str, Any], body: dict[str, Any] | None) -> dict "status": int(response["status"]), "statusText": str(response.get("statusText") or ""), "headers": _headers(headers), - "cookies": [], + "cookies": _response_cookies(headers, extra_headers), "redirectURL": next( (str(value) for name, value in headers.items() if name.lower() == "location"), "", @@ -81,7 +113,22 @@ class CdpHarRecorder: "maxResourceBufferSize": 1024 * 1024, } - def __init__(self) -> None: + def __init__( + self, + *, + capture_transport: str = "cdp", + capture_new_pages: bool = True, + playwright_request_transform: Callable[ + [str, str, dict[str, Any]], + tuple[dict[str, Any], str | None, dict[str, Any] | None], + ] + | None = None, + ) -> None: + if capture_transport not in {"cdp", "playwright"}: + raise ValueError(f"unknown HAR capture transport: {capture_transport}") + self._capture_transport = capture_transport + self._capture_new_pages = capture_new_pages + self._playwright_request_transform = playwright_request_transform self._context: Any = None self._sessions: list[Any] = [] self._requests: dict[tuple[int, str], dict[str, Any]] = {} @@ -90,6 +137,7 @@ def __init__(self) -> None: self._session_ordinals: dict[int, int] = {} self._completion_timestamps: dict[tuple[int, str, int], Any] = {} self._extra_headers: dict[tuple[int, str], list[dict[str, Any]]] = {} + self._extra_response_headers: dict[tuple[int, str], list[dict[str, Any]]] = {} self._responses: list[tuple[Any, str, dict[str, Any], dict[str, Any]]] = [] self._finished: set[tuple[int, str]] = set() self._failed: dict[tuple[int, str], dict[str, Any]] = {} @@ -98,18 +146,65 @@ def __init__(self) -> None: self._response_bodies: dict[tuple[int, str, int], dict[str, Any]] = {} self._response_body_errors: dict[tuple[int, str, int], str] = {} self._payload: str | None = None + self._playwright_navigations: list[dict[str, Any]] = [] + self._playwright_requests: dict[int, dict[str, Any]] = {} + self._playwright_request_objects: dict[int, Any] = {} + self._playwright_request_order: list[int] = [] + self._playwright_responses: dict[int, tuple[Any, dict[str, Any]]] = {} + self._playwright_bodies: dict[int, dict[str, Any]] = {} + self._playwright_failures: dict[int, str] = {} + self._playwright_pages: list[Any] = [] self._stopped = False self.errors: list[str] = [] - def start(self, context: Any) -> None: + def start(self, context: Any, page: Any | None = None) -> None: self._context = context + if self._capture_transport == "playwright": + if self._capture_new_pages: + context.on("page", self._attach_playwright_page) + for selected_page in [page] if page is not None else context.pages: + self._attach_playwright_page(selected_page) + return context.on("page", self._attach) for page in context.pages: self._attach(page) + def record_navigation_response(self, response: Any) -> None: + """Retain an observed page.goto response when CDP omits Document events.""" + + if response is None: + return + try: + request = response.request + raw_request = { + "method": str(request.method), + "url": str(request.url), + "headers": dict(request.headers), + "postData": request.post_data, + "__resource_type": "Document", + } + raw_response = { + "status": int(response.status), + "statusText": str(response.status_text), + "headers": dict(response.headers), + "mimeType": str(response.headers.get("content-type") or "").split(";", 1)[0], + } + self._playwright_navigations.append( + { + "request": _har_request(raw_request), + "response": _har_response(raw_response, None), + "_capture": {"state": "playwright-navigation-response"}, + } + ) + except Exception as exc: + self.errors.append(f"Playwright navigation response capture failed: {exc}") + def stop(self, context: Any, path: Path) -> None: if self._stopped: return + if self._capture_transport == "playwright": + self._stop_playwright(context, path) + return if self._payload is None: # Network events and command responses share the ordered CDP channel. # A command round trip pumps Playwright's sync dispatcher without @@ -147,6 +242,10 @@ def stop(self, context: Any, path: Path) -> None: key: [dict(headers) for headers in values] for key, values in self._extra_headers.items() } + extra_response_headers_snapshot = { + key: [dict(headers) for headers in values] + for key, values in self._extra_response_headers.items() + } responses = [ observation for observation in responses_snapshot @@ -197,6 +296,10 @@ def stop(self, context: Any, path: Path) -> None: extras = extra_headers_snapshot.get(key, []) if ordinal < len(extras): request["headers"] = {**request.get("headers", {}), **extras[ordinal]} + response_extras = extra_response_headers_snapshot.get(key, []) + response_extra = ( + response_extras[ordinal] if ordinal < len(response_extras) else None + ) mime_type = str((response or {}).get("mimeType") or "") if ( index == last_response_by_key[key] @@ -218,7 +321,7 @@ def stop(self, context: Any, path: Path) -> None: entry = { "request": _har_request(request), "response": ( - _har_response(response, body) + _har_response(response, body, response_extra) if response is not None else { "status": 0, @@ -271,6 +374,22 @@ def stop(self, context: Any, path: Path) -> None: # Retain raw evidence even if the supplied wall clock is invalid. pass entries.append(entry) + by_request = { + (entry["request"]["method"], entry["request"]["url"]): entry + for entry in entries + } + for fallback in self._playwright_navigations: + identity = ( + fallback["request"]["method"], + fallback["request"]["url"], + ) + existing = by_request.get(identity) + if existing is None: + entries.append(fallback) + by_request[identity] = fallback + elif existing["response"]["status"] == 0: + existing["response"] = fallback["response"] + existing["_capture"] = fallback["_capture"] for session in self._sessions: try: session.detach() @@ -290,6 +409,207 @@ def stop(self, context: Any, path: Path) -> None: path.write_text(self._payload, encoding="utf-8") self._stopped = True + @staticmethod + def _playwright_headers(value: Any) -> dict[str, Any]: + try: + return dict(value.all_headers()) + except Exception: + return dict(value.headers) + + def _record_playwright_request(self, request: Any) -> None: + key = id(request) + if key in self._playwright_requests: + return + try: + raw = { + "method": str(request.method), + "url": str(request.url), + # Use cached headers in the event callback. all_headers() sends a + # protocol command and can re-enter response/finished callbacks + # before this request has been registered. + "headers": dict(request.headers), + "postData": request.post_data, + "__resource_type": str(request.resource_type).title(), + } + self._transform_playwright_request(raw) + except Exception as exc: + self.errors.append(f"Playwright request capture failed: {exc}") + return + self._playwright_requests[key] = raw + self._playwright_request_objects[key] = request + self._playwright_request_order.append(key) + + def _transform_playwright_request(self, raw: dict[str, Any]) -> None: + if self._playwright_request_transform is None: + return + headers, post_data, response_transform = self._playwright_request_transform( + str(raw["method"]), str(raw["url"]), dict(raw.get("headers", {})) + ) + raw["headers"] = headers + raw["__response_transform"] = response_transform + if post_data is not None: + raw["postData"] = post_data + + def _record_playwright_response(self, response: Any) -> None: + try: + request = response.request + self._record_playwright_request(request) + headers = dict(response.headers) + content_type = next( + ( + str(value) + for name, value in headers.items() + if str(name).lower() == "content-type" + ), + "", + ) + self._playwright_responses[id(request)] = ( + response, + { + "status": int(response.status), + "statusText": str(response.status_text), + "headers": headers, + "mimeType": content_type.split(";", 1)[0].strip(), + }, + ) + except Exception as exc: + self.errors.append(f"Playwright response capture failed: {exc}") + + def _capture_playwright_body(self, key: int) -> None: + observation = self._playwright_responses.get(key) + if observation is None or key in self._playwright_bodies: + return + response, raw_response = observation + if not _is_json_mime_type(raw_response.get("mimeType")): + return + try: + self._playwright_bodies[key] = { + "body": response.body().decode("utf-8", errors="replace"), + "base64Encoded": False, + } + except Exception as exc: + request = self._playwright_requests.get(key, {}) + self.errors.append( + f"{request.get('url', '')}: Playwright response body unavailable: {exc}" + ) + + def _record_playwright_finished(self, request: Any) -> None: + key = id(request) + self._record_playwright_request(request) + self._capture_playwright_body(key) + + def _record_playwright_failed(self, request: Any) -> None: + key = id(request) + self._record_playwright_request(request) + try: + detail = request.failure or "request failed" + except Exception as exc: + detail = f"request failure unavailable: {exc}" + self._playwright_failures[key] = str(detail) + + def _attach_playwright_page(self, page: Any) -> None: + if page in self._playwright_pages: + return + page.on("request", self._record_playwright_request) + page.on("response", self._record_playwright_response) + page.on("requestfinished", self._record_playwright_finished) + page.on("requestfailed", self._record_playwright_failed) + self._playwright_pages.append(page) + + def _stop_playwright(self, context: Any, path: Path) -> None: + if self._payload is None: + if self._capture_new_pages: + try: + context.remove_listener("page", self._attach_playwright_page) + except Exception as exc: + self.errors.append(f"Playwright page listener detach failed: {exc}") + for page in self._playwright_pages: + for event, callback in ( + ("request", self._record_playwright_request), + ("response", self._record_playwright_response), + ("requestfinished", self._record_playwright_finished), + ("requestfailed", self._record_playwright_failed), + ): + try: + page.remove_listener(event, callback) + except Exception as exc: + self.errors.append( + f"Playwright {event} listener detach failed: {exc}" + ) + entries: list[dict[str, Any]] = [] + for key in self._playwright_request_order: + request = self._playwright_requests[key] + request_object = self._playwright_request_objects[key] + request["headers"] = self._playwright_headers(request_object) + self._transform_playwright_request(request) + observation = self._playwright_responses.get(key) + if observation is not None: + self._capture_playwright_body(key) + response_object, raw_response = observation + raw_response["headers"] = self._playwright_headers(response_object) + response_transform = request.get("__response_transform") or {} + for field in ("status", "statusText", "headers"): + if field in response_transform: + raw_response[field] = response_transform[field] + raw_response["mimeType"] = next( + ( + str(value).split(";", 1)[0].strip() + for name, value in raw_response["headers"].items() + if str(name).lower() == "content-type" + ), + "", + ) + response = _har_response( + raw_response, + self._playwright_bodies.get(key), + response_transform.get("extra_headers") + or raw_response.get("headers"), + ) + else: + response = { + "status": 0, + "statusText": "", + "headers": [], + "cookies": [], + "redirectURL": "", + "content": {"mimeType": "", "text": ""}, + } + entry = {"request": _har_request(request), "response": response} + if key in self._playwright_failures: + entry["_capture"] = { + "state": "failed", + "errorText": self._playwright_failures[key], + } + elif observation is None: + entry["_capture"] = { + "state": "incomplete", + "reason": "response-not-observed-at-finalization", + } + entries.append(entry) + represented = { + (entry["request"]["method"], entry["request"]["url"]) + for entry in entries + } + entries.extend( + fallback + for fallback in self._playwright_navigations + if (fallback["request"]["method"], fallback["request"]["url"]) + not in represented + ) + self._context = None + self._payload = json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "browser-eval", "version": "1"}, + "entries": entries, + } + }, + ensure_ascii=False, + ) + path.write_text(self._payload, encoding="utf-8") + self._stopped = True + @staticmethod def _observation_order( index: int, @@ -391,6 +711,12 @@ def _attach(self, page: Any) -> None: (session_index, str(event["requestId"])), [] ).append(event.get("headers", {})), ) + session.on( + "Network.responseReceivedExtraInfo", + lambda event: self._extra_response_headers.setdefault( + (session_index, str(event["requestId"])), [] + ).append(event.get("headers", {})), + ) session.on( "Network.loadingFinished", lambda event: self._record_finished(session, session_index, event), diff --git a/harness/browser_use/runner.py b/harness/browser_use/runner.py index af49e6895..2ba34f784 100644 --- a/harness/browser_use/runner.py +++ b/harness/browser_use/runner.py @@ -481,15 +481,25 @@ def _browser_tool_config(endpoint: str) -> dict[str, Any]: "viewport_width": BENCHMARK_VIEWPORT["width"], "viewport_height": BENCHMARK_VIEWPORT["height"], "cdp_isolated_context": True, + "locale": "en-US", "startup_timeout": 90, } -def _record_page_viewport(page: Any, evidence: dict[str, Any], phase: str) -> None: +def _record_page_viewport( + page: Any, + evidence: dict[str, Any], + phase: str, + observer: Any | None = None, +) -> None: try: - observed = page.evaluate( - "() => ({width: innerWidth, height: innerHeight, " - "device_scale_factor: devicePixelRatio})" + observed = ( + observer(page) + if observer is not None + else page.evaluate( + "() => ({width: innerWidth, height: innerHeight, " + "device_scale_factor: devicePixelRatio})" + ) ) except Exception: evidence["verified"] = False @@ -506,8 +516,9 @@ def verify() -> None: if not pages: evidence["verified"] = False raise RuntimeError("browser viewport cannot be verified without a live page") + observer = getattr(service, "viewport_observer", None) for page in pages: - _record_page_viewport(page, evidence, phase) + _record_page_viewport(page, evidence, phase, observer) service._submit(verify) @@ -524,13 +535,24 @@ def _prepare_browser_viewport(service: Any) -> dict[str, Any]: def prepare() -> None: if not getattr(service, "_owns_cdp_context", False): raise RuntimeError("CowAgent lacks the required owned CDP context contract") - _record_page_viewport(service._page, evidence, "primary_before_navigation") - # Verify inheritance using a blank child before any login/site script. - child = service._context.new_page() - try: - _record_page_viewport(child, evidence, "new_page_before_navigation") - finally: - child.close() + observer = getattr(service, "viewport_observer", None) + _record_page_viewport( + service._page, evidence, "primary_before_navigation", observer + ) + if getattr(service, "_supports_multiple_pages", True): + # Verify inheritance using a blank child before any login/site script. + child = service._context.new_page() + try: + _record_page_viewport( + child, evidence, "new_page_before_navigation", observer + ) + finally: + child.close() + evidence["page_model"] = "multiple_pages" + else: + # Some CDP implementations expose one target per browser session. + # The primary page still proves the requested viewport was applied. + evidence["page_model"] = "single_target" evidence["retained_external_context_count"] = sum( context != service._context for context in service._browser.contexts ) @@ -573,6 +595,37 @@ def _bootstrap_postmill_session(context: Any, base_url: str) -> None: response.dispose() +def _bootstrap_postmill_browser_session(context: Any, page: Any, base_url: str) -> None: + """Establish Postmill auth through the browser when API cookies are not shared.""" + + creds = SITE_CREDENTIALS[WebArenaSite.REDDIT] + page.set_extra_http_headers( + { + SITE_AUTH_HEADERS[WebArenaSite.REDDIT]: ( + f"{creds['username']}:{creds['password']}" + ) + } + ) + try: + response = page.goto(base_url, wait_until="commit", timeout=90_000) + finally: + page.set_extra_http_headers({}) + if ( + response is None + or not 200 <= response.status < 300 + or response.request.redirected_from is not None + ): + status = response.status if response is not None else "no response" + raise RuntimeError( + f"Postmill browser bootstrap requires a direct 2xx response; got {status}" + ) + if not any( + cookie.get("name") == "PHPSESSID" and cookie.get("value") + for cookie in context.cookies(base_url) + ): + raise RuntimeError("Postmill browser bootstrap did not establish a session cookie") + + def _login_context( service: Any, sites: tuple[WebArenaSite, ...], @@ -580,6 +633,26 @@ def _login_context( start_urls: list[str], viewport_evidence: dict[str, Any] | None = None, ) -> int: + def publish_layout(page: Any) -> None: + publisher = getattr(service, "_publish_page_layout", None) + if publisher is not None: + publisher(page) + + def wait_for_login_ready( + page: Any, selector: str, *, exact_text: str | None = None + ) -> None: + waiter = getattr(service, "_wait_for_login_ready", None) + if waiter is not None: + waiter(page, selector, exact_text=exact_text) + return + if exact_text is not None: + page.get_by_role("button", name=exact_text, exact=True).wait_for() + return + locator = page.locator(selector) + if "," in selector: + locator = locator.first + locator.wait_for() + def login() -> None: context = service._context assert context is not None @@ -588,52 +661,80 @@ def login() -> None: for existing_page in context.pages: if existing_page != primary_page: existing_page.close() - context.clear_cookies() - context.set_extra_http_headers( - { - SITE_AUTH_HEADERS[site]: ( - f"{SITE_CREDENTIALS[site]['username']}:{SITE_CREDENTIALS[site]['password']}" - ) - for site in sites - if site in SITE_AUTH_HEADERS and site != WebArenaSite.REDDIT - } - ) + fresh_context = getattr(service, "_fresh_context_per_task", False) + if not fresh_context: + context.clear_cookies() + auth_headers = { + SITE_AUTH_HEADERS[site]: ( + f"{SITE_CREDENTIALS[site]['username']}:{SITE_CREDENTIALS[site]['password']}" + ) + for site in sites + if site in SITE_AUTH_HEADERS and site != WebArenaSite.REDDIT + } + if auth_headers or not fresh_context: + context.set_extra_http_headers(auth_headers) for site in sites: base = benchmark.config.get_environment(site).active_url - creds = SITE_CREDENTIALS[site] - page = context.new_page() + creds = SITE_CREDENTIALS.get(site) + reuse_primary = not getattr(service, "_supports_multiple_pages", True) + page = primary_page if reuse_primary else context.new_page() try: if viewport_evidence is not None: - _record_page_viewport(page, viewport_evidence, "login_before_navigation") + _record_page_viewport( + page, + viewport_evidence, + "login_before_navigation", + getattr(service, "viewport_observer", None), + ) page.set_default_timeout(60_000) page.set_default_navigation_timeout(90_000) if site == WebArenaSite.SHOPPING: page.goto(f"{base}/customer/account", wait_until="commit") page.evaluate("localStorage.clear(); sessionStorage.clear()") page.reload(wait_until="domcontentloaded") - page.locator(".block-dashboard-info, .box-information").first.wait_for() + publish_layout(page) + wait_for_login_ready( + page, ".block-dashboard-info, .box-information" + ) elif site == WebArenaSite.SHOPPING_ADMIN: page.goto(base, wait_until="commit") page.evaluate("localStorage.clear(); sessionStorage.clear()") page.reload(wait_until="domcontentloaded") - page.locator(".page-title").wait_for() + publish_layout(page) + wait_for_login_ready(page, ".page-title") elif site == WebArenaSite.REDDIT: - _bootstrap_postmill_session(context, base) + if getattr(service, "_supports_context_request_cookie_sharing", True): + _bootstrap_postmill_session(context, base) + else: + _bootstrap_postmill_browser_session(context, page, base) page.goto(base, wait_until="commit") page.evaluate("localStorage.clear(); sessionStorage.clear()") page.reload(wait_until="domcontentloaded") - page.get_by_role("button", name=creds["username"], exact=True).wait_for() + publish_layout(page) + wait_for_login_ready( + page, "button", exact_text=creds["username"] + ) elif site == WebArenaSite.GITLAB: page.goto(f"{base}/users/sign_in", wait_until="commit") page.evaluate("localStorage.clear(); sessionStorage.clear()") page.reload(wait_until="domcontentloaded") + publish_layout(page) page.get_by_test_id("username-field").fill(creds["username"]) + publish_layout(page) page.get_by_test_id("password-field").fill(creds["password"]) - page.get_by_test_id("sign-in-button").click() + publish_layout(page) + sign_in = page.get_by_test_id("sign-in-button") + if getattr(service, "_native_login_form_submit", False): + service._submit_login_form(page, sign_in) + else: + sign_in.click() page.wait_for_url("**/", timeout=90_000) + elif site not in {WebArenaSite.MAP, WebArenaSite.WIKIPEDIA}: + raise ValueError(f"unsupported WebArena login site: {site}") page.wait_for_timeout(1_000) finally: - page.close() + if not reuse_primary: + page.close() primary_page.set_default_timeout(60_000) primary_page.set_default_navigation_timeout(90_000) primary_page.goto(start_urls[0], wait_until="domcontentloaded") diff --git a/harness/environments/webarena/map_detail.py b/harness/environments/webarena/map_detail.py index d133a39b0..4ef1918fe 100644 --- a/harness/environments/webarena/map_detail.py +++ b/harness/environments/webarena/map_detail.py @@ -18,6 +18,32 @@ OBJECT_PATH = re.compile(r"^/(node|way|relation)/([1-9][0-9]{0,18})/?$") TYPE_CODE = {"node": "N", "way": "W", "relation": "R"} NOMINATIM = "http://127.0.0.1:8085" +# Nominatim deliberately omits many route relations. This exact object is +# frozen from Wikidata Q1132510 revision 2489023584, whose P402 value is +# relation 5475586 and whose P625 coordinate is the value below. The source +# entity JSON has SHA-256 479e2cf83fe867b87eb875ddc5916af2feae1664fae45426f391a2ffd7df7185. +FROZEN_OBJECTS = { + ("relation", "5475586"): { + "osm_type": "relation", + "osm_id": 5475586, + "display_name": ( + "Lake Pontchartrain Causeway, Louisiana, United States" + ), + "class": "man_made", + "type": "bridge", + "lat": "30.199722222222", + "lon": "-90.122777777778", + "address": {"state": "Louisiana", "country": "United States"}, + "extratags": { + "description": ( + "set of two parallel bridges over Lake Pontchartrain in " + "southern Louisiana, United States" + ), + "length": "38.442 km", + "wikidata": "Q1132510", + }, + } +} STYLE = """ .map-detail {font:14px/1.45 system-ui,sans-serif;color:#263238;min-width:0} .map-detail h1 {font-size:1.45rem;margin:0 0 8px;overflow-wrap:anywhere} @@ -36,6 +62,9 @@ def lookup(kind: str, identifier: str) -> dict | None: + frozen = FROZEN_OBJECTS.get((kind, identifier)) + if frozen is not None: + return frozen query = urlencode({"osm_ids": TYPE_CODE[kind] + identifier, "format": "json", "addressdetails": 1, "extratags": 1}) with urlopen(f"{NOMINATIM}/lookup?{query}", timeout=10) as response: diff --git a/harness/trajectory_replay/catalog_acceptance.py b/harness/trajectory_replay/catalog_acceptance.py new file mode 100644 index 000000000..81c2cb6d9 --- /dev/null +++ b/harness/trajectory_replay/catalog_acceptance.py @@ -0,0 +1,716 @@ +"""Audit an ordered WebArena catalog sweep and only its failed-case repairs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from webarena_verified.core.utils.checksum import compute_evaluator_checksum +from webarena_verified.types.task import WebArenaSite + +from harness.trajectory_replay.catalog_contracts import ( + ACCEPTANCE_SCHEMA, + EXECUTION_CONTRACT_RUN_SCHEMAS, + RUN_SCHEMAS, + SCOPE_SCHEMAS, + file_sha256, + load_action_overrides, + validate_passing_evaluation, + verify_physical_evidence, +) +from harness.trajectory_replay.evaluator_transition import ( + compatible_source, + load_receipt, + load_transition, + reevaluate_pass, + validate_source_pass, +) +from harness.trajectory_replay.remote_runtime import load_remote_sites, remote_benchmark +from harness.trajectory_replay.runtime_identity import ( + assert_observed_runtime, + request_bridge_identity, + runtime_identity, +) +from harness.trajectory_replay.webarena_corrections import load_corrections +from harness.trajectory_replay.webarena_runtime_source import ( + WEBARENA_VERIFIED_REVISION, + verify_webarena_runtime_source, +) + + +def _evaluator_identity_field(binding: dict) -> str: + """Return the evaluator identity used by a recorded run contract. + + Catalog run schema /1 was emitted with both identities at different points in + its lifetime. Schema /2 removes that ambiguity for new runs while this reader + keeps both already-recorded /1 forms verifiable. + """ + has_revision = "webarena_revision" in binding + has_fixes = "evaluator_fixes_sha256" in binding + if binding.get("schema") == "lex_trajectory_replay_catalog_run/2": + if not has_revision or has_fixes: + raise ValueError("catalog run /2 must bind exactly one WebArena revision") + return "webarena_revision" + if binding.get("schema") == "lex_trajectory_replay_catalog_run/1": + if has_revision == has_fixes: + raise ValueError("catalog run /1 must bind exactly one evaluator identity") + return "webarena_revision" if has_revision else "evaluator_fixes_sha256" + return "evaluator_fixes_sha256" + + +def _read_run(directory: Path) -> tuple[dict, list[dict]]: + binding_path = directory / "inputs.json" + progress_path = directory / "progress.jsonl" + if any(path.is_symlink() or not path.is_file() for path in (binding_path, progress_path)): + raise ValueError(f"run binding or progress is missing: {directory}") + binding = json.loads(binding_path.read_text(encoding="utf-8")) + rows = [json.loads(line) for line in progress_path.read_text(encoding="utf-8").splitlines()] + order = binding.get("execution_order") + if ( + binding.get("schema") not in RUN_SCHEMAS + or not isinstance(order, list) + or [row.get("task_id") for row in rows] != order[: len(rows)] + or binding.get("run_count_per_task") != 1 + ): + raise ValueError(f"run order or protocol changed: {directory}") + if binding["schema"] in EXECUTION_CONTRACT_RUN_SCHEMAS: + required = { + "engine", + "backend", + "action_timeout_ms", + "environment_profile_sha256", + "corrections_sha256", + } + evaluator_identity = _evaluator_identity_field(binding) + required.add(evaluator_identity) + if not required <= set(binding): + raise ValueError(f"run execution contract is incomplete: {directory}") + return binding, rows + + +def _verified_attempt( + directory: Path, + row: dict, + *, + expected_data_checksum: str, + benchmark: object, + corrections: object, + run_inputs_sha256: str | None = None, + run_revision: str | None = None, + run_evaluator_fixes_sha256: str | None = None, + evaluator_transition: dict | None = None, + transition_receipt: dict[tuple[int, str, str], dict] | None = None, + runtime_binding: dict[str, object] | None = None, +) -> tuple[dict, bool]: + task_id = row["task_id"] + physical = "action_count" in row + if physical and runtime_binding is not None: + assert_observed_runtime( + runtime_binding, row.get("runtime_metadata"), task_id=task_id + ) + transition_binding = None + if physical: + evaluation_path = verify_physical_evidence(directory, row) + elif row.get("status") != "candidate_setup_failure" or not row.get("error"): + raise ValueError(f"task {task_id} has neither physical evidence nor setup error") + else: + evaluation_path = None + if row.get("status") == "pass": + if evaluation_path is None: + raise ValueError(f"task {task_id} pass lacks evaluation evidence") + evaluation = json.loads(evaluation_path.read_text(encoding="utf-8")) + current_evaluator = evaluation.get("webarena_verified_evaluator_checksum") + current_data = evaluation.get("webarena_verified_data_checksum") + if ( + current_evaluator == compute_evaluator_checksum() + and current_data == expected_data_checksum + ): + if run_revision != WEBARENA_VERIFIED_REVISION: + raise ValueError(f"task {task_id} current evaluation has a different revision") + validate_passing_evaluation( + evaluation, + task_id=task_id, + expected_data_checksum=expected_data_checksum, + benchmark=benchmark, + corrections=corrections, + ) + else: + source = ( + compatible_source( + evaluator_transition, + revision=run_revision, + evaluator_fixes_sha256=run_evaluator_fixes_sha256, + ) + if evaluator_transition is not None + else None + ) + if ( + evaluator_transition is None + or transition_receipt is None + or source is None + or run_inputs_sha256 is None + or current_evaluator != source["evaluator_checksum"] + or current_data != source["data_checksum"] + ): + raise ValueError(f"task {task_id} pass lacks evaluator transition evidence") + validate_source_pass(evaluation, task_id=task_id, source=source) + current, transition_binding = reevaluate_pass( + directory / f"task-{task_id}", + task_id=task_id, + expected_data_checksum=expected_data_checksum, + benchmark=benchmark, + corrections=corrections, + ) + del current + receipt_row = transition_receipt.get( + ( + task_id, + transition_binding["old_evaluation_sha256"], + run_inputs_sha256, + ) + ) + expected_receipt = { + "run_inputs_sha256": run_inputs_sha256, + "source": source, + **transition_binding, + } + if receipt_row != expected_receipt: + raise ValueError(f"task {task_id} evaluator transition receipt changed") + if any(item.get("status") != "pass" for item in evaluation.get("replay_assertions") or []): + raise ValueError(f"task {task_id} pass disagrees with replay assertions") + return ( + { + "task_id": task_id, + "status": row.get("status"), + "score": row.get("score"), + "progress_sha256": file_sha256(directory / "progress.jsonl"), + "evidence_sha256": row.get("evidence_sha256"), + **( + {"runtime_metadata": row["runtime_metadata"]} + if isinstance(row.get("runtime_metadata"), dict) + else {} + ), + **( + {"evaluator_transition": transition_binding} + if transition_binding is not None + else {} + ), + }, + physical, + ) + + +def _superseded_attempt( + directory: Path, + row: dict, + *, + runtime_binding: dict[str, object] | None = None, +) -> tuple[dict, bool]: + """Verify retained physical bytes without promoting an obsolete action contract.""" + + task_id = row["task_id"] + physical = "action_count" in row + if physical and runtime_binding is not None: + assert_observed_runtime( + runtime_binding, row.get("runtime_metadata"), task_id=task_id + ) + if physical: + verify_physical_evidence(directory, row) + elif row.get("status") != "candidate_setup_failure" or not row.get("error"): + raise ValueError(f"task {task_id} has neither physical evidence nor setup error") + return ( + { + "task_id": task_id, + "status": row.get("status"), + "score": row.get("score"), + "progress_sha256": file_sha256(directory / "progress.jsonl"), + "evidence_sha256": row.get("evidence_sha256"), + "superseded_by_catalog_action_change": True, + **( + {"runtime_metadata": row["runtime_metadata"]} + if isinstance(row.get("runtime_metadata"), dict) + else {} + ), + }, + physical, + ) + + +def audit( + scope_path: Path, + segment_dirs: list[Path], + repair_dirs: list[Path], + *, + replacement_dirs: list[Path] | None = None, + superseded_catalog_tasks: set[int] | None = None, + excluded_duplicate_repair_tasks: set[int] | None = None, + expected_backend: str = "browsertool", + action_overrides_paths: list[Path] | None = None, + site_map_path: Path | None = None, + evaluator_transition_path: Path | None = None, + evaluator_transition_receipt_path: Path | None = None, +) -> dict: + verify_webarena_runtime_source() + if (evaluator_transition_path is None) != (evaluator_transition_receipt_path is None): + raise ValueError("evaluator transition authority and receipt must be supplied together") + evaluator_transition = ( + load_transition(evaluator_transition_path) + if evaluator_transition_path is not None + else None + ) + if evaluator_transition_receipt_path is not None and evaluator_transition is not None: + transition_receipt, transition_exclusions = load_receipt( + evaluator_transition_receipt_path, evaluator_transition + ) + else: + transition_receipt = None + transition_exclusions: set[tuple[int, str, str]] = set() + superseded_requested = set(superseded_catalog_tasks or ()) + replacement_dirs = list(replacement_dirs or ()) + if bool(superseded_requested) != bool(replacement_dirs): + raise ValueError("catalog replacements require an exact superseded task scope") + used_transition_rows: set[tuple[int, str, str]] = set() + expected_transition_exclusions: set[tuple[int, str, str]] = set() + scope = json.loads(scope_path.read_text(encoding="utf-8")) + ids = scope.get("task_ids") + data_checksum = str(scope.get("official_dataset_sha256") or "") + if ( + scope.get("schema") not in SCOPE_SCHEMAS + or not isinstance(ids, list) + or not ids + or any(not isinstance(task_id, int) for task_id in ids) + or len(set(ids)) != len(ids) + or scope.get("task_count") != len(ids) + or not data_checksum + ): + raise ValueError("the declared WebArena task scope changed") + if not segment_dirs: + raise ValueError("acceptance segments are required") + if not superseded_requested <= set(ids): + raise ValueError("catalog replacement scope is outside the declared task scope") + states = { + task_id: {"task_id": task_id, "attempts": [], "action_sha256s": []} + for task_id in ids + } + excluded_requested = set(excluded_duplicate_repair_tasks or ()) + excluded_seen: set[int] = set() + excluded_attempts = [] + ordered_ids: list[int] = [] + bindings = [] + schemas = set() + sweep_contract = None + participant_contract = None + binary_sha256s: set[str] = set() + remote_browser_identities: dict[str, dict[str, object]] = {} + preparation_sha256s = set() + physical_attempts = 0 + replacement_seen: set[int] = set() + benchmark = remote_benchmark( + load_remote_sites(site_map_path) + if site_map_path is not None + else {site: f"http://{site.value}.invalid" for site in WebArenaSite} + ) + corrections = load_corrections() + override_sources = {} + for path in action_overrides_paths or []: + _amendments, digest = load_action_overrides(path) + if digest in override_sources: + raise ValueError("duplicate external action overrides source") + override_sources[digest] = path + used_override_sources: set[str] = set() + for kind, directories in ( + ("segment", segment_dirs), + ("repair", repair_dirs), + ("replacement", replacement_dirs), + ): + for directory in directories: + binding, rows = _read_run(directory) + run_revision = binding.get("webarena_revision") + run_evaluator_fixes_sha256 = binding.get("evaluator_fixes_sha256") + if "action_overrides_sha256" in binding: + override_digest = binding["action_overrides_sha256"] + used_override_sources.add(override_digest) + schemas.add(binding["schema"]) + backend = binding.get("backend") + engine = binding.get("engine", "chromium") + if backend != expected_backend: + raise ValueError("acceptance run used a different execution backend") + runtime_identity_key, runtime_binding = runtime_identity(binding) + request_bridge = request_bridge_identity(binding) + binary_sha256 = runtime_binding.get("binary_sha256") + remote_browser = runtime_binding.get("remote_browser") + participant = ( + engine, + backend, + binding.get("site_map_sha256"), + ( + json.dumps(remote_browser, sort_keys=True, separators=(",", ":")) + if remote_browser is not None + else None + ), + json.dumps( + request_bridge, + sort_keys=True, + separators=(",", ":"), + ), + ) + if participant_contract is None: + participant_contract = participant + elif participant != participant_contract: + raise ValueError("acceptance runs mixed engine, backend, or deployment identities") + if isinstance(binary_sha256, str): + binary_sha256s.add(binary_sha256) + elif isinstance(remote_browser, dict): + remote_browser_identities[runtime_identity_key] = remote_browser + segment_contract = ( + binding.get("catalog_sha256"), + binding.get("portable_suite_sha256"), + binding.get("site_reset_receipt_sha256"), + runtime_identity_key, + binding.get("runner_revision"), + binding.get("cowagent_revision"), + binding.get("action_timeout_ms"), + binding.get("corrections_sha256"), + binding.get("webarena_revision") or binding.get("evaluator_fixes_sha256"), + binding.get("action_overrides_sha256"), + request_bridge, + ) + if kind == "segment": + if sweep_contract is None: + sweep_contract = segment_contract + elif segment_contract != sweep_contract: + raise ValueError("ordered sweep changed its execution contract") + preparation_sha256s.add(binding["site_reset_receipt_sha256"]) + run_record = { + "kind": kind, + "inputs_sha256": file_sha256(directory / "inputs.json"), + "progress_sha256": file_sha256(directory / "progress.jsonl"), + "catalog_sha256": binding["catalog_sha256"], + "site_reset_receipt_sha256": binding["site_reset_receipt_sha256"], + "site_map_sha256": binding.get("site_map_sha256"), + "runner_revision": binding["runner_revision"], + "task_ids": [row["task_id"] for row in rows], + } + if binding["schema"] in EXECUTION_CONTRACT_RUN_SCHEMAS: + run_record.update( + { + "engine": engine, + "backend": backend, + "environment_profile_sha256": binding["environment_profile_sha256"], + "action_timeout_ms": binding["action_timeout_ms"], + "corrections_sha256": binding["corrections_sha256"], + } + ) + run_record.update(runtime_binding) + if request_bridge is not None: + run_record["request_bridge"] = request_bridge + evaluator_identity = _evaluator_identity_field(binding) + run_record[evaluator_identity] = binding[evaluator_identity] + if "action_overrides_sha256" in binding: + run_record["action_overrides_sha256"] = binding["action_overrides_sha256"] + bindings.append(run_record) + run_inputs_sha256 = run_record["inputs_sha256"] + for row in rows: + task_id = row["task_id"] + if kind == "segment": + ordered_ids.append(task_id) + if kind == "replacement": + first_replacement = task_id not in replacement_seen + valid_first = ( + first_replacement + and states[task_id]["attempts"] + and states[task_id]["attempts"][-1]["status"] == "pass" + and row.get("executed_actions_sha256") + != states[task_id]["action_sha256s"][-1] + ) + valid_repair = ( + not first_replacement + and states[task_id]["attempts"][-1]["status"] != "pass" + and row.get("executed_actions_sha256") + == states[task_id]["action_sha256s"][-1] + ) + if ( + task_id not in superseded_requested + or not row.get("executed_actions_sha256") + or not (valid_first or valid_repair) + ): + raise ValueError( + f"task {task_id} is not a valid changed-catalog replacement" + ) + attempt, physical = _verified_attempt( + directory, + row, + expected_data_checksum=data_checksum, + benchmark=benchmark, + corrections=corrections, + run_inputs_sha256=run_inputs_sha256, + run_revision=run_revision, + run_evaluator_fixes_sha256=run_evaluator_fixes_sha256, + evaluator_transition=evaluator_transition, + transition_receipt=transition_receipt, + runtime_binding=runtime_binding, + ) + if "evaluator_transition" in attempt: + used_transition_rows.add( + ( + task_id, + attempt["evaluator_transition"][ + "old_evaluation_sha256" + ], + run_inputs_sha256, + ) + ) + physical_attempts += int(physical) + states[task_id]["attempts"].append(attempt) + states[task_id]["action_sha256s"].append( + row.get("executed_actions_sha256") + ) + replacement_seen.add(task_id) + continue + if task_id in superseded_requested: + if kind == "repair" and ( + not states[task_id]["attempts"] + or states[task_id]["attempts"][-1]["status"] == "pass" + ): + raise ValueError( + f"task {task_id} superseded repair does not follow a failure" + ) + attempt, physical = _superseded_attempt( + directory, row, runtime_binding=runtime_binding + ) + if row.get("status") == "pass" and evaluator_transition is not None: + source = compatible_source( + evaluator_transition, + revision=run_revision, + evaluator_fixes_sha256=run_evaluator_fixes_sha256, + ) + if source is not None: + evaluation_path = verify_physical_evidence(directory, row) + if evaluation_path is None: + raise ValueError( + f"task {task_id} pass lacks evaluation evidence" + ) + evaluation = json.loads( + evaluation_path.read_text(encoding="utf-8") + ) + if ( + evaluation.get( + "webarena_verified_evaluator_checksum" + ) + != source["evaluator_checksum"] + or evaluation.get("webarena_verified_data_checksum") + != source["data_checksum"] + ): + raise ValueError( + f"task {task_id} has an unexpected old evaluator" + ) + expected_transition_exclusions.add( + ( + task_id, + file_sha256(evaluation_path), + run_inputs_sha256, + ) + ) + physical_attempts += int(physical) + states[task_id]["attempts"].append(attempt) + states[task_id]["action_sha256s"].append( + row.get("executed_actions_sha256") + ) + continue + if kind == "repair" and task_id in excluded_requested: + if ( + not states[task_id]["attempts"] + or states[task_id]["attempts"][-1]["status"] != "pass" + ): + raise ValueError(f"task {task_id} is not a duplicate pass") + if task_id in excluded_seen: + raise ValueError(f"task {task_id} was excluded more than once") + excluded_seen.add(task_id) + attempt, physical = _verified_attempt( + directory, + row, + expected_data_checksum=data_checksum, + benchmark=benchmark, + corrections=corrections, + run_inputs_sha256=run_inputs_sha256, + run_revision=run_revision, + run_evaluator_fixes_sha256=run_evaluator_fixes_sha256, + evaluator_transition=evaluator_transition, + transition_receipt=transition_receipt, + runtime_binding=runtime_binding, + ) + if "evaluator_transition" in attempt: + used_transition_rows.add( + ( + task_id, + attempt["evaluator_transition"]["old_evaluation_sha256"], + run_inputs_sha256, + ) + ) + physical_attempts += int(physical) + excluded_attempts.append( + { + "reason": "duplicate_execution_after_prior_pass", + "run_inputs_sha256": file_sha256(directory / "inputs.json"), + **attempt, + } + ) + continue + if kind == "repair" and ( + not states[task_id]["attempts"] + or states[task_id]["attempts"][-1]["status"] == "pass" + ): + raise ValueError(f"task {task_id} repair does not follow a failure") + if ( + states[task_id]["attempts"] + and states[task_id]["attempts"][-1]["status"] == "pass" + ): + raise ValueError(f"task {task_id} repeated after passing") + attempt, physical = _verified_attempt( + directory, + row, + expected_data_checksum=data_checksum, + benchmark=benchmark, + corrections=corrections, + run_inputs_sha256=run_inputs_sha256, + run_revision=run_revision, + run_evaluator_fixes_sha256=run_evaluator_fixes_sha256, + evaluator_transition=evaluator_transition, + transition_receipt=transition_receipt, + runtime_binding=runtime_binding, + ) + if "evaluator_transition" in attempt: + used_transition_rows.add( + ( + task_id, + attempt["evaluator_transition"]["old_evaluation_sha256"], + run_inputs_sha256, + ) + ) + physical_attempts += int(physical) + states[task_id]["attempts"].append(attempt) + states[task_id]["action_sha256s"].append( + row.get("executed_actions_sha256") + ) + if excluded_seen != excluded_requested: + raise ValueError("declared duplicate repair exclusions were not found") + if set(override_sources) - used_override_sources: + raise ValueError("external action overrides are not bound to an acceptance run") + if transition_exclusions != expected_transition_exclusions: + raise ValueError("transition exclusions differ from superseded pass evidence") + if transition_receipt is not None and used_transition_rows != set(transition_receipt): + raise ValueError("evaluator transition receipt contains unused or missing pass evidence") + if ordered_ids != ids: + raise ValueError("acceptance sweep is incomplete or changed official order") + if replacement_seen != superseded_requested: + raise ValueError("catalog replacement scope is incomplete") + rows = [ + { + "task_id": task_id, + "status": states[task_id]["attempts"][-1]["status"], + "attempts": states[task_id]["attempts"], + } + for task_id in ids + ] + result = { + "schema": ACCEPTANCE_SCHEMA, + "scope_sha256": file_sha256(scope_path), + "site_reset_receipt_sha256": sweep_contract[2], + "site_preparation_receipt_sha256s": sorted(preparation_sha256s), + "binary_sha256s": sorted(binary_sha256s), + "runs": bindings, + "excluded_attempts": excluded_attempts, + "superseded_catalog_tasks": sorted(superseded_requested), + "counts": { + "total": len(ids), + "pass": sum(row["status"] == "pass" for row in rows), + "unresolved": sum(row["status"] != "pass" for row in rows), + "physical_attempts": physical_attempts, + "excluded_duplicate_attempts": len(excluded_attempts), + }, + "rows": rows, + } + if schemas and schemas <= EXECUTION_CONTRACT_RUN_SCHEMAS: + result["participant"] = { + "engine": participant_contract[0], + "backend": participant_contract[1], + "site_map_sha256": participant_contract[2], + **( + {"binary_sha256": next(iter(binary_sha256s))} + if len(binary_sha256s) == 1 + else {"binary_sha256s": sorted(binary_sha256s)} + if binary_sha256s + else { + "remote_browser": next(iter(remote_browser_identities.values())) + } + ), + **( + {"request_bridge": json.loads(participant_contract[4])} + if participant_contract[4] != "null" + else {} + ), + } + if len(binary_sha256s) == 1: + result["binary_sha256"] = next(iter(binary_sha256s)) + if remote_browser_identities: + result["remote_browsers"] = list(remote_browser_identities.values()) + if used_override_sources: + result["action_overrides_sha256s"] = sorted(used_override_sources) + if evaluator_transition_path is not None and evaluator_transition_receipt_path is not None: + result["evaluator_transition"] = { + "authority_sha256": file_sha256(evaluator_transition_path), + "receipt_sha256": file_sha256(evaluator_transition_receipt_path), + "reevaluated_passes": len(used_transition_rows), + } + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--scope", + type=Path, + default=Path("evaluation/trajectory-replay/webarena-verified-full/scope.json"), + ) + parser.add_argument("--segment", type=Path, action="append", required=True) + parser.add_argument("--repair", type=Path, action="append", default=[]) + parser.add_argument("--replacement", type=Path, action="append", default=[]) + parser.add_argument("--supersede-catalog-task", type=int, action="append", default=[]) + parser.add_argument("--action-overrides", type=Path, action="append", default=[]) + parser.add_argument("--site-map", type=Path) + parser.add_argument("--evaluator-transition", type=Path) + parser.add_argument("--evaluator-transition-receipt", type=Path) + parser.add_argument("--exclude-duplicate-repair-task", type=int, action="append", default=[]) + parser.add_argument( + "--expected-backend", choices=("browsertool", "playwright"), default="browsertool" + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--require-all-pass", action="store_true") + args = parser.parse_args() + result = audit( + args.scope, + args.segment, + args.repair, + replacement_dirs=args.replacement, + superseded_catalog_tasks=set(args.supersede_catalog_task), + excluded_duplicate_repair_tasks=set(args.exclude_duplicate_repair_task), + expected_backend=args.expected_backend, + action_overrides_paths=args.action_overrides, + site_map_path=args.site_map, + evaluator_transition_path=args.evaluator_transition, + evaluator_transition_receipt_path=args.evaluator_transition_receipt, + ) + if args.require_all_pass and result["counts"]["unresolved"]: + raise ValueError("the acceptance sweep has unresolved cases") + if args.output.exists(): + raise ValueError("acceptance ledger output already exists") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print(json.dumps(result["counts"])) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/catalog_action_audit.py b/harness/trajectory_replay/catalog_action_audit.py new file mode 100644 index 000000000..bac6b4a11 --- /dev/null +++ b/harness/trajectory_replay/catalog_action_audit.py @@ -0,0 +1,165 @@ +"""Audit catalog evaluate actions that require explicit function-body semantics.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from harness.trajectory_replay.catalog_contracts import file_sha256, load_action_overrides +from harness.trajectory_replay.public_webarena.suite import load_suite +from harness.trajectory_replay.webarena_replay import _json_sha256 + +AUDIT_SCHEMA = "lex_webarena_replay_action_override_audit/1" + +_NODE_AUDIT = r""" +const vm = require("vm"); +const rows = JSON.parse(require("fs").readFileSync(0, "utf8")); +const findings = []; +const invalid = []; +for (const row of rows) { + try { + new vm.Script(row.script); + continue; + } catch (scriptError) { + try { + new vm.Script(`(async () => {\n${row.script}\n})`); + if (row.script_mode === "function_body") continue; + findings.push({ + task_id: row.task_id, + mode: row.mode, + action_index: row.action_index, + precondition_action_sha256: row.precondition_action_sha256, + }); + } catch (bodyError) { + invalid.push({ + task_id: row.task_id, + action_index: row.action_index, + script_error: String(scriptError.message), + function_body_error: String(bodyError.message), + }); + } + } +} +process.stdout.write(JSON.stringify({findings, invalid})); +""" + + +def _evaluate_actions(catalog: dict, suite_path: Path) -> list[dict]: + catalog_rows = catalog.get("rows") or [] + if any(row.get("mode") == "portable" for row in catalog_rows): + suite = load_suite(suite_path) + portable = { + int(task.task_id.removeprefix("webarena-")): list(task.tool_calls) + for task in suite.tasks + } + else: + portable = {} + rows = [] + for catalog_row in catalog_rows: + task_id = catalog_row["task_id"] + mode = catalog_row.get("mode") + if mode == "portable": + actions = portable[task_id] + elif mode == "direct": + actions = catalog_row.get("source_actions") or [] + else: + raise ValueError(f"task {task_id}: unknown catalog mode") + for action_index, action in enumerate(actions, 1): + if action.get("action") != "evaluate": + continue + script = action.get("script") + if not isinstance(script, str) or not script.strip(): + raise ValueError(f"task {task_id}: evaluate action has no script") + rows.append( + { + "task_id": task_id, + "mode": mode, + "action_index": action_index, + "precondition_action_sha256": _json_sha256(action), + "script_mode": action.get("script_mode"), + "script": script, + } + ) + return rows + + +def scan_function_body_actions( + catalog_path: Path, suite_path: Path, *, node_binary: str = "node" +) -> dict: + """Parse every source evaluate action under Script and async function-body grammars.""" + + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + evaluate_actions = _evaluate_actions(catalog, suite_path) + process = subprocess.run( + [node_binary, "-e", _NODE_AUDIT], + input=json.dumps(evaluate_actions, ensure_ascii=False), + text=True, + capture_output=True, + check=False, + ) + if process.returncode: + raise RuntimeError(f"JavaScript action audit failed: {process.stderr.strip()}") + result = json.loads(process.stdout) + if result.get("invalid"): + raise ValueError(f"evaluate actions are invalid under both grammars: {result['invalid']}") + return { + "schema": AUDIT_SCHEMA, + "catalog_sha256": file_sha256(catalog_path), + "portable_suite_sha256": file_sha256(suite_path), + "evaluate_action_count": len(evaluate_actions), + "function_body_findings": result["findings"], + } + + +def validate_function_body_overrides(audit: dict, action_overrides_path: Path) -> None: + """Require one exact script_mode amendment for every parser finding and no extras.""" + + if audit.get("schema") != AUDIT_SCHEMA: + raise ValueError("action override audit schema is invalid") + override_map, _digest = load_action_overrides(action_overrides_path) + finding_modes = _finding_modes(audit) + actual = [ + { + "task_id": amendment["task_id"], + "mode": finding_modes.get(amendment["task_id"], {}).get(amendment["action_index"]), + "action_index": amendment["action_index"], + "precondition_action_sha256": amendment["precondition_action_sha256"], + } + for amendments in override_map.values() + for amendment in amendments + if amendment["set"].get("script_mode") == "function_body" + ] + if actual != audit.get("function_body_findings"): + raise ValueError("function-body action overrides differ from the complete static audit") + + +def validate_catalog_action_audit( + audit_path: Path, + catalog_path: Path, + suite_path: Path, + action_overrides_path: Path, +) -> dict: + """Re-run the parser audit at release and bind it to the exact frozen inputs.""" + + if audit_path.is_symlink() or not audit_path.is_file(): + raise ValueError("action override audit must be a regular file") + audit = json.loads(audit_path.read_text(encoding="utf-8")) + if ( + audit.get("schema") != AUDIT_SCHEMA + or audit.get("catalog_sha256") != file_sha256(catalog_path) + or audit.get("portable_suite_sha256") != file_sha256(suite_path) + ): + raise ValueError("action override audit changed its catalog or suite binding") + live = scan_function_body_actions(catalog_path, suite_path) + if live != audit: + raise ValueError("action override audit differs from the live catalog scan") + validate_function_body_overrides(audit, action_overrides_path) + return audit + + +def _finding_modes(audit: dict) -> dict[int, dict[int, str]]: + modes: dict[int, dict[int, str]] = {} + for finding in audit.get("function_body_findings") or []: + modes.setdefault(finding["task_id"], {})[finding["action_index"]] = finding["mode"] + return modes diff --git a/harness/trajectory_replay/catalog_binding.py b/harness/trajectory_replay/catalog_binding.py new file mode 100644 index 000000000..f32e835b6 --- /dev/null +++ b/harness/trajectory_replay/catalog_binding.py @@ -0,0 +1,175 @@ +"""Bind a frozen WebArena action catalog to another site deployment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from harness.trajectory_replay.catalog_contracts import ( + ACTION_OVERRIDES_SCHEMA, + TRUSTED_RECORDED_HOSTS, + apply_action_overrides, + catalog_action_overrides, + file_sha256, + load_action_overrides, + load_catalog_inputs, + row_source_sites, +) +from harness.trajectory_replay.public_webarena.suite import load_suite +from harness.trajectory_replay.remote_runtime import load_remote_sites, rebind_remote_calls +from harness.trajectory_replay.webarena_replay import _json_sha256 + + +def rebind_catalog( + catalog_path: Path, + suite_path: Path, + site_map_path: Path, + *, + revised_task_ids: set[int] | None = None, + source_site_map_path: Path | None = None, + action_overrides_path: Path | None = None, + revise_action_overrides: bool = False, +) -> dict: + if revise_action_overrides and action_overrides_path is None: + raise ValueError("action override revision requires an overrides source") + source = json.loads(catalog_path.read_text(encoding="utf-8")) + suite = load_suite(suite_path) + tasks = {int(task.task_id.removeprefix("webarena-")): task for task in suite.tasks} + sites = load_remote_sites(site_map_path) + target = dict(source) + target["predecessor_catalog_sha256"] = file_sha256(catalog_path) + target["site_map_sha256"] = file_sha256(site_map_path) + target["portable_suite_sha256"] = file_sha256(suite_path) + if action_overrides_path is not None: + override_map, override_sha256 = load_action_overrides(action_overrides_path) + existing_sha256 = source.get("action_overrides_sha256") + has_existing_overrides = "action_overrides_sha256" in source or any( + row.get("action_overrides") is not None for row in source.get("rows") or [] + ) + if ( + has_existing_overrides + and existing_sha256 != override_sha256 + and not revise_action_overrides + ): + raise ValueError("action overrides differ from the predecessor catalog") + if has_existing_overrides and existing_sha256 == override_sha256: + embedded = { + row["task_id"]: amendments + for row in source.get("rows") or [] + if (amendments := catalog_action_overrides(source, row)) + } + if embedded != override_map: + raise ValueError("action overrides differ from the predecessor catalog rows") + target["action_overrides_sha256"] = override_sha256 + else: + if "action_overrides_sha256" in source or any( + row.get("action_overrides") is not None for row in source.get("rows") or [] + ): + raise ValueError("catalog rebind requires its external action overrides source") + override_map = {} + if source_site_map_path is not None: + if file_sha256(source_site_map_path) != source["site_map_sha256"]: + raise ValueError("source site map differs from the predecessor catalog") + target["source_site_urls"] = { + site.value: url for site, url in load_remote_sites(source_site_map_path).items() + } + target["rows"] = [] + revised = set(revised_task_ids or ()) + if revised and target["site_map_sha256"] != source["site_map_sha256"]: + raise ValueError("action revisions must keep the site map fixed") + for original in source["rows"]: + row = dict(original) + task_id = row["task_id"] + actions = ( + list(tasks[task_id].tool_calls) if row["mode"] == "portable" else row["source_actions"] + ) + amendments = override_map.get(task_id, []) + actions = apply_action_overrides(actions, amendments, task_id=task_id) + if amendments: + row["action_overrides"] = { + "schema": ACTION_OVERRIDES_SCHEMA, + "source_sha256": target["action_overrides_sha256"], + "amendments_sha256": _json_sha256(amendments), + "amendments": amendments, + } + else: + row.pop("action_overrides", None) + row["bound_calls_sha256"] = _json_sha256( + rebind_remote_calls( + actions, sites, set(TRUSTED_RECORDED_HOSTS), row_source_sites(target, row) + ) + ) + if task_id in revised: + if ( + row["mode"] != "portable" + or row["bound_calls_sha256"] == original["bound_calls_sha256"] + ): + raise ValueError(f"task {task_id} has no revised portable actions") + row["source_kind"] = "revised_portable_suite" + row["source_suite_sha256"] = file_sha256(suite_path) + row.pop("source_path", None) + target["rows"].append(row) + if revised - {row["task_id"] for row in target["rows"]}: + raise ValueError("revised task IDs are outside the catalog") + if set(override_map) - {row["task_id"] for row in target["rows"]}: + raise ValueError("action overrides contain tasks outside the catalog") + if revised: + target["repairs"] = list(source.get("repairs") or []) + [ + { + "kind": "portable_action_revision", + "task_ids": sorted(revised), + "suite_sha256": file_sha256(suite_path), + } + ] + if revise_action_overrides: + if not has_existing_overrides or existing_sha256 == target["action_overrides_sha256"]: + raise ValueError("action override revision must replace an existing source") + target["repairs"] = list(target.get("repairs") or []) + [ + { + "kind": "action_override_revision", + "predecessor_action_overrides_sha256": existing_sha256, + "action_overrides_sha256": target["action_overrides_sha256"], + "task_ids": sorted(override_map), + } + ] + return target + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, required=True) + parser.add_argument("--portable-suite", type=Path, required=True) + parser.add_argument("--site-map", type=Path, required=True) + parser.add_argument("--source-site-map", type=Path) + parser.add_argument("--action-overrides", type=Path) + parser.add_argument("--revise-action-overrides", action="store_true") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--revised-task-id", type=int, action="append", default=[]) + args = parser.parse_args() + if args.output.exists(): + raise FileExistsError(args.output) + result = rebind_catalog( + args.catalog, + args.portable_suite, + args.site_map, + revised_task_ids=set(args.revised_task_id), + source_site_map_path=args.source_site_map, + action_overrides_path=args.action_overrides, + revise_action_overrides=args.revise_action_overrides, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + load_catalog_inputs( + args.output, + args.portable_suite, + args.site_map, + action_overrides_path=args.action_overrides, + ) + print(json.dumps({"tasks": len(result["rows"]), "catalog_sha256": file_sha256(args.output)})) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/catalog_comparison.py b/harness/trajectory_replay/catalog_comparison.py new file mode 100644 index 000000000..4bc598e85 --- /dev/null +++ b/harness/trajectory_replay/catalog_comparison.py @@ -0,0 +1,378 @@ +"""Compile full-catalog replay evidence for the shared comparison report.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from harness.trajectory_replay.catalog_contracts import ( + TRUSTED_RECORDED_HOSTS, + apply_action_overrides, + catalog_action_overrides, + file_sha256, + load_catalog_inputs, + row_source_sites, +) +from harness.trajectory_replay.remote_runtime import rebind_remote_calls, remote_benchmark +from harness.trajectory_replay.runtime_identity import ( + assert_observed_runtime, + request_bridge_identity, + runtime_identities, + runtime_identity, +) +from harness.trajectory_replay.webarena_corrections import apply_correction, load_corrections +from harness.trajectory_replay.webarena_replay import _json_sha256 + +RESULTS_SCHEMA = "lex_webarena_catalog_comparison_results/1" +DATASET_SCHEMA = "lex_webarena_catalog_comparison_dataset/1" + +_EVIDENCE_FILENAMES = { + "network_har": "network.har", + "acceptance": "acceptance.json", + "actual_response": "actual_response.json", + "evaluation": "evaluation.json", + "failure_screenshot": "failure.png", + "trace": "trace.json", + "readiness": "readiness.json", + "outputs": "outputs.json", +} + + +def _read(path: Path) -> dict[str, Any]: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"missing or symlinked comparison input: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _artifact(path: Path, digest: str) -> dict[str, str]: + if path.is_symlink() or not path.is_file() or file_sha256(path) != digest: + raise ValueError(f"catalog evidence is missing or changed: {path}") + return {"path": str(path.resolve()), "sha256": digest} + + +def build_catalog_comparison_dataset( + *, + catalog_path: Path, + suite_path: Path, + site_map_path: Path, + action_overrides_path: Path | None = None, +) -> dict[str, Any]: + """Materialize all catalog tasks under the same contracts used by execution.""" + + catalog, sites, portable = load_catalog_inputs( + catalog_path, + suite_path, + site_map_path, + action_overrides_path=action_overrides_path, + ) + benchmark = remote_benchmark(sites) + corrections = load_corrections() + trajectories = [] + for frozen in catalog["rows"]: + task_id = int(frozen["task_id"]) + official = benchmark.get_task(task_id) + effective = ( + apply_correction(official, corrections).task + if task_id in corrections.records + else official + ) + contract = effective.model_dump(mode="json") + source_actions = ( + list(portable[task_id].tool_calls) + if frozen["mode"] == "portable" + else list(frozen["source_actions"]) + ) + source_actions = apply_action_overrides( + source_actions, + catalog_action_overrides(catalog, frozen), + task_id=task_id, + ) + actions = rebind_remote_calls( + source_actions, + sites, + set(TRUSTED_RECORDED_HOSTS), + row_source_sites(catalog, frozen), + ) + if _json_sha256(actions) != frozen["bound_calls_sha256"]: + raise ValueError(f"task {task_id}: report actions differ from executed catalog") + task = { + key: contract[key] + for key in ( + "intent", + "intent_template_id", + "revision", + "sites", + "start_urls", + ) + if key in contract + } + trajectories.append( + { + "id": f"webarena-verified-full-{task_id}", + "task_id": task_id, + "task": task, + "tool_calls": actions, + "evaluation_input": contract["eval"][0]["expected"], + "source": { + "kind": frozen.get("source_kind"), + "catalog_row_sha256": _json_sha256(frozen), + }, + "verification": { + "contract": {"kind": "official_or_reviewed_correction"}, + "model_calls": 0, + }, + } + ) + return { + "schema": DATASET_SCHEMA, + "benchmark": "WebArena-Verified", + "selection_sha256": file_sha256(catalog_path), + "counts": { + "verified_tasks": len(trajectories), + "verified_trajectories": len(trajectories), + }, + "trajectories": trajectories, + } + + +def _progress_index(evidence_root: Path) -> dict[str, tuple[Path, dict[int, dict[str, Any]]]]: + index: dict[str, tuple[Path, dict[int, dict[str, Any]]]] = {} + for progress in evidence_root.rglob("progress.jsonl"): + if progress.is_symlink() or progress.parent.is_symlink(): + raise ValueError(f"catalog progress is symlinked: {progress}") + digest = file_sha256(progress) + rows = [ + json.loads(line) + for line in progress.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + by_task = {int(row["task_id"]): row for row in rows} + if len(by_task) != len(rows): + raise ValueError(f"catalog progress contains duplicate task ids: {progress}") + if digest in index and index[digest][0] != progress.parent: + raise ValueError(f"catalog progress digest is ambiguous: {digest}") + index[digest] = (progress.parent, by_task) + return index + + +def _site_reset_index(evidence_root: Path) -> dict[str, dict[str, str]]: + index = {} + for path in evidence_root.rglob("*.json"): + name = path.name.lower() + if ( + "reset" not in name + and "receipt" not in name + or path.is_symlink() + or path.parent.is_symlink() + ): + continue + digest = file_sha256(path) + ref = _artifact(path, digest) + if digest in index and index[digest] != ref: + raise ValueError(f"site reset receipt digest is ambiguous: {digest}") + index[digest] = ref + return index + + +def _attempt( + *, + participant: dict[str, Any], + task: dict[str, Any], + summary: dict[str, Any], + progress_index: dict[str, tuple[Path, dict[int, dict[str, Any]]]], + site_resets: dict[str, dict[str, str]], + ledger_runtimes: dict[str, dict[str, object]], + ledger_request_bridge: dict[str, object] | None, +) -> dict[str, Any] | None: + if summary.get("status") == "candidate_setup_failure": + return None + task_id = int(summary["task_id"]) + progress_sha256 = str(summary["progress_sha256"]) + try: + run_dir, rows = progress_index[progress_sha256] + physical = rows[task_id] + except KeyError as error: + raise ValueError(f"task {task_id}: bound physical result is missing") from error + for key in ("status", "score"): + if physical.get(key) != summary.get(key): + raise ValueError(f"task {task_id}: acceptance summary differs from physical result") + inputs_path = run_dir / "inputs.json" + inputs = _read(inputs_path) + if inputs.get("engine") != participant["engine"]: + raise ValueError(f"task {task_id}: physical browser identity changed") + if request_bridge_identity(inputs) != ledger_request_bridge: + raise ValueError(f"task {task_id}: physical request bridge identity changed") + identity_key, physical_runtime = runtime_identity(inputs) + if ledger_runtimes.get(identity_key) != physical_runtime: + raise ValueError(f"task {task_id}: physical browser runtime identity changed") + assert_observed_runtime( + physical_runtime, physical.get("runtime_metadata"), task_id=task_id + ) + binary_sha256 = physical_runtime.get("binary_sha256") + remote_browser = physical_runtime.get("remote_browser") + actions = task["tool_calls"] + uses_current_actions = ( + physical.get("executed_actions_sha256") == _json_sha256(actions) + and physical.get("action_count") == len(actions) + ) + evidence = {} + task_dir = run_dir / f"task-{task_id}" + for name, digest in (physical.get("evidence_sha256") or {}).items(): + filename = _EVIDENCE_FILENAMES.get(name) + if filename is None: + raise ValueError(f"task {task_id}: unknown evidence kind {name}") + evidence[name] = _artifact(task_dir / filename, digest) + try: + evidence["site_reset"] = site_resets[inputs["site_reset_receipt_sha256"]] + except KeyError as error: + raise ValueError(f"task {task_id}: run site reset receipt is missing") from error + evaluation_exists = "evaluation" in evidence + error = str(physical.get("error") or "") + process_status = "fail" if error else "pass" + failure_stage = physical.get("failure_stage") + if error and not failure_stage: + failure_stage = ( + "browser_session_setup" + if "_login_context" in error + else "browser_tool_execution" + ) + evaluator_status = ( + "pass" + if physical.get("status") == "pass" + else "fail" + if evaluation_exists + else "not_run" + ) + contract_sha256 = _json_sha256( + {"task": task["task"], "evaluation_input": task["evaluation_input"]} + ) + reference_validity = { + "status": "valid", + "reason": ( + "The hash-bound official task and evaluator contract are retained independently " + "of this browser outcome." + ), + "source_sha256": contract_sha256, + } + return { + "participant_id": participant["id"], + "engine": participant["engine"], + "task_id": task_id, + "task_contract_sha256": contract_sha256, + **( + { + "planned_actions": actions, + "planned_actions_sha256": _json_sha256(actions), + } + if uses_current_actions + else { + "historical_planned_actions_sha256": physical.get( + "executed_actions_sha256" + ) + } + ), + "status": physical["status"], + "official_rule_score": physical.get("score"), + "action_count": physical["action_count"], + "completed_action_count": physical["completed_actions"], + "duration_ms": physical["duration_ms"], + "process_status": process_status, + "evaluator_status": evaluator_status, + "error": error, + **({"failure_stage": failure_stage} if failure_stage else {}), + "binary_sha256": binary_sha256, + **({"remote_browser": remote_browser} if remote_browser is not None else {}), + **( + {"request_bridge": inputs["request_bridge"]} + if inputs.get("request_bridge") is not None + else {} + ), + **( + {"runtime_metadata": physical["runtime_metadata"]} + if isinstance(physical.get("runtime_metadata"), dict) + else {} + ), + "cohort_receipt_sha256": progress_sha256, + "evidence": evidence, + "reference_validity": reference_validity, + } + + +def build_catalog_comparison_results( + *, + dataset: dict[str, Any], + dataset_sha256: str, + profile_sha256: str, + scope_sha256: str, + site_map_sha256: str, + participants: list[dict[str, Any]], +) -> dict[str, Any]: + """Build one complete logical matrix and retain every valid physical call.""" + + tasks = {int(row["task_id"]): row for row in dataset["trajectories"]} + attempts = [] + physical_executions = [] + seen_participants = set() + for participant in participants: + participant_id = participant.get("id") + if not participant_id or participant_id in seen_participants: + raise ValueError("catalog comparison participant ids must be unique") + seen_participants.add(participant_id) + ledger_path = Path(participant["ledger_path"]).expanduser().resolve() + evidence_root = Path(participant["evidence_root"]).expanduser().resolve() + ledger = _read(ledger_path) + if [int(row["task_id"]) for row in ledger.get("rows") or []] != list(tasks): + raise ValueError(f"{participant_id}: acceptance ledger task order changed") + ledger_participant = ledger.get("participant") or {} + if ledger_participant.get("engine") != participant["engine"]: + raise ValueError(f"{participant_id}: acceptance ledger browser identity changed") + ledger_runtimes = runtime_identities(ledger_participant) + ledger_request_bridge = request_bridge_identity(ledger_participant) + index = _progress_index(evidence_root) + site_resets = _site_reset_index(evidence_root) + if ledger.get("site_reset_receipt_sha256") not in site_resets: + raise ValueError(f"{participant_id}: initial site reset receipt is missing") + for accepted in ledger["rows"]: + task_id = int(accepted["task_id"]) + physical_for_task = [] + for summary in accepted.get("attempts") or []: + row = _attempt( + participant=participant, + task=tasks[task_id], + summary={"task_id": task_id, **summary}, + progress_index=index, + site_resets=site_resets, + ledger_runtimes=ledger_runtimes, + ledger_request_bridge=ledger_request_bridge, + ) + if row is not None: + physical_for_task.append(row) + physical_executions.append(row) + if not physical_for_task: + raise ValueError(f"{participant_id}/{task_id}: no physical outcome") + final = physical_for_task[-1] + if "planned_actions" not in final: + raise ValueError( + f"{participant_id}/{task_id}: final result used another catalog action set" + ) + if final["status"] != accepted["status"]: + raise ValueError(f"{participant_id}/{task_id}: final logical outcome changed") + attempts.append(final) + expected = {(participant["id"], task_id) for participant in participants for task_id in tasks} + actual = {(row["participant_id"], row["task_id"]) for row in attempts} + if actual != expected or len(attempts) != len(expected): + raise ValueError("catalog comparison matrix is incomplete") + return { + "schema": RESULTS_SCHEMA, + "dataset_sha256": dataset_sha256, + "profile_sha256": profile_sha256, + "selection_sha256": scope_sha256, + "reference_validity_sha256": hashlib.sha256( + "".join(sorted(row["task_contract_sha256"] for row in attempts)).encode() + ).hexdigest(), + "site_map_sha256": site_map_sha256, + "attempts": attempts, + "physical_executions": physical_executions, + } diff --git a/harness/trajectory_replay/catalog_consolidation.py b/harness/trajectory_replay/catalog_consolidation.py new file mode 100644 index 000000000..b2798ac45 --- /dev/null +++ b/harness/trajectory_replay/catalog_consolidation.py @@ -0,0 +1,336 @@ +"""Freeze accepted WebArena trajectories into one deployment-bound catalog.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from harness.trajectory_replay.catalog_contracts import ( + ACCEPTANCE_SCHEMAS, + ACTION_OVERRIDES_SCHEMA, + CATALOG_SCHEMA, + TRUSTED_RECORDED_HOSTS, + apply_action_overrides, + catalog_action_overrides, + file_sha256, + load_action_overrides, + load_catalog_inputs, + row_source_sites, +) +from harness.trajectory_replay.public_webarena.suite import load_suite +from harness.trajectory_replay.remote_runtime import load_remote_sites, rebind_remote_calls +from harness.trajectory_replay.webarena_replay import _json_sha256 +from harness.trajectory_replay.webarena_runtime_source import verify_webarena_runtime_source + + +def _source_files(roots: list[Path], expected: set[str]) -> dict[str, Path]: + found: dict[str, Path] = {} + for root in roots: + for path in root.rglob("*.json"): + if path.is_symlink() or path.parent.is_symlink(): + raise ValueError(f"source file is a symlink: {path}") + if not any(word in path.name for word in ("catalog", "suite", "sites")): + continue + digest = file_sha256(path) + if digest in expected: + found.setdefault(digest, path) + if set(found) != expected: + raise FileNotFoundError(f"frozen inputs missing: {sorted(expected - set(found))}") + return found + + +def _verified_run_directories( + ledger: dict, run_root: Path, selected_progress: set[str] +) -> dict[str, Path]: + runs = { + file_sha256(progress): progress.parent for progress in run_root.glob("*/progress.jsonl") + } + ledger_runs = {row["progress_sha256"]: row for row in ledger.get("runs") or []} + if len(ledger_runs) != len(ledger.get("runs") or []): + raise ValueError("accepted ledger contains duplicate run bindings") + if not selected_progress <= set(runs): + raise FileNotFoundError("accepted progress evidence is missing") + if not selected_progress <= set(ledger_runs): + raise ValueError("accepted attempts are not bound to ledger runs") + for digest in selected_progress: + inputs_path = runs[digest] / "inputs.json" + if file_sha256(inputs_path) != ledger_runs[digest]["inputs_sha256"]: + raise ValueError("run inputs changed after acceptance audit") + return runs + + +def _catalog_rows_by_id(catalog: dict) -> dict[int, dict]: + rows = catalog.get("rows") or [] + indexed = {row.get("task_id"): row for row in rows} + if len(indexed) != len(rows) or set(indexed) != set(catalog.get("task_ids") or []): + raise ValueError("source catalog task rows do not match its declared order") + return indexed + + +def _load_bound_action_overrides( + inputs: dict[str, dict], action_overrides_paths: list[Path] +) -> dict[str, tuple[Path, dict[int, list[dict]]]]: + sources: dict[str, tuple[Path, dict[int, list[dict]]]] = {} + for path in action_overrides_paths: + override_map, digest = load_action_overrides(path) + if digest in sources: + raise ValueError("duplicate external action overrides source") + sources[digest] = (path, override_map) + required = { + binding["action_overrides_sha256"] + for binding in inputs.values() + if "action_overrides_sha256" in binding + } + if set(sources) - required: + raise ValueError("external action overrides are not bound to an accepted run") + return sources + + +def _source_action_overrides_path( + binding: dict, + source_catalog: dict, + override_sources: dict[str, tuple[Path, dict[int, list[dict]]]], +) -> Path | None: + catalog_digest = source_catalog.get("action_overrides_sha256") + if binding.get("action_overrides_sha256") != catalog_digest: + raise ValueError("run action override binding differs from its source catalog") + if catalog_digest is None: + return None + source = override_sources.get(catalog_digest) + return source[0] if source is not None else None + + +def _target_action_amendments( + task_id: int, + source_amendments: list[dict], + target_override_map: dict[int, list[dict]], +) -> list[dict]: + target_amendments = target_override_map.get(task_id, []) + if sorted(source_amendments, key=lambda row: row["action_index"]) != sorted( + target_amendments, key=lambda row: row["action_index"] + ): + raise ValueError( + f"task {task_id}: target action overrides differ from its accepted execution" + ) + return target_amendments + + +def consolidate( + ledger_path: Path, + run_root: Path, + source_roots: list[Path], + site_map_path: Path, + action_overrides_paths: list[Path] | None = None, + target_action_overrides_path: Path | None = None, +) -> tuple[dict, dict]: + verify_webarena_runtime_source() + ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + if ( + ledger.get("schema") not in ACCEPTANCE_SCHEMAS + or ledger.get("counts", {}).get("pass") != ledger.get("counts", {}).get("total") + or ledger.get("counts", {}).get("unresolved") != 0 + or not ledger.get("rows") + or len({row.get("task_id") for row in ledger.get("rows", [])}) + != len(ledger.get("rows", [])) + ): + raise ValueError("accepted ledger must contain every declared passing task in order") + selected = {row["attempts"][-1]["progress_sha256"] for row in ledger["rows"]} + runs = _verified_run_directories(ledger, run_root, selected) + inputs = { + digest: json.loads((runs[digest] / "inputs.json").read_text(encoding="utf-8")) + for digest in selected + } + override_sources = _load_bound_action_overrides(inputs, action_overrides_paths or []) + if target_action_overrides_path is not None: + target_override_map, target_override_sha256 = load_action_overrides( + target_action_overrides_path + ) + else: + target_override_map = {} + target_override_sha256 = None + if override_sources and target_override_sha256 is None: + raise ValueError("consolidation requires an explicit target action overrides source") + expected = { + digest + for binding in inputs.values() + for digest in ( + binding["catalog_sha256"], + binding["portable_suite_sha256"], + binding["site_map_sha256"], + ) + } + files = _source_files(source_roots, expected) + catalogs = { + digest: json.loads(files[digest].read_text(encoding="utf-8")) + for digest in {binding["catalog_sha256"] for binding in inputs.values()} + } + catalog_rows = {digest: _catalog_rows_by_id(catalog) for digest, catalog in catalogs.items()} + suites = { + digest: { + int(task.task_id.removeprefix("webarena-")): task + for task in load_suite(files[digest]).tasks + } + for digest in {binding["portable_suite_sha256"] for binding in inputs.values()} + } + raw_suites = { + digest: { + int(task["id"].removeprefix("webarena-")): task + for task in json.loads(files[digest].read_text(encoding="utf-8"))["tasks"] + } + for digest in suites + } + for binding in inputs.values(): + source_catalog = catalogs[binding["catalog_sha256"]] + source_override_path = _source_action_overrides_path( + binding, source_catalog, override_sources + ) + if source_catalog.get("action_overrides_sha256") is None or source_override_path: + load_catalog_inputs( + files[binding["catalog_sha256"]], + files[binding["portable_suite_sha256"]], + files[binding["site_map_sha256"]], + action_overrides_path=source_override_path, + ) + target_sites = load_remote_sites(site_map_path) + rows: list[dict] = [] + portable_tasks: list[dict] = [] + for accepted in ledger["rows"]: + task_id = accepted["task_id"] + attempt = accepted["attempts"][-1] + run_digest = attempt["progress_sha256"] + binding = inputs[run_digest] + source_catalog = catalogs[binding["catalog_sha256"]] + if source_catalog["site_map_sha256"] != binding["site_map_sha256"]: + raise ValueError(f"task {task_id}: source deployment binding differs") + try: + source_row = catalog_rows[binding["catalog_sha256"]][task_id] + except KeyError as exc: + raise ValueError( + f"task {task_id}: accepted task is absent from source catalog" + ) from exc + if source_row["task_id"] != task_id or attempt["status"] != "pass": + raise ValueError(f"task {task_id}: accepted source row differs") + if source_row["mode"] == "portable": + suite_hash = binding["portable_suite_sha256"] + actions = list(suites[suite_hash][task_id].tool_calls) + portable_tasks.append(raw_suites[suite_hash][task_id]) + else: + actions = source_row["source_actions"] + source_amendments = catalog_action_overrides(source_catalog, source_row) + target_amendments = _target_action_amendments( + task_id, source_amendments, target_override_map + ) + actions = apply_action_overrides(actions, source_amendments, task_id=task_id) + source_sites = row_source_sites(source_catalog, source_row) + run_sites = load_remote_sites(files[binding["site_map_sha256"]]) + old_digest = _json_sha256( + rebind_remote_calls(actions, run_sites, set(TRUSTED_RECORDED_HOSTS), source_sites) + ) + progress_rows = [ + json.loads(line) + for line in (runs[run_digest] / "progress.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + progress = next(row for row in progress_rows if row["task_id"] == task_id) + if old_digest != progress["executed_actions_sha256"]: + raise ValueError(f"task {task_id}: accepted actions differ from source inputs") + # Older catalogs relied on the run's deployment map and did not list + # their source origins. Carry those origins into the new binding. + rebind_sources = {**run_sites, **source_sites} + row = dict(source_row) + if target_amendments: + row["action_overrides"] = { + "schema": ACTION_OVERRIDES_SCHEMA, + "source_sha256": target_override_sha256, + "amendments_sha256": _json_sha256(target_amendments), + "amendments": target_amendments, + } + else: + row.pop("action_overrides", None) + row["source_site_urls"] = {site.value: url for site, url in rebind_sources.items()} + row["bound_calls_sha256"] = _json_sha256( + rebind_remote_calls(actions, target_sites, set(TRUSTED_RECORDED_HOSTS), rebind_sources) + ) + row["accepted_progress_sha256"] = run_digest + rows.append(row) + suite = {"schema": "lex_browser_use_tasks/4", "tasks": portable_tasks} + catalog = { + "schema": CATALOG_SCHEMA, + "task_ids": [row["task_id"] for row in rows], + "site_map_sha256": file_sha256(site_map_path), + "portable_suite_sha256": _json_file_sha256(suite), + "accepted_ledger_sha256": file_sha256(ledger_path), + "rows": rows, + } + row_override_sources = { + row["action_overrides"]["source_sha256"] + for row in rows + if row.get("action_overrides") is not None + } + if len(row_override_sources) > 1: + raise ValueError("accepted rows mix action override sources") + if row_override_sources: + catalog["action_overrides_sha256"] = row_override_sources.pop() + embedded_overrides = { + row["task_id"]: row["action_overrides"]["amendments"] + for row in rows + if row.get("action_overrides") is not None + } + if embedded_overrides != target_override_map: + raise ValueError("accepted rows differ from the target action overrides source") + return catalog, suite + + +def _json_file_sha256(value: object) -> str: + payload = json.dumps(value, ensure_ascii=False, indent=2) + "\n" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--source-root", type=Path, action="append", required=True) + parser.add_argument("--site-map", type=Path, required=True) + parser.add_argument("--action-overrides", type=Path, action="append", default=[]) + parser.add_argument("--target-action-overrides", type=Path) + parser.add_argument("--output-catalog", type=Path, required=True) + parser.add_argument("--output-suite", type=Path, required=True) + args = parser.parse_args() + if args.output_catalog.exists() or args.output_suite.exists(): + raise FileExistsError("consolidated catalog or suite already exists") + catalog, suite = consolidate( + args.ledger, + args.run_root, + args.source_root, + args.site_map, + action_overrides_paths=args.action_overrides, + target_action_overrides_path=args.target_action_overrides, + ) + args.output_suite.parent.mkdir(parents=True, exist_ok=True) + args.output_catalog.parent.mkdir(parents=True, exist_ok=True) + args.output_suite.write_text( + json.dumps(suite, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + args.output_catalog.write_text( + json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + load_catalog_inputs( + args.output_catalog, + args.output_suite, + args.site_map, + accepted_ledger_path=args.ledger, + action_overrides_path=args.target_action_overrides, + ) + print( + json.dumps( + {"tasks": len(catalog["rows"]), "catalog_sha256": file_sha256(args.output_catalog)} + ) + ) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/catalog_contracts.py b/harness/trajectory_replay/catalog_contracts.py new file mode 100644 index 000000000..9807ba11a --- /dev/null +++ b/harness/trajectory_replay/catalog_contracts.py @@ -0,0 +1,808 @@ +"""Shared contracts for WebArena trajectory catalogs and replay evidence.""" + +from __future__ import annotations + +import json +import tomllib +from pathlib import Path +from urllib.parse import urlsplit + +from webarena_verified.core.utils.checksum import compute_evaluator_checksum +from webarena_verified.types.task import WebArenaSite + +from harness.trajectory_replay.public_webarena.suite import load_suite +from harness.trajectory_replay.remote_runtime import ( + file_sha256, + load_remote_sites, + rebind_remote_calls, +) +from harness.trajectory_replay.retrieval_acceptance import validate_contract +from harness.trajectory_replay.webarena_corrections import apply_correction +from harness.trajectory_replay.webarena_replay import _json_sha256 + +CATALOG_SCHEMA = "lex_webarena_replay_catalog/1" +CATALOG_SCHEMAS = frozenset({CATALOG_SCHEMA, "lex_webarena_full_replay_catalog/1"}) +DATASET_SCHEMA = "lex_webarena_replay_dataset/1" +DATASET_SCHEMAS = frozenset({DATASET_SCHEMA, "lex_webarena_verified_full_dataset/1"}) +SCOPE_SCHEMA = "lex_webarena_replay_scope/1" +SCOPE_SCHEMAS = frozenset({SCOPE_SCHEMA, "lex_webarena_verified_full_scope/1"}) +ENVIRONMENT_SCHEMA = "lex_webarena_replay_environment/1" +ENVIRONMENT_SCHEMAS = frozenset({ENVIRONMENT_SCHEMA, "lex_webarena_verified_full_environment/1"}) +REPORT_PROFILE_SCHEMA = "lex_webarena_replay_report_profile/1" +REPORT_PROFILE_SCHEMAS = frozenset( + {REPORT_PROFILE_SCHEMA, "lex_webarena_verified_full_report_profile/1"} +) +COMPARISON_RELEASE_SCHEMA = "lex_webarena_replay_comparison_release/1" +RUN_SCHEMA = "lex_trajectory_replay_catalog_run/2" +LEGACY_RUN_SCHEMAS = frozenset( + { + "lex_trajectory_replay_catalog_run/1", + "lex_webarena_full_replay_run/1", + "lex_webarena_full_replay_run/2", + } +) +RUN_SCHEMAS = frozenset({RUN_SCHEMA, *LEGACY_RUN_SCHEMAS}) +EXECUTION_CONTRACT_RUN_SCHEMAS = frozenset( + {RUN_SCHEMA, "lex_trajectory_replay_catalog_run/1", "lex_webarena_full_replay_run/2"} +) +ACTION_OVERRIDES_SCHEMA = "lex_webarena_replay_action_overrides/1" +_ACTION_OVERRIDE_FIELDS = frozenset( + { + "force", + "script", + "script_mode", + "selector", + "wait_state", + "wait_for_enabled", + "wait_for_selector_hidden", + } +) +_MAX_OVERRIDE_SCRIPT_BYTES = 64 * 1024 +ACCEPTANCE_SCHEMA = "lex_trajectory_replay_catalog_acceptance/1" +LEGACY_ACCEPTANCE_SCHEMAS = frozenset({"lex_webarena_full_acceptance_ledger/1"}) +ACCEPTANCE_SCHEMAS = frozenset({ACCEPTANCE_SCHEMA, *LEGACY_ACCEPTANCE_SCHEMAS}) +LEGACY_CATALOG_REPORT_SCHEMA = "lex_webarena_full_acceptance_report/3" +LEGACY_CATALOG_CONTRACTS_SCHEMA = "lex_webarena_full_contracts/1" +TRUSTED_RECORDED_HOSTS = frozenset( + { + "onestopmarket.example.com", + "gitlab.example.com", + "openstreetmap.example.com", + "wikipedia.example.com", + "ec2-3-133-227-75.us-east-2.compute.amazonaws.com", + "ec2-3-135-39-80.us-east-2.compute.amazonaws.com", + "host.docker.internal", + "192.168.5.2", + "127.0.0.1", + } +) +WEBARENA_FORK_REPOSITORY = "https://github.com/lexmount/webarena-verified.git" +WEBARENA_FORK_BRANCH = "lex-main" +ENVIRONMENT_PREPARATION_METHODS = frozenset( + { + "new_rainbond_application_from_pinned_images", + "component_restarts_from_pinned_images", + } +) + +_EVIDENCE_FILENAMES = { + "network_har": "network.har", + "acceptance": "acceptance.json", + "actual_response": "actual_response.json", + "evaluation": "evaluation.json", + "failure_screenshot": "failure.png", + "trace": "trace.json", + "readiness": "readiness.json", + "site_reset": "site-reset.json", + "outputs": "outputs.json", +} +_LEGACY_EVIDENCE_FILENAMES = frozenset( + { + "actions.json", + "evaluation.json", + "failure.png", + "final.png", + "network.har", + "outputs.json", + "response.json", + "trace.json", + } +) + + +def source_sites(catalog: dict) -> dict[WebArenaSite, str]: + raw = catalog.get("source_site_urls") or {} + if not isinstance(raw, dict) or not set(raw) <= {site.value for site in WebArenaSite}: + raise ValueError("catalog source site URLs are invalid") + if any(not isinstance(value, str) or not urlsplit(value).netloc for value in raw.values()): + raise ValueError("catalog source site URLs are invalid") + return {WebArenaSite(key): value for key, value in raw.items()} + + +def row_source_sites(catalog: dict, row: dict) -> dict[WebArenaSite, str]: + override = row.get("source_site_urls") + if override is not None: + return source_sites({"source_site_urls": override}) + return source_sites(catalog) + + +def _is_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(char in "0123456789abcdef" for char in value) + ) + + +def _validate_action_amendments(amendments: object) -> list[dict]: + if not isinstance(amendments, list): + raise ValueError("action override amendments must be a list") + validated: list[dict] = [] + identities: set[tuple[int, int]] = set() + for amendment in amendments: + if not isinstance(amendment, dict) or set(amendment) != { + "task_id", + "action_index", + "precondition_action_sha256", + "set", + }: + raise ValueError("action override amendment fields are invalid") + task_id = amendment["task_id"] + action_index = amendment["action_index"] + precondition = amendment["precondition_action_sha256"] + updates = amendment["set"] + if ( + type(task_id) is not int + or type(action_index) is not int + or action_index < 1 + or not _is_sha256(precondition) + or not isinstance(updates, dict) + or not updates + or not set(updates) <= _ACTION_OVERRIDE_FIELDS + ): + raise ValueError("action override amendment is invalid") + if "script_mode" in updates and updates["script_mode"] != "function_body": + raise ValueError("action override script_mode is invalid") + script = updates.get("script") + if script is not None and ( + not isinstance(script, str) + or not script.strip() + or len(script.encode("utf-8")) > _MAX_OVERRIDE_SCRIPT_BYTES + ): + raise ValueError("action override script is invalid") + hidden = updates.get("wait_for_selector_hidden") + if hidden is not None and (not isinstance(hidden, str) or not hidden): + raise ValueError("action override hidden selector is invalid") + selector = updates.get("selector") + if selector is not None and (not isinstance(selector, str) or not selector.strip()): + raise ValueError("action override selector is invalid") + wait_state = updates.get("wait_state") + if wait_state is not None and wait_state not in { + "attached", + "detached", + "visible", + "hidden", + }: + raise ValueError("action override wait state is invalid") + for flag in ("force", "wait_for_enabled"): + if flag in updates and type(updates[flag]) is not bool: + raise ValueError(f"action override {flag} flag is invalid") + identity = (task_id, action_index) + if identity in identities: + raise ValueError("action override amendments contain a duplicate action") + identities.add(identity) + validated.append(amendment) + return validated + + +def load_action_overrides(path: Path) -> tuple[dict[int, list[dict]], str]: + """Load strict, versioned amendments without consulting a replay result.""" + + if path.is_symlink() or not path.is_file(): + raise ValueError("action overrides must be a regular file") + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or set(payload) != {"schema", "amendments"}: + raise ValueError("action overrides contract fields are invalid") + if payload["schema"] != ACTION_OVERRIDES_SCHEMA: + raise ValueError("action overrides schema is invalid") + amendments = _validate_action_amendments(payload["amendments"]) + if not amendments: + raise ValueError("action overrides must contain at least one amendment") + grouped: dict[int, list[dict]] = {} + for amendment in amendments: + grouped.setdefault(amendment["task_id"], []).append(amendment) + return grouped, file_sha256(path) + + +def apply_action_overrides( + actions: list[dict], amendments: list[dict], *, task_id: int +) -> list[dict]: + """Apply hash-guarded generic action fields to one immutable source trajectory.""" + + result = [dict(action) for action in actions] + for amendment in _validate_action_amendments(amendments): + if amendment["task_id"] != task_id: + raise ValueError(f"task {task_id}: action override belongs to another task") + index = amendment["action_index"] - 1 + if index >= len(result): + raise ValueError(f"task {task_id}: action override index is outside the trajectory") + original = actions[index] + if _json_sha256(original) != amendment["precondition_action_sha256"]: + raise ValueError(f"task {task_id}: action override precondition changed") + if {"script", "script_mode"} & set(amendment["set"]) and original.get( + "action" + ) != "evaluate": + raise ValueError(f"task {task_id}: script fields require an evaluate action") + if {"force", "wait_for_enabled"} & set(amendment["set"]) and original.get( + "action" + ) not in {"click", "fill", "select"}: + raise ValueError( + f"task {task_id}: actionability fields require click, fill, or select" + ) + if "selector" in amendment["set"] and original.get("action") not in { + "click", + "fill", + "select", + "wait", + }: + raise ValueError( + f"task {task_id}: selector override requires click, fill, select, or wait" + ) + if "wait_state" in amendment["set"] and original.get("action") != "wait": + raise ValueError(f"task {task_id}: wait_state requires a wait action") + result[index].update(amendment["set"]) + return result + + +def catalog_action_overrides(catalog: dict, row: dict) -> list[dict]: + """Read and validate the self-contained amendment provenance on a catalog row.""" + + provenance = row.get("action_overrides") + source_sha256 = catalog.get("action_overrides_sha256") + if provenance is None: + return [] + if ( + not isinstance(provenance, dict) + or set(provenance) != {"schema", "source_sha256", "amendments_sha256", "amendments"} + or provenance.get("schema") != ACTION_OVERRIDES_SCHEMA + or not _is_sha256(source_sha256) + or provenance.get("source_sha256") != source_sha256 + or not _is_sha256(provenance.get("amendments_sha256")) + ): + raise ValueError(f"task {row.get('task_id')}: action override provenance is invalid") + amendments = _validate_action_amendments(provenance["amendments"]) + if not amendments or _json_sha256(amendments) != provenance["amendments_sha256"]: + raise ValueError(f"task {row.get('task_id')}: action override provenance changed") + return amendments + + +def load_catalog_inputs( + catalog_path: Path, + suite_path: Path, + site_map_path: Path, + *, + accepted_ledger_path: Path | None = None, + action_overrides_path: Path | None = None, +) -> tuple[dict, dict[WebArenaSite, str], dict[int, object]]: + """Load and fully validate one frozen catalog input set.""" + + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + sites = load_remote_sites(site_map_path) + suite = load_suite(suite_path) + tasks = {int(task.task_id.removeprefix("webarena-")): task for task in suite.tasks} + rows = catalog.get("rows") or [] + ids = [row.get("task_id") for row in rows] + if ( + catalog.get("schema") not in CATALOG_SCHEMAS + or not ids + or len(set(ids)) != len(ids) + or catalog.get("task_ids") != ids + or catalog.get("site_map_sha256") != file_sha256(site_map_path) + or catalog.get("portable_suite_sha256") != file_sha256(suite_path) + ): + raise ValueError("replay catalog binding is invalid") + if accepted_ledger_path is not None and catalog.get("accepted_ledger_sha256") != file_sha256( + accepted_ledger_path + ): + raise ValueError("replay catalog is not bound to the accepted ledger") + external_overrides: dict[int, list[dict]] | None = None + has_override_binding = "action_overrides_sha256" in catalog or any( + row.get("action_overrides") is not None for row in rows + ) + if has_override_binding and action_overrides_path is None: + raise ValueError("replay catalog requires its external action overrides source") + if action_overrides_path is not None: + external_overrides, override_sha256 = load_action_overrides(action_overrides_path) + if catalog.get("action_overrides_sha256") != override_sha256: + raise ValueError("replay catalog is not bound to the action overrides") + portable_ids = {row["task_id"] for row in rows if row.get("mode") == "portable"} + if portable_ids != set(tasks): + raise ValueError("portable suite differs from the catalog") + for row in rows: + task_id = row["task_id"] + mode = row.get("mode") + if mode == "portable": + actions = list(tasks[task_id].tool_calls) + elif mode == "direct": + actions = row.get("source_actions") + if not isinstance(actions, list) or not actions: + raise ValueError(f"task {task_id}: direct source actions are missing") + contract = row.get("retrieval_contract") + if contract is not None: + validate_contract(contract, actions) + else: + raise ValueError(f"task {task_id}: unknown catalog mode") + amendments = catalog_action_overrides(catalog, row) + if external_overrides is not None and amendments != external_overrides.get(task_id, []): + raise ValueError(f"task {task_id}: catalog action overrides differ from their source") + actions = apply_action_overrides(actions, amendments, task_id=task_id) + bound = rebind_remote_calls( + actions, sites, set(TRUSTED_RECORDED_HOSTS), row_source_sites(catalog, row) + ) + if _json_sha256(bound) != row.get("bound_calls_sha256"): + raise ValueError(f"task {task_id}: bound actions changed") + if "recorded_response" in row: + raise ValueError("recorded answers cannot enter a replay catalog") + if external_overrides is not None and set(external_overrides) - set(ids): + raise ValueError("action overrides contain tasks outside the catalog") + return catalog, sites, tasks + + +def completed_task_ids(progress_path: Path, execution_order: list[int]) -> set[int]: + """Return completed IDs only when progress is a strict order prefix.""" + + if not progress_path.exists(): + return set() + rows = [json.loads(line) for line in progress_path.read_text(encoding="utf-8").splitlines()] + observed = [row.get("task_id") for row in rows] + if observed != execution_order[: len(observed)]: + raise ValueError("progress is not a strict prefix of the declared execution order") + return set(observed) + + +def verify_physical_evidence(directory: Path, row: dict) -> Path | None: + """Hash-check every declared artifact for one completed physical attempt.""" + + task_id = row["task_id"] + evidence = row.get("evidence_sha256") + if not isinstance(evidence, dict) or not evidence: + raise ValueError(f"task {task_id} has no declared physical evidence") + task_dir = directory / f"task-{task_id}" + if task_dir.is_symlink() or not task_dir.is_dir(): + raise ValueError(f"task {task_id} evidence directory is missing") + if "trace.json" in evidence: + if not set(evidence) <= _LEGACY_EVIDENCE_FILENAMES: + raise ValueError(f"task {task_id} has unknown legacy evidence files") + names = {name: name for name in evidence} + required = {"evaluation.json", "network.har", "response.json", "trace.json"} + evaluation_path = task_dir / "evaluation.json" + else: + unknown = set(evidence) - set(_EVIDENCE_FILENAMES) + if unknown: + raise ValueError(f"task {task_id} has unknown evidence kinds: {sorted(unknown)}") + names = {name: _EVIDENCE_FILENAMES[name] for name in evidence} + required = {"evaluation", "network_har", "actual_response", "trace"} + evaluation_path = task_dir / "evaluation.json" + for name, filename in names.items(): + path = task_dir / filename + if path.is_symlink() or not path.is_file() or file_sha256(path) != evidence[name]: + raise ValueError(f"task {task_id} evidence hash changed: {filename}") + if row.get("status") == "pass" and not required <= set(evidence): + raise ValueError(f"task {task_id} pass lacks official evaluation") + return evaluation_path if evaluation_path.is_file() else None + + +def validate_environment_preparation(receipt_path: Path, environment_profile_path: Path) -> dict: + """Validate the fresh-deployment receipt against the versioned environment profile.""" + + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + profile = json.loads(environment_profile_path.read_text(encoding="utf-8")) + deployment = profile.get("deployment") or {} + images = profile.get("images") or {} + mutable_sites = ("shopping", "shopping_admin", "reddit", "gitlab") + if ( + profile.get("schema") not in ENVIRONMENT_SCHEMAS + or receipt.get("schema") != "lex_webarena_environment_preparation/1" + or receipt.get("method") not in ENVIRONMENT_PREPARATION_METHODS + or receipt.get("source_app_id") != deployment.get("source_app_id") + or receipt.get("fresh_app_id") != deployment.get("acceptance_app_id") + or receipt.get("source_app_id") == receipt.get("fresh_app_id") + or receipt.get("site_reset_between_tasks") is not deployment.get("site_reset_between_tasks") + ): + raise ValueError("environment preparation is not bound to the declared deployment") + if receipt["method"] == "component_restarts_from_pinned_images" and set( + receipt.get("reset_scope") or () + ) != set(mutable_sites): + raise ValueError("environment preparation reset scope is incomplete") + receipt_images = receipt.get("site_component_images") or {} + volumes = receipt.get("database_volume_mounts") or {} + initialization = receipt.get("initialization") or {} + statuses = receipt.get("public_http_status") or {} + for site in mutable_sites: + expected_digest = images.get(site) + if ( + not isinstance(expected_digest, str) + or not str(receipt_images.get(site) or "").endswith("@" + expected_digest) + or volumes.get(site) != 0 + or not initialization.get(site) + or statuses.get(site) != 200 + ): + raise ValueError(f"environment preparation is incomplete for {site}") + if set(receipt.get("shared_site_components") or ()) != {"map", "wikipedia"}: + raise ValueError("environment preparation has an unexpected shared-site scope") + return receipt + + +def repository_root(path: Path) -> Path: + """Find the browser-eval checkout that owns a versioned input or entry point.""" + + start = path.resolve() + candidates = (start, *start.parents) if start.is_dir() else start.parents + root = next( + ( + candidate + for candidate in candidates + if (candidate / "pyproject.toml").is_file() and (candidate / "uv.lock").is_file() + ), + None, + ) + if root is None: + raise ValueError("path is outside a browser-eval repository") + return root + + +def validate_passing_evaluation( + evaluation: dict, + *, + task_id: int, + expected_data_checksum: str, + benchmark: object, + corrections: object, + expected_evaluator_checksum: str | None = None, +) -> None: + """Require current official rules and the declared replay correction for a pass.""" + + official = benchmark.get_task(task_id) + correction = corrections.records.get(task_id) + if correction is None: + task = official + expected_correction = (None, None) + else: + applied = apply_correction(official, corrections) + task = applied.task + expected_correction = (applied.correction_id, applied.corrected_task_sha256) + actual_correction = ( + evaluation.get("contract_correction_id"), + evaluation.get("corrected_task_sha256"), + ) + evaluator_names = tuple(item.evaluator for item in task.eval) + evaluator_rows = evaluation.get("evaluators_results") or [] + if ( + evaluation.get("task_id") != task_id + or float(evaluation.get("score") or 0) != 1.0 + or str(evaluation.get("status") or "").lower() not in {"pass", "success"} + or evaluation.get("webarena_verified_evaluator_checksum") + != (expected_evaluator_checksum or compute_evaluator_checksum()) + or evaluation.get("webarena_verified_data_checksum") != expected_data_checksum + or actual_correction != expected_correction + or tuple(row.get("evaluator_name") for row in evaluator_rows) != evaluator_names + or any( + float(row.get("score") or 0) != 1.0 + or str(row.get("status") or "").lower() not in {"pass", "success"} + for row in evaluator_rows + ) + ): + raise ValueError(f"task {task_id} pass has stale or incomplete evaluator provenance") + + +def validate_catalog_report_profile(profile_path: Path, data_home: Path) -> dict: + """Validate the repository authority against its frozen local release artifacts.""" + + profile = json.loads(profile_path.read_text(encoding="utf-8")) + root = repository_root(profile_path) + dataset_path = root / str(profile.get("dataset_path") or "") + scope_path = root / str(profile.get("scope_path") or "") + if ( + profile.get("schema") not in REPORT_PROFILE_SCHEMAS + or not dataset_path.is_file() + or file_sha256(dataset_path) != profile.get("dataset_sha256") + or not scope_path.is_file() + or file_sha256(scope_path) != profile.get("scope_sha256") + ): + raise ValueError("catalog report profile changed its versioned dataset binding") + dataset = json.loads(dataset_path.read_text(encoding="utf-8")) + scope = json.loads(scope_path.read_text(encoding="utf-8")) + task_ids = scope.get("task_ids") + evaluator_source = dataset.get("evaluator_source") or {} + project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8")) + dependency_source = ( + project.get("tool", {}).get("uv", {}).get("sources", {}).get("webarena-verified") or {} + ) + lock = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) + locked_package = next( + ( + package + for package in lock.get("package", []) + if package.get("name") == "webarena-verified" + ), + {}, + ) + locked_git = (locked_package.get("source") or {}).get("git", "") + if ( + dataset.get("schema") not in DATASET_SCHEMAS + or scope.get("schema") not in SCOPE_SCHEMAS + or not isinstance(task_ids, list) + or not task_ids + or any(not isinstance(task_id, int) for task_id in task_ids) + or len(set(task_ids)) != len(task_ids) + or scope.get("task_count") != len(task_ids) + or dataset.get("task_count") != len(task_ids) + or ( + dataset.get("task_id_range") is not None + and dataset.get("task_id_range") != [min(task_ids), max(task_ids)] + ) + or scope.get("official_dataset_sha256") != dataset.get("official_dataset_sha256") + or "browser" in dataset + or evaluator_source.get("repository") != WEBARENA_FORK_REPOSITORY + or evaluator_source.get("branch") != WEBARENA_FORK_BRANCH + or dependency_source.get("git") != WEBARENA_FORK_REPOSITORY + or dependency_source.get("rev") != evaluator_source.get("revision") + or locked_git + != ( + f"{WEBARENA_FORK_REPOSITORY}?rev={evaluator_source.get('revision')}" + f"#{evaluator_source.get('revision')}" + ) + ): + raise ValueError("WebArena catalog dataset definition is invalid") + root = data_home / str(profile.get("evidence_root") or "") + if root.is_symlink() or not root.is_dir() or data_home.resolve() not in root.resolve().parents: + raise ValueError("catalog report evidence root is missing or unsafe") + artifacts = { + "acceptance_ledger": ( + profile.get("acceptance_ledger_path"), + profile.get("acceptance_ledger_sha256"), + ), + "report_manifest": ( + profile.get("report_manifest_path"), + profile.get("report_manifest_sha256"), + ), + "report_archive": ( + profile.get("report_archive_path"), + profile.get("report_archive_sha256"), + ), + "accepted_catalog": ( + (profile.get("accepted_catalog") or {}).get("path"), + (profile.get("accepted_catalog") or {}).get("sha256"), + ), + "accepted_portable_suite": ( + (profile.get("accepted_portable_suite") or {}).get("path"), + (profile.get("accepted_portable_suite") or {}).get("sha256"), + ), + } + for name, (relative, expected) in artifacts.items(): + path = Path(str(relative or "")) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"{name} path is unsafe") + target = root / path + if target.is_symlink() or not target.is_file() or file_sha256(target) != expected: + raise ValueError(f"{name} artifact changed") + ledger = json.loads((root / artifacts["acceptance_ledger"][0]).read_text(encoding="utf-8")) + accepted_counts = profile.get("accepted_counts") + if ( + not isinstance(accepted_counts, dict) + or set(accepted_counts) + != { + "total", + "pass", + "unresolved", + "physical_attempts", + "excluded_duplicate_attempts", + } + or accepted_counts.get("total") != len(task_ids) + or accepted_counts.get("pass") != len(task_ids) + or accepted_counts.get("unresolved") != 0 + or ledger.get("schema") not in ACCEPTANCE_SCHEMAS + or ledger.get("scope_sha256") != profile.get("scope_sha256") + or ledger.get("counts") != accepted_counts + ): + raise ValueError("catalog report acceptance population changed") + catalog = json.loads((root / artifacts["accepted_catalog"][0]).read_text(encoding="utf-8")) + if ( + catalog.get("schema") not in CATALOG_SCHEMAS + or catalog.get("task_ids") != task_ids + or catalog.get("accepted_ledger_sha256") != profile["acceptance_ledger_sha256"] + ): + raise ValueError("accepted catalog is not bound to the current ledger") + publication = profile.get("publication") or {} + if not all(publication.get(key) for key in ("host", "slug", "version_id", "share_url")): + raise ValueError("catalog report publication target is incomplete") + return profile + + +def validate_catalog_comparison_release(binding_path: Path, data_home: Path) -> dict: + """Validate the current multi-browser release without storing measured evidence in Git.""" + + binding = json.loads(binding_path.read_text(encoding="utf-8")) + evidence_root = Path(str(binding.get("evidence_root") or "")) + if ( + binding.get("schema") != COMPARISON_RELEASE_SCHEMA + or evidence_root.is_absolute() + or ".." in evidence_root.parts + or not evidence_root.parts + ): + raise ValueError("comparison release binding is invalid") + root = data_home / evidence_root + if root.is_symlink() or not root.is_dir() or data_home.resolve() not in root.resolve().parents: + raise ValueError("comparison release evidence root is missing or unsafe") + + required_artifacts = { + "profile", + "dataset", + "results", + "failure_review", + "execution_metadata_revision", + "report_manifest", + "publication_archive", + "publication_attestation", + "publication_receipt", + } + artifacts = binding.get("artifacts") + if not isinstance(artifacts, dict) or set(artifacts) != required_artifacts: + raise ValueError("comparison release artifact inventory is incomplete") + resolved: dict[str, Path] = {} + for name, artifact in artifacts.items(): + relative = Path(str((artifact or {}).get("path") or "")) + expected = (artifact or {}).get("sha256") + if ( + relative.is_absolute() + or ".." in relative.parts + or not relative.parts + or not _is_sha256(expected) + ): + raise ValueError(f"comparison release {name} binding is invalid") + target = root / relative + components = [ + root.joinpath(*relative.parts[:index]) + for index in range(1, len(relative.parts) + 1) + ] + if ( + any(component.is_symlink() for component in components) + or not target.is_file() + or file_sha256(target) != expected + ): + raise ValueError(f"comparison release {name} artifact changed") + resolved[name] = target + if len({path.resolve() for path in resolved.values()}) != len(resolved): + raise ValueError("comparison release artifacts must be distinct files") + + profile = json.loads(resolved["profile"].read_text(encoding="utf-8")) + dataset = json.loads(resolved["dataset"].read_text(encoding="utf-8")) + results = json.loads(resolved["results"].read_text(encoding="utf-8")) + metadata_revision = json.loads( + resolved["execution_metadata_revision"].read_text(encoding="utf-8") + ) + manifest = json.loads(resolved["report_manifest"].read_text(encoding="utf-8")) + receipt = json.loads(resolved["publication_receipt"].read_text(encoding="utf-8")) + attestation = json.loads(resolved["publication_attestation"].read_text(encoding="utf-8")) + attempts = results.get("attempts") + participants = binding.get("participants") + if ( + profile.get("schema") != "lex_webarena_trajectory_replay_profile/1" + or results.get("schema") != "lex_webarena_catalog_comparison_results/1" + or not isinstance(attempts, list) + or not isinstance(participants, list) + or not participants + ): + raise ValueError("comparison release inputs are invalid") + source_inputs = metadata_revision.get("source_inputs") + verification = metadata_revision.get("verification") or {} + if ( + metadata_revision.get("schema") != "lex_trajectory_replay_metadata_revision/1" + or metadata_revision.get("provider_calls") != 0 + or metadata_revision.get("results_sha256") != artifacts["results"]["sha256"] + or not isinstance(source_inputs, list) + or not source_inputs + or any( + not isinstance(row, dict) + or row.get("backend") != "playwright" + or not _is_sha256(row.get("sha256")) + or not row.get("path") + for row in source_inputs + ) + or verification.get("source_input_count") != len(source_inputs) + or verification.get("non_playwright_source_inputs") != 0 + or profile.get("execution_backend") != "playwright" + or (profile.get("resources") or {}).get("collection_status") != "not_recorded" + ): + raise ValueError("comparison release execution metadata revision changed") + methods = profile.get("methods") or [] + method_registry = {method.get("id"): method for method in methods if isinstance(method, dict)} + declared = {row.get("id"): row for row in participants if isinstance(row, dict)} + if ( + len(method_registry) != len(methods) + or len(declared) != len(participants) + or set(declared) != set(method_registry) + or any( + set(row) != {"id", "engine", "label", "pass", "fail"} + or row.get("engine") != method_registry[participant_id].get("engine") + or row.get("label") != method_registry[participant_id].get("label") + for participant_id, row in declared.items() + ) + ): + raise ValueError("comparison release participant registry changed") + task_count = binding.get("task_count") + trajectories = dataset.get("trajectories") or [] + dataset_task_ids = {row.get("task_id") for row in trajectories if isinstance(row, dict)} + task_ids: set[int] | None = None + for participant_id, declared_row in declared.items(): + rows = [row for row in attempts if row.get("participant_id") == participant_id] + row_task_ids = {row.get("task_id") for row in rows} + passed = sum(row.get("status") == "pass" for row in rows) + if ( + len(rows) != task_count + or len(row_task_ids) != task_count + or row_task_ids != dataset_task_ids + or (task_ids is not None and row_task_ids != task_ids) + or declared_row.get("pass") != passed + or declared_row.get("fail") != len(rows) - passed + ): + raise ValueError("comparison release participant counts changed") + task_ids = row_task_ids + logical_results = len(attempts) + physical_executions = results.get("physical_executions") + failed_results = sum(row.get("fail", 0) for row in participants) + if ( + type(task_count) is not int + or task_count <= 0 + or dataset.get("schema") != "lex_webarena_catalog_comparison_dataset/1" + or len(trajectories) != task_count + or len(dataset_task_ids) != task_count + or any(type(task_id) is not int for task_id in dataset_task_ids) + or (dataset.get("counts") or {}).get("verified_tasks") != task_count + or (dataset.get("counts") or {}).get("verified_trajectories") != task_count + or results.get("dataset_sha256") != artifacts["dataset"]["sha256"] + or binding.get("logical_results") != logical_results + or logical_results != task_count * len(declared) + or any( + row.get("participant_id") not in declared + or row.get("engine") != method_registry[row.get("participant_id")].get("engine") + for row in attempts + ) + or not isinstance(physical_executions, list) + or binding.get("physical_executions") != len(physical_executions) + or len(physical_executions) < logical_results + or any( + row.get("participant_id") not in declared + or row.get("engine") != method_registry[row.get("participant_id")].get("engine") + or row.get("task_id") not in dataset_task_ids + for row in physical_executions + ) + or manifest.get("task_count") != task_count + or manifest.get("logical_results") != logical_results + or manifest.get("failed_results") != failed_results + or manifest.get("reviewed_failures") != failed_results + or manifest.get("participants") != list(declared) + or manifest.get("results_sha256") != artifacts["results"]["sha256"] + or manifest.get("dataset_sha256") != artifacts["dataset"]["sha256"] + or manifest.get("review_sha256") != artifacts["failure_review"]["sha256"] + ): + raise ValueError("comparison release population or report binding changed") + publication = binding.get("publication") or {} + profile_publication = profile.get("publication") or {} + if ( + receipt.get("verified") is not True + or receipt.get("archive_sha256") != artifacts["publication_archive"]["sha256"] + or attestation.get("archive_sha256") != artifacts["publication_archive"]["sha256"] + or attestation.get("profile_sha256") != artifacts["profile"]["sha256"] + or attestation.get("dataset_sha256") != artifacts["dataset"]["sha256"] + or attestation.get("review_sha256") != artifacts["failure_review"]["sha256"] + or attestation.get("evidence_sha256") != artifacts["results"]["sha256"] + or publication.get("slug") != receipt.get("slug") + or publication.get("share_url") != receipt.get("share_url") + or publication.get("version_id") != receipt.get("version") + or publication.get("slug") != profile_publication.get("site_slug") + or publication.get("share_url") != profile_publication.get("share_url") + or publication.get("version_id") != profile_publication.get("expected_version") + ): + raise ValueError("comparison release publication binding changed") + return binding diff --git a/harness/trajectory_replay/catalog_release.py b/harness/trajectory_replay/catalog_release.py new file mode 100644 index 000000000..f838558a1 --- /dev/null +++ b/harness/trajectory_replay/catalog_release.py @@ -0,0 +1,87 @@ +"""Validate a WebArena catalog report profile and its frozen local artifacts.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from harness.trajectory_replay.catalog_action_audit import validate_catalog_action_audit +from harness.trajectory_replay.catalog_contracts import ( + repository_root, + validate_catalog_comparison_release, + validate_catalog_report_profile, +) +from harness.trajectory_replay.webarena_report import validate_catalog_acceptance_report +from harness.trajectory_replay.webarena_runtime_source import verify_webarena_runtime_source + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profile", + type=Path, + default=Path("evaluation/trajectory-replay/webarena-verified-full/current-report.json"), + ) + parser.add_argument( + "--data-home", + type=Path, + default=Path(os.environ.get("BROWSER_EVAL_DATA_HOME", Path.home() / ".browser-eval")) + / "evidence/current", + ) + parser.add_argument( + "--action-overrides", + type=Path, + default=Path("evaluation/trajectory-replay/webarena-verified-full/action-overrides.json"), + ) + parser.add_argument( + "--action-overrides-audit", + type=Path, + default=Path( + "evaluation/trajectory-replay/webarena-verified-full/action-overrides-audit.json" + ), + ) + parser.add_argument( + "--comparison-release", + type=Path, + default=Path( + "evaluation/trajectory-replay/webarena-verified-full/current-comparison.json" + ), + ) + args = parser.parse_args() + verify_webarena_runtime_source() + profile = validate_catalog_report_profile(args.profile, args.data_home) + root = repository_root(args.profile) + action_overrides = ( + args.action_overrides + if args.action_overrides.is_absolute() + else root / args.action_overrides + ) + action_overrides_audit = ( + args.action_overrides_audit + if args.action_overrides_audit.is_absolute() + else root / args.action_overrides_audit + ) + evidence_root = args.data_home / profile["evidence_root"] + validate_catalog_action_audit( + action_overrides_audit, + evidence_root / profile["accepted_catalog"]["path"], + evidence_root / profile["accepted_portable_suite"]["path"], + action_overrides, + ) + validate_catalog_acceptance_report( + evidence_root / Path(profile["report_manifest_path"]).parent, + evidence_root / profile["report_archive_path"], + profile=profile, + ) + comparison_release = ( + args.comparison_release + if args.comparison_release.is_absolute() + else root / args.comparison_release + ) + comparison = validate_catalog_comparison_release(comparison_release, args.data_home) + print(comparison["publication"]["share_url"]) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/catalog_runner.py b/harness/trajectory_replay/catalog_runner.py new file mode 100644 index 000000000..ac8318c9e --- /dev/null +++ b/harness/trajectory_replay/catalog_runner.py @@ -0,0 +1,775 @@ +"""Replay one frozen WebArena catalog in its declared task order.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import multiprocessing +import os +import socket +import subprocess +import sys +import tempfile +import threading +import time +import traceback +from pathlib import Path +from urllib.parse import quote, urlsplit + +from harness.trajectory_replay.catalog_contracts import ( + RUN_SCHEMA, + TRUSTED_RECORDED_HOSTS, + apply_action_overrides, + catalog_action_overrides, + completed_task_ids, + file_sha256, + load_catalog_inputs, + repository_root, + row_source_sites, + validate_environment_preparation, + verify_physical_evidence, +) +from harness.trajectory_replay.remote_execution import ( + _browser_command, + _run_task, + _wait_cdp, + finalize_timed_out_task, + sanitize_remote_text, +) +from harness.trajectory_replay.remote_runtime import rebind_remote_calls, remote_benchmark +from harness.trajectory_replay.webarena_corrections import load_corrections +from harness.trajectory_replay.webarena_runtime_source import verify_webarena_runtime_source + + +def _append(path: Path, row: dict) -> None: + with path.open("a", encoding="utf-8") as target: + target.write(json.dumps(row, ensure_ascii=False) + "\n") + target.flush() + os.fsync(target.fileno()) + + +def _task_ids( + declared: list[int], + task_from: int | None, + task_to: int | None, + selected: list[int] | None = None, +) -> list[int]: + if not declared or len(set(declared)) != len(declared): + raise ValueError("catalog task order is empty or contains duplicates") + if selected: + if task_from is not None or task_to is not None: + raise ValueError("explicit task IDs cannot be combined with a task range") + if len(set(selected)) != len(selected) or any( + task_id not in declared for task_id in selected + ): + raise ValueError("explicit task IDs must be unique catalog members") + expected = [task_id for task_id in declared if task_id in set(selected)] + if selected != expected: + raise ValueError("explicit task IDs must follow catalog order") + return selected + lower = declared[0] if task_from is None else task_from + upper = declared[-1] if task_to is None else task_to + if lower not in declared or upper not in declared: + raise ValueError("task range endpoints must belong to the catalog") + start = declared.index(lower) + end = declared.index(upper) + if start > end: + raise ValueError("task range is reversed in catalog order") + return declared[start : end + 1] + + +def _task_origins(benchmark: object, task_id: int, sites: dict) -> set[str]: + """Return only the fixture origins declared by the current official task. + + Browsers without ``Storage.clearDataForOrigin`` clear site storage by + visiting each supplied origin. Passing the whole deployment makes an + unrelated site outage invalidate every task in the cohort and does extra + cross-site work before each replay. + """ + + task = benchmark.get_task(task_id) + return { + f"{urlsplit(sites[site]).scheme}://{urlsplit(sites[site]).netloc}" + for site in task.sites + } + + +def _recover_completed_attempts(output: Path, progress: Path, ids: list[int]) -> set[int]: + """Commit a finished result left between task completion and progress append.""" + + done = completed_task_ids(progress, ids) + while len(done) < len(ids): + task_id = ids[len(done)] + task_dir = output / f"task-{task_id}" + if not task_dir.exists(): + break + result_path = task_dir / "result.json" + if result_path.is_symlink() or not result_path.is_file(): + raise ValueError( + f"task {task_id} has interrupted evidence without a committed result; " + "use a fresh prepared deployment before retrying" + ) + row = json.loads(result_path.read_text(encoding="utf-8")) + if row.get("task_id") != task_id: + raise ValueError(f"task {task_id} committed result identity changed") + verify_physical_evidence(output, row) + _append(progress, row) + done.add(task_id) + return done + + +def _load_task_selection(path: Path) -> list[int]: + if path.is_symlink() or not path.is_file(): + raise ValueError("task selection must be a regular file") + payload = json.loads(path.read_text(encoding="utf-8")) + task_ids = payload.get("task_ids") if isinstance(payload, dict) else None + if ( + not isinstance(payload, dict) + or payload.get("schema") != "lex_webarena_replay_selection/1" + or not isinstance(task_ids, list) + or not task_ids + or any(type(task_id) is not int for task_id in task_ids) + ): + raise ValueError("task selection contract is invalid") + return task_ids + + +def _load_env_values(path: Path) -> dict[str, str]: + if path.is_symlink() or not path.is_file(): + raise ValueError("credential environment file must be a regular file") + values: dict[str, str] = {} + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, value = line.split("=", 1) + values[name.strip()] = value.strip().strip("\"'") + return values + + +def _kitesurf_cdp_config( + path: Path, +) -> tuple[str, dict[str, str], dict[str, object]]: + values = _load_env_values(path) + account_id = values.get("CLOUDFLARE_ACCOUNT_ID", "").strip() + api_token = values.get("CLOUDFLARE_API_TOKEN", "").strip() + if not account_id or not api_token: + raise ValueError( + "Kitesurf requires CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN" + ) + endpoint = ( + "wss://api.cloudflare.com/client/v4/accounts/" + f"{quote(account_id, safe='')}" + "/browser-run/devtools/browser?browser=kitesurf" + ) + headers = {"Authorization": f"Bearer {api_token}"} + public_binding = { + "provider": "cloudflare-browser-run", + "browser": "kitesurf", + "protocol": "cdp", + "endpoint_contract": ( + "wss://api.cloudflare.com/client/v4/accounts/{account_id}/" + "browser-run/devtools/browser?browser=kitesurf" + ), + "account_id_sha256": hashlib.sha256(account_id.encode()).hexdigest(), + } + return endpoint, headers, public_binding + + +def _probe_kitesurf_runtime( + endpoint: str, headers: dict[str, str], *, timeout_ms: int = 120_000 +) -> dict[str, str]: + """Freeze the hosted product/version identity before any task evidence exists.""" + + def probe() -> dict[str, str]: + from harness.trajectory_replay.playwright_tool import ( # noqa: PLC0415 + PlaywrightReplayTool, + ) + + tool = PlaywrightReplayTool(endpoint, engine="kitesurf", headers=headers) + try: + metadata = tool.runtime_metadata + if not isinstance(metadata, dict): + raise RuntimeError("Kitesurf did not expose Browser.getVersion metadata") + return metadata + finally: + tool.close() + + return _run_forked_with_watchdog(probe, timeout_ms=timeout_ms) + + +def _chromium_host_resolver_rules(environment_profile: dict, engine: str) -> str | None: + """Compile deployment-neutral host overrides for Chromium, with legacy read support.""" + + if engine != "chromium": + return None + overrides = environment_profile.get("host_overrides") + if overrides is None: + legacy = environment_profile.get("browser_host_resolver_rules") + if legacy is not None and (not isinstance(legacy, str) or "\n" in legacy): + raise ValueError("legacy Chromium host resolver rules must be one line") + return legacy + if not isinstance(overrides, list) or not overrides: + raise ValueError("environment host overrides must be a nonempty list") + rules = [] + for row in overrides: + pattern = row.get("pattern") if isinstance(row, dict) else None + address = row.get("address") if isinstance(row, dict) else None + if ( + not isinstance(pattern, str) + or not pattern + or any(char in pattern for char in "\n,") + or not isinstance(address, str) + or not address + or any(char in address for char in "\n,") + ): + raise ValueError("environment host override is invalid") + rules.append(f"MAP {pattern} {address}") + return ", ".join(rules) + + +def _request_bridge_options( + environment_profile: dict, + engine: str, + task_origins: set[str], +) -> dict[str, object]: + """Bind Kitesurf's local bridge to the deployment's declared host routing.""" + + if engine != "kitesurf": + return { + "request_bridge_origins": None, + "request_bridge_host_overrides": None, + } + return { + "request_bridge_origins": task_origins, + "request_bridge_host_overrides": environment_profile.get("host_overrides") or [], + } + + +def _stop_browser(process: subprocess.Popen) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def _run_with_setup_recovery( + *, + task_id: int, + task_dir: Path, + process: subprocess.Popen, + restart_browser, + run_task, + row: dict | None = None, + task_timeout_ms: int | None = None, +) -> tuple[dict, subprocess.Popen]: + """Run one task, restarting only before physical evidence exists. + + ``_run_task`` creates ``task_dir`` immediately before browser-session work and + catches all later execution failures into a committed result. An exception + with no task directory is therefore a setup failure that is safe to retry; + once the directory exists, retrying could duplicate a physical execution. + """ + + def run_once(current: subprocess.Popen) -> dict: + if row is None or task_timeout_ms is None: + return run_task() + state = {"finished": False, "timed_out": False} + lock = threading.Lock() + + def expire() -> None: + with lock: + if state["finished"]: + return + state["timed_out"] = True + _stop_browser(current) + + started = time.monotonic() + timer = threading.Timer(task_timeout_ms / 1_000, expire) + timer.daemon = True + timer.start() + error: Exception | None = None + result: dict | None = None + try: + result = run_task() + except Exception as exc: + error = exc + finally: + with lock: + state["finished"] = True + timed_out = state["timed_out"] + timer.cancel() + timer.join() + if timed_out: + if not task_dir.is_dir(): + raise TimeoutError( + f"task {task_id} exceeded its {task_timeout_ms} ms budget before evidence began" + ) + return finalize_timed_out_task( + task_dir, + row, + duration_ms=max(task_timeout_ms, int((time.monotonic() - started) * 1_000)), + task_timeout_ms=task_timeout_ms, + ) + if error is not None: + raise error + assert result is not None + return result + + runtime = {"process": process} + + def execute() -> dict: + current = runtime["process"] + if current.poll() is not None: + current = restart_browser(current) + runtime["process"] = current + return run_once(current) + + def recover() -> None: + runtime["process"] = restart_browser(runtime["process"]) + + return ( + _run_with_pre_evidence_recovery( + task_id=task_id, + task_dir=task_dir, + run_once=execute, + recover=recover, + ), + runtime["process"], + ) + + +def _run_with_pre_evidence_recovery( + *, task_id: int, task_dir: Path, run_once, recover +) -> dict: + """Retry once only when setup failed before any physical evidence existed.""" + + try: + return run_once() + except FileExistsError: + raise + except Exception: + first_error = traceback.format_exc() + if task_dir.exists(): + (task_dir / ".trace.partial.json").unlink(missing_ok=True) + return { + "task_id": task_id, + "status": "candidate_setup_failure", + "error": first_error, + } + recover() + try: + return run_once() + except FileExistsError: + raise + except Exception: + if task_dir.exists(): + (task_dir / ".trace.partial.json").unlink(missing_ok=True) + return { + "task_id": task_id, + "status": "candidate_setup_failure", + "error": f"{first_error}\nsetup recovery failed:\n{traceback.format_exc()}", + } + + +def _fork_worker(sender, run_task) -> None: + try: + sender.send(("result", run_task())) + except BaseException: + sender.send(("error", traceback.format_exc())) + finally: + sender.close() + + +def _run_forked_with_watchdog(run_task, *, timeout_ms: int) -> dict: + """Run one hosted-browser operation behind a killable process boundary.""" + + if "fork" not in multiprocessing.get_all_start_methods(): + raise RuntimeError("hosted browser replay requires a POSIX fork process boundary") + context = multiprocessing.get_context("fork") + receiver, sender = context.Pipe(duplex=False) + worker = context.Process(target=_fork_worker, args=(sender, run_task), daemon=False) + worker.start() + sender.close() + try: + if not receiver.poll(timeout_ms / 1_000): + worker.terminate() + worker.join(timeout=10) + if worker.is_alive(): + worker.kill() + worker.join(timeout=10) + raise TimeoutError(f"hosted browser operation exceeded {timeout_ms} ms") + kind, payload = receiver.recv() + worker.join(timeout=10) + if worker.is_alive(): + worker.kill() + worker.join(timeout=10) + raise RuntimeError("hosted browser worker did not exit after returning a result") + if kind == "error": + raise RuntimeError(payload) + if worker.exitcode != 0: + raise RuntimeError(f"hosted browser worker exited with code {worker.exitcode}") + return payload + finally: + receiver.close() + + +def _run_remote_with_setup_recovery( + *, + task_id: int, + task_dir: Path, + row: dict, + task_timeout_ms: int, + runtime_metadata: dict[str, str], + remote_endpoint: str, + remote_headers: dict[str, str], + run_task, +) -> dict: + """Run a hosted task with the shared setup retry and a whole-task watchdog.""" + + def run_once() -> dict: + started = time.monotonic() + try: + return _run_forked_with_watchdog(run_task, timeout_ms=task_timeout_ms) + except TimeoutError: + if not task_dir.is_dir(): + raise + return finalize_timed_out_task( + task_dir, + row, + duration_ms=max( + task_timeout_ms, int((time.monotonic() - started) * 1_000) + ), + task_timeout_ms=task_timeout_ms, + runtime_metadata=runtime_metadata, + remote_endpoint=remote_endpoint, + remote_headers=remote_headers, + ) + + return _run_with_pre_evidence_recovery( + task_id=task_id, + task_dir=task_dir, + run_once=run_once, + recover=lambda: None, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, required=True) + parser.add_argument("--portable-suite", type=Path, required=True) + parser.add_argument("--action-overrides", type=Path) + parser.add_argument("--site-map", type=Path, required=True) + parser.add_argument("--site-reset-receipt", type=Path, required=True) + parser.add_argument("--environment-profile", type=Path, required=True) + parser.add_argument("--binary", type=Path) + parser.add_argument( + "--engine", + choices=("chromium", "moli", "lightpanda", "obscura", "kitesurf"), + default="chromium", + ) + parser.add_argument( + "--credential-env-file", + type=Path, + default=repository_root(Path(__file__)) / ".env", + help="local credential source for a remote browser; values are never persisted", + ) + parser.add_argument("--backend", choices=("browsertool", "playwright"), default="browsertool") + parser.add_argument("--cowagent", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--action-timeout-ms", type=int, default=30_000) + parser.add_argument( + "--task-timeout-ms", + type=int, + default=600_000, + help=( + "per-task execution budget; local engines additionally terminate an " + "unresponsive browser process" + ), + ) + parser.add_argument("--task-from", type=int) + parser.add_argument("--task-to", type=int) + parser.add_argument( + "--task-id", + type=int, + action="append", + help="replay only these catalog tasks, supplied once each in catalog order", + ) + parser.add_argument( + "--task-selection", + type=Path, + help="JSON selection for a large unresolved-only replay", + ) + args = parser.parse_args() + webarena_runtime = verify_webarena_runtime_source() + if args.action_timeout_ms < 1: + raise ValueError("action timeout must be positive") + if args.task_timeout_ms < 1: + raise ValueError("task timeout must be positive") + if args.engine == "kitesurf": + if args.backend != "playwright": + raise ValueError("Kitesurf trajectory replay requires the Playwright backend") + if args.binary is not None: + raise ValueError("Kitesurf is remote and does not accept --binary") + remote_endpoint, remote_headers, remote_binding = _kitesurf_cdp_config( + args.credential_env_file.expanduser() + ) + try: + remote_binding["runtime_metadata"] = _probe_kitesurf_runtime( + remote_endpoint, remote_headers + ) + except Exception as exc: + raise RuntimeError( + sanitize_remote_text(str(exc), remote_endpoint, remote_headers) + ) from None + else: + if args.binary is None: + raise ValueError("local replay engines require a browser --binary") + remote_endpoint = None + remote_headers = None + remote_binding = None + if args.backend == "browsertool": + if args.cowagent is None or not args.cowagent.is_dir(): + raise ValueError("BrowserTool backend requires --cowagent repository") + if str(args.cowagent) not in sys.path: + sys.path.insert(0, str(args.cowagent)) + catalog, sites, portable_tasks = load_catalog_inputs( + args.catalog, + args.portable_suite, + args.site_map, + action_overrides_path=args.action_overrides, + ) + if args.action_overrides is None and ( + catalog.get("action_overrides_sha256") is not None + or any(row.get("action_overrides") is not None for row in catalog.get("rows") or []) + ): + raise ValueError("catalog requires its versioned action overrides input") + validate_environment_preparation(args.site_reset_receipt, args.environment_profile) + environment_profile = json.loads(args.environment_profile.read_text(encoding="utf-8")) + corrections = load_corrections() + benchmark = remote_benchmark(sites) + if args.task_selection is not None and args.task_id: + raise ValueError("task selection file cannot be combined with explicit task IDs") + selected = _load_task_selection(args.task_selection) if args.task_selection else args.task_id + ids = _task_ids(list(catalog["task_ids"]), args.task_from, args.task_to, selected) + os.umask(0o077) + args.output.mkdir(parents=True, exist_ok=True) + root = repository_root(Path(__file__)) + binding = { + "schema": RUN_SCHEMA, + "catalog_sha256": file_sha256(args.catalog), + "portable_suite_sha256": file_sha256(args.portable_suite), + "site_map_sha256": file_sha256(args.site_map), + "site_reset_receipt_sha256": file_sha256(args.site_reset_receipt), + "environment_profile_sha256": file_sha256(args.environment_profile), + "binary_sha256": file_sha256(args.binary) if args.binary is not None else None, + "engine": args.engine, + "backend": args.backend, + "action_timeout_ms": args.action_timeout_ms, + "task_timeout_ms": args.task_timeout_ms, + "corrections_sha256": corrections.source_sha256, + "webarena_revision": webarena_runtime.revision, + "runner_revision": subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip(), + "cowagent_revision": ( + subprocess.check_output( + ["git", "-C", str(args.cowagent), "rev-parse", "HEAD"], text=True + ).strip() + if args.backend == "browsertool" + else None + ), + "execution_order": ids, + "site_reset_between_tasks": False, + "run_count_per_task": 1, + "task_selection_sha256": ( + file_sha256(args.task_selection) if args.task_selection is not None else None + ), + } + if remote_binding is not None: + binding["remote_browser"] = remote_binding + binding["request_bridge"] = { + "kind": "local_http_fulfill/1", + "origins": sorted( + { + f"{urlsplit(url).scheme}://{urlsplit(url).netloc}" + for url in sites.values() + } + ), + } + if args.action_overrides is not None: + binding["action_overrides_sha256"] = file_sha256(args.action_overrides) + binding_path = args.output / "inputs.json" + if binding_path.exists(): + if json.loads(binding_path.read_text(encoding="utf-8")) != binding: + raise ValueError("resumed run inputs changed") + else: + binding_path.write_text(json.dumps(binding, indent=2) + "\n", encoding="utf-8") + progress = args.output / "progress.jsonl" + done = _recover_completed_attempts(args.output, progress, ids) + pending = [ + row for row in catalog["rows"] if row["task_id"] in ids and row["task_id"] not in done + ] + if not pending: + return + + def bound_row(frozen: dict) -> dict: + task_id = frozen["task_id"] + source_actions = ( + list(portable_tasks[task_id].tool_calls) + if frozen["mode"] == "portable" + else frozen["source_actions"] + ) + source_actions = apply_action_overrides( + source_actions, + catalog_action_overrides(catalog, frozen), + task_id=task_id, + ) + return { + **frozen, + "actions": rebind_remote_calls( + source_actions, + sites, + set(TRUSTED_RECORDED_HOSTS), + row_source_sites(catalog, frozen), + ), + "readiness": list(frozen.get("readiness") or []), + } + + def execute_task( + frozen: dict, + row: dict, + endpoint: str, + cdp_headers: dict[str, str] | None = None, + ) -> dict: + task_id = frozen["task_id"] + task_origins = _task_origins(benchmark, task_id, sites) + return _run_task( + task_id=task_id, + row=row, + benchmark=benchmark, + endpoint=endpoint, + output=args.output, + run_id=args.output.name, + origins=task_origins, + sites=sites, + catalog=corrections, + portable_task=( + portable_tasks[task_id] if frozen["mode"] == "portable" else None + ), + portable_action_timeout_ms=args.action_timeout_ms, + backend=args.backend, + engine=args.engine, + cdp_headers=cdp_headers, + expected_runtime_metadata=( + remote_binding["runtime_metadata"] + if remote_binding is not None + else None + ), + **_request_bridge_options(environment_profile, args.engine, task_origins), + task_timeout_ms=args.task_timeout_ms, + ) + + def record_outcome(outcome: dict) -> None: + _append(progress, outcome) + print( + json.dumps( + { + key: value + for key, value in outcome.items() + if key not in {"error", "evidence_sha256"} + }, + ensure_ascii=False, + ), + flush=True, + ) + + if remote_endpoint is not None: + assert remote_headers is not None + for frozen in pending: + task_id = frozen["task_id"] + row = bound_row(frozen) + task_dir = args.output / f"task-{task_id}" + outcome = _run_remote_with_setup_recovery( + task_id=task_id, + task_dir=task_dir, + row=row, + task_timeout_ms=args.task_timeout_ms, + runtime_metadata=remote_binding["runtime_metadata"], + remote_endpoint=remote_endpoint, + remote_headers=remote_headers, + run_task=lambda frozen=frozen, row=row: execute_task( + frozen, row, remote_endpoint, remote_headers + ), + ) + if outcome.get("error"): + outcome["error"] = sanitize_remote_text( + str(outcome["error"]), remote_endpoint, remote_headers + ) + record_outcome(outcome) + return + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + endpoint = f"http://127.0.0.1:{port}" + browser_args = argparse.Namespace( + engine=args.engine, + binary=args.binary, + host_resolver_rules=_chromium_host_resolver_rules(environment_profile, args.engine), + ) + with tempfile.TemporaryDirectory(prefix="webarena-catalog-") as profile: + def start_browser() -> subprocess.Popen: + started = subprocess.Popen( + _browser_command(browser_args, port, profile), + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + _wait_cdp(endpoint, started) + except Exception: + _stop_browser(started) + raise + return started + + def restart_browser(current: subprocess.Popen) -> subprocess.Popen: + _stop_browser(current) + return start_browser() + + process = start_browser() + try: + for pending_index, frozen in enumerate(pending): + task_id = frozen["task_id"] + row = bound_row(frozen) + def run_task( + row: dict = row, + frozen: dict = frozen, + ) -> dict: + return execute_task(frozen, row, endpoint) + + outcome, process = _run_with_setup_recovery( + task_id=task_id, + task_dir=args.output / f"task-{task_id}", + process=process, + restart_browser=restart_browser, + run_task=run_task, + row=row, + task_timeout_ms=args.task_timeout_ms, + ) + record_outcome(outcome) + if ( + outcome.get("runtime_poisoned") or process.poll() is not None + ) and pending_index + 1 < len(pending): + process = restart_browser(process) + finally: + _stop_browser(process) + + +if __name__ == "__main__": + main() diff --git a/harness/trajectory_replay/evaluator_transition.py b/harness/trajectory_replay/evaluator_transition.py new file mode 100644 index 000000000..17255a3f3 --- /dev/null +++ b/harness/trajectory_replay/evaluator_transition.py @@ -0,0 +1,286 @@ +"""Re-evaluate immutable WebArena evidence across a pinned evaluator update.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from webarena_verified.core.utils.checksum import compute_evaluator_checksum + +from harness.trajectory_replay.catalog_contracts import file_sha256, validate_passing_evaluation +from harness.trajectory_replay.webarena_corrections import ( + apply_correction, + evaluate_corrected_contract, +) +from harness.trajectory_replay.webarena_runtime_source import ( + WEBARENA_VERIFIED_FORK_URL, + WEBARENA_VERIFIED_REVISION, +) + +TRANSITION_SCHEMA = "lex_webarena_evaluator_transition/1" +RECEIPT_SCHEMA = "lex_webarena_evaluator_transition_receipt/3" +SOURCE_KINDS = {"revision", "evaluator_fixes_sha256"} + + +def validate_source_pass( + evaluation: dict[str, Any], + *, + task_id: int, + source: dict[str, str], +) -> None: + """Validate immutable source-era provenance before current re-evaluation. + + Source-era correction metadata cannot be compared with the current correction + catalog because accepted fixes may have moved upstream. The transition receipt + binds the exact old evaluation, response, HAR and current re-evaluation. + """ + evaluator_rows = evaluation.get("evaluators_results") or [] + correction_id = evaluation.get("contract_correction_id") + corrected_task = evaluation.get("corrected_task_sha256") + if ( + evaluation.get("task_id") != task_id + or float(evaluation.get("score") or 0) != 1.0 + or str(evaluation.get("status") or "").lower() not in {"pass", "success"} + or evaluation.get("webarena_verified_evaluator_checksum") + != source["evaluator_checksum"] + or evaluation.get("webarena_verified_data_checksum") != source["data_checksum"] + or not isinstance(evaluator_rows, list) + or not evaluator_rows + or any( + float(row.get("score") or 0) != 1.0 + or str(row.get("status") or "").lower() not in {"pass", "success"} + for row in evaluator_rows + ) + or ((correction_id is None) != (corrected_task is None)) + or (corrected_task is not None and len(str(corrected_task)) != 64) + ): + raise ValueError(f"task {task_id} source pass has stale or incomplete provenance") + + +def json_sha256(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def load_transition(path: Path) -> dict[str, Any]: + transition = json.loads(path.read_text(encoding="utf-8")) + compatible = transition.get("compatible_from_revisions") + sources = transition.get("compatible_sources") + source_keys = { + (source.get("kind"), source.get("identity")) + for source in sources or [] + if isinstance(source, dict) + } + if ( + transition.get("schema") != TRANSITION_SCHEMA + or transition.get("repository") != WEBARENA_VERIFIED_FORK_URL + or transition.get("to_revision") != WEBARENA_VERIFIED_REVISION + or transition.get("to_evaluator_checksum") != compute_evaluator_checksum() + or not isinstance(compatible, list) + or transition.get("from_revision") not in compatible + or len(set(compatible)) != len(compatible) + or any(not isinstance(value, str) or len(value) != 40 for value in compatible) + or not isinstance(sources, list) + or not sources + or len(source_keys) != len(sources) + or any( + not isinstance(source, dict) + or source.get("kind") not in SOURCE_KINDS + or not isinstance(source.get("identity"), str) + or len(source["identity"]) + != (40 if source.get("kind") == "revision" else 64) + or any( + not isinstance(source.get(name), str) or len(source[name]) != 64 + for name in ("evaluator_checksum", "data_checksum") + ) + for source in sources + ) + or {identity for kind, identity in source_keys if kind == "revision"} + != set(compatible) + or any( + not isinstance(transition.get(key), str) or len(transition[key]) != 40 + for key in ("from_revision", "to_revision") + ) + or any( + not isinstance(transition.get(key), str) or len(transition[key]) != 64 + for key in ( + "from_evaluator_checksum", + "from_data_checksum", + "to_evaluator_checksum", + "to_data_checksum", + "fork_diff_sha256", + ) + ) + ): + raise ValueError("WebArena evaluator transition authority is invalid") + return transition + + +def compatible_source( + transition: dict[str, Any], + *, + revision: str | None, + evaluator_fixes_sha256: str | None, +) -> dict[str, str] | None: + """Resolve the exact evaluator/data identity used by an older run.""" + + candidates = [] + if isinstance(revision, str): + candidates.append(("revision", revision)) + if isinstance(evaluator_fixes_sha256, str): + candidates.append(("evaluator_fixes_sha256", evaluator_fixes_sha256)) + matches = [ + source + for source in transition["compatible_sources"] + if (source["kind"], source["identity"]) in candidates + ] + if len(matches) > 1: + raise ValueError("run matches more than one evaluator transition source") + return matches[0] if matches else None + + +def _regular_json(path: Path) -> dict[str, Any]: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"transition evidence file is missing: {path.name}") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"transition evidence file is not an object: {path.name}") + return value + + +def reevaluate_pass( + task_dir: Path, + *, + task_id: int, + expected_data_checksum: str, + benchmark: Any, + corrections: Any, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Return the current evaluation and a binding for the immutable old evidence.""" + + old_evaluation_path = task_dir / "evaluation.json" + actual_response_path = task_dir / "actual_response.json" + network_har_path = task_dir / "network.har" + old_evaluation = _regular_json(old_evaluation_path) + actual_response = _regular_json(actual_response_path) + if network_har_path.is_symlink() or not network_har_path.is_file(): + raise ValueError("transition network evidence is missing") + + official = benchmark.get_task(task_id) + correction = ( + apply_correction(official, corrections) if task_id in corrections.records else None + ) + result = ( + evaluate_corrected_contract( + benchmark, + correction, + agent_response=actual_response, + network_trace=network_har_path, + ) + if correction is not None + else benchmark.evaluate_task( + task_id=task_id, + agent_response=actual_response, + network_trace=network_har_path, + ) + ) + current = result.model_dump(mode="json") + current["replay_assertions"] = old_evaluation.get("replay_assertions") or [] + if correction is not None: + current["contract_correction_id"] = correction.correction_id + current["corrected_task_sha256"] = correction.corrected_task_sha256 + validate_passing_evaluation( + current, + task_id=task_id, + expected_data_checksum=expected_data_checksum, + benchmark=benchmark, + corrections=corrections, + ) + return current, { + "task_id": task_id, + "old_evaluation_sha256": file_sha256(old_evaluation_path), + "actual_response_sha256": file_sha256(actual_response_path), + "network_har_sha256": file_sha256(network_har_path), + "current_evaluation_sha256": json_sha256(current), + } + + +def load_receipt( + path: Path, transition: dict[str, Any] +) -> tuple[ + dict[tuple[int, str, str], dict[str, Any]], + set[tuple[int, str, str]], +]: + receipt = json.loads(path.read_text(encoding="utf-8")) + rows = receipt.get("rows") if isinstance(receipt, dict) else None + excluded = ( + receipt.get("excluded_old_evaluations", []) if isinstance(receipt, dict) else None + ) + excluded_keys = [ + ( + row.get("task_id"), + row.get("old_evaluation_sha256"), + row.get("run_inputs_sha256"), + ) + for row in excluded or [] + if isinstance(row, dict) + ] + if ( + receipt.get("schema") != RECEIPT_SCHEMA + or receipt.get("transition_sha256") != json_sha256(transition) + or receipt.get("from_revision") != transition["from_revision"] + or receipt.get("compatible_from_revisions") + != transition["compatible_from_revisions"] + or receipt.get("compatible_sources") != transition["compatible_sources"] + or receipt.get("to_revision") != transition["to_revision"] + or receipt.get("from_evaluator_checksum") != transition["from_evaluator_checksum"] + or receipt.get("from_data_checksum") != transition["from_data_checksum"] + or receipt.get("to_evaluator_checksum") != transition["to_evaluator_checksum"] + or receipt.get("to_data_checksum") != transition["to_data_checksum"] + or not isinstance(rows, list) + or not isinstance(excluded, list) + or len(excluded_keys) != len(excluded) + or excluded_keys != sorted(set(excluded_keys)) + or any( + type(task_id) is not int + or not isinstance(evaluation_sha256, str) + or len(evaluation_sha256) != 64 + or not isinstance(run_inputs_sha256, str) + or len(run_inputs_sha256) != 64 + for task_id, evaluation_sha256, run_inputs_sha256 in excluded_keys + ) + ): + raise ValueError("WebArena evaluator transition receipt is invalid") + indexed: dict[tuple[int, str, str], dict[str, Any]] = {} + for row in rows: + key = ( + row.get("task_id"), + row.get("old_evaluation_sha256"), + row.get("run_inputs_sha256"), + ) + if ( + type(key[0]) is not int + or not isinstance(key[1], str) + or len(key[1]) != 64 + or key in indexed + or any( + not isinstance(row.get(name), str) or len(row[name]) != 64 + for name in ( + "actual_response_sha256", + "network_har_sha256", + "current_evaluation_sha256", + "run_inputs_sha256", + ) + ) + or not isinstance(row.get("source"), dict) + or row["source"] not in transition["compatible_sources"] + ): + raise ValueError("WebArena evaluator transition receipt row is invalid") + indexed[key] = row + if receipt.get("row_count") != len(rows): + raise ValueError("WebArena evaluator transition receipt count changed") + if set(indexed) & set(excluded_keys): + raise ValueError("WebArena evaluator transition receipt includes excluded evidence") + return indexed, set(excluded_keys) diff --git a/harness/trajectory_replay/failure_matrix.py b/harness/trajectory_replay/failure_matrix.py index a9b1ff193..43086fd41 100644 --- a/harness/trajectory_replay/failure_matrix.py +++ b/harness/trajectory_replay/failure_matrix.py @@ -27,6 +27,137 @@ ) +def _reviewed_failures(rows: list[dict], participant: str) -> list[dict]: + selected = [ + row + for row in rows + if row["participant_id"] == participant and row["status"] != "pass" + ] + if any(not row.get("cause_code") for row in selected): + raise ValueError(f"failure profile has an unreviewed result: {participant}") + return selected + + +def _cause_groups(rows: list[dict]) -> list[tuple[str, list[dict]]]: + grouped: dict[str, list[dict]] = {} + for row in rows: + grouped.setdefault(str(row["cause_code"]), []).append(row) + return sorted(grouped.items(), key=lambda item: (-len(item[1]), item[0])) + + +def failure_signature_cards( + methods: dict, rows: list[dict], lang: str, *, participants: tuple[str, ...] +) -> str: + """Render a compact, data-bearing failure signature for summary pages.""" + zh = lang == "zh" + cards = [] + for participant in participants: + if participant not in methods: + raise ValueError(f"failure signature participant is not registered: {participant}") + failures = _reviewed_failures(rows, participant) + population = sum(row["participant_id"] == participant for row in rows) + groups = _cause_groups(failures) + dominant = groups[:3] + items = [] + for _code, members in dominant: + sample = members[0] + label = sample.get("cause_zh" if zh else "cause_en") or sample["cause_code"] + share = len(members) / len(failures) * 100 if failures else 0 + query = urlencode( + { + "participant": participant, + "status": "fail", + "cause": sample.get("cause_zh") or sample["cause_code"], + } + ) + items.append( + '
  • ' + + escape(str(label)) + + f'{len(members)}
    ' + + f'
    ' + + f'' + + ("查看这些任务" if zh else "Inspect tasks") + + "
    • " + ) + remainder = len(failures) - sum(len(members) for _, members in dominant) + if remainder: + items.append( + f'
  • {"其余类型" if zh else "Other subtypes"}' + f'{remainder}
    • ' + ) + cards.append( + f'
    {"官方失败" if zh else "Official failures"}' + f'{len(failures)} / {population}
    {escape(methods[participant]["label"])}
    ' + f'
      {"".join(items)}
    ' + ) + participant_count = len(participants) + heading = ( + f"{participant_count} 款浏览器的主导错误" + if zh + else f"Dominant failures in {participant_count} " + + ("browser" if participant_count == 1 else "browsers") + ) + note = ( + "每条失败只进入一个复核子类。这里显示各方法数量最多的三类；完整分类、原因和逐题证据见报告解读。" + if zh + else "Each failure belongs to one reviewed subtype. These are the three largest for each method; Interpretation contains every subtype, explanation, and case evidence." + ) + return ( + f'

    {heading}

    ' + f'

    {note}

    ' + + ( + f"查看 {participant_count} 章详解" + if zh + else f"Read {participant_count} browser " + + ("chapter" if participant_count == 1 else "chapters") + ) + + f'
    {"".join(cards)}
    ' + ) + + +def browser_failure_profile(methods: dict, rows: list[dict], lang: str, participant: str) -> str: + """Render one browser's complete reviewed subtype distribution and evidence links.""" + if participant not in methods: + raise ValueError(f"failure profile participant is not registered: {participant}") + zh = lang == "zh" + selected = [row for row in rows if row["participant_id"] == participant] + failures = _reviewed_failures(rows, participant) + groups = _cause_groups(failures) + if not selected or not failures: + raise ValueError(f"failure profile requires a failing population: {participant}") + maximum = max(len(members) for _, members in groups) + bars = [] + explanations = [] + for code, members in groups: + sample = min(members, key=lambda row: int(row["task_id"])) + count = len(members) + label = sample.get("cause_zh" if zh else "cause_en") or code + reason = sample.get("reason_zh" if zh else "reason_en") or label + share = count / len(failures) * 100 + width = count / maximum * 100 + bars.append( + f'
  • {escape(str(label))}' + f'
    ' + f'{count}{share:.1f}%
    • ' + ) + detail = f'result/{escape(str(sample["key"]), quote=True)}.html' + explanations.append( + f'' + ) + passed = sum(row["status"] == "pass" for row in selected) + label = methods[participant]["label"] + return ( + f'
    ' + f'
    {escape(label)}' + f'{passed}/{len(selected)}{"官方通过" if zh else "official passes"}
    ' + f'
    {len(failures)}{"条失败，全部已复核" if zh else "failures, all reviewed"}
    ' + f'
      {"".join(bars)}
    ' + f'
    {"".join(explanations)}
    ' + ) + + def failure_category_matrix(methods: dict, rows: list[dict], lang: str, *, embedded: bool = False) -> str: """Count each failed method-task once, including unreviewed failures.""" zh = lang == "zh" diff --git a/harness/trajectory_replay/interpretation.py b/harness/trajectory_replay/interpretation.py index f2eff696c..22b666879 100644 --- a/harness/trajectory_replay/interpretation.py +++ b/harness/trajectory_replay/interpretation.py @@ -7,7 +7,11 @@ from pathlib import Path from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit, urlunsplit -from harness.trajectory_replay.failure_matrix import failure_category_matrix +from harness.trajectory_replay.failure_matrix import ( + browser_failure_profile, + failure_category_matrix, + failure_signature_cards, +) def _report_page_url(report_url: str, relative: str) -> str: @@ -17,6 +21,23 @@ def _report_page_url(report_url: str, relative: str) -> str: return urlunsplit((target.scheme, target.netloc, target.path, query, target.fragment)) +def _outcome_chart(methods: dict, rows: list[dict], lang: str) -> str: + """Render exact pass counts as a slide-friendly visual, not another data table.""" + zh = lang == "zh" + entries = [] + for key, method in methods.items(): + selected = [row for row in rows if row["participant_id"] == key] + passed = sum(row["status"] == "pass" for row in selected) + rate = passed / len(selected) * 100 if selected else 0 + entries.append( + f'
  • {html.escape(method["label"])}' + f'{passed}/{len(selected)} · {rate:.1f}%
    ' + f'
    • ' + ) + caption = "官方通过数／812 题" if zh else "Official passes / 812 tasks" + return f'
      {"".join(entries)}
    {caption}
    ' + + def render_interpretation(*, source: Path, contract_audit: Path, results_sha256: str, methods: dict, rows: list[dict], lang: str, image_dir: Path, image_prefix: str, standalone: bool = False, @@ -39,16 +60,24 @@ def render_interpretation(*, source: Path, contract_audit: Path, results_sha256: if identity in seen: raise ValueError("duplicate article section") seen.add(identity) - body.append(f'

    {e(section["title"][lang])}

    ') + body.append( + f'

    {e(section["title"][lang])}

    ' + ) if section.get("data") == "outcomes": - headers = ("方法", "原始通过数", "任务总数") if lang == "zh" else ("Method", "Original passes", "Tasks") - table_rows = [] - for key, method in methods.items(): - selected = [row for row in rows if row["participant_id"] == key] - table_rows.append([method["label"], sum(row["status"] == "pass" for row in selected), len(selected)]) - body.append(_table(headers, table_rows)) + body.append(_outcome_chart(methods, rows, lang)) elif section.get("data") == "failure_matrix": body.append(failure_category_matrix(methods, rows, lang, embedded=True)) + elif section.get("data") == "failure_signatures": + participants = tuple(section.get("participants") or ()) + body.append( + failure_signature_cards( + methods, rows, lang, participants=participants + ).replace('href="interpretation.html#browser-diagnoses"', 'href="#browser-diagnoses"') + ) + elif section.get("data") == "browser_failure_profile": + body.append( + browser_failure_profile(methods, rows, lang, section["participant"]) + ) elif section.get("data") == "contract_changes": headers = ("题目", "原合同", "改动及影响") if lang == "zh" else ("Task", "Original contract", "Change and effect") body.append(_table(headers, [ @@ -113,11 +142,16 @@ def localized(value): if standalone: if not report_url.startswith("https://"): raise ValueError("offline article requires the bound HTTPS report URL") - # Only the registry-generated matrix creates links in this renderer. + # Data visuals create report-relative evidence links. Preserve the share token + # in the script-free standalone article. def absolute_result_link(match): target = _report_page_url(report_url, html.unescape(match[1])) return f'href="{e(target)}"' - result = re.sub(r'href="(results\.html[^"]*)"', absolute_result_link, result) + result = re.sub( + r'href="((?:results\.html|result/[^"?#]+\.html)[^"]*)"', + absolute_result_link, + result, + ) return title, result diff --git a/harness/trajectory_replay/playwright_tool.py b/harness/trajectory_replay/playwright_tool.py new file mode 100644 index 000000000..3c2ab7a1b --- /dev/null +++ b/harness/trajectory_replay/playwright_tool.py @@ -0,0 +1,1328 @@ +"""CowAgent-free execution of frozen BrowserTool replay calls over Chromium CDP.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import ipaddress +import json +import re +import signal +import threading +import time +from dataclasses import dataclass +from typing import Any +from urllib.parse import urljoin, urlsplit + +import requests +from playwright.sync_api import sync_playwright +from requests.adapters import HTTPAdapter + +from harness.browser_use.runner import BENCHMARK_VIEWPORT +from harness.trajectory_replay.public_webarena.executor import execute_steps + +SNAPSHOT_NAVIGATION_SETTLE_TIMEOUT_MS = 1_500 +SNAPSHOT_NAVIGATION_GRACE_MS = 300 +REQUEST_BODY_CAPTURE_BINDING = "__browserEvalCaptureRequestBody" +REQUEST_BODY_CAPTURE_HEADER = "x-browser-eval-request-body-id" +NATIVE_FORM_REDIRECT_HEADER = "x-browser-eval-native-form-location" +REQUEST_BODY_CAPTURE_LIMIT_BYTES = 20_000_000 +REQUEST_BODY_CAPTURE_MAX_PENDING = 256 +REQUEST_BODY_CAPTURE_TTL_SECONDS = 120.0 +_REQUEST_BODY_CAPTURE_ID = re.compile(r"[A-Za-z0-9._:-]{1,128}\Z") + + +class _HostOverrideAdapter(HTTPAdapter): + """Connect selected HTTP hosts to a pinned address without changing URL identity.""" + + def __init__(self, overrides: tuple[tuple[str, str], ...]): + super().__init__() + self._overrides = overrides + + def _address_for(self, hostname: str) -> str | None: + hostname = hostname.lower() + for pattern, address in self._overrides: + if hostname == pattern or ( + pattern.startswith("*.") and hostname.endswith(pattern[1:]) + ): + return address + return None + + def get_connection_with_tls_context( + self, + request: Any, + verify: bool | str, + proxies: dict[str, str] | None = None, + cert: Any = None, + ) -> Any: + parsed = urlsplit(request.url) + address = self._address_for(parsed.hostname or "") + if address is None: + return super().get_connection_with_tls_context( + request, verify, proxies=proxies, cert=cert + ) + if parsed.scheme != "http": + raise ValueError("request bridge host overrides support HTTP origins only") + request.headers["Host"] = parsed.netloc + return self.poolmanager.connection_from_host( + scheme="http", + host=address, + port=parsed.port or 80, + ) + + +def _validated_host_overrides( + overrides: list[dict[str, str]] | None, +) -> tuple[tuple[str, str], ...]: + normalized: list[tuple[str, str]] = [] + for row in overrides or []: + pattern = row.get("pattern") if isinstance(row, dict) else None + address = row.get("address") if isinstance(row, dict) else None + if ( + not isinstance(pattern, str) + or not isinstance(address, str) + or not pattern + or pattern != pattern.lower() + or pattern.count("*") > 1 + or ("*" in pattern and not pattern.startswith("*.")) + or any(char in pattern for char in "/:,\n\r\t ") + ): + raise ValueError("request bridge host override pattern is invalid") + try: + ipaddress.ip_address(address) + except ValueError: + raise ValueError("request bridge host override address is invalid") from None + normalized.append((pattern, address)) + if len({pattern for pattern, _ in normalized}) != len(normalized): + raise ValueError("request bridge host override pattern is duplicated") + return tuple( + sorted( + normalized, + key=lambda item: ( + item[0].startswith("*."), + -len(item[0]), + item[0], + ), + ) + ) + + +def _http_origin(url: str) -> str: + parsed = urlsplit(url) + scheme = parsed.scheme.lower() + hostname = (parsed.hostname or "").lower() + if scheme not in {"http", "https"} or not hostname: + return "" + host = f"[{hostname}]" if ":" in hostname else hostname + port = parsed.port + if port is not None and not ( + (scheme == "http" and port == 80) or (scheme == "https" and port == 443) + ): + host = f"{host}:{port}" + return f"{scheme}://{host}" + + +def _navigation_interrupted_snapshot(exc: Exception) -> bool: + message = str(exc).lower() + return any( + marker in message + for marker in ( + "navigation to finish", + "execution context was destroyed", + "cannot find context with specified id", + "frame was detached", + "inspected target navigated or closed", + ) + ) + + +@dataclass +class ReplayResult: + status: str + result: str + + +def _request_body_capture_script(origins: frozenset[str]) -> str: + """Capture selected fetch bodies before Kitesurf drops them from CDP.""" + + return f""" +(() => {{ + const origins = new Set({json.dumps(sorted(origins))}); + const binding = {json.dumps(REQUEST_BODY_CAPTURE_BINDING)}; + const header = {json.dumps(REQUEST_BODY_CAPTURE_HEADER)}; + const nativeRedirectHeader = {json.dumps(NATIVE_FORM_REDIRECT_HEADER)}; + const limit = {REQUEST_BODY_CAPTURE_LIMIT_BYTES}; + if (globalThis.__browserEvalRequestBodyCaptureInstalled) return; + globalThis.__browserEvalRequestBodyCaptureInstalled = true; + const prefix = `${{Date.now().toString(36)}}-${{Math.random().toString(36).slice(2)}}`; + let sequence = 0; + let keepaliveBodyBytes = 0; + + const selected = url => {{ + try {{ return origins.has(new URL(url, document.baseURI).origin); }} + catch (_) {{ return false; }} + }}; + const byteLength = text => {{ + if (text.length > limit) {{ + throw new Error('request body exceeds the capture limit'); + }} + const size = new TextEncoder().encode(text).byteLength; + if (size > limit) {{ + throw new Error('request body exceeds the capture limit'); + }} + return size; + }}; + const checkedText = text => {{ + byteLength(text); + return text; + }}; + const urlEncodedLength = text => {{ + let size = 0; + for (const character of text) {{ + size += /^[A-Za-z0-9*._-]$/.test(character) ? 1 : byteLength(character) * 3; + if (size > limit) throw new Error('request body exceeds the capture limit'); + }} + return size; + }}; + const encode = body => {{ + if (typeof body === 'string') return {{kind: 'text', text: checkedText(body)}}; + if (body instanceof URLSearchParams) {{ + let count = 0; + let total = 0; + for (const pair of body.entries()) {{ + if (count >= 10000) {{ + throw new Error('request body has too many URLSearchParams fields'); + }} + total += urlEncodedLength(String(pair[0])); + total += urlEncodedLength(String(pair[1])); + total += count === 0 ? 1 : 2; + if (total > limit) {{ + throw new Error('request body exceeds the capture limit'); + }} + count += 1; + }} + return {{kind: 'text', text: checkedText(body.toString())}}; + }} + if (body instanceof FormData) {{ + const entries = []; + let total = 0; + for (const pair of body.entries()) {{ + if (entries.length >= 10000) {{ + throw new Error('request body has too many FormData fields'); + }} + const name = String(pair[0]); + const value = pair[1]; + total += byteLength(name); + if (typeof value === 'string') {{ + total += byteLength(value); + entries.push({{name, value}}); + }} else {{ + if (!(value instanceof File) || value.size !== 0) {{ + throw new Error('request body capture supports only empty FormData files'); + }} + total += byteLength(value.name); + total += byteLength(value.type); + entries.push({{ + name, + file: {{name: value.name, type: value.type, base64: ''}}, + }}); + }} + if (total > limit) {{ + throw new Error('request body exceeds the capture limit'); + }} + }} + return {{kind: 'form-data', entries}}; + }} + throw new Error(`unsupported request body ${{Object.prototype.toString.call(body)}}`); + }}; + const capture = async (method, url, body, source = 'fetch') => {{ + const absolute = new URL(url, document.baseURI).href; + if (body === undefined || body === null || !selected(absolute)) return null; + const requestId = `${{prefix}}-${{sequence += 1}}`; + await globalThis[binding]({{ + requestId, + method: String(method || 'GET').toUpperCase(), + url: absolute, + body: encode(body), + source, + }}); + return requestId; + }}; + + const originalFetch = globalThis.fetch; + globalThis.fetch = async function(input, init) {{ + const options = init || {{}}; + const method = options.method || (input && input.method) || 'GET'; + const url = typeof input === 'string' ? input : String(input && input.url || input); + const hasInitBody = Object.prototype.hasOwnProperty.call(options, 'body'); + const hasBody = hasInitBody + ? options.body !== undefined && options.body !== null + : input instanceof Request && input.body !== null; + if (hasInitBody) {{ + const normalizedMethod = String(method).toUpperCase(); + if ( + hasBody + && (normalizedMethod === 'GET' || normalizedMethod === 'HEAD') + ) {{ + throw new TypeError('Request with GET/HEAD method cannot have body.'); + }} + }} + let keepaliveSize = 0; + const effectiveKeepalive = Object.prototype.hasOwnProperty.call(options, 'keepalive') + ? Boolean(options.keepalive) + : input instanceof Request && input.keepalive; + if (hasBody && effectiveKeepalive) {{ + const absolute = new URL(url, document.baseURI).href; + const probe = hasInitBody + ? new Request(input instanceof Request ? input : absolute, options) + : input.clone(); + keepaliveSize = (await probe.clone().arrayBuffer()).byteLength; + if (keepaliveBodyBytes + keepaliveSize > 65536) {{ + throw new TypeError('Failed to fetch'); + }} + keepaliveBodyBytes += keepaliveSize; + }} + try {{ + if (hasInitBody) {{ + const requestId = await capture(method, url, options.body); + if (requestId !== null) {{ + const inherited = input instanceof Request ? input.headers : undefined; + const headers = new Headers(options.headers || inherited); + if (!headers.has('content-type')) {{ + if (options.body instanceof URLSearchParams) {{ + headers.set('content-type', 'application/x-www-form-urlencoded;charset=UTF-8'); + }} else if (typeof options.body === 'string') {{ + headers.set('content-type', 'text/plain;charset=UTF-8'); + }} + }} + headers.set(header, requestId); + return await originalFetch.call(this, input, {{...options, headers, body: ''}}); + }} + }} else if (input instanceof Request && input.body !== null && selected(url)) {{ + throw new Error('request body capture requires the body in fetch init'); + }} + return await originalFetch.call(this, input, init); + }} finally {{ + keepaliveBodyBytes -= keepaliveSize; + }} + }}; + + const originalSubmit = HTMLFormElement.prototype.submit; + HTMLFormElement.prototype.submit = function() {{ + const method = String(this.method || 'get').toUpperCase(); + const url = this.action || document.URL; + if (method !== 'POST' || !selected(url) || (this.target && this.target !== '_self')) {{ + return originalSubmit.call(this); + }} + const enctype = String(this.enctype || 'application/x-www-form-urlencoded').toLowerCase(); + const formData = new FormData(this); + let body; + if (enctype === 'application/x-www-form-urlencoded') {{ + body = new URLSearchParams(); + for (const [name, value] of formData.entries()) {{ + const normalizedName = String(name).replace(/\\r\\n|\\r|\\n/g, '\\r\\n'); + const rawValue = typeof value === 'string' ? value : value.name; + body.append(normalizedName, String(rawValue).replace(/\\r\\n|\\r|\\n/g, '\\r\\n')); + }} + }} else if (enctype === 'multipart/form-data') {{ + body = formData; + }} else {{ + throw new Error(`unsupported native form enctype ${{enctype}}`); + }} + void (async () => {{ + const requestId = await capture(method, url, body, 'native-form'); + const headers = new Headers(); + headers.set(header, requestId); + const response = await originalFetch(url, {{ + method, + credentials: 'same-origin', + body, + headers, + redirect: 'manual', + }}); + const redirect = response.headers.get(nativeRedirectHeader); + if (!redirect) throw new Error('native form bridge returned no supported redirect'); + location.assign(redirect); + }})(); + }}; +}})(); +""" + + +class PlaywrightReplayTool: + """Expose the small BrowserTool contract consumed by the shared replay runner. + + The shared snapshot-ref path still dispatches DOM click/fill/select itself, + preserving its compiled-ref behavior on either backend. + """ + + def __init__( + self, + endpoint: str, + *, + engine: str = "chromium", + headers: dict[str, str] | None = None, + request_bridge_origins: set[str] | None = None, + request_bridge_host_overrides: list[dict[str, str]] | None = None, + ): + if engine not in {"chromium", "moli", "lightpanda", "obscura", "kitesurf"}: + raise ValueError(f"unknown Playwright replay engine: {engine}") + self._engine = engine + self._supports_multiple_pages = engine not in {"lightpanda", "obscura", "kitesurf"} + self._supports_network_reset_cdp = engine not in {"obscura", "kitesurf"} + self._supports_storage_cdp = engine not in {"lightpanda", "obscura", "kitesurf"} + self._native_login_form_submit = engine == "kitesurf" + self._fresh_context_per_task = True + self._supports_context_request_cookie_sharing = engine not in { + "lightpanda", + "kitesurf", + } + self.har_capture_transport = "playwright" if engine == "kitesurf" else "cdp" + self.har_request_transform = ( + self.transform_playwright_har_request if engine == "kitesurf" else None + ) + self._layout_sessions: dict[Any, Any] = {} + self._viewport_sessions: dict[Any, Any] = {} + self._layout_connection_poisoned = False + self._playwright = sync_playwright().start() + self._browser = None + self._context = None + self.runtime_metadata: dict[str, Any] | None = None + self.request_bridge_stats = {"requests": 0, "failures": 0} + self._request_bridge_session = None + self._request_bridge_origins: frozenset[str] = frozenset() + self._request_bridge_bodies: dict[str, dict[str, Any]] = {} + self._request_bridge_body_bytes = 0 + self._request_bridge_har_bodies: dict[str, dict[str, Any]] = {} + self._request_bridge_har_body_bytes = 0 + try: + connect_options: dict[str, Any] = {"timeout": 90_000} + if headers is not None: + connect_options["headers"] = dict(headers) + self._browser = self._playwright.chromium.connect_over_cdp( + endpoint, **connect_options + ) + if engine == "kitesurf": + browser_session = self._browser.new_browser_cdp_session() + try: + version = browser_session.send("Browser.getVersion") + finally: + browser_session.detach() + self.runtime_metadata = { + key: version.get(key) + for key in ( + "protocolVersion", + "product", + "revision", + "userAgent", + "jsVersion", + ) + if version.get(key) is not None + } + if len(self._browser.contexts) != 1: + raise RuntimeError("Kitesurf did not expose exactly one default context") + self._context = self._browser.contexts[0] + if len(self._context.pages) != 1: + raise RuntimeError("Kitesurf did not expose exactly one default page") + self._page = self._context.pages[0] + self._install_request_bridge( + request_bridge_origins or set(), + request_bridge_host_overrides, + ) + self._page.set_viewport_size(dict(BENCHMARK_VIEWPORT)) + # Kitesurf applies emulation to the next document; its initial + # provider-owned about:blank retains the service default height. + self._page.goto("about:blank", wait_until="load", timeout=10_000) + else: + self._context = self._browser.new_context( + viewport=dict(BENCHMARK_VIEWPORT), locale="en-US" + ) + self._page = self._context.new_page() + except Exception: + self.close() + raise + self._owns_cdp_context = True + + def _install_request_bridge( + self, + origins: set[str], + host_overrides: list[dict[str, str]] | None = None, + ) -> None: + """Fetch selected private origins from the local Playwright driver.""" + + parsed_origins = [urlsplit(origin) for origin in origins] + if any( + parsed.scheme not in {"http", "https"} + or not parsed.netloc + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + for parsed in parsed_origins + ): + raise ValueError("request bridge origins must be bare HTTP(S) origins") + normalized = {_http_origin(origin) for origin in origins} + if not normalized: + return + self._request_bridge_origins = frozenset(normalized) + self._request_bridge_bodies = {} + self._request_bridge_body_bytes = 0 + self._request_bridge_har_bodies = {} + self._request_bridge_har_body_bytes = 0 + self._request_bridge_session = requests.Session() + self._request_bridge_session.trust_env = False + normalized_overrides = _validated_host_overrides(host_overrides) + if normalized_overrides: + for scheme in ("http://", "https://"): + self._request_bridge_session.mount( + scheme, _HostOverrideAdapter(normalized_overrides) + ) + self._context.expose_binding( + REQUEST_BODY_CAPTURE_BINDING, self._capture_request_body + ) + self._context.add_init_script( + _request_body_capture_script(self._request_bridge_origins) + ) + + def selected(url: str) -> bool: + return _http_origin(url) in normalized + + def bridge(route) -> None: + self.request_bridge_stats["requests"] += 1 + try: + request = route.request + headers = { + name: value + for name, value in request.all_headers().items() + if name.lower() + not in { + "accept-encoding", + "connection", + "content-length", + "host", + "proxy-connection", + "signature", + "signature-agent", + "signature-input", + "transfer-encoding", + REQUEST_BODY_CAPTURE_HEADER, + } + } + request_headers = request.all_headers() + capture_id = request_headers.get(REQUEST_BODY_CAPTURE_HEADER) + data = request.post_data_buffer + content_type = None + if capture_id is not None: + data, content_type = self._request_bridge_body_for_route( + capture_id, request.method, request.url + ) + if content_type is not None: + headers = { + name: value + for name, value in headers.items() + if name.lower() != "content-type" + } + headers["content-type"] = content_type + response = self._request_bridge_session.request( + method=request.method, + url=request.url, + headers=headers, + data=data, + allow_redirects=False, + timeout=60, + ) + if capture_id is not None: + self._record_request_bridge_redirect( + capture_id, request.method, request.url, response + ) + self._record_request_bridge_response( + capture_id, + request.method, + request.url, + response, + ) + self._sync_request_bridge_cookies( + str(response.url), response.cookies + ) + response_headers = { + name: value + for name, value in response.headers.items() + if name.lower() + not in { + "connection", + "content-encoding", + "content-length", + "set-cookie", + "transfer-encoding", + } + } + source = self._request_bridge_har_bodies.get( + capture_id or "", {} + ).get("source") + if source == "native-form": + redirect_url = self._native_form_redirect_url(request.url, response) + route.fulfill( + status=200, + headers={NATIVE_FORM_REDIRECT_HEADER: redirect_url}, + body=b"", + ) + else: + route.fulfill( + status=response.status_code, + headers=response_headers, + body=response.content, + ) + except Exception: + self.request_bridge_stats["failures"] += 1 + try: + route.abort("failed") + except Exception: + pass + raise + + self._context.route(selected, bridge) + + def transform_playwright_har_request( + self, method: str, url: str, headers: dict[str, Any] + ) -> tuple[dict[str, Any], str | None, dict[str, Any] | None]: + """Restore a Kitesurf fetch body and remove its private correlation header.""" + + capture_id = next( + ( + str(value) + for name, value in headers.items() + if str(name).lower() == REQUEST_BODY_CAPTURE_HEADER + ), + None, + ) + cleaned = { + name: value + for name, value in headers.items() + if str(name).lower() != REQUEST_BODY_CAPTURE_HEADER + } + normalized_url = urlsplit(url)._replace(fragment="").geturl() + request_key = (method.upper(), normalized_url) + record = self._request_bridge_har_bodies.get(capture_id or "") + if record is None: + return cleaned, None, None + if request_key not in record["allowed_requests"]: + raise RuntimeError("HAR request body does not match its routed request") + response_transform = record.get("response_transforms", {}).get(request_key) + if request_key not in record["body_requests"]: + return cleaned, None, response_transform + if record["content_type"] is not None: + cleaned = { + name: value + for name, value in cleaned.items() + if str(name).lower() != "content-type" + } + cleaned["content-type"] = record["content_type"] + return cleaned, record["post_data"], response_transform + + def _request_bridge_body_for_route( + self, request_id: str, method: str, url: str + ) -> tuple[bytes | None, str | None]: + normalized_url = urlsplit(url)._replace(fragment="").geturl() + request_key = (method.upper(), normalized_url) + self._purge_captured_request_bodies() + if request_id in self._request_bridge_bodies: + source = self._request_bridge_bodies[request_id].get("source", "fetch") + captured = self._pop_captured_request_body(request_id, method, url) + data, content_type = self._decode_captured_request_body( + request_id, captured + ) + if ( + self._request_bridge_har_body_bytes + len(data) + > REQUEST_BODY_CAPTURE_LIMIT_BYTES + ): + raise RuntimeError("HAR request bodies exceed the total size limit") + self._request_bridge_har_bodies[request_id] = { + "post_data": data.decode("utf-8"), + "body_bytes": data, + "content_type": content_type, + "allowed_requests": {request_key}, + "body_requests": {request_key}, + "source": source, + "response_transforms": {}, + } + self._request_bridge_har_body_bytes += len(data) + return data, content_type + record = self._request_bridge_har_bodies.get(request_id) + if record is None or request_key not in record["allowed_requests"]: + raise RuntimeError("request body capture ID is missing or expired") + if request_key not in record["body_requests"]: + return None, None + return record["body_bytes"], record["content_type"] + + def _record_request_bridge_response( + self, request_id: str, method: str, url: str, response: Any + ) -> None: + record = self._request_bridge_har_bodies.get(request_id) + if record is None: + return + raw_cookie = next( + ( + str(value) + for name, value in response.headers.items() + if str(name).lower() == "set-cookie" and "mage-messages=" in str(value) + ), + "", + ) + request_key = ( + method.upper(), + urlsplit(url)._replace(fragment="").geturl(), + ) + if request_key not in record["allowed_requests"]: + raise RuntimeError("response evidence does not match its routed request") + transform: dict[str, Any] = {} + if raw_cookie: + transform["extra_headers"] = {"set-cookie": raw_cookie} + if record.get("source") == "native-form": + transform.update( + { + "status": int(response.status_code), + "statusText": str(getattr(response, "reason", "") or ""), + "headers": { + str(name): str(value) + for name, value in response.headers.items() + if str(name).lower() + not in { + "connection", + "content-encoding", + "content-length", + "set-cookie", + "transfer-encoding", + } + }, + } + ) + if transform: + record["response_transforms"][request_key] = transform + + def _native_form_redirect_url(self, request_url: str, response: Any) -> str: + if int(response.status_code) not in {302, 303}: + raise RuntimeError( + f"native form bridge requires a 302/303 redirect; got {response.status_code}" + ) + location = response.headers.get("location") + if not location: + raise RuntimeError("native form bridge redirect has no location") + target = urljoin(request_url, str(location)) + if _http_origin(target) != _http_origin(request_url): + raise RuntimeError("native form bridge rejected a cross-origin redirect") + return target + + def _record_request_bridge_redirect( + self, request_id: str, method: str, url: str, response: Any + ) -> None: + record = self._request_bridge_har_bodies.get(request_id) + if record is None or int(response.status_code) not in {301, 302, 303, 307, 308}: + return + location = next( + ( + str(value) + for name, value in response.headers.items() + if str(name).lower() == "location" + ), + "", + ) + if not location: + return + current_method = method.upper() + status = int(response.status_code) + if status == 303 and current_method != "HEAD": + next_method = "GET" + elif status in {301, 302} and current_method == "POST": + next_method = "GET" + else: + next_method = current_method + next_url = urlsplit(urljoin(url, location))._replace(fragment="").geturl() + next_key = (next_method, next_url) + record["allowed_requests"].add(next_key) + current_key = ( + current_method, + urlsplit(url)._replace(fragment="").geturl(), + ) + if next_method == current_method and current_key in record["body_requests"]: + record["body_requests"].add(next_key) + + def _capture_request_body(self, _source: Any, payload: Any) -> None: + if not isinstance(payload, dict): + raise ValueError("request body capture payload must be an object") + method = payload.get("method") + url = payload.get("url") + body = payload.get("body") + source = payload.get("source", "fetch") + request_id = payload.get("requestId") + if ( + not isinstance(request_id, str) + or _REQUEST_BODY_CAPTURE_ID.fullmatch(request_id) is None + ): + raise ValueError("request body capture has an invalid request ID") + if not isinstance(method, str) or not isinstance(url, str): + raise ValueError("request body capture is missing its method or URL") + method = method.upper() + parsed = urlsplit(url) + origin = _http_origin(url) + if origin not in self._request_bridge_origins: + raise ValueError("request body capture origin is not allowed") + if not isinstance(body, dict): + raise ValueError("request body capture is missing its body") + if source not in {"fetch", "native-form"}: + raise ValueError("request body capture has an invalid source") + body_size = self._validate_captured_request_body(body) + normalized_url = parsed._replace(fragment="").geturl() + self._purge_captured_request_bodies() + if ( + request_id in self._request_bridge_bodies + or request_id in self._request_bridge_har_bodies + ): + raise ValueError("request body capture reused a request ID") + if ( + len(self._request_bridge_bodies) + len(self._request_bridge_har_bodies) + >= REQUEST_BODY_CAPTURE_MAX_PENDING + ): + raise RuntimeError("request body capture count exceeded its bound") + if ( + self._request_bridge_body_bytes + + self._request_bridge_har_body_bytes + + body_size + > REQUEST_BODY_CAPTURE_LIMIT_BYTES + ): + raise RuntimeError("pending request bodies exceed the total size limit") + self._request_bridge_bodies[request_id] = { + "captured_at": time.monotonic(), + "method": method, + "url": normalized_url, + "body": body, + "size": body_size, + "source": source, + } + self._request_bridge_body_bytes += body_size + + @staticmethod + def _validate_captured_request_body(body: dict[str, Any]) -> int: + kind = body.get("kind") + if kind == "text": + text = body.get("text") + if not isinstance(text, str): + raise ValueError("captured text body is invalid") + size = len(text.encode("utf-8")) + if size > REQUEST_BODY_CAPTURE_LIMIT_BYTES: + raise ValueError("captured text body exceeds the size limit") + return size + if kind == "form-data": + entries = body.get("entries") + if not isinstance(entries, list) or len(entries) > 10_000: + raise ValueError("captured form body is invalid") + total = 0 + for entry in entries: + if not isinstance(entry, dict) or not isinstance(entry.get("name"), str): + raise ValueError("captured form entry is invalid") + if "value" in entry: + if not isinstance(entry["value"], str): + raise ValueError("captured form value is invalid") + total += len(entry["value"].encode("utf-8")) + elif "file" in entry: + file = entry["file"] + if ( + not isinstance(file, dict) + or not isinstance(file.get("name"), str) + or not isinstance(file.get("type"), str) + or file.get("base64") != "" + ): + raise ValueError("captured form supports only empty files") + total += len(file["name"].encode("utf-8")) + total += len(file["type"].encode("utf-8")) + else: + raise ValueError("captured form entry has no value") + total += len(entry["name"].encode("utf-8")) + if total > REQUEST_BODY_CAPTURE_LIMIT_BYTES: + raise ValueError("captured form body exceeds the size limit") + return total + raise ValueError(f"unsupported captured request body: {body.get('type', kind)}") + + def _purge_captured_request_bodies(self) -> None: + cutoff = time.monotonic() - REQUEST_BODY_CAPTURE_TTL_SECONDS + expired = [ + request_id + for request_id, capture in self._request_bridge_bodies.items() + if capture["captured_at"] < cutoff + ] + for request_id in expired: + capture = self._request_bridge_bodies.pop(request_id) + self._request_bridge_body_bytes -= capture["size"] + + def _pop_captured_request_body( + self, request_id: str, method: str, url: str + ) -> dict[str, Any]: + self._purge_captured_request_bodies() + parsed = urlsplit(url) + normalized_url = parsed._replace(fragment="").geturl() + capture = self._request_bridge_bodies.pop(request_id, None) + if capture is None: + raise RuntimeError("request body capture ID is missing or expired") + self._request_bridge_body_bytes -= capture["size"] + if capture["method"] != method.upper() or capture["url"] != normalized_url: + raise RuntimeError("request body capture does not match the routed request") + return capture["body"] + + @staticmethod + def _decode_captured_request_body( + request_id: str, body: dict[str, Any] + ) -> tuple[bytes, str | None]: + kind = body["kind"] + if kind == "text": + return body["text"].encode("utf-8"), None + boundary = f"browser-eval-{hashlib.sha256(request_id.encode()).hexdigest()[:24]}" + chunks: list[bytes] = [] + for entry in body["entries"]: + name = entry["name"] + if "\r" in name or "\n" in name: + raise ValueError("captured form field name contains a newline") + escaped_name = name.replace('"', "%22") + disposition = f'Content-Disposition: form-data; name="{escaped_name}"' + content = b"" + content_type = "" + if "value" in entry: + content = entry["value"].encode("utf-8") + else: + file = entry["file"] + filename = file["name"] + if "\r" in filename or "\n" in filename: + raise ValueError("captured form filename contains a newline") + escaped_filename = filename.replace('"', "%22") + disposition += f'; filename="{escaped_filename}"' + content_type = file["type"] or "application/octet-stream" + if "\r" in content_type or "\n" in content_type: + raise ValueError("captured form content type contains a newline") + content = base64.b64decode(file["base64"], validate=True) + chunks.extend((f"--{boundary}\r\n".encode(), disposition.encode(), b"\r\n")) + if content_type: + chunks.extend((f"Content-Type: {content_type}\r\n".encode(),)) + chunks.extend((b"\r\n", content, b"\r\n")) + chunks.append(f"--{boundary}--\r\n".encode()) + return b"".join(chunks), f"multipart/form-data; boundary={boundary}" + + def _sync_request_bridge_cookies( + self, response_url: str, response_cookies: Any + ) -> None: + parsed = urlsplit(response_url) + origin = _http_origin(response_url) + if origin not in self._request_bridge_origins: + raise RuntimeError("request bridge cookie response origin is not allowed") + host = (parsed.hostname or "").lower() + cookies = [] + for cookie in response_cookies: + item: dict[str, Any] = { + "name": cookie.name, + "value": cookie.value, + "secure": bool(cookie.secure), + "httpOnly": any( + str(name).lower() == "httponly" for name in cookie._rest + ), + } + if cookie.domain_specified: + cookie_domain = str(cookie.domain).lstrip(".").lower() + if host != cookie_domain and not host.endswith(f".{cookie_domain}"): + raise RuntimeError("request bridge rejected an invalid cookie domain") + item["domain"] = cookie.domain + item["path"] = cookie.path or "/" + else: + if (cookie.path or "/") != "/": + raise RuntimeError( + "request bridge cannot preserve a non-root host-only cookie path" + ) + item["url"] = f"{origin}/" + if cookie.expires is not None: + item["expires"] = float(cookie.expires) + same_site = next( + ( + value + for name, value in cookie._rest.items() + if str(name).lower() == "samesite" + ), + None, + ) + if isinstance(same_site, str) and same_site.capitalize() in { + "Strict", + "Lax", + "None", + }: + item["sameSite"] = same_site.capitalize() + cookies.append(item) + if cookies: + self._context.add_cookies(cookies) + + def _request_login_form( + self, + *, + source_origin: str, + method: str, + url: str, + fields: list[Any], + referer: str, + ) -> Any: + current_method = method + current_url = url + current_fields: list[Any] | None = fields + current_referer = referer + for _ in range(11): + parsed = urlsplit(current_url) + if f"{parsed.scheme}://{parsed.netloc}" != source_origin: + raise RuntimeError("login form bridge rejected a cross-origin redirect") + kwargs: dict[str, Any] = { + "method": current_method, + "url": current_url, + "headers": {"referer": current_referer}, + "allow_redirects": False, + "timeout": 60, + } + if current_fields is not None: + kwargs["params" if current_method == "get" else "data"] = current_fields + response = self._request_bridge_session.request(**kwargs) + self._sync_request_bridge_cookies(str(response.url), response.cookies) + if response.status_code not in {301, 302, 303, 307, 308}: + return response + location = response.headers.get("location") + if not location: + raise RuntimeError("login form bridge redirect has no location") + current_referer = str(response.url) + current_url = urljoin(str(response.url), location) + if response.status_code == 303 or ( + response.status_code in {301, 302} and current_method == "post" + ): + current_method = "get" + current_fields = None + raise RuntimeError("login form bridge exceeded ten redirects") + + def _submit_login_form(self, page: Any, submitter: Any) -> str: + form = submitter.evaluate( + """element => { + const form = element.form; + if (!form) throw new Error('login submitter has no form'); + const fields = Array.from(new FormData(form), ([name, value]) => [ + name, + typeof value === 'string' ? value : value.name, + ]); + if (element.name) fields.push([element.name, element.value || '']); + return {action: form.action, method: form.method || 'get', fields}; + }""" + ) + action = str(form.get("action") or "") + method = str(form.get("method") or "get").lower() + fields = form.get("fields") + source = urlsplit(page.url) + source_origin = f"{source.scheme}://{source.netloc}" + target = urlsplit(action) + if ( + method not in {"get", "post"} + or not isinstance(fields, list) + or target.scheme != source.scheme + or target.netloc != source.netloc + ): + raise RuntimeError("login form bridge rejected an invalid submission") + response = self._request_login_form( + source_origin=source_origin, + method=method, + url=action, + fields=fields, + referer=page.url, + ) + final = urlsplit(str(response.url)) + if ( + not 200 <= response.status_code < 300 + or final.scheme != source.scheme + or final.netloc != source.netloc + ): + raise RuntimeError( + f"login form bridge returned invalid status or origin: {response.status_code}" + ) + page.goto(str(response.url), wait_until="domcontentloaded", timeout=90_000) + return str(response.url) + + def _wait_for_login_ready( + self, + page: Any, + selector: str, + *, + exact_text: str | None = None, + timeout_ms: int = 60_000, + ) -> None: + """Wait for a login marker without Kitesurf's broken Locator visibility path.""" + + if self._engine != "kitesurf": + if exact_text is not None: + page.get_by_role( + "button", name=exact_text, exact=True + ).wait_for(timeout=timeout_ms) + return + locator = page.locator(selector) + if "," in selector: + locator = locator.first + locator.wait_for(timeout=timeout_ms) + return + deadline = time.monotonic() + timeout_ms / 1_000 + while True: + ready = page.evaluate( + """({selector, exactText}) => Array.from( + document.querySelectorAll(selector) + ).some(element => { + if (exactText !== null && element.textContent.trim() !== exactText) { + return false; + } + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.visibility !== 'hidden' && style.display !== 'none' + && rect.width > 0 && rect.height > 0; + })""", + {"selector": selector, "exactText": exact_text}, + ) + if ready: + return + if time.monotonic() >= deadline: + raise TimeoutError(f"login readiness marker did not appear: {selector}") + page.wait_for_timeout(100) + + def _get_service(self) -> PlaywrightReplayTool: + return self + + def _submit(self, callback): + return callback() + + @property + def layout_observer(self): + return self._publish_page_layout if self._engine == "moli" else None + + @property + def viewport_observer(self): + if self._engine != "obscura": + return None + return self._observe_page_viewport + + @property + def document_observer(self): + return self._observe_document if self._engine == "kitesurf" else None + + def _observe_document( + self, page: Any, timeout_ms: int, limit: int + ) -> tuple[str, str]: + if timeout_ms < 1: + raise ValueError("document observation timeout must be positive") + if threading.current_thread() is not threading.main_thread() or not hasattr( + signal, "setitimer" + ): + raise RuntimeError( + "Kitesurf document observation requires the POSIX worker main thread" + ) + if signal.getitimer(signal.ITIMER_REAL) != (0.0, 0.0): + raise RuntimeError("Kitesurf document observation found an active process timer") + previous_handler = signal.getsignal(signal.SIGALRM) + + def expire(_signum: int, _frame: Any) -> None: + raise TimeoutError( + f"Kitesurf document observation exceeded {timeout_ms} ms" + ) + + signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, timeout_ms / 1_000) + try: + value = page.evaluate( + """limit => ({ + title: document.title || '', + body: (document.body?.innerText || '').slice(0, limit), + })""", + limit, + ) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if not isinstance(value, dict): + raise RuntimeError("Kitesurf document observation returned an invalid payload") + return str(value.get("title") or ""), str(value.get("body") or "") + + def _observe_page_viewport(self, page: Any) -> dict[str, int | float]: + """Read renderer state through CDP instead of echoing Playwright config.""" + + sessions = getattr(self, "_viewport_sessions", None) + if sessions is None: + sessions = self._viewport_sessions = {} + session = sessions.get(page) + if session is None: + session = self._context.new_cdp_session(page) + sessions[page] = session + metrics = session.send("Page.getLayoutMetrics") + viewport = metrics.get("cssVisualViewport") or metrics.get("visualViewport") + if not isinstance(viewport, dict): + raise RuntimeError("Obscura returned no observed visual viewport") + runtime = session.send( + "Runtime.evaluate", + {"expression": "devicePixelRatio", "returnByValue": True}, + ) + scale = (runtime.get("result") or {}).get("value") + width = viewport.get("clientWidth") + height = viewport.get("clientHeight") + if not all(isinstance(value, (int, float)) for value in (width, height, scale)): + raise RuntimeError("Obscura returned incomplete viewport metrics") + return { + "width": width, + "height": height, + "device_scale_factor": scale, + } + + @property + def failure_capture_available(self) -> bool: + """Whether another renderer command can still complete reliably.""" + + return not self._layout_connection_poisoned + + def _publish_page_layout(self, page: Any, timeout_ms: int = 5_000) -> None: + """Publish Moli's current layout before Playwright consumes geometry. + + Moli keeps ordinary geometry reads side effect free. Its explicit CDP + extension publishes the same layout tree without paint, rasterization, + or screenshot encoding. + """ + + if getattr(self, "_engine", "chromium") != "moli": + return + if timeout_ms < 1: + raise TimeoutError("Moli layout publication budget exhausted") + sessions = getattr(self, "_layout_sessions", None) + if sessions is None: + sessions = self._layout_sessions = {} + session = sessions.get(page) + if session is None: + session = self._context.new_cdp_session(page) + sessions[page] = session + + async def publish() -> dict[str, Any]: + return await asyncio.wait_for( + session._impl_obj.send("Page.getLayoutMetrics", {"publishLayout": True}), + timeout=timeout_ms / 1_000, + ) + + try: + result = session._sync(publish()) + except TimeoutError as exc: + sessions.pop(page, None) + # The renderer still owns the timed-out CDP command. Sending a + # screenshot or graceful close behind it can block forever, so let + # the catalog runner terminate the isolated browser process. + self._layout_connection_poisoned = True + raise TimeoutError("Moli layout publication timed out") from exc + if not isinstance(result, dict) or not isinstance(result.get("layoutViewport"), dict): + raise RuntimeError("Moli layout publication returned no layout metrics") + + def _snapshot(self, selector: str, timeout: int) -> str: + deadline = time.monotonic() + timeout / 1_000 + + def remaining(maximum: int | None = None) -> int: + value = int((deadline - time.monotonic()) * 1_000) + if value < 1: + raise TimeoutError("snapshot action budget exhausted") + return value if maximum is None else min(value, maximum) + + def observe(maximum: int | None = None) -> str: + observation_deadline = deadline + if maximum is not None: + observation_deadline = min(observation_deadline, time.monotonic() + maximum / 1_000) + while True: + try: + self._page.wait_for_load_state( + "domcontentloaded", + timeout=min(remaining(maximum), SNAPSHOT_NAVIGATION_SETTLE_TIMEOUT_MS), + ) + except Exception: + pass + self._publish_page_layout(self._page, remaining()) + locator = self._page.locator(selector).first + wait_ms = int((observation_deadline - time.monotonic()) * 1_000) + if wait_ms < 1: + raise TimeoutError("snapshot selector did not become visible") + try: + locator.wait_for(state="visible", timeout=min(wait_ms, 1_000)) + except Exception as exc: + if _navigation_interrupted_snapshot(exc): + raise + if ( + "timeout" not in str(exc).lower() + or time.monotonic() >= observation_deadline + ): + raise + continue + return locator.aria_snapshot(timeout=remaining(maximum)) + + try: + return observe() + except Exception as exc: + if not _navigation_interrupted_snapshot(exc): + raise + self._page.wait_for_load_state( + "domcontentloaded", + timeout=remaining(SNAPSHOT_NAVIGATION_SETTLE_TIMEOUT_MS), + ) + self._page.wait_for_timeout(min(remaining(), SNAPSHOT_NAVIGATION_GRACE_MS)) + return observe() + + def execute(self, arguments: dict[str, Any]) -> ReplayResult: + action = arguments.get("action") + if action == "snapshot": + try: + selector = str(arguments.get("selector") or "body") + timeout = int(arguments.get("timeout_ms", 30_000)) + if timeout < 1: + raise ValueError("snapshot timeout_ms must be positive") + value = self._snapshot(selector, timeout) + if not isinstance(value, str) or not value.strip(): + raise RuntimeError("snapshot returned an empty observation") + return ReplayResult("success", value[: int(arguments.get("max_chars", 40_000))]) + except Exception as exc: + return ReplayResult("error", f"Snapshot failed: {exc}") + records: list[dict[str, Any]] = [] + try: + execute_steps( + self._page, + [arguments], + trace=records, + layout_observer=self.layout_observer, + document_observer=self.document_observer, + ) + except Exception as exc: + return ReplayResult("error", f"{type(exc).__name__}: {exc}") + value = records[0]["output"] + if not isinstance(value, str): + value = json.dumps(value, ensure_ascii=False) + return ReplayResult("success", value) + + def close(self) -> None: + bridge_close_errors: list[BaseException] = [] + if getattr(self, "_request_bridge_session", None) is not None: + try: + unroute_all = getattr(self._context, "unroute_all", None) + if unroute_all is not None: + unroute_all(behavior="wait") + except BaseException as exc: + bridge_close_errors.append(exc) + try: + self._request_bridge_session.close() + except BaseException as exc: + bridge_close_errors.append(exc) + try: + if not getattr(self, "_layout_connection_poisoned", False): + try: + if self._context is not None and self._engine != "kitesurf": + self._context.close() + finally: + if self._browser is not None: + self._browser.close() + except BaseException as exc: + for bridge_error in bridge_close_errors: + exc.add_note(f"request bridge close also failed: {bridge_error}") + raise + finally: + self._playwright.stop() + if bridge_close_errors: + raise bridge_close_errors[0] diff --git a/harness/trajectory_replay/public_webarena/executor.py b/harness/trajectory_replay/public_webarena/executor.py index 6a0f7c7ee..c6dcec76b 100644 --- a/harness/trajectory_replay/public_webarena/executor.py +++ b/harness/trajectory_replay/public_webarena/executor.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Iterable from contextlib import nullcontext from typing import Any +from urllib.parse import urlsplit from .suite import validate_selector from .targets import locator_for @@ -34,12 +35,34 @@ }); }""" +NAVIGATION_NETWORK_IDLE_TIMEOUT_MS = 1_500 +NAVIGATION_RENDER_GRACE_MS = 300 +TEXT_OBSERVATION_MAX_CHARS = 40_000 +# BrowserTool returns complete JSON evaluate results. Keep that retrieval +# contract while placing a finite ceiling on one captured observation. +STRUCTURED_OBSERVATION_MAX_BYTES = 4 * 1024 * 1024 + + +def _same_document_url(left: str, right: str) -> bool: + first = urlsplit(left) + second = urlsplit(right) + return ( + first.scheme, + first.netloc, + first.path or "/", + first.query, + ) == ( + second.scheme, + second.netloc, + second.path or "/", + second.query, + ) + def _completed_action_navigation_timeout(exc: Exception) -> bool: message = str(exc).lower() return ( - "click action done" in message - and "waiting for scheduled navigations to finish" in message + "click action done" in message and "waiting for scheduled navigations to finish" in message ) @@ -47,6 +70,41 @@ def _body_text(page: Any, timeout: int, limit: int) -> str: return page.locator("body").first.inner_text(timeout=timeout)[:limit] +def _evaluate_script(page: Any, script: str, script_mode: Any) -> Any: + if script_mode is None: + return page.evaluate(script) + if script_mode == "function_body": + return page.evaluate(f"(async () => {{\n{script}\n}})()") + raise ValueError("evaluate script_mode must be function_body when provided") + + +def _bounded_evaluate_observation(value: Any, requested_max_chars: Any = None) -> str: + """Serialize evaluate observations without producing partial JSON.""" + + structured = not isinstance(value, str) + if structured: + observation = json.dumps(value, ensure_ascii=False, allow_nan=False) + else: + observation = value + try: + json.loads(observation) + except (TypeError, ValueError): + pass + else: + structured = True + limit = TEXT_OBSERVATION_MAX_CHARS if requested_max_chars is None else int(requested_max_chars) + if limit < 1: + raise ValueError("observation max_chars must be positive") + if structured: + if requested_max_chars is not None and len(observation) > limit: + raise ValueError(f"structured observation exceeds the {limit}-character evidence limit") + byte_length = len(observation.encode("utf-8")) + if byte_length > STRUCTURED_OBSERVATION_MAX_BYTES: + raise ValueError("structured observation exceeds the 4194304-byte evidence limit") + return observation + return observation[:limit] + + def _actionability_probe(page: Any, selector: str) -> dict[str, Any]: """Capture post-failure DOM/actionability evidence without affecting scoring.""" return page.evaluate( @@ -88,6 +146,65 @@ def _page_for_tab(page: Any, tab_index: int, timeout: int) -> Any: return context.pages[tab_index] +def _prepare_geometry_action( + page: Any, + call: dict[str, Any], + action: str, + publish_layout: Callable[[], None], + remaining: Callable[[], int], + enabled: bool, +) -> None: + """Wait for one actionable layout, then let Playwright dispatch exactly once.""" + + if not enabled: + return + require_enabled = action not in {"hover"} + while True: + publish_layout() + probe_call = {**call, "timeout_ms": remaining(), "target_timeout_ms": remaining()} + try: + locator = locator_for(page, probe_call, wait_for_target=False) + except ValueError as exc: + if "matched 0" not in str(exc): + raise + else: + if action == "press": + budget = remaining() + call["timeout_ms"] = budget + call["target_timeout_ms"] = min( + int(call.get("target_timeout_ms", budget)), budget + ) + return + ready = locator.is_visible() and (not require_enabled or locator.is_enabled()) + if ready and action == "fill": + ready = locator.is_editable() + if ready: + if action in {"fill", "select"}: + budget = remaining() + call["timeout_ms"] = budget + call["target_timeout_ms"] = min( + int(call.get("target_timeout_ms", budget)), budget + ) + return + try: + trial_timeout = min(500, remaining()) + if action == "hover": + locator.hover(timeout=trial_timeout, trial=True) + else: + locator.click(timeout=trial_timeout, trial=True) + except Exception as exc: + if "timeout" not in str(exc).lower(): + raise + else: + budget = remaining() + call["timeout_ms"] = budget + call["target_timeout_ms"] = min( + int(call.get("target_timeout_ms", budget)), budget + ) + return + page.wait_for_timeout(min(100, remaining())) + + def execute_steps( page: Any, tool_calls: Iterable[dict[str, Any]], @@ -95,22 +212,24 @@ def execute_steps( trace: list[dict[str, Any]] | None = None, outputs: dict[str, str] | None = None, page_observer: Callable[[Any], None] | None = None, + layout_observer: Callable[[Any, int], None] | None = None, + navigation_response_observer: Callable[[Any], None] | None = None, + document_observer: Callable[[Any, int, int], tuple[str, str]] | None = None, + trace_observer: Callable[[list[dict[str, Any]]], None] | None = None, task_timeout_ms: int | None = None, + default_action_timeout_ms: int = 30_000, ) -> dict[str, str]: """Execute declared calls; implicit readiness waits are internal and not calls.""" if task_timeout_ms is not None and task_timeout_ms < 1: raise ValueError("task_timeout_ms must be positive when provided") + if default_action_timeout_ms < 1: + raise ValueError("default_action_timeout_ms must be positive") captured = outputs if outputs is not None else {} records = trace if trace is not None else [] - deadline = ( - time.monotonic() + task_timeout_ms / 1000 - if task_timeout_ms is not None - else None - ) + deadline = time.monotonic() + task_timeout_ms / 1000 if task_timeout_ms is not None else None for index, call in enumerate(tool_calls, start=1): action = call["action"] - default_timeout = 30_000 - declared_timeout = int(call.get("timeout_ms", default_timeout)) + declared_timeout = int(call.get("timeout_ms", default_action_timeout_ms)) if deadline is None: timeout = declared_timeout else: @@ -121,6 +240,21 @@ def execute_steps( ) timeout = min(declared_timeout, remaining_ms) started = time.perf_counter() + action_deadline = started + timeout / 1_000 + + def action_remaining( + deadline: float = action_deadline, + action_name: str = action, + ) -> int: + remaining = int((deadline - time.perf_counter()) * 1_000) + if remaining < 1: + raise TimeoutError(f"{action_name} action budget exhausted") + return remaining + + def operation_timeout(original_timeout: int = timeout) -> int: + return action_remaining() if layout_observer is not None else original_timeout + + locator_call = {**call, "timeout_ms": timeout} record = { "index": index, "action": action, @@ -131,40 +265,125 @@ def execute_steps( "output": "", } records.append(record) + if trace_observer is not None: + trace_observer(records) try: validate_selector(call) - tab_index = int(call.get("tab_index", 0)) - action_page = _page_for_tab(page, tab_index, timeout) + if action == "navigate" and call.get("new_page"): + context = getattr(page, "context", None) + if context is None or not callable(getattr(context, "new_page", None)): + raise RuntimeError("navigate new_page requires a browser context") + action_page = context.new_page() + else: + tab_index = int(call.get("tab_index", 0)) + action_page = _page_for_tab( + page, + tab_index, + action_remaining() if layout_observer is not None else timeout, + ) if page_observer is not None: page_observer(action_page) + + def publish_layout(current_page: Any = action_page) -> None: + if layout_observer is not None: + layout_observer(current_page, action_remaining()) + + hidden_selector = call.get("wait_for_selector_hidden") + if hidden_selector is not None: + if not isinstance(hidden_selector, str) or not hidden_selector: + raise ValueError("wait_for_selector_hidden must be a non-empty selector") + if layout_observer is None: + action_page.locator(hidden_selector).wait_for(state="hidden", timeout=timeout) + else: + while True: + publish_layout() + try: + action_page.locator(hidden_selector).wait_for( + state="hidden", timeout=min(action_remaining(), 500) + ) + break + except Exception as exc: + if "timeout" not in str(exc).lower(): + raise + action_remaining() observation = "" if action == "navigate": + prior_url = str(getattr(action_page, "url", "")) response = action_page.goto( call["url"], wait_until=call.get("wait_until", "domcontentloaded"), - timeout=timeout, + timeout=action_remaining(), ) + if ( + response is None + and prior_url + and _same_document_url(prior_url, str(call["url"])) + ): + response = action_page.reload( + wait_until=call.get("wait_until", "domcontentloaded"), + timeout=action_remaining(), + ) + if navigation_response_observer is not None: + navigation_response_observer(response) + # BrowserTool returns after DOMContentLoaded, then gives ordinary + # background requests a short chance to settle. Network idle is + # deliberately best effort: analytics or streaming requests must + # not turn a completed navigation into a failed replay action. + try: + action_page.wait_for_load_state( + "networkidle", + timeout=min(action_remaining(), NAVIGATION_NETWORK_IDLE_TIMEOUT_MS), + ) + except Exception: + pass + try: + action_page.wait_for_timeout( + min(action_remaining(), NAVIGATION_RENDER_GRACE_MS) + ) + except TimeoutError: + pass + publish_layout() limit = int(call.get("max_chars", 40_000)) - title_fn = getattr(action_page, "title", None) - title = title_fn() if callable(title_fn) else "" + if document_observer is None: + title = str( + action_page.locator("html").first.evaluate( + "element => element.ownerDocument.title", + None, + timeout=action_remaining(), + ) + or "" + ) + body = _body_text(action_page, action_remaining(), limit) + else: + title, body = document_observer( + action_page, action_remaining(), limit + ) status = getattr(response, "status", "") if response is not None else "" - body = _body_text(action_page, timeout, limit) observation = ( f"Navigated to: {getattr(action_page, 'url', call['url'])}\n" f"Title: {title}\nStatus: {status}\n\n--- Page Snapshot ---\n{body}" ) elif action == "click": + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) navigation = ( action_page.expect_navigation( - wait_until=call.get("wait_until", "domcontentloaded"), timeout=timeout + wait_until=call.get("wait_until", "domcontentloaded"), + timeout=operation_timeout(), ) if call.get("wait_for_navigation") else nullcontext() ) with navigation: try: - locator_for(action_page, call).click( - timeout=timeout, + locator_for(action_page, locator_call).click( + timeout=operation_timeout(), no_wait_after=bool(call.get("no_wait_after", False)), ) except Exception as exc: @@ -179,20 +398,47 @@ def execute_steps( } observation = "Clicked successfully." elif action == "double_click": - locator_for(action_page, call).dblclick( - timeout=timeout, + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).dblclick( + timeout=operation_timeout(), no_wait_after=bool(call.get("no_wait_after", False)), ) observation = "Double-clicked successfully." elif action == "fill": if (call.get("source") or {}).get("unresolved_external_input"): raise ValueError("trajectory contains unresolved external input") - locator_for(action_page, call).fill(str(call.get("text", "")), timeout=timeout) + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).fill( + str(call.get("text", "")), timeout=operation_timeout() + ) observation = "Filled text successfully." elif action == "select": + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) navigation = ( action_page.expect_navigation( - wait_until=call.get("wait_until", "domcontentloaded"), timeout=timeout + wait_until=call.get("wait_until", "domcontentloaded"), + timeout=operation_timeout(), ) if call.get("wait_for_navigation") else nullcontext() @@ -204,23 +450,47 @@ def execute_steps( if isinstance(selected, list) else str(selected) ) - locator_for(action_page, call).select_option( + locator_for(action_page, locator_call).select_option( selected, - timeout=timeout, + timeout=operation_timeout(), no_wait_after=bool(call.get("no_wait_after", False)), ) observation = f"Selected option {call.get('value', '')}." elif action == "check": - locator_for(action_page, call).check(timeout=timeout) + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).check(timeout=operation_timeout()) observation = "Checked successfully." elif action == "uncheck": - locator_for(action_page, call).uncheck(timeout=timeout) + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).uncheck(timeout=operation_timeout()) observation = "Unchecked successfully." elif action == "hover": - locator_for(action_page, call).hover(timeout=timeout) + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).hover(timeout=operation_timeout()) observation = "Hovered successfully." elif action == "get_text": - locator = locator_for(action_page, call) + locator = locator_for(action_page, locator_call) expected_text = call.get("wait_for_text") if isinstance(expected_text, str): wait_timeout = min(timeout, int(call.get("wait_timeout_ms", 5_000))) @@ -242,14 +512,16 @@ def execute_steps( observation = observation.strip() observation = observation[: int(call.get("max_chars", 40_000))] elif action == "get_all_text": - values = locator_for(action_page, call, all_matches=True).all_text_contents() + values = locator_for( + action_page, locator_call, all_matches=True + ).all_text_contents() if call.get("strip"): values = [value.strip() for value in values] max_items = int(call.get("max_items", 1_000)) max_chars = int(call.get("max_chars", 40_000)) observation = json.dumps(values[:max_items], ensure_ascii=False)[:max_chars] elif action == "get_table_column": - locator = locator_for(action_page, call) + locator = locator_for(action_page, locator_call) options = { "header": str(call["header"]), "maxItems": int(call.get("max_items", 1_000)), @@ -262,10 +534,15 @@ def execute_steps( ) action_deadline = time.monotonic() + wait_timeout / 1000 while True: + publish_layout() + evaluate_timeout = min(timeout, wait_timeout) + if layout_observer is not None: + loop_remaining = max(1, int((action_deadline - time.monotonic()) * 1_000)) + evaluate_timeout = min(action_remaining(), loop_remaining) values = locator.evaluate( TABLE_COLUMN_EXPRESSION, options, - timeout=min(timeout, wait_timeout), + timeout=evaluate_timeout, ) if wait_pattern is None or any( re.search(str(wait_pattern), str(value), flags=re.IGNORECASE) @@ -278,16 +555,16 @@ def execute_steps( f"{wait_pattern!r}" ) time.sleep(0.05) - observation = json.dumps(values, ensure_ascii=False)[: int( - call.get("max_chars", 40_000) - )] + observation = json.dumps(values, ensure_ascii=False)[ + : int(call.get("max_chars", 40_000)) + ] elif action == "get_attribute": - value = locator_for(action_page, call).get_attribute( + value = locator_for(action_page, locator_call).get_attribute( str(call["name"]), timeout=timeout ) observation = "" if value is None else str(value) elif action == "get_all_attribute": - values = locator_for(action_page, call, all_matches=True).evaluate_all( + values = locator_for(action_page, locator_call, all_matches=True).evaluate_all( "(elements, name) => elements.map(element => element.getAttribute(name))", str(call["name"]), ) @@ -295,7 +572,7 @@ def execute_steps( max_chars = int(call.get("max_chars", 40_000)) observation = json.dumps(values[:max_items], ensure_ascii=False)[:max_chars] elif action == "get_property": - value = locator_for(action_page, call).evaluate( + value = locator_for(action_page, locator_call).evaluate( "(element, name) => element[name]", str(call["name"]), timeout=timeout ) if isinstance(value, bool): @@ -315,17 +592,80 @@ def execute_steps( action_deadline = time.monotonic() + wait_timeout / 1000 observation = "" while time.monotonic() < action_deadline: - observation = locator.inner_text(timeout=min(1_000, wait_timeout)) + publish_layout() + inner_timeout = min(1_000, wait_timeout) + if layout_observer is not None: + loop_remaining = max( + 1, int((action_deadline - time.monotonic()) * 1_000) + ) + inner_timeout = min(inner_timeout, action_remaining(), loop_remaining) + observation = locator.inner_text(timeout=inner_timeout) if re.search(wait_pattern, observation, flags=re.IGNORECASE): break time.sleep(0.05) else: - raise TimeoutError( - f"{selector} did not match text regex {wait_pattern!r}" - ) + raise TimeoutError(f"{selector} did not match text regex {wait_pattern!r}") else: - observation = locator.inner_text(timeout=timeout) + publish_layout() + observation = locator.inner_text(timeout=operation_timeout()) observation = observation[: int(call.get("max_chars", 40_000))] + elif action == "evaluate": + script = call.get("script") + if not isinstance(script, str) or not script.strip(): + raise ValueError("evaluate requires a non-empty script") + try: + value = _evaluate_script(action_page, script, call.get("script_mode")) + except Exception as exc: + pattern = call.get("accept_navigation_url_regex") + if not ( + isinstance(pattern, str) + and pattern + and "Execution context was destroyed" in str(exc) + ): + raise + action_page.wait_for_url( + re.compile(pattern), wait_until="domcontentloaded", timeout=timeout + ) + if not re.search(pattern, str(action_page.url)): + raise ValueError("evaluate navigation ended at the wrong URL") from exc + record["accepted_navigation"] = { + "url": str(action_page.url), + "expected_url_regex": pattern, + "error": f"{type(exc).__name__}: {exc}", + } + observation = f"Navigated to: {action_page.url}" + else: + observation = _bounded_evaluate_observation(value, call.get("max_chars")) + elif action == "wait_for_url": + action_page.wait_for_url( + str(call["url"]), + wait_until=call.get("wait_until", "domcontentloaded"), + timeout=timeout, + ) + observation = f"URL ready: {getattr(action_page, 'url', '')}" + elif action == "wait": + wait_selector = call.get("selector") + if wait_selector is not None: + wait_timeout = min(timeout, int(call.get("timeout", timeout))) + wait_state = str(call.get("wait_state", "visible")) + if wait_state not in {"attached", "detached", "visible", "hidden"}: + raise ValueError( + "wait_state must be attached, detached, visible, or hidden" + ) + action_page.locator(str(wait_selector)).first.wait_for( + state=wait_state, timeout=wait_timeout + ) + observation = f"Selector {wait_state}: {wait_selector}" + else: + requested_ms = ( + int(call["timeout"]) + if "timeout" in call + else round(float(call["seconds"]) * 1000) + ) + if requested_ms > timeout: + raise TimeoutError("wait exceeds the remaining action budget") + action_page.wait_for_timeout(requested_ms) + observation = f"Waited {requested_ms} ms." elif action == "back": action_page.go_back(wait_until=call.get("wait_until", "commit"), timeout=timeout) observation = f"Navigated back to: {getattr(action_page, 'url', '')}" @@ -333,6 +673,7 @@ def execute_steps( action_page.go_forward(wait_until=call.get("wait_until", "commit"), timeout=timeout) observation = f"Navigated forward to: {getattr(action_page, 'url', '')}" elif action == "scroll": + publish_layout() direction = str(call.get("direction", "down")) amount = int(call.get("amount", 700)) dx = amount if direction == "right" else -amount if direction == "left" else 0 @@ -342,9 +683,17 @@ def execute_steps( elif action == "press": target = call.get("target") or call.get("selector") if target: - locator_for(action_page, call).press( + _prepare_geometry_action( + action_page, + locator_call, + action, + publish_layout, + action_remaining, + layout_observer is not None, + ) + locator_for(action_page, locator_call).press( str(call["key"]), - timeout=timeout, + timeout=operation_timeout(), no_wait_after=bool(call.get("no_wait_after", False)), ) else: @@ -370,6 +719,8 @@ def execute_steps( raise finally: record["duration_ms"] = round((time.perf_counter() - started) * 1000, 3) + if trace_observer is not None: + trace_observer(records) return captured diff --git a/harness/trajectory_replay/public_webarena/qualification.py b/harness/trajectory_replay/public_webarena/qualification.py index 62bf88297..76cb583a0 100644 --- a/harness/trajectory_replay/public_webarena/qualification.py +++ b/harness/trajectory_replay/public_webarena/qualification.py @@ -441,6 +441,9 @@ def _apply_material_change(task: dict[str, Any], adjudication: dict[str, Any]) - "observed_review_authors_matching", "observed_currency_range", "observed_not_found", + "observed_action_not_allowed", + "observed_action_blocked", + "observed_permission_denied", }: raise ValueError(f"{task['id']}: response builder must use observed outputs") task["ground_truth"]["response_builder"] = deepcopy(response_builder) diff --git a/harness/trajectory_replay/public_webarena/response_builder.py b/harness/trajectory_replay/public_webarena/response_builder.py index 2c93441dd..c9a423a0c 100644 --- a/harness/trajectory_replay/public_webarena/response_builder.py +++ b/harness/trajectory_replay/public_webarena/response_builder.py @@ -250,8 +250,74 @@ def build_agent_response(task: BrowserTask, outputs: dict[str, str]) -> dict[str raise ValueError("observed_not_found requires a non-empty evidence_regex") if re.search(evidence_regex, outputs[output_name], flags=re.IGNORECASE) is None: raise ValueError("observed_not_found evidence regex did not match observed output") + forbidden_regex = config.get("forbidden_regex") + if forbidden_regex is not None: + if not isinstance(forbidden_regex, str) or not forbidden_regex: + raise ValueError("observed_not_found forbidden_regex must be non-empty") + if re.search(forbidden_regex, outputs[output_name], flags=re.IGNORECASE): + raise ValueError("observed_not_found forbidden evidence was present") retrieved_data = None response_status = "NOT_FOUND_ERROR" + elif kind == "observed_action_not_allowed": + output_name = str(config.get("output") or "") + if output_name not in outputs: + raise ValueError(f"response builder output is missing: {output_name}") + evidence = json.loads(outputs[output_name]) + if not isinstance(evidence, dict): + raise ValueError("observed_action_not_allowed output must contain a JSON object") + if evidence.get("scope_complete") is not True: + raise ValueError("observed_action_not_allowed requires complete observed scope") + matches = evidence.get("matches") + if not isinstance(matches, list): + raise ValueError("observed_action_not_allowed requires an observed matches array") + if matches: + raise ValueError("observed_action_not_allowed found an actionable match") + retrieved_data = None + response_status = "ACTION_NOT_ALLOWED_ERROR" + elif kind == "observed_action_blocked": + output_name = str(config.get("output") or "") + if output_name not in outputs: + raise ValueError(f"response builder output is missing: {output_name}") + evidence = json.loads(outputs[output_name]) + if not isinstance(evidence, dict): + raise ValueError("observed_action_blocked output must contain a JSON object") + if evidence.get("scope_complete") is not True: + raise ValueError("observed_action_blocked requires complete observed scope") + if evidence.get("allowed") is not False: + raise ValueError("observed_action_blocked requires an explicit blocked result") + blockers = evidence.get("blockers") + if not isinstance(blockers, list) or not blockers: + raise ValueError("observed_action_blocked requires observed blockers") + retrieved_data = None + response_status = "ACTION_NOT_ALLOWED_ERROR" + elif kind == "observed_permission_denied": + output_name = str(config.get("output") or "") + if output_name not in outputs: + raise ValueError(f"response builder output is missing: {output_name}") + evidence = json.loads(outputs[output_name]) + if not isinstance(evidence, dict): + raise ValueError("observed_permission_denied output must contain a JSON object") + if evidence.get("scope_complete") is not True: + raise ValueError("observed_permission_denied requires complete observed scope") + if evidence.get("allowed") is not False: + raise ValueError("observed_permission_denied requires an explicit denied result") + access_levels = evidence.get("access_levels") + if not isinstance(access_levels, list) or not access_levels: + raise ValueError("observed_permission_denied requires observed access levels") + if any( + level is not None and (type(level) is not int or level < 0) for level in access_levels + ): + raise ValueError( + "observed_permission_denied access levels must be non-negative integers or null" + ) + required_access_level = evidence.get("required_access_level") + if type(required_access_level) is not int or required_access_level < 1: + raise ValueError("observed_permission_denied requires a positive required access level") + concrete_access_levels = [level for level in access_levels if level is not None] + if any(level >= required_access_level for level in concrete_access_levels): + raise ValueError("observed_permission_denied found sufficient access") + retrieved_data = None + response_status = "PERMISSION_DENIED_ERROR" else: raise ValueError(f"unsupported response builder kind: {kind!r}") return { diff --git a/harness/trajectory_replay/public_webarena/suite.py b/harness/trajectory_replay/public_webarena/suite.py index fa67ca9d4..81e74dd32 100644 --- a/harness/trajectory_replay/public_webarena/suite.py +++ b/harness/trajectory_replay/public_webarena/suite.py @@ -37,6 +37,9 @@ "back", "forward", "scroll", + "evaluate", + "wait_for_url", + "wait", } ) @@ -146,6 +149,44 @@ def _validate_step(task_id: str, index: int, step: Any) -> dict[str, Any]: ("http://", "https://", "{fixture_origin}/") ): raise ValueError(f"{task_id}: navigate step {index} requires an HTTP(S) URL") + if "new_page" in step and ( + action != "navigate" or type(step["new_page"]) is not bool + ): + raise ValueError( + f"{task_id}: new_page is only valid as a boolean on navigate steps" + ) + if action == "navigate" and step.get("new_page") and "tab_index" in step: + raise ValueError( + f"{task_id}: navigate step {index} cannot combine new_page with tab_index" + ) + if action == "evaluate" and not str(step.get("script") or "").strip(): + raise ValueError(f"{task_id}: evaluate call {index} requires a non-empty script") + if action == "wait_for_url" and not str(step.get("url") or "").strip(): + raise ValueError(f"{task_id}: wait_for_url call {index} requires a URL pattern") + if action == "wait": + selector = step.get("selector") + milliseconds = step.get("timeout") + seconds = step.get("seconds") + if selector is not None: + if not isinstance(selector, str) or not selector.strip() or seconds is not None: + raise ValueError(f"{task_id}: wait call {index} has an invalid selector wait") + if step.get("wait_state", "visible") not in { + "attached", + "detached", + "visible", + "hidden", + }: + raise ValueError(f"{task_id}: wait call {index} has an invalid wait state") + if milliseconds is not None and ( + type(milliseconds) not in {int, float} or milliseconds <= 0 + ): + raise ValueError(f"{task_id}: wait call {index} requires a positive timeout") + else: + if (milliseconds is None) == (seconds is None): + raise ValueError(f"{task_id}: wait call {index} requires exactly one duration") + duration = milliseconds if milliseconds is not None else seconds + if type(duration) not in {int, float} or duration <= 0: + raise ValueError(f"{task_id}: wait call {index} requires a positive duration") try: validate_selector(step) except ValueError as exc: diff --git a/harness/trajectory_replay/public_webarena/targets.py b/harness/trajectory_replay/public_webarena/targets.py index 6a1591bde..3086a2691 100644 --- a/harness/trajectory_replay/public_webarena/targets.py +++ b/harness/trajectory_replay/public_webarena/targets.py @@ -36,7 +36,13 @@ def validate_target(step: dict[str, Any]) -> None: raise ValueError("role target name must be a string") -def locator_for(page: Any, step: dict[str, Any], *, all_matches: bool = False) -> Any: +def locator_for( + page: Any, + step: dict[str, Any], + *, + all_matches: bool = False, + wait_for_target: bool = True, +) -> Any: """Resolve the common target schema to a Playwright locator. Legacy selectors retain their old ``first`` behavior. Structured targets are @@ -73,7 +79,7 @@ def locator_for(page: Any, step: dict[str, Any], *, all_matches: bool = False) - if bool(step.get("allow_many", False)): return locator.first count = locator.count() - if count == 0: + if count == 0 and wait_for_target: first = locator.first wait_for = getattr(first, "wait_for", None) if callable(wait_for): diff --git a/harness/trajectory_replay/remote_execution.py b/harness/trajectory_replay/remote_execution.py index 10026dea4..5d9252c05 100644 --- a/harness/trajectory_replay/remote_execution.py +++ b/harness/trajectory_replay/remote_execution.py @@ -1,6 +1,7 @@ """Shared browser launch, task execution and evidence verification for remote replay.""" from __future__ import annotations + import argparse import json import subprocess @@ -9,12 +10,22 @@ import urllib.request import uuid from pathlib import Path + +from webarena_verified.types.task import WebArenaSite + from harness.browser_use.har import CdpHarRecorder from harness.browser_use.runner import ( _browser_tool_config, _login_context, _prepare_browser_viewport, ) +from harness.trajectory_replay.catalog_contracts import ( + apply_action_overrides, + catalog_action_overrides, +) +from harness.trajectory_replay.public_webarena.executor import execute_steps +from harness.trajectory_replay.public_webarena.response_builder import build_agent_response +from harness.trajectory_replay.public_webarena.suite import BrowserTask from harness.trajectory_replay.remote_runtime import ( file_sha256, rebind_remote_calls, @@ -31,14 +42,161 @@ _click_was_dispatched_before_navigation_timeout, _execute_replay_call, _json_sha256, + _replay_step_status, ) +NETWORK_SETTLE_MS = 2_000 + + +def sanitize_remote_text( + message: str, endpoint: str, headers: dict[str, str] | None +) -> str: + """Remove hosted-browser account and credentials before evidence is persisted.""" + + sanitized = message.replace(endpoint, "") + sanitized = sanitized.replace(endpoint.split("?", 1)[0], "") + for value in (headers or {}).values(): + sanitized = sanitized.replace(value, "") + if " " in value: + sanitized = sanitized.replace( + value.split(" ", 1)[1], "" + ) + return sanitized + + +def _sanitize_remote_payload(value: object, endpoint: str, headers: dict[str, str]) -> object: + if isinstance(value, str): + return sanitize_remote_text(value, endpoint, headers) + if isinstance(value, list): + return [_sanitize_remote_payload(item, endpoint, headers) for item in value] + if isinstance(value, dict): + return { + key: _sanitize_remote_payload(item, endpoint, headers) + for key, item in value.items() + } + return value + + +def finalize_timed_out_task( + task_dir: Path, + row: dict, + *, + duration_ms: int, + task_timeout_ms: int, + runtime_metadata: dict[str, str] | None = None, + remote_endpoint: str | None = None, + remote_headers: dict[str, str] | None = None, +) -> dict: + """Commit a browser runtime failure after an external process watchdog fires.""" + + partial_path = task_dir / ".trace.partial.json" + trace_path = task_dir / "trace.json" + records_path = partial_path if partial_path.is_file() else trace_path + records = ( + json.loads(records_path.read_text(encoding="utf-8")) + if records_path.is_file() + else [] + ) + if records and records[-1].get("status") == "running": + active = records[-1] + active["status"] = "fail" + active["duration_ms"] = duration_ms + active["error"] = ( + f"ExternalTaskTimeout: browser stopped responding during action " + f"{active.get('index')} ({active.get('action')}) and exceeded the " + f"{task_timeout_ms} ms task budget" + ) + else: + records.append( + { + "index": None, + "action": "task_timeout", + "arguments": {}, + "status": "fail", + "duration_ms": duration_ms, + "output": "", + "error": ( + "ExternalTaskTimeout: browser stopped responding before the active " + f"action boundary was persisted and exceeded the {task_timeout_ms} ms " + "task budget" + ), + } + ) + if remote_endpoint is not None and remote_headers is not None: + records = _sanitize_remote_payload( # type: ignore[assignment] + records, remote_endpoint, remote_headers + ) + message = records[-1]["error"] + hashes = { + "trace": _write(trace_path, records), + "readiness": _write(task_dir / "readiness.json", []), + "outputs": _write(task_dir / "outputs.json", {}), + } + partial_path.unlink(missing_ok=True) + result = { + "task_id": row["task_id"], + "status": "fail", + "score": None, + "action_count": len(row["actions"]), + "executed_actions_sha256": _json_sha256(row["actions"]), + "omitted_diagnostic_observer_sha256": row.get( + "omitted_diagnostic_observer_sha256" + ), + "completed_actions": sum( + record.get("status") in {"pass", "success"} for record in records + ), + "duration_ms": duration_ms, + "error": message, + "failure_stage": "browser_tool_execution", + "runtime_poisoned": True, + **( + {"runtime_metadata": runtime_metadata} + if runtime_metadata is not None + else {} + ), + "evidence_sha256": hashes, + } + result_path = task_dir / "result.json" + temporary = task_dir / ".result.json.tmp" + temporary.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + temporary.chmod(0o600) + temporary.replace(result_path) + return result + + +def _effective_retrieval_calls(task_id: int, row: dict, calls: list[dict]) -> list[dict]: + """Apply the catalog's frozen amendments without changing contract source calls.""" + + provenance = row.get("action_overrides") + catalog = { + "action_overrides_sha256": ( + provenance.get("source_sha256") if isinstance(provenance, dict) else None + ) + } + return apply_action_overrides( + calls, + catalog_action_overrides(catalog, row), + task_id=task_id, + ) + + +def _finalize_network_recorder( + service, recorder, har_path: Path, *, settle_ms: int = NETWORK_SETTLE_MS +) -> None: + """Let dispatched background requests finish before freezing their statuses.""" + + service._submit(lambda: service._page.wait_for_timeout(settle_ms)) + service._submit(lambda: recorder.stop(service._context, har_path)) + def _write(path: Path, value: object) -> str: path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") path.chmod(0o600) return file_sha256(path) + def _verify_result_evidence(output: Path, rows: list[dict]) -> None: for row in rows: directory = output / f"task-{row['task_id']}" @@ -54,6 +212,7 @@ def _verify_result_evidence(output: Path, rows: list[dict]) -> None: "trace": "trace.json", "readiness": "readiness.json", "site_reset": "site-reset.json", + "outputs": "outputs.json", }.get(name) if filename is None: raise ValueError(f"task {row['task_id']} has unknown evidence kind {name}") @@ -61,10 +220,11 @@ def _verify_result_evidence(output: Path, rows: list[dict]) -> None: if path.is_symlink() or not path.is_file() or file_sha256(path) != expected: raise ValueError(f"task {row['task_id']} evidence hash changed: {filename}") + def _wait_cdp(endpoint: str, process: subprocess.Popen) -> None: for _ in range(300): if process.poll() is not None: - raise RuntimeError(f"Moli exited before CDP was ready: {process.returncode}") + raise RuntimeError(f"browser exited before CDP was ready: {process.returncode}") try: with urllib.request.urlopen(endpoint + "/json/version", timeout=1) as response: if response.status == 200: @@ -73,9 +233,10 @@ def _wait_cdp(endpoint: str, process: subprocess.Popen) -> None: time.sleep(0.1) raise TimeoutError("browser CDP did not become ready") + def _browser_command(args: argparse.Namespace, port: int, profile: str) -> list[str]: if args.engine == "chromium": - return [ + command = [ str(args.binary), "--headless=new", "--no-first-run", @@ -88,18 +249,48 @@ def _browser_command(args: argparse.Namespace, port: int, profile: str) -> list[ "--remote-debugging-address=127.0.0.1", "about:blank", ] - return [ - str(args.binary), - "serve", - "--host", - "127.0.0.1", - "--port", - str(port), - "--layout", - "--font", - "--profile-dir", - profile, - ] + host_resolver_rules = getattr(args, "host_resolver_rules", None) + if host_resolver_rules: + if not isinstance(host_resolver_rules, str) or "\n" in host_resolver_rules: + raise ValueError("Chromium host resolver rules must be one line") + command.insert(-1, f"--host-resolver-rules={host_resolver_rules}") + return command + if args.engine == "moli": + return [ + str(args.binary), + "serve", + "--host", + "127.0.0.1", + "--port", + str(port), + "--layout", + "--font", + "--profile-dir", + profile, + ] + if args.engine == "lightpanda": + return [ + str(args.binary), + "serve", + "--host", + "127.0.0.1", + "--port", + str(port), + ] + if args.engine == "obscura": + return [ + str(args.binary), + "serve", + "--host", + "127.0.0.1", + "--port", + str(port), + "--storage-dir", + profile, + "--allow-private-network", + ] + raise ValueError(f"unknown local replay engine: {args.engine}") + def _task_type(task: object) -> str: expected = [ @@ -111,9 +302,44 @@ def _task_type(task: object) -> str: raise ValueError(f"task {task.task_id}: current-run response type unavailable") return expected[0] + def _read_retrieval_contract( task_id: int, row: dict, sites: dict, candidate_path: Path | None = None ) -> dict: + embedded_contract = row.get("retrieval_contract") + embedded_calls = row.get("source_actions") + if embedded_contract is not None or embedded_calls is not None: + if not isinstance(embedded_contract, dict) or not isinstance(embedded_calls, list): + raise ValueError(f"task {task_id}: embedded retrieval input is incomplete") + candidate = { + "task_id": task_id, + "tool_calls": embedded_calls, + "retrieval_contract": embedded_contract, + } + if embedded_contract.get("task_id") != task_id: + raise ValueError(f"task {task_id}: retrieval contract identity mismatch") + source_sites = { + WebArenaSite(site): url for site, url in (row.get("source_site_urls") or {}).items() + } + rebound = rebind_remote_calls( + _effective_retrieval_calls(task_id, row, embedded_calls), + sites, + { + "onestopmarket.example.com", + "gitlab.example.com", + "openstreetmap.example.com", + "wikipedia.example.com", + "ec2-3-133-227-75.us-east-2.compute.amazonaws.com", + "ec2-3-135-39-80.us-east-2.compute.amazonaws.com", + "host.docker.internal", + "127.0.0.1", + "192.168.5.2", + }, + source_sites, + ) + if _json_sha256(rebound) != row["bound_calls_sha256"]: + raise ValueError(f"task {task_id}: retrieval actions differ from frozen bound calls") + return candidate source_revision = row.get("source_revision") or {} path = candidate_path if path is None and source_revision: @@ -127,7 +353,7 @@ def _read_retrieval_contract( if candidate["task_id"] != task_id or candidate["retrieval_contract"]["task_id"] != task_id: raise ValueError(f"task {task_id}: retrieval contract identity mismatch") rebound = rebind_remote_calls( - candidate["tool_calls"], + _effective_retrieval_calls(task_id, row, candidate["tool_calls"]), sites, {"127.0.0.1", "192.168.5.2", "host.docker.internal"}, ) @@ -135,6 +361,7 @@ def _read_retrieval_contract( raise ValueError(f"task {task_id}: retrieval actions differ from frozen bound calls") return candidate + def _evaluate_assertions(service: object, contract: object) -> list[dict]: selectors = { selector @@ -158,6 +385,7 @@ def _evaluate_assertions(service: object, contract: object) -> list[dict]: ) ) + def _run_task( *, task_id: int, @@ -170,23 +398,79 @@ def _run_task( sites: dict, catalog: object, candidate_override_path: Path | None = None, + portable_task: BrowserTask | None = None, + portable_action_timeout_ms: int = 30_000, + backend: str = "browsertool", + engine: str = "chromium", + cdp_headers: dict[str, str] | None = None, + expected_runtime_metadata: dict[str, str] | None = None, + request_bridge_origins: set[str] | None = None, + request_bridge_host_overrides: list[dict[str, str]] | None = None, + task_timeout_ms: int = 300_000, ) -> dict: - from agent.tools.browser.browser_tool import BrowserTool # noqa: PLC0415 - + if task_timeout_ms < 1: + raise ValueError("task_timeout_ms must be positive") official = benchmark.get_task(task_id) correction = apply_correction(official, catalog) if task_id in catalog.records else None - tool = BrowserTool(config=_browser_tool_config(endpoint)) - recorder = CdpHarRecorder() + if backend == "browsertool": + if cdp_headers: + raise ValueError("BrowserTool replay does not support remote CDP headers") + from agent.tools.browser.browser_tool import BrowserTool # noqa: PLC0415 + + tool = BrowserTool(config=_browser_tool_config(endpoint)) + elif backend == "playwright": + from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool + + tool = PlaywrightReplayTool( + endpoint, + engine=engine, + headers=cdp_headers, + request_bridge_origins=request_bridge_origins, + request_bridge_host_overrides=request_bridge_host_overrides, + ) + else: + raise ValueError(f"unknown replay backend: {backend}") + if ( + expected_runtime_metadata is not None + and getattr(tool, "runtime_metadata", None) != expected_runtime_metadata + ): + observed = getattr(tool, "runtime_metadata", None) + tool.close() + raise RuntimeError( + "hosted browser version changed before task execution: " + f"expected {expected_runtime_metadata!r}, observed {observed!r}" + ) + recorder = CdpHarRecorder( + capture_transport=getattr(tool, "har_capture_transport", "cdp"), + capture_new_pages=getattr(tool, "_supports_multiple_pages", True), + playwright_request_transform=getattr(tool, "har_request_transform", None), + ) task_dir = output / f"task-{task_id}" task_dir.mkdir(mode=0o700) + partial_trace_path = task_dir / ".trace.partial.json" har_path = task_dir / "network.har" records: list[dict] = [] + portable_outputs: dict[str, str] = {} readiness_records: list[dict] = [] hashes: dict[str, str] = {} started = time.perf_counter() + deadline = started + task_timeout_ms / 1_000 + + def remaining_task_ms() -> int: + remaining_ms = int((deadline - time.perf_counter()) * 1_000) + if remaining_ms < 1: + raise TimeoutError(f"task exceeded its {task_timeout_ms} ms execution budget") + return remaining_ms + + def bounded_call(call: dict) -> dict: + remaining_ms = remaining_task_ms() + declared = int(call.get("timeout_ms", portable_action_timeout_ms)) + return {**call, "timeout_ms": min(declared, remaining_ms)} + status = "fail" score = None error = "" + failure_stage = "browser_session_setup" service = None recorder_started = False try: @@ -198,32 +482,88 @@ def _run_task( benchmark.config.render_url(list(official.start_urls), list(official.sites)) ) _login_context(service, tuple(official.sites), benchmark, start_urls, viewport) - service._submit(lambda: recorder.start(service._context)) + service._submit(lambda: recorder.start(service._context, service._page)) recorder_started = True - for index, action in enumerate(row["actions"], 1): + failure_stage = "browser_tool_execution" + if portable_task is not None: + def persist_partial_trace(current_records: list[dict]) -> None: + temporary = partial_trace_path.with_suffix(".tmp") + temporary.write_text( + json.dumps(current_records, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + temporary.chmod(0o600) + temporary.replace(partial_trace_path) + + service._submit( + lambda: execute_steps( + service._page, + row["actions"], + trace=records, + outputs=portable_outputs, + task_timeout_ms=remaining_task_ms(), + default_action_timeout_ms=portable_action_timeout_ms, + layout_observer=getattr(service, "layout_observer", None), + navigation_response_observer=recorder.record_navigation_response, + document_observer=getattr(service, "document_observer", None), + trace_observer=persist_partial_trace, + ) + ) + for index, action in enumerate(() if portable_task is not None else row["actions"], 1): for readiness in row["readiness"]: if readiness["before_action"] == index: - ready_status, ready_result = _execute_replay_call(tool, readiness["input"]) + readiness_input = bounded_call(readiness["input"]) + ready_status, ready_result = _execute_replay_call(tool, readiness_input) + ready_effective_status, ready_observation_error = _replay_step_status( + readiness["input"], ready_status, ready_result, dispatched=False + ) readiness_records.append( - {"before_action": index, "status": ready_status, "result": ready_result} + { + "before_action": index, + "status": ready_effective_status, + "tool_status": ready_status, + "result": ready_result, + **( + { + "observation_status": "error", + "observation_error": ready_observation_error, + } + if ready_observation_error is not None + else {} + ), + } ) - if ready_status != "success": - raise RuntimeError(f"readiness before step {index} failed: {ready_result}") + if ready_effective_status != "success": + detail = ready_observation_error or ready_result + raise RuntimeError(f"readiness before step {index} failed: {detail}") call_started = time.perf_counter() diagnostics = {} - result_status, result = _execute_replay_call(tool, action, diagnostics=diagnostics) + result_status, result = _execute_replay_call( + tool, bounded_call(action), diagnostics=diagnostics + ) dispatched = _click_was_dispatched_before_navigation_timeout( action, result_status, result ) + effective_status, observation_error = _replay_step_status( + action, result_status, result, dispatched=dispatched + ) records.append( { "index": index, "arguments": action, - "status": "success" if dispatched else result_status, + "status": effective_status, "tool_status": result_status, "duration_ms": round((time.perf_counter() - call_started) * 1000, 3), "result": result, **({"action_diagnostics": diagnostics} if diagnostics else {}), + **( + { + "observation_status": "error", + "observation_error": observation_error, + } + if observation_error is not None + else {} + ), **( {"recovery": "click_dispatched_navigation_wait_timeout"} if dispatched @@ -231,23 +571,33 @@ def _run_task( ), } ) - if result_status != "success" and not dispatched: - try: - records[-1]["failure_ui_state"] = _capture_failure_ui_state(service, action) - except Exception as observation_exc: - records[-1]["failure_ui_state_error"] = ( - f"{type(observation_exc).__name__}: {observation_exc}" - )[:300] - raise RuntimeError(f"step {index} failed: {result}") + if effective_status != "success": + if getattr(service, "failure_capture_available", True): + try: + records[-1]["failure_ui_state"] = _capture_failure_ui_state( + service, action + ) + except Exception as observation_exc: + records[-1]["failure_ui_state_error"] = ( + f"{type(observation_exc).__name__}: {observation_exc}" + )[:300] + detail = observation_error or result + raise RuntimeError(f"step {index} failed: {detail}") if correction is not None and correction.replay_assertions: assertions = _evaluate_assertions(service, correction) else: assertions = [] - service._submit(lambda: service._page.wait_for_timeout(500)) - service._submit(lambda: recorder.stop(service._context, har_path)) + settle_ms = max( + [NETWORK_SETTLE_MS] + + [int(action.get("network_settle_ms", 0)) for action in row["actions"]] + ) + failure_stage = "evidence_finalization" + _finalize_network_recorder(service, recorder, har_path, settle_ms=settle_ms) recorder_started = False hashes["network_har"] = file_sha256(har_path) - if official.is_retrieve_task: + if portable_task is not None: + actual = build_agent_response(portable_task, portable_outputs) + elif official.is_retrieve_task: candidate = _read_retrieval_contract(task_id, row, sites, candidate_override_path) # Contract step indices refer to the source calls; recorded results are # from the equivalent remote-bound calls executed in this run. @@ -276,6 +626,7 @@ def _run_task( if actual is None: status = "unsupported" else: + failure_stage = "official_evaluator_execution" evaluation_result = ( evaluate_corrected_contract( benchmark, correction, agent_response=actual, network_trace=har_path @@ -300,13 +651,16 @@ def _run_task( ) except Exception: error = traceback.format_exc() - if service is not None and recorder_started: + failure_capture_available = bool( + service is not None and getattr(service, "failure_capture_available", True) + ) + if failure_capture_available and recorder_started: try: service._submit(lambda: recorder.stop(service._context, har_path)) hashes["network_har"] = file_sha256(har_path) except Exception: pass - if service is not None: + if failure_capture_available: try: service._submit( lambda: service._page.screenshot(path=str(task_dir / "failure.png")) @@ -318,18 +672,55 @@ def _run_task( try: tool.close() except Exception as exc: - error += f"\nBrowserTool close failed: {exc}" + error += f"\n{backend} close failed: {exc}" + if cdp_headers: + error = sanitize_remote_text(error, endpoint, cdp_headers) + records = _sanitize_remote_payload(records, endpoint, cdp_headers) # type: ignore[assignment] + readiness_records = _sanitize_remote_payload( # type: ignore[assignment] + readiness_records, endpoint, cdp_headers + ) + portable_outputs = _sanitize_remote_payload( # type: ignore[assignment] + portable_outputs, endpoint, cdp_headers + ) hashes["trace"] = _write(task_dir / "trace.json", records) + partial_trace_path.unlink(missing_ok=True) hashes["readiness"] = _write(task_dir / "readiness.json", readiness_records) - return { + if portable_task is not None: + hashes["outputs"] = _write(task_dir / "outputs.json", portable_outputs) + outcome = { "task_id": task_id, "status": status, "score": score, "action_count": len(row["actions"]), "executed_actions_sha256": _json_sha256(row["actions"]), "omitted_diagnostic_observer_sha256": row.get("omitted_diagnostic_observer_sha256"), - "completed_actions": sum(record["status"] == "success" for record in records), + "completed_actions": sum( + record["status"] == ("pass" if portable_task is not None else "success") + for record in records + ), "duration_ms": round((time.perf_counter() - started) * 1000, 3), "error": error, + **({"failure_stage": failure_stage} if error else {}), + "runtime_poisoned": bool( + service is not None and not getattr(service, "failure_capture_available", True) + ), + **( + {"runtime_metadata": tool.runtime_metadata} + if getattr(tool, "runtime_metadata", None) is not None + else {} + ), + **( + {"request_bridge_stats": tool.request_bridge_stats} + if request_bridge_origins + else {} + ), "evidence_sha256": hashes, } + result_path = task_dir / "result.json" + result_tmp = task_dir / ".result.json.tmp" + result_tmp.write_text( + json.dumps(outcome, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + result_tmp.chmod(0o600) + result_tmp.replace(result_path) + return outcome diff --git a/harness/trajectory_replay/remote_runtime.py b/harness/trajectory_replay/remote_runtime.py index 141b935da..d6b722145 100644 --- a/harness/trajectory_replay/remote_runtime.py +++ b/harness/trajectory_replay/remote_runtime.py @@ -19,16 +19,27 @@ from harness.browser_use.runner import SITE_CREDENTIALS, SITE_PORTS +READ_ONLY_PORTS = {8888: WebArenaSite.WIKIPEDIA, 3000: WebArenaSite.MAP} +SUPPORTED_REMOTE_SITES = frozenset(SITE_PORTS) | frozenset(READ_ONLY_PORTS.values()) + def load_remote_sites(path: Path) -> dict[WebArenaSite, str]: value = json.loads(path.read_text(encoding="utf-8")) if value.get("schema") != "lex_webarena_remote_sites/1": raise ValueError("unsupported remote site map") raw = value.get("sites") - if not isinstance(raw, dict) or set(raw) != {site.value for site in SITE_PORTS}: - raise ValueError("remote site map must cover exactly the four mutable sites") + if ( + not isinstance(raw, dict) + or not {site.value for site in SITE_PORTS} <= set(raw) + or not set(raw) <= {site.value for site in SUPPORTED_REMOTE_SITES} + ): + raise ValueError( + "remote site map must cover the four mutable sites and optional read-only sites" + ) sites: dict[WebArenaSite, str] = {} - for site in SITE_PORTS: + for site in sorted(SUPPORTED_REMOTE_SITES, key=lambda item: item.value): + if site.value not in raw: + continue url = raw[site.value] if not isinstance(url, str): raise ValueError(f"invalid remote URL for {site.value}") @@ -56,8 +67,27 @@ def remote_benchmark(sites: dict[WebArenaSite, str]) -> WebArenaVerified: config=WebArenaVerifiedConfig( environments={ site: EnvironmentConfig( - urls=[url], - credentials=SITE_CREDENTIALS[site], + # Reverse proxies can terminate HTTPS while an application + # still emits same-host HTTP requests from its configured + # public base URL. Both transports identify the same bound + # fixture and must derender to the same site token. + urls=list( + dict.fromkeys( + [ + url, + urlunsplit( + ( + "http" if urlsplit(url).scheme == "https" else "https", + urlsplit(url).netloc, + urlsplit(url).path, + urlsplit(url).query, + urlsplit(url).fragment, + ) + ), + ] + ) + ), + credentials=SITE_CREDENTIALS.get(site), use_header_login=site == WebArenaSite.SHOPPING_ADMIN, ) for site, url in sites.items() @@ -70,6 +100,7 @@ def rebind_remote_calls( calls: list[dict[str, Any]], sites: dict[WebArenaSite, str], trusted_source_hosts: set[str], + source_sites: dict[WebArenaSite, str] | None = None, ) -> list[dict[str, Any]]: """Bind frozen WebArena URLs while tolerating HTTP-to-HTTPS proxy upgrades. @@ -78,13 +109,17 @@ def rebind_remote_calls( semantics; only the transport scheme becomes an explicit alternative. """ by_port = {port: site for site, ports in SITE_PORTS.items() for port in ports} + by_port.update(READ_ONLY_PORTS) + by_source_origin = {urlsplit(url).netloc: site for site, url in (source_sites or {}).items()} def rebind_url(value: str) -> str: parsed = urlsplit(value) - if parsed.scheme not in {"http", "https"} or parsed.hostname not in trusted_source_hosts: + if parsed.scheme not in {"http", "https"}: return value - site = by_port.get(parsed.port) - if site is None: + site = by_source_origin.get(parsed.netloc) + if site is None and parsed.hostname in trusted_source_hosts: + site = by_port.get(parsed.port) + if site is None or site not in sites: return value target = urlsplit(sites[site]) return urlunsplit( @@ -95,6 +130,10 @@ def rebind_url(value: str) -> str: r"(?P\[href\s*=\s*)(?P['\"])(?Phttps?://[^'\"]+)" r"(?P=quote)(?P\])" ) + embedded_url_pattern = re.compile(r"https?://[^\s\"'\])]+") + + def rebind_embedded_urls(value: str) -> str: + return embedded_url_pattern.sub(lambda match: rebind_url(match.group()), value) def rebind_href(match: re.Match[str]) -> str: url = match.group("url") @@ -118,13 +157,35 @@ def rebind_href(match: re.Match[str]) -> str: # URL-valued form inputs refer to the same fixture deployment as # navigation URLs. Do not rewrite prose or arbitrary external URLs. text = call.get("text") - if call.get("action") == "fill" and isinstance(text, str) and not any( - char.isspace() for char in text + if ( + call.get("action") == "fill" + and isinstance(text, str) + and not any(char.isspace() for char in text) ): call["text"] = rebind_url(text) selector = call.get("selector") if isinstance(selector, str): call["selector"] = href_pattern.sub(rebind_href, selector) + target = call.get("target") + if isinstance(target, dict): + rebound_target = dict(target) + for key in ("value", "name"): + value = rebound_target.get(key) + if isinstance(value, str): + rebound_target[key] = rebind_embedded_urls(value) + call["target"] = rebound_target + script = call.get("script") + if call.get("action") == "evaluate" and isinstance(script, str): + # Some qualified actions submit site URLs inside a JavaScript + # request body. Rewrite only origins explicitly bound to a source + # deployment; leave the rest of the program intact. + for source_site, source_url in (source_sites or {}).items(): + source_origin = urlsplit(source_url) + target_origin = urlsplit(sites[source_site]) + old = f"{source_origin.scheme}://{source_origin.netloc}" + new = f"{target_origin.scheme}://{target_origin.netloc}" + script = script.replace(old, new) + call["script"] = script rebound.append(call) return rebound diff --git a/harness/trajectory_replay/report_compaction.py b/harness/trajectory_replay/report_compaction.py new file mode 100644 index 000000000..eb87728c3 --- /dev/null +++ b/harness/trajectory_replay/report_compaction.py @@ -0,0 +1,213 @@ +"""Compact generated replay detail files into query-addressed static bundles.""" + +# ruff: noqa: E501 + +from __future__ import annotations + +import json +import re +import shutil +from pathlib import Path + +SHARD_COUNT = 16 + + +def _task_id(value: str) -> int: + match = re.search(r"(?:^|-)(\d+)$", value) + if match is None: + raise ValueError(f"compact route has no task id: {value}") + return int(match.group(1)) + + +def _rewrite_links(text: str, *, detail_kind: str | None = None) -> str: + text = re.sub( + r'href="(?:\.\./)?task/(\d+)\.html([#?][^"]*)?"', + lambda match: f'href="task.html?id={match.group(1)}{match.group(2) or ""}"', + text, + ) + text = re.sub( + r'href="(?:\.\./)?result/([A-Za-z0-9_.-]+)\.html([#?][^"]*)?"', + lambda match: f'href="result.html?key={match.group(1)}{match.group(2) or ""}"', + text, + ) + if detail_kind == "result": + text = re.sub( + r'href="([A-Za-z0-9_.-]+-\d+)\.html([#?][^"]*)?"', + lambda match: f'href="result.html?key={match.group(1)}{match.group(2) or ""}"', + text, + ) + text = re.sub( + r'href="(?:\.\./)*data/evidence/([A-Za-z0-9_.-]+)\.json"', + lambda match: f'href="evidence-raw.html?key={match.group(1)}"', + text, + ) + return ( + text.replace('="../', '="') + .replace(">下载净化 JSON<", ">查看净化 JSON<") + .replace(">Download sanitized JSON<", ">Inspect sanitized JSON<") + ) + + +def _javascript_source(source: str) -> str: + doubled = chr(92) * 2 + while doubled in source: + source = source.replace(doubled, chr(92)) + return source + + +def _loader(kind: str) -> str: + parameter = "id" if kind == "task" else "key" + return _javascript_source(f'''轨迹重放详情

    正在读取详情…

    ''') + + +def _raw_evidence_loader() -> str: + return _javascript_source(f'''单结果净化 JSON

    返回结果

    单结果净化 JSON

    正在读取…
    ''') + + +def _data_fetch_adapter() -> str: + return _javascript_source(rf'''(()=>{{ +const originalFetch=window.fetch.bind(window);const compactCache=new Map(); +const compactRoute=raw=>{{ + const url=new URL(raw,location.href);const match=url.pathname.match(/data\/(trajectory|evidence)\/([A-Za-z0-9_.-]+)\.json$/); + if(!match)return null; + const kind=match[1];const key=match[2];const id=(key.match(/(?:^|-)(\\d+)$/)||[])[1]; + if(id===undefined)return null; + return {{kind,key,shard:String(Number(id)%{SHARD_COUNT}).padStart(2,'0')}}; +}}; +window.fetch=async(input,init)=>{{ + const route=compactRoute(typeof input==='string'?input:input.url); + if(!route)return originalFetch(input,init); + const cacheKey=route.kind+'-'+route.shard; + if(!compactCache.has(cacheKey))compactCache.set(cacheKey,originalFetch('data/'+route.kind+'-'+route.shard+'.json').then(response=>{{if(!response.ok)throw Error(`HTTP ${{response.status}}`);return response.json()}})); + const value=(await compactCache.get(cacheKey))[route.key]; + return value===undefined?new Response('',{{status:404}}):new Response(JSON.stringify(value),{{status:200,headers:{{'content-type':'application/json'}}}}); +}}; +document.addEventListener('click',event=>{{ + const anchor=event.target.closest?.('a[href]');if(!anchor)return; + const route=compactRoute(anchor.href);if(route?.kind!=='evidence')return; + event.preventDefault(); + location.href='evidence-raw.html?key='+encodeURIComponent(route.key); +}},true); +}})(); +''') + + +def compact_report(source: Path, destination: Path) -> dict[str, int]: + if destination.exists(): + raise FileExistsError(destination) + destination.mkdir(parents=True) + excluded = {"task", "result"} + for path in source.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(source) + if relative.parts[0] in excluded or relative.parts[:2] in { + ("data", "trajectory"), + ("data", "evidence"), + }: + continue + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + if path.suffix == ".html": + target.write_text(_rewrite_links(path.read_text(encoding="utf-8")), encoding="utf-8") + else: + shutil.copy2(path, target) + + pages = destination / "pages" + pages.mkdir() + for kind in ("task", "result"): + buckets = [{} for _ in range(SHARD_COUNT)] + for path in sorted((source / kind).glob("*.html")): + key = path.stem + buckets[_task_id(key) % SHARD_COUNT][key] = _rewrite_links( + path.read_text(encoding="utf-8"), detail_kind=kind + ) + for index, rows in enumerate(buckets): + (pages / f"{kind}-{index:02d}.json").write_text( + json.dumps(rows, ensure_ascii=False, separators=(",", ":")), encoding="utf-8" + ) + (destination / f"{kind}.html").write_text(_loader(kind), encoding="utf-8") + (destination / "evidence-raw.html").write_text( + _raw_evidence_loader(), encoding="utf-8" + ) + + (destination / "data").mkdir(exist_ok=True) + for kind in ("trajectory", "evidence"): + shards = [{} for _ in range(SHARD_COUNT)] + for path in sorted((source / f"data/{kind}").glob("*.json")): + key = path.stem + shards[_task_id(key) % SHARD_COUNT][key] = json.loads( + path.read_text(encoding="utf-8") + ) + for index, rows in enumerate(shards): + (destination / f"data/{kind}-{index:02d}.json").write_text( + json.dumps(rows, ensure_ascii=False, separators=(",", ":")), encoding="utf-8" + ) + app = destination / "assets/app.js" + app.write_text(_data_fetch_adapter() + app.read_text(encoding="utf-8"), encoding="utf-8") + files = [path for path in destination.rglob("*") if path.is_file()] + return { + "files": len(files), + "task_routes": len(list((source / "task").glob("*.html"))), + "result_routes": len(list((source / "result").glob("*.html"))), + } + + +def validate_compact_report(report: Path, *, max_files: int) -> dict[str, int]: + files = [path for path in report.rglob("*") if path.is_file()] + if not files or len(files) > max_files: + raise ValueError(f"compact report file count is outside 1..{max_files}: {len(files)}") + required = [ + report / "index.html", + report / "task.html", + report / "result.html", + report / "evidence-raw.html", + ] + required += [ + report / f"pages/{kind}-{index:02d}.json" + for kind in ("task", "result") + for index in range(SHARD_COUNT) + ] + required += [ + report / f"data/{kind}-{index:02d}.json" + for kind in ("trajectory", "evidence") + for index in range(SHARD_COUNT) + ] + if any(not path.is_file() for path in required): + raise ValueError("compact report route bundles are incomplete") + manifest = json.loads((report / "report-manifest.json").read_text(encoding="utf-8")) + task_routes = sum( + len(json.loads((report / f"pages/task-{index:02d}.json").read_text())) + for index in range(SHARD_COUNT) + ) + result_routes = sum( + len(json.loads((report / f"pages/result-{index:02d}.json").read_text())) + for index in range(SHARD_COUNT) + ) + if task_routes != manifest["task_count"] or result_routes != manifest["logical_results"]: + raise ValueError("compact report route coverage differs from its manifest") + return {"files": len(files), "task_routes": task_routes, "result_routes": result_routes} diff --git a/harness/trajectory_replay/runtime_identity.py b/harness/trajectory_replay/runtime_identity.py new file mode 100644 index 000000000..20d08f8d0 --- /dev/null +++ b/harness/trajectory_replay/runtime_identity.py @@ -0,0 +1,131 @@ +"""Validate the browser runtime identity shared by replay audit and comparison.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any +from urllib.parse import urlsplit + +REMOTE_RUNTIME_FIELDS = ( + "protocolVersion", + "product", + "revision", + "userAgent", + "jsVersion", +) + + +def request_bridge_identity(binding: dict[str, Any]) -> dict[str, object] | None: + bridge = binding.get("request_bridge") + engine = binding.get("engine") + if bridge is None: + if engine == "kitesurf": + raise ValueError("Kitesurf run is missing its request bridge identity") + return None + if engine is not None and engine != "kitesurf": + raise ValueError("request bridge is only valid for Kitesurf runs") + origins = bridge.get("origins") if isinstance(bridge, dict) else None + if ( + not isinstance(bridge, dict) + or bridge.get("kind") != "local_http_fulfill/1" + or not isinstance(origins, list) + or not origins + or origins != sorted(set(origins)) + ): + raise ValueError("run request bridge identity is invalid") + for origin in origins: + parsed = urlsplit(origin) if isinstance(origin, str) else None + if ( + parsed is None + or parsed.scheme not in {"http", "https"} + or not parsed.netloc + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError("run request bridge origin is invalid") + return bridge + + +def runtime_identity(binding: dict[str, Any]) -> tuple[str, dict[str, object]]: + """Return one credential-free, hashable local or hosted runtime identity.""" + + binary = binding.get("binary_sha256") + remote = binding.get("remote_browser") + if isinstance(binary, str) and binary: + if remote is not None: + raise ValueError("run mixes local and remote browser identities") + return f"binary:{binary}", {"binary_sha256": binary} + if binary is not None: + raise ValueError("run local browser identity is invalid") + required = { + "provider", + "browser", + "protocol", + "endpoint_contract", + "account_id_sha256", + "runtime_metadata", + } + if not isinstance(remote, dict) or required - set(remote): + raise ValueError("run browser runtime identity is missing") + for key in required - {"runtime_metadata"}: + if not isinstance(remote.get(key), str) or not remote[key]: + raise ValueError(f"run remote browser identity has invalid {key}") + account_digest = remote["account_id_sha256"] + if len(account_digest) != 64 or any(char not in "0123456789abcdef" for char in account_digest): + raise ValueError("run remote browser account identity is invalid") + metadata = remote["runtime_metadata"] + if ( + not isinstance(metadata, dict) + or set(metadata) != set(REMOTE_RUNTIME_FIELDS) + or any( + not isinstance(metadata.get(key), str) or not metadata[key] + for key in REMOTE_RUNTIME_FIELDS + ) + ): + raise ValueError("run remote browser version identity is incomplete") + serialized = json.dumps(remote, sort_keys=True, separators=(",", ":")) + if any(marker in serialized.lower() for marker in ("authorization", "bearer ", "token")): + raise ValueError("run remote browser identity contains credentials") + digest = hashlib.sha256(serialized.encode()).hexdigest() + return f"remote:{digest}", {"remote_browser": remote} + + +def runtime_identities(binding: dict[str, Any]) -> dict[str, dict[str, object]]: + """Return every runtime allowed by an acceptance participant contract.""" + + binaries = binding.get("binary_sha256s") + if binaries is None: + key, public = runtime_identity(binding) + return {key: public} + if ( + binding.get("binary_sha256") is not None + or binding.get("remote_browser") is not None + or not isinstance(binaries, list) + or not binaries + or len(set(binaries)) != len(binaries) + or any(not isinstance(binary, str) or not binary for binary in binaries) + ): + raise ValueError("participant browser runtime identities are invalid") + return { + f"binary:{binary}": {"binary_sha256": binary} + for binary in binaries + } + + +def assert_observed_runtime( + public_identity: dict[str, object], observed: object, *, task_id: int +) -> None: + """Reject a hosted result produced by a runtime other than its frozen identity.""" + + remote = public_identity.get("remote_browser") + if remote is None: + if observed is not None: + raise ValueError(f"task {task_id}: local result unexpectedly records remote runtime") + return + assert isinstance(remote, dict) + if observed != remote["runtime_metadata"]: + raise ValueError(f"task {task_id}: observed remote browser version changed") diff --git a/harness/trajectory_replay/template_transfer.py b/harness/trajectory_replay/template_transfer.py new file mode 100644 index 000000000..0675dcc16 --- /dev/null +++ b/harness/trajectory_replay/template_transfer.py @@ -0,0 +1,21 @@ +"""Substitute frozen task parameters in replay candidate values.""" + +from __future__ import annotations + +from typing import Any + + +def transfer_values(value: Any, replacements: list[tuple[str, str]]) -> Any: + if isinstance(value, list): + return [transfer_values(item, replacements) for item in value] + if isinstance(value, dict): + return {key: transfer_values(item, replacements) for key, item in value.items()} + if not isinstance(value, str): + return value + result = value + for before, after in replacements: + if before == result: + result = after + elif len(before) >= 3 and not before.isdigit(): + result = result.replace(before, after) + return result diff --git a/harness/trajectory_replay/webarena_corrections.py b/harness/trajectory_replay/webarena_corrections.py index 788a6c16f..bc486d432 100644 --- a/harness/trajectory_replay/webarena_corrections.py +++ b/harness/trajectory_replay/webarena_corrections.py @@ -11,8 +11,10 @@ import copy import hashlib import json +import re from collections.abc import Mapping from dataclasses import dataclass +from importlib.resources import files from pathlib import Path from types import MappingProxyType from typing import Any @@ -22,6 +24,11 @@ from webarena_verified.types.eval import TaskEvalContext, TaskEvalResult from webarena_verified.types.task import WebArenaVerifiedTask +from harness.trajectory_replay.webarena_runtime_source import ( + WEBARENA_VERIFIED_FORK_URL, + verify_webarena_runtime_source, +) + SCHEMA = "lex_webarena_replay_task_corrections/1" DEFAULT_CORRECTIONS_PATH = ( Path(__file__).resolve().parents[2] @@ -131,7 +138,11 @@ class CorrectedReplayContract: network_alternatives: Mapping[int, tuple[WebArenaVerifiedTask, ...]] -def load_corrections(path: str | Path = DEFAULT_CORRECTIONS_PATH) -> CorrectionCatalog: +def load_corrections( + path: str | Path = DEFAULT_CORRECTIONS_PATH, + *, + require_current_upstream: bool = True, +) -> CorrectionCatalog: """Load and validate the replay-only correction catalog.""" source_path = Path(path).expanduser().resolve() @@ -141,9 +152,24 @@ def load_corrections(path: str | Path = DEFAULT_CORRECTIONS_PATH) -> CorrectionC raise ValueError(f"unsupported WebArena correction schema: {payload.get('schema')!r}") upstream = payload.get("upstream") if not isinstance(upstream, dict) or not all( - upstream.get(key) for key in ("benchmark", "package_version", "dataset_sha256") + upstream.get(key) for key in ("benchmark", "dataset_sha256") ): raise ValueError("correction catalog must identify its upstream benchmark and dataset") + if require_current_upstream: + identity = verify_webarena_runtime_source() + dataset_sha256 = hashlib.sha256( + files("webarena_verified") + .joinpath("assets/dataset/webarena-verified.json") + .read_bytes() + ).hexdigest() + if ( + upstream.get("benchmark") != "WebArena-Verified" + or upstream.get("repository") != WEBARENA_VERIFIED_FORK_URL + or upstream.get("branch") != "lex-main" + or upstream.get("revision") != identity.revision + or upstream.get("dataset_sha256") != dataset_sha256 + ): + raise ValueError("correction catalog targets a different WebArena fork revision") records: dict[int, Mapping[str, Any]] = {} record_sha256s: dict[int, str] = {} for record in payload.get("corrections") or []: @@ -265,28 +291,353 @@ def apply_correction( ): raise ValueError(f"task {task.task_id}: incomplete independent reference") corrected["eval"][0]["expected"] = copy.deepcopy(expected_response) + elif observation.get("schema") == "trajectory_replay_route_event_reference/1": + corrected_url = corrected["eval"][1]["expected"]["url"] + coordinate_match = re.search( + r"-?\d+(?:\.\d+)?,-?\d+(?:\.\d+)?;" + r"-?\d+(?:\.\d+)?,-?\d+(?:\.\d+)?", + corrected_url, + ) + if ( + observation.get("task_intent") != original.get("intent") + or observation.get("official_expected_url") + != original["eval"][1]["expected"]["url"] + or observation.get("corrected_expected_url") != corrected_url + or observation.get("official_navigation_only") + != original["eval"][1]["navigation_only"] + or observation.get("corrected_navigation_only") + != corrected["eval"][1]["navigation_only"] + or observation.get("observed_event_sequence") + != [ + "GET /directions (HTML navigation)", + "GET /osrm/routed-car/route/v1/driving/{origin};{destination} " + "(route XHR, HTTP 200)", + ] + or coordinate_match is None + or coordinate_match.group(0) + not in str(observation.get("observed_correct_direction_request") or "") + or len(str(observation.get("observed_har_sha256") or "")) != 64 + ): + raise ValueError(f"task {task.task_id}: incomplete route-order reference") + elif observation.get("schema") == "trajectory_replay_navigation_xhr_reference/1": + expected_status = original["eval"][1]["expected"]["query_params"][ + "filters[status]" + ][0] + if ( + observation.get("expected_filter_status") != expected_status + or observation.get("official_navigation_only") + != original["eval"][1]["navigation_only"] + or observation.get("corrected_navigation_only") + != corrected["eval"][1]["navigation_only"] + or observation.get("observed_request_contains") + != f"filters%5Bstatus%5D={expected_status}" + or len(str(observation.get("observed_har_sha256") or "")) != 64 + ): + raise ValueError(f"task {task.task_id}: incomplete navigation-XHR reference") + elif ( + observation.get("schema") + == "trajectory_replay_repeated_network_event_reference/1" + ): + original_members = original["eval"][2:] + corrected_members = corrected["eval"][2:] + member_ids = [row["expected"]["post_data"]["user_id"] for row in original_members] + if ( + observation.get("required_member_ids") != member_ids + or observation.get("official_last_event_only") + != [row["last_event_only"] for row in original_members] + or observation.get("corrected_last_event_only") + != [row["last_event_only"] for row in corrected_members] + or any( + row["expected"]["url"] != observation.get("endpoint_pattern") + for row in original_members + ) + or ( + observation.get("observed_har_sha256") is not None + and len(str(observation["observed_har_sha256"])) != 64 + ) + ): + raise ValueError( + f"task {task.task_id}: incomplete repeated-network-event reference" + ) + elif observation.get("schema") == "trajectory_replay_reddit_forum_reference/1": + observed = observation.get("observation") or {} + original_expected = original["eval"][1]["expected"] + corrected_expected = corrected["eval"][1]["expected"] + if ( + original.get("intent") + != 'Subscribe to the "machine learning" forum from the page of the newest post in that forum.' + or original_expected.get("url") + != [ + "__REDDIT__/f/machine learning/subscribe.json", + "__REDDIT__/f/machine learning/unsubscribe.json", + ] + or corrected_expected.get("url") + != [ + "__REDDIT__/f/MachineLearning/subscribe.json", + "__REDDIT__/f/MachineLearning/unsubscribe.json", + ] + or observed.get("matched_forum_href") != "/f/MachineLearning" + or observed.get("canonical_forum_href") != "/f/MachineLearning" + or observed.get("newest_post_href") + != "/f/MachineLearning/120613/d-yan-lecun-s-recent-recommendations" + or any( + len(str(observed.get(key) or "")) != 64 + for key in ("search_response_sha256", "newest_response_sha256") + ) + ): + raise ValueError(f"task {task.task_id}: incomplete Reddit forum reference") + elif observation.get("schema") == "trajectory_replay_map_live_reference/1": + bindings = observation.get("source_evidence_sha256") or {} + comparisons = observation.get("contract_values") or [] + if ( + observation.get("task_intent") != original.get("intent") + or not isinstance(observation.get("browser_call_count"), int) + or observation.get("browser_call_count") < 1 + or not isinstance(bindings, dict) + or not {"network_har", "trace", "outputs", "readiness"} + <= set(bindings) + or any( + not isinstance(value, str) or len(value) != 64 + for value in bindings.values() + ) + or not isinstance(comparisons, list) + or not comparisons + or any( + not isinstance(item, dict) + or not isinstance(item.get("path"), str) + or _resolve(original, item["path"]) != item.get("official") + or _resolve(corrected, item["path"]) != item.get("corrected") + for item in comparisons + ) + or observation.get("observed_response") is None + or observation.get("derivation", {}).get("kind") + not in { + "bounded_nominatim_osrm/1", + "frozen_wikipedia_nominatim_osrm/1", + "nominatim_spatial_distance/1", + "frozen_wikipedia_relation_ids/1", + "nominatim_object_geometry/1", + } + ): + raise ValueError(f"task {task.task_id}: incomplete live map reference") + elif observation.get("schema") == "trajectory_replay_gitlab_live_reference/1": + bindings = observation.get("source_evidence_sha256") or {} + query = observation.get("query") or {} + issue = observation.get("selected_issue") or {} + comparisons = observation.get("contract_values") or [] + if ( + observation.get("task_intent") != original.get("intent") + or observation.get("browser_call_count") != 1 + or not isinstance(bindings, dict) + or not {"network_har", "actual_response", "evaluation", "trace"} + <= set(bindings) + or any( + not isinstance(value, str) or len(value) != 64 + for value in bindings.values() + ) + or query.get("authenticated_user_id") != query.get("author_id") + or issue.get("author_id") != query.get("authenticated_user_id") + or query.get("title_search") != "feature" + or query.get("order_by") != "created_at" + or query.get("sort") != "desc" + or "feature" not in str(issue.get("title") or "").lower() + or issue.get("state") not in {"opened", "closed"} + or not isinstance(comparisons, list) + or not comparisons + or any( + not isinstance(item, dict) + or not isinstance(item.get("path"), str) + or _resolve(original, item["path"]) != item.get("official") + or _resolve(corrected, item["path"]) != item.get("corrected") + for item in comparisons + ) + or observation.get("observed_response", {}).get("retrieved_data") + != [issue.get("state") == "closed"] + ): + raise ValueError(f"task {task.task_id}: incomplete live GitLab reference") + elif observation.get("schema") == "trajectory_replay_gitlab_repository_reference/1": + bindings = observation.get("source_evidence_sha256") or {} + query = observation.get("query") or {} + project = observation.get("selected_project") or {} + page_sizes = observation.get("commit_page_sizes") or [] + commit_count = observation.get("commit_count") + observed_post = observation.get("observed_post") or {} + comparisons = observation.get("contract_values") or [] + expected_post = corrected["eval"][1]["expected"]["post_data"] + topic = str(original.get("instantiation_dict", {}).get("topic") or "") + normalized_topic = re.sub(r"[^a-z0-9]", "", topic.lower()) + normalized_path = re.sub( + r"[^a-z0-9]", "", str(project.get("path_with_namespace") or "").lower() + ) + if ( + observation.get("task_intent") != original.get("intent") + or observation.get("browser_call_count") != 1 + or not isinstance(bindings, dict) + or not {"network_har", "trace", "outputs", "readiness"} + <= set(bindings) + or any( + not isinstance(value, str) or len(value) != 64 + for value in bindings.values() + ) + or query.get("search_term") != topic + or query.get("simple") is not True + or query.get("per_page") != 100 + or query.get("result_count") != 1 + or not isinstance(project.get("id"), int) + or not normalized_topic + or normalized_topic not in normalized_path + or not isinstance(page_sizes, list) + or not page_sizes + or any(type(size) is not int or size < 1 or size > 100 for size in page_sizes) + or any(size != 100 for size in page_sizes[:-1]) + or page_sizes[-1] >= 100 + or commit_count != sum(page_sizes) + or observed_post.get("kind") != "repo" + or observed_post.get("title") != project.get("path_with_namespace") + or not str(observed_post.get("url") or "").endswith( + "/" + str(project.get("path_with_namespace") or "") + ) + or observed_post.get("body") != f"{commit_count} commits already!" + or expected_post.get("submission[title]") != observed_post.get("title") + or not str(expected_post.get("submission[url]") or "").endswith( + "/" + str(project.get("path_with_namespace") or "") + ) + or expected_post.get("submission[body]") != observed_post.get("body") + or not isinstance(comparisons, list) + or not comparisons + or any( + not isinstance(item, dict) + or not isinstance(item.get("path"), str) + or _resolve(original, item["path"]) != item.get("official") + or _resolve(corrected, item["path"]) != item.get("corrected") + for item in comparisons + ) + ): + raise ValueError( + f"task {task.task_id}: incomplete live GitLab repository reference" + ) + elif ( + observation.get("schema") + == "trajectory_replay_magento_create_options_reference/1" + ): + source = observation.get("official_source") or {} + source_observation = observation.get("source_observation") or {} + required_option = observation.get("required_option") or {} + original_expected = original["eval"][1]["expected"] + corrected_expected = corrected["eval"][1]["expected"] + if ( + observation.get("task_intent") != original.get("intent") + or observation.get("official_endpoint") != original_expected.get("url") + or observation.get("corrected_endpoint") != corrected_expected.get("url") + or corrected_expected.get("response_status") != 200 + or corrected_expected.get("post_data") + != { + "options[0][label]": required_option.get("label"), + "options[0][attribute_id]": required_option.get("attribute_id"), + "options[0][is_new]": required_option.get("is_new"), + } + or source.get("repository") != "https://github.com/magento/magento2" + or source.get("path") + != "app/code/Magento/ConfigurableProduct/view/adminhtml/web/js/variations/steps/attributes_values.js" + or len(str(source.get("revision") or "")) != 40 + or len(str(source.get("sha256") or "")) != 64 + or source_observation + != { + "method": "saveOptions", + "request_method": "POST", + "request_url_binding": "createOptionsUrl", + "request_body_root": "options", + "success_status_class": 200, + } + ): + raise ValueError( + f"task {task.task_id}: incomplete Magento create-options reference" + ) + elif ( + observation.get("schema") + == "trajectory_replay_magento_mass_status_reference/1" + ): + bindings = observation.get("source_evidence_sha256") or {} + event = observation.get("observed_event") or {} + official_contract = observation.get("official_contract") or {} + corrected_contract = observation.get("corrected_contract") or {} + original_expected = original["eval"][1]["expected"] + corrected_expected = corrected["eval"][1]["expected"] + if ( + observation.get("task_intent") != original.get("intent") + or observation.get("browser_call_count") != 3 + or not isinstance(bindings, dict) + or not { + "network_har", + "actual_response", + "evaluation", + "trace", + "readiness", + "outputs", + } + <= set(bindings) + or any( + not isinstance(value, str) or len(value) != 64 + for value in bindings.values() + ) + or official_contract + != { + "url": original_expected.get("url"), + "post_data": original_expected.get("post_data"), + "http_method": original_expected.get("http_method"), + "response_status": original_expected.get("response_status"), + "navigation_only": original["eval"][1].get("navigation_only"), + } + or corrected_contract + != { + "url": corrected_expected.get("url"), + "post_data": corrected_expected.get("post_data"), + "http_method": corrected_expected.get("http_method"), + "response_status": corrected_expected.get("response_status"), + "navigation_only": corrected["eval"][1].get("navigation_only"), + } + or event.get("method") != "POST" + or event.get("url") + != "__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/" + or event.get("response_status") != 302 + or event.get("updated_record_count") != 16 + or event.get("post_data") + != { + "excluded": "false", + "filters[placeholder]": "true", + "search": "hollister", + "namespace": "product_listing", + } + or observation.get("derivation", {}).get("kind") + != "native_magento_filtered_mass_status/1" + ): + raise ValueError( + f"task {task.task_id}: incomplete Magento mass-status reference" + ) elif observation.get("derivation", {}).get("kind") == "qualified_catalog_price_range/1": source = observation.get("independent_source") or {} products = source.get("qualified_products") or [] if ( - observation.get("schema") != "trajectory_replay_revised_reference/1" - or source.get("total_count") != source.get("returned_items") - or type(source.get("total_count")) is not int - or source["total_count"] < len(products) - or not products - or len({row.get("sku") for row in products}) != len(products) - or len({row.get("name") for row in products}) != len(products) - or any( - not row.get("sku") - or not row.get("name") - or type(row.get("price_cents")) is not int - or row["price_cents"] < 0 - or row.get("currency") != "USD" - or not row.get("qualification_evidence") - for row in products - ) + observation.get("schema") != "trajectory_replay_revised_reference/1" + or source.get("total_count") != source.get("returned_items") + or type(source.get("total_count")) is not int + or source["total_count"] < len(products) + or not products + or len({row.get("sku") for row in products}) != len(products) + or len({row.get("name") for row in products}) != len(products) + or any( + not row.get("sku") + or not row.get("name") + or type(row.get("price_cents")) is not int + or row["price_cents"] < 0 + or row.get("currency") != "USD" + or not row.get("qualification_evidence") + for row in products + ) ): - raise ValueError(f"task {task.task_id}: incomplete qualified catalog reference") + raise ValueError( + f"task {task.task_id}: incomplete qualified catalog reference" + ) prices = [row["price_cents"] for row in products] expected = { "task_type": "RETRIEVE", @@ -301,7 +652,9 @@ def apply_correction( "error_details": None, } if observation.get("expected_response") != expected: - raise ValueError(f"task {task.task_id}: catalog reference derivation differs") + raise ValueError( + f"task {task.task_id}: catalog reference derivation differs" + ) corrected["eval"][0]["expected"] = expected else: raise ValueError(f"task {task.task_id}: unsupported independent reference") diff --git a/harness/trajectory_replay/webarena_replay.py b/harness/trajectory_replay/webarena_replay.py index abd717d5f..ecb3cb2a2 100644 --- a/harness/trajectory_replay/webarena_replay.py +++ b/harness/trajectory_replay/webarena_replay.py @@ -65,6 +65,10 @@ "waiting for scheduled navigations to finish", ) +_STATE_CLEAR_NAVIGATION_SETTLE_TIMEOUT_MS = 1_500 +_STATE_CLEAR_NAVIGATION_GRACE_MS = 300 +_STATE_CLEAR_MAX_ATTEMPTS = 3 + def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -131,7 +135,51 @@ def _execute_replay_call( def execute() -> None: page = service._page - locator = page.locator(selector).first + layout_observer = getattr(service, "layout_observer", None) + timeout_ms = int(public_arguments.get("timeout_ms", 60_000)) + deadline = time.monotonic() + timeout_ms / 1000 + + def remaining() -> int: + budget = int((deadline - time.monotonic()) * 1000) + if budget < 1: + raise TimeoutError("compiled snapshot-ref action timed out") + return budget + + if layout_observer is not None: + layout_observer(page, remaining()) + locator = None + if layout_observer is not None and selector.endswith(":visible"): + stable = page.locator(selector.removesuffix(":visible")) + count = getattr(stable, "count", None) + if callable(count) and count() == 1: + locator = stable.first + if not bool(public_arguments.get("force", False)): + visible = page.locator(selector) + while visible.count() != 1: + page.wait_for_timeout(min(100, remaining())) + layout_observer(page, remaining()) + if locator is None: + locator = page.locator(selector).first + if public_arguments.get("wait_for_enabled"): + while True: + state = locator.evaluate( + """element => ({ + connected: element.isConnected, + disabled: Boolean(element.disabled), + effectivelyDisabled: element.matches(':disabled'), + ariaDisabled: element.getAttribute('aria-disabled') + })""" + ) + if ( + state.get("connected") + and not state.get("disabled") + and not state.get("effectivelyDisabled") + and state.get("ariaDisabled") != "true" + ): + break + page.wait_for_timeout(min(100, remaining())) + if layout_observer is not None: + layout_observer(page, remaining()) if action == "click": if diagnostics is None: locator.evaluate("element => element.click()") @@ -152,25 +200,60 @@ def execute() -> None: diagnostics["click_dispatch_state"] = state page.wait_for_timeout(500) elif action == "fill": - platform = locator.evaluate( - "element => { element.focus(); return navigator.platform; }" - ) - modifier = ( - "Meta" - if str(platform).startswith(("Mac", "iPhone", "iPad", "iPod")) - else "Control" - ) - page.keyboard.press(f"{modifier}+A") - page.keyboard.press("Backspace") - page.keyboard.insert_text(str(public_arguments.get("text", ""))) + text = str(public_arguments.get("text", "")) + if public_arguments.get("force"): + state = locator.evaluate( + "(element, expected) => { " + "const setter = Object.getOwnPropertyDescriptor(" + "Object.getPrototypeOf(element), 'value')?.set; " + "if (!setter) throw new Error('target has no native value setter'); " + "setter.call(element, expected); " + "element.dispatchEvent(new Event('input', {bubbles: true})); " + "element.dispatchEvent(new Event('change', {bubbles: true})); " + "if (element.value !== expected) " + "throw new Error('filled value was not committed'); " + "return {value: element.value}; }", + text, + ) + else: + platform = locator.evaluate( + "element => { element.focus(); return navigator.platform; }" + ) + modifier = ( + "Meta" + if str(platform).startswith(("Mac", "iPhone", "iPad", "iPod")) + else "Control" + ) + page.keyboard.press(f"{modifier}+A") + page.keyboard.press("Backspace") + page.keyboard.insert_text(text) + state = locator.evaluate( + "(element, expected) => { " + "element.dispatchEvent(new Event('input', {bubbles: true})); " + "element.dispatchEvent(new Event('change', {bubbles: true})); " + "if (element.value !== expected) " + "throw new Error('filled value was not committed'); " + "return {value: element.value}; }", + text, + ) + if diagnostics is not None: + diagnostics["fill_dispatch_state"] = state else: value = str(public_arguments.get("value", "")) - locator.evaluate( + state = locator.evaluate( "(element, value) => { if (element.tagName.toLowerCase() !== 'select') " - "throw new Error('target is not a select element'); element.value = value; " - "element.dispatchEvent(new Event('change', {bubbles: true})); }", + "throw new Error('target is not a select element'); " + "const before = element.value; " + "element.value = value; if (element.value !== value) " + "throw new Error('select option value was not found'); " + "element.dispatchEvent(new Event('input', {bubbles: true})); " + "element.dispatchEvent(new Event('change', {bubbles: true})); " + "return {before, after: element.value, selectedText: " + "element.selectedOptions[0]?.textContent ?? null}; }", value, ) + if diagnostics is not None: + diagnostics["select_dispatch_state"] = state service._submit(execute) except Exception as exc: @@ -287,6 +370,36 @@ def _rebind_calls( return rebound +def _clear_page_storage(page: Any) -> None: + script = ( + "async () => {" + "localStorage.clear(); sessionStorage.clear();" + "if ('caches' in globalThis) {" + "for (const key of await caches.keys()) await caches.delete(key);" + "}" + "if ('indexedDB' in globalThis && indexedDB.databases) {" + "for (const db of await indexedDB.databases()) {" + "if (db.name) indexedDB.deleteDatabase(db.name);" + "}" + "}" + "}" + ) + for attempt in range(_STATE_CLEAR_MAX_ATTEMPTS): + try: + page.evaluate(script) + return + except Exception as exc: + if ( + "execution context was destroyed" not in str(exc).lower() + or attempt + 1 == _STATE_CLEAR_MAX_ATTEMPTS + ): + raise + page.wait_for_load_state( + "domcontentloaded", timeout=_STATE_CLEAR_NAVIGATION_SETTLE_TIMEOUT_MS + ) + page.wait_for_timeout(_STATE_CLEAR_NAVIGATION_GRACE_MS) + + def _clear_browser_state(service: Any, origins: set[str]) -> None: """Fail closed unless all persistent browser state is cleared before a task.""" @@ -295,22 +408,40 @@ def clear() -> None: page = service._page if context is None or page is None: raise RuntimeError("browser context is not ready for state reset") + if getattr(service, "_fresh_context_per_task", False): + return context.clear_cookies() context.clear_permissions() - cdp = context.new_cdp_session(page) + supports_network_reset_cdp = getattr( + service, "_supports_network_reset_cdp", True + ) + supports_storage_cdp = getattr(service, "_supports_storage_cdp", True) + cdp = ( + context.new_cdp_session(page) + if supports_network_reset_cdp or supports_storage_cdp + else None + ) try: - cdp.send("Network.enable") - cdp.send("Network.clearBrowserCache") - cdp.send("Network.clearBrowserCookies") - for origin in sorted(origins): - cdp.send( - "Storage.clearDataForOrigin", - {"origin": origin, "storageTypes": "all"}, - ) + if supports_network_reset_cdp: + cdp.send("Network.enable") + cdp.send("Network.clearBrowserCache") + cdp.send("Network.clearBrowserCookies") + if supports_storage_cdp: + for origin in sorted(origins): + cdp.send( + "Storage.clearDataForOrigin", + {"origin": origin, "storageTypes": "all"}, + ) + else: + for origin in sorted(origins): + page.goto(origin, wait_until="domcontentloaded", timeout=30_000) + _clear_page_storage(page) page.goto("about:blank", wait_until="load", timeout=10_000) - cdp.send("Page.resetNavigationHistory") + if supports_storage_cdp: + cdp.send("Page.resetNavigationHistory") finally: - cdp.detach() + if cdp is not None: + cdp.detach() service._submit(clear) @@ -421,21 +552,28 @@ def _run_one( # noqa: PLR0913 for index, arguments in enumerate(calls, start=1): call_started = time.perf_counter() diagnostics: dict[str, Any] = {} - result_status, result_text = _execute_replay_call(tool, arguments, diagnostics=diagnostics) + result_status, result_text = _execute_replay_call( + tool, arguments, diagnostics=diagnostics + ) dispatched_before_timeout = _click_was_dispatched_before_navigation_timeout( arguments, result_status, result_text ) + effective_status, observation_error = _replay_step_status( + arguments, + result_status, + result_text, + dispatched=dispatched_before_timeout, + ) record = { "index": index, "arguments": arguments, - "status": "success" if dispatched_before_timeout else result_status, + "status": effective_status, "tool_status": result_status, "duration_ms": round((time.perf_counter() - call_started) * 1000, 3), "result": result_text, } if diagnostics: record["action_diagnostics"] = diagnostics - observation_error = _embedded_observation_error(arguments, result_status, result_text) if observation_error is not None: record["observation_status"] = "error" record["observation_error"] = observation_error @@ -445,14 +583,15 @@ def _run_one( # noqa: PLR0913 "action_retried": False, } records.append(record) - if result_status != "success" and not dispatched_before_timeout: + if effective_status != "success": try: record["failure_ui_state"] = _capture_failure_ui_state(service, arguments) except Exception as observation_exc: record["failure_ui_state_error"] = ( f"{type(observation_exc).__name__}: {observation_exc}" )[:300] - raise RuntimeError(f"step {index} failed: {result_text}") + detail = observation_error or result_text + raise RuntimeError(f"step {index} failed: {detail}") timings["trajectory_replay_ms"] = (time.perf_counter() - replay_started) * 1000 _verify_session_viewport(service, viewport_evidence, "after_trajectory") process_status = "pass" @@ -707,12 +846,12 @@ def _attempt_needs_retry(attempt: dict[str, Any]) -> bool: def _embedded_observation_error( arguments: dict[str, Any], result_status: str, result_text: str ) -> str | None: - """Expose BrowserTool snapshot degradation without rewriting task outcome. + """Expose BrowserTool snapshot degradation as a replay failure. BrowserTool currently returns a successful tool status with a bracketed error payload when the page execution context changes during a snapshot. The action - remains completed, but the observation is unavailable and must be explicit in - evidence so diagnosis can identify the first degraded step. + remains completed, but the required observation is unavailable. Both replay + runners use this classification to fail the step and retain the exact error. """ if arguments.get("action") != "snapshot" or result_status != "success": @@ -723,6 +862,17 @@ def _embedded_observation_error( return result_text[len(prefix) :].removesuffix("]").strip() +def _replay_step_status( + arguments: dict[str, Any], result_status: str, result_text: str, *, dispatched: bool +) -> tuple[str, str | None]: + """Apply the same snapshot failure policy in every replay runner.""" + + observation_error = _embedded_observation_error(arguments, result_status, result_text) + if observation_error is not None: + return "error", observation_error + return ("success" if dispatched else result_status), None + + def _diagnostic_locator(selector: str) -> str: """Expand visibility/ordinal filters without confusing CSS with locators. @@ -744,7 +894,8 @@ def _diagnostic_locator(selector: str) -> str: elif char in ("'", '"'): quote = char elif selector.startswith(":visible", index) and ( - index + 8 == len(selector) or not (selector[index + 8].isalnum() or selector[index + 8] in "_-\\") + index + 8 == len(selector) + or not (selector[index + 8].isalnum() or selector[index + 8] in "_-\\") ): index += 8 continue diff --git a/harness/trajectory_replay/webarena_report.py b/harness/trajectory_replay/webarena_report.py index 1b16e2e53..e4ec24e6a 100644 --- a/harness/trajectory_replay/webarena_report.py +++ b/harness/trajectory_replay/webarena_report.py @@ -4,29 +4,43 @@ import hashlib import html +import importlib.resources import json import math import re import shutil import statistics -from collections import defaultdict +import zipfile +from collections import Counter, defaultdict from pathlib import Path from typing import Any from urllib.parse import urlencode, urlsplit -from harness.trajectory_replay.failure_matrix import failure_category_matrix +from harness.trajectory_replay.catalog_contracts import ( + LEGACY_CATALOG_CONTRACTS_SCHEMA, + LEGACY_CATALOG_REPORT_SCHEMA, +) +from harness.trajectory_replay.evaluator_transition import json_sha256, reevaluate_pass +from harness.trajectory_replay.failure_matrix import ( + failure_category_matrix, + failure_signature_cards, +) from harness.trajectory_replay.interpretation import render_interpretation, standalone_document from harness.trajectory_replay.participants import ( bind_participant, participant_registry, resolve_participant_id, ) +from harness.trajectory_replay.remote_runtime import load_remote_sites, remote_benchmark +from harness.trajectory_replay.runtime_identity import request_bridge_identity +from harness.trajectory_replay.webarena_corrections import load_corrections from harness.trajectory_replay.webarena_review import ( load_validated_failure_review, programmatic_failure, ) LOCAL_PATH_RE = re.compile(r"/(?:Users|home|root|private|tmp|var|opt)/[^\s\"']+") +WEB_URL_RE = re.compile(r"https?://[^\s\"']+") # This notice describes one retained historical cohort, not trajectory replay in # general. Keying all three immutable inputs prevents a future requalified run @@ -151,8 +165,19 @@ def _clean(value: Any) -> Any: if isinstance(value, list): return [_clean(item) for item in value] if isinstance(value, str): - value = LOCAL_PATH_RE.sub("[local path removed]", value) - value = re.sub(r"sk-[A-Za-z0-9_-]{12,}", "[secret removed]", value) + url_spans = [match.span() for match in WEB_URL_RE.finditer(value)] + + def redact_local_path(match: re.Match[str]) -> str: + if any(start <= match.start() and match.end() <= end for start, end in url_spans): + return match.group(0) + return "[local path removed]" + + value = LOCAL_PATH_RE.sub(redact_local_path, value) + value = re.sub( + r"(? dict[int, str def require_all_reference_validity( profile: dict[str, Any], results: dict[str, Any], dataset: dict[str, Any] ) -> None: - """Reject a formal three-method release with any unverified task reference.""" + """Reject a formal replay release with any unverified task reference.""" if (profile.get("scoring") or {}).get("require_all_reference_validity") is not True: return @@ -564,6 +589,11 @@ def _metrics_table( [{"id": row["id"], "value": row["valid_reference_passed"]} for row in methods], "value", higher=True, ) + metric_keys = ["duration", "action_latency"] + if any(row["cpu"]["n"] for row in methods): + metric_keys.append("cpu") + if any(row["memory"]["n"] for row in methods): + metric_keys.append("memory") metric_winners = { metric: { stat: _unique_winners( @@ -573,13 +603,21 @@ def _metrics_table( ) if metric == "action_latency" else set() for stat in ("mean", "p50", "p95") } - for metric in ("duration", "action_latency", "cpu", "memory") + for metric in metric_keys } - headers = ( - ("方法", "官方通过", "官方通过率", "可核验题通过", "全部任务端到端耗时", "单次动作耗时", "CPU 时间", "平均内存") - if lang == "zh" - else ("Method", "Official passes", "Official pass rate", "Passes on verifiable tasks", "End-to-end time, all tasks", "Per-action latency", "CPU time", "Mean memory") - ) + metric_headers = { + "duration": ("全部任务端到端耗时", "End-to-end time, all tasks"), + "action_latency": ("单次动作耗时", "Per-action latency"), + "cpu": ("CPU 时间", "CPU time"), + "memory": ("平均内存", "Mean memory"), + } + headers = [ + "方法" if lang == "zh" else "Method", + "官方通过" if lang == "zh" else "Official passes", + "官方通过率" if lang == "zh" else "Official pass rate", + "可核验题通过" if lang == "zh" else "Passes on verifiable tasks", + *(metric_headers[key][0 if lang == "zh" else 1] for key in metric_keys), + ] if not show_valid_reference: headers = headers[:3] + headers[4:] body = [] @@ -603,21 +641,14 @@ def time_value(value: float | None) -> str: f'{row["valid_reference_passed"]}/{row["valid_reference_total"]}', key in valid_winners, lang=lang, ), - _metric_lines( - row["duration"], time_value, key, metric_winners["duration"], lang=lang - ), - _metric_lines( - row["action_latency"], - time_value, - key, - metric_winners["action_latency"], - lang=lang, - ), - _metric_lines(row["cpu"], time_value, key, metric_winners["cpu"], lang=lang), - _metric_lines( - row["memory"], _format_mib, key, metric_winners["memory"], lang=lang - ), ] + for metric in metric_keys: + formatter = _format_mib if metric == "memory" else time_value + cells.append( + _metric_lines( + row[metric], formatter, key, metric_winners[metric], lang=lang + ) + ) body.append( "" + f'{_e(cells[0])}' @@ -644,10 +675,14 @@ def _resource_table(methods: list[dict[str, Any]], lang: str) -> str: ) rows = [] for key, label_zh, label_en, formatter in metrics: + if not any(method[key]["n"] for method in methods): + continue cells = [_e(label_zh if zh else label_en)] for method in methods: cells.append(_metric_lines(method[key], formatter, method["id"], {stat: set() for stat in ("mean", "p50", "p95")}, lang=lang)) rows.append("" + "".join(f"{cell}" for cell in cells) + "") + if not rows: + return "" headers = ("资源指标" if zh else "Resource metric", *(method["label"] for method in methods)) return f'
    {"".join(f"" for h in headers)}{"".join(rows)}
    {_e(h)}
    ' @@ -665,6 +700,8 @@ def _successful_distribution_table(methods: list[dict[str, Any]], lang: str) -> ("cpu_utilization", "CPU 平均使用率", "Mean CPU utilization", lambda v: "—" if v is None else f"{v:.1f}%")) rows = [] for name, label_zh, label_en, formatter in metrics: + if name != "duration" and not any(method["successful"][name]["n"] for method in methods): + continue cells = [_e(label_zh if zh else label_en)] + [ _metric_lines(method["successful"][name], formatter, method["id"], {stat: set() for stat in ("mean", "p50", "p95")}, lang=lang) @@ -702,6 +739,18 @@ def _public_evidence( site_reset = _load_artifact((attempt.get("evidence") or {}).get("site_reset")) readiness = _load_artifact((attempt.get("evidence") or {}).get("readiness")) failed = programmatic_failure(attempt) + if failed["stage"] == "browser_tool_execution" and failed.get("step") is None: + failed_trace_step = next( + ( + step.get("index") + for step in reversed(trace) + if step.get("status") in {"fail", "error"} + and isinstance(step.get("index"), int) + ), + None, + ) + if failed_trace_step is not None: + failed = {**failed, "step": failed_trace_step} if failed["stage"] == "official_evaluation" and evaluation: missing_events = [ result for result in evaluation.get("evaluators_results") or [] @@ -777,6 +826,7 @@ def _public_evidence( "supplemental_diagnostic": _supplemental_diagnostic(review, int(attempt["task_id"])), "screenshot": screenshot, } + public = _clean(public) _write_json(output / "data/evidence" / f"{key}.json", public) return public @@ -804,7 +854,10 @@ def _detail_page( key = f'{attempt["participant_id"]}-{attempt["task_id"]}' report_root = "../" if zh else "../../" stage_labels = { - "browser_tool_execution": ("浏览器动作执行", "Browser action execution"), + "browser_session_setup": ("浏览器会话初始化", "Browser session setup"), + "browser_tool_execution": ("Playwright 动作执行", "Playwright action execution"), + "evidence_finalization": ("执行证据收尾", "Evidence finalization"), + "official_evaluator_execution": ("官方评测器执行", "Official evaluator execution"), "side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"), "official_evaluation": ("官方验收", "Official evaluation"), "pass": ("无", "None"), @@ -823,7 +876,14 @@ def _detail_page( + (' aria-current="page"' if peer_key == key else '') + f'>{_e(peer["browser"])}{peer_status}' ) - actual_by_index = {int(step["index"]): step for step in evidence["trajectory"]} + # A watchdog can preserve a task-level timeout when an older runner did not + # checkpoint the active action. Keep that evidence without pretending it + # belongs to a numbered trajectory step. + actual_by_index = { + index: step + for step in evidence["trajectory"] + if isinstance((index := step.get("index")), int) and not isinstance(index, bool) + } steps = [] for number, planned in enumerate(attempt["planned_actions"], 1): actual = actual_by_index.get(number) @@ -861,8 +921,14 @@ def _detail_page( ) failure_panel = "" if attempt["status"] != "pass": - step_link = f'{"跳到失败动作" if zh else "Go to failed action"} #{failed_index}' if failed_index else f'{"查看触发动作" if zh else "Inspect trigger action"} #{trigger_index}' if trigger_index else f'{"查看已完成动作" if zh else "View completed actions"}' - action_locator = f'{failed_action} #{failed_index}' if failed_index else ((f'{("触发动作" if zh else "Trigger action")} #{trigger_index}; {("请求未满足验收条件" if zh else "request does not meet acceptance")}') if trigger_index else ("所有动作执行后" if zh else "After all actions")) + unknown_active_action = ( + failed_index is None + and trigger_index is None + and int(attempt.get("completed_action_count") or 0) + < int(attempt.get("action_count") or 0) + ) + step_link = f'{"跳到失败动作" if zh else "Go to failed action"} #{failed_index}' if failed_index else f'{"查看触发动作" if zh else "Inspect trigger action"} #{trigger_index}' if trigger_index else f'{"查看已保存轨迹证据" if zh else "View saved trajectory evidence"}' + action_locator = f'{failed_action} #{failed_index}' if failed_index else ((f'{("触发动作" if zh else "Trigger action")} #{trigger_index}; {("请求未满足验收条件" if zh else "request does not meet acceptance")}') if trigger_index else (("活动动作未记录" if zh else "Active action not recorded") if unknown_active_action else ("所有动作执行后" if zh else "After all actions"))) expected_network = f'

    {"要求匹配的网络条件" if zh else "Required network match"}

    {_e(json.dumps(failed["expected_network"], ensure_ascii=False, indent=2))}
    ' if failed.get("expected_network") else "" failure_panel = f'''

    {"失败定位与处理" if zh else "Failure diagnosis and treatment"}

    {"失败环节" if zh else "Failure stage"}{_e(stage)}
    {"失败位置" if zh else "Failure location"}{_e(action_locator)}
    {"复核分类" if zh else "Reviewed category"}{_e(label)}

    {"程序原始判定" if zh else "Exact programmatic finding"}

    {_e(failed["message"])}
    {expected_network}

    {"复核判断与依据" if zh else "Reviewed cause and basis"}

    {_e(reason)}

    {"处理方" if zh else "Owner"}: {_e(review.get(f"remediation_owner_{lang}", review.get("remediation_owner")) if review else "—")} · {"状态" if zh else "Status"}: {_e(review.get(f"remediation_status_{lang}", review.get("remediation_status")) if review else "—")}

    {step_link}
    ''' failure_panel += _supplemental_diagnostic_html(evidence.get("supplemental_diagnostic"), lang) @@ -999,6 +1065,7 @@ def _assets(output: Path) -> None: .mermaid svg{min-width:850px;height:auto!important}.flow-fallback{font-size:14px}.flow-node{min-width:170px} .review-table table{min-width:1120px}.review-table td:first-child{min-width:145px;font-weight:700}.review-table td:nth-child(n+2){min-width:230px;max-width:320px}.review-table td{overflow-wrap:anywhere} .successful-distributions table{min-width:1100px}.successful-distributions th:first-child,.successful-distributions td:first-child{min-width:140px;white-space:nowrap}.successful-distributions th:not(:first-child),.successful-distributions td:not(:first-child){min-width:170px} +.report-table-tools{display:flex;align-items:end;gap:10px;flex-wrap:wrap;margin:10px 0}.report-table-tools label{display:grid;gap:4px;color:var(--muted);font-size:13px;font-weight:700}.report-table-tools input{min-width:220px;padding:8px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--ink);font:inherit}.report-table-tools button{padding:8px 11px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--accent);font-weight:700}.report-table-count{color:var(--muted);font-size:13px}.table-sort{display:flex;align-items:center;gap:6px;width:100%;border:0;background:transparent;color:inherit;padding:0;text-align:left;font:inherit;font-weight:800;cursor:pointer}.table-sort::after{content:'↕';color:var(--muted);font-size:11px}.table-sort[data-direction=ascending]::after{content:'↑';color:var(--accent)}.table-sort[data-direction=descending]::after{content:'↓';color:var(--accent)} .trajectory-columns,.aligned-step{display:grid;grid-template-columns:48px minmax(0,1fr) minmax(0,1fr);gap:12px;align-items:start} .trajectory-columns{padding:0 18px 10px;color:var(--muted);font-size:13px}.aligned-step{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:15px 18px} .aligned-step.failed-step{border:2px solid var(--bad);background:color-mix(in srgb,var(--bad) 5%,var(--panel))}.aligned-standard,.aligned-actual{min-width:0;display:grid;gap:7px}.aligned-standard{border-right:1px solid var(--line);padding-right:12px}.aligned-step code{color:var(--muted);font:13px/1.45 ui-monospace,SFMono-Regular,monospace;overflow-wrap:anywhere}.aligned-step button{justify-self:start;border:0;background:transparent;color:var(--accent);font:inherit;font-weight:700;padding:4px 0;text-align:left}.aligned-actual-head{display:flex;align-items:center;gap:10px;flex-wrap:wrap}.aligned-actual-head span:last-child{margin-left:auto;color:var(--muted)}.aligned-actual p{margin:0;color:var(--muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.status-muted{color:var(--muted)} @@ -1019,6 +1086,83 @@ def _assets(output: Path) -> None: localStorage.setItem('report-theme',root.dataset.theme); }); const fetchJson=async path=>{const response=await fetch(path);if(!response.ok)throw Error(`HTTP ${response.status}`);return response.json()}; +const enhanceTables=()=>{ + const excluded='[data-results] table,[data-dataset-table] table,[data-task-matrix] table'; + document.querySelectorAll('.table-scroll>table').forEach((table,tableIndex)=>{ + const headerRows=table.tHead?.rows||[]; + const body=table.tBodies[0]; + if(headerRows.length!==1||!body||body.rows.length<2)return; + const headers=[...headerRows[0].cells]; + const rows=[...body.rows]; + if(rows.some(row=>row.cells.length!==headers.length))return; + const original=new Map(rows.map((row,index)=>[row,index])); + const external=table.matches(excluded); + let query=''; + let sortColumn=-1; + let direction='ascending'; + let count=null; + const value=(cell)=>{ + const explicit=cell.dataset.sort; + if(explicit!==undefined&&explicit!=='')return {missing:false,numeric:true,value:Number(explicit)}; + const text=cell.textContent.trim(); + if(!text||text==='—')return {missing:true,numeric:false,value:''}; + const compact=text.replaceAll(',','').replace(/\s+/g,' '); + const ratio=compact.match(/^(-?\d+(?:\.\d+)?)\s*\/\s*(-?\d+(?:\.\d+)?)/); + if(ratio)return {missing:false,numeric:true,value:Number(ratio[1])/Number(ratio[2]||1)}; + const number=compact.match(/^(-?\d+(?:\.\d+)?)\s*(?:%|ms|s|秒|MiB|MB|KiB|次|条|个|calls?)?(?:\s|$)/i); + if(number)return {missing:false,numeric:true,value:Number(number[1])}; + return {missing:false,numeric:false,value:compact.toLocaleLowerCase()}; + }; + const apply=()=>{ + const ordered=[...rows].sort((left,right)=>{ + if(sortColumn<0)return original.get(left)-original.get(right); + const a=value(left.cells[sortColumn]); + const b=value(right.cells[sortColumn]); + if(a.missing!==b.missing)return a.missing?1:-1; + const compared=a.numeric&&b.numeric?a.value-b.value:String(a.value).localeCompare(String(b.value),lang); + return direction==='ascending'?compared:-compared; + }); + let shown=0; + ordered.forEach(row=>{ + body.append(row); + const matches=!query||row.textContent.toLocaleLowerCase().includes(query); + if(!external)row.hidden=!matches; + if(matches&&!row.hidden)shown++; + }); + if(count)count.textContent=zh?`显示 ${shown} / ${rows.length} 行`:`Showing ${shown} / ${rows.length} rows`; + }; + headers.forEach((header,column)=>{ + const button=document.createElement('button'); + button.type='button';button.className='table-sort';button.textContent=header.textContent.trim(); + button.setAttribute('aria-label',(zh?'按此列排序：':'Sort by column: ')+button.textContent); + header.replaceChildren(button); + button.addEventListener('click',()=>{ + direction=sortColumn===column&&direction==='ascending'?'descending':'ascending'; + sortColumn=column; + headers.forEach((item,index)=>{ + item.setAttribute('aria-sort',index===column?direction:'none'); + item.querySelector('.table-sort')?.removeAttribute('data-direction'); + }); + button.dataset.direction=direction;apply(); + }); + header.setAttribute('aria-sort','none'); + }); + if(!external){ + const tools=document.createElement('div');tools.className='report-table-tools';tools.dataset.tableControls=String(tableIndex); + const label=document.createElement('label');label.textContent=zh?'筛选此表':'Filter this table'; + const input=document.createElement('input');input.type='search';input.placeholder=zh?'输入关键词':'Enter keywords'; + input.dataset.tableFilter=String(tableIndex);label.append(input);tools.append(label); + const reset=document.createElement('button');reset.type='button';reset.textContent=zh?'重置':'Reset';tools.append(reset); + count=document.createElement('span');count.className='report-table-count';count.setAttribute('role','status');tools.append(count); + table.closest('.table-scroll').before(tools); + table.dataset.tableControls=String(tableIndex); + input.addEventListener('input',()=>{query=input.value.trim().toLocaleLowerCase();apply()}); + reset.addEventListener('click',()=>{query='';sortColumn=-1;direction='ascending';input.value='';headers.forEach(item=>{item.setAttribute('aria-sort','none');item.querySelector('.table-sort')?.removeAttribute('data-direction')});apply();input.focus()}); + } + table.dataset.enhanced='true';apply(); + }); +}; +enhanceTables(); const datasetTable=document.querySelector('[data-dataset-table]'); if(datasetTable){ const rows=[...datasetTable.querySelectorAll('tbody tr')]; @@ -1105,7 +1249,7 @@ def _assets(output: Path) -> None: if(!result)return; const detail=result.detail_url||'result/'+result.key+'.html'; dialog.querySelector('[data-dialog-title]').textContent=result.browser+' · '+(zh?'任务':'Task')+' #'+result.task_id; - dialog.querySelector('[data-dialog-stage]').textContent=result.failure_stage==='browser_tool_execution'?(zh?'浏览器动作':'Browser action'):result.failure_stage==='side_effect_not_observed'?(zh?'网络验收条件未满足':'Network acceptance not satisfied'):result.failure_stage==='official_evaluation'?(zh?'官方验收':'Official evaluation'):(zh?'无':'None'); + dialog.querySelector('[data-dialog-stage]').textContent=result.failure_stage==='browser_session_setup'?(zh?'浏览器会话初始化':'Browser session setup'):result.failure_stage==='browser_tool_execution'?(zh?'Playwright 动作':'Playwright action'):result.failure_stage==='evidence_finalization'?(zh?'执行证据收尾':'Evidence finalization'):result.failure_stage==='official_evaluator_execution'?(zh?'官方评测器执行':'Official evaluator execution'):result.failure_stage==='side_effect_not_observed'?(zh?'网络验收条件未满足':'Network acceptance not satisfied'):result.failure_stage==='official_evaluation'?(zh?'官方验收':'Official evaluation'):(zh?'无':'None'); dialog.querySelector('[data-dialog-error]').textContent=zh?'正在读取完整证据…':'Loading complete evidence…'; delete dialog.dataset.loaded; dialog.querySelector('[data-dialog-link]').href=detail; @@ -1116,7 +1260,7 @@ def _assets(output: Path) -> None: const step=evidence.programmatic_failure?.step; const trigger=evidence.programmatic_failure?.trigger_step; const failedAction=(evidence.trajectory||[]).find(action=>action.index===step); - dialog.querySelector('[data-dialog-step]').textContent=step?(failedAction?.arguments?.action||'')+' #'+step:trigger?(zh?'触发动作':'Trigger action')+' #'+trigger:(zh?'动作完成后':'After actions completed'); + dialog.querySelector('[data-dialog-step]').textContent=step?(failedAction?.arguments?.action||'')+' #'+step:trigger?(zh?'触发动作':'Trigger action')+' #'+trigger:evidence.programmatic_failure?.stage==='browser_session_setup'?(zh?'轨迹开始前':'Before trajectory'):(zh?'动作完成后':'After actions completed'); dialog.querySelector('[data-dialog-step-link]').href=detail+(step?'#step-'+step:trigger?'#step-'+trigger:'#trajectory'); dialog.querySelector('[data-dialog-category]').textContent=(zh?evidence.review?.label_zh:evidence.review?.label_en)||(zh?'未复核':'Not reviewed'); const exact=step?(failedAction?.result||evidence.programmatic_failure?.message||''):evidence.programmatic_failure?.stage==='side_effect_not_observed'?(evidence.programmatic_failure.message+'\n\nExpected network event:\n'+JSON.stringify(evidence.programmatic_failure.expected_network,null,2)):evidence.official_evaluation?JSON.stringify(evidence.official_evaluation,null,2):(evidence.programmatic_failure?.message||''); @@ -1210,6 +1354,16 @@ def _assets(output: Path) -> None: }''' (output / "assets").mkdir(parents=True, exist_ok=True) css += ( + "\n.summary-slide{border:1px solid var(--line);background:var(--panel);border-radius:20px;padding:clamp(20px,3vw,34px);margin:28px 0;box-shadow:var(--shadow)}" + ".section-heading{display:flex;align-items:end;justify-content:space-between;gap:24px;margin-bottom:20px}.section-heading h2{margin:0 0 8px}.section-heading p{margin:0;max-width:760px}" + ".failure-signature-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px}.failure-signature-grid article{border:1px solid var(--line);border-radius:16px;padding:18px;background:var(--bg)}" + ".failure-signature-grid header{display:flex;justify-content:space-between;gap:16px;align-items:end;margin-bottom:14px}.failure-signature-grid header div span,.failure-signature-grid header div strong{display:block}.failure-signature-grid header div strong{font-size:1.55rem}.failure-signature-grid header b{text-align:right}" + ".failure-signature-grid ul{list-style:none;padding:0;margin:0;display:grid;gap:12px}.failure-signature-grid li>div:first-child{display:flex;justify-content:space-between;gap:10px}.failure-mini-track,.failure-bar-track,.outcome-track{height:9px;background:color-mix(in srgb,var(--line) 65%,transparent);border-radius:999px;overflow:hidden;margin:5px 0}.failure-mini-track i,.failure-bar-track i,.outcome-track i{display:block;height:100%;background:var(--accent);border-radius:inherit}" + ".failure-signature-grid li a{font-size:.88rem}.failure-remainder{display:flex;justify-content:space-between;color:var(--muted)}" + ".interpretation>.presentation-slide{margin:30px 0;padding:clamp(24px,4vw,50px);background:var(--panel);border:1px solid var(--line);border-radius:24px;box-shadow:var(--shadow)}.interpretation>.presentation-slide>h2{max-width:980px;font-size:clamp(1.7rem,3vw,2.7rem);margin-top:0}" + ".outcome-chart ol{list-style:none;padding:0;margin:24px 0;display:grid;gap:17px}.outcome-chart li>div:first-child{display:flex;justify-content:space-between;gap:18px}.outcome-chart li strong{font-size:1.08rem}.outcome-track{height:18px;margin-top:7px}.outcome-chart figcaption{text-align:right;color:var(--muted)}" + ".browser-failure-profile{display:grid;gap:22px;margin:26px 0}.browser-profile-overview{display:grid;grid-template-columns:1.2fr 1fr;gap:14px}.browser-profile-overview>div{border:1px solid var(--line);border-radius:16px;padding:20px;background:var(--bg)}.browser-profile-overview span,.browser-profile-overview small{display:block}.browser-profile-overview strong{display:block;font-size:clamp(2rem,5vw,4rem);line-height:1.05;margin:8px 0}" + ".failure-bars{list-style:none;padding:0;margin:0;display:grid;gap:9px}.failure-bars li{display:grid;grid-template-columns:minmax(180px,1.4fr) minmax(180px,2fr) 54px 58px;gap:12px;align-items:center}.failure-bars strong,.failure-bars small{text-align:right}.failure-explanations{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px}.failure-explanations article{border:1px solid var(--line);border-radius:15px;padding:18px;background:var(--bg)}.failure-explanations header{display:flex;justify-content:space-between;gap:14px}.failure-explanations h3{font-size:1.05rem;margin:0}.failure-explanations p{font-size:.94rem}.failure-explanations a{font-size:.9rem}" "\n.diagnostic-observations table{min-width:840px}" ".diagnostic-observations td:first-child{white-space:nowrap;min-width:100px}" ".diagnostic-observations td code{overflow-wrap:anywhere}" @@ -1217,6 +1371,8 @@ def _assets(output: Path) -> None: ".diagnostic-timeline li{border:1px solid #888;border-radius:10px;padding:12px;min-width:0}" ".diagnostic-timeline strong,.diagnostic-timeline span{display:block}" ".fixed36-diagnostic figure img{display:block;max-width:100%;height:auto;border:1px solid var(--line);border-radius:8px}" + "@media(max-width:760px){.section-heading{display:block}.section-heading>a{display:inline-block;margin-top:12px}.failure-signature-grid,.failure-explanations,.browser-profile-overview{grid-template-columns:1fr}.failure-bars li{grid-template-columns:1fr 52px}.failure-bars li .failure-bar-track{grid-column:1/-1;grid-row:2}.failure-bars li small{grid-column:2}.interpretation>.presentation-slide{padding:22px 18px}}" + "@media print{@page{size:13.333in 7.5in;margin:.42in}.interpretation>.hero,.interpretation>.presentation-slide{break-after:page;break-inside:avoid;box-shadow:none;border-color:#bbb}.failure-explanations article{break-inside:avoid}}" ) (output / "assets/report.css").write_text(css, encoding="utf-8") (output / "assets/app.js").write_text(js, encoding="utf-8") @@ -1248,18 +1404,24 @@ def build_webarena_replay_report( methods = participant_registry(profile["methods"]) if len(methods) < 2 or profile.get("affected_task_methods"): raise ValueError("current report requires complete registered methods") + execution_backend = profile.get("execution_backend") + if execution_backend is not None and ( + execution_backend != "playwright" + or any(method.get("backend") != execution_backend for method in profile["methods"]) + ): + raise ValueError("replay report requires one Playwright backend for every method") execution = profile.get("execution") or {} if ( execution.get("method_parallelism") != 1 or execution.get("task_parallelism") != 1 - or type(execution.get("reset_before_every_task")) is not bool + or execution.get("reset_before_every_task") is not False or execution.get("reset_before_every_method") is not True or execution.get("teardown_between_methods") is not True or execution.get("retry_failed_task") is not False ): - raise ValueError("three-method report requires the declared common execution protocol") + raise ValueError("replay report requires the declared common execution protocol") if diagnostic_closure_path is not None: - raise ValueError("diagnostic-only cohorts cannot enter the three-method report") + raise ValueError("diagnostic-only cohorts cannot enter the replay report") validated = load_validated_failure_review(results_path, review_path, methods=profile["methods"]) results = validated["results"] reviews = validated["by_result"] @@ -1477,6 +1639,8 @@ def build_webarena_replay_report( if reference_method is None or reference_row is None: raise ValueError("profile qualification reference participant must be registered") reference_label = _e(reference_method["label"]) + method_count = len(methods) + browser_count = len({method["engine"] for method in methods.values()}) reviewed_failure_count = len(reviews) unverifiable_ids = sorted(task_id for task_id, status in task_validity.items() if status == "unverifiable") valid_reference_count = len(tasks) - len(unverifiable_ids) @@ -1526,18 +1690,21 @@ def build_webarena_replay_report( if zh else f"WebArena-Verified · {task_count} tasks · model calls 0" ) - stage_rows = [] - for engine, method in methods.items(): - for code, label_pair in (("browser_tool_execution", ("浏览器动作", "Browser action")), ("side_effect_not_observed", ("网络验收条件未满足", "Network acceptance not satisfied")), ("official_evaluation", ("官方验收", "Official evaluation"))): - count = sum(row["participant_id"] == engine and row["status"] != "pass" and row["failure_stage"] == code for row in public_rows) - if count: - stage_rows.append(f'{_e(method["label"])}{label_pair[0 if zh else 1]}{count}{"下钻到失败结果" if zh else "Inspect failures"} →') - stage_table = f'
    {"".join(stage_rows)}
    {"方法" if zh else "Method"}{"失败环节" if zh else "Failure stage"}{"结果数" if zh else "Outcomes"}{"逐题证据" if zh else "Per-task evidence"}
    ' - stage_section = ( - f'

    {"失败发生在哪个环节" if zh else "Where failures occurred"}

    ' - f'

    {"动作报错与网络验收失败分别统计。后者表示工具调用返回成功，但网络记录中没有满足全部验收条件的请求：可能是请求未发生，也可能是 URL、参数或请求头不匹配。逐题详情给出相关动作、验收规则和复核原因。" if zh else "Action errors and network-acceptance failures are counted separately. In the latter case the tool call returned success, but no recorded request satisfied all criteria: the request may be absent, or its URL, parameters or headers may not match. Result details show related actions, evaluator rules, and reviewed causes."}

    ' - + stage_table + '
    ' - ) if stage_rows else '' + failure_participants = tuple( + engine + for engine in methods + if any( + row["participant_id"] == engine and row["status"] != "pass" + for row in public_rows + ) + ) + failure_signatures = ( + failure_signature_cards( + methods, public_rows, lang, participants=failure_participants + ) + if failure_participants + else "" + ) validity_notice = _historical_replay_validity_notice( **validity_identity, lang=lang ) @@ -1556,10 +1723,23 @@ def build_webarena_replay_report( if zh else f"All methods cover the same {task_count} tasks. Task details identify acceptance rules and declared corrections; the review page states the independent review scope. " ) + has_resource_samples = any( + row[metric]["n"] + for row in method_rows + for metric in ("cpu", "memory", "memory_peak", "network_rx", "network_tx", "cpu_utilization") + ) core_metrics_note += ( - "提前失败会缩短工作量，因此总耗时和资源不按低者判优；单次动作耗时按实际工具调用统计。" - if zh else - "Early failures reduce work, so total time and resources are not ranked by lower values; per-action latency uses actual tool calls." + ( + "提前失败会缩短工作量，因此总耗时和资源不按低者判优；单次动作耗时按实际 Playwright 调用统计。" + if has_resource_samples + else "提前失败会缩短工作量，因此总耗时不按低者判优；单次动作耗时按实际 Playwright 调用统计。" + ) + if zh + else ( + "Early failures reduce work, so total time and resources are not ranked by lower values; per-action latency uses actual Playwright calls." + if has_resource_samples + else "Early failures reduce work, so total time is not ranked by lower values; per-action latency uses actual Playwright calls." + ) ) core_metrics_table = _metrics_table( method_rows, lang, show_valid_reference=bool(unverifiable_ids) @@ -1567,14 +1747,18 @@ def build_webarena_replay_report( measurement_note = (profile.get("measurement_notes") or {}).get(lang) if measurement_note: core_metrics_note += " " + _e(measurement_note) - summary_body = f'''

    {eyebrow}

    {"浏览器轨迹重放评测" if zh else "Browser Trajectory Replay"}

    {f"所有浏览器配置分别完成同一 {task_count} 题的冻结轨迹重放。CowAgent 只提供共同的 BrowserTool 操作层，Agent 不规划动作，模型不参与；因此结果比较的是 BrowserTool 与浏览器的组合，不能单独归因于浏览器内核。方法版本、轨迹与站点准备见环境说明。" if zh else f"All browser configurations replay frozen paths for the same {task_count} tasks. CowAgent supplies only the shared BrowserTool action layer: no Agent plans actions and no model is called. Results compare the BrowserTool–browser combinations, not browser engines in isolation. Environment documents versions, paths and site preparation."}

    {"逐题查看结果与报错" if zh else "Explore outcomes and exact errors"}{"查看任务与标准轨迹" if zh else "Task contracts and fixed trajectories"}
    + resource_table = _resource_table(method_rows, lang) + resource_section = ( + f'

    {"资源与网络" if zh else "Resources and network"}

    {f"下表覆盖全部 {task_count} 题；n 表示有观测值的样本数，缺失不计为零。真实零值保留。网络传输量和 CPU 使用率不直接判优，低数值也可能来自提前失败、缓存或等待。" if zh else f"These per-task samples cover all {task_count} tasks. Each n counts observed values; missing samples are not zero, and measured zeroes are retained. Network bytes and CPU utilization are not ranked: lower values may reflect early failure, caching, or waiting."}

    {"展开资源与网络统计" if zh else "Open resource and network statistics"}{resource_table}
    ' + if resource_table + else f'

    {"资源数据" if zh else "Resource data"}

    {"本轮没有记录进程级 CPU、内存和网络采样，因此这些指标不可用，报告不展示空值表或资源排名。端到端耗时和动作耗时来自现有轨迹记录，无需重新执行任务。" if zh else "This run did not record process-level CPU, memory, or network samples. Those metrics are unavailable, so the report does not show empty tables or resource rankings. End-to-end and action latency come from the retained trajectories and require no rerun."}

    ' + ) + summary_body = f'''

    {eyebrow}

    {"浏览器轨迹重放评测" if zh else "Browser Trajectory Replay"}

    {f"{browser_count} 款浏览器组成 {method_count} 种方法配置，分别覆盖同一 {task_count} 题的冻结轨迹重放。六种方法均由同一 Playwright 重放器通过 CDP 执行 BrowserTool 动作契约；本轮没有调用 CowAgent BrowserTool，也没有 Agent 或模型参与。结果比较的是 Playwright 重放器与浏览器的组合配置，不能单独归因于浏览器内核。方法版本、轨迹与站点准备见环境说明。" if zh else f"{browser_count} browsers form {method_count} method configurations, each covering the same {task_count} frozen trajectory replays. All six methods use the same Playwright replay adapter over CDP to execute the BrowserTool action contract; this run does not invoke CowAgent BrowserTool, an Agent, or a model. Results compare Playwright-adapter–browser configurations, not browser engines in isolation. Environment documents versions, paths and site preparation."}

    {"逐题查看结果与报错" if zh else "Explore outcomes and exact errors"}{"查看任务与标准轨迹" if zh else "Task contracts and fixed trajectories"}
    {validity_notice} {f'

    {"查看 74 条历史取数验收输入的逐条哈希" if zh else "Inspect hashes for all 74 historical retrieval inputs"}

    ' if legacy_provenance is not None else ''}

    {"核心指标" if zh else "Core metrics"}

    {core_metrics_note}

    {core_metrics_table}
    -

    {"资源与网络" if zh else "Resources and network"}

    {f"下表覆盖全部 {task_count} 题；n 表示有观测值的样本数，缺失不计为零。真实零值保留。网络传输量和 CPU 使用率不直接判优，低数值也可能来自提前失败、缓存或等待。" if zh else f"These per-task samples cover all {task_count} tasks. Each n counts observed values; missing samples are not zero, and measured zeroes are retained. Network bytes and CPU utilization are not ranked: lower values may reflect early failure, caching, or waiting."}

    {"展开资源与网络统计" if zh else "Open resource and network statistics"}{_resource_table(method_rows, lang)}
    - {stage_section} -

    {"浏览器动作耗时" if zh else "Browser action latency"}

    {"均值、P50、P95 按单次调用统计；调用次数只说明工作量。下钻到结果详情可查看具体动作耗时和工具返回。" if zh else "Mean, P50, and P95 are per-call statistics; call counts only describe workload. Open a task result for the individual action timing and full tool response."}

    {"展开各类动作耗时" if zh else "Open latency by action type"}{action_table}
    ''' - summary_body += f'

    {"各方法成功任务的条件分布" if zh else "Conditional distributions on each method’s successful tasks"}

    {"各列仅含该方法成功的任务，任务集合与样本数可能不同，不用于排名，也不替代上方共同任务全集的统计。" if zh else "Each column includes only that method’s successful tasks. Task sets and sample counts may differ; these distributions are not ranked and do not replace the all-task statistics above."}

    {_successful_distribution_table(method_rows, lang)}
    ' + {resource_section} + {failure_signatures}''' (root / "index.html").write_text(_shell(lang, "index", "浏览器轨迹重放" if zh else "Browser Trajectory Replay", summary_body, current_route="index.html"), encoding="utf-8") headers = ["任务" if zh else "Task", "站点" if zh else "Site", "任务目标" if zh else "Task goal", "类型" if zh else "Type", "轨迹来源" if zh else "Trajectory source", "标准动作" if zh else "Fixed actions", "下钻" if zh else "Drill down"] @@ -1607,12 +1791,12 @@ def build_webarena_replay_report( for row in public_rows: cause = row["cause_zh"] if zh else row["cause_en"] status = _verdict_label(row["status"], row.get("semantic_status"), lang) - stage_label = {"browser_tool_execution": "浏览器动作" if zh else "Browser action", "side_effect_not_observed": "网络验收条件未满足" if zh else "Network acceptance not satisfied", "official_evaluation": "官方验收" if zh else "Official evaluation", "pass": "—"}.get(row["failure_stage"], row["failure_stage"]) + stage_label = {"browser_session_setup": "浏览器会话初始化" if zh else "Browser session setup", "browser_tool_execution": "Playwright 动作" if zh else "Playwright action", "evidence_finalization": "执行证据收尾" if zh else "Evidence finalization", "official_evaluator_execution": "官方评测器执行" if zh else "Official evaluator execution", "side_effect_not_observed": "网络验收条件未满足" if zh else "Network acceptance not satisfied", "official_evaluation": "官方验收" if zh else "Official evaluation", "pass": "—"}.get(row["failure_stage"], row["failure_stage"]) detail_url = row.get("detail_url") or f'result/{row["key"]}.html' result_rows.append(f'#{row["task_id"]}{_e(row["browser"])}{_e(row["sites"][0] if row["sites"] else "")}{status}{row["completed_actions"]}/{row["action_count"]}{_e(stage_label)}{_e(cause) if cause else "—"}{"证据页" if zh else "Evidence page"}') browser_options = "".join(f'' for key, item in comparison_methods.items()) site_options = "".join(f'' for site in sorted({s for row in public_rows for s in row["sites"]})) - stage_labels = {"browser_tool_execution": ("浏览器动作", "Browser action"), "side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"), "official_evaluation": ("官方验收", "Official evaluation")} + stage_labels = {"browser_session_setup": ("浏览器会话初始化", "Browser session setup"), "browser_tool_execution": ("Playwright 动作", "Playwright action"), "evidence_finalization": ("执行证据收尾", "Evidence finalization"), "official_evaluator_execution": ("官方评测器执行", "Official evaluator execution"), "side_effect_not_observed": ("网络验收条件未满足", "Network acceptance not satisfied"), "official_evaluation": ("官方验收", "Official evaluation")} present_stages = {row["failure_stage"] for row in public_rows if row["status"] == "fail"} stage_options = "".join(f'' for code in stage_labels if code in present_stages) causes = {row["cause_zh"]: row["cause_zh"] if zh else row["cause_en"] for row in public_rows if row["cause_zh"]} @@ -1623,7 +1807,7 @@ def build_webarena_replay_report( fail_links = [] passed_engines = [] failed_engines = [] - for engine, method in comparison_methods.items(): + for engine, _method in comparison_methods.items(): row = next((item for item in rows_by_task[task_id] if item["participant_id"] == engine), None) if row is None: raise ValueError(f"same-task method result is missing: {engine}/{task_id}") @@ -1643,9 +1827,19 @@ def build_webarena_replay_report( matrix_checks = "".join(f'' for engine, item in comparison_methods.items()) result_matrix = f'

    {"同题方法对比" if zh else "Same-task method comparison"}

    {"按官方通过和失败分组展示所有方法；点开任一结果查看该方法的轨迹与执行证据。" if zh else "All methods are grouped by official pass and fail. Open any outcome for its method-specific path and execution evidence."}

    {"参与比较的方法（至少选一个）" if zh else "Comparison methods (select at least one)"}
    {matrix_checks}
    {"".join(result_matrix_rows)}
    {"任务" if zh else "Task"}{"站点" if zh else "Site"}{"官方通过" if zh else "Official pass"}{"失败" if zh else "Fail"}
    ' results_body = f'''

    {len(public_rows)} {"条逻辑结果" if zh else "logical outcomes"}

    {"结果与失败诊断" if zh else "Results and failure diagnosis"}

    {"同题对比支持选择基准和多个方法；下方筛选表默认只显示失败，点击“诊断”查看原始判定、复核原因与执行证据。" if zh else "Choose a baseline and multiple methods for same-task comparison. The inventory starts with failures; open Diagnose for the exact finding, reviewed cause, and execution evidence."}

    {result_matrix}

    {''.join(result_rows)}
    {"任务" if zh else "Task"}{"方法" if zh else "Method"}{"站点" if zh else "Site"}{"判定" if zh else "Verdict"}{"动作" if zh else "Actions"}{"失败环节" if zh else "Failure stage"}{"复核分类" if zh else "Reviewed category"}{"详情" if zh else "Detail"}
    {"失败环节" if zh else "Failure stage"}
    {"动作定位" if zh else "Action locator"}
    {"复核分类" if zh else "Reviewed category"}

    {"程序判定与验收规则" if zh else "Programmatic finding and evaluator rule"}

    {"复核原因与处理" if zh else "Reviewed cause and treatment"}
    {"打开对应动作" if zh else "Open the relevant action"}{"打开完整详情" if zh else "Open full detail"}
    '''
+        diagnostic_details = (
+            failure_category_matrix(comparison_methods, public_rows, lang)
+            + f'
    {"Playwright 动作耗时" if zh else "Playwright action latency"}
    '
+            + f'
    {"均值、P50、P95 按单次 Playwright 调用统计；调用次数只说明工作量。逐题详情保留每个动作的耗时和返回内容。" if zh else "Mean, P50, and P95 are per Playwright call; call counts describe workload. Result details retain every action duration and response."}
    '
+            + f'
    {"展开各类动作耗时" if zh else "Open latency by action type"}{action_table}
    '
+            + f'
    {"各方法成功任务的条件分布" if zh else "Conditional distributions on each method’s successful tasks"}
    '
+            + f'
    {"各列仅含该方法成功的任务，任务集合和样本数可能不同。这组数据用于诊断，不参与方法排名。" if zh else "Each column contains only that method’s successful tasks. Task sets and sample counts may differ, so these distributions are diagnostic and not ranked."}
    '
+            + _successful_distribution_table(method_rows, lang)
+            + "
    "
+        )
         results_body = results_body.replace(
             '
    ',
-            failure_category_matrix(comparison_methods, public_rows, lang) + '
    ',
+            diagnostic_details + '
    ',
             1,
         ).replace('
    ', '
    ', 1)
         results_body = results_body.replace("结果与失败诊断", "结果分析").replace("Results and failure diagnosis", "Results analysis")
@@ -1670,7 +1864,7 @@ def build_webarena_replay_report(
   V --> X[{"绑定证据的结果" if zh else "Evidence-bound result"}]
   X --> T[{"销毁浏览器与站点" if zh else "Teardown browser and sites"}]
   T --> R'''
-        environment_body = f'''
    {"环境与评测流程" if zh else "Environment and evaluation flow"}
    {"192.168.5.2 是 Colima 虚拟机中的宿主网关：浏览器容器通过它访问运行在同一台 Mac 虚拟化环境里的 WebArena 站点。它不是公网测试网站。" if zh else "192.168.5.2 is the host gateway inside the Colima VM. Browser containers use it to reach the WebArena sites running in the same Mac virtualization environment; it is not a public test website."}
    {"执行拓扑" if zh else "Execution topology"}
    {fallback}{topology}
    {"隔离规则" if zh else "Isolation rules"}
    1
    {"一套固定端口站点环境，因此只允许一个方法运行。" if zh else "One fixed-port site stack permits only one running method."}
    {task_count}
    {"每方法完成全部任务，方法开始前恢复站点；任务间不重置，失败题不单独重试。" if zh else "Each method completes all tasks after one site reset; there is no per-task reset or isolated retry of failed tasks."}
    0
    {"没有 Agent 决策和模型请求；输入是冻结的 BrowserTool 动作。" if zh else "No Agent decisions or model calls; inputs are frozen BrowserTool actions."}
    '''
+        environment_body = f'''
    {"环境与评测流程" if zh else "Environment and evaluation flow"}
    {"192.168.5.2 是 Colima 虚拟机中的宿主网关：浏览器容器通过它访问运行在同一台 Mac 虚拟化环境里的 WebArena 站点。它不是公网测试网站。" if zh else "192.168.5.2 is the host gateway inside the Colima VM. Browser containers use it to reach the WebArena sites running in the same Mac virtualization environment; it is not a public test website."}
    {"执行拓扑" if zh else "Execution topology"}
    {fallback}{topology}
    {"隔离规则" if zh else "Isolation rules"}
    1
    {"一套固定端口站点环境，因此只允许一个方法运行。" if zh else "One fixed-port site stack permits only one running method."}
    {task_count}
    {"每种方法覆盖全部任务，方法开始前恢复站点；任务之间不重置。每题只保留一个最终逻辑结果，额外物理执行单独计数，输入未变且已经通过的任务不再执行。" if zh else "Each method covers every task after one site reset, with no reset between tasks. One final logical outcome is retained per task; extra physical calls are counted separately, and an unchanged task is not run again after passing."}
    0
    {"没有 Agent 决策和模型请求；Playwright 通过 CDP 重放冻结的 BrowserTool 动作契约。" if zh else "No Agent decisions or model calls; Playwright replays the frozen BrowserTool action contract over CDP."}
    '''
         environment_body = environment_body.replace("环境与评测流程", "环境与方法").replace("Environment and evaluation flow", "Environment and method")
         if "Rainbond" in str((profile.get("execution") or {}).get("site_stack") or ""):
             diagram_script = environment_body[environment_body.index('"""
+    (output / "index.html").write_text(
+        _catalog_shell("全量轨迹重放结果", index_body, participant), encoding="utf-8"
+    )
+
+    runtime_method = (
+        "托管浏览器的服务、账户匿名哈希及 Browser.getVersion 版本字段"
+        if hosted_runtime
+        else "本地浏览器二进制哈希"
+    )
+    revision_method = (
+        "托管浏览器运行时在开始前冻结版本身份，每个物理结果都必须与该身份一致。"
+        if hosted_runtime
+        else "轨迹修复期间使用多个二进制版本，但每个版本均在接受账本中单独绑定。"
+    )
+    bridge_method = (
+        f"Kitesurf 远端浏览器无法直接访问私网 Rainbond 地址；本轮用 Playwright request bridge 从执行机访问账本绑定的 {len(request_bridge['origins'])} 个站点 origin，并以原 URL、请求方法、请求头和请求体回填响应。桥接身份随每个执行段和公开证据保存。"
+        if request_bridge is not None
+        else ""
+    )
+    method_body = f"""
    方法与边界
    执行方法
    评测集按官方题号 {task_ids[0]}–{task_ids[-1]} 排列。每题执行冻结轨迹，由 {html.escape(engine.capitalize())} 浏览器引擎通过 {html.escape(backend.capitalize())} 协议后端完成动作。接受标准是 WebArena-Verified 官方 evaluator 返回 score 1.0。
    初始扫描中断后，后续执行只选择接受账本里尚未通过的题。最终接受账本绑定每个物理调用、{runtime_method}、站点映射、轨迹、实际响应、网络证据和 evaluator 结果。
    {f'
    {html.escape(bridge_method)}
    ' if bridge_method else ''}
    代码与评测边界
    确认属于评测集或 evaluator 的通用修复维护在 lexmount/webarena-verified 的 lex-main 分支；轨迹、执行器、站点绑定、证据聚合和报告维护在 browser-eval。评测答案和通过条件没有为浏览器适配而放宽。
    旧 evaluator 版本下的通过证据通过固定实际响应和 HAR 离线重评，浏览器动作没有重跑。{revision_method}
    结果范围
    该结果证明当前 {html.escape(participant)} 配置下 {task_count} 题各自至少成功一次。812-case 全量 split 与既有 90-case 抽样 split 数据独立；两者共享轨迹执行和 evaluator 基础模块，并分别绑定自己的证据与发布目标。
    """
+    (output / "method.html").write_text(
+        _catalog_shell("方法与边界", method_body, participant), encoding="utf-8"
+    )
+
+    _catalog_write_json(
+        output / "dataset-contracts.json",
+        {"schema": LEGACY_CATALOG_CONTRACTS_SCHEMA, "tasks": contracts},
+    )
+    dataset_rows = "".join(
+        f'{task["task_id"]}'
+        f'{html.escape("、".join(CATALOG_SITE_LABELS[site] for site in task["sites"]))}'
+        f'{html.escape(task["intent"])}'
+        for task in contracts
+    )
+    dataset_body = f"""
    数据集与轨迹合同
    {task_count} 题按 WebArena-Verified 官方顺序冻结。此页只展示任务定义；判分和执行证据在逐题结果页。
    {dataset_rows}
    题号网站任务
    下载完整任务、evaluator 与固定动作合同
    """
+    (output / "dataset.html").write_text(
+        _catalog_shell("数据集与轨迹合同", dataset_body, participant), encoding="utf-8"
+    )
+    (output / "contract.html").write_text(
+        _catalog_shell("评测合同", dataset_body, participant), encoding="utf-8"
+    )
+    viewer = """
    证据查看器
    返回结果
    加载中…
    """
+    (output / "evidence.html").write_text(
+        _catalog_shell("证据查看器", viewer, participant), encoding="utf-8"
+    )
+    notice = '
    English edition not generated
    This release is Chinese only.
    返回中文版
    '
+    (output / "en").mkdir()
+    (output / "en" / "index.html").write_text(
+        _catalog_shell("English edition not generated", notice, participant),
+        encoding="utf-8",
+    )
+
+    manifest = {
+        "schema": LEGACY_CATALOG_REPORT_SCHEMA,
+        "generated_languages": ["zh"],
+        "participant": {
+            "engine": engine,
+            "backend": backend,
+            **(
+                {"request_bridge": request_bridge}
+                if request_bridge is not None
+                else {}
+            ),
+        },
+        "ledger_sha256": _catalog_file_sha256(ledger_path),
+        "scope_sha256": _catalog_file_sha256(scope_path),
+        "catalog_sha256": _catalog_file_sha256(catalog_path),
+        "portable_suite_sha256": _catalog_file_sha256(suite_path),
+        "task_count": len(official),
+        "pass": counts["pass"],
+        "initial_physical_attempts": initial_physical,
+        "initial_pass": first_pass,
+        "initial_setup_failures": setup_failure,
+        "physical_attempts": counts["physical_attempts"],
+        "passed_case_reruns": counts["excluded_duplicate_attempts"],
+        "binary_count": binary_count,
+        "site_counts": site_counts,
+        "route_counts": {"case": len(official), "evidence_json": len(official)},
+    }
+    _catalog_write_json(output / "manifest.json", manifest)
+    with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as bundle:
+        for path in sorted(output.rglob("*")):
+            if path.is_file():
+                bundle.write(path, path.relative_to(output))
+    validate_catalog_acceptance_report(output, archive)
+    return {
+        "output": str(output),
+        "archive": str(archive),
+        "manifest_sha256": _catalog_file_sha256(output / "manifest.json"),
+        "archive_sha256": _catalog_file_sha256(archive),
+        "archive_bytes": archive.stat().st_size,
+    }
+
+
+def validate_catalog_acceptance_report(
+    output: Path, archive: Path, *, profile: dict[str, Any] | None = None
+) -> dict:
+    """Validate the public boundary of a catalog report package."""
+
+    manifest = _catalog_read_json(output / "manifest.json")
+    task_count = int(manifest.get("task_count") or 0)
+    if (
+        manifest.get("schema") != LEGACY_CATALOG_REPORT_SCHEMA
+        or manifest.get("generated_languages") != ["zh"]
+        or manifest.get("pass") != task_count
+        or task_count <= 0
+        or len(list((output / "case").glob("*.html"))) != task_count
+        or len(list((output / "evidence").glob("*.json"))) != task_count
+    ):
+        raise ValueError("catalog report manifest or route inventory is invalid")
+    expected_participant = (
+        {
+            "engine": profile.get("browser"),
+            "backend": profile.get("execution_backend"),
+            **(
+                {"request_bridge": profile["request_bridge"]}
+                if profile.get("request_bridge") is not None
+                else {}
+            ),
+        }
+        if profile is not None
+        else None
+    )
+    if profile is not None and (
+        manifest.get("participant") != expected_participant
+        or manifest.get("ledger_sha256") != profile.get("acceptance_ledger_sha256")
+        or manifest.get("catalog_sha256")
+        != (profile.get("accepted_catalog") or {}).get("sha256")
+        or manifest.get("portable_suite_sha256")
+        != (profile.get("accepted_portable_suite") or {}).get("sha256")
+        or manifest.get("task_count") != (profile.get("accepted_counts") or {}).get("total")
+        or manifest.get("physical_attempts")
+        != (profile.get("accepted_counts") or {}).get("physical_attempts")
+        or manifest.get("passed_case_reruns")
+        != (profile.get("accepted_counts") or {}).get("excluded_duplicate_attempts")
+    ):
+        raise ValueError("catalog report presentation changed its release binding")
+    missing = []
+    for path in output.rglob("*"):
+        if not path.is_file() or path.suffix not in {".html", ".json", ".css", ".js"}:
+            continue
+        text = path.read_text(encoding="utf-8")
+        if any(marker in text for marker in ("/Users/", "file://", "/private/tmp")):
+            raise ValueError(f"catalog report leaks a private path: {path}")
+        if path.suffix != ".html":
+            continue
+
+        for value in re.findall(
+            r'<(?:a|script|img|link)\b[^>]*\b(?:href|src)="([^"]+)"', text
+        ):
+            if value.startswith(("http:", "https:", "#", "data:", "javascript:")):
+                continue
+            relative = value.split("?", 1)[0].split("#", 1)[0]
+            if relative.startswith("/") or ".." in Path(relative).parts:
+                raise ValueError(f"catalog report contains an unsafe asset path: {value}")
+            if relative and not (output / relative).resolve().is_file():
+                missing.append((str(path.relative_to(output)), value))
+    if missing:
+        raise ValueError(f"catalog report has missing local links: {missing[:5]}")
+    index = (output / "index.html").read_text(encoding="utf-8")
+    method = (output / "method.html").read_text(encoding="utf-8")
+    if (
+        f'{task_count} / {task_count}' not in index
+        or f'{manifest["physical_attempts"]:,} 次物理执行' not in index
+        or "通过后不再执行" not in index
+        or "lexmount/webarena-verified" not in method
+        or "评测答案和通过条件没有为浏览器适配而放宽" not in method
+    ):
+        raise ValueError("catalog report presentation contract is incomplete")
+    with zipfile.ZipFile(archive) as bundle:
+        corrupt = bundle.testzip()
+        if corrupt is not None:
+            raise ValueError(f"catalog report archive is corrupt: {corrupt}")
+    return manifest
diff --git a/harness/trajectory_replay/webarena_review.py b/harness/trajectory_replay/webarena_review.py
index e9c3f3d26..184889537 100644
--- a/harness/trajectory_replay/webarena_review.py
+++ b/harness/trajectory_replay/webarena_review.py
@@ -13,6 +13,19 @@
 REVIEW_SCHEMA = "lex_webarena_trajectory_replay_failure_review/1"
 
 
+def _concise_process_error(message: str) -> str:
+    """Keep the final exception and tool call log, not mutable source excerpts."""
+
+    starts = [
+        match.start()
+        for match in re.finditer(
+            r"(?m)^(?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*(?:Error|Exception): .+$",
+            message,
+        )
+    ]
+    return message[starts[-1] :].strip() if starts else message.strip()
+
+
 def _sha256(path: Path) -> str:
     return hashlib.sha256(path.read_bytes()).hexdigest()
 
@@ -23,10 +36,24 @@ def programmatic_failure(attempt: dict[str, Any]) -> dict[str, Any]:
     if attempt.get("process_status") != "pass":
         message = str(attempt.get("error") or "BrowserTool execution failed")
         match = re.search(r"RuntimeError: step (\d+) failed: ([^\n]+)", message)
+        stage = str(attempt.get("failure_stage") or "browser_tool_execution")
+        if stage not in {
+            "browser_session_setup",
+            "browser_tool_execution",
+            "evidence_finalization",
+            "official_evaluator_execution",
+        }:
+            raise ValueError(f"unsupported process failure stage: {stage}")
         return {
-            "stage": "browser_tool_execution",
-            "step": int(match.group(1)) if match else None,
-            "message": match.group(2) if match else message.strip(),
+            "stage": stage,
+            "step": (
+                int(attempt["failure_step"])
+                if attempt.get("failure_step") is not None
+                else int(match.group(1))
+                if match
+                else None
+            ),
+            "message": match.group(2) if match else _concise_process_error(message),
         }
     if attempt.get("evaluator_status") != "pass":
         return {
diff --git a/harness/trajectory_replay/webarena_runtime_source.py b/harness/trajectory_replay/webarena_runtime_source.py
new file mode 100644
index 000000000..0deaedfbf
--- /dev/null
+++ b/harness/trajectory_replay/webarena_runtime_source.py
@@ -0,0 +1,119 @@
+"""Fail closed unless WebArena-Verified comes from the pinned Lexmount fork."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import importlib.util
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlsplit
+
+WEBARENA_VERIFIED_DISTRIBUTION = "webarena-verified"
+WEBARENA_VERIFIED_MODULE = "webarena_verified"
+WEBARENA_VERIFIED_FORK_URL = "https://github.com/lexmount/webarena-verified.git"
+WEBARENA_VERIFIED_REVISION = "da7b992eb10de78000f11a7f34fd4590c171fbd6"
+
+
+@dataclass(frozen=True)
+class WebArenaRuntimeSourceIdentity:
+    repository_url: str
+    revision: str
+    module_root: Path
+    distribution_root: Path
+
+
+def _canonical_repository_url(value: object) -> str:
+    if not isinstance(value, str):
+        raise RuntimeError("WebArena direct_url repository URL is missing")
+    parsed = urlsplit(value)
+    if (
+        parsed.scheme != "https"
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.port is not None
+        or parsed.query
+        or parsed.fragment
+    ):
+        raise RuntimeError("WebArena direct_url repository URL is invalid")
+    path = parsed.path.rstrip("/")
+    if path.endswith(".git"):
+        path = path[:-4]
+    if not parsed.hostname or not path:
+        raise RuntimeError("WebArena direct_url repository URL is invalid")
+    return f"https://{parsed.hostname.lower()}{path.lower()}"
+
+
+def verify_webarena_runtime_source(
+    *,
+    expected_repository: str = WEBARENA_VERIFIED_FORK_URL,
+    expected_revision: str = WEBARENA_VERIFIED_REVISION,
+) -> WebArenaRuntimeSourceIdentity:
+    """Verify VCS provenance and the import path before WebArena work begins."""
+
+    try:
+        distribution = importlib.metadata.distribution(WEBARENA_VERIFIED_DISTRIBUTION)
+    except importlib.metadata.PackageNotFoundError as exc:
+        raise RuntimeError("the pinned WebArena-Verified distribution is not installed") from exc
+
+    raw_direct_url = distribution.read_text("direct_url.json")
+    try:
+        direct_url = json.loads(raw_direct_url) if raw_direct_url is not None else None
+    except (TypeError, json.JSONDecodeError) as exc:
+        raise RuntimeError("WebArena direct_url metadata is malformed") from exc
+    if not isinstance(direct_url, dict):
+        raise RuntimeError("WebArena direct_url metadata is missing")
+    vcs = direct_url.get("vcs_info")
+    if not isinstance(vcs, dict) or vcs.get("vcs") != "git":
+        raise RuntimeError("WebArena direct_url metadata is not a Git checkout")
+    revision = vcs.get("commit_id")
+    requested_revision = vcs.get("requested_revision")
+    if (
+        not isinstance(revision, str)
+        or not isinstance(requested_revision, str)
+        or revision != expected_revision
+        or requested_revision != expected_revision
+    ):
+        raise RuntimeError("WebArena runtime revision differs from the pinned fork revision")
+    repository_url = direct_url.get("url")
+    if _canonical_repository_url(repository_url) != _canonical_repository_url(expected_repository):
+        raise RuntimeError("WebArena runtime repository is not the Lexmount fork")
+
+    files = distribution.files
+    package_init = next(
+        (
+            entry
+            for entry in files or ()
+            if str(entry).replace("\\", "/") == f"{WEBARENA_VERIFIED_MODULE}/__init__.py"
+        ),
+        None,
+    )
+    if package_init is None:
+        raise RuntimeError("WebArena distribution does not own its import package")
+    distribution_root = Path(distribution.locate_file("")).resolve()
+    expected_init = Path(distribution.locate_file(package_init)).resolve()
+    try:
+        expected_init.relative_to(distribution_root)
+    except ValueError as exc:
+        raise RuntimeError("WebArena distribution package path escapes its installation") from exc
+    if not expected_init.is_file():
+        raise RuntimeError("WebArena distribution package entry point is missing")
+    expected_module_root = expected_init.parent
+    try:
+        spec = importlib.util.find_spec(WEBARENA_VERIFIED_MODULE)
+    except (AttributeError, ImportError, ValueError) as exc:
+        raise RuntimeError("WebArena import metadata is malformed") from exc
+    if spec is None or spec.origin is None:
+        raise RuntimeError("WebArena import package is missing")
+    module_origin = Path(spec.origin).resolve()
+    search_locations = tuple(
+        Path(value).resolve() for value in (spec.submodule_search_locations or ())
+    )
+    if module_origin != expected_init or search_locations != (expected_module_root,):
+        raise RuntimeError("WebArena import path does not belong to the pinned distribution")
+    return WebArenaRuntimeSourceIdentity(
+        repository_url=str(repository_url),
+        revision=revision,
+        module_root=expected_module_root,
+        distribution_root=distribution_root,
+    )
diff --git a/pyproject.toml b/pyproject.toml
index a0df2f2ca..14aa21abc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,7 +11,7 @@ dependencies = [
 browser-harness-eval = "harness.web_fetch.cli:main"
 
 [tool.uv.sources]
-webarena-verified = { git = "https://github.com/lexmount/webarena-verified.git", rev = "7d060ec92b611d66b48d667acb9d92c35cfbdbda" }
+webarena-verified = { git = "https://github.com/lexmount/webarena-verified.git", rev = "da7b992eb10de78000f11a7f34fd4590c171fbd6" }
 
 [project.optional-dependencies]
 headless-eval = [
diff --git a/research/trajectory-replay-magento-configurable-create-options-reference.json b/research/trajectory-replay-magento-configurable-create-options-reference.json
new file mode 100644
index 000000000..0666e5cfd
--- /dev/null
+++ b/research/trajectory-replay-magento-configurable-create-options-reference.json
@@ -0,0 +1,41 @@
+{
+  "schema": "trajectory_replay_magento_create_options_reference/1",
+  "task_id": 549,
+  "task_intent": "Add a new size XXXL to green Minerva LumaTech V-Tee",
+  "official_endpoint": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144",
+  "corrected_endpoint": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$",
+  "captured_at": "2026-09-23T19:45:45+08:00",
+  "subject": "Magento configurable-product wizard new attribute option contract",
+  "official_source": {
+    "repository": "https://github.com/magento/magento2",
+    "revision": "efd7420a8d0758777032ee2f6fcc6c922d8bb3f9",
+    "path": "app/code/Magento/ConfigurableProduct/view/adminhtml/web/js/variations/steps/attributes_values.js",
+    "sha256": "088fb13a9b60cf160d40adbcd4dcdb7325da9239f2c241063794a9befa6aec3d"
+  },
+  "source_observation": {
+    "method": "saveOptions",
+    "request_method": "POST",
+    "request_url_binding": "createOptionsUrl",
+    "request_body_root": "options",
+    "success_status_class": 200
+  },
+  "required_option": {
+    "attribute_id": "144",
+    "label": "XXXL",
+    "is_new": "true"
+  },
+  "live_observation": {
+    "site": "shopping_admin",
+    "task_id": 549,
+    "request_method": "POST",
+    "request_path": "/admin/catalog/product_attribute/createOptions/",
+    "response_status": 200,
+    "new_option": {
+      "attribute_id": "144",
+      "label": "XXXL",
+      "is_new": "true"
+    },
+    "network_har_sha256": "d3974443c857afd700927b96568700ddd1c6d5ffa2fff516a161d0037e4fd7df",
+    "browser_call_count": 1
+  }
+}
diff --git a/research/trajectory-replay-task179-live-issue-reference.json b/research/trajectory-replay-task179-live-issue-reference.json
new file mode 100644
index 000000000..cf544ea0e
--- /dev/null
+++ b/research/trajectory-replay-task179-live-issue-reference.json
@@ -0,0 +1,56 @@
+{
+  "schema": "trajectory_replay_gitlab_live_reference/1",
+  "task_id": 179,
+  "task_intent": "Get whether my latest created issue with feature in its title is closed. Return a boolean (true if closed, false if opened).",
+  "source_sweep": "low-cost-repairs-sweep-v195",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "29a8d82629341664d164974931e2592f6932a106b3048867e1a395dd81d11e2c",
+    "actual_response": "2df64f5659039975483987e1c6d0c4dfb91f8429d1097f8aab3776366c0e6510",
+    "evaluation": "63caa57f13fec0242600ffa72ab84d3f775ab674f23142f153a9cd007974b8ca",
+    "trace": "b92f86f93d71d3e65a30e8b39dfa8ec9ca44cc8c079d641f9b61000eab5d0496",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "30c299758f3c5ab13b6efde6d03f35bf46c949c3b9f71817e4060d2796f742db"
+  },
+  "query": {
+    "authenticated_user_id": 2330,
+    "authenticated_username": "byteblaze",
+    "scope": "all",
+    "author_id": 2330,
+    "title_search": "feature",
+    "order_by": "created_at",
+    "sort": "desc"
+  },
+  "selected_issue": {
+    "id": 83747,
+    "iid": 39,
+    "project_id": 179,
+    "title": "[Feature suggestion] WCAG trash panda mode",
+    "author_id": 2330,
+    "created_at": "2023-01-22T19:23:16.000Z",
+    "state": "opened"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      false
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data",
+      "official": [
+        true
+      ],
+      "corrected": [
+        false
+      ]
+    }
+  ],
+  "finding": "The authenticated user's latest authored issue with feature in its title is open in the deployed GitLab state. The frozen expected value says it is closed.",
+  "regression_ids": [
+    "tr-179-accept-live-authored-issue-state",
+    "tr-179-reject-opposite-issue-state"
+  ]
+}
diff --git a/research/trajectory-replay-task18-map-live-reference.json b/research/trajectory-replay-task18-map-live-reference.json
new file mode 100644
index 000000000..edd82ce91
--- /dev/null
+++ b/research/trajectory-replay-task18-map-live-reference.json
@@ -0,0 +1,61 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 18,
+  "task_intent": "Get the time for walking and driving route from AMC Waterfront to Univ of Pittsburgh. Return a list of objects with keys \"mode\" (driving or walking) and \"duration\" (in HH:MM:SS format) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v117",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "643fadf78059364952fbdb16fe4a54905630af24d707564e773ce8dc43814f8b",
+    "actual_response": "b6f9933720d12b20d3345925ca449223fee45e8365e08441e9b889a162bcc231",
+    "evaluation": "5b181a9aea5cfbb4185627dbfcfc4a632f78e2b7985260d466e0b0dd63d3ed8f",
+    "trace": "c7ca74f043e3f99654b8c32946ef142003e1d6418bfbf32d6a2c01abf9558fbf",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "bdfe828060f6a69c47081cfcaab705d8606e9e3c8cdd4222de82a58a6d015c8f"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "mode": "driving",
+        "duration": "876.7s"
+      },
+      {
+        "mode": "walking",
+        "duration": "6431.6s"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data",
+      "official": [
+        {
+          "mode": "driving",
+          "duration": "2min"
+        },
+        {
+          "mode": "walking",
+          "duration": "16min"
+        }
+      ],
+      "corrected": [
+        {
+          "mode": "driving",
+          "duration": "876.7s"
+        },
+        {
+          "mode": "walking",
+          "duration": "6431.6s"
+        }
+      ]
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-18-accept-live-map-reference",
+    "tr-18-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task252-map-live-reference.json b/research/trajectory-replay-task252-map-live-reference.json
new file mode 100644
index 000000000..612cf25db
--- /dev/null
+++ b/research/trajectory-replay-task252-map-live-reference.json
@@ -0,0 +1,44 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 252,
+  "task_intent": "Get the coordinates of Tokyo Japanese Food Store in Pittsburgh. Return an object with keys \"latitude\" and \"longitude\" (as decimal degrees) only, without any additional details.",
+  "source_sweep": "map-live-sweep-v120",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "85eed90fbf4300ef5b7496be378bd451dd85f58e3cccdb9e3c73eaff70d3d0b4",
+    "actual_response": "ab1900f422950e07184a1d2f6687abc8ce26c157df1ff9da03557cebfb41ca98",
+    "evaluation": "e679f761ec1bf62acb69d5d1f8823b9e824c293fe576b7e4145cc9a53ce09d8f",
+    "trace": "56512069021373508ba23165d45636194988efc249181a9da8d8990b300a1e26",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "4c4080057956f6c043782482cf776125b14dd7beb4aa3ce71db705a8ee83fda0"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "latitude": "40.45760025",
+        "longitude": "-79.92932994166013"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0/latitude",
+      "official": "40.45761",
+      "corrected": "40.45760025"
+    },
+    {
+      "path": "/eval/0/expected/retrieved_data/0/longitude",
+      "official": "-79.92934",
+      "corrected": "-79.92932994166013"
+    }
+  ],
+  "derivation": {
+    "kind": "nominatim_object_geometry/1"
+  },
+  "regression_ids": [
+    "tr-252-accept-live-map-reference",
+    "tr-252-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task265-map-live-reference.json b/research/trajectory-replay-task265-map-live-reference.json
new file mode 100644
index 000000000..b40706fde
--- /dev/null
+++ b/research/trajectory-replay-task265-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 265,
+  "task_intent": "Get the relation ID of the closest national park to Boston and the distance to drive there. Return a list of objects with keys \"relation_id\" (integer) and \"distance\" (numeric value with unit km or m, e.g., 2.4km, 500m) only, without any additional details. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "source_sweep": "map-live-sweep-v126",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "49520968ec7c7f1267169f3ccc5ee78c9733086210b598de28351dfc9c8b2bb5",
+    "actual_response": "9654bb1aff02655bc99546abeafe658b1ca22a55817151cdf098eb571f7893f7",
+    "evaluation": "f6a01be0cad7123823472919120f9a26ae6f2652fec37a2800d261f2f4ba2210",
+    "trace": "994fb58c2cdf5597ce2e44a00783ef943353e00fe6911e4748a1aaaa4dcde88d",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "b58f411c230cb689493b2de93f71fa618290a22363d31fac28c788f9e6ae7a69"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "relation_id": 2176999,
+        "distance": "459445.3m"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "^.*/route/v1/.*/-68.2177005,44.3494709;-71.0579762,42.3603713.*$",
+      "corrected": "^.*/route/v1/.*/-68.2177005318788,44.3494709082008;-71.05768162905511,42.360400999999996.*$"
+    }
+  ],
+  "derivation": {
+    "kind": "frozen_wikipedia_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-265-accept-live-map-reference",
+    "tr-265-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task266-map-live-reference.json b/research/trajectory-replay-task266-map-live-reference.json
new file mode 100644
index 000000000..7088febb2
--- /dev/null
+++ b/research/trajectory-replay-task266-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 266,
+  "task_intent": "Get the relation ID of the closest national park to the largest city in Maine and the distance to drive there. Return a list of objects with keys \"relation_id\" (integer) and \"distance\" (numeric value with unit km or m, e.g., 2.4km, 500m) only, without any additional details. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "source_sweep": "map-live-sweep-v126",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "5fe85e84aee96355b71afbccc75ee93977c23443d904199273d7468ef39186ec",
+    "actual_response": "a6c296e928e8221bff6e4da75e33ae9802421cf93c475bf773961bacd12c7aa7",
+    "evaluation": "dcf03d1504888358987df5b573a9a5e5fe112149a2e480e9bc5657da4f769e0d",
+    "trace": "a35938377ecb3643b67b5c7252cc1942b1097f4ddf6e58bb902ff6825e66641f",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "82520969148cc96f6f7b4b790e8186e70bfd92c913dfa44ed583122a399b6108"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "relation_id": 2176999,
+        "distance": "289743.7m"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "^.*/route/v1/.*/-68.2177005,44.3494709;-70.2545299,43.6599147.*$",
+      "corrected": "^.*/route/v1/.*/-68.2177005318788,44.3494709082008;-70.25452991354058,43.659914650000005.*$"
+    }
+  ],
+  "derivation": {
+    "kind": "frozen_wikipedia_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-266-accept-live-map-reference",
+    "tr-266-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task267-map-live-reference.json b/research/trajectory-replay-task267-map-live-reference.json
new file mode 100644
index 000000000..daddf6916
--- /dev/null
+++ b/research/trajectory-replay-task267-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 267,
+  "task_intent": "Get the relation ID of the closest national park to the hometown of Stephen King and the time to drive there. Return a list of objects with keys \"relation_id\" (integer) and \"duration\" (in HH:MM:SS format) only, without any additional details. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "source_sweep": "map-live-sweep-v126",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "a3d4f74d1da89a878a59e213dcc3a651409ba43c29564ef2e36f55675547bd51",
+    "actual_response": "92b9d825b24a2edb53c9fe47bf8cdaa32f3dbd691a63ab603f52dce4cc932ea0",
+    "evaluation": "1137bc6d5683bea123068442901258035d501309c849c229334e52db054048c0",
+    "trace": "84153a386d524265463e384a02127a7854f72b3beb11636f4a1ffb5fd0b34d3f",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "59eff0161f83773c01571377d30898a0633f75abc4d3fce215d83bd645c19dc3"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "relation_id": 2176999,
+        "duration": "5550.7s"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "^.*/route/.*/-68.2177005,44.3494709;-68.767507,44.8030715.*$",
+      "corrected": "^.*/route/.*/-68.2177005318788,44.3494709082008;-68.76750570908236,44.803065340439595.*$"
+    }
+  ],
+  "derivation": {
+    "kind": "frozen_wikipedia_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-267-accept-live-map-reference",
+    "tr-267-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task268-map-live-reference.json b/research/trajectory-replay-task268-map-live-reference.json
new file mode 100644
index 000000000..0548b21d1
--- /dev/null
+++ b/research/trajectory-replay-task268-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 268,
+  "task_intent": "Get the relation ID of the closest national park to Vinalhaven, ME and the time to bike there. Return a list of objects with keys \"relation_id\" (integer) and \"duration\" (in HH:MM:SS format) only, without any additional details. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "source_sweep": "map-live-sweep-v126",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "8f6651fed7fa0a3d85d3e882d3bd2023aa04354501d00ce1e191f18a243612c6",
+    "actual_response": "ad5c1a9a12aa2dcdbe8f5483121bc96a99d888156e6683b559b88de173c7cdc0",
+    "evaluation": "61022919accaefad8a9a1889ade8b8ed986586a497258dd4d4dc02e1de041b1c",
+    "trace": "7d15f90a877190b103adcacdb65e738317d479a72d5763fdb0a2eb202ffde373",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "e9597fd7c06108c868715b346e72d67b584b4b623b5b83e098a231f72a9d6bcc"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "relation_id": 2176999,
+        "duration": "39504.1s"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "^.*/route/v1/.*/-68.2177005,44.3494709;-68.8315387,44.0478975.*$",
+      "corrected": "^.*/route/v1/.*/-68.2177005318788,44.3494709082008;-68.83153867121342,44.047897450225626.*$"
+    }
+  ],
+  "derivation": {
+    "kind": "frozen_wikipedia_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-268-accept-live-map-reference",
+    "tr-268-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task280-revised-reference.json b/research/trajectory-replay-task280-revised-reference.json
new file mode 100644
index 000000000..b91af47d3
--- /dev/null
+++ b/research/trajectory-replay-task280-revised-reference.json
@@ -0,0 +1,257 @@
+{
+  "schema": "trajectory_replay_revised_reference/1",
+  "task_id": 280,
+  "observed_at": "2026-09-23",
+  "derivation": {
+    "kind": "qualified_catalog_price_range/1",
+    "selection": "Complete name-match population for Anker, retaining names that also contain charger; this is the observable title rule shared by every official member."
+  },
+  "independent_source": {
+    "kind": "public_storefront_graphql",
+    "path": "/graphql",
+    "filter": {
+      "name": {
+        "match": "Anker"
+      }
+    },
+    "page_size": 100,
+    "total_count": 28,
+    "returned_items": 28,
+    "source_observation_sha256": "01bd5d981c53d17370317b12824a5b389c3151cedebcb74c16df0d6c24cba94c",
+    "candidate_binding": {
+      "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-280.json",
+      "sha256": "e975838d2a64045b57ebb62a336aa17ed193270f976fbce3e2fd2f8c4264b70b",
+      "contract_sha256": "c6bd0706f1826cb5597e78c68e25649e1453ffea62d79c0b76bc76506512bf51",
+      "canonical_calls_sha256": "e8c06d05e08447cc88534eb74a89956254a5ee159626de258c414512a0a588e7"
+    },
+    "evidence_sha256": {
+      "trace.json": "0695f0e22a469203854bf74216c445c096528fb26e9378ee68b18e0be24fa9e7",
+      "acceptance.json": "b56b5fded8911d8a4cd16150660f2e7593a5f60e7a37dea51bc70928f8fe2476",
+      "network.har": "3b6ccc3d87a4122af35c287d0b77710e6f1985a57b783d0fccce1493fd57f7ed",
+      "readiness.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+      "actual_response.json": "8dfde6ae3e0b25fe5a197ea1b0c427b9e3be0e028c109f4afebee4805f95e484",
+      "evaluation.json": "35b623bd852af7412ed0783b93f0e9214fee2969b468a9021d34f2d3e10f7aab"
+    },
+    "qualified_products": [
+      {
+        "sku": "B08LPPD3NL",
+        "name": "Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/magnetic-wireless-charger-anker-wireless-charger-with-5ft-built-in-usb-c-cable-powerwave-magnetic-pad-7-5w-charging-for-iphone-13-13-pro-13-pro-max-13-mini-12-12-pro-no-ac-adapter.html",
+        "price_cents": 2399,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter)"
+        }
+      },
+      {
+        "sku": "B09C5SC9FW",
+        "name": "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-usb-c-charger-40w-521-charger-nano-pro-piq-3-0-durable-compact-fast-charger-not-foldable-for-iphone-13-13-mini-13-pro-13-pro-max-12-galaxy-pixel-4-3-ipad-ipad-mini-cable-not-included.html",
+        "price_cents": 3599,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)"
+        }
+      },
+      {
+        "sku": "B00VH8ZW02",
+        "name": "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-40w-5-port-usb-wall-charger-powerport-5-for-iphone-xs-xs-max-xr-x-8-7-6-plus-ipad-pro-air-2-mini-galaxy-s9-s8-edge-plus-note-8-7-lg-nexus-htc-and-more-black-ak-a2124111.html",
+        "price_cents": 2999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)"
+        }
+      },
+      {
+        "sku": "B08J48CPMJ",
+        "name": "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-10w-max-wireless-charger-313-wireless-charger-pad-qi-certified-wireless-charging-7-5w-for-iphone-12-12-pro-12-mini-12-pro-max-10w-for-galaxy-s10-s9-s8-s9-plus-note-9-no-ac-adapter.html",
+        "price_cents": 899,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)"
+        }
+      },
+      {
+        "sku": "B09F671M86",
+        "name": "Anker Outlet Extender with Stepless Dimming Night Light,Surge Protector, 331 Outlet Extender with 6 AC Outlets and 2 USB Ports, 18W USB Wall Charger Plug for College Dorm Room, Home Office and Kitchen",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-outlet-extender-with-stepless-dimming-night-light-surge-protector-331-outlet-extender-with-6-ac-outlets-and-2-usb-ports-18w-usb-wall-charger-plug-for-college-dorm-room-home-office-and-kitchen.html",
+        "price_cents": 1899,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker Outlet Extender with Stepless Dimming Night Light,Surge Protector, 331 Outlet Extender with 6 AC Outlets and 2 USB Ports, 18W USB Wall Charger Plug for College Dorm Room, Home Office and Kitchen"
+        }
+      },
+      {
+        "sku": "B09FJWQ15T",
+        "name": "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/5ft-micro-usb-charger-cord-cable-fit-for-anker-powercore-5000-10000-20100-13000-26800-mini-3350-fusion-ii-15000-redux-20000-slim-10000-astro-e1-ac-replacement-power-adapter-supply.html",
+        "price_cents": 899,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply"
+        }
+      },
+      {
+        "sku": "B08T67HDH1",
+        "name": "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/usb-c-charger-anker-20w-piq-3-0-fast-charger-with-foldable-plug-powerport-iii-charger-for-iphone-13-13-mini-13-pro-13-pro-max-12-11-ipad-ipad-mini-magsafe-and-more-cable-not-included.html",
+        "price_cents": 1599,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)"
+        }
+      },
+      {
+        "sku": "B07DFGKFM3",
+        "name": "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/iphone-12-charger-gan-tech-anker-30w-compact-usb-c-wall-charger-with-power-delivery-powerport-atom-for-iphone-12-mini-pro-pro-max-11-x-xs-xr-ipad-pro-macbook-12-pixel-galaxy.html",
+        "price_cents": 2999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy"
+        }
+      },
+      {
+        "sku": "B01IUSYF8G",
+        "name": "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-quick-charge-3-0-39w-dual-usb-wall-charger-powerport-speed-2-for-galaxy-s10-s9-s8-edge-plus-note-8-7-and-poweriq-for-iphone-xs-xs-max-xr-x-8-plus-ipad-pro-air-2-mini-lg-nexus-htc-and-more.html",
+        "price_cents": 2599,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More"
+        }
+      },
+      {
+        "sku": "B0832KMRLT",
+        "name": "USB C Cable 60W 10ft, Anker Powerline III USB-C to USB-C Cable 2.0, USB C Charger Cable for MacBook Pro 2020, iPad Pro 2020, Switch, Samsung Galaxy S20 Plus S9 S8 Plus, Pixel, and More (White)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/usb-c-cable-60w-10ft-anker-powerline-iii-usb-c-to-usb-c-cable-2-0-usb-c-charger-cable-for-macbook-pro-2020-ipad-pro-2020-switch-samsung-galaxy-s20-plus-s9-s8-plus-pixel-and-more-white.html",
+        "price_cents": 1999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "USB C Cable 60W 10ft, Anker Powerline III USB-C to USB-C Cable 2.0, USB C Charger Cable for MacBook Pro 2020, iPad Pro 2020, Switch, Samsung Galaxy S20 Plus S9 S8 Plus, Pixel, and More (White)"
+        }
+      },
+      {
+        "sku": "B09J4YHB7J",
+        "name": "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/ 13 Pro/ 13 Pro Max/ 12, Galaxy S21, Note 20, iPad Pro, Pixel, and More",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-usb-c-charger-30w-711-charger-compact-fast-charger-not-foldable-for-macbook-air-iphone-13-13-mini-13-pro-13-pro-max-12-galaxy-s21-note-20-ipad-pro-pixel-and-more.html",
+        "price_cents": 2999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/ 13 Pro/ 13 Pro Max/ 12, Galaxy S21, Note 20, iPad Pro, Pixel, and More"
+        }
+      },
+      {
+        "sku": "B01N0X3NL5",
+        "name": "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-powercore-speed-20000-20000mah-qualcomm-quick-charge-3-0-poweriq-portable-charger-with-quick-charge-recharging-power-bank-for-samsung-iphone-ipad-and-more-black-a1278.html",
+        "price_cents": 5999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)"
+        }
+      },
+      {
+        "sku": "B08XQRCPG4",
+        "name": "USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/usb-c-super-fast-charger-anker-25w-pd-wall-charger-fast-charging-for-samsung-galaxy-s21-s21-s21-ultra-s20-z-flip-note20-20-ultra-note10-10-s9-s8-s10e-ipad-pro-12-9-and-more-cable-not-included.html",
+        "price_cents": 1999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included)"
+        }
+      },
+      {
+        "sku": "B071YMZ4LD",
+        "name": "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/usb-charger-anker-elite-dual-port-24w-wall-charger-powerport-2-with-poweriq-and-foldable-plug-for-iphone-11-xs-xs-max-xr-x-8-7-6-plus-ipad-pro-air-2-mini-3-mini-4-samsung-s4-s5-and-more.html",
+        "price_cents": 1399,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More"
+        }
+      },
+      {
+        "sku": "B01MSM6DE7",
+        "name": "Anker iPhone Charger Cable, Powerline II Lightning Cable (10ft), Durable Cable, MFi Certified for iPhone X / 8/8 Plus /7/7 Plus / 6/6 Plus / 5s (White), iPad 8, and More",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-iphone-charger-cable-powerline-ii-lightning-cable-10ft-durable-cable-mfi-certified-for-iphone-x-8-8-plus-7-7-plus-6-6-plus-5s-white-ipad-8-and-more.html",
+        "price_cents": 1499,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker iPhone Charger Cable, Powerline II Lightning Cable (10ft), Durable Cable, MFi Certified for iPhone X / 8/8 Plus /7/7 Plus / 6/6 Plus / 5s (White), iPad 8, and More"
+        }
+      },
+      {
+        "sku": "B07DBXZZN3",
+        "name": "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/anker-wireless-charger-313-wireless-charger-stand-qi-certified-for-iphone-12-12-pro-max-se-11-11-pro-11-pro-max-xr-xs-max-10w-fast-charging-galaxy-s20-s10-no-ac-adapter.html",
+        "price_cents": 1949,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)"
+        }
+      },
+      {
+        "sku": "B07GWR9V1N",
+        "name": "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/usb-c-charger-anker-30w-2-port-fast-charger-with-18w-usb-c-power-adapter-foldable-powerport-pd-2-charger-for-ipad-pro-iphone-11-11-pro-11-pro-max-xs-max-xr-x-pixel-galaxy-and-more.html",
+        "price_cents": 2999,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name contains both Anker and charger, matching the official task's catalog-name qualification",
+          "matched_name": "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More"
+        }
+      }
+    ]
+  },
+  "expected_response": {
+    "task_type": "RETRIEVE",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "names": [
+          "Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter)",
+          "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)",
+          "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)",
+          "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)",
+          "Anker Outlet Extender with Stepless Dimming Night Light,Surge Protector, 331 Outlet Extender with 6 AC Outlets and 2 USB Ports, 18W USB Wall Charger Plug for College Dorm Room, Home Office and Kitchen",
+          "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply",
+          "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)",
+          "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy",
+          "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More",
+          "USB C Cable 60W 10ft, Anker Powerline III USB-C to USB-C Cable 2.0, USB C Charger Cable for MacBook Pro 2020, iPad Pro 2020, Switch, Samsung Galaxy S20 Plus S9 S8 Plus, Pixel, and More (White)",
+          "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/ 13 Pro/ 13 Pro Max/ 12, Galaxy S21, Note 20, iPad Pro, Pixel, and More",
+          "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)",
+          "USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included)",
+          "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More",
+          "Anker iPhone Charger Cable, Powerline II Lightning Cable (10ft), Durable Cable, MFi Certified for iPhone X / 8/8 Plus /7/7 Plus / 6/6 Plus / 5s (White), iPad 8, and More",
+          "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)",
+          "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More"
+        ],
+        "min": 8.99,
+        "max": 59.99
+      }
+    ],
+    "error_details": null
+  },
+  "root_cause": "The frozen reference contains 12 title-matched Anker charger products. The deployed catalog's complete 28-item Anker population now contains 17 names meeting the same title rule, so the frozen list omits five current products.",
+  "score_effect": "Reference membership changes from 12 to 17 products; price bounds remain USD 8.99–59.99. Historical results remain unchanged."
+}
diff --git a/research/trajectory-replay-task282-revised-reference.json b/research/trajectory-replay-task282-revised-reference.json
new file mode 100644
index 000000000..06ff66b91
--- /dev/null
+++ b/research/trajectory-replay-task282-revised-reference.json
@@ -0,0 +1,161 @@
+{
+  "schema": "trajectory_replay_revised_reference/1",
+  "task_id": 282,
+  "observed_at": "2026-09-23",
+  "derivation": {
+    "kind": "qualified_catalog_price_range/1",
+    "selection": "Complete name-match population for Nike, retaining names that begin with Nike and explicitly identify slide or slides."
+  },
+  "independent_source": {
+    "kind": "public_storefront_graphql",
+    "path": "/graphql",
+    "filter": {
+      "name": {
+        "match": "Nike"
+      }
+    },
+    "page_size": 100,
+    "total_count": 148,
+    "returned_items": 148,
+    "source_observation_sha256": "10906db0bf14fb0e13d2462a76978470e7a245265cf2156a2fc0424a07bcba54",
+    "candidate_binding": {
+      "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-282.json",
+      "sha256": "b38cc2c5827f6b3ee8da3c0c074af364f1618762ab7223e3eaf76416dfa3cee1",
+      "contract_sha256": "04834c78c79bebd4b88cb66feb08e3c758139388f78ab75d7b730cd6977fb6c0",
+      "canonical_calls_sha256": "df6bdfcba113c601868a052ad247754cacabf2d5b86d2c838994ce6d31cb75f4"
+    },
+    "evidence_sha256": {
+      "trace.json": "442b408984070614bc500d1b8a374c8595ba265776c2673959f5b662fcc7e06f",
+      "acceptance.json": "11636bed51ec9a3c9b1cc075630c0e862b7eaa6f9180ef36426b3ebea6809582",
+      "network.har": "c6d3a9c828070a201e83664e66f882015081f35995ece54f1f55c6cf83a63d6a",
+      "readiness.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+      "actual_response.json": "fe7b5253bedcb0368259f106b694089aa265a1f902c522149a612dc153c7fdc0",
+      "evaluation.json": "bf9524d469aee4fc35b35b9c47678bec8cc35b82f97d10894cc63ca555fe6472"
+    },
+    "qualified_products": [
+      {
+        "sku": "B091MQTJ5V",
+        "name": "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-men-s-benassi-solarsoft-slide-athletic-sandal-midnight-navy-blue-numeric-8.html",
+        "price_cents": 9065,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)"
+        }
+      },
+      {
+        "sku": "B07PWQF2J4",
+        "name": "Nike Men's Air Max Camden Slide Sandal",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-men-s-air-max-camden-slide-sandal.html",
+        "price_cents": 6900,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Men's Air Max Camden Slide Sandal"
+        }
+      },
+      {
+        "sku": "B091G4NDPD",
+        "name": "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-men-s-benassi-solarsoft-slide-athletic-sandal-black-white-numeric-14.html",
+        "price_cents": 3995,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)"
+        }
+      },
+      {
+        "sku": "B084VVK71Y",
+        "name": "Nike Jordan Men's Break Slide Red AR6374-602",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-jordan-men-s-break-slide-red-ar6374-602.html",
+        "price_cents": 4281,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Jordan Men's Break Slide Red AR6374-602"
+        }
+      },
+      {
+        "sku": "B07GKXTVLN",
+        "name": "Nike Men's Benassi JDI Fanny Pack Slides",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-men-s-benassi-jdi-fanny-pack-slides.html",
+        "price_cents": 5494,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Men's Benassi JDI Fanny Pack Slides"
+        }
+      },
+      {
+        "sku": "B081TH3QK6",
+        "name": "Nike Offcourt Slide Mens Bq4639-002 Size 12",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-offcourt-slide-mens-bq4639-002-size-12.html",
+        "price_cents": 4861,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Offcourt Slide Mens Bq4639-002 Size 12"
+        }
+      },
+      {
+        "sku": "B096WH29TZ",
+        "name": "Nike Victori One Slide Mens Style : Dd9559-300",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-victori-one-slide-mens-style-dd9559-300.html",
+        "price_cents": 6100,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Victori One Slide Mens Style : Dd9559-300"
+        }
+      },
+      {
+        "sku": "B08QBDYN14",
+        "name": "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-victori-one-mens-comfort-slide-cn9675-003-midnight-navy-midnight-navy-white-numeric-10.html",
+        "price_cents": 3686,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)"
+        }
+      },
+      {
+        "sku": "B0838KJT69",
+        "name": "Nike Owaysis Mens Slide Casual Sandal Ct5545-002 Size 6 Black/Black-Black",
+        "url": "http://webarena-slice-warm-1-195-shopping.dev.lexmount.net/nike-owaysis-mens-slide-casual-sandal-ct5545-002-size-6-black-black-black.html",
+        "price_cents": 7399,
+        "currency": "USD",
+        "qualification_evidence": {
+          "rule": "name begins with Nike and explicitly identifies a slide or slides",
+          "matched_name": "Nike Owaysis Mens Slide Casual Sandal Ct5545-002 Size 6 Black/Black-Black"
+        }
+      }
+    ]
+  },
+  "expected_response": {
+    "task_type": "RETRIEVE",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "names": [
+          "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)",
+          "Nike Men's Air Max Camden Slide Sandal",
+          "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)",
+          "Nike Jordan Men's Break Slide Red AR6374-602",
+          "Nike Men's Benassi JDI Fanny Pack Slides",
+          "Nike Offcourt Slide Mens Bq4639-002 Size 12",
+          "Nike Victori One Slide Mens Style : Dd9559-300",
+          "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)",
+          "Nike Owaysis Mens Slide Casual Sandal Ct5545-002 Size 6 Black/Black-Black"
+        ],
+        "min": 36.86,
+        "max": 90.65
+      }
+    ],
+    "error_details": null
+  },
+  "root_cause": "The frozen reference includes Nike womens Benassi Just Do It, which is absent from the deployed catalog's complete 148-item Nike population. The current population instead includes Nike Owaysis Mens Slide, changing both membership and the minimum price.",
+  "score_effect": "Reference membership remains nine products but replaces the absent Benassi item with the current Owaysis slide; the price range changes from USD 27.60–90.65 to USD 36.86–90.65. Historical results remain unchanged."
+}
diff --git a/research/trajectory-replay-task32-map-live-reference.json b/research/trajectory-replay-task32-map-live-reference.json
new file mode 100644
index 000000000..7c8095947
--- /dev/null
+++ b/research/trajectory-replay-task32-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 32,
+  "task_intent": "I will arrive at Pittsburgh Airport soon. Find a Hilton hotel in the vicinity, if available, and get me its name and the walking distance to the nearest supermarket own by a local company from the hotel. Return a list of objects with keys \"hotel\" (hotel name) and \"distance\" (numeric value with unit km or m, e.g., 2.4km, 500m) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v125",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "c5ff5707eebf826985982f828fb609284b3e5afbe2fc6420ffad5f758e4e16e9",
+    "actual_response": "2953d899f9d1fcfa9e5d494d0aabd6effc2f6cc2ca13dec275d2f5f281696cfe",
+    "evaluation": "b445d618226767fd8c5188b6226dca8dd36969aa595b38eaac820fbfcfc1c393",
+    "trace": "7dbf71113a8607a9f407d2e16c48f2f9f2db5de381735bd1d673cb8d86d487b7",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "378ffcdcdce1a635c8b6204ede4eafe1faffe4997672a323a78455d59d43ce09"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "hotel": "DoubleTree by Hilton Hotel Pittsburgh Airport",
+        "distance": "1946.9m"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0/distance",
+      "official": "2km",
+      "corrected": "1946.9m"
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-32-accept-live-map-reference",
+    "tr-32-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task34-map-live-reference.json b/research/trajectory-replay-task34-map-live-reference.json
new file mode 100644
index 000000000..3b5347d9c
--- /dev/null
+++ b/research/trajectory-replay-task34-map-live-reference.json
@@ -0,0 +1,62 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 34,
+  "task_intent": "I will arrive at Pittsburgh Airport soon. Find a Hyatt hotel in the vicinity, if available, and get me its name and the shortest walking time to a supermarket from the hotel. Return a list of objects with keys \"hotel_name\" (hotel name only) and \"travel_time\" (in HH:MM:SS format) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v127",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "e828beef584f7890cf55813769ad46abee710df23a0d420b04c1a07b5a9c43f0",
+    "actual_response": "cee0743d0cfd866012b67f68b1d51dc820ae583f4aaeee6b4044d0c551455afa",
+    "evaluation": "4afd243ec2342b705d61a78d5cb5e819e3433dc054185ac857d256d1a87546b4",
+    "trace": "d8bd779f79276754d319bdd66aee148eb4ff4c85827fc974e24558d846d61368",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "125f9ccf1d7a30236225c92777288a10d0d750d4255a950f1d057d534018ef2a"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "hotel_name": "Hyatt Regency Pittsburgh International Airport",
+        "travel_time": "9892.3s"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/results_schema/items/properties",
+      "official": {
+        "hotel": {
+          "type": "string",
+          "format": "location-name"
+        },
+        "information": {
+          "type": "string",
+          "format": "duration"
+        }
+      },
+      "corrected": {
+        "hotel_name": {
+          "type": "string",
+          "format": "location-name"
+        },
+        "travel_time": {
+          "type": "string",
+          "format": "duration"
+        }
+      }
+    },
+    {
+      "path": "/eval/0/expected/retrieved_data/0/travel_time",
+      "official": "3h 30min",
+      "corrected": "9892.3s"
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-34-accept-live-map-reference",
+    "tr-34-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task35-map-live-reference.json b/research/trajectory-replay-task35-map-live-reference.json
new file mode 100644
index 000000000..951615f33
--- /dev/null
+++ b/research/trajectory-replay-task35-map-live-reference.json
@@ -0,0 +1,57 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 35,
+  "task_intent": "I will arrive at Pittsburgh Airport soon. Find a Hyatt hotel in the vicinity, if available, and get me its name and the minimal driving time to a supermarket from the hotel. Return a list of objects with keys \"hotel_name\" (hotel name only) and \"travel_time\" (in HH:MM:SS format) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v125",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "0356180a421eea87978d157d22bde1c753ef2954ee823969b34d93278ff75d6d",
+    "actual_response": "bb1a44fd9ae0e6c5e11299ff389849bd331a3282a6c4b1a02872c2c9c0932af4",
+    "evaluation": "29a4c67452fc0ca64920f629b9afde855d17d82fe68c4d6e9a8b94ec43dbc1c6",
+    "trace": "f493ba34c8d195579f5caf7095c63d41974d6c3007e8d0246290eabfdbc836e5",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "df9e49282affc68c9763de94c9f89afa04fe23bf6b064be284778afc0180a59e"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "hotel_name": "Hyatt Regency Pittsburgh International Airport",
+        "travel_time": "864.7s"
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/results_schema/items/properties",
+      "official": {
+        "hotel": {
+          "type": "string",
+          "format": "location-name"
+        },
+        "information": {
+          "type": "string",
+          "format": "duration"
+        }
+      },
+      "corrected": {
+        "hotel_name": {
+          "type": "string",
+          "format": "location-name"
+        },
+        "travel_time": {
+          "type": "string",
+          "format": "duration"
+        }
+      }
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-35-accept-live-map-reference",
+    "tr-35-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task363-map-live-reference.json b/research/trajectory-replay-task363-map-live-reference.json
new file mode 100644
index 000000000..0c632bca3
--- /dev/null
+++ b/research/trajectory-replay-task363-map-live-reference.json
@@ -0,0 +1,36 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 363,
+  "task_intent": "Measure distance between Carnegie Mellon University and Carnegie Music Hall by walking. Return the value as a string (numeric value with unit km or m, e.g., 2.4km, 500m) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v117",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "87b8e36c7eb9d96e38a36b2d57033836d0b0809ede03d165ba234018a5f26c24",
+    "actual_response": "b6f55bd02474296e12be174da5a6738b57ac9ca84e1cec3b1cc308734c81fd0f",
+    "evaluation": "8e128aa88f6504952d95bbffa4167334fad06982ff53f56f53ed09724f98022f",
+    "trace": "77cac636a0fe850fe2e7a6df4d88e88d3b49fc557fcc862b6c061acf515d07af",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "18a53355083e8bcd0d1aeb2c87ccac70bd3b04e920f367d9306931091f1428ba"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      "778.3m"
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0",
+      "official": "748m",
+      "corrected": "778.3m"
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-363-accept-live-map-reference",
+    "tr-363-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task367-map-live-reference.json b/research/trajectory-replay-task367-map-live-reference.json
new file mode 100644
index 000000000..3bb4f43b3
--- /dev/null
+++ b/research/trajectory-replay-task367-map-live-reference.json
@@ -0,0 +1,36 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 367,
+  "task_intent": "Measure distance between Carnegie Mellon University and CVS (closest one) by walking. Return the value as a string (numeric value with unit km or m, e.g., 2.4km, 500m) only, without any additional details. (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v120",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "8518b7f53b88fdbeacd01ebfe17849649383b25c705a2fba91517e3f5b4301b8",
+    "actual_response": "e6d2dd6b8fa8d434115982aebaa593e9fabab8581c9b3a26e392b472c573db2d",
+    "evaluation": "0674949893bfea88ff1f535b1d091475780dcd7f6e66b2816f6585fa7c9da131",
+    "trace": "95eb4cd6ecf4a112a75900dad675c12235375632ba854e967ecbc1e767c1d3b4",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "81ac721104b0e8c034b112cd27a018da5f8cceef9ac9601c55617cab518fc6b5"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      "1454.8m"
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0",
+      "official": "1.4km",
+      "corrected": "1454.8m"
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-367-accept-live-map-reference",
+    "tr-367-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task383-map-live-reference.json b/research/trajectory-replay-task383-map-live-reference.json
new file mode 100644
index 000000000..9ef43365e
--- /dev/null
+++ b/research/trajectory-replay-task383-map-live-reference.json
@@ -0,0 +1,54 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 383,
+  "task_intent": "I am arriving at Pittsburgh Airport. Find if there is a Hyatt hotel nearby. If there is, get me its name and the names of any supermarkets that are within 15 minutes driving from the hotel. Return a list of objects with keys \"hotel\" and \"supermarkets\". (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v127",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "b54f4491da0d3a22a7fac4f47acc43f11307c58beae802a5f6a16483b2c982c7",
+    "actual_response": "d62bbe6d76089f843c271c8b2dde6c5960cec2f456d05d5b9cfc56f9182f7a27",
+    "evaluation": "818aa0b9e6c4c4266b4ae60d0648f7deb5861e1570254be03728f523db3acaa9",
+    "trace": "8bcc67e32900583e223a20784d89a7154480f0f658c8769bbb2e1412a62c3260",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "e7c71f3ffb0b99d6f83e392d5801317edbb523314326ddbf39b7bff29f84358d"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      {
+        "hotel": "Hyatt Regency Pittsburgh International Airport",
+        "supermarkets": [
+          "Shop & Save",
+          "ALDI",
+          "Walmart Supercenter",
+          "Gordon Food Service",
+          "Magnet International Market"
+        ]
+      }
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0/supermarkets",
+      "official": [
+        "Giant Eagle",
+        "ALDI"
+      ],
+      "corrected": [
+        "Shop & Save",
+        "ALDI",
+        "Walmart Supercenter",
+        "Gordon Food Service",
+        "Magnet International Market"
+      ]
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-383-accept-live-map-reference",
+    "tr-383-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task423-magento-mass-status-reference.json b/research/trajectory-replay-task423-magento-mass-status-reference.json
new file mode 100644
index 000000000..921a88500
--- /dev/null
+++ b/research/trajectory-replay-task423-magento-mass-status-reference.json
@@ -0,0 +1,54 @@
+{
+  "schema": "trajectory_replay_magento_mass_status_reference/1",
+  "task_id": 423,
+  "task_intent": "Set all Hollister shirts to on-sale status",
+  "browser_call_count": 3,
+  "source_evidence_sha256": {
+    "network_har": "702f56d930b39a3a3fe374a9adf1f71a2895369bbcbfedafe3eb7b1324f19c8a",
+    "actual_response": "1a4a8a8dd855bd1eb3a9da3a2d1ebcefd1661db389cba564d1c6e0107075e1f8",
+    "evaluation": "1e24312c2cd60ae76e4462608d0d430533106d1ca6bf1299d2fcec68f8aed9bd",
+    "trace": "5e3354bb5b0d394290459bd72a2e531f9f65c0a7eb8490813b3801571c3aaa9d",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "3cfa67817cde36eb101d0fbb3d191080a5fa6a5c811ea8000b376e9317ef9726"
+  },
+  "observed_event": {
+    "method": "POST",
+    "url": "__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/",
+    "response_status": 302,
+    "post_data": {
+      "excluded": "false",
+      "filters[placeholder]": "true",
+      "search": "hollister",
+      "namespace": "product_listing"
+    },
+    "updated_record_count": 16
+  },
+  "official_contract": {
+    "url": "^__SHOPPING_ADMIN__/catalog/product/save/id/126/type/configurable/store/0/set/\\d+/back/edit$",
+    "post_data": {
+      "report_type": "created_at_order",
+      "from": "02/1/2023",
+      "to": "02/28/2023"
+    },
+    "http_method": "GET",
+    "response_status": 302,
+    "navigation_only": true
+  },
+  "corrected_contract": {
+    "url": "^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$",
+    "post_data": {
+      "excluded": "false",
+      "search": "hollister",
+      "namespace": "product_listing"
+    },
+    "http_method": "POST",
+    "response_status": 302,
+    "navigation_only": false
+  },
+  "derivation": {
+    "kind": "native_magento_filtered_mass_status/1",
+    "filter": "hollister",
+    "selected_scope": "all 16 filtered records",
+    "status": "enabled"
+  }
+}
diff --git a/research/trajectory-replay-task429-map-live-reference.json b/research/trajectory-replay-task429-map-live-reference.json
new file mode 100644
index 000000000..cc17291cf
--- /dev/null
+++ b/research/trajectory-replay-task429-map-live-reference.json
@@ -0,0 +1,39 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 429,
+  "task_intent": "On the map site, view the info page for the college or colleges where The Chair was filmed (open each in a separate tab) (use the provided wiki site to look up any needed information).",
+  "source_sweep": "map-college-sweep-v224",
+  "browser_call_count": 2,
+  "source_evidence_sha256": {
+    "network_har": "0a1b5d8380c73edefbbbbecd1c7775a76597212b57d4da159db7d855cc66083a",
+    "actual_response": "f5030a247a4d34da3abde0ef1f538e3a9a1bf67d41b32a18a829895f9c2f5cef",
+    "evaluation": "86046a586979df43697933fcf60a2c3633471dff9673b947f72028fafd3b3d41",
+    "trace": "b815841521da196697a923266edd201e8379234a5c5fde57b40b168dc99b85a1",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "565b9a0eeb468d65be5ce904bc948198cf42119e3c875f0e2f4bbb7196608d3d"
+  },
+  "observed_response": {
+    "task_type": "navigate",
+    "status": "SUCCESS",
+    "retrieved_data": null
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "__MAP__/relation/583390395",
+      "corrected": "__MAP__/way/583390395"
+    },
+    {
+      "path": "/eval/2/expected/url",
+      "official": "__MAP__/relation/172206707",
+      "corrected": "__MAP__/way/172206707"
+    }
+  ],
+  "derivation": {
+    "kind": "nominatim_object_geometry/1"
+  },
+  "regression_ids": [
+    "tr-429-accept-live-way-identities",
+    "tr-429-reject-relation-kind-for-way-identities"
+  ]
+}
diff --git a/research/trajectory-replay-task430-map-live-reference.json b/research/trajectory-replay-task430-map-live-reference.json
new file mode 100644
index 000000000..a2f35ed15
--- /dev/null
+++ b/research/trajectory-replay-task430-map-live-reference.json
@@ -0,0 +1,34 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 430,
+  "task_intent": "On the map site, view the info page for the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh (use the provided wiki site to look up any needed information).",
+  "source_sweep": "map-college-sweep-v224",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "4bd9109a90f1108b4caad986cdb29fab3cbcbdbd8085cc56f3ceb2eb7b6339ac",
+    "actual_response": "f5030a247a4d34da3abde0ef1f538e3a9a1bf67d41b32a18a829895f9c2f5cef",
+    "evaluation": "793d11ec6a240f96a6d9d31654c85c51a098c14d1fff20bbf851244f30ac8c39",
+    "trace": "3e4182aa4665238ae9c11940896b6e982308ee2fee917e8ee98c69c11af58e23",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "fe74bcfcd9cd00bfe24a3aed31daf875b946a41d1d9cfbb537857c1bfca1913b"
+  },
+  "observed_response": {
+    "task_type": "navigate",
+    "status": "SUCCESS",
+    "retrieved_data": null
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/url",
+      "official": "__MAP__/TASK_430_MAP_RELATION_ID",
+      "corrected": "__MAP__/way/583390395"
+    }
+  ],
+  "derivation": {
+    "kind": "nominatim_object_geometry/1"
+  },
+  "regression_ids": [
+    "tr-430-accept-live-washington-jefferson-way",
+    "tr-430-reject-unresolved-placeholder"
+  ]
+}
diff --git a/research/trajectory-replay-task510-quantity-revision.json b/research/trajectory-replay-task510-quantity-revision.json
index c37571913..dabae0b82 100644
--- a/research/trajectory-replay-task510-quantity-revision.json
+++ b/research/trajectory-replay-task510-quantity-revision.json
@@ -7,6 +7,7 @@
   "predecessor_correction_catalog_sha256": "f77d148498e50710fdb7b1205dcb7770d2e746d6856112cb8bfa173989333a9e",
   "predecessor_correction_record_sha256": "5a141413e71ad077c19515596b7b4a904fac427bdaa43de656a8f978aab7839e",
   "revised_correction_catalog_sha256": "e764f4b2828fa97d98cecc8752b31329871e612be1fff13878410d41e8414f9b",
+  "revised_catalog_path": "evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-post-510.json",
   "revised_correction_record_sha256": "5b57108f9589f43d9ce5db9869793c77012413e0f93985d39d3124959e14759c",
   "regression_ids": [
     "tr-510-quantity-one",
diff --git a/research/trajectory-replay-task550-magento-configurable-create-options-reference.json b/research/trajectory-replay-task550-magento-configurable-create-options-reference.json
new file mode 100644
index 000000000..12d149897
--- /dev/null
+++ b/research/trajectory-replay-task550-magento-configurable-create-options-reference.json
@@ -0,0 +1,28 @@
+{
+  "schema": "trajectory_replay_magento_create_options_reference/1",
+  "task_id": 550,
+  "task_intent": "Add a new size XXS to blue and purple Nona Fitness Tank",
+  "official_endpoint": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144",
+  "corrected_endpoint": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$",
+  "captured_at": "2026-09-23T19:45:45+08:00",
+  "subject": "Magento configurable-product wizard new size option contract for task 550",
+  "official_source": {
+    "repository": "https://github.com/magento/magento2",
+    "revision": "efd7420a8d0758777032ee2f6fcc6c922d8bb3f9",
+    "path": "app/code/Magento/ConfigurableProduct/view/adminhtml/web/js/variations/steps/attributes_values.js",
+    "sha256": "088fb13a9b60cf160d40adbcd4dcdb7325da9239f2c241063794a9befa6aec3d"
+  },
+  "source_observation": {
+    "method": "saveOptions",
+    "request_method": "POST",
+    "request_url_binding": "createOptionsUrl",
+    "request_body_root": "options",
+    "success_status_class": 200
+  },
+  "required_option": {
+    "attribute_id": "144",
+    "label": "XXS",
+    "is_new": "true"
+  },
+  "browser_call_count": 0
+}
diff --git a/research/trajectory-replay-task599-reddit-forum-reference.json b/research/trajectory-replay-task599-reddit-forum-reference.json
new file mode 100644
index 000000000..13b097a6b
--- /dev/null
+++ b/research/trajectory-replay-task599-reddit-forum-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_reddit_forum_reference/1",
+  "task_id": 599,
+  "captured_at": "2026-09-23T00:00:00+08:00",
+  "site": "reddit",
+  "observation": {
+    "search_url": "__REDDIT__/search?q=machine+learning",
+    "search_response_sha256": "20e9b0638f457ea208039d424f44c1f3d1bc06266b5b11c94d1190e0f0ca016f",
+    "matched_forum_href": "/f/MachineLearning",
+    "newest_url": "__REDDIT__/f/MachineLearning/new",
+    "newest_response_sha256": "d84e9d97417aaaeb6bcc9766692f28ae5015ccbac750b49bf21e90ca8b5e6b29",
+    "canonical_forum_href": "/f/MachineLearning",
+    "newest_post_href": "/f/MachineLearning/120613/d-yan-lecun-s-recent-recommendations"
+  },
+  "finding": "The deployed forum slug is case-sensitive MachineLearning. The official contract uses the nonexistent path machine learning while retaining the post ID and slug from the MachineLearning forum."
+}
diff --git a/research/trajectory-replay-task61-map-live-reference.json b/research/trajectory-replay-task61-map-live-reference.json
new file mode 100644
index 000000000..b7c3af523
--- /dev/null
+++ b/research/trajectory-replay-task61-map-live-reference.json
@@ -0,0 +1,36 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 61,
+  "task_intent": "Get the closest restaurant(s) to CMU Sorrells Library",
+  "source_sweep": "map-live-sweep-v128",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "77ae52fe84fa898cfcd960f61094d5c4a5b3fae71f862507fa3c05308f065de2",
+    "actual_response": "02677d978fd0dc8a99b394094cc4fba897a2b6669ffd23dcc0c84038e6c7b30d",
+    "evaluation": "b3279fd4e76ccf5cc330ed4b1fcec1a7918ec249f8038a206f69a7977c813daf",
+    "trace": "f633fcc18c2d5c45ff88d8f01ac35517b03b3b90262a2a0509a5dce5ed1b182f",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "812d2589ab738989d5801a067a499e3d275074f8464d80a8ed54a33edb6cefa3"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      "Hunan Express"
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/0",
+      "official": "La Prima Espresso",
+      "corrected": "Hunan Express"
+    }
+  ],
+  "derivation": {
+    "kind": "nominatim_spatial_distance/1"
+  },
+  "regression_ids": [
+    "tr-61-accept-live-map-reference",
+    "tr-61-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task677-navigation-xhr-reference.json b/research/trajectory-replay-task677-navigation-xhr-reference.json
new file mode 100644
index 000000000..55ce995ef
--- /dev/null
+++ b/research/trajectory-replay-task677-navigation-xhr-reference.json
@@ -0,0 +1,11 @@
+{
+  "schema": "trajectory_replay_navigation_xhr_reference/1",
+  "task_id": 677,
+  "expected_event_kind": "XHR after the sales order page navigation",
+  "expected_filter_status": "processing",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_har_sha256": "b5e186a51982ade7556fd378957a863d76bcfae36ad46c5a1eb65ddfc9aba28a",
+  "observed_request_contains": "filters%5Bstatus%5D=processing",
+  "finding": "The official contract expects the Magento grid XHR but navigation_only selects only the preceding HTML document navigation."
+}
diff --git a/research/trajectory-replay-task678-navigation-xhr-reference.json b/research/trajectory-replay-task678-navigation-xhr-reference.json
new file mode 100644
index 000000000..04a57be1a
--- /dev/null
+++ b/research/trajectory-replay-task678-navigation-xhr-reference.json
@@ -0,0 +1,11 @@
+{
+  "schema": "trajectory_replay_navigation_xhr_reference/1",
+  "task_id": 678,
+  "expected_event_kind": "XHR after the sales order page navigation",
+  "expected_filter_status": "canceled",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_har_sha256": "147ac90777dbfe287b406183747083d66b16804da51113557ccb08e2f67eb368",
+  "observed_request_contains": "filters%5Bstatus%5D=canceled",
+  "finding": "The official contract expects the Magento grid XHR but navigation_only selects only the preceding HTML document navigation."
+}
diff --git a/research/trajectory-replay-task679-navigation-xhr-reference.json b/research/trajectory-replay-task679-navigation-xhr-reference.json
new file mode 100644
index 000000000..bd6d4bd41
--- /dev/null
+++ b/research/trajectory-replay-task679-navigation-xhr-reference.json
@@ -0,0 +1,11 @@
+{
+  "schema": "trajectory_replay_navigation_xhr_reference/1",
+  "task_id": 679,
+  "expected_event_kind": "XHR after the sales order page navigation",
+  "expected_filter_status": "complete",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_har_sha256": "50bb8137b07185bbf4e3a6e631922ca6a5429b44af7e1487bace24b6f8295f1f",
+  "observed_request_contains": "filters%5Bstatus%5D=complete",
+  "finding": "The official contract expects the Magento grid XHR but navigation_only selects only the preceding HTML document navigation."
+}
diff --git a/research/trajectory-replay-task680-navigation-xhr-reference.json b/research/trajectory-replay-task680-navigation-xhr-reference.json
new file mode 100644
index 000000000..b003959f0
--- /dev/null
+++ b/research/trajectory-replay-task680-navigation-xhr-reference.json
@@ -0,0 +1,11 @@
+{
+  "schema": "trajectory_replay_navigation_xhr_reference/1",
+  "task_id": 680,
+  "expected_event_kind": "XHR after the sales order page navigation",
+  "expected_filter_status": "holded",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_har_sha256": "cd27b443abeed1baa09ae6f68bad311dcbb0700af851ccfb0738aa390cca6173",
+  "observed_request_contains": "filters%5Bstatus%5D=holded",
+  "finding": "The official contract expects the Magento grid XHR but navigation_only selects only the preceding HTML document navigation."
+}
diff --git a/research/trajectory-replay-task682-live-repository-reference.json b/research/trajectory-replay-task682-live-repository-reference.json
new file mode 100644
index 000000000..28b7c2e85
--- /dev/null
+++ b/research/trajectory-replay-task682-live-repository-reference.json
@@ -0,0 +1,47 @@
+{
+  "schema": "trajectory_replay_gitlab_repository_reference/1",
+  "task_id": 682,
+  "task_intent": "Find a GitLab repository related to chatGPT and create a new post in the discussion forum with the URL of the repository to the most relevant forum. Use the repository path as the post title and \" commit(s) already!\" as the post body.",
+  "source_sweep": "cross-site-posts-sweep-v192",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "642237d854e6e98465e701ac3dbe3b11e6b0aea5b4aa9b0e959bf381e431bcb9",
+    "failure_screenshot": "0772ffd0591ac6426df99ccb391a5673e383de7853701c54b3883d6da5264ee5",
+    "trace": "fd3beaba537ee23e72e465dff437f2f5bc6fdb8800ec2c02babf05e15c7e10f8",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "9408a1ff2f89f10839662d22e0dac099399dcbbd662a3403815a263620237861"
+  },
+  "query": {
+    "search_term": "chatGPT",
+    "simple": true,
+    "per_page": 100,
+    "result_count": 1
+  },
+  "selected_project": {
+    "id": 175,
+    "path_with_namespace": "convexegg/chatgpt",
+    "web_url": "http://localhost:8023/convexegg/chatgpt"
+  },
+  "commit_page_sizes": [
+    1
+  ],
+  "commit_count": 1,
+  "observed_post": {
+    "title": "convexegg/chatgpt",
+    "url": "http://localhost:8023/convexegg/chatgpt",
+    "body": "1 commits already!",
+    "kind": "repo"
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/post_data/submission[body]",
+      "official": "193 commits already!",
+      "corrected": "1 commits already!"
+    }
+  ],
+  "finding": "The deployed GitLab search returns exactly one chatGPT project (convexegg/chatgpt); complete commit pagination yields 1 commits.",
+  "regression_ids": [
+    "tr-682-accept-live-repository-post",
+    "tr-682-reject-frozen-repository-post"
+  ]
+}
diff --git a/research/trajectory-replay-task683-live-repository-reference.json b/research/trajectory-replay-task683-live-repository-reference.json
new file mode 100644
index 000000000..2bf2dafd6
--- /dev/null
+++ b/research/trajectory-replay-task683-live-repository-reference.json
@@ -0,0 +1,54 @@
+{
+  "schema": "trajectory_replay_gitlab_repository_reference/1",
+  "task_id": 683,
+  "task_intent": "Find a GitLab repository related to metaseq and create a new post in the discussion forum with the URL of the repository to the most relevant forum. Use the repository path as the post title and \" commit(s) already!\" as the post body.",
+  "source_sweep": "cross-site-posts-sweep-v192",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "d33e6fa8f1b938528a3e25bb40d3d4bbba38ac27233b718a85be3d866b9aa140",
+    "failure_screenshot": "15f88b7e9443642af335fa6989b8f1e74902161e12ecaa71fbdea654002e83f9",
+    "trace": "197a0e52f59db47c879d33145ba9ac71700a817ee41673a4eed1126efa776536",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "a5cc6f7c9cc3c1658da651aa2a2464899bbb5c96b92a45e6b3d6c702759fca52"
+  },
+  "query": {
+    "search_term": "metaseq",
+    "simple": true,
+    "per_page": 100,
+    "result_count": 1
+  },
+  "selected_project": {
+    "id": 33,
+    "path_with_namespace": "root/metaseq",
+    "web_url": "http://localhost:8023/root/metaseq"
+  },
+  "commit_page_sizes": [
+    100,
+    100,
+    72
+  ],
+  "commit_count": 272,
+  "observed_post": {
+    "title": "root/metaseq",
+    "url": "http://localhost:8023/root/metaseq",
+    "body": "272 commits already!",
+    "kind": "repo"
+  },
+  "contract_values": [
+    {
+      "path": "/eval/1/expected/post_data/submission[url]",
+      "official": "__GITLAB__/facebook/metaseq",
+      "corrected": "__GITLAB__/root/metaseq"
+    },
+    {
+      "path": "/eval/1/expected/post_data/submission[title]",
+      "official": "facebook/metaseq",
+      "corrected": "root/metaseq"
+    }
+  ],
+  "finding": "The deployed GitLab search returns exactly one metaseq project (root/metaseq); complete commit pagination yields 272 commits.",
+  "regression_ids": [
+    "tr-683-accept-live-repository-post",
+    "tr-683-reject-frozen-repository-post"
+  ]
+}
diff --git a/research/trajectory-replay-task737-route-order-reference.json b/research/trajectory-replay-task737-route-order-reference.json
new file mode 100644
index 000000000..452eb8cfa
--- /dev/null
+++ b/research/trajectory-replay-task737-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 737,
+  "task_intent": "View the route on the map from Carnegie Mellon University to the home stadium of Philadelphia 76ers. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "official_expected_url": "^.*/route/v1/.*/-75.1718916,39.9011873;-79.9427192,40.4441897.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1718916,39.9011873.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-79.9427192,40.4441897;-75.1718916,39.9011873?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "25f7ef44f5bbb47b355ccc3269bef1f59c4b29a8053b4615199c331fda55c50f",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task738-route-order-reference.json b/research/trajectory-replay-task738-route-order-reference.json
new file mode 100644
index 000000000..612c8f12f
--- /dev/null
+++ b/research/trajectory-replay-task738-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 738,
+  "task_intent": "View the route on the map from Carnegie Mellon University to the home stadium of Philadelphia 76ers in the 70s. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "official_expected_url": "^.*/route/v1/.*/-75.1712951,39.9042046;-79.9427192,40.4441897.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1712951,39.9042046.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-79.9427192,40.4441897;-75.1712951,39.9042046?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "d577a056721fbcfb4027d51668bdd17a3e6a332362e6afe22a01aa00f6e337e1",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task739-route-order-reference.json b/research/trajectory-replay-task739-route-order-reference.json
new file mode 100644
index 000000000..6a4319fca
--- /dev/null
+++ b/research/trajectory-replay-task739-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 739,
+  "task_intent": "View the route on the map from Carnegie Mellon University to the home stadium of Yankees in the 80s. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "official_expected_url": "^.*/route/v1/.*/-73.9265212,40.8295828;-79.9427192,40.4441897.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9265212,40.8295828.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-79.9427192,40.4441897;-73.9265212,40.8295828?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "77e7a1d6f5db3017125c4b6c29909186266108889ac38a75e134c347d38d6d74",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task740-route-order-reference.json b/research/trajectory-replay-task740-route-order-reference.json
new file mode 100644
index 000000000..7e244e3b6
--- /dev/null
+++ b/research/trajectory-replay-task740-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 740,
+  "task_intent": "View the route on the map from Carnegie Mellon University to the home stadium of NYC NBA team. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "official_expected_url": "^.*/route/v1/.*/-73.9935443,40.7505085;-79.9427192,40.4441897.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9935443,40.7505085.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-79.9427192,40.4441897;-73.9935443,40.7505085?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "4fd013ebfee32c31d63887c310c71baac48cfde0658828429302f255d71d52b8",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task741-route-order-reference.json b/research/trajectory-replay-task741-route-order-reference.json
new file mode 100644
index 000000000..3f16cd0bd
--- /dev/null
+++ b/research/trajectory-replay-task741-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 741,
+  "task_intent": "View the route on the map from Carnegie Mellon University to the home stadium of Boston home NBA team. (Use the OSRM direction service and the provided wiki to look up any needed information and search both source and destination by coordinates from the place official page on the wiki.)",
+  "official_expected_url": "^.*/route/v1/.*/-71.0621475,42.3662922;-79.9427192,40.4441897.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-79.9427192,40.4441897;-71.0621475,42.3662922.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-79.9427192,40.4441897;-71.0621475,42.3662922?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "1ba77e89105b2cc5f25eee55658634ae1c13793cce05923e6c225f795292ea9c",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task742-multi-member-reference.json b/research/trajectory-replay-task742-multi-member-reference.json
new file mode 100644
index 000000000..8c45f21dd
--- /dev/null
+++ b/research/trajectory-replay-task742-multi-member-reference.json
@@ -0,0 +1,20 @@
+{
+  "schema": "trajectory_replay_repeated_network_event_reference/1",
+  "task_id": 742,
+  "endpoint_pattern": "^__GITLAB__/api/v4/projects/\\d+/members$",
+  "required_member_ids": [
+    5,
+    278
+  ],
+  "official_last_event_only": [
+    true,
+    true
+  ],
+  "corrected_last_event_only": [
+    false,
+    false
+  ],
+  "proof_kind": "official_multi-event-contract_and_evaluator_last-event-selection",
+  "observed_har_sha256": "a06b21010d6bf90d59f4e96fb32c3f5806e6db9832ae3b76e59308f7cd9661fb",
+  "observed_project_id": 221
+}
diff --git a/research/trajectory-replay-task743-multi-member-reference.json b/research/trajectory-replay-task743-multi-member-reference.json
new file mode 100644
index 000000000..9dfa1c8d5
--- /dev/null
+++ b/research/trajectory-replay-task743-multi-member-reference.json
@@ -0,0 +1,18 @@
+{
+  "schema": "trajectory_replay_repeated_network_event_reference/1",
+  "task_id": 743,
+  "endpoint_pattern": "^__GITLAB__/api/v4/projects/\\d+/members$",
+  "required_member_ids": [
+    5,
+    278
+  ],
+  "official_last_event_only": [
+    true,
+    true
+  ],
+  "corrected_last_event_only": [
+    false,
+    false
+  ],
+  "proof_kind": "official_multi-event-contract_and_evaluator_last-event-selection"
+}
diff --git a/research/trajectory-replay-task745-multi-member-reference.json b/research/trajectory-replay-task745-multi-member-reference.json
new file mode 100644
index 000000000..8da0a634f
--- /dev/null
+++ b/research/trajectory-replay-task745-multi-member-reference.json
@@ -0,0 +1,21 @@
+{
+  "schema": "trajectory_replay_repeated_network_event_reference/1",
+  "task_id": 745,
+  "endpoint_pattern": "^__GITLAB__/api/v4/projects/\\d+/members$",
+  "required_member_ids": [
+    2367,
+    43,
+    5
+  ],
+  "official_last_event_only": [
+    true,
+    true,
+    true
+  ],
+  "corrected_last_event_only": [
+    false,
+    false,
+    false
+  ],
+  "proof_kind": "official_multi-event-contract_and_evaluator_last-event-selection"
+}
diff --git a/research/trajectory-replay-task746-multi-member-reference.json b/research/trajectory-replay-task746-multi-member-reference.json
new file mode 100644
index 000000000..00cbb3e2d
--- /dev/null
+++ b/research/trajectory-replay-task746-multi-member-reference.json
@@ -0,0 +1,21 @@
+{
+  "schema": "trajectory_replay_repeated_network_event_reference/1",
+  "task_id": 746,
+  "endpoint_pattern": "^__GITLAB__/api/v4/projects/\\d+/members$",
+  "required_member_ids": [
+    2367,
+    43,
+    5
+  ],
+  "official_last_event_only": [
+    true,
+    true,
+    true
+  ],
+  "corrected_last_event_only": [
+    false,
+    false,
+    false
+  ],
+  "proof_kind": "official_multi-event-contract_and_evaluator_last-event-selection"
+}
diff --git a/research/trajectory-replay-task759-route-order-reference.json b/research/trajectory-replay-task759-route-order-reference.json
new file mode 100644
index 000000000..24fedb808
--- /dev/null
+++ b/research/trajectory-replay-task759-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 759,
+  "task_intent": "Show on the map the route and driving time from the city where my E-commerce customer Sophia Young lives to New York City. (Use the OSRM direction service.)",
+  "official_expected_url": "^.*/route/v1/.*/-71.060511,42.3554334;-1.4869496,55.0252998.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-1.4869496,55.0252998;-71.060511,42.3554334.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-1.4869496,55.0252998;-71.060511,42.3554334?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "75953e4f602a68e2c73435a74e09357b9ddb7359ca15b98f65762d503b5a63da",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task760-route-order-reference.json b/research/trajectory-replay-task760-route-order-reference.json
new file mode 100644
index 000000000..78d73759b
--- /dev/null
+++ b/research/trajectory-replay-task760-route-order-reference.json
@@ -0,0 +1,16 @@
+{
+  "schema": "trajectory_replay_route_event_reference/1",
+  "task_id": 760,
+  "task_intent": "Show on the map the route and driving time from Allentown, PA to the city where my E-commerce customer Amanda Kim lives. (Use the OSRM direction service.)",
+  "official_expected_url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.4041622,40.0757384.*$",
+  "corrected_expected_url": "^.*/route/v1/.*/-74.4041622,40.0757384;-75.4716115,40.6022552.*$",
+  "observed_correct_direction_request": "http://grfdfd7b-8080-ye7izl3p.dev.lexmount.net/osrm/routed-car/route/v1/driving/-74.4041622,40.0757384;-75.4716115,40.6022552?overview=false&geometries=polyline&steps=true",
+  "observed_har_sha256": "a3c43f0cb068e0f61dc8c38c9a74c374e38451d1b04a1de3cdd565f2e986c8b8",
+  "finding": "The official evaluator selects only the HTML navigation even though its expected URL describes the later OSRM route XHR, and its coordinate pairs reverse the task’s origin and destination.",
+  "official_navigation_only": true,
+  "corrected_navigation_only": false,
+  "observed_event_sequence": [
+    "GET /directions (HTML navigation)",
+    "GET /osrm/routed-car/route/v1/driving/{origin};{destination} (route XHR, HTTP 200)"
+  ]
+}
diff --git a/research/trajectory-replay-task9-map-live-reference.json b/research/trajectory-replay-task9-map-live-reference.json
new file mode 100644
index 000000000..4ba8d5163
--- /dev/null
+++ b/research/trajectory-replay-task9-map-live-reference.json
@@ -0,0 +1,40 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 9,
+  "task_intent": "Get the name, state, and zip code of all international airports that are within a driving distance of 30 km to Carnegie Art Museum. Return a list of objects with keys \"name\", \"state\", and \"postcode\". (Use the OSRM direction service.)",
+  "source_sweep": "map-live-sweep-v129",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "0db2ea9f39116af859f83dac60c378f365dbd22df3c53906c263876de46aa2a6",
+    "actual_response": "15259d424bd360178fbe18c7b2fc1d5d8de3fe1e9c9f8b5518fd453a2c964dbd",
+    "evaluation": "5504441711b60b4d4d7eba11a888a98a326777a8866963dd985ecc9c065f9199",
+    "trace": "49166abd182bf4fa34c8a3d7a080e5c12ccb412e097e2d55f00274a1f4bd53cb",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "4593009ffeed11944c0e43084086d50b1406bae69767c9519f13da88931000e7"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": []
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data",
+      "official": [
+        {
+          "name": "Pittsburgh International Airport",
+          "state": "Pennsylvania",
+          "postcode": "15231"
+        }
+      ],
+      "corrected": []
+    }
+  ],
+  "derivation": {
+    "kind": "bounded_nominatim_osrm/1"
+  },
+  "regression_ids": [
+    "tr-9-accept-live-map-reference",
+    "tr-9-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/research/trajectory-replay-task90-map-live-reference.json b/research/trajectory-replay-task90-map-live-reference.json
new file mode 100644
index 000000000..0b077bc21
--- /dev/null
+++ b/research/trajectory-replay-task90-map-live-reference.json
@@ -0,0 +1,41 @@
+{
+  "schema": "trajectory_replay_map_live_reference/1",
+  "task_id": 90,
+  "task_intent": "Get the relation IDs of each US state that borders Pennsylvania. Return a list of integers only, without any additional details.",
+  "source_sweep": "map-live-sweep-v127",
+  "browser_call_count": 1,
+  "source_evidence_sha256": {
+    "network_har": "95d065623d799b3ccc3861f9cd59043caed08a65643e85041fc2d8c3213fb085",
+    "actual_response": "0ec650fdffdb303f92ed0e4b8f9596af8a82eca7ba4e3721811bac2522faa763",
+    "evaluation": "a09aa2fcc0b8c07cb8e8c35662f60d3c95a1214ef13f3ea02e7183e42949eec5",
+    "trace": "95d019ea87ecfdb28ae2d86b37e40c03c5d8bf389e4d25165eb6b98497000bb4",
+    "readiness": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
+    "outputs": "261ba4e597a83fe5a19989afe133be6b9d6e68a79c37b4c7eb8fa6c3beb8a2d2"
+  },
+  "observed_response": {
+    "task_type": "retrieve",
+    "status": "SUCCESS",
+    "retrieved_data": [
+      162061,
+      162112,
+      61320,
+      224951,
+      162110,
+      162068
+    ]
+  },
+  "contract_values": [
+    {
+      "path": "/eval/0/expected/retrieved_data/2",
+      "official": 175905,
+      "corrected": 61320
+    }
+  ],
+  "derivation": {
+    "kind": "frozen_wikipedia_relation_ids/1"
+  },
+  "regression_ids": [
+    "tr-90-accept-live-map-reference",
+    "tr-90-reject-nearby-wrong-map-reference"
+  ]
+}
diff --git a/scripts/audit_webarena_catalog_acceptance.py b/scripts/audit_webarena_catalog_acceptance.py
new file mode 100644
index 000000000..962e909c2
--- /dev/null
+++ b/scripts/audit_webarena_catalog_acceptance.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Audit WebArena catalog results through the shared harness."""
+
+from harness.trajectory_replay.catalog_acceptance import main
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/audit_webarena_catalog_actions.py b/scripts/audit_webarena_catalog_actions.py
new file mode 100644
index 000000000..79303a4b5
--- /dev/null
+++ b/scripts/audit_webarena_catalog_actions.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Audit evaluate syntax and verify the catalog's action override data."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from harness.trajectory_replay.catalog_action_audit import (
+    scan_function_body_actions,
+    validate_function_body_overrides,
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--catalog", type=Path, required=True)
+    parser.add_argument("--portable-suite", type=Path, required=True)
+    parser.add_argument("--action-overrides", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.output.exists():
+        raise FileExistsError(args.output)
+    audit = scan_function_body_actions(args.catalog, args.portable_suite)
+    validate_function_body_overrides(audit, args.action_overrides)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(audit, indent=2) + "\n", encoding="utf-8")
+    print(
+        json.dumps(
+            {
+                "evaluate_actions": audit["evaluate_action_count"],
+                "function_body_findings": len(audit["function_body_findings"]),
+            }
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/build_revised_webarena_dataset.py b/scripts/build_revised_webarena_dataset.py
index 9d86cd818..8187d37b2 100644
--- a/scripts/build_revised_webarena_dataset.py
+++ b/scripts/build_revised_webarena_dataset.py
@@ -28,6 +28,8 @@ def revise(
     corrections_path: Path,
     task_ids: list[int],
     selection_path: Path | None = None,
+    *,
+    require_current_upstream: bool = True,
 ) -> dict[str, Any]:
     if not task_ids or len(task_ids) != len(set(task_ids)):
         raise ValueError("revision task IDs must be nonempty and unique")
@@ -56,7 +58,10 @@ def revise(
         base["selection_sha256"] = _sha256(selection_path)
         base["counts"]["verified_tasks"] = len(selected_ids)
         base["counts"]["verified_trajectories"] = len(selected_ids)
-    catalog = load_corrections(corrections_path)
+    catalog = load_corrections(
+        corrections_path,
+        require_current_upstream=require_current_upstream,
+    )
     if not set(task_ids) - {67, 243} <= set(catalog.records):
         raise ValueError("missing reviewed correction")
     benchmark = _configure_benchmark()
diff --git a/scripts/build_webarena_catalog_comparison_inputs.py b/scripts/build_webarena_catalog_comparison_inputs.py
new file mode 100644
index 000000000..d8238730d
--- /dev/null
+++ b/scripts/build_webarena_catalog_comparison_inputs.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Build shared report inputs from full-catalog acceptance ledgers."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from harness.trajectory_replay.catalog_comparison import (
+    build_catalog_comparison_dataset,
+    build_catalog_comparison_results,
+)
+from harness.trajectory_replay.catalog_contracts import file_sha256
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, required=True)
+    parser.add_argument("--dataset-output", type=Path, required=True)
+    parser.add_argument("--results-output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.dataset_output.exists() or args.results_output.exists():
+        raise FileExistsError("catalog comparison output already exists")
+    config = json.loads(args.config.read_text(encoding="utf-8"))
+    dataset = build_catalog_comparison_dataset(
+        catalog_path=Path(config["catalog_path"]),
+        suite_path=Path(config["suite_path"]),
+        site_map_path=Path(config["site_map_path"]),
+        action_overrides_path=(
+            Path(config["action_overrides_path"])
+            if config.get("action_overrides_path")
+            else None
+        ),
+    )
+    args.dataset_output.parent.mkdir(parents=True, exist_ok=True)
+    args.dataset_output.write_text(
+        json.dumps(dataset, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+    )
+    results = build_catalog_comparison_results(
+        dataset=dataset,
+        dataset_sha256=file_sha256(args.dataset_output),
+        profile_sha256=file_sha256(Path(config["profile_path"])),
+        scope_sha256=file_sha256(Path(config["scope_path"])),
+        site_map_sha256=file_sha256(Path(config["site_map_path"])),
+        participants=config["participants"],
+    )
+    args.results_output.parent.mkdir(parents=True, exist_ok=True)
+    args.results_output.write_text(
+        json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+    )
+    print(
+        json.dumps(
+            {
+                "tasks": len(dataset["trajectories"]),
+                "logical_results": len(results["attempts"]),
+                "physical_executions": len(results["physical_executions"]),
+                "dataset_sha256": file_sha256(args.dataset_output),
+                "results_sha256": file_sha256(args.results_output),
+            }
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/build_webarena_catalog_report.py b/scripts/build_webarena_catalog_report.py
new file mode 100644
index 000000000..018c736a4
--- /dev/null
+++ b/scripts/build_webarena_catalog_report.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Build the full WebArena catalog report from accepted evidence."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from harness.trajectory_replay.webarena_report import build_catalog_acceptance_report
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--evidence-root", type=Path, required=True)
+    parser.add_argument("--ledger", type=Path, required=True)
+    parser.add_argument("--scope", type=Path, required=True)
+    parser.add_argument("--catalog", type=Path, required=True)
+    parser.add_argument("--portable-suite", type=Path, required=True)
+    parser.add_argument("--site-map", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--archive", type=Path, required=True)
+    args = parser.parse_args()
+    result = build_catalog_acceptance_report(
+        evidence_root=args.evidence_root,
+        ledger_path=args.ledger,
+        scope_path=args.scope,
+        catalog_path=args.catalog,
+        suite_path=args.portable_suite,
+        site_map_path=args.site_map,
+        output=args.output,
+        archive=args.archive,
+    )
+    print(json.dumps(result, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/build_webarena_replay_report.py b/scripts/build_webarena_replay_report.py
index f8aec2bc6..13c7f5eae 100644
--- a/scripts/build_webarena_replay_report.py
+++ b/scripts/build_webarena_replay_report.py
@@ -10,50 +10,83 @@
 from pathlib import Path
 
 from harness.reporting.assets import copy_bound_assets
-from harness.web_fetch.storage import data_home
 from harness.trajectory_replay.webarena_report import build_webarena_replay_report
-
+from harness.web_fetch.storage import data_home
 
 PROFILE = Path("evaluation/trajectory-replay/webarena-verified/current-run.json")
 
 
+def _verified_dataset_path(path: Path, digest: str) -> Path:
+    if (
+        path.is_symlink()
+        or any(parent.is_symlink() for parent in path.parents)
+        or not path.is_file()
+    ):
+        raise ValueError("bound corrected dataset is missing or symlinked")
+    if hashlib.sha256(path.read_bytes()).hexdigest() != digest:
+        raise ValueError("bound corrected dataset changed")
+    return path
+
+
 def _bound_dataset_path(profile: dict) -> Path:
     dataset = profile.get("dataset") or {}
     path_value = dataset.get("path")
     digest = dataset.get("sha256")
     if not isinstance(path_value, str) or not path_value or not isinstance(digest, str):
-        raise ValueError("three-method report profile requires a bound corrected dataset")
+        raise ValueError("replay report profile requires a bound corrected dataset")
     path = Path(path_value)
-    if path.is_symlink() or any(parent.is_symlink() for parent in path.parents) or not path.is_file():
-        raise ValueError("bound corrected dataset is missing or symlinked")
-    if hashlib.sha256(path.read_bytes()).hexdigest() != digest:
-        raise ValueError("bound corrected dataset changed")
-    return path
+    return _verified_dataset_path(path, digest)
 
 
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--results", required=True)
+    parser.add_argument("--profile", type=Path, default=PROFILE)
+    parser.add_argument("--dataset", type=Path)
+    parser.add_argument("--review", type=Path)
     parser.add_argument("--output", default="reports/trajectory-replay")
-    parser.add_argument("--content", type=Path, default=data_home() / "report-assets/trajectory-replay",
-                        help="Git-external frozen article and illustrations")
-    parser.add_argument("--include-english", action="store_true", help="Explicitly generate the English report and its assets")
+    parser.add_argument(
+        "--content",
+        type=Path,
+        default=data_home() / "report-assets/trajectory-replay",
+        help="Git-external frozen article and illustrations",
+    )
+    parser.add_argument(
+        "--content-bindings",
+        type=Path,
+        default=Path("research/trajectory-replay-content.json"),
+    )
+    parser.add_argument("--interpretation-audit", type=Path)
+    parser.add_argument(
+        "--include-english",
+        action="store_true",
+        help="Explicitly generate the English report and its assets",
+    )
     args = parser.parse_args()
-    profile = json.loads(PROFILE.read_text(encoding="utf-8"))
-    dataset_path = _bound_dataset_path(profile)
-    bindings = json.loads(Path("research/trajectory-replay-content.json").read_text())
+    profile = json.loads(args.profile.read_text(encoding="utf-8"))
+    dataset_path = args.dataset or _bound_dataset_path(profile)
+    if args.dataset is not None:
+        expected = (profile.get("dataset") or {}).get("sha256")
+        if not isinstance(expected, str):
+            raise ValueError("explicit report dataset requires a profile hash binding")
+        dataset_path = _verified_dataset_path(dataset_path, expected)
+    review_path = args.review or Path("research/trajectory-replay-failure-review.json")
+    bindings = json.loads(args.content_bindings.read_text())
     with tempfile.TemporaryDirectory(prefix="replay-content-") as directory:
         content = Path(directory).resolve()
         copy_bound_assets(args.content, content, bindings)
         manifest = build_webarena_replay_report(
             results_path=args.results,
             dataset_path=dataset_path,
-            review_path="research/trajectory-replay-failure-review.json",
-            profile_path=PROFILE,
+            review_path=review_path,
+            profile_path=args.profile,
             output_dir=args.output,
             include_english=args.include_english,
             interpretation_path=content / "trajectory-replay-interpretation.json",
-            interpretation_audit_path=Path("research/trajectory-replay-contract-audit.json"),
+            interpretation_audit_path=(
+                args.interpretation_audit
+                or Path("research/trajectory-replay-contract-audit.json")
+            ),
         )
     print(json.dumps(manifest, ensure_ascii=False))
 
diff --git a/scripts/check_track_naming.py b/scripts/check_track_naming.py
index 73f11578d..904cd1bb7 100644
--- a/scripts/check_track_naming.py
+++ b/scripts/check_track_naming.py
@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import re
 from pathlib import Path
 
 
@@ -16,7 +17,20 @@
     "evaluation/sources",
 }
 EXCLUDED_PARTS = {".pytest_cache", ".ruff_cache", ".venv", "__pycache__"}
-TEXT_SUFFIXES = {"", ".css", ".html", ".js", ".json", ".md", ".py", ".sh", ".toml", ".txt", ".yaml", ".yml"}
+TEXT_SUFFIXES = {
+    "",
+    ".css",
+    ".html",
+    ".js",
+    ".json",
+    ".md",
+    ".py",
+    ".sh",
+    ".toml",
+    ".txt",
+    ".yaml",
+    ".yml",
+}
 FORBIDDEN = (
     "browser_" + "agent",
     "browser-" + "agent",
@@ -27,14 +41,24 @@
     "Browser-" + "agent",
     "Browser-" + "kernel",
 )
+CATALOG_REPLAY_ENTRYPOINTS = {
+    "scripts/audit_webarena_catalog_acceptance.py",
+    "scripts/audit_webarena_catalog_actions.py",
+    "scripts/build_webarena_catalog_comparison_inputs.py",
+    "scripts/build_webarena_catalog_report.py",
+    "scripts/check_webarena_catalog_release.py",
+    "scripts/consolidate_webarena_catalog.py",
+    "scripts/rebind_webarena_catalog.py",
+    "scripts/run_webarena_catalog.py",
+}
 
 
-def main() -> None:
+def repository_contract_findings(root: Path) -> list[str]:
     findings: list[str] = []
-    for path in ROOT.rglob("*"):
+    for path in root.rglob("*"):
         if not path.is_file() or path.suffix.lower() not in TEXT_SUFFIXES:
             continue
-        relative = path.relative_to(ROOT)
+        relative = path.relative_to(root)
         name = relative.as_posix()
         if EXCLUDED_PARTS.intersection(relative.parts):
             continue
@@ -44,9 +68,74 @@ def main() -> None:
         for term in FORBIDDEN:
             if term in text:
                 findings.append(f"{name}: retired track name {term!r}")
+    catalog_replay_entrypoints = {
+        path.relative_to(root).as_posix()
+        for path in (root / "scripts").glob("*webarena_catalog*.py")
+    }
+    if catalog_replay_entrypoints != CATALOG_REPLAY_ENTRYPOINTS:
+        findings.append(
+            "WebArena catalog replay entry points changed: "
+            f"unexpected={sorted(catalog_replay_entrypoints - CATALOG_REPLAY_ENTRYPOINTS)}, "
+            f"missing={sorted(CATALOG_REPLAY_ENTRYPOINTS - catalog_replay_entrypoints)}"
+        )
+    specialized_replay_entrypoints = sorted(
+        path.relative_to(root).as_posix()
+        for path in (root / "scripts").glob("*webarena*full*.py")
+    )
+    if specialized_replay_entrypoints:
+        findings.append(
+            "dataset size belongs in catalog data, not specialized entry points: "
+            + ", ".join(specialized_replay_entrypoints)
+        )
+    specialized_catalog_modules = sorted(
+        path.relative_to(root).as_posix()
+        for path in (root / "harness" / "trajectory_replay").glob("*full*catalog*.py")
+    )
+    if specialized_catalog_modules:
+        findings.append(
+            "catalog contracts belong in the shared pipeline, not full-dataset modules: "
+            + ", ".join(specialized_catalog_modules)
+        )
+    contracts_path = root / "harness" / "trajectory_replay" / "catalog_contracts.py"
+    schema_implementation_paths = [
+        *(root / "harness" / "trajectory_replay").glob("*.py"),
+        *(root / "scripts").glob("*.py"),
+    ]
+    legacy_schema_prefixes = (
+        "lex_webarena_" + "full_",
+        "lex_webarena_verified_" + "full_",
+    )
+    for path in schema_implementation_paths:
+        if path == contracts_path:
+            continue
+        text = path.read_text(encoding="utf-8", errors="ignore")
+        if any(prefix in text for prefix in legacy_schema_prefixes):
+            findings.append(
+                f"{path.relative_to(root).as_posix()}: legacy full schemas are read-only contracts"
+            )
+    makefile = root / "Makefile"
+    if makefile.is_file() and re.search(
+        r"(?m)^[A-Za-z0-9_.-]*webarena-full[A-Za-z0-9_.-]*\s*:",
+        makefile.read_text(encoding="utf-8"),
+    ):
+        findings.append("Makefile: catalog release targets must not encode one dataset size")
+    repair_programs = sorted(
+        path.relative_to(root).as_posix()
+        for path in (root / "scripts").glob("repair_webarena_*.py")
+    )
+    if repair_programs:
+        findings.append(
+            "case repair belongs in catalog data and the shared replay pipeline, not programs: "
+            + ", ".join(repair_programs)
+        )
+    return findings
+
+
+def main() -> None:
+    findings = repository_contract_findings(ROOT)
     if findings:
         raise SystemExit("\n".join(findings))
-    print("Track naming check passed")
+    print("Track contract checks passed")
 
 
 if __name__ == "__main__":
diff --git a/scripts/check_webarena_catalog_release.py b/scripts/check_webarena_catalog_release.py
new file mode 100644
index 000000000..aaafe8448
--- /dev/null
+++ b/scripts/check_webarena_catalog_release.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Validate a WebArena catalog release through the shared harness."""
+
+from harness.trajectory_replay.catalog_release import main
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/check_webarena_replay_report.py b/scripts/check_webarena_replay_report.py
index 341983663..dab402aa8 100644
--- a/scripts/check_webarena_replay_report.py
+++ b/scripts/check_webarena_replay_report.py
@@ -12,13 +12,20 @@
 import re
 from html.parser import HTMLParser
 from pathlib import Path
-
-from harness.trajectory_replay.report_languages import language_roots, report_languages
 from urllib.parse import parse_qs, unquote, urlsplit
 
 from bs4 import BeautifulSoup
 
-from harness.trajectory_replay.webarena_report import LOCAL_PATH_RE
+from harness.trajectory_replay.report_languages import language_roots, report_languages
+from harness.trajectory_replay.webarena_report import LOCAL_PATH_RE, WEB_URL_RE
+
+
+def _contains_local_path_outside_web_url(text: str) -> bool:
+    url_spans = [match.span() for match in WEB_URL_RE.finditer(text)]
+    return any(
+        not any(start <= match.start() and match.end() <= end for start, end in url_spans)
+        for match in LOCAL_PATH_RE.finditer(text)
+    )
 
 
 def _text_without_verified_embedded_images(text: str, image_digests: set[str]) -> str:
@@ -35,7 +42,11 @@ def replace(match):
     return re.sub(r'(?:src|href)="data:image/png;base64,([^"]+)"', replace, text)
 
 
-def _check_interpretation(root: Path) -> None:
+def _check_interpretation(
+    root: Path,
+    results: list[dict] | None = None,
+    participants: list[str] | None = None,
+) -> None:
     sections = []
     for language_root in language_roots(root):
         integrated = BeautifulSoup((language_root / "interpretation.html").read_text(), "html.parser")
@@ -52,6 +63,32 @@ def _check_interpretation(root: Path) -> None:
             raise ValueError("article needs illustrations embedded in its offline view")
         if not integrated.select('a[href="article.html"][download]'):
             raise ValueError("article download is missing")
+        if any("browser-effect-pipeline" in image.get("src", "") for image in article.select("img")):
+            raise ValueError("interpretation retains the non-data-bearing pipeline image")
+        authored_sections = article.select(":scope > section[id]")
+        if not authored_sections or any("presentation-slide" not in section.get("class", []) for section in authored_sections):
+            raise ValueError("interpretation sections are not presentation-ready")
+        if results is not None and participants is not None:
+            failing = [
+                participant for participant in participants
+                if any(row["participant_id"] == participant and row["status"] != "pass" for row in results)
+            ]
+            profiles = article.select("[data-browser-failure-profile]")
+            if [profile["data-browser-failure-profile"] for profile in profiles] != failing:
+                raise ValueError("interpretation must contain one ordered profile per failing participant")
+            for profile in profiles:
+                participant = profile["data-browser-failure-profile"]
+                expected = {
+                    row["cause_code"] for row in results
+                    if row["participant_id"] == participant and row["status"] != "pass"
+                }
+                bars = {item["data-cause"] for item in profile.select(".failure-bars [data-cause]")}
+                details = {item["data-cause-detail"] for item in profile.select("[data-cause-detail]")}
+                if bars != expected or details != expected or any(
+                    detail.select_one('a[href^="result/"]') is None
+                    for detail in profile.select("[data-cause-detail]")
+                ):
+                    raise ValueError("browser failure profile is incomplete or lacks evidence links")
         online_links = article.select('[data-failure-matrix] a')
         offline_links = standalone.select('[data-failure-matrix] a')
         if len(online_links) != len(offline_links):
@@ -362,8 +399,10 @@ def _check_result_detail_semantics(text: str, evidence: dict, status: str) -> No
         or html.escape(str(failure["message"]), quote=True) not in text
         or (failure.get("stage") in {"official_evaluation", "side_effect_not_observed"}
             and not evidence.get("official_evaluation"))
-        or (failure.get("stage") == "browser_tool_execution"
-            and not evidence.get("trajectory"))
+        or (
+            failure.get("stage") == "browser_tool_execution"
+            and not isinstance(evidence.get("trajectory"), list)
+        )
     ):
         raise ValueError("failure detail lacks exact error or evaluation logs")
 
@@ -413,6 +452,17 @@ def _check_failure_matrix(page: str, results: list[dict], participants: list[str
                 raise ValueError("failure matrix inventory anchor is missing")
 
 
+def _check_formal_execution_scope(manifest: dict) -> None:
+    """Require method-level reset without hard-coding a benchmark split size."""
+
+    if manifest.get("formal_reference_validity") is not True:
+        return
+    if int(manifest.get("task_count") or 0) < 1 or manifest.get(
+        "site_reset_between_tasks"
+    ) is not False:
+        raise ValueError("formal report requires a complete method-reset task cohort")
+
+
 def check(report_dir: Path) -> dict[str, int]:
     root = report_dir.resolve()
     manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8"))
@@ -435,6 +485,8 @@ def check(report_dir: Path) -> dict[str, int]:
     required_manifest_fields = (
         "task_count",
         "participant_count",
+        "browser_count",
+        "method_count",
         "participants",
         "logical_results",
         "failed_results",
@@ -448,10 +500,18 @@ def check(report_dir: Path) -> dict[str, int]:
     participants = manifest.get("participants") or []
     task_count = int(manifest.get("task_count") or 0)
     participant_count = int(manifest.get("participant_count") or 0)
+    browser_count = int(manifest.get("browser_count") or 0)
+    method_count = int(manifest.get("method_count") or 0)
+    participant_engines = {
+        str(row["participant_id"]): str(row["engine"]) for row in results
+    }
     expected_results = task_count * participant_count
     if (
         task_count < 1
         or participant_count < 2
+        or method_count != participant_count
+        or browser_count != len(set(participant_engines.values()))
+        or not 1 <= browser_count <= method_count
         or len(participants) != participant_count
         or len(set(participants)) != participant_count
         or "historical_participants" in manifest
@@ -462,6 +522,23 @@ def check(report_dir: Path) -> dict[str, int]:
         raise ValueError(
             "report must contain the complete manifest-declared participant-task matrix"
         )
+    home = (root / "index.html").read_text(encoding="utf-8")
+    if f"{browser_count} 款浏览器组成 {method_count} 种方法配置" not in home:
+        raise ValueError("report home does not state the browser and method populations")
+    home_soup = BeautifulSoup(home, "html.parser")
+    signature = home_soup.select_one("[data-failure-signatures]")
+    failing_participants = [
+        participant for participant in participants
+        if any(row["participant_id"] == participant and row["status"] != "pass" for row in results)
+    ]
+    if (failing_participants and signature is None) or (
+        signature is not None
+        and [card["data-method"] for card in signature.select("[data-method]")]
+        != failing_participants
+    ):
+        raise ValueError("home must show one compact failure signature per failing participant")
+    if "失败发生在哪个环节" in home or home_soup.select_one("[data-failure-matrix]") is not None:
+        raise ValueError("home duplicates detailed result analysis")
     task_ids = {int(item["task_id"]) for item in results}
     result_keys = {(str(item["participant_id"]), int(item["task_id"])) for item in results}
     expected_keys = {(participant, task_id) for participant in participants for task_id in task_ids}
@@ -479,14 +556,62 @@ def check(report_dir: Path) -> dict[str, int]:
     metrics = json.loads((root / "data/metrics.json").read_text(encoding="utf-8"))
     _check_metric_populations(metrics, results, participants)
     _check_core_comparison(root, metrics, results, participants)
+    has_resource_samples = any(
+        row[metric]["n"]
+        for row in metrics["methods"]
+        for metric in (
+            "cpu",
+            "memory",
+            "memory_peak",
+            "network_rx",
+            "network_tx",
+            "cpu_utilization",
+        )
+    )
+    if not has_resource_samples:
+        for language_root in language_roots(root):
+            page = (language_root / "index.html").read_text(encoding="utf-8")
+            expected = (
+                "本轮没有记录进程级 CPU、内存和网络采样"
+                if language_root == root
+                else "did not record process-level CPU, memory, or network samples"
+            )
+            if expected not in page or "n=0/" in page:
+                raise ValueError(
+                    "unavailable resources must be stated once instead of rendered as empty metrics"
+                )
+    for language_root in language_roots(root):
+        environment = (language_root / "environment.html").read_text(encoding="utf-8")
+        identity_heading = (
+            "方法身份与修复序列"
+            if language_root == root
+            else "Method identity and repair sequence"
+        )
+        if identity_heading not in environment:
+            raise ValueError("environment page is missing method identity and repair sequence")
     for language_root in language_roots(root):
-        _check_failure_matrix((language_root / "results.html").read_text(encoding="utf-8"), results, participants)
+        results_page = (language_root / "results.html").read_text(encoding="utf-8")
+        _check_failure_matrix(results_page, results, participants)
+        if "successful-distributions" not in results_page or (
+            "Playwright 动作耗时" not in results_page
+            and "Playwright action latency" not in results_page
+        ):
+            raise ValueError("result analysis is missing its detailed latency distributions")
+    _check_formal_execution_scope(manifest)
     if manifest.get("formal_reference_validity") is True:
-        if task_count != 90 or manifest.get("site_reset_between_tasks") is not False:
-            raise ValueError("formal report requires the complete method-reset 90-task cohort")
+        if manifest.get("execution_backend") != "playwright":
+            raise ValueError("formal replay report must bind the Playwright backend")
         for language_root in language_roots(root):
             environment = (language_root / "environment.html").read_text(encoding="utf-8")
-            if "Rainbond" not in environment or "Colima" in environment:
+            if (
+                "Rainbond" not in environment
+                or "Colima" in environment
+                or "Playwright" not in environment
+                or any(
+                    str(row["browser"]) not in environment
+                    for row in results
+                )
+            ):
                 raise ValueError("formal environment page does not describe the measured deployment")
         for participant in participants:
             for task_id in task_ids:
@@ -497,7 +622,7 @@ def check(report_dir: Path) -> dict[str, int]:
     if metrics.get("affected36_methods"):
         raise ValueError("report must not contain a diagnostic-only comparison")
     if any((root / route).exists() for route in ("diagnostics.html", "fixes.html", "data/diagnostic-closure.json")):
-        raise ValueError("three-method report contains a separate diagnostic appendix")
+        raise ValueError("replay report contains a separate diagnostic appendix")
     failures = [item for item in results if item["status"] != "pass"]
     if manifest["failed_results"] != len(failures) or manifest["reviewed_failures"] != len(
         failures
@@ -579,7 +704,7 @@ def check(report_dir: Path) -> dict[str, int]:
         "evidence-viewer.html",
     )
     if manifest.get("interpretation"):
-        _check_interpretation(root)
+        _check_interpretation(root, results, participants)
         core_names += ("interpretation.html", "article.html")
     core, chinese_details, english_details = _semantic_page_inventory(root, core_names)
     forbidden = (
@@ -652,7 +777,9 @@ def check(report_dir: Path) -> dict[str, int]:
         for path in root.rglob("*")
         if path.is_file() and path.suffix in {".html", ".json", ".js", ".css"}
     )
-    if LOCAL_PATH_RE.search(distributable) or re.search(r"sk-[A-Za-z0-9_-]{12,}", distributable):
+    if _contains_local_path_outside_web_url(distributable) or re.search(
+        r"(? dict[str, int]:
                 "review history must describe review rounds, not enumerate per-result audits"
             )
     js = (root / "assets/app.js").read_text(encoding="utf-8")
+    if (
+        "const enhanceTables=()=>" not in js
+        or "input.dataset.tableFilter" not in js
+        or "table.dataset.enhanced='true'" not in js
+    ):
+        raise ValueError("report tables must expose reusable sorting and filtering controls")
     if "const task=params.get('task');" not in js or "row.dataset.task===task" not in js:
         raise ValueError("dataset-to-results drilldown must select an exact task ID")
     if (
diff --git a/scripts/check_webarena_replay_visual.py b/scripts/check_webarena_replay_visual.py
index 6a9083344..7f4d63938 100644
--- a/scripts/check_webarena_replay_visual.py
+++ b/scripts/check_webarena_replay_visual.py
@@ -286,7 +286,10 @@ def check(report: Path, receipt: Path) -> None:
                         timeout=5_000,
                     )
                     assert route_page.locator("[data-step-actual]").inner_text()
-                    assert route_page.locator("[data-step-result]").inner_text()
+                    assert (
+                        route_page.locator("[data-step-result]").inner_text()
+                        or route_page.locator("[data-step-error]").inner_text()
+                    )
                     route_page.locator("[data-close-step]").click()
                     if (report / "interpretation.html").is_file():
                         route_page.locator("nav a[data-report-interpretation]").click()
@@ -336,7 +339,10 @@ def check(report: Path, receipt: Path) -> None:
                         dialog_box = modal_page.locator("#step-dialog").bounding_box()
                         assert dialog_box and dialog_box["x"] >= 0
                         assert dialog_box["x"] + dialog_box["width"] <= 390
-                        assert modal_page.locator("[data-step-result]").inner_text()
+                        assert (
+                            modal_page.locator("[data-step-result]").inner_text()
+                            or modal_page.locator("[data-step-error]").inner_text()
+                        )
                         modal_page.locator("#step-dialog").screenshot(
                             path=str(screenshot_dir / f"step-dialog-{language}.png")
                         )
@@ -356,6 +362,9 @@ def check(report: Path, receipt: Path) -> None:
                     fix_page.locator("h1").wait_for()
                     fix_page.close()
                 results = json.loads((report / "data/results.json").read_text(encoding="utf-8"))
+                expected_failure_profiles = len({
+                    item["participant_id"] for item in results if item["status"] != "pass"
+                })
                 passed_key = next((item["key"] for item in results if item["status"] == "pass"), None)
                 if passed_key is not None:
                     passed_page = browser.new_page()
@@ -459,6 +468,34 @@ def check(report: Path, receipt: Path) -> None:
                                 ).last.get_attribute("href")
                                 page.locator('[data-dataset-filter="q"]').fill("")
                                 page.locator('[data-dataset-filter="type"]').select_option("")
+                            if route.endswith("index.html") and not route.startswith("en/"):
+                                metrics = page.locator("table.metrics[data-enhanced=true]")
+                                metrics.wait_for(timeout=5_000)
+                                labels = metrics.locator("tbody tr td:first-child")
+                                original = labels.all_inner_texts()
+                                expected = page.evaluate(
+                                    "items => [...items].sort((left, right) => "
+                                    "left.toLocaleLowerCase().localeCompare("
+                                    "right.toLocaleLowerCase(), 'zh'))",
+                                    original,
+                                )
+                                metrics.locator("thead .table-sort").first.click()
+                                ascending = labels.all_inner_texts()
+                                metrics.locator("thead .table-sort").first.click()
+                                descending = labels.all_inner_texts()
+                                assert ascending == expected
+                                assert descending == list(reversed(expected))
+                                controls_id = metrics.get_attribute("data-table-controls")
+                                assert controls_id is not None
+                                table_filter = page.locator(
+                                    f'[data-table-controls="{controls_id}"] [data-table-filter]'
+                                )
+                                table_filter.fill("Kitesurf")
+                                assert metrics.locator("tbody tr:not([hidden])").count() == 1
+                                assert "Kitesurf" in metrics.locator(
+                                    "tbody tr:not([hidden]) td:first-child"
+                                ).inner_text()
+                                table_filter.fill("")
                             if route.endswith("diagnostics.html"):
                                 page.wait_for_function(
                                     "() => document.querySelector('[data-closure-count]')"
@@ -506,6 +543,11 @@ def check(report: Path, receipt: Path) -> None:
                                     timeout=5_000
                                 )
                                 page.locator(".dialog-head button").click()
+                            if route.endswith("index.html") and not route.startswith("en/"):
+                                assert page.locator("[data-failure-signatures] [data-method]").count() == expected_failure_profiles
+                            if route.endswith(("interpretation.html", "article.html")):
+                                assert page.locator("[data-browser-failure-profile]").count() == expected_failure_profiles
+                                assert page.locator("[data-browser-failure-profile] [data-cause-detail] a").count() > expected_failure_profiles
                             if route.endswith("article.html"):
                                 # Script-free exports follow the OS preference, not
                                 # the dashboard's JavaScript theme attribute.
@@ -521,7 +563,8 @@ def check(report: Path, receipt: Path) -> None:
 const root=document.documentElement;
 const clipped=[];
 const layoutSelector='h1,h2,.filters input,.filters select,'+
-  '.diagnosis-grid article,.summary-grid article,.fact-strip>div';
+  '.diagnosis-grid article,.summary-grid article,.fact-strip>div,'+
+  '.failure-signature-grid article,.failure-explanations article,.failure-bars li';
 for(const element of document.querySelectorAll(layoutSelector)){
   if(!element.getClientRects().length)continue;
   const box=element.getBoundingClientRect();
@@ -588,6 +631,8 @@ def check(report: Path, receipt: Path) -> None:
                                     "index.html",
                                     "results.html",
                                     "environment.html",
+                                    "interpretation.html",
+                                    "article.html",
                                 }
                                 or route.startswith(("result/", "task/"))
                                 or route.endswith("diagnostics.html")
diff --git a/scripts/consolidate_webarena_catalog.py b/scripts/consolidate_webarena_catalog.py
new file mode 100644
index 000000000..55ec312f0
--- /dev/null
+++ b/scripts/consolidate_webarena_catalog.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Consolidate accepted WebArena catalog evidence through the shared harness."""
+
+from harness.trajectory_replay.catalog_consolidation import main
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/package_webarena_replay_artifact_site.py b/scripts/package_webarena_replay_artifact_site.py
index f11989f5a..df075403e 100644
--- a/scripts/package_webarena_replay_artifact_site.py
+++ b/scripts/package_webarena_replay_artifact_site.py
@@ -9,6 +9,7 @@
 import zipfile
 from pathlib import Path
 
+from harness.trajectory_replay.report_compaction import validate_compact_report
 from harness.trajectory_replay.webarena_report import require_all_reference_validity
 
 if __package__:
@@ -68,12 +69,21 @@ def package(
     evidence: Path,
     output: Path,
     visual_receipt: Path,
+    *,
+    package_report: Path | None = None,
+    profile_path: Path | None = None,
+    review_path: Path | None = None,
+    attestation_path: Path | None = None,
 ) -> dict:
+    packaged = package_report or report
+    profile_path = profile_path or PROFILE
+    review_path = review_path or REVIEW
+    attestation_path = attestation_path or ATTESTATION
     if not evidence.is_file():
         raise ValueError(f"missing replay evidence: {evidence}")
     summary = validate_release(report, visual_receipt)
     report_manifest = json.loads((report / "report-manifest.json").read_text(encoding="utf-8"))
-    profile = json.loads(PROFILE.read_text(encoding="utf-8"))
+    profile = json.loads(profile_path.read_text(encoding="utf-8"))
     dataset_path = _bound_dataset_path(profile)
     if (profile.get("scoring") or {}).get("require_all_reference_validity") is True:
         require_all_reference_validity(
@@ -87,14 +97,20 @@ def package(
     ):
         raise ValueError("report and publication inputs refer to different result or dataset bytes")
     if report_manifest.get("diagnostic_closure_sha256"):
-        raise ValueError("diagnostic-only appendices cannot enter the three-method release")
-    files = sorted(path for path in report.rglob("*") if path.is_file())
+        raise ValueError("diagnostic-only appendices cannot enter the replay release")
+    if packaged != report:
+        compact = validate_compact_report(packaged, max_files=MAX_FILES)
+        if compact["task_routes"] != report_manifest["task_count"] or compact[
+            "result_routes"
+        ] != report_manifest["logical_results"]:
+            raise ValueError("compact publication routes differ from the validated report")
+    files = sorted(path for path in packaged.rglob("*") if path.is_file())
     if not files or len(files) > MAX_FILES:
         raise ValueError(f"Artifact Site file count is outside 1..{MAX_FILES}: {len(files)}")
     output.parent.mkdir(parents=True, exist_ok=True)
     with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
         for path in files:
-            relative = path.relative_to(report)
+            relative = path.relative_to(packaged)
             if path.is_symlink() or SKIP_PARTS.intersection(relative.parts):
                 raise ValueError(f"unsafe publication path: {relative}")
             if any(part.startswith(".") for part in relative.parts):
@@ -109,15 +125,18 @@ def package(
         "archive_bytes": output.stat().st_size,
         "report": str(report),
         "report_tree_sha256": tree_sha256(report),
+        "packaged_report": str(packaged),
+        "packaged_report_tree_sha256": tree_sha256(packaged),
         "file_count": len(files),
-        "profile_sha256": sha256(PROFILE),
+        "profile_sha256": sha256(profile_path),
         "dataset_sha256": sha256(dataset_path),
-        "review_sha256": sha256(REVIEW),
+        "review_sha256": sha256(review_path),
         "evidence": str(evidence),
         "evidence_sha256": sha256(evidence),
         **summary,
     }
-    ATTESTATION.write_text(
+    attestation_path.parent.mkdir(parents=True, exist_ok=True)
+    attestation_path.write_text(
         json.dumps(attestation, ensure_ascii=False, indent=2) + "\n",
         encoding="utf-8",
     )
@@ -127,9 +146,13 @@ def package(
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--report", type=Path, default=Path("reports/trajectory-replay"))
+    parser.add_argument("--package-report", type=Path)
     parser.add_argument("--evidence", type=Path, required=True)
     parser.add_argument("--output", type=Path, default=ARCHIVE)
     parser.add_argument("--visual-receipt", type=Path, default=VISUAL_RECEIPT)
+    parser.add_argument("--profile", type=Path, default=PROFILE)
+    parser.add_argument("--review", type=Path, default=REVIEW)
+    parser.add_argument("--attestation", type=Path, default=ATTESTATION)
     args = parser.parse_args()
     print(
         json.dumps(
@@ -138,6 +161,10 @@ def main() -> None:
                 args.evidence,
                 args.output,
                 args.visual_receipt,
+                package_report=args.package_report,
+                profile_path=args.profile,
+                review_path=args.review,
+                attestation_path=args.attestation,
             ),
             ensure_ascii=False,
         )
diff --git a/scripts/publish_webarena_replay_artifact_site.py b/scripts/publish_webarena_replay_artifact_site.py
index 6503c2e27..d33901ac8 100644
--- a/scripts/publish_webarena_replay_artifact_site.py
+++ b/scripts/publish_webarena_replay_artifact_site.py
@@ -157,6 +157,7 @@ def validate_package() -> dict:
         raise ValueError("missing validated replay package or attestation")
     attestation = json.loads(ATTESTATION.read_text(encoding="utf-8"))
     report = Path(str(attestation.get("report") or ""))
+    packaged_report = Path(str(attestation.get("packaged_report") or report))
     evidence = Path(str(attestation.get("evidence") or ""))
     profile = json.loads(PROFILE.read_text(encoding="utf-8"))
     dataset_path = _bound_dataset_path(profile)
@@ -170,6 +171,9 @@ def validate_package() -> dict:
         "schema": attestation.get("schema") == "lex_webarena_replay_publication_attestation/1",
         "archive": sha256(ARCHIVE) == attestation.get("archive_sha256"),
         "report": report.is_dir() and tree_sha256(report) == attestation.get("report_tree_sha256"),
+        "packaged_report": packaged_report.is_dir()
+        and tree_sha256(packaged_report)
+        == attestation.get("packaged_report_tree_sha256", attestation.get("report_tree_sha256")),
         "profile": sha256(PROFILE) == attestation.get("profile_sha256"),
         "dataset": sha256(dataset_path) == attestation.get("dataset_sha256"),
         "review": sha256(REVIEW) == attestation.get("review_sha256"),
@@ -599,4 +603,19 @@ def main(*, create: bool) -> None:
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--create", action="store_true")
-    main(create=parser.parse_args().create)
+    parser.add_argument("--archive", type=Path, default=ARCHIVE)
+    parser.add_argument("--attestation", type=Path, default=ATTESTATION)
+    parser.add_argument("--receipt", type=Path, default=RECEIPT)
+    parser.add_argument("--profile", type=Path, default=PROFILE)
+    parser.add_argument("--review", type=Path, default=REVIEW)
+    parser.add_argument("--binding-journal", type=Path, default=BINDING_JOURNAL)
+    parser.add_argument("--operation-journal", type=Path, default=OPERATION_JOURNAL)
+    args = parser.parse_args()
+    ARCHIVE = args.archive
+    ATTESTATION = args.attestation
+    RECEIPT = args.receipt
+    PROFILE = args.profile
+    REVIEW = args.review
+    BINDING_JOURNAL = args.binding_journal
+    OPERATION_JOURNAL = args.operation_journal
+    main(create=args.create)
diff --git a/scripts/rebind_webarena_catalog.py b/scripts/rebind_webarena_catalog.py
new file mode 100644
index 000000000..cbf6fab57
--- /dev/null
+++ b/scripts/rebind_webarena_catalog.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Rebind a WebArena catalog through the shared harness."""
+
+from harness.trajectory_replay.catalog_binding import main
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/reevaluate_webarena_evaluator_transition.py b/scripts/reevaluate_webarena_evaluator_transition.py
new file mode 100644
index 000000000..656abfc17
--- /dev/null
+++ b/scripts/reevaluate_webarena_evaluator_transition.py
@@ -0,0 +1,162 @@
+"""Build a receipt proving old passing evidence still passes the pinned evaluator."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from harness.trajectory_replay.catalog_contracts import file_sha256, verify_physical_evidence
+from harness.trajectory_replay.evaluator_transition import (
+    RECEIPT_SCHEMA,
+    compatible_source,
+    json_sha256,
+    load_transition,
+    reevaluate_pass,
+)
+from harness.trajectory_replay.remote_runtime import remote_benchmark
+from harness.trajectory_replay.webarena_corrections import load_corrections
+from harness.trajectory_replay.webarena_runtime_source import verify_webarena_runtime_source
+
+
+def _old_evaluation_identity(value: str) -> tuple[int, str, str]:
+    parts = value.split("/")
+    if len(parts) != 3 or not parts[0].isdecimal() or any(
+        len(digest) != 64 for digest in parts[1:]
+    ):
+        raise argparse.ArgumentTypeError(
+            "expected TASK_ID/EVALUATION_SHA256/RUN_INPUTS_SHA256"
+        )
+    try:
+        int(parts[1], 16)
+        int(parts[2], 16)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError(
+            "expected TASK_ID/EVALUATION_SHA256/RUN_INPUTS_SHA256"
+        ) from exc
+    return int(parts[0]), parts[1], parts[2]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--transition", type=Path, required=True)
+    parser.add_argument("--scope", type=Path, required=True)
+    parser.add_argument("--site-map", type=Path, required=True)
+    parser.add_argument("--run", type=Path, action="append", required=True)
+    parser.add_argument(
+        "--exclude-old-evaluation",
+        type=_old_evaluation_identity,
+        action="append",
+        default=[],
+        metavar="TASK_ID/EVALUATION_SHA256/RUN_INPUTS_SHA256",
+    )
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    verify_webarena_runtime_source()
+    transition = load_transition(args.transition)
+    scope = json.loads(args.scope.read_text(encoding="utf-8"))
+    sites = json.loads(args.site_map.read_text(encoding="utf-8"))["sites"]
+    benchmark = remote_benchmark(sites)
+    corrections = load_corrections()
+    rows = []
+    identities = set()
+    excluded = set(args.exclude_old_evaluation)
+    observed_excluded = set()
+    for directory in args.run:
+        inputs_path = directory / "inputs.json"
+        run_inputs_sha256 = file_sha256(inputs_path)
+        progress_path = directory / "progress.jsonl"
+        inputs = json.loads(inputs_path.read_text(encoding="utf-8"))
+        source = compatible_source(
+            transition,
+            revision=inputs.get("webarena_revision"),
+            evaluator_fixes_sha256=inputs.get("evaluator_fixes_sha256"),
+        )
+        if source is None:
+            continue
+        for row in map(json.loads, progress_path.read_text(encoding="utf-8").splitlines()):
+            if row.get("status") != "pass":
+                continue
+            evaluation_path = verify_physical_evidence(directory, row)
+            if evaluation_path is None:
+                raise ValueError(f"task {row['task_id']} pass lacks evaluation evidence")
+            old = json.loads(evaluation_path.read_text(encoding="utf-8"))
+            if (
+                old.get("webarena_verified_evaluator_checksum")
+                != source["evaluator_checksum"]
+                or old.get("webarena_verified_data_checksum")
+                != source["data_checksum"]
+            ):
+                raise ValueError(f"task {row['task_id']} has an unexpected old evaluator")
+            identity = (
+                row["task_id"],
+                file_sha256(evaluation_path),
+                run_inputs_sha256,
+            )
+            if identity in excluded:
+                observed_excluded.add(identity)
+                continue
+            current, binding = reevaluate_pass(
+                directory / f"task-{row['task_id']}",
+                task_id=row["task_id"],
+                expected_data_checksum=scope["official_dataset_sha256"],
+                benchmark=benchmark,
+                corrections=corrections,
+            )
+            del current
+            identity = (
+                binding["task_id"],
+                binding["old_evaluation_sha256"],
+                run_inputs_sha256,
+            )
+            if identity in identities:
+                continue
+            identities.add(identity)
+            rows.append(
+                {
+                    "run_inputs_sha256": run_inputs_sha256,
+                    "source": source,
+                    **binding,
+                }
+            )
+    rows.sort(
+        key=lambda row: (
+            row["task_id"],
+            row["old_evaluation_sha256"],
+            row["run_inputs_sha256"],
+        )
+    )
+    if observed_excluded != excluded:
+        raise ValueError("excluded transition evaluations were not found as old passes")
+    receipt = {
+        "schema": RECEIPT_SCHEMA,
+        "transition_sha256": json_sha256(transition),
+        "from_revision": transition["from_revision"],
+        "compatible_from_revisions": transition["compatible_from_revisions"],
+        "compatible_sources": transition["compatible_sources"],
+        "from_evaluator_checksum": transition["from_evaluator_checksum"],
+        "from_data_checksum": transition["from_data_checksum"],
+        "to_revision": transition["to_revision"],
+        "to_evaluator_checksum": transition["to_evaluator_checksum"],
+        "to_data_checksum": transition["to_data_checksum"],
+        "row_count": len(rows),
+        "excluded_old_evaluations": [
+            {
+                "task_id": task_id,
+                "old_evaluation_sha256": digest,
+                "run_inputs_sha256": run_inputs_sha256,
+            }
+            for task_id, digest, run_inputs_sha256 in sorted(excluded)
+        ],
+        "rows": rows,
+    }
+    if args.output.exists():
+        raise ValueError("transition receipt output already exists")
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"rows": len(rows), "output": str(args.output)}))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/run_webarena_catalog.py b/scripts/run_webarena_catalog.py
new file mode 100644
index 000000000..a9154b4f1
--- /dev/null
+++ b/scripts/run_webarena_catalog.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""Replay a frozen WebArena catalog through the shared harness."""
+
+from harness.trajectory_replay.catalog_runner import main
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/verify_webarena_map.py b/scripts/verify_webarena_map.py
index 321fcb871..4211d19a3 100644
--- a/scripts/verify_webarena_map.py
+++ b/scripts/verify_webarena_map.py
@@ -66,12 +66,19 @@ def get(path: str) -> bytes:
         assert name in detail and path.rsplit("/", 1)[1] in detail
         assert "Not Found" not in detail
         checks.append({"feature": "object-detail", "path": path, "status": "pass"})
+    causeway = get("/relation/5475586").decode("utf-8")
+    assert "Lake Pontchartrain Causeway" in causeway and "5475586" in causeway
+    checks.append({
+        "feature": "frozen-route-relation",
+        "path": "/relation/5475586",
+        "status": "pass",
+    })
     try:
-        get("/relation/5475586")
+        get("/relation/999999999999")
     except HTTPError as error:
         assert error.code == 404
     else:
-        raise AssertionError("A missing frozen object must not be fabricated")
+        raise AssertionError("An unknown frozen object must not be fabricated")
     checks.append({"feature": "unavailable-object", "status": "pass"})
     return {"base_url": base, "status": "pass", "checks": checks}
 
diff --git a/tests/browser_use/test_har.py b/tests/browser_use/test_har.py
index 78db55ec8..2c6abc4bd 100644
--- a/tests/browser_use/test_har.py
+++ b/tests/browser_use/test_har.py
@@ -2,6 +2,576 @@
 import json
 
 from harness.browser_use.har import CdpHarRecorder, _har_request, _har_response, _is_json_mime_type
+from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+
+def test_request_bridge_fulfillment_is_recorded_once_with_complete_json(
+    tmp_path, monkeypatch
+) -> None:
+    class EventSource:
+        def __init__(self):
+            self.handlers = {}
+
+        def on(self, event, callback):
+            self.handlers[event] = callback
+
+        def remove_listener(self, *_args):
+            pass
+
+    page = EventSource()
+
+    class Context(EventSource):
+        pages = [page]
+
+        def expose_binding(self, _name, callback):
+            self.capture = callback
+
+        def add_init_script(self, _script):
+            pass
+
+        def route(self, selector, handler):
+            self.selector = selector
+            self.route_handler = handler
+
+    class Request:
+        method = "POST"
+        url = "http://private.example.test/save"
+        headers = {
+            "content-type": "application/json",
+            "x-browser-eval-request-body-id": "capture-1",
+        }
+        post_data = None
+        post_data_buffer = None
+        resource_type = "xhr"
+        failure = None
+
+        def all_headers(self):
+            return dict(self.headers)
+
+    request = Request()
+
+    class HarResponse:
+        status = 201
+        status_text = "Created"
+        headers = {"content-type": "application/json"}
+
+        def __init__(self):
+            self.request = request
+
+        def all_headers(self):
+            return dict(self.headers)
+
+        def body(self):
+            return b'{"saved":true}'
+
+    har_response = HarResponse()
+
+    class BridgeResponse:
+        status_code = 201
+        headers = {
+            "content-type": "application/json",
+            "content-length": "14",
+            "set-cookie": (
+                "PHPSESSID=private; Path=/\r\n"
+                "mage-messages=product%20added; Path=/"
+            ),
+        }
+        content = b'{"saved":true}'
+        url = request.url
+        cookies = []
+
+    class BridgeSession:
+        trust_env = True
+        cookies = []
+
+        def request(self, **kwargs):
+            assert kwargs == {
+                "method": "POST",
+                "url": request.url,
+                "headers": {"content-type": "application/json"},
+                "data": b'{"enabled":true}',
+                "allow_redirects": False,
+                "timeout": 60,
+            }
+            return BridgeResponse()
+
+        def close(self):
+            pass
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", BridgeSession
+    )
+    context = Context()
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = context
+    tool._page = page
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge({"http://private.example.test"})
+    context.capture(
+        {},
+        {
+            "requestId": "capture-1",
+            "method": "POST",
+            "url": request.url,
+            "body": {"kind": "text", "text": '{"enabled":true}'},
+        },
+    )
+    recorder = CdpHarRecorder(
+        capture_transport="playwright",
+        playwright_request_transform=tool.transform_playwright_har_request,
+    )
+    recorder.start(context)
+
+    class Route:
+        def __init__(self):
+            self.request = request
+
+        def fulfill(self, **kwargs):
+            assert kwargs == {
+                "status": 201,
+                "headers": {"content-type": "application/json"},
+                "body": b'{"saved":true}',
+            }
+            page.handlers["request"](request)
+            page.handlers["response"](har_response)
+            page.handlers["requestfinished"](request)
+
+    assert context.selector(request.url) is True
+    context.route_handler(Route())
+    path = tmp_path / "bridged.har"
+    recorder.stop(context, path)
+
+    entries = json.loads(path.read_text())["log"]["entries"]
+    assert len(entries) == 1
+    assert entries[0]["request"]["postData"]["text"] == '{"enabled":true}'
+    assert all(
+        header["name"].lower() != "x-browser-eval-request-body-id"
+        for header in entries[0]["request"]["headers"]
+    )
+    assert entries[0]["response"]["status"] == 201
+    assert entries[0]["response"]["content"]["text"] == '{"saved":true}'
+    assert entries[0]["response"]["cookies"] == [
+        {"name": "mage-messages", "value": "product%20added"}
+    ]
+    assert "private" not in json.dumps(entries[0]["response"])
+    assert tool.request_bridge_stats == {"requests": 1, "failures": 0}
+    assert recorder.errors == []
+
+
+def test_native_form_har_restores_real_redirect_and_safe_cookie(tmp_path) -> None:
+    class Request:
+        method = "POST"
+        url = "http://shop.test/wishlist/index/add/"
+        headers = {
+            "content-type": "application/x-www-form-urlencoded",
+            "x-browser-eval-request-body-id": "native-1",
+        }
+        post_data = None
+
+        def all_headers(self):
+            return dict(self.headers)
+
+    request = Request()
+
+    class Response:
+        status = 200
+        status_text = "OK"
+        headers = {"x-browser-eval-native-form-location": "http://shop.test/wishlist/"}
+
+        def all_headers(self):
+            return dict(self.headers)
+
+        def body(self):
+            return b""
+
+    response = Response()
+    response.request = request
+
+    def transform(method, url, headers):
+        assert (method, url) == (request.method, request.url)
+        assert headers["x-browser-eval-request-body-id"] == "native-1"
+        return (
+            {"content-type": "application/x-www-form-urlencoded"},
+            "product=42",
+            {
+                "status": 302,
+                "statusText": "Found",
+                "headers": {"location": "/wishlist/"},
+                "extra_headers": {
+                    "set-cookie": (
+                        "PHPSESSID=private; Path=/\r\n"
+                        "mage-messages=product%20added; Path=/"
+                    )
+                },
+            },
+        )
+
+    recorder = CdpHarRecorder(
+        capture_transport="playwright",
+        capture_new_pages=False,
+        playwright_request_transform=transform,
+    )
+    key = id(request)
+    recorder._playwright_requests[key] = {
+        "method": request.method,
+        "url": request.url,
+        "headers": dict(request.headers),
+        "postData": None,
+        "__resource_type": "Fetch",
+    }
+    recorder._playwright_request_objects[key] = request
+    recorder._playwright_request_order.append(key)
+    recorder._playwright_responses[key] = (
+        response,
+        {
+            "status": response.status,
+            "statusText": response.status_text,
+            "headers": dict(response.headers),
+            "mimeType": "",
+        },
+    )
+
+    path = tmp_path / "native-form.har"
+    recorder.stop(object(), path)
+    entry = json.loads(path.read_text())["log"]["entries"][0]
+
+    assert entry["request"]["postData"]["text"] == "product=42"
+    assert entry["response"]["status"] == 302
+    assert entry["response"]["statusText"] == "Found"
+    assert entry["response"]["redirectURL"] == "/wishlist/"
+    assert entry["response"]["cookies"] == [
+        {"name": "mage-messages", "value": "product%20added"}
+    ]
+    serialized = json.dumps(entry)
+    assert "x-browser-eval-native-form-location" not in serialized
+    assert "PHPSESSID" not in serialized
+
+
+def test_request_bridge_preserves_redirect_and_browser_generated_get(
+    tmp_path, monkeypatch
+) -> None:
+    class EventSource:
+        def __init__(self):
+            self.handlers = {}
+
+        def on(self, event, callback):
+            self.handlers[event] = callback
+
+        def remove_listener(self, *_args):
+            pass
+
+    page = EventSource()
+
+    class Context(EventSource):
+        pages = [page]
+
+        def expose_binding(self, _name, _callback):
+            pass
+
+        def add_init_script(self, _script):
+            pass
+
+        def route(self, selector, handler):
+            self.selector = selector
+            self.route_handler = handler
+
+    class Request:
+        headers = {}
+        post_data = None
+        post_data_buffer = None
+        resource_type = "document"
+        failure = None
+
+        def __init__(self, method, url):
+            self.method = method
+            self.url = url
+
+        def all_headers(self):
+            return {}
+
+    first_request = Request("POST", "http://private.example.test/start")
+    second_request = Request("GET", "http://private.example.test/target")
+
+    class BridgeResponse:
+        cookies = []
+
+        def __init__(self, request):
+            self.url = request.url
+            if request is first_request:
+                self.status_code = 302
+                self.headers = {"location": "/target", "content-length": "0"}
+                self.content = b""
+            else:
+                self.status_code = 200
+                self.headers = {"content-type": "text/html"}
+                self.content = b"Target"
+
+    class BridgeSession:
+        trust_env = True
+
+        def request(self, **kwargs):
+            assert kwargs["allow_redirects"] is False
+            request = first_request if kwargs["url"].endswith("/start") else second_request
+            return BridgeResponse(request)
+
+        def close(self):
+            pass
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", BridgeSession
+    )
+    context = Context()
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = context
+    tool._page = page
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge({"http://private.example.test"})
+    recorder = CdpHarRecorder(capture_transport="playwright")
+    recorder.start(context)
+
+    class HarResponse:
+        status_text = ""
+
+        def __init__(self, request, status, headers, body):
+            self.request = request
+            self.status = status
+            self.headers = headers
+            self._body = body
+
+        def all_headers(self):
+            return dict(self.headers)
+
+        def body(self):
+            return self._body
+
+    fulfilled = []
+
+    class Route:
+        def __init__(self, request):
+            self.request = request
+
+        def fulfill(self, **kwargs):
+            fulfilled.append(kwargs)
+            page.handlers["request"](self.request)
+            page.handlers["response"](
+                HarResponse(
+                    self.request,
+                    kwargs["status"],
+                    kwargs["headers"],
+                    kwargs["body"],
+                )
+            )
+            page.handlers["requestfinished"](self.request)
+
+    context.route_handler(Route(first_request))
+    context.route_handler(Route(second_request))
+    path = tmp_path / "redirect.har"
+    recorder.stop(context, path)
+
+    entries = json.loads(path.read_text())["log"]["entries"]
+    assert [(row["request"]["method"], row["request"]["url"]) for row in entries] == [
+        ("POST", first_request.url),
+        ("GET", second_request.url),
+    ]
+    assert [row["response"]["status"] for row in entries] == [302, 200]
+    assert fulfilled[0]["headers"]["location"] == "/target"
+    assert tool.request_bridge_stats == {"requests": 2, "failures": 0}
+
+
+def test_playwright_transport_records_mutation_and_json_body_without_cdp_session(
+    tmp_path,
+) -> None:
+    class EventSource:
+        def __init__(self):
+            self.handlers = {}
+            self.removed = []
+
+        def on(self, event, callback):
+            self.handlers[event] = callback
+
+        def remove_listener(self, event, callback):
+            assert self.handlers[event] == callback
+            self.removed.append(event)
+
+    page = EventSource()
+
+    class Context(EventSource):
+        pages = [page]
+
+        def new_cdp_session(self, _page):
+            raise AssertionError("Playwright capture must not attach a second CDP session")
+
+    class Request:
+        method = "POST"
+        url = "http://example.test/save"
+        headers = {"content-type": "application/json"}
+        post_data = '{"enabled":true}'
+        resource_type = "xhr"
+        failure = None
+
+        def all_headers(self):
+            return dict(self.headers)
+
+    class Response:
+        request = Request()
+        status = 200
+        status_text = "OK"
+        headers = {"content-type": "application/json; charset=utf-8"}
+
+        def all_headers(self):
+            return dict(self.headers)
+
+        def body(self):
+            return b'{"saved":true}'
+
+    context = Context()
+    recorder = CdpHarRecorder(capture_transport="playwright")
+    recorder.start(context)
+    response = Response()
+    page.handlers["request"](response.request)
+    page.handlers["response"](response)
+    page.handlers["requestfinished"](response.request)
+    path = tmp_path / "playwright.har"
+    recorder.stop(context, path)
+
+    entries = json.loads(path.read_text())["log"]["entries"]
+    assert entries == [
+        {
+            "request": {
+                "method": "POST",
+                "url": "http://example.test/save",
+                "headers": [
+                    {"name": "content-type", "value": "application/json"}
+                ],
+                "cookies": [],
+                "postData": {
+                    "mimeType": "application/json",
+                    "text": '{"enabled":true}',
+                },
+            },
+            "response": {
+                "status": 200,
+                "statusText": "OK",
+                "headers": [
+                    {
+                        "name": "content-type",
+                        "value": "application/json; charset=utf-8",
+                    }
+                ],
+                "cookies": [],
+                "redirectURL": "",
+                "content": {
+                    "mimeType": "application/json",
+                    "text": '{"saved":true}',
+                },
+            },
+        }
+    ]
+    assert recorder.errors == []
+    assert context.removed == ["page"]
+    assert set(page.removed) == {
+        "request",
+        "response",
+        "requestfinished",
+        "requestfailed",
+    }
+
+
+def test_playwright_transport_keeps_failed_request_as_status_zero(tmp_path) -> None:
+    class EventSource:
+        def __init__(self):
+            self.handlers = {}
+
+        def on(self, event, callback):
+            self.handlers[event] = callback
+
+        def remove_listener(self, *_args):
+            pass
+
+    page = EventSource()
+
+    class Context(EventSource):
+        pages = [page]
+
+    class Request:
+        method = "GET"
+        url = "http://example.test/unavailable"
+        headers = {}
+        post_data = None
+        resource_type = "document"
+        failure = "net::ERR_CONNECTION_REFUSED"
+
+        def all_headers(self):
+            return {}
+
+    context = Context()
+    recorder = CdpHarRecorder(capture_transport="playwright")
+    recorder.start(context)
+    request = Request()
+    page.handlers["request"](request)
+    page.handlers["requestfailed"](request)
+    path = tmp_path / "failed.har"
+    recorder.stop(context, path)
+
+    entry = json.loads(path.read_text())["log"]["entries"][0]
+    assert entry["response"]["status"] == 0
+    assert entry["_capture"] == {
+        "state": "failed",
+        "errorText": "net::ERR_CONNECTION_REFUSED",
+    }
+
+
+def test_navigation_response_fills_a_missing_cdp_document_event(tmp_path) -> None:
+    class Context:
+        pages = []
+
+        def on(self, *_args):
+            pass
+
+        def remove_listener(self, *_args):
+            pass
+
+    class Request:
+        method = "GET"
+        url = "http://map.example.test/"
+        headers = {"accept": "text/html"}
+        post_data = None
+
+    class Response:
+        request = Request()
+        status = 200
+        status_text = "OK"
+        headers = {"content-type": "text/html; charset=utf-8"}
+
+    context = Context()
+    recorder = CdpHarRecorder()
+    recorder.start(context)
+    recorder.record_navigation_response(Response())
+    path = tmp_path / "navigation.har"
+    recorder.stop(context, path)
+
+    entries = json.loads(path.read_text())["log"]["entries"]
+    assert entries == [
+        {
+            "request": {
+                "method": "GET",
+                "url": "http://map.example.test/",
+                "headers": [{"name": "accept", "value": "text/html"}],
+                "cookies": [],
+            },
+            "response": {
+                "status": 200,
+                "statusText": "OK",
+                "headers": [
+                    {"name": "content-type", "value": "text/html; charset=utf-8"}
+                ],
+                "cookies": [],
+                "redirectURL": "",
+                "content": {"mimeType": "text/html", "text": ""},
+            },
+            "_capture": {"state": "playwright-navigation-response"},
+        }
+    ]
 
 
 def test_capture_explicitly_retains_navigation_json_with_finite_limits(tmp_path) -> None:
@@ -97,6 +667,24 @@ def test_har_preserves_multipart_post_data_and_json_response() -> None:
     assert response["content"]["text"] == '{"items_qty":1}'
 
 
+def test_har_preserves_only_evaluator_safe_response_cookie_from_extra_info() -> None:
+    response = _har_response(
+        {"status": 302, "headers": {"Location": "/wishlist"}},
+        None,
+        {
+            "Set-Cookie": (
+                "PHPSESSID=secret; Path=/\r\n"
+                "mage-messages=product%20added; Path=/"
+            )
+        },
+    )
+
+    assert response["cookies"] == [
+        {"name": "mage-messages", "value": "product%20added"}
+    ]
+    assert "secret" not in json.dumps(response)
+
+
 def test_json_mime_matching_uses_case_insensitive_essence() -> None:
     for value in (
         "application/json",
diff --git a/tests/browser_use/test_runner.py b/tests/browser_use/test_runner.py
index 6846252e0..10c91eaff 100644
--- a/tests/browser_use/test_runner.py
+++ b/tests/browser_use/test_runner.py
@@ -13,6 +13,7 @@
     QEMU_X86_64_HOST_PATH,
     SITE_AUTH_HEADERS,
     _allow_official_container_hostname,
+    _bootstrap_postmill_browser_session,
     _bootstrap_postmill_session,
     _browser_tool_config,
     _configure_benchmark,
@@ -43,6 +44,7 @@
 def test_browser_config_requests_owned_context_and_supported_viewport_keys() -> None:
     config = _browser_tool_config("http://localhost:9222")
     assert config["cdp_isolated_context"] is True
+    assert config["locale"] == "en-US"
     assert (config["viewport_width"], config["viewport_height"]) == (1280, 900)
     assert "viewport" not in config
 
@@ -85,6 +87,22 @@ def test_viewport_receipt_measures_primary_and_new_page_before_navigation() -> N
     assert evidence["observations"][-1]["phase"] == "after_login"
 
 
+def test_viewport_receipt_supports_a_declared_single_target_session() -> None:
+    service = _viewport_service()
+    service._supports_multiple_pages = False
+    service._context.new_page = lambda: (_ for _ in ()).throw(
+        AssertionError("single-target sessions must not create a child page")
+    )
+
+    evidence = _prepare_browser_viewport(service)
+
+    assert evidence["verified"] is True
+    assert evidence["page_model"] == "single_target"
+    assert [row["phase"] for row in evidence["observations"]] == [
+        "primary_before_navigation"
+    ]
+
+
 @pytest.mark.parametrize("width,height", [(780, 437), (1920, 1080), (1280, 720)])
 def test_viewport_readback_rejects_unapplied_configuration(width, height) -> None:
     with pytest.raises(RuntimeError, match="viewport mismatch"):
@@ -252,6 +270,278 @@ def _submit(self, callback):
     assert primary.url == "http://official.test/start"
 
 
+def test_login_context_publishes_layout_before_admin_visibility_check() -> None:
+    events = []
+
+    class Page:
+        def close(self):
+            pass
+
+        def set_default_timeout(self, _value):
+            pass
+
+        def set_default_navigation_timeout(self, _value):
+            pass
+
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, _script):
+            pass
+
+        def reload(self, **_kwargs):
+            events.append(("reload",))
+
+        def locator(self, selector):
+            class Locator:
+                def wait_for(self):
+                    events.append(("visible", selector))
+
+            return Locator()
+
+        def wait_for_timeout(self, _value):
+            pass
+
+    primary = Page()
+    login_page = Page()
+    context = SimpleNamespace(
+        pages=[primary],
+        clear_cookies=lambda: None,
+        set_extra_http_headers=lambda _headers: None,
+        new_page=lambda: login_page,
+    )
+    service = SimpleNamespace(
+        _context=context,
+        _page=primary,
+        _submit=lambda callback: callback(),
+        _publish_page_layout=lambda page: events.append(("layout", page)),
+    )
+    benchmark = SimpleNamespace(
+        config=SimpleNamespace(
+            get_environment=lambda _site: SimpleNamespace(
+                active_url="http://fixture.test/admin"
+            )
+        )
+    )
+
+    _login_context(
+        service,
+        (WebArenaSite.SHOPPING_ADMIN,),
+        benchmark,
+        ["http://fixture.test/start"],
+    )
+
+    assert events.index(("reload",)) < events.index(("layout", login_page))
+    assert events.index(("layout", login_page)) < events.index(("visible", ".page-title"))
+
+
+def test_login_context_reuses_primary_page_for_single_target_session() -> None:
+    events = []
+
+    class Page:
+        def close(self):
+            raise AssertionError("the primary page must remain open")
+
+        def set_default_timeout(self, _value):
+            pass
+
+        def set_default_navigation_timeout(self, _value):
+            pass
+
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, _script):
+            pass
+
+        def reload(self, **_kwargs):
+            events.append(("reload",))
+
+        def locator(self, selector):
+            return SimpleNamespace(wait_for=lambda: events.append(("visible", selector)))
+
+        def wait_for_timeout(self, _value):
+            pass
+
+    primary = Page()
+    context = SimpleNamespace(
+        pages=[primary],
+        clear_cookies=lambda: None,
+        set_extra_http_headers=lambda _headers: None,
+        new_page=lambda: (_ for _ in ()).throw(
+            AssertionError("single-target sessions must reuse the primary page")
+        ),
+    )
+    service = SimpleNamespace(
+        _context=context,
+        _page=primary,
+        _supports_multiple_pages=False,
+        _submit=lambda callback: callback(),
+    )
+    benchmark = SimpleNamespace(
+        config=SimpleNamespace(
+            get_environment=lambda _site: SimpleNamespace(
+                active_url="http://fixture.test/admin"
+            )
+        )
+    )
+
+    _login_context(
+        service,
+        (WebArenaSite.SHOPPING_ADMIN,),
+        benchmark,
+        ["http://fixture.test/start"],
+    )
+
+    assert events[-1] == ("goto", "http://fixture.test/start")
+
+
+def test_login_context_can_use_native_provider_login_submit() -> None:
+    events = []
+
+    class Locator:
+        def __init__(self, test_id):
+            self.test_id = test_id
+
+        def fill(self, value):
+            events.append(("fill", self.test_id, value))
+
+        def click(self, **kwargs):
+            events.append(("click", self.test_id, kwargs))
+
+        def evaluate(self, script):
+            events.append(("evaluate", self.test_id, script))
+
+    class Page:
+        def close(self):
+            raise AssertionError("the primary page must remain open")
+
+        def set_default_timeout(self, _value):
+            pass
+
+        def set_default_navigation_timeout(self, _value):
+            pass
+
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, _script):
+            pass
+
+        def reload(self, **_kwargs):
+            pass
+
+        def get_by_test_id(self, test_id):
+            return Locator(test_id)
+
+        def wait_for_url(self, pattern, **_kwargs):
+            events.append(("wait_for_url", pattern))
+
+        def wait_for_timeout(self, _value):
+            pass
+
+    primary = Page()
+    context = SimpleNamespace(
+        pages=[primary],
+        clear_cookies=lambda: None,
+        set_extra_http_headers=lambda _headers: None,
+        new_page=lambda: (_ for _ in ()).throw(
+            AssertionError("single-target sessions must reuse the primary page")
+        ),
+    )
+    service = SimpleNamespace(
+        _context=context,
+        _page=primary,
+        _supports_multiple_pages=False,
+        _native_login_form_submit=True,
+        _submit_login_form=lambda page, submitter: events.append(
+            ("submit_login_form", page is primary, submitter.test_id)
+        ),
+        _submit=lambda callback: callback(),
+    )
+    benchmark = SimpleNamespace(
+        config=SimpleNamespace(
+            get_environment=lambda _site: SimpleNamespace(
+                active_url="http://fixture.test"
+            )
+        )
+    )
+
+    _login_context(
+        service,
+        (WebArenaSite.GITLAB,),
+        benchmark,
+        ["http://fixture.test/start"],
+    )
+
+    assert ("submit_login_form", True, "sign-in-button") in events
+    assert events[-1] == ("goto", "http://fixture.test/start")
+
+
+def test_login_context_uses_service_viewport_observer_without_page_script() -> None:
+    class Page:
+        def evaluate(self, _script):
+            raise AssertionError("the service viewport observer must replace page evaluation")
+
+        def set_default_timeout(self, _value):
+            pass
+
+        def set_default_navigation_timeout(self, _value):
+            pass
+
+        def goto(self, _url, **_kwargs):
+            pass
+
+        def wait_for_timeout(self, _value):
+            pass
+
+    primary = Page()
+    context = SimpleNamespace(
+        pages=[primary],
+        clear_cookies=lambda: (_ for _ in ()).throw(
+            AssertionError("fresh contexts must not repeat cookie cleanup")
+        ),
+        set_extra_http_headers=lambda _headers: (_ for _ in ()).throw(
+            AssertionError("an empty authentication header set must not be written")
+        ),
+    )
+    service = SimpleNamespace(
+        _context=context,
+        _page=primary,
+        _supports_multiple_pages=False,
+        _fresh_context_per_task=True,
+        _submit=lambda callback: callback(),
+        viewport_observer=lambda page: {
+            "width": 1280,
+            "height": 900,
+            "device_scale_factor": 1,
+        },
+    )
+    benchmark = SimpleNamespace(
+        config=SimpleNamespace(
+            get_environment=lambda _site: SimpleNamespace(active_url="http://fixture.test")
+        )
+    )
+    evidence = {"observations": [], "verified": True}
+
+    _login_context(
+        service,
+        (WebArenaSite.MAP,),
+        benchmark,
+        ["http://fixture.test/start"],
+        evidence,
+    )
+
+    assert evidence["observations"] == [
+        {
+            "phase": "login_before_navigation",
+            "width": 1280,
+            "height": 900,
+            "device_scale_factor": 1,
+        }
+    ]
+
+
 @pytest.mark.parametrize(
     "status,cookie,accepted",
     [(200, True, True), (302, True, False), (401, False, False), (200, False, False)],
@@ -281,6 +571,52 @@ def get(url, **kwargs):
     assert calls[1] == "dispose"
 
 
+@pytest.mark.parametrize(
+    "status,redirected,cookie,accepted",
+    [
+        (200, False, True, True),
+        (302, False, True, False),
+        (200, True, True, False),
+        (200, False, False, False),
+    ],
+)
+def test_postmill_browser_bootstrap_uses_one_temporary_page_header(
+    status, redirected, cookie, accepted
+) -> None:
+    events = []
+    request = SimpleNamespace(redirected_from=object() if redirected else None)
+    response = SimpleNamespace(status=status, request=request)
+
+    class Page:
+        def set_extra_http_headers(self, headers):
+            events.append(("headers", headers))
+
+        def goto(self, url, **kwargs):
+            events.append(("goto", url, kwargs))
+            return response
+
+    context = SimpleNamespace(
+        cookies=lambda url: (
+            [{"name": "PHPSESSID", "value": "fixture-session"}] if cookie else []
+        )
+    )
+    page = Page()
+    if accepted:
+        _bootstrap_postmill_browser_session(context, page, "http://fixture.test")
+    else:
+        with pytest.raises(RuntimeError, match="Postmill browser bootstrap"):
+            _bootstrap_postmill_browser_session(context, page, "http://fixture.test")
+
+    assert events[0][0] == "headers"
+    assert set(events[0][1]) == {SITE_AUTH_HEADERS[WebArenaSite.REDDIT]}
+    assert events[1] == (
+        "goto",
+        "http://fixture.test",
+        {"wait_until": "commit", "timeout": 90_000},
+    )
+    assert events[2] == ("headers", {})
+
+
 @pytest.mark.parametrize("include_shopping", [False, True])
 @pytest.mark.parametrize("identity_valid", [False, True])
 def test_postmill_login_keeps_auth_out_of_browser_requests_and_checks_identity(
diff --git a/tests/environments/test_webarena_map_detail.py b/tests/environments/test_webarena_map_detail.py
index dce01f5a7..975309058 100644
--- a/tests/environments/test_webarena_map_detail.py
+++ b/tests/environments/test_webarena_map_detail.py
@@ -9,7 +9,12 @@
 
 import pytest
 
-from harness.environments.webarena.map_detail import DetailHandler, lookup, render
+from harness.environments.webarena.map_detail import (
+    DetailHandler,
+    FROZEN_OBJECTS,
+    lookup,
+    render,
+)
 
 
 @pytest.mark.parametrize("kind,identifier,name", [
@@ -54,7 +59,7 @@ def test_missing_object_stays_missing_and_details_escape_untrusted_tags():
         thread.start()
         try:
             with pytest.raises(HTTPError) as error:
-                urlopen(f"http://127.0.0.1:{server.server_port}/relation/5475586")
+                urlopen(f"http://127.0.0.1:{server.server_port}/relation/999999999999")
             assert error.value.code == 404
         finally:
             server.shutdown()
@@ -76,3 +81,13 @@ def read(self):
     with patch("harness.environments.webarena.map_detail.urlopen", return_value=Response()):
         assert lookup("node", "7")["osm_type"] == "node"
         assert lookup("relation", "7") is None
+
+
+def test_frozen_route_relation_uses_the_bound_real_osm_identity():
+    place = lookup("relation", "5475586")
+    assert place is FROZEN_OBJECTS[("relation", "5475586")]
+    assert place["display_name"].startswith("Lake Pontchartrain Causeway")
+    assert place["lat"] == "30.199722222222"
+    assert place["lon"] == "-90.122777777778"
+    assert place["extratags"]["wikidata"] == "Q1132510"
+    assert "5475586" in render("relation", "5475586", place).decode()
diff --git a/tests/test_runtime_entrypoints.py b/tests/test_runtime_entrypoints.py
index 1713c490a..07807abfd 100644
--- a/tests/test_runtime_entrypoints.py
+++ b/tests/test_runtime_entrypoints.py
@@ -5,6 +5,7 @@
 import tempfile
 import unittest
 
+from scripts.check_track_naming import repository_contract_findings
 from scripts.run_headless_benchmark import require_expected_provider_versions
 from harness.web_fetch.publication_target import current_publication_target, require_publication_site, require_publication_version
 
@@ -13,6 +14,29 @@
 
 
 class RuntimeEntrypointTests(unittest.TestCase):
+    def test_catalog_replay_rejects_specialized_modules_schemas_targets_and_repairs(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            scripts = root / "scripts"
+            scripts.mkdir()
+            (scripts / "run_webarena_full_chromium.py").write_text("")
+            (scripts / "repair_webarena_case_42.py").write_text("")
+            harness = root / "harness" / "trajectory_replay"
+            harness.mkdir(parents=True)
+            (harness / "full_catalog.py").write_text("")
+            (harness / "writer.py").write_text(
+                'SCHEMA = "lex_webarena_verified_full_replay_run/3"\n'
+            )
+            (root / "Makefile").write_text("webarena-full-release-check:\n\ttrue\n")
+
+            findings = "\n".join(repository_contract_findings(root))
+
+            self.assertIn("run_webarena_full_chromium.py", findings)
+            self.assertIn("repair_webarena_case_42.py", findings)
+            self.assertIn("full_catalog.py", findings)
+            self.assertIn("legacy full schemas are read-only", findings)
+            self.assertIn("must not encode one dataset size", findings)
+
     def test_repository_pins_python_and_uv(self) -> None:
         self.assertEqual((ROOT / ".python-version").read_text().strip(), "3.11")
         self.assertTrue((ROOT / "uv.lock").is_file())
diff --git a/tests/trajectory_replay/public_webarena/test_executor.py b/tests/trajectory_replay/public_webarena/test_executor.py
index 1caf24cd2..5c0efad24 100644
--- a/tests/trajectory_replay/public_webarena/test_executor.py
+++ b/tests/trajectory_replay/public_webarena/test_executor.py
@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import json
 from contextlib import contextmanager
+from copy import deepcopy
 from unittest.mock import MagicMock
 
 import pytest
@@ -18,6 +20,9 @@ def __init__(self):
         self.url = "about:blank"
         self.mouse = self
         self.keyboard = self
+        self.visible = True
+        self.enabled = True
+        self.trial_ready = True
 
     def goto(self, url, **kwargs):
         self.calls.append(("navigate", url))
@@ -31,7 +36,20 @@ class Response:
     def title(self):
         return "Example"
 
+    def wait_for_url(self, url, **kwargs):
+        self.calls.append(("wait_for_url", url, kwargs))
+        self.url = "https://example.com/reports/ready"
+
+    def wait_for_timeout(self, milliseconds):
+        self.calls.append(("wait", milliseconds))
+
+    def wait_for_load_state(self, state, **kwargs):
+        self.calls.append(("load_state", state, kwargs))
+
     def evaluate(self, script, argument=None):
+        self.calls.append(("evaluate", script, argument))
+        if script == "() => ({ok: true, count: 2})":
+            return {"ok": True, "count": 2}
         return {"found": True, "rect": {"width": 0}}
 
     def locator(self, selector):
@@ -44,7 +62,29 @@ def first(self):
                 return self
 
             def click(self, **kwargs):
-                page.calls.append(("click", selector, kwargs))
+                kind = "trial_click" if kwargs.get("trial") else "click"
+                page.calls.append((kind, selector, kwargs))
+                if kwargs.get("trial") and not page.trial_ready:
+                    raise TimeoutError("trial timeout: element is covered by an overlay")
+
+            def is_visible(self):
+                page.calls.append(("is_visible", selector))
+                return page.visible
+
+            def is_enabled(self):
+                page.calls.append(("is_enabled", selector))
+                return page.enabled
+
+            def is_editable(self):
+                page.calls.append(("is_editable", selector))
+                return page.enabled
+
+            def hover(self, **kwargs):
+                kind = "trial_hover" if kwargs.get("trial") else "hover"
+                page.calls.append((kind, selector, kwargs))
+
+            def wait_for(self, **kwargs):
+                page.calls.append(("wait_for", selector, kwargs))
 
             def dblclick(self, **kwargs):
                 page.calls.append(("double_click", selector, kwargs))
@@ -152,6 +192,10 @@ def test_execute_steps_counts_only_declared_browser_operations():
     assert "Navigated to: https://example.com" in outputs["page"]
     assert page.calls == [
         ("navigate", "https://example.com"),
+        ("load_state", "networkidle", {"timeout": 1500}),
+        ("wait", 300),
+        ("first", "html"),
+        ("get_property", "html", None),
         ("first", "body"),
         ("get_text", "body"),
         ("first", "h1"),
@@ -159,6 +203,63 @@ def test_execute_steps_counts_only_declared_browser_operations():
     ]
 
 
+def test_navigate_can_use_provider_document_observer_without_locators():
+    page = FakePage()
+    observed = []
+
+    output = execute_steps(
+        page,
+        ({"action": "navigate", "url": "https://example.com", "save_as": "page"},),
+        document_observer=lambda selected, timeout, limit: (
+            observed.append((selected, timeout, limit)) or ("Provider title", "Provider body")
+        ),
+    )
+
+    assert "Title: Provider title" in output["page"]
+    assert output["page"].endswith("Provider body")
+    assert observed[0][0] is page
+    assert observed[0][2] == 40_000
+    assert not any(call[0] in {"first", "get_property", "get_text"} for call in page.calls)
+
+
+def test_trace_observer_receives_running_and_terminal_action_states():
+    snapshots = []
+
+    execute_steps(
+        FakePage(),
+        ({"action": "get_text", "selector": "h1", "save_as": "heading"},),
+        trace_observer=lambda records: snapshots.append(deepcopy(records)),
+    )
+
+    assert [snapshot[-1]["status"] for snapshot in snapshots] == ["running", "pass"]
+    assert snapshots[-1][-1]["duration_ms"] is not None
+
+
+def test_navigate_can_create_a_new_page_and_publish_it_to_observers():
+    first = FakePage()
+    second = FakePage()
+
+    class Context:
+        pages = [first]
+
+        def new_page(self):
+            self.pages.append(second)
+            return second
+
+    first.context = Context()
+    observed = []
+
+    execute_steps(
+        first,
+        ({"action": "navigate", "url": "https://example.com/second", "new_page": True},),
+        page_observer=observed.append,
+    )
+
+    assert observed == [second]
+    assert first.calls == []
+    assert second.calls[0] == ("navigate", "https://example.com/second")
+
+
 def test_trace_has_one_record_per_declared_tool_call():
     trace = []
     execute_steps(
@@ -172,6 +273,330 @@ def test_trace_has_one_record_per_declared_tool_call():
     assert trace[0]["status"] == "pass"
 
 
+def test_evaluate_runs_declared_page_script_and_captures_json():
+    page = FakePage()
+
+    outputs = execute_steps(
+        page,
+        (
+            {
+                "action": "evaluate",
+                "script": "() => ({ok: true, count: 2})",
+                "save_as": "result",
+            },
+        ),
+    )
+
+    assert outputs["result"] == '{"ok": true, "count": 2}'
+    assert page.calls == [("evaluate", "() => ({ok: true, count: 2})", None)]
+
+
+def test_evaluate_keeps_large_structured_json_complete_for_retrieval():
+    value = {"scope": True, "complete": True, "records": ["x" * 1_000] * 50}
+
+    class StructuredPage(FakePage):
+        def evaluate(self, script, argument=None):
+            self.calls.append(("evaluate", script, argument))
+            return value
+
+    outputs = execute_steps(
+        StructuredPage(),
+        ({"action": "evaluate", "script": "collect()", "save_as": "retrieval"},),
+    )
+
+    assert len(outputs["retrieval"]) > 40_000
+    assert json.loads(outputs["retrieval"]) == value
+
+
+def test_evaluate_never_slices_structured_json_at_an_explicit_limit():
+    value = {"records": ["x" * 100]}
+
+    class StructuredPage(FakePage):
+        def evaluate(self, script, argument=None):
+            return value
+
+    trace = []
+    with pytest.raises(ValueError, match="structured observation exceeds"):
+        execute_steps(
+            StructuredPage(),
+            ({"action": "evaluate", "script": "collect()", "max_chars": 20},),
+            trace=trace,
+        )
+
+    assert trace[0]["status"] == "fail"
+    assert trace[0]["output"] == ""
+
+
+def test_evaluate_keeps_parseable_json_strings_atomic_but_bounds_plain_text():
+    values = iter((json.dumps({"record": "x" * 100}), "x" * 50_000))
+
+    class StringPage(FakePage):
+        def evaluate(self, script, argument=None):
+            return next(values)
+
+    page = StringPage()
+    structured = execute_steps(
+        page,
+        ({"action": "evaluate", "script": "json()", "save_as": "result"},),
+    )
+    plain = execute_steps(
+        page,
+        ({"action": "evaluate", "script": "text()", "save_as": "result"},),
+    )
+
+    assert json.loads(structured["result"])["record"] == "x" * 100
+    assert len(plain["result"]) == 40_000
+
+
+def test_evaluate_structured_limit_counts_utf8_bytes():
+    value = {"records": ["😀" * 1_100_000]}
+
+    class StructuredPage(FakePage):
+        def evaluate(self, script, argument=None):
+            return value
+
+    with pytest.raises(ValueError, match="4194304-byte evidence limit"):
+        execute_steps(
+            StructuredPage(),
+            ({"action": "evaluate", "script": "collect()"},),
+        )
+
+
+def test_evaluate_explicit_function_body_executes_side_effect_once():
+    class ReturnPage(FakePage):
+        def __init__(self):
+            super().__init__()
+            self.effects = 0
+
+        def evaluate(self, script, argument=None):
+            self.calls.append(("evaluate", script, argument))
+            self.effects += 1
+            return self.effects
+
+    page = ReturnPage()
+    outputs = execute_steps(
+        page,
+        (
+            {
+                "action": "evaluate",
+                "script": "window.count += 1; return window.count",
+                "script_mode": "function_body",
+                "save_as": "count",
+            },
+        ),
+    )
+
+    assert outputs == {"count": "1"}
+    assert page.effects == 1
+    assert len(page.calls) == 1
+    assert page.calls[0][1].startswith("(async () => {")
+
+
+def test_evaluate_never_retries_default_script_after_illegal_return():
+    class ReturnPage(FakePage):
+        def evaluate(self, script, argument=None):
+            self.calls.append(("evaluate", script, argument))
+            raise RuntimeError("SyntaxError: Illegal return statement")
+
+    page = ReturnPage()
+    with pytest.raises(RuntimeError, match="Illegal return statement"):
+        execute_steps(
+            page,
+            ({"action": "evaluate", "script": "sideEffect(); return true"},),
+        )
+    assert page.calls == [("evaluate", "sideEffect(); return true", None)]
+
+
+def test_evaluate_rejects_unknown_script_mode_before_page_execution():
+    page = FakePage()
+    with pytest.raises(ValueError, match="script_mode"):
+        execute_steps(
+            page,
+            ({"action": "evaluate", "script": "1", "script_mode": "retry"},),
+        )
+    assert page.calls == []
+
+
+def test_evaluate_does_not_retry_a_runtime_error():
+    class BrokenPage(FakePage):
+        def evaluate(self, script, argument=None):
+            self.calls.append(("evaluate", script, argument))
+            raise RuntimeError("ReferenceError: missing is not defined")
+
+    page = BrokenPage()
+    with pytest.raises(RuntimeError, match="ReferenceError"):
+        execute_steps(page, ({"action": "evaluate", "script": "missing()"},))
+    assert page.calls == [("evaluate", "missing()", None)]
+
+
+def test_navigation_network_idle_is_best_effort_and_keeps_render_grace():
+    class BusyPage(FakePage):
+        def wait_for_load_state(self, state, **kwargs):
+            self.calls.append(("load_state", state, kwargs))
+            raise TimeoutError("page keeps a streaming request open")
+
+    page = BusyPage()
+    responses = []
+    execute_steps(
+        page,
+        ({"action": "navigate", "url": "https://example.com"},),
+        navigation_response_observer=responses.append,
+    )
+    assert ("load_state", "networkidle", {"timeout": 1500}) in page.calls
+    assert ("wait", 300) in page.calls
+    assert len(responses) == 1
+    assert responses[0].status == 200
+
+
+def test_same_document_navigation_without_response_reloads_for_network_evidence():
+    class SamePage(FakePage):
+        def __init__(self):
+            super().__init__()
+            self.url = "https://example.com/"
+
+        def goto(self, url, **kwargs):
+            self.calls.append(("navigate", url, kwargs))
+            self.url = url + "/"
+            return None
+
+        def reload(self, **kwargs):
+            self.calls.append(("reload", kwargs))
+
+            class Response:
+                status = 200
+
+            return Response()
+
+    page = SamePage()
+    responses = []
+    execute_steps(
+        page,
+        ({"action": "navigate", "url": "https://example.com"},),
+        navigation_response_observer=responses.append,
+    )
+
+    assert any(call[0] == "reload" for call in page.calls)
+    assert len(responses) == 1
+    assert responses[0].status == 200
+
+
+def test_form_submission_evaluate_continues_only_after_expected_navigation():
+    class FormPage(FakePage):
+        def evaluate(self, script, argument=None):
+            self.calls.append(("evaluate", script, argument))
+            raise RuntimeError(
+                "Execution context was destroyed, most likely because of a navigation"
+            )
+
+        def wait_for_url(self, url, **kwargs):
+            self.calls.append(("wait_for_url", url.pattern, kwargs))
+            self.url = "http://gitlab.example/byteblaze/new_project"
+
+    page = FormPage()
+    trace = []
+    outputs = execute_steps(
+        page,
+        [
+            {
+                "action": "evaluate",
+                "script": "() => document.querySelector('form').submit()",
+                "accept_navigation_url_regex": r"/byteblaze/new_project$",
+                "save_as": "submitted",
+            },
+            {"action": "get_text", "selector": "body", "save_as": "landed"},
+        ],
+        trace=trace,
+    )
+
+    assert outputs["submitted"].endswith("/byteblaze/new_project")
+    assert outputs["landed"] == "Example Domain"
+    assert trace[0]["accepted_navigation"]["expected_url_regex"] == r"/byteblaze/new_project$"
+
+
+def test_form_submission_evaluate_rejects_wrong_navigation():
+    class WrongPage(FakePage):
+        def evaluate(self, script, argument=None):
+            raise RuntimeError(
+                "Execution context was destroyed, most likely because of a navigation"
+            )
+
+        def wait_for_url(self, url, **kwargs):
+            self.url = "http://gitlab.example/byteblaze/another_project"
+
+    with pytest.raises(ValueError, match="wrong URL"):
+        execute_steps(
+            WrongPage(),
+            [
+                {
+                    "action": "evaluate",
+                    "script": "() => document.querySelector('form').submit()",
+                    "accept_navigation_url_regex": r"/byteblaze/new_project$",
+                }
+            ],
+        )
+
+
+def test_action_can_wait_for_loading_mask_to_disappear_before_clicking():
+    page = FakePage()
+    execute_steps(
+        page,
+        (
+            {
+                "action": "click",
+                "selector": "button.filters",
+                "wait_for_selector_hidden": ".loading-mask",
+            },
+        ),
+    )
+    assert page.calls[:2] == [
+        ("wait_for", ".loading-mask", {"state": "hidden", "timeout": 30000}),
+        ("first", "button.filters"),
+    ]
+
+
+def test_wait_selector_uses_visible_readiness_instead_of_sleeping():
+    page = FakePage()
+
+    execute_steps(
+        page,
+        ({"action": "wait", "selector": "div.message-success:visible", "timeout": 12_000},),
+    )
+
+    assert page.calls == [
+        ("first", "div.message-success:visible"),
+        (
+            "wait_for",
+            "div.message-success:visible",
+            {"state": "visible", "timeout": 12_000},
+        ),
+    ]
+
+
+def test_wait_selector_can_use_attached_readiness_for_non_geometric_status_messages():
+    page = FakePage()
+
+    execute_steps(
+        page,
+        (
+            {
+                "action": "wait",
+                "selector": "div.message-success",
+                "wait_state": "attached",
+                "timeout": 12_000,
+            },
+        ),
+    )
+
+    assert page.calls == [
+        ("first", "div.message-success"),
+        (
+            "wait_for",
+            "div.message-success",
+            {"state": "attached", "timeout": 12_000},
+        ),
+    ]
+
+
 def test_default_action_timeout_is_thirty_seconds():
     page = FakePage()
 
@@ -190,6 +615,37 @@ def click(self, **kwargs):
     assert page.calls == [("timeout", 30_000)]
 
 
+def test_wait_for_url_uses_declared_readiness_contract():
+    page = FakePage()
+
+    execute_steps(
+        page,
+        (
+            {
+                "action": "wait_for_url",
+                "url": "**/reports/**",
+                "wait_until": "domcontentloaded",
+                "timeout_ms": 120_000,
+            },
+        ),
+    )
+
+    assert page.calls == [
+        (
+            "wait_for_url",
+            "**/reports/**",
+            {"wait_until": "domcontentloaded", "timeout": 120_000},
+        )
+    ]
+
+
+@pytest.mark.parametrize("duration,expected", [({"timeout": 700}, 700), ({"seconds": 2}, 2000)])
+def test_wait_preserves_recorded_duration(duration, expected):
+    page = FakePage()
+    execute_steps(page, ({"action": "wait", **duration},))
+    assert page.calls == [("wait", expected)]
+
+
 def test_task_wall_clock_budget_caps_each_action_timeout(monkeypatch):
     page = FakePage()
 
@@ -216,6 +672,45 @@ def click(self, **kwargs):
     assert page.calls == [("timeout", 4_800)]
 
 
+def test_initial_sweep_can_shorten_only_undeclared_action_waits():
+    page = FakePage()
+    execute_steps(
+        page,
+        (
+            {"action": "click", "selector": "#first"},
+            {"action": "click", "selector": "#second", "timeout_ms": 25_000},
+        ),
+        default_action_timeout_ms=15_000,
+    )
+    timeouts = [call[2]["timeout"] for call in page.calls if call[0] == "click"]
+    assert timeouts == [15_000, 25_000]
+
+
+def test_shorter_action_budget_also_caps_missing_structured_target_wait():
+    page = FakePage()
+    observed = {}
+
+    class MissingLocator:
+        @property
+        def first(self):
+            return self
+
+        def count(self):
+            return 0
+
+        def wait_for(self, **kwargs):
+            observed.update(kwargs)
+
+    page.get_by_text = lambda *_args, **_kwargs: MissingLocator()
+    with pytest.raises(ValueError, match="matched 0"):
+        execute_steps(
+            page,
+            ({"action": "click", "target": {"by": "text", "value": "missing"}},),
+            default_action_timeout_ms=15_000,
+        )
+    assert observed["timeout"] == 15_000
+
+
 def test_task_wall_clock_budget_stops_before_next_action(monkeypatch):
     monkeypatch.setattr(
         "harness.trajectory_replay.public_webarena.executor.time.monotonic",
@@ -403,6 +898,122 @@ def observe(action_page):
     assert page.calls[1] == ("navigate", "https://example.com")
 
 
+def test_layout_observer_runs_before_actions_and_after_navigation():
+    page = FakePage()
+
+    def publish_layout(action_page, timeout_ms):
+        assert action_page is page
+        assert timeout_ms > 0
+        page.calls.append(("layout", page.url))
+
+    execute_steps(
+        page,
+        (
+            {"action": "navigate", "url": "https://example.com"},
+            {"action": "click", "selector": "#target"},
+        ),
+        layout_observer=publish_layout,
+    )
+
+    navigation = page.calls.index(("navigate", "https://example.com"))
+    first_layout = page.calls.index(("layout", "https://example.com"))
+    assert navigation < first_layout
+    assert ("layout", "about:blank") not in page.calls
+    click = next(index for index, call in enumerate(page.calls) if call[0] == "click")
+    action_layout = max(
+        index
+        for index, call in enumerate(page.calls[:click])
+        if call == ("layout", "https://example.com")
+    )
+    locator_resolution = page.calls.index(("first", "#target"))
+    assert action_layout < locator_resolution < click
+
+
+def test_dynamic_geometry_is_refreshed_before_one_side_effect_dispatch():
+    page = FakePage()
+    page.visible = False
+    publications = 0
+
+    def publish_layout(action_page, timeout_ms):
+        nonlocal publications
+        assert action_page is page
+        assert timeout_ms > 0
+        publications += 1
+        if publications == 2:
+            page.visible = True
+
+    execute_steps(
+        page,
+        ({"action": "click", "selector": "#late"},),
+        layout_observer=publish_layout,
+    )
+
+    assert publications == 2
+    assert sum(call[0] == "click" for call in page.calls) == 1
+
+
+def test_dynamic_overlay_is_refreshed_before_one_side_effect_dispatch():
+    page = FakePage()
+    page.trial_ready = False
+    publications = 0
+
+    def publish_layout(action_page, timeout_ms):
+        nonlocal publications
+        assert action_page is page
+        assert timeout_ms > 0
+        publications += 1
+        if publications == 2:
+            page.trial_ready = True
+
+    execute_steps(
+        page,
+        ({"action": "click", "selector": "#covered"},),
+        layout_observer=publish_layout,
+    )
+
+    assert publications == 2
+    assert sum(call[0] == "trial_click" for call in page.calls) == 2
+    assert sum(call[0] == "click" for call in page.calls) == 1
+
+
+def test_fill_select_and_press_keep_native_non_click_actionability():
+    page = FakePage()
+    page.trial_ready = False
+
+    execute_steps(
+        page,
+        (
+            {"action": "fill", "selector": "#field", "text": "value"},
+            {"action": "select", "selector": "#choice", "value": "one"},
+            {"action": "press", "selector": "#field", "key": "Enter"},
+        ),
+        layout_observer=lambda _page, _timeout: None,
+    )
+
+    assert not any(call[0].startswith("trial_") for call in page.calls)
+    assert sum(call[0] == "fill" for call in page.calls) == 1
+    assert sum(call[0] == "select" for call in page.calls) == 1
+    assert sum(call[0] == "press_target" for call in page.calls) == 1
+
+
+@pytest.mark.parametrize(("visible", "enabled"), ((False, True), (True, False)))
+def test_targeted_press_preserves_native_visibility_and_enabled_semantics(
+    visible, enabled
+):
+    page = FakePage()
+    page.visible = visible
+    page.enabled = enabled
+
+    execute_steps(
+        page,
+        ({"action": "press", "selector": "#field", "key": "Enter"},),
+        layout_observer=lambda _page, _timeout: None,
+    )
+
+    assert sum(call[0] == "press_target" for call in page.calls) == 1
+    assert not any(call[0] in {"is_visible", "is_enabled"} for call in page.calls)
+
+
 def test_history_steps_wait_for_commit_to_support_bfcache_restores():
     page = FakePage()
 
diff --git a/tests/trajectory_replay/public_webarena/test_response_builder.py b/tests/trajectory_replay/public_webarena/test_response_builder.py
index 2c1c18d59..8fa1564f2 100644
--- a/tests/trajectory_replay/public_webarena/test_response_builder.py
+++ b/tests/trajectory_replay/public_webarena/test_response_builder.py
@@ -233,9 +233,7 @@ def test_response_builder_filters_aligned_observed_rows_by_requested_value():
     )
 
     assert response is not None
-    assert response["retrieved_data"] == [
-        {"name": "Ada", "email": "ada@example.test"}
-    ]
+    assert response["retrieved_data"] == [{"name": "Ada", "email": "ada@example.test"}]
 
 
 def test_response_builder_ranks_observed_labels_by_numeric_score():
@@ -294,8 +292,7 @@ def test_response_builder_deduplicates_review_authors_across_observed_pages():
         ),
         {
             "page_two": '["Review by Ada\\nLow quality"]',
-            "page_one": '["Review by Grace\\nQuality issue", '
-            '"Review by Ada\\nQuality issue"]',
+            "page_one": '["Review by Grace\\nQuality issue", "Review by Ada\\nQuality issue"]',
         },
     )
 
@@ -359,6 +356,37 @@ def test_response_builder_reports_not_found_only_from_explicit_observed_evidence
     }
 
 
+def test_response_builder_reports_not_found_when_required_page_lacks_target_state():
+    response = build_agent_response(
+        _task(
+            {
+                "kind": "observed_not_found",
+                "task_type": "navigate",
+                "output": "page",
+                "evidence_regex": r"my orders",
+                "forbidden_regex": r"\bprocessing\b",
+            }
+        ),
+        {"page": "My Orders\nOrder #1 Complete"},
+    )
+    assert response is not None
+    assert response["status"] == "NOT_FOUND_ERROR"
+
+    with pytest.raises(ValueError, match="forbidden evidence"):
+        build_agent_response(
+            _task(
+                {
+                    "kind": "observed_not_found",
+                    "task_type": "navigate",
+                    "output": "page",
+                    "evidence_regex": r"my orders",
+                    "forbidden_regex": r"\bprocessing\b",
+                }
+            ),
+            {"page": "My Orders\nOrder #2 Processing"},
+        )
+
+
 @pytest.mark.parametrize(
     ("builder", "outputs"),
     (
@@ -385,3 +413,140 @@ def test_response_builder_reports_not_found_only_from_explicit_observed_evidence
 def test_response_builder_not_found_fails_closed_without_matching_evidence(builder, outputs):
     with pytest.raises(ValueError, match="observed_not_found"):
         build_agent_response(_task(builder), outputs)
+
+
+def test_response_builder_reports_action_not_allowed_only_from_complete_empty_scope():
+    builder = {
+        "kind": "observed_action_not_allowed",
+        "task_type": "mutate",
+        "output": "review_scope",
+    }
+
+    response = build_agent_response(
+        _task(builder),
+        {"review_scope": '{"scope_complete": true, "matches": [], "total": 7}'},
+    )
+
+    assert response == {
+        "task_type": "mutate",
+        "status": "ACTION_NOT_ALLOWED_ERROR",
+        "retrieved_data": None,
+    }
+
+
+@pytest.mark.parametrize(
+    "evidence",
+    [
+        '{"scope_complete": false, "matches": []}',
+        '{"scope_complete": true, "matches": ["review 42"]}',
+        '{"scope_complete": true}',
+    ],
+)
+def test_response_builder_action_not_allowed_fails_closed(evidence):
+    builder = {
+        "kind": "observed_action_not_allowed",
+        "task_type": "mutate",
+        "output": "review_scope",
+    }
+
+    with pytest.raises(ValueError):
+        build_agent_response(_task(builder), {"review_scope": evidence})
+
+
+def test_response_builder_reports_action_blocked_only_from_observed_blocker():
+    builder = {
+        "kind": "observed_action_blocked",
+        "task_type": "mutate",
+        "output": "merge_request_scope",
+    }
+
+    response = build_agent_response(
+        _task(builder),
+        {
+            "merge_request_scope": (
+                '{"scope_complete":true,"allowed":false,"blockers":[{"id":17,"state":"opened"}]}'
+            )
+        },
+    )
+
+    assert response == {
+        "task_type": "mutate",
+        "status": "ACTION_NOT_ALLOWED_ERROR",
+        "retrieved_data": None,
+    }
+
+
+@pytest.mark.parametrize(
+    "evidence",
+    [
+        '{"scope_complete":false,"allowed":false,"blockers":[{"id":17}]}',
+        '{"scope_complete":true,"allowed":true,"blockers":[{"id":17}]}',
+        '{"scope_complete":true,"allowed":false,"blockers":[]}',
+        '{"scope_complete":true,"allowed":false}',
+    ],
+)
+def test_response_builder_action_blocked_fails_closed(evidence):
+    builder = {
+        "kind": "observed_action_blocked",
+        "task_type": "mutate",
+        "output": "merge_request_scope",
+    }
+
+    with pytest.raises(ValueError):
+        build_agent_response(_task(builder), {"merge_request_scope": evidence})
+
+
+@pytest.mark.parametrize(
+    "evidence",
+    [
+        (
+            '{"scope_complete":true,"allowed":false,'
+            '"access_levels":[null,20],"required_access_level":40}'
+        ),
+        (
+            '{"scope_complete":true,"allowed":false,'
+            '"access_levels":[null,null],"required_access_level":40}'
+        ),
+    ],
+)
+def test_response_builder_reports_permission_denied_from_absent_or_insufficient_access(
+    evidence,
+):
+    builder = {
+        "kind": "observed_permission_denied",
+        "task_type": "retrieve",
+        "output": "project_permissions",
+    }
+
+    response = build_agent_response(
+        _task(builder),
+        {"project_permissions": evidence},
+    )
+
+    assert response == {
+        "task_type": "retrieve",
+        "status": "PERMISSION_DENIED_ERROR",
+        "retrieved_data": None,
+    }
+
+
+@pytest.mark.parametrize(
+    "evidence",
+    [
+        '{"scope_complete":false,"allowed":false,"access_levels":[null],"required_access_level":40}',
+        '{"scope_complete":true,"allowed":true,"access_levels":[null],"required_access_level":40}',
+        '{"scope_complete":true,"allowed":false,"access_levels":[],"required_access_level":40}',
+        '{"scope_complete":true,"allowed":false,"access_levels":[40],"required_access_level":40}',
+        '{"scope_complete":true,"allowed":false,"access_levels":[false],"required_access_level":40}',
+        '{"scope_complete":true,"allowed":false,"access_levels":[20],"required_access_level":true}',
+    ],
+)
+def test_response_builder_permission_denied_fails_closed(evidence):
+    builder = {
+        "kind": "observed_permission_denied",
+        "task_type": "retrieve",
+        "output": "project_permissions",
+    }
+
+    with pytest.raises(ValueError):
+        build_agent_response(_task(builder), {"project_permissions": evidence})
diff --git a/tests/trajectory_replay/public_webarena/test_suite.py b/tests/trajectory_replay/public_webarena/test_suite.py
index 38826a563..ce353c291 100644
--- a/tests/trajectory_replay/public_webarena/test_suite.py
+++ b/tests/trajectory_replay/public_webarena/test_suite.py
@@ -98,6 +98,110 @@ def test_load_suite_requires_visible_header_for_table_column(tmp_path):
         load_suite(path)
 
 
+def test_load_suite_requires_url_pattern_for_wait_for_url(tmp_path):
+    path = tmp_path / "tasks.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_browser_use_tasks/3",
+                "tasks": [_task("bad", [{"action": "wait_for_url", "url": ""}])],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="requires a URL pattern"):
+        load_suite(path)
+
+
+@pytest.mark.parametrize(
+    "call",
+    [
+        {"action": "click", "selector": "a", "new_page": True},
+        {"action": "navigate", "url": "https://example.com", "new_page": "yes"},
+        {
+            "action": "navigate",
+            "url": "https://example.com",
+            "new_page": True,
+            "tab_index": 1,
+        },
+    ],
+)
+def test_load_suite_rejects_invalid_new_page_contract(tmp_path, call):
+    path = tmp_path / "tasks.json"
+    path.write_text(
+        json.dumps({"schema": "lex_browser_use_tasks/3", "tasks": [_task("bad", [call])]}),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="new_page"):
+        load_suite(path)
+
+
+@pytest.mark.parametrize("duration", [{}, {"timeout": 0}, {"seconds": -1}, {"timeout": 1, "seconds": 1}])
+def test_load_suite_rejects_invalid_wait_duration(tmp_path, duration):
+    path = tmp_path / "tasks.json"
+    path.write_text(
+        json.dumps({"schema": "lex_browser_use_tasks/3", "tasks": [_task("bad", [{"action": "wait", **duration}])]}),
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError, match="wait call 1"):
+        load_suite(path)
+
+
+def test_load_suite_accepts_selector_wait_with_timeout(tmp_path):
+    path = tmp_path / "tasks.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_browser_use_tasks/3",
+                "tasks": [
+                    _task(
+                        "ready",
+                        [
+                            {
+                                "action": "wait",
+                                "selector": "div.message-success:visible",
+                                "timeout": 30_000,
+                            }
+                        ],
+                    )
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    assert load_suite(path).tasks[0].tool_calls[0]["selector"].startswith("div.message")
+
+
+def test_load_suite_accepts_attached_selector_wait(tmp_path):
+    path = tmp_path / "tasks.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_browser_use_tasks/3",
+                "tasks": [
+                    _task(
+                        "ready",
+                        [
+                            {
+                                "action": "wait",
+                                "selector": "div.message-success",
+                                "wait_state": "attached",
+                                "timeout": 30_000,
+                            }
+                        ],
+                    )
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    assert load_suite(path).tasks[0].tool_calls[0]["wait_state"] == "attached"
+
+
 @pytest.mark.parametrize("action", ["click", "fill", "select"])
 @pytest.mark.parametrize("selector", [None, "", "  ", 12])
 def test_load_suite_rejects_ref_only_and_invalid_selectors(tmp_path, action, selector):
diff --git a/tests/trajectory_replay/public_webarena/test_webarena_evaluator.py b/tests/trajectory_replay/public_webarena/test_webarena_evaluator.py
index 03bdbd7e6..6007482e3 100644
--- a/tests/trajectory_replay/public_webarena/test_webarena_evaluator.py
+++ b/tests/trajectory_replay/public_webarena/test_webarena_evaluator.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+from urllib.parse import urlencode
 
 import pytest
 
@@ -204,10 +205,12 @@ def test_official_webarena_verified_evaluator_can_read_safe_flash_cookie(tmp_pat
 
 
 @pytest.mark.parametrize(
-    "directions_engine",
-    ["fossgis_osrm_car", "fossgis_osrm_bicycle"],
+    ("directions_engine", "expected_status"),
+    [("fossgis_osrm_car", "pass"), ("fossgis_osrm_bicycle", "fail")],
 )
-def test_official_task_760_has_navigate_get_xhr_evaluator_defect(tmp_path, directions_engine):
+def test_official_task_760_checks_the_hoboken_car_route(
+    tmp_path, directions_engine, expected_status
+):
     config = tmp_path / "config.json"
     config.write_text(
         json.dumps(
@@ -221,13 +224,35 @@ def test_official_task_760_has_navigate_get_xhr_evaluator_defect(tmp_path, direc
         encoding="utf-8",
     )
     har = tmp_path / "network.har"
-    route = "/route/v1/driving/-75.4716115,40.6022552;-74.4041622,40.0757384"
+    route = (
+        "/route/v1/driving/"
+        "-75.44225386838299,40.651163100000005;-74.0323752,40.7433066"
+    )
     har.write_text(
         json.dumps(
-            {
-                "log": {
-                    "entries": [
-                        {
+                {
+                    "log": {
+                        "entries": [
+                            {
+                                "request": {
+                                    "method": "GET",
+                                    "url": "http://map.test/directions",
+                                    "headers": [
+                                        {"name": "Accept", "value": "text/html"},
+                                        {"name": "Sec-Fetch-Dest", "value": "document"},
+                                        {"name": "Sec-Fetch-Mode", "value": "navigate"},
+                                        {"name": "Sec-Fetch-User", "value": "?1"},
+                                    ],
+                                },
+                                "response": {
+                                    "status": 200,
+                                    "redirectURL": "",
+                                    "headers": [],
+                                    "cookies": [],
+                                    "content": {"mimeType": "text/html", "text": ""},
+                                },
+                            },
+                            {
                             "request": {
                                 "method": "GET",
                                 "url": f"http://map.test{route}",
@@ -264,7 +289,7 @@ def test_official_task_760_has_navigate_get_xhr_evaluator_defect(tmp_path, direc
         difficulty="simple",
         ground_truth={"expected_actions": []},
         source={"url": "https://example.test/760.trace.zip"},
-        evaluator={"kind": "webarena_verified", "task_id": 760, "revision": 2},
+        evaluator={"kind": "webarena_verified", "task_id": 760, "revision": 6},
         environment={"kind": "webarena_verified"},
     )
     evidence = TaskEvidence(
@@ -276,13 +301,10 @@ def test_official_task_760_has_navigate_get_xhr_evaluator_defect(tmp_path, direc
 
     result = WebArenaVerifiedEvaluator(config_path=config).evaluate(task, evidence)
 
-    # v1.2.3 selects only main-frame navigation events for every navigate+GET
-    # task, so it never considers this expected OSRM XHR for either engine.
-    assert result.status == "fail"
-    assert result.checks[1]["actual"] == []
+    assert result.status == expected_status
 
 
-def test_official_task_678_has_navigate_get_xhr_evaluator_defect(tmp_path):
+def test_official_task_678_checks_the_order_grid_xhr(tmp_path):
     config = tmp_path / "config.json"
     config.write_text(
         json.dumps(
@@ -301,10 +323,29 @@ def test_official_task_678_has_navigate_get_xhr_evaluator_defect(tmp_path):
     )
     har.write_text(
         json.dumps(
-            {
-                "log": {
-                    "entries": [
-                        {
+                {
+                    "log": {
+                        "entries": [
+                            {
+                                "request": {
+                                    "method": "GET",
+                                    "url": "http://shopping-admin.test/admin/sales/order/",
+                                    "headers": [
+                                        {"name": "Accept", "value": "text/html"},
+                                        {"name": "Sec-Fetch-Dest", "value": "document"},
+                                        {"name": "Sec-Fetch-Mode", "value": "navigate"},
+                                        {"name": "Sec-Fetch-User", "value": "?1"},
+                                    ],
+                                },
+                                "response": {
+                                    "status": 200,
+                                    "redirectURL": "",
+                                    "headers": [],
+                                    "cookies": [],
+                                    "content": {"mimeType": "text/html", "text": ""},
+                                },
+                            },
+                            {
                             "request": {
                                 "method": "GET",
                                 "url": f"http://shopping-admin.test/admin/mui/index/render/?{query}",
@@ -338,7 +379,7 @@ def test_official_task_678_has_navigate_get_xhr_evaluator_defect(tmp_path):
         difficulty="simple",
         ground_truth={"expected_actions": []},
         source={"url": "https://example.test/678.trace.zip"},
-        evaluator={"kind": "webarena_verified", "task_id": 678, "revision": 2},
+        evaluator={"kind": "webarena_verified", "task_id": 678, "revision": 3},
         environment={"kind": "webarena_verified"},
     )
     evidence = TaskEvidence(
@@ -350,11 +391,16 @@ def test_official_task_678_has_navigate_get_xhr_evaluator_defect(tmp_path):
 
     result = WebArenaVerifiedEvaluator(config_path=config).evaluate(task, evidence)
 
-    assert result.status == "fail"
-    assert result.checks[1]["actual"] == []
+    assert result.status == "pass"
 
 
-def test_official_task_811_has_invalid_single_item_alternatives_schema(tmp_path):
+@pytest.mark.parametrize(
+    ("assignee_ids", "expected_status"),
+    [([2330], "pass"), ([2331], "fail")],
+)
+def test_official_task_811_compares_singleton_array_as_an_array(
+    tmp_path, assignee_ids, expected_status
+):
     config = tmp_path / "config.json"
     config.write_text(
         json.dumps({"environments": {"__GITLAB__": {"urls": ["http://gitlab.test"]}}}),
@@ -376,7 +422,9 @@ def test_official_task_811_has_invalid_single_item_alternatives_schema(tmp_path)
                                 "headers": [{"name": "content-type", "value": "application/json"}],
                                 "postData": {
                                     "mimeType": "application/json",
-                                    "text": '{"issue":{"assignee_ids":[2330]}}',
+                                    "text": json.dumps(
+                                        {"issue": {"assignee_ids": assignee_ids}}
+                                    ),
                                 },
                             },
                             "response": {
@@ -402,7 +450,7 @@ def test_official_task_811_has_invalid_single_item_alternatives_schema(tmp_path)
         difficulty="simple",
         ground_truth={"expected_actions": []},
         source={"url": "https://example.test/811.trace.zip"},
-        evaluator={"kind": "webarena_verified", "task_id": 811, "revision": 2},
+        evaluator={"kind": "webarena_verified", "task_id": 811, "revision": 3},
         environment={"kind": "webarena_verified"},
     )
     evidence = TaskEvidence(
@@ -414,6 +462,183 @@ def test_official_task_811_has_invalid_single_item_alternatives_schema(tmp_path)
 
     result = WebArenaVerifiedEvaluator(config_path=config).evaluate(task, evidence)
 
-    assert result.status == "error"
-    assert result.checks[1]["status"] == "error"
-    assert "Alternatives require 2+ items" in str(result.checks[1]["error_msg"])
+    assert result.status == expected_status
+    assert result.checks[1]["status"] == (
+        "success" if expected_status == "pass" else "failure"
+    )
+
+
+def _reddit_entry(url, post_data):
+    return {
+        "request": {
+            "method": "POST",
+            "url": url,
+            "headers": [{"name": "content-type", "value": "application/x-www-form-urlencoded"}],
+            "postData": {
+                "mimeType": "application/x-www-form-urlencoded",
+                "text": urlencode(post_data),
+                "params": [{"name": key, "value": str(value)} for key, value in post_data.items()],
+            },
+        },
+        "response": {
+            "status": 302,
+            "redirectURL": "",
+            "headers": [],
+            "cookies": [],
+            "content": {"mimeType": "text/html", "text": ""},
+        },
+    }
+
+
+def _official_task(task_id):
+    return BrowserTask(
+        task_id=f"webarena-{task_id}",
+        title=f"Official task {task_id}",
+        tool_calls=(),
+        assertions=(),
+        tags=(),
+        difficulty="simple",
+        ground_truth={"expected_actions": []},
+        source={"url": f"https://example.test/{task_id}.trace.zip"},
+        evaluator={"kind": "webarena_verified", "task_id": task_id, "revision": 2},
+        environment={"kind": "webarena_verified"},
+    )
+
+
+@pytest.mark.parametrize(
+    ("retrieved_data", "expected_status"),
+    [(None, "pass"), ([0], "fail")],
+)
+def test_official_task_319_preserves_null_retrieved_data_key(
+    tmp_path, retrieved_data, expected_status
+):
+    config = tmp_path / "config.json"
+    config.write_text(
+        json.dumps({"environments": {"__SHOPPING__": {"urls": ["http://shopping.test"]}}}),
+        encoding="utf-8",
+    )
+    har = tmp_path / "network.har"
+    har.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "GET",
+                                "url": "http://shopping.test/sales/order/history/",
+                                "headers": [],
+                            },
+                            "response": {
+                                "status": 200,
+                                "redirectURL": "",
+                                "headers": [],
+                                "cookies": [],
+                                "content": {"mimeType": "text/html", "text": "ok"},
+                            },
+                        }
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    result = WebArenaVerifiedEvaluator(config_path=config).evaluate(
+        _official_task(319),
+        TaskEvidence(
+            outputs={},
+            process_passed=True,
+            agent_response={
+                "task_type": "retrieve",
+                "status": "NOT_FOUND_ERROR",
+                "retrieved_data": retrieved_data,
+            },
+            network_har_path=har,
+        ),
+    )
+
+    assert result.status == expected_status
+
+
+def test_official_task_604_binds_dynamic_forum_id_from_the_submitted_form(tmp_path):
+    config = tmp_path / "config.json"
+    config.write_text(
+        json.dumps({"environments": {"__REDDIT__": {"urls": ["http://reddit.test"]}}}),
+        encoding="utf-8",
+    )
+    har = tmp_path / "network.har"
+    har.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        _reddit_entry(
+                            "http://reddit.test/submit",
+                            {
+                                "submission[title]": "what is the SOTA web navigation agent repo",
+                                "submission[forum]": "10043",
+                            },
+                        )
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = WebArenaVerifiedEvaluator(config_path=config).evaluate(
+        _official_task(604),
+        TaskEvidence(
+            outputs={},
+            process_passed=True,
+            agent_response={"task_type": "mutate", "status": "SUCCESS", "retrieved_data": None},
+            network_har_path=har,
+        ),
+    )
+
+    assert result.status == "pass"
+
+
+@pytest.mark.parametrize(
+    ("comment_key", "expected_status"),
+    [("reply_to_submission_69[comment]", "pass"), ("reply_to_submission_70[comment]", "fail")],
+)
+def test_official_task_611_requires_one_post_id_in_comment_url_and_form_key(
+    tmp_path, comment_key, expected_status
+):
+    config = tmp_path / "config.json"
+    config.write_text(
+        json.dumps({"environments": {"__REDDIT__": {"urls": ["http://reddit.test"]}}}),
+        encoding="utf-8",
+    )
+    har = tmp_path / "network.har"
+    har.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        _reddit_entry(
+                            "http://reddit.test/submit",
+                            {"submission[title]": "Harry Potter", "submission[forum]": "10037"},
+                        ),
+                        _reddit_entry(
+                            "http://reddit.test/f/books/69/-/comment",
+                            {comment_key: "Wonderful journey"},
+                        ),
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = WebArenaVerifiedEvaluator(config_path=config).evaluate(
+        _official_task(611),
+        TaskEvidence(
+            outputs={},
+            process_passed=True,
+            agent_response={"task_type": "mutate", "status": "SUCCESS", "retrieved_data": None},
+            network_har_path=har,
+        ),
+    )
+
+    assert result.status == expected_status
diff --git a/tests/trajectory_replay/test_audit_webarena_catalog_acceptance.py b/tests/trajectory_replay/test_audit_webarena_catalog_acceptance.py
new file mode 100644
index 000000000..aec894aaa
--- /dev/null
+++ b/tests/trajectory_replay/test_audit_webarena_catalog_acceptance.py
@@ -0,0 +1,712 @@
+import hashlib
+import json
+
+import pytest
+
+from harness.trajectory_replay import catalog_acceptance as acceptance
+from harness.trajectory_replay.webarena_runtime_source import WEBARENA_VERIFIED_REVISION
+
+
+@pytest.fixture(autouse=True)
+def _isolate_acceptance_contract_tests(monkeypatch):
+    monkeypatch.setattr(acceptance, "load_corrections", lambda: object())
+    monkeypatch.setattr(acceptance, "verify_webarena_runtime_source", lambda: None)
+
+
+def _run(
+    tmp_path,
+    name,
+    ids,
+    statuses,
+    *,
+    preparation="same-reset",
+    backend="browsertool",
+    catalog="catalog",
+    schema="lex_trajectory_replay_catalog_run/2",
+    engine=None,
+    binary="same-chromium",
+    action_overrides=None,
+):
+    directory = tmp_path / name
+    directory.mkdir()
+    inputs = {
+        "schema": schema,
+        "execution_order": ids,
+        "run_count_per_task": 1,
+        "site_reset_receipt_sha256": preparation,
+        "binary_sha256": binary,
+        "catalog_sha256": catalog,
+        "portable_suite_sha256": "suite",
+        "site_map_sha256": "sites",
+        "backend": backend,
+        "runner_revision": "revision",
+        "cowagent_revision": "cowagent",
+    }
+    if action_overrides is not None:
+        inputs["action_overrides_sha256"] = action_overrides
+    if schema in {
+        "lex_trajectory_replay_catalog_run/1",
+        "lex_trajectory_replay_catalog_run/2",
+        "lex_webarena_full_replay_run/2",
+    }:
+        inputs.update(
+            {
+                "engine": engine or "chromium",
+                "action_timeout_ms": 30_000,
+                "environment_profile_sha256": "environment",
+                "corrections_sha256": "corrections",
+                **(
+                    {"webarena_revision": WEBARENA_VERIFIED_REVISION}
+                    if schema in {
+                        "lex_trajectory_replay_catalog_run/1",
+                        "lex_trajectory_replay_catalog_run/2",
+                    }
+                    else {"evaluator_fixes_sha256": "fixes"}
+                ),
+            }
+        )
+    (directory / "inputs.json").write_text(json.dumps(inputs))
+    (directory / "progress.jsonl").write_text(
+        "".join(
+            json.dumps(
+                {
+                    "task_id": task_id,
+                    "status": statuses.get(task_id, "pass"),
+                    "score": 1,
+                    "action_count": 1,
+                }
+            )
+            + "\n"
+            for task_id in ids
+        )
+    )
+    return directory
+
+
+def test_remote_runtime_identity_is_bound_without_a_binary_or_credentials() -> None:
+    remote = {
+        "provider": "cloudflare-browser-run",
+        "browser": "kitesurf",
+        "protocol": "cdp",
+        "endpoint_contract": "wss://example/{account_id}?browser=kitesurf",
+        "account_id_sha256": "a" * 64,
+        "runtime_metadata": {
+            "protocolVersion": "1.3",
+            "product": "Chrome/145.0.0.0",
+            "revision": "@kitesurf",
+            "userAgent": "Kitesurf",
+            "jsVersion": "14.5",
+        },
+    }
+    identity, public = acceptance.runtime_identity(
+        {"binary_sha256": None, "remote_browser": remote}
+    )
+    assert identity.startswith("remote:")
+    assert public == {"remote_browser": remote}
+
+
+def test_remote_runtime_identity_rejects_persisted_credentials() -> None:
+    with pytest.raises(ValueError, match="contains credentials"):
+        acceptance.runtime_identity(
+            {
+                "binary_sha256": None,
+                "remote_browser": {
+                    "provider": "cloudflare-browser-run",
+                    "browser": "kitesurf",
+                    "protocol": "cdp",
+                    "endpoint_contract": "wss://example/{account_id}",
+                    "account_id_sha256": "a" * 64,
+                    "runtime_metadata": {
+                        "protocolVersion": "1.3",
+                        "product": "Chrome/145.0.0.0",
+                        "revision": "@kitesurf",
+                        "userAgent": "Kitesurf",
+                        "jsVersion": "14.5",
+                    },
+                    "Authorization": "Bearer private",
+                },
+            }
+        )
+
+
+def test_acceptance_rejects_observed_hosted_version_drift() -> None:
+    remote = {
+        "provider": "cloudflare-browser-run",
+        "browser": "kitesurf",
+        "protocol": "cdp",
+        "endpoint_contract": "wss://example/{account_id}?browser=kitesurf",
+        "account_id_sha256": "a" * 64,
+        "runtime_metadata": {
+            "protocolVersion": "1.3",
+            "product": "Chrome/145.0.0.0",
+            "revision": "@kitesurf",
+            "userAgent": "Kitesurf",
+            "jsVersion": "14.5",
+        },
+    }
+    observed = {**remote["runtime_metadata"], "product": "Chrome/146.0.0.0"}
+
+    with pytest.raises(ValueError, match="observed remote browser version changed"):
+        acceptance.assert_observed_runtime(
+            {"remote_browser": remote}, observed, task_id=7
+        )
+
+
+def test_request_bridge_identity_is_credential_free_and_canonical() -> None:
+    bridge = {
+        "kind": "local_http_fulfill/1",
+        "origins": ["http://a.example.test", "https://b.example.test"],
+    }
+    assert acceptance.request_bridge_identity(
+        {"engine": "kitesurf", "request_bridge": bridge}
+    ) == bridge
+
+    with pytest.raises(ValueError, match="missing its request bridge"):
+        acceptance.request_bridge_identity({"engine": "kitesurf"})
+
+    with pytest.raises(ValueError, match="only valid for Kitesurf"):
+        acceptance.request_bridge_identity(
+            {"engine": "chromium", "request_bridge": bridge}
+        )
+
+    with pytest.raises(ValueError, match="origin is invalid"):
+        acceptance.request_bridge_identity(
+            {
+                "engine": "kitesurf",
+                "request_bridge": {
+                    "kind": "local_http_fulfill/1",
+                    "origins": ["http://user:password@a.example.test"],
+                }
+            }
+        )
+
+
+def _scope(tmp_path, ids=None):
+    ids = list(range(812)) if ids is None else ids
+    path = tmp_path / "scope.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_replay_scope/1",
+                "official_dataset_sha256": "official-data",
+                "task_count": len(ids),
+                "task_ids": ids,
+            }
+        )
+    )
+    return path
+
+
+def _fake_verified(directory, row, **_kwargs):
+    return (dict(row, progress_sha256="progress", evidence_sha256={}), True)
+
+
+def _action_overrides(tmp_path):
+    path = tmp_path / "action-overrides.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_replay_action_overrides/1",
+                "amendments": [
+                    {
+                        "task_id": 7,
+                        "action_index": 1,
+                        "precondition_action_sha256": "a" * 64,
+                        "set": {"wait_for_selector_hidden": ".loading-mask"},
+                    }
+                ],
+            }
+        )
+        + "\n"
+    )
+    return path, hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def test_acceptance_still_reads_historical_run_schemas(tmp_path):
+    legacy_v1 = _run(
+        tmp_path,
+        "legacy-v1",
+        [4],
+        {},
+        schema="lex_webarena_full_replay_run/1",
+    )
+    legacy_v2 = _run(
+        tmp_path,
+        "legacy-v2",
+        [4],
+        {},
+        schema="lex_webarena_full_replay_run/2",
+    )
+
+    assert acceptance._read_run(legacy_v1)[0]["schema"].endswith("/1")
+    assert acceptance._read_run(legacy_v2)[0]["schema"].endswith("/2")
+
+
+def test_acceptance_defers_revision_compatibility_to_physical_evidence(tmp_path, monkeypatch):
+    def reject_unknown(_directory, _row, **kwargs):
+        assert kwargs["run_revision"] == "0" * 40
+        raise ValueError("pass lacks evaluator transition evidence")
+
+    monkeypatch.setattr(acceptance, "_verified_attempt", reject_unknown)
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {})
+    inputs = sweep / "inputs.json"
+    payload = json.loads(inputs.read_text())
+    payload["webarena_revision"] = "0" * 40
+    inputs.write_text(json.dumps(payload))
+
+    with pytest.raises(ValueError, match="lacks evaluator transition evidence"):
+        acceptance.audit(scope, [sweep], [])
+
+
+def test_catalog_run_v1_with_legacy_evaluator_hash_remains_readable(tmp_path):
+    run = _run(
+        tmp_path,
+        "catalog-v1",
+        [7],
+        {},
+        schema="lex_trajectory_replay_catalog_run/1",
+    )
+    inputs = run / "inputs.json"
+    payload = json.loads(inputs.read_text())
+    payload["evaluator_fixes_sha256"] = payload.pop("webarena_revision")
+    inputs.write_text(json.dumps(payload))
+
+    binding, _rows = acceptance._read_run(run)
+
+    assert acceptance._evaluator_identity_field(binding) == "evaluator_fixes_sha256"
+
+
+def test_acceptance_allows_declared_predecessor_only_with_paired_transition_receipt(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    transition = {
+        "from_revision": "0" * 40,
+        "to_revision": WEBARENA_VERIFIED_REVISION,
+    }
+    monkeypatch.setattr(acceptance, "load_transition", lambda _path: transition)
+    monkeypatch.setattr(acceptance, "load_receipt", lambda _path, _transition: ({}, set()))
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {7: "fail"})
+    inputs = sweep / "inputs.json"
+    payload = json.loads(inputs.read_text())
+    payload["webarena_revision"] = transition["from_revision"]
+    inputs.write_text(json.dumps(payload))
+    authority = tmp_path / "transition.json"
+    receipt = tmp_path / "receipt.json"
+    authority.write_text("{}")
+    receipt.write_text("{}")
+
+    ledger = acceptance.audit(
+        scope,
+        [sweep],
+        [],
+        evaluator_transition_path=authority,
+        evaluator_transition_receipt_path=receipt,
+    )
+
+    assert ledger["counts"]["unresolved"] == 1
+    assert ledger["evaluator_transition"]["reevaluated_passes"] == 0
+
+
+def test_acceptance_requires_transition_authority_and_receipt_together(tmp_path):
+    with pytest.raises(ValueError, match="must be supplied together"):
+        acceptance.audit(
+            _scope(tmp_path, [7]),
+            [],
+            [],
+            evaluator_transition_path=tmp_path / "transition.json",
+        )
+
+
+def test_acceptance_replaces_only_declared_changed_catalog_tasks(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    monkeypatch.setattr(acceptance, "_superseded_attempt", _fake_verified)
+    scope = _scope(tmp_path, [7, 8])
+    sweep = _run(tmp_path, "sweep", [7, 8], {})
+    replacement = _run(tmp_path, "replacement", [8], {}, catalog="new-catalog")
+    progress = replacement / "progress.jsonl"
+    rows = [json.loads(line) for line in progress.read_text().splitlines()]
+    rows[0]["executed_actions_sha256"] = "new-actions"
+    progress.write_text("".join(json.dumps(row) + "\n" for row in rows))
+    sweep_progress = sweep / "progress.jsonl"
+    rows = [json.loads(line) for line in sweep_progress.read_text().splitlines()]
+    for row in rows:
+        row["executed_actions_sha256"] = f'old-{row["task_id"]}'
+    sweep_progress.write_text("".join(json.dumps(row) + "\n" for row in rows))
+
+    ledger = acceptance.audit(
+        scope,
+        [sweep],
+        [],
+        replacement_dirs=[replacement],
+        superseded_catalog_tasks={8},
+    )
+
+    assert ledger["superseded_catalog_tasks"] == [8]
+    assert len(ledger["rows"][1]["attempts"]) == 2
+
+
+def test_catalog_replacement_excludes_only_superseded_transition_evidence(
+    tmp_path, monkeypatch
+):
+    source = {
+        "kind": "revision",
+        "identity": "0" * 40,
+        "evaluator_checksum": "old-evaluator",
+        "data_checksum": "old-data",
+    }
+    transition = {"compatible_sources": [source]}
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {})
+    replacement = _run(tmp_path, "replacement", [7], {}, catalog="new-catalog")
+    for directory, action in ((sweep, "old-actions"), (replacement, "new-actions")):
+        progress = directory / "progress.jsonl"
+        row = json.loads(progress.read_text())
+        row["executed_actions_sha256"] = action
+        progress.write_text(json.dumps(row) + "\n")
+    old_evaluation = sweep / "old-evaluation.json"
+    old_evaluation.write_text(
+        json.dumps(
+            {
+                "webarena_verified_evaluator_checksum": source["evaluator_checksum"],
+                "webarena_verified_data_checksum": source["data_checksum"],
+            }
+        )
+    )
+    replacement_evaluation = replacement / "replacement-evaluation.json"
+    replacement_evaluation.write_bytes(old_evaluation.read_bytes())
+    old_identity = (
+        7,
+        acceptance.file_sha256(old_evaluation),
+        acceptance.file_sha256(sweep / "inputs.json"),
+    )
+    replacement_identity = (
+        7,
+        acceptance.file_sha256(replacement_evaluation),
+        acceptance.file_sha256(replacement / "inputs.json"),
+    )
+    assert old_identity[:2] == replacement_identity[:2]
+    assert old_identity != replacement_identity
+    monkeypatch.setattr(acceptance, "load_transition", lambda _path: transition)
+    monkeypatch.setattr(
+        acceptance,
+        "load_receipt",
+        lambda _path, _transition: ({replacement_identity: {}}, {old_identity}),
+    )
+    monkeypatch.setattr(acceptance, "compatible_source", lambda *_args, **_kwargs: source)
+    monkeypatch.setattr(
+        acceptance,
+        "verify_physical_evidence",
+        lambda directory, _row: (
+            replacement_evaluation if directory == replacement else old_evaluation
+        ),
+    )
+    monkeypatch.setattr(acceptance, "_superseded_attempt", _fake_verified)
+
+    def verified(directory, row, **_kwargs):
+        assert directory == replacement
+        return (
+            {
+                **row,
+                "progress_sha256": "progress",
+                "evidence_sha256": {},
+                "evaluator_transition": {
+                    "old_evaluation_sha256": replacement_identity[1]
+                },
+            },
+            True,
+        )
+
+    monkeypatch.setattr(acceptance, "_verified_attempt", verified)
+    authority = tmp_path / "transition.json"
+    receipt = tmp_path / "receipt.json"
+    authority.write_text("{}")
+    receipt.write_text("{}")
+
+    ledger = acceptance.audit(
+        scope,
+        [sweep],
+        [],
+        replacement_dirs=[replacement],
+        superseded_catalog_tasks={7},
+        evaluator_transition_path=authority,
+        evaluator_transition_receipt_path=receipt,
+    )
+
+    assert ledger["counts"]["pass"] == 1
+    assert ledger["evaluator_transition"]["reevaluated_passes"] == 1
+
+
+def test_acceptance_rejects_replacement_with_unchanged_actions(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    monkeypatch.setattr(acceptance, "_superseded_attempt", _fake_verified)
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {})
+    replacement = _run(tmp_path, "replacement", [7], {}, catalog="new-catalog")
+    for directory in (sweep, replacement):
+        progress = directory / "progress.jsonl"
+        row = json.loads(progress.read_text())
+        row["executed_actions_sha256"] = "same-actions"
+        progress.write_text(json.dumps(row) + "\n")
+
+    with pytest.raises(ValueError, match="changed-catalog replacement"):
+        acceptance.audit(
+            scope,
+            [sweep],
+            [],
+            replacement_dirs=[replacement],
+            superseded_catalog_tasks={7},
+        )
+
+
+def test_acceptance_allows_repair_only_after_failed_catalog_replacement(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    monkeypatch.setattr(acceptance, "_superseded_attempt", _fake_verified)
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {})
+    replacement = _run(
+        tmp_path, "replacement", [7], {7: "fail"}, catalog="new-catalog"
+    )
+    repair = _run(tmp_path, "replacement-repair", [7], {}, catalog="new-catalog")
+    for directory, action in (
+        (sweep, "old-actions"),
+        (replacement, "new-actions"),
+        (repair, "new-actions"),
+    ):
+        progress = directory / "progress.jsonl"
+        row = json.loads(progress.read_text())
+        row["executed_actions_sha256"] = action
+        progress.write_text(json.dumps(row) + "\n")
+
+    ledger = acceptance.audit(
+        scope,
+        [sweep],
+        [],
+        replacement_dirs=[replacement, repair],
+        superseded_catalog_tasks={7},
+    )
+
+    assert ledger["counts"]["pass"] == 1
+    assert len(ledger["rows"][0]["attempts"]) == 3
+
+
+def test_acceptance_records_hash_bound_overrides_without_retaining_historical_file(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    path, digest = _action_overrides(tmp_path)
+    scope = _scope(tmp_path, [7])
+    sweep = _run(tmp_path, "sweep", [7], {}, action_overrides=digest)
+
+    ledger = acceptance.audit(scope, [sweep], [])
+    assert ledger["action_overrides_sha256s"] == [digest]
+
+    ledger = acceptance.audit(scope, [sweep], [], action_overrides_paths=[path])
+    assert ledger["action_overrides_sha256s"] == [digest]
+    assert ledger["runs"][0]["action_overrides_sha256"] == digest
+
+
+def test_acceptance_allows_binary_revisions_only_in_targeted_repairs(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path, [7, 8])
+    sweep = _run(tmp_path, "sweep", [7, 8], {8: "fail"})
+    repair = _run(tmp_path, "repair", [8], {}, binary="repaired-chromium")
+
+    ledger = acceptance.audit(scope, [sweep], [repair])
+
+    assert ledger["binary_sha256s"] == ["repaired-chromium", "same-chromium"]
+    assert ledger["participant"]["binary_sha256s"] == [
+        "repaired-chromium",
+        "same-chromium",
+    ]
+    assert "binary_sha256" not in ledger["participant"]
+
+
+def test_current_evaluation_rejects_an_unknown_declared_revision(tmp_path, monkeypatch):
+    task = tmp_path / "task-7"
+    task.mkdir()
+    evaluation = task / "evaluation.json"
+    evaluation.write_text(
+        json.dumps(
+            {
+                "webarena_verified_evaluator_checksum": "current-evaluator",
+                "webarena_verified_data_checksum": "current-data",
+            }
+        ),
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(acceptance, "verify_physical_evidence", lambda *_args: evaluation)
+    monkeypatch.setattr(acceptance, "compute_evaluator_checksum", lambda: "current-evaluator")
+
+    with pytest.raises(ValueError, match="current evaluation has a different revision"):
+        acceptance._verified_attempt(
+            tmp_path,
+            {"task_id": 7, "status": "pass", "action_count": 1},
+            expected_data_checksum="current-data",
+            benchmark=object(),
+            corrections=object(),
+            run_revision="0" * 40,
+        )
+
+
+def test_acceptance_keeps_one_ordered_sweep_and_only_repairs_a_failure(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    early = _run(tmp_path, "early", list(range(400)), {66: "fail"})
+    late = _run(tmp_path, "late", list(range(400, 812)), {})
+    repair = _run(tmp_path, "repair", [66], {})
+
+    ledger = acceptance.audit(scope, [early, late], [repair])
+
+    assert ledger["counts"] == {
+        "total": 812,
+        "pass": 812,
+        "unresolved": 0,
+        "physical_attempts": 813,
+        "excluded_duplicate_attempts": 0,
+    }
+    assert [attempt["status"] for attempt in ledger["rows"][66]["attempts"]] == [
+        "fail",
+        "pass",
+    ]
+    assert len(ledger["rows"][67]["attempts"]) == 1
+    assert ledger["schema"] == "lex_trajectory_replay_catalog_acceptance/1"
+
+
+def test_acceptance_supports_a_noncontiguous_catalog_scope(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    ids = [4, 11, 23]
+    scope = _scope(tmp_path, ids)
+    sweep = _run(tmp_path, "sweep", ids, {11: "fail"})
+    repair = _run(tmp_path, "repair", [11], {})
+
+    ledger = acceptance.audit(scope, [sweep], [repair])
+
+    assert [row["task_id"] for row in ledger["rows"]] == ids
+    assert ledger["counts"]["total"] == 3
+
+
+def test_acceptance_binds_repair_on_a_fresh_site_without_repeating_passes(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    sweep = _run(tmp_path, "sweep", list(range(812)), {496: "fail"})
+    repair = _run(tmp_path, "repair", [496], {}, preparation="spare-slice")
+
+    ledger = acceptance.audit(scope, [sweep], [repair])
+
+    assert ledger["counts"]["pass"] == 812
+    assert ledger["site_preparation_receipt_sha256s"] == ["same-reset", "spare-slice"]
+    assert ledger["runs"][1]["site_reset_receipt_sha256"] == "spare-slice"
+
+
+@pytest.mark.parametrize(
+    ("change", "message"),
+    [
+        ({"preparation": "other"}, "ordered sweep changed its execution contract"),
+        ({"catalog": "other"}, "ordered sweep changed its execution contract"),
+        ({"binary": "other"}, "ordered sweep changed its execution contract"),
+        ({"backend": "playwright"}, "different execution backend"),
+    ],
+)
+def test_acceptance_rejects_mixed_sweep_contracts(tmp_path, monkeypatch, change, message):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    early = _run(tmp_path, "early", list(range(400)), {})
+    late = _run(tmp_path, "late", list(range(400, 812)), {}, **change)
+
+    with pytest.raises(ValueError, match=message):
+        acceptance.audit(scope, [early, late], [])
+
+
+def test_acceptance_rejects_rerunning_a_pass(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    sweep = _run(tmp_path, "sweep", list(range(812)), {})
+    repeated = _run(tmp_path, "repeated", [66], {})
+
+    with pytest.raises(ValueError, match="repair does not follow a failure"):
+        acceptance.audit(scope, [sweep], [repeated])
+
+
+def test_acceptance_records_an_explicit_duplicate_without_replacing_first_pass(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    sweep = _run(tmp_path, "sweep", list(range(812)), {500: "fail"})
+    repair = _run(tmp_path, "repair", [499, 500], {})
+
+    ledger = acceptance.audit(scope, [sweep], [repair], excluded_duplicate_repair_tasks={499})
+
+    assert len(ledger["rows"][499]["attempts"]) == 1
+    assert [row["status"] for row in ledger["rows"][500]["attempts"]] == [
+        "fail",
+        "pass",
+    ]
+    assert ledger["counts"]["physical_attempts"] == 814
+    assert ledger["excluded_attempts"][0]["task_id"] == 499
+
+
+def test_acceptance_rejects_an_order_gap(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    gap = _run(tmp_path, "gap", [task_id for task_id in range(812) if task_id != 500], {})
+
+    with pytest.raises(ValueError, match="incomplete or changed official order"):
+        acceptance.audit(scope, [gap], [])
+
+
+def test_acceptance_rejects_tampered_failure_evidence(tmp_path):
+    run = tmp_path / "run"
+    task = run / "task-7"
+    task.mkdir(parents=True)
+    trace = task / "trace.json"
+    trace.write_text("[]\n", encoding="utf-8")
+    row = {
+        "task_id": 7,
+        "status": "fail",
+        "score": 0,
+        "action_count": 1,
+        "evidence_sha256": {"trace": hashlib.sha256(trace.read_bytes()).hexdigest()},
+    }
+    (run / "progress.jsonl").write_text(json.dumps(row) + "\n", encoding="utf-8")
+    trace.write_text('[{"changed":true}]\n', encoding="utf-8")
+
+    with pytest.raises(ValueError, match="evidence hash changed"):
+        acceptance._verified_attempt(
+            run,
+            row,
+            expected_data_checksum="official-data",
+            benchmark=object(),
+            corrections=object(),
+        )
+
+
+def test_acceptance_records_v2_participant_and_execution_contract(tmp_path, monkeypatch):
+    monkeypatch.setattr(acceptance, "_verified_attempt", _fake_verified)
+    scope = _scope(tmp_path)
+    sweep = _run(
+        tmp_path,
+        "sweep",
+        list(range(812)),
+        {},
+        schema="lex_webarena_full_replay_run/2",
+        engine="moli",
+    )
+
+    ledger = acceptance.audit(scope, [sweep], [])
+
+    assert ledger["participant"] == {
+        "engine": "moli",
+        "backend": "browsertool",
+        "binary_sha256": "same-chromium",
+        "site_map_sha256": "sites",
+    }
+    assert ledger["runs"][0]["action_timeout_ms"] == 30_000
+    assert ledger["runs"][0]["environment_profile_sha256"] == "environment"
diff --git a/tests/trajectory_replay/test_catalog_action_audit.py b/tests/trajectory_replay/test_catalog_action_audit.py
new file mode 100644
index 000000000..684125db3
--- /dev/null
+++ b/tests/trajectory_replay/test_catalog_action_audit.py
@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay import catalog_release
+from harness.trajectory_replay.catalog_action_audit import (
+    scan_function_body_actions,
+    validate_catalog_action_audit,
+    validate_function_body_overrides,
+)
+from harness.trajectory_replay.catalog_contracts import file_sha256
+from harness.trajectory_replay.webarena_replay import _json_sha256
+
+
+def _write_inputs(tmp_path: Path, scripts: list[str]) -> tuple[Path, Path]:
+    rows = []
+    for task_id, script in enumerate(scripts, 1):
+        action = {"action": "evaluate", "script": script}
+        rows.append(
+            {
+                "task_id": task_id,
+                "mode": "direct",
+                "source_actions": [action],
+                "bound_calls_sha256": _json_sha256([action]),
+            }
+        )
+    catalog = tmp_path / "catalog.json"
+    catalog.write_text(json.dumps({"task_ids": list(range(1, len(rows) + 1)), "rows": rows}))
+    suite = tmp_path / "suite.json"
+    suite.write_text(json.dumps({"schema": "lex_browser_use_tasks/4", "tasks": []}))
+    return catalog, suite
+
+
+def test_static_audit_only_selects_script_invalid_function_body_valid_evaluate(tmp_path: Path):
+    catalog, suite = _write_inputs(
+        tmp_path,
+        [
+            "() => { return true; }",
+            "const value = 1; return value;",
+        ],
+    )
+
+    audit = scan_function_body_actions(catalog, suite)
+
+    assert audit["evaluate_action_count"] == 2
+    assert [(row["task_id"], row["action_index"]) for row in audit["function_body_findings"]] == [
+        (2, 1)
+    ]
+
+
+def test_static_audit_rejects_script_invalid_under_both_grammars(tmp_path: Path):
+    catalog, suite = _write_inputs(tmp_path, ["return }"])
+    with pytest.raises(ValueError, match="invalid under both grammars"):
+        scan_function_body_actions(catalog, suite)
+
+
+def test_static_audit_accepts_embedded_function_body_mode(tmp_path: Path):
+    catalog, suite = _write_inputs(tmp_path, ["const value = 1; return value;"])
+    payload = json.loads(catalog.read_text(encoding="utf-8"))
+    payload["rows"][0]["source_actions"][0]["script_mode"] = "function_body"
+    catalog.write_text(json.dumps(payload), encoding="utf-8")
+
+    audit = scan_function_body_actions(catalog, suite)
+
+    assert audit["evaluate_action_count"] == 1
+    assert audit["function_body_findings"] == []
+
+
+def _write_overrides(path: Path, audit: dict) -> None:
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_replay_action_overrides/1",
+                "amendments": [
+                    {
+                        "task_id": row["task_id"],
+                        "action_index": row["action_index"],
+                        "precondition_action_sha256": row["precondition_action_sha256"],
+                        "set": {"script_mode": "function_body"},
+                    }
+                    for row in audit["function_body_findings"]
+                ],
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+
+def test_release_validation_binds_audit_to_catalog_suite_and_live_scan(tmp_path: Path):
+    catalog, suite = _write_inputs(tmp_path, ["return true;"])
+    audit = scan_function_body_actions(catalog, suite)
+    audit_path = tmp_path / "audit.json"
+    audit_path.write_text(json.dumps(audit), encoding="utf-8")
+    overrides = tmp_path / "overrides.json"
+    _write_overrides(overrides, audit)
+
+    assert validate_catalog_action_audit(audit_path, catalog, suite, overrides) == audit
+
+    changed = dict(audit, catalog_sha256="0" * 64)
+    audit_path.write_text(json.dumps(changed), encoding="utf-8")
+    with pytest.raises(ValueError, match="catalog or suite binding"):
+        validate_catalog_action_audit(audit_path, catalog, suite, overrides)
+
+
+def test_release_validation_detects_new_unamended_top_level_return(tmp_path: Path):
+    catalog, suite = _write_inputs(tmp_path, ["return true;"])
+    audit = scan_function_body_actions(catalog, suite)
+    overrides = tmp_path / "overrides.json"
+    _write_overrides(overrides, audit)
+
+    payload = json.loads(catalog.read_text(encoding="utf-8"))
+    action = {"action": "evaluate", "script": "const next = 2; return next;"}
+    payload["task_ids"].append(2)
+    payload["rows"].append(
+        {
+            "task_id": 2,
+            "mode": "direct",
+            "source_actions": [action],
+            "bound_calls_sha256": _json_sha256([action]),
+        }
+    )
+    catalog.write_text(json.dumps(payload), encoding="utf-8")
+    audit["catalog_sha256"] = file_sha256(catalog)
+    audit_path = tmp_path / "audit.json"
+    audit_path.write_text(json.dumps(audit), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="differs from the live catalog scan"):
+        validate_catalog_action_audit(audit_path, catalog, suite, overrides)
+
+
+def test_versioned_override_data_matches_complete_812_action_audit():
+    root = Path(__file__).resolve().parents[2]
+    directory = root / "evaluation/trajectory-replay/webarena-verified-full"
+    audit = json.loads((directory / "action-overrides-audit.json").read_text(encoding="utf-8"))
+
+    assert audit["evaluate_action_count"] == 804
+    assert len(audit["function_body_findings"]) == 21
+    assert (
+        len({(row["task_id"], row["action_index"]) for row in audit["function_body_findings"]})
+        == 21
+    )
+    validate_function_body_overrides(audit, directory / "action-overrides.json")
+
+
+def test_catalog_release_gate_runs_live_action_audit(monkeypatch, tmp_path: Path):
+    profile = tmp_path / "current-report.json"
+    data_home = tmp_path / "evidence"
+    seen = {}
+    monkeypatch.setattr(catalog_release, "verify_webarena_runtime_source", lambda: None)
+    monkeypatch.setattr(catalog_release, "repository_root", lambda _path: tmp_path)
+    monkeypatch.setattr(
+        catalog_release,
+        "validate_catalog_report_profile",
+        lambda _profile, _data_home: {
+            "evidence_root": "run",
+                "accepted_catalog": {"path": "catalog.json"},
+                "accepted_portable_suite": {"path": "suite.json"},
+                "report_manifest_path": "report/manifest.json",
+                "report_archive_path": "report.zip",
+                "publication": {"share_url": "https://artifact.example/report"},
+        },
+    )
+    monkeypatch.setattr(
+        catalog_release,
+        "validate_catalog_action_audit",
+        lambda *args: seen.setdefault("paths", args),
+    )
+    monkeypatch.setattr(
+        catalog_release,
+        "validate_catalog_acceptance_report",
+        lambda *args, **kwargs: seen.setdefault("report", (args, kwargs)),
+    )
+    monkeypatch.setattr(
+        catalog_release,
+        "validate_catalog_comparison_release",
+        lambda *args: {
+            "publication": {"share_url": "https://artifact.example/comparison"}
+        },
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "release-check",
+            "--profile",
+            str(profile),
+            "--data-home",
+            str(data_home),
+        ],
+    )
+
+    catalog_release.main()
+
+    assert seen["paths"] == (
+        tmp_path
+        / "evaluation/trajectory-replay/webarena-verified-full/action-overrides-audit.json",
+        data_home / "run/catalog.json",
+        data_home / "run/suite.json",
+        tmp_path / "evaluation/trajectory-replay/webarena-verified-full/action-overrides.json",
+    )
+    assert seen["report"][0] == (
+        data_home / "run/report",
+        data_home / "run/report.zip",
+    )
diff --git a/tests/trajectory_replay/test_catalog_comparison.py b/tests/trajectory_replay/test_catalog_comparison.py
new file mode 100644
index 000000000..b78bd46ff
--- /dev/null
+++ b/tests/trajectory_replay/test_catalog_comparison.py
@@ -0,0 +1,469 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from harness.trajectory_replay import catalog_comparison
+from harness.trajectory_replay.catalog_contracts import file_sha256
+from harness.trajectory_replay.webarena_replay import _json_sha256
+
+
+def _write(path: Path, value: object) -> str:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value) + "\n", encoding="utf-8")
+    return file_sha256(path)
+
+
+def test_site_preparation_index_accepts_a_restart_receipt_name(tmp_path: Path) -> None:
+    receipt = tmp_path / "four-site-restart-receipt-v2.json"
+    digest = _write(receipt, {"schema": "lex_webarena_environment_preparation/1"})
+
+    assert catalog_comparison._site_reset_index(tmp_path)[digest]["path"] == str(receipt)
+
+
+def test_catalog_dataset_uses_the_executed_catalog_actions(monkeypatch, tmp_path: Path) -> None:
+    actions = [{"action": "navigate", "url": "https://target.example/task"}]
+    catalog = {
+        "rows": [
+            {
+                "task_id": 0,
+                "mode": "direct",
+                "source_kind": "reviewed",
+                "source_actions": actions,
+                "source_site_urls": {},
+                "bound_calls_sha256": _json_sha256(actions),
+            }
+        ]
+    }
+    monkeypatch.setattr(
+        catalog_comparison,
+        "load_catalog_inputs",
+        lambda *_args, **_kwargs: (catalog, {}, {}),
+    )
+    official = SimpleNamespace(
+        model_dump=lambda **_kwargs: {
+            "intent": "Open the task",
+            "intent_template_id": 1,
+            "revision": 1,
+            "sites": ["wikipedia"],
+            "start_urls": ["https://target.example/task"],
+            "eval": [{"expected": {"task_type": "NAVIGATE", "status": "SUCCESS"}}],
+        }
+    )
+    monkeypatch.setattr(
+        catalog_comparison,
+        "remote_benchmark",
+        lambda _sites: SimpleNamespace(get_task=lambda _task_id: official),
+    )
+    monkeypatch.setattr(
+        catalog_comparison,
+        "load_corrections",
+        lambda: SimpleNamespace(records={}),
+    )
+    monkeypatch.setattr(
+        catalog_comparison,
+        "rebind_remote_calls",
+        lambda calls, *_args: calls,
+    )
+    monkeypatch.setattr(catalog_comparison, "row_source_sites", lambda *_args: {})
+
+    catalog_path = tmp_path / "catalog.json"
+    _write(catalog_path, catalog)
+
+    dataset = catalog_comparison.build_catalog_comparison_dataset(
+        catalog_path=catalog_path,
+        suite_path=tmp_path / "suite.json",
+        site_map_path=tmp_path / "sites.json",
+    )
+
+    assert dataset["counts"]["verified_tasks"] == 1
+    assert dataset["trajectories"][0]["tool_calls"] == actions
+    assert dataset["trajectories"][0]["evaluation_input"]["status"] == "SUCCESS"
+
+
+def _participant_fixture(
+    root: Path,
+    *,
+    participant_id: str,
+    engine: str,
+    rows: list[dict],
+    site_reset: Path,
+    remote_browser: dict | None = None,
+    request_bridge: dict | None = None,
+) -> dict:
+    local_site_reset = root / "site-reset.json"
+    _write(local_site_reset, json.loads(site_reset.read_text(encoding="utf-8")))
+    ledger_rows = []
+    for task_id, specification in enumerate(rows):
+        attempts = []
+        if specification.get("setup_failure"):
+            attempts.append(
+                {
+                    "status": "candidate_setup_failure",
+                    "score": None,
+                    "progress_sha256": "0" * 64,
+                }
+            )
+        run = root / f"{participant_id}-{task_id}"
+        action = {"action": "navigate", "url": f"https://example.test/{task_id}"}
+        error = specification.get("error", "")
+        status = specification["status"]
+        evidence = {
+            "trace": _write(run / f"task-{task_id}" / "trace.json", []),
+            "readiness": _write(run / f"task-{task_id}" / "readiness.json", []),
+        }
+        if specification.get("evaluation", status == "pass"):
+            evidence["evaluation"] = _write(
+                run / f"task-{task_id}" / "evaluation.json",
+                {"status": status, "score": 1.0 if status == "pass" else 0.0},
+            )
+        physical = {
+            "task_id": task_id,
+            "status": status,
+            "score": 1.0 if status == "pass" else None,
+            "action_count": 1,
+            "completed_actions": 0 if error else 1,
+            "duration_ms": 10 + task_id,
+            "executed_actions_sha256": _json_sha256([action]),
+            "error": error,
+            **(
+                {"failure_stage": specification["failure_stage"]}
+                if specification.get("failure_stage")
+                else {}
+            ),
+            "evidence_sha256": evidence,
+            **(
+                {"runtime_metadata": remote_browser["runtime_metadata"]}
+                if remote_browser is not None
+                else {}
+            ),
+        }
+        progress = run / "progress.jsonl"
+        progress.parent.mkdir(parents=True, exist_ok=True)
+        progress.write_text(json.dumps(physical) + "\n", encoding="utf-8")
+        _write(
+            run / "inputs.json",
+            {
+                "engine": engine,
+                "binary_sha256": (
+                    None if remote_browser is not None else participant_id[0] * 64
+                ),
+                **(
+                    {"remote_browser": remote_browser}
+                    if remote_browser is not None
+                    else {}
+                ),
+                **(
+                    {"request_bridge": request_bridge}
+                    if request_bridge is not None
+                    else {}
+                ),
+                "site_reset_receipt_sha256": file_sha256(local_site_reset),
+            },
+        )
+        attempts.append(
+            {
+                "status": status,
+                "score": physical["score"],
+                "progress_sha256": file_sha256(progress),
+            }
+        )
+        ledger_rows.append({"task_id": task_id, "status": status, "attempts": attempts})
+    ledger = root / f"{participant_id}-ledger.json"
+    _write(
+        ledger,
+        {
+            "participant": {
+                "engine": engine,
+                "backend": "playwright",
+                **(
+                    {"remote_browser": remote_browser}
+                    if remote_browser is not None
+                    else {"binary_sha256": participant_id[0] * 64}
+                ),
+                **(
+                    {"request_bridge": request_bridge}
+                    if request_bridge is not None
+                    else {}
+                ),
+            },
+            "site_reset_receipt_sha256": file_sha256(local_site_reset),
+            "rows": ledger_rows,
+        },
+    )
+    return {
+        "id": participant_id,
+        "engine": engine,
+        "ledger_path": str(ledger),
+        "evidence_root": str(root),
+    }
+
+
+def _kitesurf_identity() -> dict:
+    return {
+        "provider": "cloudflare-browser-run",
+        "browser": "kitesurf",
+        "protocol": "cdp",
+        "endpoint_contract": "wss://example/{account_id}?browser=kitesurf",
+        "account_id_sha256": "a" * 64,
+        "runtime_metadata": {
+            "protocolVersion": "1.3",
+            "product": "Chrome/145.0.0.0",
+            "revision": "@kitesurf",
+            "userAgent": "Kitesurf",
+            "jsVersion": "14.5",
+        },
+    }
+
+
+def _kitesurf_request_bridge() -> dict:
+    return {
+        "kind": "local_http_fulfill/1",
+        "origins": ["http://private.example.test"],
+    }
+
+
+def test_catalog_results_reject_hosted_runtime_version_drift(tmp_path: Path) -> None:
+    dataset = {
+        "trajectories": [
+            {
+                "task_id": 0,
+                "task": {"intent": "Task 0", "sites": ["wikipedia"]},
+                "evaluation_input": {"task_type": "NAVIGATE", "status": "SUCCESS"},
+                "tool_calls": [
+                    {"action": "navigate", "url": "https://example.test/0"}
+                ],
+            }
+        ]
+    }
+    reset = tmp_path / "site-reset.json"
+    _write(reset, {"status": "ready"})
+    participant = _participant_fixture(
+        tmp_path / "remote",
+        participant_id="kitesurf",
+        engine="kitesurf",
+        rows=[{"status": "pass"}],
+        site_reset=reset,
+        remote_browser=_kitesurf_identity(),
+        request_bridge=_kitesurf_request_bridge(),
+    )
+    progress = tmp_path / "remote" / "kitesurf-0" / "progress.jsonl"
+    physical = json.loads(progress.read_text(encoding="utf-8"))
+    physical["runtime_metadata"]["product"] = "Chrome/146.0.0.0"
+    progress.write_text(json.dumps(physical) + "\n", encoding="utf-8")
+    ledger = Path(participant["ledger_path"])
+    payload = json.loads(ledger.read_text(encoding="utf-8"))
+    payload["rows"][0]["attempts"][0]["progress_sha256"] = file_sha256(progress)
+    ledger.write_text(json.dumps(payload) + "\n", encoding="utf-8")
+
+    with pytest.raises(ValueError, match="observed remote browser version changed"):
+        catalog_comparison.build_catalog_comparison_results(
+            dataset=dataset,
+            dataset_sha256="1" * 64,
+            profile_sha256="2" * 64,
+            scope_sha256="3" * 64,
+            site_map_sha256="4" * 64,
+            participants=[participant],
+        )
+
+
+def test_catalog_results_reject_request_bridge_drift(tmp_path: Path) -> None:
+    dataset = {
+        "trajectories": [
+            {
+                "task_id": 0,
+                "task": {"intent": "Task 0", "sites": ["wikipedia"]},
+                "evaluation_input": {"task_type": "NAVIGATE", "status": "SUCCESS"},
+                "tool_calls": [
+                    {"action": "navigate", "url": "https://example.test/0"}
+                ],
+            }
+        ]
+    }
+    reset = tmp_path / "site-reset.json"
+    _write(reset, {"status": "ready"})
+    bridge = _kitesurf_request_bridge()
+    participant = _participant_fixture(
+        tmp_path / "remote",
+        participant_id="kitesurf",
+        engine="kitesurf",
+        rows=[{"status": "pass"}],
+        site_reset=reset,
+        remote_browser=_kitesurf_identity(),
+        request_bridge=bridge,
+    )
+    inputs = tmp_path / "remote" / "kitesurf-0" / "inputs.json"
+    payload = json.loads(inputs.read_text(encoding="utf-8"))
+    payload["request_bridge"]["origins"] = ["http://other.example.test"]
+    inputs.write_text(json.dumps(payload) + "\n", encoding="utf-8")
+
+    with pytest.raises(ValueError, match="physical request bridge identity changed"):
+        catalog_comparison.build_catalog_comparison_results(
+            dataset=dataset,
+            dataset_sha256="1" * 64,
+            profile_sha256="2" * 64,
+            scope_sha256="3" * 64,
+            site_map_sha256="4" * 64,
+            participants=[participant],
+        )
+
+
+def test_catalog_results_accept_each_binary_in_bound_repair_chain(tmp_path: Path) -> None:
+    dataset = {
+        "trajectories": [
+            {
+                "task_id": task_id,
+                "task": {"intent": f"Task {task_id}", "sites": ["wikipedia"]},
+                "evaluation_input": {"task_type": "NAVIGATE", "status": "SUCCESS"},
+                "tool_calls": [
+                    {
+                        "action": "navigate",
+                        "url": f"https://example.test/{task_id}",
+                    }
+                ],
+            }
+            for task_id in range(2)
+        ]
+    }
+    reset = tmp_path / "site-reset.json"
+    _write(reset, {"status": "ready"})
+    participant = _participant_fixture(
+        tmp_path / "moli",
+        participant_id="moli",
+        engine="moli",
+        rows=[{"status": "pass"}, {"status": "pass"}],
+        site_reset=reset,
+    )
+    first_binary = "m" * 64
+    second_binary = "b" * 64
+    second_inputs = tmp_path / "moli" / "moli-1" / "inputs.json"
+    inputs = json.loads(second_inputs.read_text(encoding="utf-8"))
+    inputs["binary_sha256"] = second_binary
+    second_inputs.write_text(json.dumps(inputs) + "\n", encoding="utf-8")
+    ledger = Path(participant["ledger_path"])
+    payload = json.loads(ledger.read_text(encoding="utf-8"))
+    payload["participant"].pop("binary_sha256")
+    payload["participant"]["binary_sha256s"] = [first_binary, second_binary]
+    ledger.write_text(json.dumps(payload) + "\n", encoding="utf-8")
+
+    results = catalog_comparison.build_catalog_comparison_results(
+        dataset=dataset,
+        dataset_sha256="1" * 64,
+        profile_sha256="2" * 64,
+        scope_sha256="3" * 64,
+        site_map_sha256="4" * 64,
+        participants=[participant],
+    )
+
+    assert {row["binary_sha256"] for row in results["attempts"]} == {
+        first_binary,
+        second_binary,
+    }
+
+
+def test_catalog_results_keep_one_logical_result_and_valid_physical_history(
+    tmp_path: Path,
+) -> None:
+    dataset = {
+        "trajectories": [
+            {
+                "task_id": task_id,
+                "task": {"intent": f"Task {task_id}", "sites": ["wikipedia"]},
+                "evaluation_input": {"task_type": "NAVIGATE", "status": "SUCCESS"},
+                "tool_calls": [
+                    {"action": "navigate", "url": f"https://example.test/{task_id}"}
+                ],
+            }
+            for task_id in range(2)
+        ]
+    }
+    site_reset = tmp_path / "site-reset.json"
+    _write(site_reset, {"status": "ready"})
+    participants = [
+        _participant_fixture(
+            tmp_path / "a",
+            participant_id="moli",
+            engine="moli",
+            rows=[{"status": "pass"}, {"status": "pass"}],
+            site_reset=site_reset,
+        ),
+        _participant_fixture(
+            tmp_path / "b",
+            participant_id="lightpanda",
+            engine="lightpanda",
+            rows=[
+                {"status": "pass", "setup_failure": True},
+                {
+                    "status": "fail",
+                    "error": "Traceback in _login_context: element is outside of the viewport",
+                    "evaluation": False,
+                },
+            ],
+            site_reset=site_reset,
+        ),
+    ]
+
+    results = catalog_comparison.build_catalog_comparison_results(
+        dataset=dataset,
+        dataset_sha256="1" * 64,
+        profile_sha256="2" * 64,
+        scope_sha256="3" * 64,
+        site_map_sha256="4" * 64,
+        participants=participants,
+    )
+
+    assert len(results["attempts"]) == 4
+    assert len(results["physical_executions"]) == 4
+    lightpanda = {
+        row["task_id"]: row
+        for row in results["attempts"]
+        if row["participant_id"] == "lightpanda"
+    }
+    assert lightpanda[0]["status"] == "pass"
+    assert lightpanda[1]["process_status"] == "fail"
+    assert lightpanda[1]["failure_stage"] == "browser_session_setup"
+    assert lightpanda[1]["evaluator_status"] == "not_run"
+    assert lightpanda[1]["evidence"]["site_reset"]["sha256"] == file_sha256(
+        site_reset
+    )
+
+
+def test_catalog_results_preserve_runner_failure_stage(tmp_path: Path) -> None:
+    dataset = {
+        "trajectories": [
+            {
+                "task_id": 0,
+                "task": {"intent": "Task 0", "sites": ["wikipedia"]},
+                "evaluation_input": {"task_type": "NAVIGATE", "status": "SUCCESS"},
+                "tool_calls": [{"action": "navigate", "url": "https://example.test/0"}],
+            }
+        ]
+    }
+    site_reset = tmp_path / "site-reset.json"
+    _write(site_reset, {"status": "ready"})
+    participant = _participant_fixture(
+        tmp_path / "a",
+        participant_id="obscura",
+        engine="obscura",
+        rows=[
+            {
+                "status": "fail",
+                "error": "Evaluator crashed",
+                "evaluation": False,
+                "failure_stage": "official_evaluator_execution",
+            }
+        ],
+        site_reset=site_reset,
+    )
+    results = catalog_comparison.build_catalog_comparison_results(
+        dataset=dataset,
+        dataset_sha256="1" * 64,
+        profile_sha256="2" * 64,
+        scope_sha256="3" * 64,
+        site_map_sha256="4" * 64,
+        participants=[participant],
+    )
+    assert results["attempts"][0]["failure_stage"] == "official_evaluator_execution"
diff --git a/tests/trajectory_replay/test_catalog_contracts.py b/tests/trajectory_replay/test_catalog_contracts.py
new file mode 100644
index 000000000..74b0f378b
--- /dev/null
+++ b/tests/trajectory_replay/test_catalog_contracts.py
@@ -0,0 +1,438 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay.catalog_contracts import (
+    ACCEPTANCE_SCHEMAS,
+    RUN_SCHEMAS,
+    file_sha256,
+    repository_root,
+    validate_catalog_comparison_release,
+    validate_catalog_report_profile,
+    validate_environment_preparation,
+)
+
+
+def test_current_contracts_keep_historical_run_and_acceptance_schemas_readable():
+    assert "lex_trajectory_replay_catalog_run/1" in RUN_SCHEMAS
+    assert "lex_webarena_full_replay_run/1" in RUN_SCHEMAS
+    assert "lex_webarena_full_replay_run/2" in RUN_SCHEMAS
+    assert "lex_webarena_full_acceptance_ledger/1" in ACCEPTANCE_SCHEMAS
+
+
+def _files(tmp_path):
+    profile = tmp_path / "environment.json"
+    profile.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_replay_environment/1",
+                "deployment": {
+                    "acceptance_app_id": 202,
+                    "source_app_id": 195,
+                    "site_reset_between_tasks": False,
+                },
+                "images": {
+                    site: f"sha256:{site}"
+                    for site in ("shopping", "shopping_admin", "reddit", "gitlab")
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+    receipt = tmp_path / "receipt.json"
+    receipt.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_environment_preparation/1",
+                "method": "new_rainbond_application_from_pinned_images",
+                "source_app_id": 195,
+                "fresh_app_id": 202,
+                "site_component_images": {
+                    site: f"registry.example/image@sha256:{site}"
+                    for site in ("shopping", "shopping_admin", "reddit", "gitlab")
+                },
+                "database_volume_mounts": {
+                    site: 0 for site in ("shopping", "shopping_admin", "reddit", "gitlab")
+                },
+                "initialization": {
+                    site: "ready" for site in ("shopping", "shopping_admin", "reddit", "gitlab")
+                },
+                "public_http_status": {
+                    site: 200 for site in ("shopping", "shopping_admin", "reddit", "gitlab")
+                },
+                "shared_site_components": ["map", "wikipedia"],
+                "site_reset_between_tasks": False,
+            }
+        ),
+        encoding="utf-8",
+    )
+    return receipt, profile
+
+
+def test_environment_preparation_is_bound_to_versioned_profile(tmp_path):
+    receipt, profile = _files(tmp_path)
+    assert validate_environment_preparation(receipt, profile)["fresh_app_id"] == 202
+
+
+def test_environment_preparation_still_reads_legacy_profile_schema(tmp_path):
+    receipt, profile = _files(tmp_path)
+    payload = json.loads(profile.read_text(encoding="utf-8"))
+    payload["schema"] = "lex_webarena_verified_full_environment/1"
+    profile.write_text(json.dumps(payload), encoding="utf-8")
+
+    assert validate_environment_preparation(receipt, profile)["fresh_app_id"] == 202
+
+
+def test_environment_preparation_accepts_complete_pinned_component_restart(tmp_path):
+    receipt, profile = _files(tmp_path)
+    payload = json.loads(receipt.read_text(encoding="utf-8"))
+    payload["method"] = "component_restarts_from_pinned_images"
+    payload["reset_scope"] = ["shopping", "shopping_admin", "reddit", "gitlab"]
+    receipt.write_text(json.dumps(payload), encoding="utf-8")
+
+    assert validate_environment_preparation(receipt, profile)["method"].startswith("component_")
+
+
+def test_environment_preparation_rejects_partial_component_restart(tmp_path):
+    receipt, profile = _files(tmp_path)
+    payload = json.loads(receipt.read_text(encoding="utf-8"))
+    payload["method"] = "component_restarts_from_pinned_images"
+    payload["reset_scope"] = ["shopping"]
+    receipt.write_text(json.dumps(payload), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="reset scope"):
+        validate_environment_preparation(receipt, profile)
+
+
+def test_repository_root_is_independent_of_current_working_directory():
+    root = repository_root(Path(__file__))
+    assert (root / "pyproject.toml").is_file()
+    assert (root / "harness/trajectory_replay/webarena_runtime_source.py").is_file()
+
+
+@pytest.mark.parametrize(
+    ("path", "key", "value"),
+    [
+        ("receipt", "fresh_app_id", 203),
+        ("receipt", "method", "unverified_reset"),
+        ("profile", "schema", "unknown"),
+    ],
+)
+def test_environment_preparation_rejects_unbound_receipts(tmp_path, path, key, value):
+    receipt, profile = _files(tmp_path)
+    target = receipt if path == "receipt" else profile
+    payload = json.loads(target.read_text(encoding="utf-8"))
+    payload[key] = value
+    target.write_text(json.dumps(payload), encoding="utf-8")
+    with pytest.raises(ValueError, match="declared deployment"):
+        validate_environment_preparation(receipt, profile)
+
+
+def _write_json(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value), encoding="utf-8")
+    return path
+
+
+def test_report_profile_owns_accepted_counts_instead_of_validator_code(tmp_path):
+    repository = tmp_path / "repository"
+    (repository / "pyproject.toml").parent.mkdir(parents=True)
+    revision = "a" * 40
+    fork = "https://github.com/lexmount/webarena-verified.git"
+    (repository / "pyproject.toml").write_text(
+        f"[project]\nname='fixture'\n[tool.uv.sources]\n"
+        f"webarena-verified = {{ git = '{fork}', rev = '{revision}' }}\n"
+    )
+    (repository / "uv.lock").write_text(
+        f"version = 1\n[[package]]\nname = 'webarena-verified'\n"
+        f"source = {{ git = '{fork}?rev={revision}#{revision}' }}\n"
+    )
+    task_ids = [4, 11, 23]
+    official_checksum = "official-data"
+    dataset = _write_json(
+        repository / "evaluation/full/dataset.json",
+        {
+            "schema": "lex_webarena_replay_dataset/1",
+            "official_dataset_sha256": official_checksum,
+            "task_count": len(task_ids),
+            "evaluator_source": {
+                "repository": fork,
+                "branch": "lex-main",
+                "revision": revision,
+            },
+        },
+    )
+    scope = _write_json(
+        repository / "evaluation/full/scope.json",
+        {
+            "schema": "lex_webarena_replay_scope/1",
+            "official_dataset_sha256": official_checksum,
+            "task_count": len(task_ids),
+            "task_ids": task_ids,
+        },
+    )
+    data_home = tmp_path / "data"
+    evidence = data_home / "trajectory-replay/run"
+    counts = {
+        "total": len(task_ids),
+        "pass": len(task_ids),
+        "unresolved": 0,
+        "physical_attempts": 854,
+        "excluded_duplicate_attempts": 0,
+    }
+    ledger = _write_json(
+        evidence / "ledger.json",
+        {
+            "schema": "lex_trajectory_replay_catalog_acceptance/1",
+            "scope_sha256": file_sha256(scope),
+            "counts": counts,
+        },
+    )
+    artifacts = {
+        "manifest.json": {},
+        "report.zip": {"fixture": True},
+        "suite.json": {"tasks": []},
+    }
+    for name, value in artifacts.items():
+        _write_json(evidence / name, value)
+    catalog = _write_json(
+        evidence / "catalog.json",
+        {
+            "schema": "lex_webarena_replay_catalog/1",
+            "task_ids": task_ids,
+            "accepted_ledger_sha256": file_sha256(ledger),
+        },
+    )
+    profile = _write_json(
+        repository / "evaluation/full/current-report.json",
+        {
+            "schema": "lex_webarena_replay_report_profile/1",
+            "dataset_path": "evaluation/full/dataset.json",
+            "dataset_sha256": file_sha256(dataset),
+            "scope_path": "evaluation/full/scope.json",
+            "scope_sha256": file_sha256(scope),
+            "evidence_root": "trajectory-replay/run",
+            "acceptance_ledger_path": "ledger.json",
+            "acceptance_ledger_sha256": file_sha256(ledger),
+            "accepted_counts": counts,
+            "report_manifest_path": "manifest.json",
+            "report_manifest_sha256": file_sha256(evidence / "manifest.json"),
+            "report_archive_path": "report.zip",
+            "report_archive_sha256": file_sha256(evidence / "report.zip"),
+            "accepted_catalog": {
+                "path": "catalog.json",
+                "sha256": file_sha256(catalog),
+            },
+            "accepted_portable_suite": {
+                "path": "suite.json",
+                "sha256": file_sha256(evidence / "suite.json"),
+            },
+            "publication": {
+                "host": "https://artifact.example",
+                "slug": "fixture",
+                "version_id": "v1",
+                "share_url": "https://artifact.example/v/fixture",
+            },
+        },
+    )
+
+    assert validate_catalog_report_profile(profile, data_home)["accepted_counts"] == counts
+
+    changed = json.loads(profile.read_text())
+    changed["accepted_counts"]["physical_attempts"] += 1
+    profile.write_text(json.dumps(changed))
+    with pytest.raises(ValueError, match="acceptance population changed"):
+        validate_catalog_report_profile(profile, data_home)
+
+    changed = json.loads(dataset.read_text())
+    changed["evaluator_source"]["repository"] = "https://github.com/ServiceNow/webarena-verified.git"
+    dataset.write_text(json.dumps(changed))
+    changed_profile = json.loads(profile.read_text())
+    changed_profile["accepted_counts"] = counts
+    changed_profile["dataset_sha256"] = file_sha256(dataset)
+    profile.write_text(json.dumps(changed_profile))
+    with pytest.raises(ValueError, match="dataset definition is invalid"):
+        validate_catalog_report_profile(profile, data_home)
+
+
+def test_comparison_release_binds_population_review_and_publication(tmp_path):
+    data_home = tmp_path / "data"
+    evidence = data_home / "trajectory-replay/comparison"
+    profile = _write_json(
+        evidence / "profile.json",
+        {
+            "schema": "lex_webarena_trajectory_replay_profile/1",
+            "execution_backend": "playwright",
+            "resources": {"collection_status": "not_recorded"},
+            "methods": [
+                {"id": "moli", "engine": "moli", "label": "Moli"},
+                {"id": "other", "engine": "other", "label": "Other"},
+            ],
+            "publication": {
+                "site_slug": "site",
+                "share_url": "https://artifact.example/v/share",
+                "expected_version": "ver_2",
+            },
+        },
+    )
+    dataset = _write_json(
+        evidence / "dataset.json",
+        {
+            "schema": "lex_webarena_catalog_comparison_dataset/1",
+            "counts": {"verified_tasks": 2, "verified_trajectories": 2},
+            "trajectories": [{"task_id": 1}, {"task_id": 2}],
+        },
+    )
+    review = _write_json(evidence / "review.json", {"reviewed": 1})
+    attempts = [
+        {"participant_id": "moli", "engine": "moli", "task_id": 1, "status": "pass"},
+        {"participant_id": "moli", "engine": "moli", "task_id": 2, "status": "pass"},
+        {"participant_id": "other", "engine": "other", "task_id": 1, "status": "pass"},
+        {"participant_id": "other", "engine": "other", "task_id": 2, "status": "fail"},
+    ]
+    results = _write_json(
+        evidence / "results.json",
+        {
+            "schema": "lex_webarena_catalog_comparison_results/1",
+            "dataset_sha256": file_sha256(dataset),
+            "attempts": attempts,
+            "physical_executions": [
+                *attempts,
+                {"participant_id": "other", "engine": "other", "task_id": 2},
+            ],
+        },
+    )
+    manifest = _write_json(
+        evidence / "report/manifest.json",
+        {
+            "task_count": 2,
+            "logical_results": 4,
+            "failed_results": 1,
+            "reviewed_failures": 1,
+            "participants": ["moli", "other"],
+            "results_sha256": file_sha256(results),
+            "dataset_sha256": file_sha256(dataset),
+            "review_sha256": file_sha256(review),
+        },
+    )
+    archive = evidence / "report.zip"
+    archive.write_bytes(b"archive")
+    receipt = _write_json(
+        evidence / "receipt.json",
+        {
+            "verified": True,
+            "slug": "site",
+            "share_url": "https://artifact.example/v/share",
+            "version": "ver_2",
+            "archive_sha256": file_sha256(archive),
+        },
+    )
+    attestation = _write_json(
+        evidence / "attestation.json",
+        {
+            "archive_sha256": file_sha256(archive),
+            "profile_sha256": file_sha256(profile),
+            "dataset_sha256": file_sha256(dataset),
+            "review_sha256": file_sha256(review),
+            "evidence_sha256": file_sha256(results),
+        },
+    )
+    metadata_revision = _write_json(
+        evidence / "execution-metadata-revision.json",
+        {
+            "schema": "lex_trajectory_replay_metadata_revision/1",
+            "source_profile_sha256": "1" * 64,
+            "profile_sha256": file_sha256(profile),
+            "source_results_sha256": "2" * 64,
+            "results_sha256": file_sha256(results),
+            "provider_calls": 0,
+            "source_inputs": [
+                {
+                    "path": "/evidence/run/inputs.json",
+                    "sha256": "3" * 64,
+                    "backend": "playwright",
+                    "engine": "moli",
+                }
+            ],
+            "verification": {
+                "source_input_count": 1,
+                "non_playwright_source_inputs": 0,
+                "resource_samples_in_results": 0,
+            },
+        },
+    )
+    paths = {
+        "profile": profile,
+        "dataset": dataset,
+        "results": results,
+        "failure_review": review,
+        "execution_metadata_revision": metadata_revision,
+        "report_manifest": manifest,
+        "publication_archive": archive,
+        "publication_attestation": attestation,
+        "publication_receipt": receipt,
+    }
+    binding = _write_json(
+        tmp_path / "current-comparison.json",
+        {
+            "schema": "lex_webarena_replay_comparison_release/1",
+            "evidence_root": "trajectory-replay/comparison",
+            "task_count": 2,
+            "logical_results": 4,
+            "physical_executions": 5,
+            "participants": [
+                {
+                    "id": "moli",
+                    "engine": "moli",
+                    "label": "Moli",
+                    "pass": 2,
+                    "fail": 0,
+                },
+                {
+                    "id": "other",
+                    "engine": "other",
+                    "label": "Other",
+                    "pass": 1,
+                    "fail": 1,
+                },
+            ],
+            "artifacts": {
+                name: {"path": str(path.relative_to(evidence)), "sha256": file_sha256(path)}
+                for name, path in paths.items()
+            },
+            "publication": {
+                "slug": "site",
+                "share_url": "https://artifact.example/v/share",
+                "version_id": "ver_2",
+            },
+        },
+    )
+
+    assert validate_catalog_comparison_release(binding, data_home)["logical_results"] == 4
+
+    binding_payload = json.loads(binding.read_text(encoding="utf-8"))
+    revision = json.loads(metadata_revision.read_text(encoding="utf-8"))
+    revision["provider_calls"] = 1
+    metadata_revision.write_text(json.dumps(revision), encoding="utf-8")
+    binding_payload["artifacts"]["execution_metadata_revision"]["sha256"] = (
+        file_sha256(metadata_revision)
+    )
+    binding.write_text(json.dumps(binding_payload), encoding="utf-8")
+    with pytest.raises(ValueError, match="execution metadata revision changed"):
+        validate_catalog_comparison_release(binding, data_home)
+
+    revision["provider_calls"] = 0
+    metadata_revision.write_text(json.dumps(revision), encoding="utf-8")
+    binding_payload["artifacts"]["execution_metadata_revision"]["sha256"] = (
+        file_sha256(metadata_revision)
+    )
+    binding.write_text(json.dumps(binding_payload), encoding="utf-8")
+
+    changed = json.loads(results.read_text(encoding="utf-8"))
+    changed["attempts"][3]["status"] = "pass"
+    results.write_text(json.dumps(changed), encoding="utf-8")
+    with pytest.raises(ValueError, match="results artifact changed"):
+        validate_catalog_comparison_release(binding, data_home)
diff --git a/tests/trajectory_replay/test_catalog_report.py b/tests/trajectory_replay/test_catalog_report.py
new file mode 100644
index 000000000..e1b6f06aa
--- /dev/null
+++ b/tests/trajectory_replay/test_catalog_report.py
@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay import webarena_report
+
+
+def _write_json(path: Path, value: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value) + "\n", encoding="utf-8")
+
+
+@pytest.mark.parametrize("hosted", [False, True])
+def test_build_catalog_report_keeps_setup_failures_out_of_physical_count(
+    monkeypatch, tmp_path: Path, hosted: bool
+) -> None:
+    runtime_metadata = {
+        "protocolVersion": "1.3",
+        "product": "Chrome/145.0.0.0",
+        "revision": "@kitesurf",
+        "userAgent": "Kitesurf",
+        "jsVersion": "14.5",
+    }
+    remote_browser = {
+        "provider": "cloudflare-browser-run",
+        "browser": "kitesurf",
+        "protocol": "cdp",
+        "endpoint_contract": "wss://example/{account_id}?browser=kitesurf",
+        "account_id_sha256": "a" * 64,
+        "runtime_metadata": runtime_metadata,
+    }
+    request_bridge = {
+        "kind": "local_http_fulfill/1",
+        "origins": ["http://private.example.test"],
+    }
+    current_evaluation = {"score": 1.0, "status": "pass", "evaluator": "current"}
+    transition = {
+        "task_id": 0,
+        "old_evaluation_sha256": "1" * 64,
+        "actual_response_sha256": "2" * 64,
+        "network_har_sha256": "3" * 64,
+        "current_evaluation_sha256": webarena_report.json_sha256(current_evaluation),
+    }
+    evidence = tmp_path / "evidence"
+    run = evidence / "repair"
+    progress = run / "progress.jsonl"
+    progress.parent.mkdir(parents=True)
+    progress.write_text(
+        json.dumps(
+            {
+                "task_id": 0,
+                "status": "pass",
+                "score": 1.0,
+                "duration_ms": 1250,
+                "action_count": 1,
+                "completed_actions": 1,
+                "executed_actions_sha256": "a" * 64,
+                **({"runtime_metadata": runtime_metadata} if hosted else {}),
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    digest = hashlib.sha256(progress.read_bytes()).hexdigest()
+    for name, value in {
+        "evaluation.json": {"score": 1.0, "status": "pass"},
+        "actual_response.json": {"task_type": "retrieve", "status": "SUCCESS"},
+        "trace.json": {"events": []},
+    }.items():
+        _write_json(run / "task-0" / name, value)
+
+    ledger = tmp_path / "ledger.json"
+    _write_json(
+        ledger,
+        {
+            "counts": {
+                "total": 1,
+                "pass": 1,
+                "unresolved": 0,
+                "physical_attempts": 1,
+                "excluded_duplicate_attempts": 0,
+            },
+            "participant": {
+                "engine": "kitesurf" if hosted else "moli",
+                "backend": "playwright",
+                **(
+                    {"remote_browser": remote_browser}
+                    if hosted
+                    else {"binary_sha256": "b" * 64}
+                ),
+                **({"request_bridge": request_bridge} if hosted else {}),
+            },
+            "binary_sha256s": [] if hosted else ["b" * 64],
+            **({"remote_browsers": [remote_browser]} if hosted else {}),
+            "runs": [{"kind": "segment"}, {"kind": "repair"}],
+            "rows": [
+                {
+                    "task_id": 0,
+                    "attempts": [
+                        {
+                            "status": "candidate_setup_failure",
+                            "progress_sha256": "c" * 64,
+                        },
+                        {
+                            "status": "pass",
+                            "progress_sha256": digest,
+                            **(
+                                {"runtime_metadata": runtime_metadata}
+                                if hosted
+                                else {"binary_sha256": "b" * 64}
+                            ),
+                            "evaluator_transition": transition,
+                        },
+                    ],
+                }
+            ],
+        },
+    )
+    catalog = tmp_path / "catalog.json"
+    _write_json(
+        catalog,
+        {
+            "task_ids": [0],
+            "rows": [
+                {
+                    "task_id": 0,
+                    "mode": "direct",
+                    "source_kind": "recorded",
+                    "source_actions": [{"action": "navigate", "url": "https://example.test"}],
+                }
+            ],
+        },
+    )
+    suite = tmp_path / "suite.json"
+    _write_json(suite, {"tasks": []})
+    scope = tmp_path / "scope.json"
+    _write_json(scope, {"task_ids": [0], "official_dataset_sha256": "data"})
+    site_map = tmp_path / "sites.json"
+    _write_json(site_map, {"sites": {}})
+    monkeypatch.setattr(
+        webarena_report,
+        "_catalog_official_tasks",
+        lambda: [
+            {
+                "task_id": 0,
+                "intent": "Open the example",
+                "sites": ["wikipedia"],
+                "eval": [{"evaluator": "URLExactEvaluator"}],
+            }
+        ],
+    )
+    monkeypatch.setattr(webarena_report, "load_remote_sites", lambda _path: {})
+    monkeypatch.setattr(webarena_report, "remote_benchmark", lambda _sites: object())
+    monkeypatch.setattr(webarena_report, "load_corrections", lambda: object())
+    monkeypatch.setattr(
+        webarena_report,
+        "reevaluate_pass",
+        lambda *_args, **_kwargs: (current_evaluation, transition),
+    )
+    output = tmp_path / "report"
+    archive = tmp_path / "report.zip"
+
+    result = webarena_report.build_catalog_acceptance_report(
+        evidence_root=evidence,
+        ledger_path=ledger,
+        scope_path=scope,
+        catalog_path=catalog,
+        suite_path=suite,
+        site_map_path=site_map,
+        output=output,
+        archive=archive,
+    )
+
+    assert result["archive_sha256"] == webarena_report._catalog_file_sha256(archive)
+    assert "1 / 1" in (output / "index.html").read_text(encoding="utf-8")
+    assert "物理执行：1 次" in (output / "case/000.html").read_text(encoding="utf-8")
+    assert '' in (output / "case/000.html").read_text(encoding="utf-8")
+    public = json.loads((output / "evidence/000.json").read_text())
+    assert public["physical_attempt_count"] == 1
+    if hosted:
+        assert public["participant"]["remote_browser"] == remote_browser
+        assert public["participant"]["runtime_metadata"] == runtime_metadata
+        assert public["participant"]["request_bridge"] == request_bridge
+        assert "1 个浏览器运行时身份" in (output / "index.html").read_text()
+        assert "Browser.getVersion" in (output / "method.html").read_text()
+        assert "Playwright request bridge" in (output / "method.html").read_text()
+        manifest = json.loads((output / "manifest.json").read_text())
+        assert manifest["participant"]["request_bridge"] == request_bridge
+    else:
+        assert public["participant"]["binary_sha256"] == "b" * 64
+    assert public["evaluation"] == current_evaluation
+    assert public["evaluator_transition"] == transition
+    profile = {
+        "browser": "kitesurf" if hosted else "moli",
+        "execution_backend": "playwright",
+        **({"request_bridge": request_bridge} if hosted else {}),
+        "acceptance_ledger_sha256": webarena_report._catalog_file_sha256(ledger),
+        "accepted_catalog": {"sha256": webarena_report._catalog_file_sha256(catalog)},
+        "accepted_portable_suite": {
+            "sha256": webarena_report._catalog_file_sha256(suite)
+        },
+        "accepted_counts": {
+            "total": 1,
+            "physical_attempts": 1,
+            "excluded_duplicate_attempts": 0,
+        },
+    }
+    assert (
+        webarena_report.validate_catalog_acceptance_report(
+            output, archive, profile=profile
+        )["pass"]
+        == 1
+    )
+    profile["browser"] = "chromium"
+    with pytest.raises(ValueError, match="presentation changed its release binding"):
+        webarena_report.validate_catalog_acceptance_report(
+            output, archive, profile=profile
+        )
diff --git a/tests/trajectory_replay/test_consolidate_webarena_catalog.py b/tests/trajectory_replay/test_consolidate_webarena_catalog.py
new file mode 100644
index 000000000..8ce64307e
--- /dev/null
+++ b/tests/trajectory_replay/test_consolidate_webarena_catalog.py
@@ -0,0 +1,137 @@
+import hashlib
+import json
+
+import pytest
+
+from harness.trajectory_replay.catalog_consolidation import (
+    _catalog_rows_by_id,
+    _load_bound_action_overrides,
+    _source_action_overrides_path,
+    _target_action_amendments,
+    _verified_run_directories,
+)
+
+
+def test_consolidation_rejects_inputs_changed_after_acceptance(tmp_path):
+    run = tmp_path / "run"
+    run.mkdir()
+    progress = run / "progress.jsonl"
+    progress.write_text('{"task_id":0,"status":"pass"}\n', encoding="utf-8")
+    inputs = run / "inputs.json"
+    inputs.write_text('{"catalog":"accepted"}\n', encoding="utf-8")
+    progress_hash = hashlib.sha256(progress.read_bytes()).hexdigest()
+    inputs_hash = hashlib.sha256(inputs.read_bytes()).hexdigest()
+    ledger = {
+        "runs": [
+            {
+                "progress_sha256": progress_hash,
+                "inputs_sha256": inputs_hash,
+            }
+        ]
+    }
+
+    assert _verified_run_directories(ledger, tmp_path, {progress_hash}) == {progress_hash: run}
+    inputs.write_text('{"catalog":"tampered"}\n', encoding="utf-8")
+    with pytest.raises(ValueError, match="inputs changed"):
+        _verified_run_directories(ledger, tmp_path, {progress_hash})
+
+
+def test_consolidation_rejects_duplicate_ledger_run_bindings(tmp_path):
+    row = {"progress_sha256": "same", "inputs_sha256": "hash"}
+    with pytest.raises(ValueError, match="duplicate run bindings"):
+        _verified_run_directories({"runs": [row, row]}, tmp_path, set())
+
+
+def test_consolidation_indexes_non_contiguous_catalog_tasks_by_identity():
+    rows = [{"task_id": 7}, {"task_id": 20}]
+    assert _catalog_rows_by_id({"task_ids": [7, 20], "rows": rows}) == {7: rows[0], 20: rows[1]}
+
+
+def test_consolidation_rejects_catalog_identity_mismatch():
+    with pytest.raises(ValueError, match="declared order"):
+        _catalog_rows_by_id({"task_ids": [7, 20], "rows": [{"task_id": 7}, {"task_id": 7}]})
+
+
+def _action_overrides(tmp_path):
+    path = tmp_path / "action-overrides.json"
+    path.write_text(
+        json.dumps(
+            {
+                "schema": "lex_webarena_replay_action_overrides/1",
+                "amendments": [
+                    {
+                        "task_id": 7,
+                        "action_index": 1,
+                        "precondition_action_sha256": "a" * 64,
+                        "set": {"wait_for_selector_hidden": ".loading-mask"},
+                    }
+                ],
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return path, hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+@pytest.mark.parametrize("bound_hash", [None, "invalid", "a" * 64])
+def test_consolidation_can_use_catalog_embedded_historical_overrides(tmp_path, bound_hash):
+    inputs = {"run": {"action_overrides_sha256": bound_hash}}
+    assert _load_bound_action_overrides(inputs, []) == {}
+
+
+def test_consolidation_accepts_the_hash_bound_external_action_overrides(tmp_path):
+    path, digest = _action_overrides(tmp_path)
+    sources = _load_bound_action_overrides({"run": {"action_overrides_sha256": digest}}, [path])
+    assert sources[digest][0] == path
+    assert set(sources[digest][1]) == {7}
+
+
+def test_consolidation_maps_multiple_run_override_versions_by_digest(tmp_path):
+    first, first_digest = _action_overrides(tmp_path)
+    second = tmp_path / "action-overrides-v2.json"
+    payload = json.loads(first.read_text(encoding="utf-8"))
+    payload["amendments"][0]["task_id"] = 8
+    second.write_text(json.dumps(payload) + "\n", encoding="utf-8")
+    second_digest = hashlib.sha256(second.read_bytes()).hexdigest()
+
+    sources = _load_bound_action_overrides(
+        {
+            "first": {"action_overrides_sha256": first_digest},
+            "second": {"action_overrides_sha256": second_digest},
+        },
+        [first, second],
+    )
+    assert set(sources) == {first_digest, second_digest}
+
+    partial = _load_bound_action_overrides(
+        {
+            "first": {"action_overrides_sha256": first_digest},
+            "second": {"action_overrides_sha256": second_digest},
+        },
+        [first],
+    )
+    assert set(partial) == {first_digest}
+
+
+def test_consolidation_rejects_run_and_catalog_override_hash_mismatch(tmp_path):
+    path, digest = _action_overrides(tmp_path)
+    sources = {digest: (path, {7: []})}
+    with pytest.raises(ValueError, match="binding differs from its source catalog"):
+        _source_action_overrides_path(
+            {"action_overrides_sha256": digest},
+            {"action_overrides_sha256": "b" * 64},
+            sources,
+        )
+
+
+def test_consolidation_rejects_target_amendment_not_used_by_accepted_task():
+    new_amendment = {
+        "task_id": 7,
+        "action_index": 1,
+        "precondition_action_sha256": "a" * 64,
+        "set": {"wait_for_selector_hidden": ".loading-mask"},
+    }
+    with pytest.raises(ValueError, match="target action overrides differ"):
+        _target_action_amendments(7, [], {7: [new_amendment]})
+    assert _target_action_amendments(7, [new_amendment], {7: [new_amendment]}) == [new_amendment]
diff --git a/tests/trajectory_replay/test_contract_boundary_controls.py b/tests/trajectory_replay/test_contract_boundary_controls.py
index 92aabff18..bd0106aa3 100644
--- a/tests/trajectory_replay/test_contract_boundary_controls.py
+++ b/tests/trajectory_replay/test_contract_boundary_controls.py
@@ -6,14 +6,18 @@
 from urllib.parse import urlencode
 
 import pytest
+from webarena_verified.types.task import WebArenaSite
 
 from harness.browser_use.runner import _configure_benchmark
+from harness.trajectory_replay.remote_runtime import remote_benchmark
 from harness.trajectory_replay.webarena_corrections import apply_correction, evaluate_corrected_contract, load_corrections
 
 
 ARCHIVED = Path("evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json")
 SHOP = "http://host.docker.internal:7770"
 ADMIN = "http://host.docker.internal:7780/admin"
+MAP = "http://host.docker.internal:3000"
+WIKIPEDIA = "http://host.docker.internal:8888"
 PRODUCT = "Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer)"
 RESPONSE = {"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None}
 
@@ -29,32 +33,195 @@ def entry(url, *, data=None, body=None, status=200):
 
 
 def scores(tmp_path, task_id, entries, catalog_path=ARCHIVED):
-    benchmark = _configure_benchmark()
+    benchmark = (
+        remote_benchmark({
+            WebArenaSite.SHOPPING: SHOP,
+            WebArenaSite.SHOPPING_ADMIN: ADMIN,
+            WebArenaSite.REDDIT: "http://host.docker.internal:9999",
+            WebArenaSite.GITLAB: "http://host.docker.internal:8023",
+            WebArenaSite.MAP: MAP,
+            WebArenaSite.WIKIPEDIA: WIKIPEDIA,
+        })
+        if task_id in {429, 430}
+        else _configure_benchmark()
+    )
     task = benchmark.get_task(task_id)
-    corrected = apply_correction(task, load_corrections(catalog_path))
+    catalog = load_corrections(
+        catalog_path,
+        require_current_upstream=Path(catalog_path) != ARCHIVED,
+    )
     path = tmp_path / f"synthetic-{task_id}.har"
     path.write_text(json.dumps({"log": {"version": "1.2", "creator": {"name": "counterexample", "version": "1"},
                                         "entries": entries}}))
-    return (benchmark.evaluate_task(task_id=task_id, agent_response=RESPONSE, network_trace=path).score,
-            evaluate_corrected_contract(benchmark, corrected, agent_response=RESPONSE, network_trace=path).score)
-
-
-@pytest.mark.parametrize("attributes,new_product,name,expected", [
-    ("Size: S, Color: Brown", 1, "Phoebe Zipper Sweatshirt", 1),
-    ("Size: L, Color: Brown", 1, "Phoebe Zipper Sweatshirt", 0),
-    ("Size: S, Color: Red", 1, "Phoebe Zipper Sweatshirt", 0),
-    ("Size: S, Color: Brown", 0, "Phoebe Zipper Sweatshirt", 0),
-    ("Size: S, Color: Brown", 1, "Different product", 0),
-    ("", 1, "Phoebe Zipper Sweatshirt", 0),
-], ids=["existing-brown-new-small-variation", "wrong-size", "wrong-color", "not-new", "wrong-product", "missing-attributes"])
-def test_archived_547_keeps_required_variation_fields(tmp_path, attributes, new_product, name, expected):
-    values = {"product[name]": name, "configurable-matrix-serialized": json.dumps([
-        {"newProduct": new_product, "attributes": attributes}
-    ])}
-    entries = [entry(ADMIN + "/catalog/product/save/id/1130/type/configurable/store/0/set/4/back/edit",
-                     data=values, status=302)]
-    # No global attribute-save request: the accepted equivalent reuses Brown.
-    assert scores(tmp_path, 547, entries) == (0, expected)
+    response = (
+        {"task_type": "NAVIGATE", "status": "SUCCESS", "retrieved_data": None}
+        if task_id in {429, 430}
+        else RESPONSE
+    )
+    official = benchmark.evaluate_task(
+        task_id=task_id, agent_response=response, network_trace=path
+    ).score
+    if task_id not in catalog.records:
+        return (official, official)
+    corrected = apply_correction(task, catalog)
+    return (
+        official,
+        evaluate_corrected_contract(
+            benchmark, corrected, agent_response=response, network_trace=path
+        ).score,
+    )
+
+
+
+@pytest.mark.parametrize("rows,expected", [
+    ([
+        {"newProduct": 1, "attributes": "Size: S, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: M, Color: Blue"},
+    ], 1),
+    ([{"newProduct": 1, "attributes": "Size: S, Color: Blue"}], 0),
+    ([
+        {"newProduct": 1, "attributes": "Size: S, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: L, Color: Blue"},
+    ], 0),
+    ([
+        {"newProduct": 0, "attributes": "Size: S, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: M, Color: Blue"},
+    ], 0),
+], ids=["tr-548-accept-existing-blue-small-medium-variations",
+        "tr-548-reject-missing-medium-blue",
+        "tr-548-reject-wrong-size-blue",
+        "tr-548-reject-non-new-small-blue"])
+def test_current_548_reuses_existing_blue_and_preserves_seeded_name(tmp_path, rows, expected):
+    values = {
+        "product[name]": "Frankie  Sweatshirt",
+        "configurable-matrix-serialized": json.dumps(rows),
+    }
+    events = [entry(
+        ADMIN + "/catalog/product/save/id/110/type/configurable/store/0/set/9/back/edit",
+        data=values,
+        status=302,
+    )]
+    assert scores(
+        tmp_path,
+        548,
+        events,
+        Path("evaluation/trajectory-replay/webarena-verified/task-corrections.json"),
+    ) == (expected, expected)
+
+
+@pytest.mark.parametrize("task_id,new_label,attribute_id,rows,expected", [
+    (549, "XXXL", "144", [{"newProduct": 1, "attributes": "Size: XXXL, Color: Green"}], 1),
+    (549, "XXL", "144", [{"newProduct": 1, "attributes": "Size: XXXL, Color: Green"}], 0),
+    (549, "XXXL", "93", [{"newProduct": 1, "attributes": "Size: XXXL, Color: Green"}], 0),
+    (549, "XXXL", "144", [{"newProduct": 1, "attributes": "Size: XXXL, Color: Blue"}], 0),
+    (550, "XXS", "144", [
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Purple"},
+    ], 1),
+    (550, "XS", "144", [
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Purple"},
+    ], 0),
+    (550, "XXS", "93", [
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Blue"},
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Purple"},
+    ], 0),
+    (550, "XXS", "144", [
+        {"newProduct": 1, "attributes": "Size: XXS, Color: Blue"},
+    ], 0),
+], ids=[
+    "tr-549-accept-native-create-options-and-required-variations",
+    "tr-549-reject-wrong-new-size", "tr-549-reject-wrong-attribute",
+    "tr-549-reject-wrong-variation",
+    "tr-550-accept-native-create-options-and-required-variations",
+    "tr-550-reject-wrong-new-size", "tr-550-reject-wrong-attribute",
+    "tr-550-reject-incomplete-variations",
+])
+def test_current_configurable_size_tasks_use_native_create_options(
+    tmp_path, task_id, new_label, attribute_id, rows, expected
+):
+    product_id, name = {
+        549: (1492, "Minerva LumaTech™ V-Tee"),
+        550: (1732, "Nona Fitness Tank"),
+    }[task_id]
+    events = [
+        entry(
+            ADMIN + "/catalog/product_attribute/createOptions/?isAjax=true",
+            data={
+                "options[0][label]": new_label,
+                "options[0][attribute_id]": attribute_id,
+                "options[0][is_new]": "true",
+            },
+            status=200,
+        ),
+        entry(
+            ADMIN + f"/catalog/product/save/id/{product_id}/type/configurable/store/0/set/9/back/edit",
+            data={
+                "product[name]": name,
+                "configurable-matrix-serialized": json.dumps(rows),
+            },
+            status=302,
+        ),
+    ]
+    assert scores(
+        tmp_path,
+        task_id,
+        events,
+        Path("evaluation/trajectory-replay/webarena-verified/task-corrections.json"),
+    ) == (expected, expected)
+
+
+@pytest.mark.parametrize("status,search,excluded,expected", [
+    ("1", "hollister", "false", 1),
+    ("0", "hollister", "false", 0),
+    ("1", "abercrombie", "false", 0),
+    ("1", "hollister", "true", 0),
+], ids=[
+    "tr-423-accept-filtered-enable-all-hollister",
+    "tr-423-reject-disable-hollister",
+    "tr-423-reject-wrong-brand-filter",
+    "tr-423-reject-excluded-filtered-records",
+])
+def test_current_423_uses_native_filtered_mass_status(
+    tmp_path, status, search, excluded, expected
+):
+    events = [entry(
+        ADMIN + f"/catalog/product/massStatus/status/{status}/",
+        data={
+            "excluded": excluded,
+            "filters[placeholder]": "true",
+            "search": search,
+            "namespace": "product_listing",
+        },
+        status=302,
+    )]
+    assert scores(
+        tmp_path,
+        423,
+        events,
+        Path("evaluation/trajectory-replay/webarena-verified/task-corrections.json"),
+    ) == (expected, expected)
+
+
+@pytest.mark.parametrize("task_id,paths,expected", [
+    (429, ["/way/583390395", "/way/172206707"], 1),
+    (429, ["/relation/583390395", "/relation/172206707"], 0),
+    (430, ["/way/583390395"], 1),
+    (430, ["/relation/583390395"], 0),
+], ids=["tr-429-accept-live-way-identities",
+        "tr-429-reject-relation-kind-for-way-identities",
+        "tr-430-accept-live-washington-jefferson-way",
+        "tr-430-reject-unresolved-placeholder"])
+def test_current_college_map_corrections_use_live_object_kinds(tmp_path, task_id, paths, expected):
+    events = [entry(WIKIPEDIA + "/wiki/The_Chair_(2021_TV_series)")] + [
+        entry(MAP + path) for path in paths
+    ]
+    assert scores(
+        tmp_path,
+        task_id,
+        events,
+        Path("evaluation/trajectory-replay/webarena-verified/task-corrections.json"),
+    ) == (0, expected)
 
 
 def cart_entries(quantity=1, *, name=PRODUCT, include_totals=True):
@@ -86,8 +253,8 @@ def test_current_510_restores_upstream_quantity_and_name(tmp_path, values, expec
 
 def test_510_revision_receipt_binds_rules_and_candidate():
     receipt = json.loads(Path('research/trajectory-replay-task510-quantity-revision.json').read_text())
-    before = load_corrections(receipt['predecessor_catalog_path'])
-    after = load_corrections()
+    before = load_corrections(receipt['predecessor_catalog_path'], require_current_upstream=False)
+    after = load_corrections(receipt['revised_catalog_path'], require_current_upstream=False)
     assert receipt['predecessor_correction_catalog_sha256'] == before.source_sha256
     assert receipt['predecessor_correction_record_sha256'] == before.record_sha256s[510]
     assert receipt['revised_correction_catalog_sha256'] == after.source_sha256
diff --git a/tests/trajectory_replay/test_evaluator_transition.py b/tests/trajectory_replay/test_evaluator_transition.py
new file mode 100644
index 000000000..bac5ca4e1
--- /dev/null
+++ b/tests/trajectory_replay/test_evaluator_transition.py
@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay import evaluator_transition
+from scripts.reevaluate_webarena_evaluator_transition import _old_evaluation_identity
+
+
+def test_source_pass_allows_a_superseded_correction_bound_by_transition() -> None:
+    source = {
+        "evaluator_checksum": "a" * 64,
+        "data_checksum": "b" * 64,
+    }
+    evaluator_transition.validate_source_pass(
+        {
+            "task_id": 566,
+            "score": 1.0,
+            "status": "success",
+            "webarena_verified_evaluator_checksum": source["evaluator_checksum"],
+            "webarena_verified_data_checksum": source["data_checksum"],
+            "contract_correction_id": "superseded-upstream-fix",
+            "corrected_task_sha256": "c" * 64,
+            "evaluators_results": [{"score": 1.0, "status": "success"}],
+        },
+        task_id=566,
+        source=source,
+    )
+
+
+def test_source_pass_rejects_a_failed_evaluator_row() -> None:
+    source = {
+        "evaluator_checksum": "a" * 64,
+        "data_checksum": "b" * 64,
+    }
+    with pytest.raises(ValueError, match="source pass"):
+        evaluator_transition.validate_source_pass(
+            {
+                "task_id": 7,
+                "score": 1.0,
+                "status": "success",
+                "webarena_verified_evaluator_checksum": source["evaluator_checksum"],
+                "webarena_verified_data_checksum": source["data_checksum"],
+                "evaluators_results": [{"score": 0.0, "status": "failure"}],
+            },
+            task_id=7,
+            source=source,
+        )
+
+
+def _transition():
+    return evaluator_transition.load_transition(
+        Path(
+            "evaluation/trajectory-replay/webarena-verified-full/evaluator-transition.json"
+        )
+    )
+
+
+def _receipt(transition):
+    return {
+        "schema": evaluator_transition.RECEIPT_SCHEMA,
+        "transition_sha256": evaluator_transition.json_sha256(transition),
+        "from_revision": transition["from_revision"],
+        "compatible_from_revisions": transition["compatible_from_revisions"],
+        "compatible_sources": transition["compatible_sources"],
+        "from_evaluator_checksum": transition["from_evaluator_checksum"],
+        "from_data_checksum": transition["from_data_checksum"],
+        "to_revision": transition["to_revision"],
+        "to_evaluator_checksum": transition["to_evaluator_checksum"],
+        "to_data_checksum": transition["to_data_checksum"],
+        "row_count": 1,
+        "excluded_old_evaluations": [],
+        "rows": [
+            {
+                "task_id": 7,
+                "old_evaluation_sha256": "1" * 64,
+                "actual_response_sha256": "2" * 64,
+                "network_har_sha256": "3" * 64,
+                "current_evaluation_sha256": "4" * 64,
+                "run_inputs_sha256": "5" * 64,
+                "source": transition["compatible_sources"][0],
+            }
+        ],
+    }
+
+
+def test_transition_authority_is_bound_to_the_installed_fork_revision():
+    transition = _transition()
+
+    assert transition["from_revision"] != transition["to_revision"]
+    assert transition["to_evaluator_checksum"] == (
+        evaluator_transition.compute_evaluator_checksum()
+    )
+
+
+def test_transition_resolves_revision_and_legacy_evaluator_sources():
+    transition = _transition()
+
+    revision = evaluator_transition.compatible_source(
+        transition,
+        revision="4c3b14e5cda91889f5ee883c8661bccd1d96c1af",
+        evaluator_fixes_sha256=None,
+    )
+    legacy = evaluator_transition.compatible_source(
+        transition,
+        revision=None,
+        evaluator_fixes_sha256=(
+            "58b451900c239ed6fb9a9a99633409bc56190676b9f2273ed45c472e8eb1ff08"
+        ),
+    )
+
+    assert revision["evaluator_checksum"] == (
+        "7f6b1efd6ae2f88643f623a8ab4b1aeb107c0fa912b59f5ffd753de20337a226"
+    )
+    assert legacy["data_checksum"] == (
+        "7ab0ef7e61af98efc5e16ba2b4303ddc0b8f5b66b5c88c58cb76e89e9235b57a"
+    )
+
+
+def test_transition_exclusion_cli_binds_evaluation_and_run():
+    assert _old_evaluation_identity(f"7/{'1' * 64}/{'2' * 64}") == (
+        7,
+        "1" * 64,
+        "2" * 64,
+    )
+    with pytest.raises(
+        argparse.ArgumentTypeError,
+        match="EVALUATION_SHA256/RUN_INPUTS_SHA256",
+    ):
+        _old_evaluation_identity(f"7/{'1' * 64}")
+
+
+def test_transition_receipt_binds_each_old_evaluation_once(tmp_path):
+    transition = _transition()
+    receipt = _receipt(transition)
+    path = tmp_path / "receipt.json"
+    path.write_text(json.dumps(receipt))
+
+    indexed, excluded = evaluator_transition.load_receipt(path, transition)
+
+    assert indexed[(7, "1" * 64, "5" * 64)]["current_evaluation_sha256"] == "4" * 64
+    assert excluded == set()
+
+
+def test_transition_receipt_excludes_only_the_bound_old_evaluation(tmp_path):
+    transition = _transition()
+    receipt = _receipt(transition)
+    receipt["excluded_old_evaluations"] = [
+        {
+            "task_id": 7,
+            "old_evaluation_sha256": "1" * 64,
+            "run_inputs_sha256": "6" * 64,
+        }
+    ]
+    path = tmp_path / "receipt.json"
+    path.write_text(json.dumps(receipt))
+
+    indexed, excluded = evaluator_transition.load_receipt(path, transition)
+
+    assert set(indexed) == {(7, "1" * 64, "5" * 64)}
+    assert excluded == {(7, "1" * 64, "6" * 64)}
+
+
+def test_transition_receipt_distinguishes_identical_evaluations_from_different_runs(
+    tmp_path,
+):
+    transition = _transition()
+    receipt = _receipt(transition)
+    replacement = {
+        **receipt["rows"][0],
+        "run_inputs_sha256": "6" * 64,
+    }
+    receipt["rows"].append(replacement)
+    receipt["row_count"] = 2
+    path = tmp_path / "receipt.json"
+    path.write_text(json.dumps(receipt))
+
+    indexed, excluded = evaluator_transition.load_receipt(path, transition)
+
+    assert set(indexed) == {
+        (7, "1" * 64, "5" * 64),
+        (7, "1" * 64, "6" * 64),
+    }
+    assert excluded == set()
+
+
+@pytest.mark.parametrize("change", ["extra_row", "wrong_transition", "wrong_count"])
+def test_transition_receipt_rejects_unbound_or_duplicate_evidence(tmp_path, change):
+    transition = _transition()
+    receipt = _receipt(transition)
+    if change == "extra_row":
+        receipt["rows"].append(dict(receipt["rows"][0]))
+        receipt["row_count"] = 2
+    elif change == "wrong_transition":
+        receipt["transition_sha256"] = "0" * 64
+    else:
+        receipt["row_count"] = 2
+    path = tmp_path / "receipt.json"
+    path.write_text(json.dumps(receipt))
+
+    with pytest.raises(ValueError, match="transition receipt"):
+        evaluator_transition.load_receipt(path, transition)
diff --git a/tests/trajectory_replay/test_failure_matrix.py b/tests/trajectory_replay/test_failure_matrix.py
index 7ce59b0e1..4516c381f 100644
--- a/tests/trajectory_replay/test_failure_matrix.py
+++ b/tests/trajectory_replay/test_failure_matrix.py
@@ -3,13 +3,59 @@
 import pytest
 from bs4 import BeautifulSoup
 
-from harness.trajectory_replay.failure_matrix import failure_category_matrix
+from harness.trajectory_replay.failure_matrix import (
+    browser_failure_profile,
+    failure_category_matrix,
+    failure_signature_cards,
+)
 
 
 def _row(method, task, code="", status="fail"):
     return dict(participant_id=method, task_id=task, status=status,
                 cause_code=code, cause_zh=code + "中文" if code else "",
-                cause_en=code + " English" if code else "")
+                cause_en=code + " English" if code else "",
+                reason_zh=code + "原因" if code else "",
+                reason_en=code + " reason" if code else "",
+                key=f"{method}-{task}")
+
+
+def test_failure_signatures_keep_only_three_leading_causes_and_link_to_analysis():
+    methods = {"one": {"label": "One"}}
+    rows = [_row("one", task, code) for task, code in enumerate(("a", "a", "b", "c", "d"))]
+    soup = BeautifulSoup(
+        failure_signature_cards(methods, rows, "zh", participants=("one",)),
+        "html.parser",
+    )
+    assert len(soup.select('[data-method="one"] li:not(.failure-remainder)')) == 3
+    assert soup.select_one(".failure-remainder strong").text == "1"
+    assert soup.select_one("h2").text == "1 款浏览器的主导错误"
+    assert soup.select_one(".section-heading>a").text == "查看 1 章详解"
+    assert soup.select_one(".section-heading>a")["href"] == "interpretation.html#browser-diagnoses"
+
+
+def test_failure_signature_copy_uses_registered_participant_count_in_english():
+    methods = {"one": {"label": "One"}, "two": {"label": "Two"}}
+    rows = [_row("one", 1, "a"), _row("two", 1, "b")]
+    soup = BeautifulSoup(
+        failure_signature_cards(
+            methods, rows, "en", participants=("one", "two")
+        ),
+        "html.parser",
+    )
+
+    assert soup.select_one("h2").text == "Dominant failures in 2 browsers"
+    assert soup.select_one(".section-heading>a").text == "Read 2 browser chapters"
+
+
+def test_browser_profile_covers_every_reviewed_subtype_with_evidence():
+    methods = {"one": {"label": "One"}}
+    rows = [_row("one", 0, status="pass"), _row("one", 1, "a"), _row("one", 2, "b")]
+    soup = BeautifulSoup(browser_failure_profile(methods, rows, "zh", "one"), "html.parser")
+    assert {item["data-cause"] for item in soup.select(".failure-bars [data-cause]")} == {"a", "b"}
+    assert {item["data-cause-detail"] for item in soup.select("[data-cause-detail]")} == {"a", "b"}
+    assert {link["href"] for link in soup.select("[data-cause-detail] a")} == {
+        "result/one-1.html", "result/one-2.html"
+    }
 
 
 def test_article_embedding_keeps_matrix_without_a_duplicate_section_heading():
diff --git a/tests/trajectory_replay/test_playwright_tool.py b/tests/trajectory_replay/test_playwright_tool.py
new file mode 100644
index 000000000..6f5ff59f6
--- /dev/null
+++ b/tests/trajectory_replay/test_playwright_tool.py
@@ -0,0 +1,1977 @@
+"""Exercise the CowAgent-free path against a real Chromium page."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import http.server
+import json
+import socket
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+import pytest
+import requests
+
+from harness.browser_use.runner import _prepare_browser_viewport
+from harness.trajectory_replay.public_webarena.executor import execute_steps
+from harness.trajectory_replay.remote_execution import _browser_command, _wait_cdp
+from harness.trajectory_replay.webarena_replay import _execute_replay_call
+
+
+def test_request_bridge_host_override_preserves_logical_http_identity() -> None:
+    from harness.trajectory_replay.playwright_tool import _HostOverrideAdapter
+
+    adapter = _HostOverrideAdapter(
+        (("*.dev.lexmount.net", "10.2.62.142"),)
+    )
+    calls = []
+
+    class PoolManager:
+        def connection_from_host(self, **kwargs):
+            calls.append(kwargs)
+            return "pool"
+
+    adapter.poolmanager = PoolManager()
+    prepared = requests.Session().prepare_request(
+        requests.Request(
+            "POST",
+            "http://webarena-shopping.dev.lexmount.net:8080/graphql",
+        )
+    )
+
+    assert adapter.get_connection_with_tls_context(prepared, True) == "pool"
+    assert prepared.url == "http://webarena-shopping.dev.lexmount.net:8080/graphql"
+    assert prepared.headers["Host"] == "webarena-shopping.dev.lexmount.net:8080"
+    assert calls == [{"scheme": "http", "host": "10.2.62.142", "port": 8080}]
+
+    https = requests.Session().prepare_request(
+        requests.Request("GET", "https://webarena-shopping.dev.lexmount.net/")
+    )
+    with pytest.raises(ValueError, match="HTTP origins only"):
+        adapter.get_connection_with_tls_context(https, True)
+
+
+def test_request_bridge_installation_routes_matching_https_to_fail_closed() -> None:
+    from harness.trajectory_replay.playwright_tool import (
+        PlaywrightReplayTool,
+        _HostOverrideAdapter,
+    )
+
+    class Context:
+        def expose_binding(self, *_args):
+            pass
+
+        def add_init_script(self, *_args):
+            pass
+
+        def route(self, *_args):
+            pass
+
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = Context()
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge(
+        {"https://shop.test"},
+        [{"pattern": "shop.test", "address": "127.0.0.1"}],
+    )
+    try:
+        assert isinstance(
+            tool._request_bridge_session.adapters["https://"],
+            _HostOverrideAdapter,
+        )
+        with pytest.raises(ValueError, match="HTTP origins only"):
+            tool._request_bridge_session.get("https://shop.test/", timeout=1)
+    finally:
+        tool._request_bridge_session.close()
+
+
+def test_request_bridge_host_override_routes_bytes_and_keeps_cookie_domain() -> None:
+    from harness.trajectory_replay.playwright_tool import _HostOverrideAdapter
+
+    observed = []
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            observed.append((self.path, self.headers["Host"]))
+            self.send_response(200)
+            self.send_header("Set-Cookie", "sid=value; Domain=shop.test; Path=/")
+            self.end_headers()
+            self.wfile.write(b"ok")
+
+        def log_message(self, *_args):
+            pass
+
+    server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        session = requests.Session()
+        session.trust_env = False
+        session.mount(
+            "http://",
+            _HostOverrideAdapter((("shop.test", "127.0.0.1"),)),
+        )
+        response = session.get(
+            f"http://shop.test:{server.server_port}/resource", timeout=5
+        )
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+    assert response.status_code == 200
+    assert response.url == f"http://shop.test:{server.server_port}/resource"
+    assert response.cookies.get("sid", domain=".shop.test", path="/") == "value"
+    assert observed == [("/resource", f"shop.test:{server.server_port}")]
+
+
+def test_request_bridge_host_override_prefers_exact_then_specific_wildcard() -> None:
+    from harness.trajectory_replay.playwright_tool import (
+        _HostOverrideAdapter,
+        _validated_host_overrides,
+    )
+
+    adapter = _HostOverrideAdapter(
+        _validated_host_overrides(
+            [
+                {"pattern": "*.dev.lexmount.net", "address": "10.0.0.1"},
+                {"pattern": "*.sub.dev.lexmount.net", "address": "10.0.0.2"},
+                {"pattern": "shop.sub.dev.lexmount.net", "address": "10.0.0.3"},
+            ]
+        )
+    )
+
+    assert adapter._address_for("shop.sub.dev.lexmount.net") == "10.0.0.3"
+    assert adapter._address_for("other.sub.dev.lexmount.net") == "10.0.0.2"
+    assert adapter._address_for("other.dev.lexmount.net") == "10.0.0.1"
+
+
+@pytest.mark.parametrize(
+    "overrides, message",
+    [
+        ([{"pattern": "shop.*.test", "address": "10.2.62.142"}], "pattern"),
+        ([{"pattern": "*.dev.lexmount.net", "address": "internal.test"}], "address"),
+        (
+            [
+                {"pattern": "*.dev.lexmount.net", "address": "10.2.62.142"},
+                {"pattern": "*.dev.lexmount.net", "address": "10.2.62.143"},
+            ],
+            "duplicated",
+        ),
+    ],
+)
+def test_request_bridge_host_override_rejects_ambiguous_inputs(
+    overrides, message
+) -> None:
+    from harness.trajectory_replay.playwright_tool import _validated_host_overrides
+
+    with pytest.raises(ValueError, match=message):
+        _validated_host_overrides(overrides)
+
+
+def _unit_tool(page, context, *, engine="chromium"):
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._page = page
+    tool._context = context
+    tool._engine = engine
+    return tool
+
+
+def test_obscura_viewport_verification_does_not_open_a_second_page(monkeypatch) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    class Page:
+        def evaluate(self, _script):
+            raise AssertionError("Obscura viewport verification must not execute page script")
+
+    page = Page()
+
+    class Context:
+        def __init__(self):
+            self.page_count = 0
+
+        def new_page(self):
+            self.page_count += 1
+            if self.page_count > 1:
+                raise AssertionError("Obscura viewport verification must reuse the primary page")
+            return page
+
+        def close(self):
+            pass
+
+        def new_cdp_session(self, selected):
+            assert selected is page
+
+            class Session:
+                def send(self, method, params=None):
+                    if method == "Page.getLayoutMetrics":
+                        return {
+                            "cssVisualViewport": {
+                                "clientWidth": 1280,
+                                "clientHeight": 900,
+                            }
+                        }
+                    assert method == "Runtime.evaluate"
+                    assert params == {
+                        "expression": "devicePixelRatio",
+                        "returnByValue": True,
+                    }
+                    return {"result": {"value": 1}}
+
+            return Session()
+
+    context = Context()
+
+    class Browser:
+        contexts = []
+
+        def new_context(self, **kwargs):
+            assert kwargs == {"viewport": {"width": 1280, "height": 900}, "locale": "en-US"}
+            return context
+
+        def close(self):
+            pass
+
+    class Playwright:
+        class Chromium:
+            def connect_over_cdp(self, *_args, **_kwargs):
+                return Browser()
+
+        chromium = Chromium()
+
+        def stop(self):
+            pass
+
+    class Starter:
+        def start(self):
+            return Playwright()
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.sync_playwright", lambda: Starter()
+    )
+
+    tool = PlaywrightReplayTool("http://127.0.0.1:1", engine="obscura")
+    try:
+        viewport = _prepare_browser_viewport(tool)
+    finally:
+        tool.close()
+
+    assert context.page_count == 1
+    assert viewport["page_model"] == "single_target"
+    assert viewport["verified"] is True
+    assert viewport["observations"][0]["device_scale_factor"] == 1
+
+
+def test_kitesurf_reuses_authenticated_default_target_without_second_cdp_attach(
+    monkeypatch,
+) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    events = []
+
+    class Page:
+        url = "about:blank"
+
+        def on(self, event, handler):
+            self.handlers = getattr(self, "handlers", {})
+            self.handlers[event] = handler
+
+        def set_viewport_size(self, viewport):
+            events.append(("viewport", viewport))
+
+        def goto(self, url, **kwargs):
+            events.append(("goto", url, kwargs))
+
+        def evaluate(self, _script):
+            return {"width": 1280, "height": 900, "device_scale_factor": 1}
+
+    page = Page()
+
+    class Context:
+        pages = [page]
+
+        def expose_binding(self, name, callback):
+            events.append(("binding", name, callback))
+
+        def add_init_script(self, script):
+            events.append(("init_script", script))
+
+        def route(self, selector, handler):
+            events.append(("route", selector, handler))
+
+        def new_cdp_session(self, _page):
+            raise AssertionError("Kitesurf must not attach a second CDP session")
+
+        def close(self):
+            raise AssertionError("Kitesurf default context must not be closed separately")
+
+    context = Context()
+
+    class Browser:
+        contexts = [context]
+
+        def new_browser_cdp_session(self):
+            class Session:
+                def send(self, method):
+                    events.append(("browser_cdp", method))
+                    return {
+                        "protocolVersion": "1.3",
+                        "product": "Chrome/145.0.0.0",
+                        "revision": "@kitesurf",
+                        "userAgent": "Kitesurf",
+                        "jsVersion": "14.5",
+                    }
+
+                def detach(self):
+                    events.append(("browser_cdp", "detach"))
+
+            return Session()
+
+        def new_context(self, **_kwargs):
+            raise AssertionError("Kitesurf must reuse its provider-owned default context")
+
+        def close(self):
+            events.append(("browser", "close"))
+
+    class Playwright:
+        class Chromium:
+            def connect_over_cdp(self, endpoint, **kwargs):
+                events.append(("connect", endpoint, kwargs))
+                return Browser()
+
+        chromium = Chromium()
+
+        def stop(self):
+            events.append(("playwright", "stop"))
+
+    class Starter:
+        def start(self):
+            return Playwright()
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.sync_playwright", lambda: Starter()
+    )
+
+    class BridgeResponse:
+        status_code = 201
+        headers = {"content-type": "application/json", "content-length": "14"}
+        content = b'{"saved":true}'
+        url = "http://private.test/save"
+        cookies = []
+
+    class BridgeSession:
+        trust_env = True
+
+        def __init__(self):
+            self.calls = 0
+            self.cookies = []
+
+        def request(self, **kwargs):
+            self.calls += 1
+            events.append(("bridge", "request", kwargs))
+            if self.calls == 2:
+                raise RuntimeError("local fetch failed")
+            return BridgeResponse()
+
+        def close(self):
+            events.append(("bridge", "close"))
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", BridgeSession
+    )
+    tool = PlaywrightReplayTool(
+        "wss://provider.example/cdp",
+        engine="kitesurf",
+        headers={"Authorization": "Bearer secret"},
+        request_bridge_origins={"http://private.test"},
+    )
+    assert tool.har_capture_transport == "playwright"
+    assert tool.runtime_metadata == {
+        "protocolVersion": "1.3",
+        "product": "Chrome/145.0.0.0",
+        "revision": "@kitesurf",
+        "userAgent": "Kitesurf",
+        "jsVersion": "14.5",
+    }
+    assert tool._supports_multiple_pages is False
+    assert tool._supports_network_reset_cdp is False
+    assert tool._supports_storage_cdp is False
+    assert tool._native_login_form_submit is True
+    binding_event = next(event for event in events if event[0] == "binding")
+    assert binding_event[1] == "__browserEvalCaptureRequestBody"
+    assert "globalThis.fetch = async function" in next(
+        event[1] for event in events if event[0] == "init_script"
+    )
+    init_script = next(event[1] for event in events if event[0] == "init_script")
+    assert "headers, body: ''" in init_script
+    assert "application/x-www-form-urlencoded;charset=UTF-8" in init_script
+    assert "text/plain;charset=UTF-8" in init_script
+    route_event = next(event for event in events if event[0] == "route")
+    predicate, handler = route_event[1:]
+    assert predicate("http://private.test/path") is True
+    assert predicate("https://public.test/path") is False
+
+    class Request:
+        method = "POST"
+        url = "http://private.test/save"
+        post_data_buffer = None
+        resource_type = "xhr"
+
+        def __init__(self, capture_id):
+            self.capture_id = capture_id
+
+        def all_headers(self):
+            return {
+                "content-type": "application/json",
+                "content-length": "16",
+                "signature": "private-signature",
+                "x-browser-eval-request-body-id": self.capture_id,
+            }
+
+    class Route:
+        request = Request("capture-1")
+
+        def fulfill(self, **kwargs):
+            events.append(("bridge", "fulfill", kwargs))
+
+    captured_payload = {
+        "requestId": "capture-1",
+        "method": "POST",
+        "url": "http://private.test/save",
+        "body": {"kind": "text", "text": '{"enabled":true}'},
+    }
+    binding_event[2]({}, captured_payload)
+    handler(Route())
+    assert tool.request_bridge_stats == {"requests": 1, "failures": 0}
+
+    class FailedRoute:
+        request = Request("capture-2")
+
+        def abort(self, reason):
+            events.append(("bridge", "abort", reason))
+
+    binding_event[2]({}, {**captured_payload, "requestId": "capture-2"})
+    with pytest.raises(RuntimeError, match="local fetch failed"):
+        handler(FailedRoute())
+    assert tool.request_bridge_stats == {"requests": 2, "failures": 1}
+    tool.close()
+
+    assert events == [
+        (
+            "connect",
+            "wss://provider.example/cdp",
+            {"timeout": 90_000, "headers": {"Authorization": "Bearer secret"}},
+        ),
+        ("browser_cdp", "Browser.getVersion"),
+        ("browser_cdp", "detach"),
+        ("binding", "__browserEvalCaptureRequestBody", binding_event[2]),
+        ("init_script", next(event[1] for event in events if event[0] == "init_script")),
+        ("route", predicate, handler),
+        ("viewport", {"width": 1280, "height": 900}),
+        ("goto", "about:blank", {"wait_until": "load", "timeout": 10_000}),
+        (
+            "bridge",
+            "request",
+            {
+                "method": "POST",
+                "url": "http://private.test/save",
+                "headers": {"content-type": "application/json"},
+                "data": b'{"enabled":true}',
+                "allow_redirects": False,
+                "timeout": 60,
+            },
+        ),
+        (
+            "bridge",
+            "fulfill",
+            {
+                "status": 201,
+                "headers": {"content-type": "application/json"},
+                "body": b'{"saved":true}',
+            },
+        ),
+        (
+            "bridge",
+            "request",
+            {
+                "method": "POST",
+                "url": "http://private.test/save",
+                "headers": {"content-type": "application/json"},
+                "data": b'{"enabled":true}',
+                "allow_redirects": False,
+                "timeout": 60,
+            },
+        ),
+        ("bridge", "abort", "failed"),
+        ("bridge", "close"),
+        ("browser", "close"),
+        ("playwright", "stop"),
+    ]
+
+
+def test_kitesurf_request_bridge_rebuilds_captured_form_data(monkeypatch) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    class Context:
+        def expose_binding(self, _name, callback):
+            self.capture = callback
+
+        def add_init_script(self, script):
+            assert "body instanceof FormData" in script
+
+        def route(self, _selector, handler):
+            self.bridge = handler
+
+    context = Context()
+    calls = []
+
+    class Response:
+        status_code = 201
+        headers = {"content-type": "application/json"}
+        content = b"{}"
+        url = "http://private.test/upload"
+        cookies = []
+
+    class Session:
+        trust_env = True
+
+        def request(self, **kwargs):
+            calls.append(kwargs)
+            return Response()
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", Session
+    )
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = context
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge({"http://private.test"})
+    context.capture(
+        {},
+        {
+            "requestId": "form-1",
+            "method": "POST",
+            "url": "http://private.test/upload",
+            "body": {
+                "kind": "form-data",
+                "entries": [
+                    {"name": "title", "value": "hello"},
+                    {
+                        "name": "image",
+                        "file": {
+                            "name": "",
+                            "type": "application/octet-stream",
+                            "base64": "",
+                        },
+                    },
+                ],
+            },
+        },
+    )
+
+    class Request:
+        method = "POST"
+        url = "http://private.test/upload"
+        post_data_buffer = None
+
+        def all_headers(self):
+            return {
+                "content-type": "multipart/form-data; boundary=browser-boundary",
+                "x-csrf-token": "csrf",
+                "x-browser-eval-request-body-id": "form-1",
+            }
+
+    class Route:
+        request = Request()
+
+        def fulfill(self, **_kwargs):
+            pass
+
+    context.bridge(Route())
+
+    assert len(calls) == 1
+    call = calls[0]
+    assert call["method"] == "POST"
+    assert call["url"] == "http://private.test/upload"
+    assert call["headers"]["x-csrf-token"] == "csrf"
+    assert call["headers"]["content-type"].startswith(
+        "multipart/form-data; boundary=browser-eval-"
+    )
+    assert b'name="title"\r\n\r\nhello\r\n' in call["data"]
+    assert (
+        b'name="image"; filename=""\r\nContent-Type: application/octet-stream\r\n\r\n\r\n'
+        in call["data"]
+    )
+    assert call["data"].endswith(b"--\r\n")
+    assert call["allow_redirects"] is False
+    assert call["timeout"] == 60
+
+
+def test_kitesurf_request_capture_ids_prevent_same_url_body_reuse() -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._request_bridge_origins = frozenset({"http://private.test"})
+    tool._request_bridge_bodies = {}
+    tool._request_bridge_body_bytes = 0
+    tool._request_bridge_har_bodies = {}
+    tool._request_bridge_har_body_bytes = 0
+    for request_id, value in (("aborted-1", "old"), ("sent-2", "new")):
+        tool._capture_request_body(
+            {},
+            {
+                "requestId": request_id,
+                "method": "POST",
+                "url": "http://private.test/save",
+                "body": {"kind": "text", "text": value},
+            },
+        )
+
+    assert tool._pop_captured_request_body(
+        "sent-2", "POST", "http://private.test/save"
+    ) == {"kind": "text", "text": "new"}
+    assert "aborted-1" in tool._request_bridge_bodies
+    with pytest.raises(RuntimeError, match="does not match"):
+        tool._pop_captured_request_body(
+            "aborted-1", "PUT", "http://private.test/save"
+        )
+
+
+def test_kitesurf_native_form_body_uses_capture_id(monkeypatch) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    class Context:
+        def expose_binding(self, _name, callback):
+            self.capture = callback
+
+        def add_init_script(self, script):
+            assert "HTMLFormElement.prototype.submit" in script
+
+        def route(self, _selector, handler):
+            self.bridge = handler
+
+    context = Context()
+    calls = []
+
+    class Response:
+        status_code = 302
+        headers = {"location": "/issues/1"}
+        content = b""
+        url = "http://private.test/issues"
+        cookies = []
+
+    class Session:
+        trust_env = True
+
+        def request(self, **kwargs):
+            calls.append(kwargs)
+            return Response()
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", Session
+    )
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = context
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge({"http://private.test"})
+    context.capture(
+        {},
+        {
+            "requestId": "native-form-1",
+            "method": "POST",
+            "url": "http://private.test/issues",
+            "source": "native-form",
+            "body": {"kind": "text", "text": "issue%5Btitle%5D=hello"},
+        },
+    )
+
+    class Request:
+        method = "POST"
+        url = "http://private.test/issues"
+        post_data_buffer = None
+
+        def all_headers(self):
+            return {
+                "content-type": "application/x-www-form-urlencoded",
+                "x-browser-eval-request-body-id": "native-form-1",
+            }
+
+    class Route:
+        request = Request()
+
+        def fulfill(self, **_kwargs):
+            pass
+
+    context.bridge(Route())
+
+    assert calls[0]["data"] == b"issue%5Btitle%5D=hello"
+    headers, body, response_headers = tool.transform_playwright_har_request(
+        "POST",
+        "http://private.test/issues",
+        {"x-browser-eval-request-body-id": "native-form-1"},
+    )
+    assert headers == {}
+    assert body == "issue%5Btitle%5D=hello"
+    assert response_headers == {
+        "status": 302,
+        "statusText": "",
+        "headers": {"location": "/issues/1"},
+    }
+
+
+def test_kitesurf_request_bridge_encodes_empty_form_data() -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    data, content_type = PlaywrightReplayTool._decode_captured_request_body(
+        "empty-form", {"kind": "form-data", "entries": []}
+    )
+    boundary = content_type.removeprefix("multipart/form-data; boundary=")
+    assert data == f"--{boundary}--\r\n".encode()
+
+
+@pytest.mark.parametrize("content_type", ["text/plain\r\nX-Test: injected", "text/plain\n"])
+def test_kitesurf_request_bridge_rejects_newlines_in_file_content_type(
+    content_type: str,
+) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    body = {
+        "kind": "form-data",
+        "entries": [
+            {
+                "name": "upload",
+                "file": {"name": "note.txt", "type": content_type, "base64": ""},
+            }
+        ],
+    }
+    with pytest.raises(ValueError, match="content type contains a newline"):
+        PlaywrightReplayTool._decode_captured_request_body("unsafe-type", body)
+
+
+def test_kitesurf_request_body_id_survives_post_redirect_as_bodyless_get(
+    monkeypatch,
+) -> None:
+    from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+    class Context:
+        def expose_binding(self, _name, callback):
+            self.capture = callback
+
+        def add_init_script(self, _script):
+            pass
+
+        def route(self, _selector, handler):
+            self.bridge = handler
+
+    context = Context()
+    calls = []
+
+    class Response:
+        cookies = []
+
+        def __init__(self, status, url, headers):
+            self.status_code = status
+            self.url = url
+            self.headers = headers
+            self.content = b""
+
+    class Session:
+        trust_env = True
+
+        def request(self, **kwargs):
+            calls.append(kwargs)
+            if kwargs["url"].endswith("/start"):
+                return Response(302, kwargs["url"], {"location": "/final"})
+            if kwargs["url"].endswith("/keep"):
+                return Response(307, kwargs["url"], {"location": "/kept"})
+            return Response(200, kwargs["url"], {"content-type": "text/plain"})
+
+    monkeypatch.setattr(
+        "harness.trajectory_replay.playwright_tool.requests.Session", Session
+    )
+    tool = PlaywrightReplayTool.__new__(PlaywrightReplayTool)
+    tool._context = context
+    tool.request_bridge_stats = {"requests": 0, "failures": 0}
+    tool._install_request_bridge({"http://private.test"})
+    context.capture(
+        {},
+        {
+            "requestId": "redirect-1",
+            "method": "POST",
+            "url": "http://private.test/start",
+            "body": {"kind": "text", "text": "enabled=true"},
+        },
+    )
+
+    class Request:
+        post_data_buffer = None
+
+        def __init__(self, method, url):
+            self.method = method
+            self.url = url
+
+        def all_headers(self):
+            return {"x-browser-eval-request-body-id": "redirect-1"}
+
+    class Route:
+        def __init__(self, request):
+            self.request = request
+
+        def fulfill(self, **_kwargs):
+            pass
+
+    context.bridge(Route(Request("POST", "http://private.test/start")))
+    context.bridge(Route(Request("GET", "http://private.test/final")))
+
+    context.capture(
+        {},
+        {
+            "requestId": "redirect-2",
+            "method": "POST",
+            "url": "http://private.test/keep",
+            "body": {"kind": "text", "text": "kept=true"},
+        },
+    )
+
+    class KeptRequest(Request):
+        def all_headers(self):
+            return {"x-browser-eval-request-body-id": "redirect-2"}
+
+    context.bridge(Route(KeptRequest("POST", "http://private.test/keep")))
+    context.bridge(Route(KeptRequest("POST", "http://private.test/kept")))
+
+    assert [call["data"] for call in calls] == [
+        b"enabled=true",
+        None,
+        b"kept=true",
+        b"kept=true",
+    ]
+    original_headers, original_body, _ = tool.transform_playwright_har_request(
+        "POST",
+        "http://private.test/start",
+        {"x-browser-eval-request-body-id": "redirect-1"},
+    )
+    redirect_headers, redirect_body, _ = tool.transform_playwright_har_request(
+        "GET",
+        "http://private.test/final",
+        {"x-browser-eval-request-body-id": "redirect-1"},
+    )
+    assert original_headers == redirect_headers == {}
+    assert original_body == "enabled=true"
+    assert redirect_body is None
+    _, kept_body, _ = tool.transform_playwright_har_request(
+        "POST",
+        "http://private.test/kept",
+        {"x-browser-eval-request-body-id": "redirect-2"},
+    )
+    assert kept_body == "kept=true"
+
+
+def test_kitesurf_request_capture_bounds_pending_and_retained_records(
+    monkeypatch,
+) -> None:
+    from harness.trajectory_replay import playwright_tool as module
+
+    tool = module.PlaywrightReplayTool.__new__(module.PlaywrightReplayTool)
+    tool._request_bridge_origins = frozenset({"http://private.test"})
+    tool._request_bridge_bodies = {}
+    tool._request_bridge_body_bytes = 0
+    tool._request_bridge_har_bodies = {"finished": {}}
+    tool._request_bridge_har_body_bytes = 0
+    monkeypatch.setattr(module, "REQUEST_BODY_CAPTURE_MAX_PENDING", 1)
+
+    with pytest.raises(RuntimeError, match="count exceeded"):
+        tool._capture_request_body(
+            {},
+            {
+                "requestId": "next",
+                "method": "POST",
+                "url": "http://private.test/save",
+                "body": {"kind": "text", "text": ""},
+            },
+        )
+    monkeypatch.setattr(module, "REQUEST_BODY_CAPTURE_MAX_PENDING", 2)
+    with pytest.raises(ValueError, match="reused"):
+        tool._capture_request_body(
+            {},
+            {
+                "requestId": "finished",
+                "method": "POST",
+                "url": "http://private.test/save",
+                "body": {"kind": "text", "text": ""},
+            },
+        )
+
+
+def test_kitesurf_login_form_is_submitted_by_local_bridge() -> None:
+    events = []
+
+    class Submitter:
+        def evaluate(self, script):
+            assert "new FormData(form)" in script
+            return {
+                "action": "http://gitlab.test/users/sign_in",
+                "method": "post",
+                "fields": [
+                    ["user[login]", "alice"],
+                    ["user[password]", "secret"],
+                    ["authenticity_token", "csrf"],
+                ],
+            }
+
+    class RedirectResponse:
+        status_code = 302
+        url = "http://gitlab.test/users/sign_in"
+        headers = {"location": "/"}
+        cookies = []
+
+    class FinalResponse:
+        status_code = 200
+        url = "http://gitlab.test/"
+        headers = {}
+        cookies = []
+
+    class Session:
+        responses = iter((RedirectResponse(), FinalResponse()))
+
+        def request(self, **kwargs):
+            events.append(("request", kwargs))
+            return next(self.responses)
+
+    class Page:
+        url = "http://gitlab.test/users/sign_in"
+
+        def goto(self, url, **kwargs):
+            events.append(("goto", url, kwargs))
+
+    tool = _unit_tool(Page(), object(), engine="kitesurf")
+    tool._request_bridge_session = Session()
+    tool._sync_request_bridge_cookies = lambda url, cookies: events.append(
+        ("cookies", url, list(cookies))
+    )
+
+    final_url = tool._submit_login_form(tool._page, Submitter())
+
+    assert final_url == "http://gitlab.test/"
+    assert events == [
+        (
+            "request",
+            {
+                "method": "post",
+                "url": "http://gitlab.test/users/sign_in",
+                "headers": {"referer": "http://gitlab.test/users/sign_in"},
+                "allow_redirects": False,
+                "timeout": 60,
+                "data": [
+                    ["user[login]", "alice"],
+                    ["user[password]", "secret"],
+                    ["authenticity_token", "csrf"],
+                ],
+            },
+        ),
+        ("cookies", "http://gitlab.test/users/sign_in", []),
+        (
+            "request",
+            {
+                "method": "get",
+                "url": "http://gitlab.test/",
+                "headers": {"referer": "http://gitlab.test/users/sign_in"},
+                "allow_redirects": False,
+                "timeout": 60,
+            },
+        ),
+        ("cookies", "http://gitlab.test/", []),
+        (
+            "goto",
+            "http://gitlab.test/",
+            {"wait_until": "domcontentloaded", "timeout": 90_000},
+        ),
+    ]
+
+
+def test_request_bridge_cookie_sync_preserves_host_only_scope() -> None:
+    added = []
+    context = type("Context", (), {"add_cookies": lambda self, rows: added.extend(rows)})()
+    tool = _unit_tool(object(), context, engine="kitesurf")
+    tool._request_bridge_origins = frozenset({"http://shop.test"})
+    cookie = type(
+        "Cookie",
+        (),
+        {
+            "name": "session",
+            "value": "one",
+            "path": "/",
+            "secure": False,
+            "expires": None,
+            "domain": "shop.test",
+            "domain_specified": False,
+            "_rest": {"HttpOnly": None, "SameSite": "Lax"},
+        },
+    )()
+
+    tool._sync_request_bridge_cookies("http://shop.test/account", [cookie])
+
+    assert added == [
+        {
+            "name": "session",
+            "value": "one",
+            "url": "http://shop.test/",
+            "secure": False,
+            "httpOnly": True,
+            "sameSite": "Lax",
+        }
+    ]
+
+
+def test_request_bridge_cookie_sync_rejects_nonroot_host_only_path() -> None:
+    tool = _unit_tool(
+        object(), type("Context", (), {"add_cookies": lambda *_args: None})(), engine="kitesurf"
+    )
+    tool._request_bridge_origins = frozenset({"http://shop.test"})
+    cookie = type(
+        "Cookie",
+        (),
+        {
+            "name": "session",
+            "value": "one",
+            "path": "/account",
+            "secure": False,
+            "expires": None,
+            "domain": "shop.test",
+            "domain_specified": False,
+            "_rest": {},
+        },
+    )()
+
+    with pytest.raises(RuntimeError, match="non-root host-only cookie path"):
+        tool._sync_request_bridge_cookies("http://shop.test/account", [cookie])
+
+
+def test_request_bridge_cookie_sync_preserves_explicit_domain_scope() -> None:
+    added = []
+    context = type("Context", (), {"add_cookies": lambda self, rows: added.extend(rows)})()
+    tool = _unit_tool(object(), context, engine="kitesurf")
+    tool._request_bridge_origins = frozenset({"https://admin.shop.test"})
+    cookie = type(
+        "Cookie",
+        (),
+        {
+            "name": "admin",
+            "value": "two",
+            "path": "/",
+            "secure": True,
+            "expires": 1_900_000_000,
+            "domain": ".shop.test",
+            "domain_specified": True,
+            "_rest": {},
+        },
+    )()
+
+    tool._sync_request_bridge_cookies("https://admin.shop.test/", [cookie])
+
+    assert added == [
+        {
+            "name": "admin",
+            "value": "two",
+            "domain": ".shop.test",
+            "path": "/",
+            "secure": True,
+            "httpOnly": False,
+            "expires": 1_900_000_000.0,
+        }
+    ]
+
+
+def test_request_bridge_cookie_sync_rejects_cross_origin_response() -> None:
+    tool = _unit_tool(object(), object(), engine="kitesurf")
+    tool._request_bridge_origins = frozenset({"http://shop.test"})
+
+    with pytest.raises(RuntimeError, match="origin is not allowed"):
+        tool._sync_request_bridge_cookies("http://metadata.invalid/", [])
+
+
+def test_login_form_bridge_rejects_cross_origin_redirect_before_fetch() -> None:
+    calls = []
+
+    class Redirect:
+        status_code = 302
+        url = "http://gitlab.test/users/sign_in"
+        headers = {"location": "http://metadata.invalid/latest"}
+        cookies = []
+
+    class Session:
+        def request(self, **kwargs):
+            calls.append(kwargs)
+            if len(calls) > 1:
+                raise AssertionError("cross-origin redirect must not be fetched")
+            return Redirect()
+
+    tool = _unit_tool(object(), object(), engine="kitesurf")
+    tool._request_bridge_session = Session()
+    tool._sync_request_bridge_cookies = lambda *_args: None
+
+    with pytest.raises(RuntimeError, match="cross-origin redirect"):
+        tool._request_login_form(
+            source_origin="http://gitlab.test",
+            method="post",
+            url="http://gitlab.test/users/sign_in",
+            fields=[["user[login]", "alice"]],
+            referer="http://gitlab.test/users/sign_in",
+        )
+
+    assert len(calls) == 1
+
+
+def test_bridge_close_failure_still_closes_browser_and_playwright() -> None:
+    events = []
+
+    class BridgeSession:
+        def close(self):
+            events.append("bridge")
+            raise RuntimeError("bridge close failed")
+
+    tool = _unit_tool(object(), object(), engine="kitesurf")
+    tool._request_bridge_session = BridgeSession()
+    tool._layout_connection_poisoned = False
+    tool._browser = type("Browser", (), {"close": lambda self: events.append("browser")})()
+    tool._playwright = type(
+        "Playwright", (), {"stop": lambda self: events.append("playwright")}
+    )()
+
+    with pytest.raises(RuntimeError, match="bridge close failed"):
+        tool.close()
+
+    assert events == ["bridge", "browser", "playwright"]
+
+
+def test_kitesurf_login_readiness_uses_dom_polling() -> None:
+    events = []
+
+    class Page:
+        def evaluate(self, script, argument):
+            events.append(("evaluate", argument))
+            assert "document.querySelectorAll(selector)" in script
+            return sum(event[0] == "evaluate" for event in events) == 2
+
+        def wait_for_timeout(self, timeout):
+            events.append(("wait", timeout))
+
+        def locator(self, _selector):
+            raise AssertionError("Kitesurf readiness must not use Locator.wait_for")
+
+    tool = _unit_tool(Page(), object(), engine="kitesurf")
+    tool._wait_for_login_ready(
+        tool._page, "button", exact_text="alice", timeout_ms=1_000
+    )
+
+    assert events == [
+        ("evaluate", {"selector": "button", "exactText": "alice"}),
+        ("wait", 100),
+        ("evaluate", {"selector": "button", "exactText": "alice"}),
+    ]
+
+
+def test_kitesurf_document_observer_uses_page_evaluate_without_locator() -> None:
+    class Page:
+        def evaluate(self, script, limit):
+            assert "document.body?.innerText" in script
+            assert limit == 123
+            return {"title": "Dashboard", "body": "Visible body"}
+
+        def locator(self, _selector):
+            raise AssertionError("Kitesurf document observation must not use Locator")
+
+    tool = _unit_tool(Page(), object(), engine="kitesurf")
+
+    assert tool.document_observer(tool._page, 1_000, 123) == (
+        "Dashboard",
+        "Visible body",
+    )
+
+
+def test_kitesurf_document_observer_enforces_action_timeout() -> None:
+    class Page:
+        def evaluate(self, _script, _limit):
+            time.sleep(1)
+
+    tool = _unit_tool(Page(), object(), engine="kitesurf")
+
+    started = time.monotonic()
+    with pytest.raises(TimeoutError, match="exceeded 20 ms"):
+        tool.document_observer(tool._page, 20, 123)
+    assert time.monotonic() - started < 0.5
+
+
+def test_obscura_viewport_verification_rejects_configured_but_unapplied_size() -> None:
+    class Session:
+        def send(self, method, _params=None):
+            if method == "Page.getLayoutMetrics":
+                return {
+                    "cssVisualViewport": {"clientWidth": 800, "clientHeight": 600}
+                }
+            return {"result": {"value": 1}}
+
+    page = type("Page", (), {"viewport_size": {"width": 1280, "height": 900}})()
+    context = type(
+        "Context",
+        (),
+        {"new_cdp_session": lambda self, selected: Session()},
+    )()
+    tool = _unit_tool(page, context, engine="obscura")
+    tool._owns_cdp_context = True
+    tool._supports_multiple_pages = False
+    tool._browser = type("Browser", (), {"contexts": [context]})()
+    tool._submit = lambda callback: callback()
+
+    with pytest.raises(RuntimeError, match="viewport mismatch"):
+        _prepare_browser_viewport(tool)
+
+
+def test_moli_layout_publication_uses_explicit_geometry_only_cdp():
+    events = []
+
+    class Impl:
+        async def send(self, method, params):
+            events.append(("send", method, params))
+            return {"layoutViewport": {"clientWidth": 1280, "clientHeight": 720}}
+
+    class Session:
+        _impl_obj = Impl()
+
+        def _sync(self, coroutine):
+            return asyncio.run(coroutine)
+
+    page = object()
+    context = type(
+        "Context",
+        (),
+        {
+            "new_cdp_session": lambda self, selected: (
+                events.append(("session", selected)) or Session()
+            )
+        },
+    )()
+
+    _unit_tool(page, context, engine="moli")._publish_page_layout(page)
+
+    assert events == [
+        ("session", page),
+        ("send", "Page.getLayoutMetrics", {"publishLayout": True}),
+    ]
+
+
+def test_chromium_layout_publication_is_a_strict_noop():
+    class Context:
+        def new_cdp_session(self, _page):
+            raise AssertionError("Chromium must not receive Moli layout publication")
+
+    tool = _unit_tool(object(), Context())
+    assert tool.layout_observer is None
+    tool._publish_page_layout(object())
+
+
+def test_moli_layout_publication_fails_when_renderer_returns_no_metrics():
+    class Impl:
+        async def send(self, _method, _params):
+            return {}
+
+    class Session:
+        _impl_obj = Impl()
+
+        def _sync(self, coroutine):
+            return asyncio.run(coroutine)
+
+    context = type("Context", (), {"new_cdp_session": lambda self, _page: Session()})()
+
+    with pytest.raises(RuntimeError, match="returned no layout metrics"):
+        _unit_tool(object(), context, engine="moli")._publish_page_layout(object())
+
+
+def test_moli_layout_publication_obeys_its_budget():
+    class Impl:
+        async def send(self, _method, _params):
+            await asyncio.sleep(1)
+            return {"layoutViewport": {"clientWidth": 1280, "clientHeight": 720}}
+
+    class Session:
+        _impl_obj = Impl()
+
+        def _sync(self, coroutine):
+            return asyncio.run(coroutine)
+
+    context = type("Context", (), {"new_cdp_session": lambda self, _page: Session()})()
+    started = time.monotonic()
+
+    tool = _unit_tool(object(), context, engine="moli")
+    with pytest.raises(TimeoutError, match="publication timed out"):
+        tool._publish_page_layout(object(), timeout_ms=20)
+
+    assert time.monotonic() - started < 0.2
+    assert tool.failure_capture_available is False
+
+
+def test_poisoned_layout_connection_skips_graceful_close():
+    class MustNotClose:
+        def close(self):
+            raise AssertionError("poisoned renderer connection must not receive another command")
+
+    tool = _unit_tool(object(), MustNotClose(), engine="moli")
+    tool._browser = MustNotClose()
+    stops = []
+    tool._playwright = type("Playwright", (), {"stop": lambda self: stops.append("stop")})()
+    tool._layout_connection_poisoned = True
+
+    tool.close()
+    assert stops == ["stop"]
+
+
+def test_snapshot_retries_native_observation_after_navigation_settles():
+    events = []
+
+    class Locator:
+        @property
+        def first(self):
+            return self
+
+        def wait_for(self, **kwargs):
+            events.append(("visible", kwargs))
+
+        def aria_snapshot(self, **kwargs):
+            events.append(("aria_snapshot", kwargs))
+            if sum(event[0] == "aria_snapshot" for event in events) == 1:
+                raise RuntimeError("Execution context was destroyed because of a navigation")
+            return "- main: New profile document"
+
+    class Page:
+        def locator(self, selector):
+            events.append(("locator", selector))
+            return Locator()
+
+        def wait_for_load_state(self, state, **kwargs):
+            events.append(("load_state", state, kwargs))
+
+        def wait_for_timeout(self, milliseconds):
+            events.append(("wait", milliseconds))
+
+    result = _unit_tool(Page(), object()).execute({"action": "snapshot"})
+
+    assert result.status == "success"
+    assert result.result == "- main: New profile document"
+    assert sum(event[0] == "aria_snapshot" for event in events) == 2
+    assert any(event[:2] == ("load_state", "domcontentloaded") for event in events)
+
+
+def test_snapshot_never_returns_old_document_when_navigation_does_not_settle():
+    events = []
+
+    class Locator:
+        @property
+        def first(self):
+            return self
+
+        def wait_for(self, **kwargs):
+            events.append("visible")
+
+        def aria_snapshot(self, **kwargs):
+            events.append("aria")
+            raise RuntimeError("Execution context was destroyed because of a navigation")
+
+    class Page:
+        def locator(self, selector):
+            return Locator()
+
+        def wait_for_load_state(self, state, **kwargs):
+            events.append("load_state")
+            raise TimeoutError("new document did not reach DOMContentLoaded")
+
+    result = _unit_tool(Page(), object()).execute({"action": "snapshot"})
+
+    assert result.status == "error"
+    assert "did not reach DOMContentLoaded" in result.result
+    assert events == ["load_state", "visible", "aria", "load_state"]
+
+
+def test_snapshot_does_not_hide_a_non_navigation_observation_error():
+    class Locator:
+        @property
+        def first(self):
+            return self
+
+        def wait_for(self, **kwargs):
+            pass
+
+        def aria_snapshot(self, **kwargs):
+            raise RuntimeError("Accessibility tree serialization failed")
+
+    class Page:
+        def locator(self, selector):
+            return Locator()
+
+    result = _unit_tool(Page(), object()).execute({"action": "snapshot"})
+    assert result.status == "error"
+    assert "Accessibility tree serialization failed" in result.result
+
+
+def test_snapshot_hidden_selector_fails_before_observation():
+    events = []
+
+    class Locator:
+        @property
+        def first(self):
+            return self
+
+        def wait_for(self, **kwargs):
+            events.append(("visible", kwargs))
+            raise TimeoutError("element remained hidden")
+
+        def aria_snapshot(self, **kwargs):
+            events.append(("aria", kwargs))
+            return "old hidden document"
+
+    class Page:
+        def locator(self, selector):
+            return Locator()
+
+    result = _unit_tool(Page(), object()).execute(
+        {"action": "snapshot", "selector": "#hidden", "timeout_ms": 20}
+    )
+    assert result.status == "error"
+    assert "remained hidden" in result.result
+    assert events
+    assert all(event[0] == "visible" for event in events)
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_kitesurf_native_form_capture_freezes_one_submission_snapshot() -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        NATIVE_FORM_REDIRECT_HEADER,
+        REQUEST_BODY_CAPTURE_BINDING,
+        REQUEST_BODY_CAPTURE_HEADER,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    posts = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(_request_body_capture_script(frozenset({"http://capture.test"})))
+
+        gets = []
+
+        def route_request(route):
+            request = route.request
+            if request.method == "POST":
+                posts.append(
+                    {
+                        "url": request.url,
+                        "headers": request.all_headers(),
+                        "body": request.post_data,
+                    }
+                )
+                route.fulfill(
+                    status=200,
+                    headers={NATIVE_FORM_REDIRECT_HEADER: "http://capture.test/save"},
+                    body="",
+                )
+                return
+            gets.append({"url": request.url, "resource_type": request.resource_type})
+            if request.url.endswith("/form"):
+                route.fulfill(
+                    status=200,
+                    content_type="text/html",
+                    body=(
+                        '
    '
+                        ''
+                        ''
+                        "
    "
+                    ),
+                )
+                return
+            route.fulfill(status=200, content_type="text/html", body='
    done
    ')
+
+        context.route("http://capture.test/**", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/form")
+        page.evaluate(
+            """() => {
+                const form = document.querySelector('form');
+                let count = 0;
+                form.addEventListener('formdata', event => {
+                    event.formData.append('nonce', String(++count));
+                });
+                form.elements.note.value = 'a\\nb';
+                form.submit();
+                form.action = '/other';
+                form.elements.x.value = 'two';
+                form.remove();
+            }"""
+        )
+        page.wait_for_url("http://capture.test/save")
+        browser.close()
+
+    assert len(captures) == 1
+    assert captures[0]["url"] == "http://capture.test/save"
+    assert captures[0]["body"] == {
+        "kind": "text",
+        "text": "note=a%0D%0Ab&x=one&nonce=1",
+    }
+    assert len(posts) == 1
+    assert posts[0]["url"] == "http://capture.test/save"
+    assert posts[0]["body"] == "note=a%0D%0Ab&x=one&nonce=1"
+    assert posts[0]["headers"][REQUEST_BODY_CAPTURE_HEADER] == captures[0]["requestId"]
+    assert gets == [
+        {"url": "http://capture.test/form", "resource_type": "document"},
+        {"url": "http://capture.test/save", "resource_type": "document"},
+    ]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_kitesurf_fetch_capture_keeps_body_only_in_local_bridge() -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        REQUEST_BODY_CAPTURE_BINDING,
+        REQUEST_BODY_CAPTURE_HEADER,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    posts = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(
+            _request_body_capture_script(frozenset({"http://capture.test"}))
+        )
+
+        def route_request(route):
+            request = route.request
+            if request.method == "POST":
+                posts.append(
+                    {
+                        "headers": request.all_headers(),
+                        "body": request.post_data,
+                    }
+                )
+                route.fulfill(status=200, content_type="application/json", body="{}")
+                return
+            route.fulfill(status=200, content_type="text/html", body="
    ready
    ")
+
+        context.route("http://capture.test/**", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/")
+        status = page.evaluate(
+            """async () => (await fetch('/graphql', {
+                method: 'POST',
+                headers: {'content-type': 'application/json'},
+                body: JSON.stringify({page: 1}),
+            })).status"""
+        )
+        browser.close()
+
+    assert status == 200
+    assert len(captures) == 1
+    assert captures[0]["body"] == {"kind": "text", "text": '{"page":1}'}
+    assert len(posts) == 1
+    assert posts[0]["body"] is None
+    assert posts[0]["headers"]["content-type"] == "application/json"
+    assert posts[0]["headers"][REQUEST_BODY_CAPTURE_HEADER] == captures[0]["requestId"]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+@pytest.mark.parametrize("method", ["GET", "HEAD"])
+def test_kitesurf_fetch_capture_rejects_get_and_head_bodies(method: str) -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        REQUEST_BODY_CAPTURE_BINDING,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    requests = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(
+            _request_body_capture_script(frozenset({"http://capture.test"}))
+        )
+
+        def route_request(route):
+            requests.append(route.request.url)
+            route.fulfill(status=200, content_type="text/html", body="
    ready
    ")
+
+        context.route("http://capture.test/**", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/")
+        result = page.evaluate(
+            """async method => {
+                try {
+                    await fetch('/query', {method, body: 'invalid'});
+                    return 'resolved';
+                } catch (error) {
+                    return [error.name, error.message];
+                }
+            }""",
+            method,
+        )
+        browser.close()
+
+    assert result[0] == "TypeError"
+    assert "GET/HEAD" in result[1]
+    assert captures == []
+    assert requests == ["http://capture.test/"]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_kitesurf_fetch_capture_preserves_keepalive_body_limit() -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        REQUEST_BODY_CAPTURE_BINDING,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    requests = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(
+            _request_body_capture_script(frozenset({"http://capture.test"}))
+        )
+
+        def route_request(route):
+            requests.append(
+                {
+                    "url": route.request.url,
+                    "method": route.request.method,
+                    "body": route.request.post_data,
+                }
+            )
+            route.fulfill(status=200, content_type="text/html", body="
    ready
    ")
+
+        context.route("http://capture.test/**", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/")
+        result = page.evaluate(
+            """async () => {
+                const run = async body => {
+                    try {
+                        const response = await fetch('/save', {
+                            method: 'POST', keepalive: true, body,
+                        });
+                        return ['resolved', response.status];
+                    } catch (error) {
+                        return [error.name, error.message];
+                    }
+                };
+                const emptyForm = new FormData();
+                emptyForm.set('value', '');
+                const multipartOverhead = (
+                    await new Request('/save', {method: 'POST', body: emptyForm})
+                        .arrayBuffer()
+                ).byteLength;
+                const oversizedForm = new FormData();
+                oversizedForm.set('value', 'x'.repeat(65537 - multipartOverhead));
+                return [
+                    await run('x'.repeat(65536)),
+                    await run('x'.repeat(65537)),
+                    await run(oversizedForm),
+                ];
+            }"""
+        )
+        browser.close()
+
+    assert result[0] == ["resolved", 200]
+    assert result[1][0] == "TypeError"
+    assert result[2][0] == "TypeError"
+    assert len(captures) == 1
+    assert len(captures[0]["body"]["text"]) == 65536
+    assert requests == [
+        {"url": "http://capture.test/", "method": "GET", "body": None},
+        {"url": "http://capture.test/save", "method": "POST", "body": None},
+    ]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_kitesurf_fetch_capture_shares_and_releases_keepalive_budget() -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        REQUEST_BODY_CAPTURE_BINDING,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    held_routes = []
+    requested_urls = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(
+            _request_body_capture_script(frozenset({"http://capture.test"}))
+        )
+
+        def route_request(route):
+            requested_urls.append(route.request.url)
+            if route.request.url.endswith("/hold"):
+                held_routes.append(route)
+                return
+            route.fulfill(
+                status=200,
+                headers={"access-control-allow-origin": "*"},
+                content_type="text/html",
+                body="
    ready
    ",
+            )
+
+        context.route("**/*", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/")
+
+        for first, second in (
+            ("http://outside.test/hold", "http://capture.test/rejected"),
+            ("http://capture.test/hold", "http://outside.test/rejected"),
+        ):
+            result = page.evaluate(
+                """async ([first, second]) => {
+                    window.pendingKeepalive = fetch(first, {
+                        method: 'POST', keepalive: true, body: 'x'.repeat(40000),
+                    }).then(() => 'resolved', error => error.name);
+                    await new Promise(resolve => setTimeout(resolve, 50));
+                    try {
+                        await fetch(second, {
+                            method: 'POST', keepalive: true, body: 'x'.repeat(40000),
+                        });
+                        return 'resolved';
+                    } catch (error) {
+                        return error.name;
+                    }
+                }""",
+                [first, second],
+            )
+            assert result == "TypeError"
+            assert held_routes
+            held_routes.pop(0).abort("failed")
+            assert page.evaluate("window.pendingKeepalive") == "TypeError"
+            assert (
+                page.evaluate(
+                    """async () => (await fetch('/recovered', {
+                        method: 'POST', keepalive: true, body: 'x'.repeat(40000),
+                    })).status"""
+                )
+                == 200
+            )
+
+        browser.close()
+
+    assert held_routes == []
+    assert "http://capture.test/rejected" not in requested_urls
+    assert "http://outside.test/rejected" not in requested_urls
+    assert requested_urls.count("http://capture.test/recovered") == 2
+    assert len(captures) == 3
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_kitesurf_fetch_capture_allows_replacement_body_for_used_request() -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    from playwright.sync_api import sync_playwright
+
+    from harness.trajectory_replay.playwright_tool import (
+        REQUEST_BODY_CAPTURE_BINDING,
+        _request_body_capture_script,
+    )
+
+    captures = []
+    posts = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(executable_path=str(binary), headless=True)
+        context = browser.new_context()
+        context.expose_binding(
+            REQUEST_BODY_CAPTURE_BINDING,
+            lambda _source, payload: captures.append(payload),
+        )
+        context.add_init_script(
+            _request_body_capture_script(frozenset({"http://capture.test"}))
+        )
+
+        def route_request(route):
+            if route.request.method == "POST":
+                posts.append((route.request.url, route.request.post_data))
+            route.fulfill(
+                status=200,
+                headers={"access-control-allow-origin": "*"},
+                content_type="text/html",
+                body="
    ready
    ",
+            )
+
+        context.route("**/*", route_request)
+        page = context.new_page()
+        page.goto("http://capture.test/")
+        results = page.evaluate(
+            """async () => {
+                const replace = async url => {
+                    const request = new Request(url, {method: 'POST', body: 'old'});
+                    await request.text();
+                    try {
+                        return ['ok', (await fetch(request, {body: 'new'})).status];
+                    } catch (error) {
+                        return [error.name, error.message];
+                    }
+                };
+                return [
+                    await replace('http://capture.test/selected'),
+                    await replace('http://outside.test/native'),
+                ];
+            }"""
+        )
+        browser.close()
+
+    assert results == [["ok", 200], ["ok", 200]]
+    assert captures[0]["body"] == {"kind": "text", "text": "new"}
+    assert posts == [
+        ("http://capture.test/selected", None),
+        ("http://outside.test/native", "new"),
+    ]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="uses the pinned macOS Chromium")
+def test_playwright_backend_executes_portable_and_snapshot_ref_calls(tmp_path: Path) -> None:
+    binary = Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
+    if not binary.is_file():
+        pytest.skip("Chrome is unavailable")
+    page_path = tmp_path / "form.html"
+    page_path.write_text(
+        """
    
+        """,
+        encoding="utf-8",
+    )
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+    endpoint = f"http://127.0.0.1:{port}"
+    process = subprocess.Popen(
+        _browser_command(
+            argparse.Namespace(engine="chromium", binary=binary),
+            port,
+            str(tmp_path / "chrome-profile"),
+        ),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        _wait_cdp(endpoint, process)
+        from harness.trajectory_replay.playwright_tool import PlaywrightReplayTool
+
+        tool = PlaywrightReplayTool(endpoint)
+        try:
+            assert _prepare_browser_viewport(tool)["verified"]
+            started = time.monotonic()
+            status, _ = _execute_replay_call(
+                tool,
+                {"action": "navigate", "url": page_path.as_uri(), "timeout_ms": 1_000},
+            )
+            assert status == "success"
+            assert time.monotonic() - started <= 1.5
+            status, result = _execute_replay_call(
+                tool,
+                {
+                    "action": "snapshot",
+                    "selector": "#hidden",
+                    "timeout_ms": 100,
+                },
+            )
+            assert status == "error"
+            assert "hidden" in result.lower()
+            status, _ = _execute_replay_call(
+                tool,
+                {
+                    "action": "fill",
+                    "selector": "#name",
+                    "text": "Chromium replay",
+                    "replay_mode": "snapshot_ref_semantics",
+                },
+            )
+            assert status == "success"
+            status, _ = _execute_replay_call(
+                tool,
+                {
+                    "action": "click",
+                    "selector": "#save",
+                    "replay_mode": "snapshot_ref_semantics",
+                },
+            )
+            assert status == "success"
+            status, result = _execute_replay_call(
+                tool,
+                {"action": "evaluate", "script": "document.querySelector('#result').textContent"},
+            )
+            assert (status, result) == ("success", "Chromium replay")
+            status, result = _execute_replay_call(
+                tool,
+                {"action": "evaluate", "script": "({saved:true,count:1})"},
+            )
+            assert status == "success"
+            assert json.loads(result) == {"saved": True, "count": 1}
+            output: dict[str, str] = {}
+            tool._submit(
+                lambda: execute_steps(
+                    tool._page,
+                    [{"action": "get_text", "selector": "#result", "save_as": "saved"}],
+                    outputs=output,
+                )
+            )
+            assert output == {"saved": "Chromium replay"}
+        finally:
+            tool.close()
+    finally:
+        process.terminate()
+        process.wait(timeout=10)
diff --git a/tests/trajectory_replay/test_remote_candidate_cohort_diagnostic.py b/tests/trajectory_replay/test_remote_candidate_cohort_diagnostic.py
index 9af16ccd6..fe730c1d8 100644
--- a/tests/trajectory_replay/test_remote_candidate_cohort_diagnostic.py
+++ b/tests/trajectory_replay/test_remote_candidate_cohort_diagnostic.py
@@ -9,14 +9,15 @@
 import pytest
 from webarena_verified.types.task import WebArenaSite
 
-from harness.trajectory_replay.remote_runtime import rebind_remote_calls
-from harness.trajectory_replay.webarena_replay import _json_sha256
+from harness.trajectory_replay.catalog_runner import _chromium_host_resolver_rules
 from harness.trajectory_replay.remote_execution import (
     _browser_command,
     _read_retrieval_contract,
     _task_type,
     _verify_result_evidence,
 )
+from harness.trajectory_replay.remote_runtime import rebind_remote_calls
+from harness.trajectory_replay.webarena_replay import _json_sha256
 
 
 def _sites() -> dict[WebArenaSite, str]:
@@ -68,6 +69,114 @@ def test_retrieval_contract_uses_the_qualified_source_revision() -> None:
     assert _read_retrieval_contract(task_id, row, sites) == candidate
 
 
+def test_retrieval_contract_can_be_frozen_inside_a_catalog() -> None:
+    task_id = 108
+    candidate = json.loads(
+        (
+            Path("evaluation/trajectory-replay/webarena-verified/retrieval-candidates")
+            / f"task-{task_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    sites = _sites()
+    rebound = rebind_remote_calls(
+        candidate["tool_calls"],
+        sites,
+        {"127.0.0.1", "192.168.5.2", "host.docker.internal"},
+    )
+    row = {
+        "source_actions": candidate["tool_calls"],
+        "retrieval_contract": candidate["retrieval_contract"],
+        "bound_calls_sha256": _json_sha256(rebound),
+    }
+    assert _read_retrieval_contract(task_id, row, sites) == candidate
+    row["retrieval_contract"] = {**row["retrieval_contract"], "task_id": 109}
+    with pytest.raises(ValueError, match="identity mismatch"):
+        _read_retrieval_contract(task_id, row, sites)
+
+
+def test_embedded_retrieval_contract_validates_effective_calls_after_action_override() -> None:
+    task_id = 28
+    sites = _sites()
+    source_action = {
+        "action": "evaluate",
+        "script": "const value = 1; return value;",
+    }
+    source_calls = [source_action]
+    contract = {
+        "task_id": task_id,
+        "canonical_calls_sha256": _json_sha256(source_calls),
+    }
+    amendment = {
+        "task_id": task_id,
+        "action_index": 1,
+        "precondition_action_sha256": _json_sha256(source_action),
+        "set": {"script_mode": "function_body"},
+    }
+    effective_calls = [{**source_action, "script_mode": "function_body"}]
+    overrides_sha256 = "a" * 64
+    row = {
+        "source_actions": source_calls,
+        "retrieval_contract": contract,
+        "bound_calls_sha256": _json_sha256(effective_calls),
+        "action_overrides": {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "source_sha256": overrides_sha256,
+            "amendments_sha256": _json_sha256([amendment]),
+            "amendments": [amendment],
+        },
+    }
+
+    candidate = _read_retrieval_contract(task_id, row, sites)
+
+    assert candidate["tool_calls"] == source_calls
+    assert candidate["retrieval_contract"]["canonical_calls_sha256"] == _json_sha256(source_calls)
+
+
+def test_embedded_retrieval_contract_rejects_stale_override_precondition() -> None:
+    task_id = 28
+    action = {"action": "evaluate", "script": "return true;"}
+    amendment = {
+        "task_id": task_id,
+        "action_index": 1,
+        "precondition_action_sha256": "0" * 64,
+        "set": {"script_mode": "function_body"},
+    }
+    row = {
+        "source_actions": [action],
+        "retrieval_contract": {"task_id": task_id},
+        "bound_calls_sha256": "0" * 64,
+        "action_overrides": {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "source_sha256": "a" * 64,
+            "amendments_sha256": _json_sha256([amendment]),
+            "amendments": [amendment],
+        },
+    }
+
+    with pytest.raises(ValueError, match="precondition changed"):
+        _read_retrieval_contract(task_id, row, _sites())
+
+
+def test_embedded_retrieval_contract_rebinds_its_recorded_deployment() -> None:
+    task_id = 243
+    sites = _sites()
+    source_sites = {WebArenaSite.GITLAB: "https://old-gitlab.example.test"}
+    calls = [{"action": "navigate", "url": "https://old-gitlab.example.test/owner/project"}]
+    contract = {"task_id": task_id}
+    rebound = rebind_remote_calls(calls, sites, set(), source_sites)
+    row = {
+        "source_actions": calls,
+        "source_site_urls": {site.value: url for site, url in source_sites.items()},
+        "retrieval_contract": contract,
+        "bound_calls_sha256": _json_sha256(rebound),
+    }
+    assert rebound[0]["url"] == "https://gitlab.example.test/owner/project"
+    assert _read_retrieval_contract(task_id, row, sites)["tool_calls"] == calls
+    row["source_site_urls"] = {"gitlab": "https://different-gitlab.example.test"}
+    with pytest.raises(ValueError, match="differ from frozen bound calls"):
+        _read_retrieval_contract(task_id, row, sites)
+
+
 def test_response_type_comes_from_task_contract() -> None:
     expected = SimpleNamespace(model_dump=lambda **kwargs: {"task_type": "MUTATE"})
     task = SimpleNamespace(
@@ -101,6 +210,76 @@ def test_chromium_qualification_uses_a_separate_ephemeral_profile() -> None:
     assert moli[-2:] == ["--profile-dir", "/tmp/qualification-profile"]
 
 
+def test_comparison_browsers_use_their_native_cdp_serve_contracts() -> None:
+    lightpanda = _browser_command(
+        SimpleNamespace(engine="lightpanda", binary=Path("/opt/lightpanda")),
+        9222,
+        "/tmp/qualification-profile",
+    )
+    obscura = _browser_command(
+        SimpleNamespace(engine="obscura", binary=Path("/opt/obscura")),
+        9222,
+        "/tmp/qualification-profile",
+    )
+    assert lightpanda == [
+        "/opt/lightpanda",
+        "serve",
+        "--host",
+        "127.0.0.1",
+        "--port",
+        "9222",
+    ]
+    assert obscura == [
+        "/opt/obscura",
+        "serve",
+        "--host",
+        "127.0.0.1",
+        "--port",
+        "9222",
+        "--storage-dir",
+        "/tmp/qualification-profile",
+        "--allow-private-network",
+    ]
+
+
+def test_chromium_can_bind_profile_declared_private_ingress() -> None:
+    rules = _chromium_host_resolver_rules(
+        {
+            "host_overrides": [
+                {"pattern": "*.dev.lexmount.net", "address": "10.2.62.142"}
+            ]
+        },
+        "chromium",
+    )
+    chrome = _browser_command(
+        SimpleNamespace(
+            engine="chromium",
+            binary=Path("/opt/chrome"),
+            host_resolver_rules=rules,
+        ),
+        9222,
+        "/tmp/qualification-profile",
+    )
+    assert "--host-resolver-rules=MAP *.dev.lexmount.net 10.2.62.142" in chrome
+
+
+def test_environment_host_overrides_are_not_compiled_for_other_engines() -> None:
+    profile = {
+        "host_overrides": [
+            {"pattern": "*.dev.lexmount.net", "address": "10.2.62.142"}
+        ]
+    }
+    assert _chromium_host_resolver_rules(profile, "moli") is None
+
+
+def test_chromium_still_reads_legacy_host_resolver_profile() -> None:
+    profile = {"browser_host_resolver_rules": "MAP legacy.example 127.0.0.1"}
+    assert (
+        _chromium_host_resolver_rules(profile, "chromium")
+        == "MAP legacy.example 127.0.0.1"
+    )
+
+
 def test_formal_evidence_boundary_rejects_missing_or_changed_files(tmp_path: Path) -> None:
     import hashlib
 
diff --git a/tests/trajectory_replay/test_remote_network_finalization.py b/tests/trajectory_replay/test_remote_network_finalization.py
new file mode 100644
index 000000000..cbf8c1718
--- /dev/null
+++ b/tests/trajectory_replay/test_remote_network_finalization.py
@@ -0,0 +1,74 @@
+from pathlib import Path
+
+from harness.trajectory_replay.remote_execution import (
+    NETWORK_SETTLE_MS,
+    _finalize_network_recorder,
+    _replay_step_status,
+)
+
+
+def test_network_finalization_waits_for_background_response_before_stop() -> None:
+    events = []
+
+    class Page:
+        def wait_for_timeout(self, value):
+            events.append(("wait", value))
+
+    class Recorder:
+        def stop(self, context, path):
+            events.append(("stop", context, path))
+
+    class Service:
+        _page = Page()
+        _context = "context"
+
+        def _submit(self, function):
+            return function()
+
+    path = Path("network.har")
+    _finalize_network_recorder(Service(), Recorder(), path)
+    assert events == [("wait", NETWORK_SETTLE_MS), ("stop", "context", path)]
+    assert NETWORK_SETTLE_MS >= 2_000
+
+
+def test_network_finalization_accepts_task_specific_slow_response_window() -> None:
+    events = []
+
+    class Page:
+        def wait_for_timeout(self, value):
+            events.append(("wait", value))
+
+    class Recorder:
+        def stop(self, context, path):
+            events.append(("stop", context, path))
+
+    class Service:
+        _page = Page()
+        _context = "context"
+
+        def _submit(self, function):
+            return function()
+
+    path = Path("network.har")
+    _finalize_network_recorder(Service(), Recorder(), path, settle_ms=10_000)
+    assert events == [("wait", 10_000), ("stop", "context", path)]
+
+
+def test_remote_runner_fails_browsertool_embedded_snapshot_error() -> None:
+    status, error = _replay_step_status(
+        {"action": "snapshot"},
+        "success",
+        "[Snapshot error: Execution context was destroyed]",
+        dispatched=False,
+    )
+    assert status == "error"
+    assert error == "Execution context was destroyed"
+
+
+def test_remote_runner_keeps_successful_snapshot_and_dispatched_click_semantics() -> None:
+    assert _replay_step_status(
+        {"action": "snapshot"}, "success", "Page: ready", dispatched=False
+    ) == ("success", None)
+    assert _replay_step_status(
+        {"action": "click"}, "error", "navigation timeout", dispatched=True
+    ) == ("success", None)
diff --git a/tests/trajectory_replay/test_remote_runtime.py b/tests/trajectory_replay/test_remote_runtime.py
index 73424a8b9..827e8f655 100644
--- a/tests/trajectory_replay/test_remote_runtime.py
+++ b/tests/trajectory_replay/test_remote_runtime.py
@@ -1,4 +1,5 @@
 import json
+from urllib.parse import urlencode
 
 import pytest
 from webarena_verified.types.task import WebArenaSite
@@ -7,6 +8,7 @@
     file_sha256,
     load_remote_sites,
     rebind_remote_calls,
+    remote_benchmark,
     validate_remote_cohort_reset_receipt,
     validate_remote_reset_receipt,
 )
@@ -30,6 +32,175 @@ def _site_map(tmp_path):
     return path
 
 
+def test_six_site_remote_map_accepts_read_only_sites_and_rebinds_them(tmp_path):
+    site_map = _site_map(tmp_path)
+    payload = json.loads(site_map.read_text())
+    payload["sites"].update(
+        {
+            "map": "http://map.example/",
+            "wikipedia": "http://wiki.example/",
+        }
+    )
+    site_map.write_text(json.dumps(payload))
+    sites = load_remote_sites(site_map)
+    assert set(sites) == {
+        WebArenaSite.SHOPPING,
+        WebArenaSite.SHOPPING_ADMIN,
+        WebArenaSite.REDDIT,
+        WebArenaSite.GITLAB,
+        WebArenaSite.MAP,
+        WebArenaSite.WIKIPEDIA,
+    }
+    assert remote_benchmark(sites).config.get_environment(WebArenaSite.MAP).active_url == (
+        "http://map.example"
+    )
+    result = rebind_remote_calls(
+        [
+            {"action": "navigate", "url": "http://host.docker.internal:3000/search?q=boston"},
+            {"action": "navigate", "url": "http://host.docker.internal:8888/wiki/Boston"},
+        ],
+        sites,
+        {"host.docker.internal"},
+    )
+    assert [row["url"] for row in result] == [
+        "http://map.example/search?q=boston",
+        "http://wiki.example/wiki/Boston",
+    ]
+
+
+def test_remote_benchmark_matches_same_host_network_event_after_transport_change(tmp_path):
+    site_map = _site_map(tmp_path)
+    payload = json.loads(site_map.read_text())
+    payload["sites"]["shopping_admin"] = "https://admin.example/admin"
+    site_map.write_text(json.dumps(payload))
+    benchmark = remote_benchmark(load_remote_sites(site_map))
+    trace = tmp_path / "shipment.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "POST",
+                                "url": (
+                                    "http://admin.example/admin/admin/order_shipment/"
+                                    "save/order_id/299/"
+                                ),
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/x-www-form-urlencoded",
+                                    "text": urlencode(
+                                        {
+                                            "tracking[1][carrier_code]": "fedex",
+                                            "tracking[1][number]": "8974568499",
+                                        }
+                                    ),
+                                },
+                            },
+                            "response": {
+                                "status": 302,
+                                "redirectURL": "http://admin.example/admin/sales/order/",
+                                "headers": [],
+                                "content": {},
+                            },
+                        }
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = benchmark.evaluate_task(
+        task_id=496,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+    assert float(result.score) == 1.0, result.model_dump_json(indent=2)
+
+
+def test_remote_benchmark_matches_distinct_bodies_sent_to_one_endpoint(tmp_path):
+    benchmark = remote_benchmark(load_remote_sites(_site_map(tmp_path)))
+    task = benchmark.get_task(567)
+    expected = [item.expected for item in task.eval if item.evaluator == "NetworkEventEvaluator"]
+    trace = tmp_path / "members.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": item.http_method,
+                                "url": item.url.replace("__GITLAB__", "http://gitlab.example"),
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/json",
+                                    "text": json.dumps(dict(item.post_data)),
+                                },
+                            },
+                            "response": {
+                                "status": item.response_status,
+                                "headers": [],
+                                "content": {},
+                            },
+                        }
+                        for item in expected
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = benchmark.evaluate_task(
+        task_id=567,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+    assert float(result.score) == 1.0, result.model_dump_json(indent=2)
+
+
+def test_remote_benchmark_rejects_missing_required_body_on_repeated_endpoint(tmp_path):
+    benchmark = remote_benchmark(load_remote_sites(_site_map(tmp_path)))
+    task = benchmark.get_task(567)
+    expected = [item.expected for item in task.eval if item.evaluator == "NetworkEventEvaluator"]
+    trace = tmp_path / "members-missing-first.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": item.http_method,
+                                "url": item.url.replace("__GITLAB__", "http://gitlab.example"),
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/json",
+                                    "text": json.dumps(dict(item.post_data)),
+                                },
+                            },
+                            "response": {
+                                "status": item.response_status,
+                                "headers": [],
+                                "content": {},
+                            },
+                        }
+                        for item in expected[1:]
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = benchmark.evaluate_task(
+        task_id=567,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+    assert float(result.score) == 0.0, result.model_dump_json(indent=2)
+
+
 def test_rebind_remote_calls_preserves_path_query_and_non_site_urls(tmp_path):
     sites = load_remote_sites(_site_map(tmp_path))
     calls = [
@@ -65,6 +236,58 @@ def test_rebind_remote_calls_preserves_path_query_and_non_site_urls(tmp_path):
     assert calls[0]["url"].startswith("http://host.docker.internal")
 
 
+def test_rebinds_absolute_urls_from_a_previous_deployment(tmp_path):
+    sites = load_remote_sites(_site_map(tmp_path))
+    previous = {
+        WebArenaSite.SHOPPING_ADMIN: "http://old-slice.example/admin",
+    }
+    calls = [
+        {
+            "action": "navigate",
+            "url": "http://old-slice.example/admin/catalog/product/edit/id/110/",
+        },
+        {
+            "action": "navigate",
+            "url": "http://unrelated.example/admin/catalog/product/edit/id/110/",
+        },
+    ]
+
+    rebound = rebind_remote_calls(calls, sites, set(), previous)
+
+    assert rebound[0]["url"] == "http://admin.example/admin/catalog/product/edit/id/110/"
+    assert rebound[1] == calls[1]
+
+
+def test_rebind_keeps_optional_source_site_when_target_omits_it(tmp_path):
+    sites = load_remote_sites(_site_map(tmp_path))
+    previous = {WebArenaSite.WIKIPEDIA: "http://old-wikipedia.example"}
+    calls = [
+        {
+            "action": "navigate",
+            "url": "http://old-wikipedia.example/wiki/Browser_automation",
+        }
+    ]
+
+    assert rebind_remote_calls(calls, sites, set(), previous) == calls
+
+
+def test_rebinds_site_urls_inside_evaluate_request_bodies(tmp_path):
+    sites = load_remote_sites(_site_map(tmp_path))
+    previous = {WebArenaSite.REDDIT: "http://old-reddit.example"}
+    script = (
+        "async () => fetch('/api/submit', {body: "
+        "JSON.stringify({url:'http://old-reddit.example/f/space/123', "
+        "external:'https://unrelated.example/f/space/123'})})"
+    )
+    calls = [{"action": "evaluate", "script": script}]
+
+    rebound = rebind_remote_calls(calls, sites, set(), previous)
+
+    assert "http://reddit.example/f/space/123" in rebound[0]["script"]
+    assert "https://unrelated.example/f/space/123" in rebound[0]["script"]
+    assert calls[0]["script"] == script
+
+
 def test_rebind_url_form_input_without_rewriting_prose_or_external_links(tmp_path):
     calls = [
         {"action": "fill", "text": "http://192.168.5.2:9999/submission_images/a.jpg?x=1#image"},
@@ -78,6 +301,135 @@ def test_rebind_url_form_input_without_rewriting_prose_or_external_links(tmp_pat
     assert calls[0]["text"].startswith("http://192.168.5.2")
 
 
+def test_rebinds_recorded_url_inside_portable_playwright_target(tmp_path):
+    sites = load_remote_sites(_site_map(tmp_path))
+    source = "http://ec2-3-135-39-80.us-east-2.compute.amazonaws.com:7780"
+    calls = [
+        {
+            "action": "click",
+            "target": {
+                "by": "playwright",
+                "value": f'internal:attr=[title="{source}/admin/review/product/"i]',
+            },
+        }
+    ]
+    result = rebind_remote_calls(calls, sites, {"ec2-3-135-39-80.us-east-2.compute.amazonaws.com"})
+    assert "http://admin.example/admin/review/product/" in result[0]["target"]["value"]
+    assert calls[0]["target"]["value"].startswith('internal:attr=[title="http://ec2-')
+
+
+@pytest.mark.parametrize(
+    ("reddit_origin", "expected_score"),
+    [("http://reddit.example", 1.0), ("http://wrong.example", 0.0)],
+    ids=["markdown-deployment-url", "markdown-wrong-origin"],
+)
+def test_remote_benchmark_derenders_urls_inside_markdown(tmp_path, reddit_origin, expected_score):
+    benchmark = remote_benchmark(load_remote_sites(_site_map(tmp_path)))
+    task = benchmark.get_task(562)
+    project = task.eval[1].expected.post_data
+    expected_content = task.eval[2].expected.post_data["$.actions[0].content"]
+    content = expected_content.replace("__REDDIT__", reddit_origin)
+    commit = {
+        "branch": "main",
+        "actions": [{"action": "create", "file_path": "README.md", "content": content}],
+    }
+    trace = tmp_path / f"task-562-{expected_score}.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "POST",
+                                "url": "http://gitlab.example/projects",
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/x-www-form-urlencoded",
+                                    "text": urlencode(
+                                        {
+                                            key: value
+                                            for key, value in project.items()
+                                            if value is not None
+                                        }
+                                    ),
+                                },
+                            },
+                            "response": {"status": 302, "headers": [], "content": {}},
+                        },
+                        {
+                            "request": {
+                                "method": "POST",
+                                "url": (
+                                    "http://gitlab.example/api/v4/projects/"
+                                    "byteblaze%2FAwesome_DIY_ideas/repository/commits"
+                                ),
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/json",
+                                    "text": json.dumps(commit),
+                                },
+                            },
+                            "response": {"status": 201, "headers": [], "content": {}},
+                        },
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = benchmark.evaluate_task(
+        task_id=562,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+    assert float(result.score) == expected_score, result.model_dump_json(indent=2)
+
+
+@pytest.mark.parametrize(
+    ("gitlab_origin", "expected_score"),
+    [("http://gitlab.example", 1.0), ("http://wrong.example", 0.0)],
+    ids=["string-deployment-url", "string-wrong-origin"],
+)
+def test_remote_benchmark_derenders_url_valued_form_strings(
+    tmp_path, gitlab_origin, expected_score
+):
+    benchmark = remote_benchmark(load_remote_sites(_site_map(tmp_path)))
+    task = benchmark.get_task(684)
+    post_data = dict(task.eval[1].expected.post_data)
+    post_data["submission[url]"] = post_data["submission[url]"].replace("__GITLAB__", gitlab_origin)
+    trace = tmp_path / f"task-684-{expected_score}.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "POST",
+                                "url": "http://reddit.example/submit/LifeProTips",
+                                "headers": [],
+                                "postData": {
+                                    "mimeType": "application/x-www-form-urlencoded",
+                                    "text": urlencode(post_data),
+                                },
+                            },
+                            "response": {"status": 302, "headers": [], "content": {}},
+                        }
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = benchmark.evaluate_task(
+        task_id=684,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+    assert float(result.score) == expected_score, result.model_dump_json(indent=2)
+
+
 def test_remote_reset_requires_pod_replacement_and_readiness(tmp_path):
     site_path = _site_map(tmp_path)
     receipt_path = tmp_path / "reset.json"
@@ -254,10 +606,15 @@ def test_cohort_reset_requires_bound_fast_read_only_site_probes(tmp_path):
         "site_map_sha256": file_sha256(site_path),
         "reset_scope": "once_before_original36_candidate_cohort",
         "site_reset_between_tasks": False,
-        "sites": {site.value: dict(row) for site in (
-            WebArenaSite.SHOPPING, WebArenaSite.SHOPPING_ADMIN,
-            WebArenaSite.REDDIT, WebArenaSite.GITLAB,
-        )},
+        "sites": {
+            site.value: dict(row)
+            for site in (
+                WebArenaSite.SHOPPING,
+                WebArenaSite.SHOPPING_ADMIN,
+                WebArenaSite.REDDIT,
+                WebArenaSite.GITLAB,
+            )
+        },
         "site_readiness": [probe],
     }
     receipt["sites"]["reddit"]["baseline_check"] = {"row_count": 0}
@@ -267,24 +624,33 @@ def test_cohort_reset_requires_bound_fast_read_only_site_probes(tmp_path):
         "site_map_sha256": file_sha256(site_path),
         "app_id": 195,
         "reset_scope": "once_before_original36_candidate_cohort",
-        "required_readiness_probes": [{
-            "site": "reddit", "path": "/f/DIY/post", "url": url, "max_stable_ms": 5000,
-        }],
+        "required_readiness_probes": [
+            {
+                "site": "reddit",
+                "path": "/f/DIY/post",
+                "url": url,
+                "max_stable_ms": 5000,
+            }
+        ],
     }
     assert validate_remote_cohort_reset_receipt(receipt_path, **kwargs) == receipt
     recovered = json.loads(json.dumps(receipt))
-    recovered["site_readiness"][0].update({
-        "cold_http_status": 500,
-        "recovery_http_status": 200,
-        "recovery_ms": 15_000,
-    })
+    recovered["site_readiness"][0].update(
+        {
+            "cold_http_status": 500,
+            "recovery_http_status": 200,
+            "recovery_ms": 15_000,
+        }
+    )
     receipt_path.write_text(json.dumps(recovered))
     assert validate_remote_cohort_reset_receipt(receipt_path, **kwargs) == recovered
     timed_out = json.loads(json.dumps(recovered))
-    timed_out["site_readiness"][0].update({
-        "cold_http_status": 0,
-        "cold_error": "timeout",
-    })
+    timed_out["site_readiness"][0].update(
+        {
+            "cold_http_status": 0,
+            "cold_error": "timeout",
+        }
+    )
     receipt_path.write_text(json.dumps(timed_out))
     assert validate_remote_cohort_reset_receipt(receipt_path, **kwargs) == timed_out
     del timed_out["site_readiness"][0]["cold_error"]
diff --git a/tests/trajectory_replay/test_report_compaction.py b/tests/trajectory_replay/test_report_compaction.py
new file mode 100644
index 000000000..0d15bd9c1
--- /dev/null
+++ b/tests/trajectory_replay/test_report_compaction.py
@@ -0,0 +1,56 @@
+import json
+from pathlib import Path
+
+from harness.trajectory_replay.report_compaction import compact_report, validate_compact_report
+
+
+def test_compaction_preserves_logical_routes_under_file_limit(tmp_path: Path) -> None:
+    source = tmp_path / "source"
+    (source / "task").mkdir(parents=True)
+    (source / "result").mkdir()
+    (source / "data/trajectory").mkdir(parents=True)
+    (source / "data/evidence").mkdir()
+    (source / "assets").mkdir()
+    (source / "index.html").write_text('task')
+    (source / "task/1.html").write_text('result')
+    (source / "result/browser-1.html").write_text(
+        'task'
+        '下载净化 JSON'
+        ''
+    )
+    (source / "data/trajectory/1.json").write_text('{"task_id":1}')
+    (source / "data/trajectory/browser-1.json").write_text('{"task_id":1}')
+    (source / "data/evidence/browser-1.json").write_text("{}")
+    (source / "assets/app.js").write_text("window.ready=true")
+    (source / "report-manifest.json").write_text(
+        json.dumps({"task_count": 1, "logical_results": 1})
+    )
+
+    target = tmp_path / "compact"
+    summary = compact_report(source, target)
+
+    assert summary == {"files": 70, "task_routes": 1, "result_routes": 1}
+    assert 'href="task.html?id=1"' in (target / "index.html").read_text()
+    result_bundle = json.loads((target / "pages/result-01.json").read_text())
+    assert 'href="task.html?id=1"' in result_bundle["browser-1"]
+    assert 'href="evidence-raw.html?key=browser-1"' in result_bundle["browser-1"]
+    assert "查看净化 JSON" in result_bundle["browser-1"]
+    assert "data/'+route.kind+'-" in (target / "assets/app.js").read_text()
+    assert r"match(/(?:^|-)(\d+)$/)" in (target / "result.html").read_text()
+    assert r"match(/(?:^|-)(\d+)$/)" in (target / "assets/app.js").read_text()
+    assert r"data\/(trajectory|evidence)\/" in (target / "assets/app.js").read_text()
+    assert "trajectory|evidence" in (target / "assets/app.js").read_text()
+    app = (target / "assets/app.js").read_text()
+    assert "event.preventDefault()" in app
+    assert "location.href='evidence-raw.html?key='" in app
+    raw = (target / "evidence-raw.html").read_text()
+    assert "const value=(await response.json())[key]" in raw
+    assert "JSON.stringify(value,null,2)" in raw
+    assert not (target / "data/evidence/browser-1.json").exists()
+    evidence_bundle = json.loads((target / "data/evidence-01.json").read_text())
+    assert evidence_bundle == {"browser-1": {}}
+    assert validate_compact_report(target, max_files=80) == {
+        "files": 70,
+        "task_routes": 1,
+        "result_routes": 1,
+    }
diff --git a/tests/trajectory_replay/test_retrieval_other_candidates.py b/tests/trajectory_replay/test_retrieval_other_candidates.py
index f78e053e2..5f91d6ce2 100644
--- a/tests/trajectory_replay/test_retrieval_other_candidates.py
+++ b/tests/trajectory_replay/test_retrieval_other_candidates.py
@@ -51,7 +51,7 @@ def test_current_read_observation_binds_exact_canonical_prefix_and_action(task_i
         validate_contract(contract, changed)
 
 
-@pytest.mark.parametrize("task_id", [28, 124, 132, 168, 225, 279, 303, 386])
+@pytest.mark.parametrize("task_id", [28, 124, 132, 168, 225, 279, 280, 282, 303, 386])
 def test_candidate_has_no_frozen_answer_or_mutating_network_call(task_id: int):
     candidate = json.loads((ROOT / f"task-{task_id}.json").read_text())
     script = candidate["tool_calls"][-1]["script"]
@@ -65,7 +65,7 @@ def test_candidate_has_no_frozen_answer_or_mutating_network_call(task_id: int):
     assert candidate["retrieval_contract"]["projection"] != {"source": "reference", "pointer": "/answer"}
 
 
-@pytest.mark.parametrize("task_id", [124, 279, 386])
+@pytest.mark.parametrize("task_id", [124, 279, 280, 282, 386])
 def test_shopping_candidates_require_declared_total_and_linked_products(task_id: int):
     script = json.loads((ROOT / f"task-{task_id}.json").read_text())["tool_calls"][-1]["script"]
     assert "rows.length!==total" in script
@@ -75,6 +75,21 @@ def test_shopping_candidates_require_declared_total_and_linked_products(task_id:
     assert "seen.has(row.sku)" in script
 
 
+@pytest.mark.parametrize("task_id", [280, 282])
+def test_current_catalog_candidates_bind_both_actions(task_id: int):
+    candidate = json.loads((ROOT / f"task-{task_id}.json").read_text())
+    calls, contract = candidate["tool_calls"], candidate["retrieval_contract"]
+    assert [call["action"] for call in calls] == ["navigate", "evaluate"]
+    assert contract["canonical_calls_sha256"] == digest(calls)
+    assert contract["sources"][0]["action_sha256"] == digest(calls[-1])
+    validate_contract(contract, calls)
+
+    changed = copy.deepcopy(calls)
+    changed[-1]["script"] += "\n// changed"
+    with pytest.raises(ValueError, match="canonical actions"):
+        validate_contract(contract, changed)
+
+
 def test_task168_absence_requires_complete_authenticated_personal_project_set():
     candidate = json.loads((ROOT / "task-168.json").read_text())
     calls, contract = candidate["tool_calls"], candidate["retrieval_contract"]
diff --git a/tests/trajectory_replay/test_revised_webarena_dataset.py b/tests/trajectory_replay/test_revised_webarena_dataset.py
index 9f1a339ca..93cb0fcfe 100644
--- a/tests/trajectory_replay/test_revised_webarena_dataset.py
+++ b/tests/trajectory_replay/test_revised_webarena_dataset.py
@@ -17,7 +17,13 @@
 
 
 def test_revised_dataset_is_complete_bound_and_reproducible() -> None:
-    result = revise(BASE, FROZEN_CORRECTIONS, [67, 225, 243], SELECTION)
+    result = revise(
+        BASE,
+        FROZEN_CORRECTIONS,
+        [67, 225, 243],
+        SELECTION,
+        require_current_upstream=False,
+    )
     committed = json.loads(REVISED.read_text(encoding="utf-8"))
     original = json.loads(BASE.read_text(encoding="utf-8"))
     assert result == committed
@@ -55,7 +61,13 @@ def test_revised_dataset_is_complete_bound_and_reproducible() -> None:
 
 
 def test_unrelated_catalog_updates_preserve_frozen_task_contracts() -> None:
-    historical = revise(BASE, FROZEN_CORRECTIONS, [67, 225, 243], SELECTION)
+    historical = revise(
+        BASE,
+        FROZEN_CORRECTIONS,
+        [67, 225, 243],
+        SELECTION,
+        require_current_upstream=False,
+    )
     current = revise(BASE, CORRECTIONS, [67, 225, 243], SELECTION)
     assert current["revision"]["correction_catalog_sha256"] == hashlib.sha256(
         CORRECTIONS.read_bytes()
diff --git a/tests/trajectory_replay/test_run_webarena_catalog.py b/tests/trajectory_replay/test_run_webarena_catalog.py
new file mode 100644
index 000000000..b0f6cacf6
--- /dev/null
+++ b/tests/trajectory_replay/test_run_webarena_catalog.py
@@ -0,0 +1,1113 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay import catalog_runner
+from harness.trajectory_replay.catalog_binding import rebind_catalog
+from harness.trajectory_replay.catalog_contracts import (
+    TRUSTED_RECORDED_HOSTS,
+    apply_action_overrides,
+    catalog_action_overrides,
+    completed_task_ids,
+    file_sha256,
+    load_action_overrides,
+    load_catalog_inputs,
+    row_source_sites,
+)
+from harness.trajectory_replay.catalog_runner import (
+    _kitesurf_cdp_config,
+    _load_task_selection,
+    _recover_completed_attempts,
+    _request_bridge_options,
+    _run_remote_with_setup_recovery,
+    _run_with_setup_recovery,
+    _task_ids,
+    _task_origins,
+)
+from harness.trajectory_replay.remote_execution import (
+    _sanitize_remote_payload,
+    finalize_timed_out_task,
+    sanitize_remote_text,
+)
+from harness.trajectory_replay.remote_runtime import rebind_remote_calls
+from harness.trajectory_replay.webarena_replay import _json_sha256
+
+_REMOTE_RUNTIME = {
+    "protocolVersion": "1.3",
+    "product": "Chrome/145.0.0.0",
+    "revision": "@kitesurf",
+    "userAgent": "Kitesurf",
+    "jsVersion": "14.5",
+}
+_REMOTE_ENDPOINT = (
+    "wss://api.cloudflare.com/client/v4/accounts/private-account/"
+    "browser-run/devtools/browser?browser=kitesurf"
+)
+
+
+def test_catalog_runner_selects_profile_host_overrides_only_for_kitesurf() -> None:
+    origins = {"http://shop.test"}
+    overrides = [{"pattern": "*.dev.lexmount.net", "address": "10.2.62.142"}]
+    profile = {"host_overrides": overrides}
+
+    assert _request_bridge_options(profile, "kitesurf", origins) == {
+        "request_bridge_origins": origins,
+        "request_bridge_host_overrides": overrides,
+    }
+    assert _request_bridge_options(profile, "chromium", origins) == {
+        "request_bridge_origins": None,
+        "request_bridge_host_overrides": None,
+    }
+_REMOTE_HEADERS = {"Authorization": "Bearer private-token"}
+
+
+def test_task_ids_preserve_requested_official_order():
+    assert _task_ids([3, 7, 11, 20], 7, 20) == [7, 11, 20]
+    assert _task_ids([3, 7, 11, 20], None, None) == [3, 7, 11, 20]
+    assert _task_ids([3, 7, 11, 20], None, None, [7, 20]) == [7, 20]
+
+
+def test_kitesurf_cdp_config_uses_official_endpoint_without_persisting_token(
+    tmp_path: Path,
+) -> None:
+    credentials = tmp_path / ".env"
+    credentials.write_text(
+        "CLOUDFLARE_ACCOUNT_ID=account/id\nCLOUDFLARE_API_TOKEN=private-token\n",
+        encoding="utf-8",
+    )
+
+    endpoint, headers, binding = _kitesurf_cdp_config(credentials)
+
+    assert endpoint == (
+        "wss://api.cloudflare.com/client/v4/accounts/account%2Fid/"
+        "browser-run/devtools/browser?browser=kitesurf"
+    )
+    assert headers == {"Authorization": "Bearer private-token"}
+    serialized = json.dumps(binding)
+    assert binding["browser"] == "kitesurf"
+    assert binding["protocol"] == "cdp"
+    assert "private-token" not in serialized
+    assert "account/id" not in serialized
+
+
+def test_remote_errors_redact_endpoint_account_and_token() -> None:
+    sanitized = sanitize_remote_text(
+        f"connect {_REMOTE_ENDPOINT} with Bearer private-token failed",
+        _REMOTE_ENDPOINT,
+        _REMOTE_HEADERS,
+    )
+    assert "private-account" not in sanitized
+    assert "private-token" not in sanitized
+    assert sanitized.count("") == 1
+    evidence = _sanitize_remote_payload(
+        {
+            "trace": [
+                {
+                    "error": f"connect {_REMOTE_ENDPOINT}",
+                    "result": "Authorization: Bearer private-token",
+                }
+            ]
+        },
+        _REMOTE_ENDPOINT,
+        _REMOTE_HEADERS,
+    )
+    serialized = json.dumps(evidence)
+    assert "private-account" not in serialized
+    assert "private-token" not in serialized
+
+
+def test_external_timeout_commits_the_active_action_and_hashes(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-7"
+    task_dir.mkdir()
+    (task_dir / ".trace.partial.json").write_text(
+        json.dumps(
+            [
+                {
+                    "index": 1,
+                    "action": "navigate",
+                    "arguments": {"action": "navigate", "url": "http://example.test/start"},
+                    "status": "pass",
+                    "duration_ms": 10,
+                    "output": "ok",
+                },
+                {
+                    "index": 2,
+                    "action": "navigate",
+                    "arguments": {"action": "navigate", "url": "http://example.test"},
+                    "status": "running",
+                    "duration_ms": None,
+                    "output": "",
+                }
+            ]
+        ),
+        encoding="utf-8",
+    )
+    row = {
+        "task_id": 7,
+        "actions": [{"action": "navigate", "url": "http://example.test"}],
+        "omitted_diagnostic_observer_sha256": None,
+    }
+
+    result = finalize_timed_out_task(
+        task_dir, row, duration_ms=360_000, task_timeout_ms=300_000
+    )
+
+    trace = json.loads((task_dir / "trace.json").read_text(encoding="utf-8"))
+    assert trace[-1]["status"] == "fail"
+    assert "action 2 (navigate)" in trace[-1]["error"]
+    assert result["failure_stage"] == "browser_tool_execution"
+    assert result["completed_actions"] == 1
+    assert result["runtime_poisoned"] is True
+    assert not (task_dir / ".trace.partial.json").exists()
+    for kind, digest in result["evidence_sha256"].items():
+        filename = {
+            "trace": "trace.json",
+            "readiness": "readiness.json",
+            "outputs": "outputs.json",
+        }[kind]
+        assert file_sha256(task_dir / filename) == digest
+
+
+def test_remote_setup_recovery_never_repeats_after_evidence_exists(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-7"
+    calls = tmp_path / "calls"
+
+    def run_task():
+        with calls.open("a", encoding="utf-8") as target:
+            target.write("call\n")
+        task_dir.mkdir()
+        (task_dir / ".trace.partial.json").write_text(
+            "raw hosted trace with private credentials", encoding="utf-8"
+        )
+        raise RuntimeError("browser action failed")
+
+    result = _run_remote_with_setup_recovery(
+        task_id=7,
+        task_dir=task_dir,
+        row={"task_id": 7, "actions": []},
+        task_timeout_ms=1_000,
+        runtime_metadata=_REMOTE_RUNTIME,
+        remote_endpoint=_REMOTE_ENDPOINT,
+        remote_headers=_REMOTE_HEADERS,
+        run_task=run_task,
+    )
+
+    assert calls.read_text(encoding="utf-8").splitlines() == ["call"]
+    assert result["status"] == "candidate_setup_failure"
+    assert "browser action failed" in result["error"]
+    assert not (task_dir / ".trace.partial.json").exists()
+
+
+def test_remote_setup_recovery_retries_connection_before_evidence(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-8"
+    calls = tmp_path / "calls"
+
+    def run_task():
+        count = len(calls.read_text(encoding="utf-8").splitlines()) if calls.exists() else 0
+        with calls.open("a", encoding="utf-8") as target:
+            target.write("call\n")
+        if count == 0:
+            raise ConnectionError("provider handshake failed")
+        return {"task_id": 8, "status": "fail"}
+
+    result = _run_remote_with_setup_recovery(
+        task_id=8,
+        task_dir=task_dir,
+        row={"task_id": 8, "actions": []},
+        task_timeout_ms=1_000,
+        runtime_metadata=_REMOTE_RUNTIME,
+        remote_endpoint=_REMOTE_ENDPOINT,
+        remote_headers=_REMOTE_HEADERS,
+        run_task=run_task,
+    )
+
+    assert calls.read_text(encoding="utf-8").splitlines() == ["call", "call"]
+    assert result == {"task_id": 8, "status": "fail"}
+
+
+def test_remote_watchdog_commits_timeout_and_returns(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-9"
+    row = {
+        "task_id": 9,
+        "actions": [{"action": "evaluate", "script": "while(true){}"}],
+    }
+
+    def run_task():
+        task_dir.mkdir()
+        (task_dir / ".trace.partial.json").write_text(
+            json.dumps(
+                [
+                    {
+                        "index": 0,
+                        "action": "navigate",
+                        "arguments": {},
+                        "status": "success",
+                        "duration_ms": 1,
+                        "output": (
+                            f"connect {_REMOTE_ENDPOINT} with "
+                            "Authorization: Bearer private-token"
+                        ),
+                    },
+                    {
+                        "index": 1,
+                        "action": "evaluate",
+                        "arguments": row["actions"][0],
+                        "status": "running",
+                        "duration_ms": None,
+                        "output": "",
+                    }
+                ]
+            ),
+            encoding="utf-8",
+        )
+        time.sleep(10)
+
+    outcome = _run_remote_with_setup_recovery(
+        task_id=9,
+        task_dir=task_dir,
+        row=row,
+        task_timeout_ms=50,
+        runtime_metadata=_REMOTE_RUNTIME,
+        remote_endpoint=_REMOTE_ENDPOINT,
+        remote_headers=_REMOTE_HEADERS,
+        run_task=run_task,
+    )
+
+    assert outcome["status"] == "fail"
+    assert outcome["runtime_poisoned"] is True
+    assert outcome["runtime_metadata"] == _REMOTE_RUNTIME
+    assert outcome["error"].startswith("ExternalTaskTimeout:")
+    assert json.loads((task_dir / "result.json").read_text()) == outcome
+    trace_text = (task_dir / "trace.json").read_text(encoding="utf-8")
+    assert "private-account" not in trace_text
+    assert "private-token" not in trace_text
+
+
+def test_external_timeout_counts_source_success_actions(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-8"
+    task_dir.mkdir()
+    (task_dir / ".trace.partial.json").write_text(
+        json.dumps(
+            [
+                {
+                    "index": 1,
+                    "action": "navigate",
+                    "arguments": {},
+                    "status": "success",
+                    "duration_ms": 10,
+                    "output": "ok",
+                },
+                {
+                    "index": 2,
+                    "action": "evaluate",
+                    "arguments": {},
+                    "status": "running",
+                    "duration_ms": None,
+                    "output": "",
+                },
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = finalize_timed_out_task(
+        task_dir,
+        {"task_id": 8, "actions": [{}, {}]},
+        duration_ms=300_000,
+        task_timeout_ms=300_000,
+    )
+
+    assert result["completed_actions"] == 1
+
+
+@pytest.mark.parametrize("bounds", [(-1, 1), (2, 1), (0, 812)])
+def test_task_ids_reject_out_of_catalog_range(bounds):
+    with pytest.raises(ValueError, match="catalog"):
+        _task_ids([0, 1, 2], *bounds)
+
+
+@pytest.mark.parametrize("selected", [[2, 1], [1, 1], [1, 4]])
+def test_task_ids_reject_invalid_explicit_repairs(selected):
+    with pytest.raises(ValueError, match="task IDs"):
+        _task_ids([0, 1, 2], None, None, selected)
+
+
+def test_task_ids_reject_mixed_range_and_explicit_selection():
+    with pytest.raises(ValueError, match="cannot be combined"):
+        _task_ids([0, 1, 2], 0, 2, [1])
+
+
+def test_task_origins_excludes_unrelated_deployments() -> None:
+    from webarena_verified.types.task import WebArenaSite
+
+    task = type(
+        "Task",
+        (),
+        {"sites": (WebArenaSite.GITLAB, WebArenaSite.WIKIPEDIA)},
+    )()
+    benchmark = type(
+        "Benchmark",
+        (),
+        {"get_task": lambda self, task_id: task},
+    )()
+    sites = {
+        WebArenaSite.GITLAB: "https://gitlab.example.test/root/path",
+        WebArenaSite.WIKIPEDIA: "http://wikipedia.example.test",
+        WebArenaSite.MAP: "http://unavailable-map.example.test",
+    }
+
+    assert _task_origins(benchmark, 17, sites) == {
+        "https://gitlab.example.test",
+        "http://wikipedia.example.test",
+    }
+
+
+def test_task_selection_is_schema_bound(tmp_path: Path):
+    selection = tmp_path / "selection.json"
+    selection.write_text(
+        json.dumps({"schema": "lex_webarena_replay_selection/1", "task_ids": [1, 7]}),
+        encoding="utf-8",
+    )
+    assert _load_task_selection(selection) == [1, 7]
+
+    selection.write_text(json.dumps({"task_ids": [1, 7]}), encoding="utf-8")
+    with pytest.raises(ValueError, match="contract"):
+        _load_task_selection(selection)
+
+
+class _FakeBrowserProcess:
+    def __init__(self, returncode: int | None = None):
+        self.returncode = returncode
+        self.stopped = threading.Event()
+
+    def poll(self) -> int | None:
+        return self.returncode
+
+    def terminate(self) -> None:
+        self.returncode = -15
+        self.stopped.set()
+
+    def kill(self) -> None:
+        self.returncode = -9
+        self.stopped.set()
+
+    def wait(self, timeout: int | None = None) -> int:
+        assert self.stopped.wait(timeout)
+        assert self.returncode is not None
+        return self.returncode
+
+
+def test_task_watchdog_kills_browser_commits_timeout_and_returns(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-11"
+    process = _FakeBrowserProcess()
+    row = {
+        "task_id": 11,
+        "actions": [{"action": "evaluate", "script": "while(true){}"}],
+    }
+
+    def run_task() -> dict:
+        task_dir.mkdir()
+        (task_dir / ".trace.partial.json").write_text(
+            json.dumps(
+                [
+                    {
+                        "index": 0,
+                        "action": "evaluate",
+                        "arguments": row["actions"][0],
+                        "status": "running",
+                        "duration_ms": None,
+                        "output": "",
+                    }
+                ]
+            ),
+            encoding="utf-8",
+        )
+        assert process.stopped.wait(1)
+        return {"task_id": 11, "status": "fail", "error": "browser disconnected"}
+
+    outcome, returned_process = _run_with_setup_recovery(
+        task_id=11,
+        task_dir=task_dir,
+        process=process,
+        restart_browser=lambda _current: _FakeBrowserProcess(),
+        run_task=run_task,
+        row=row,
+        task_timeout_ms=20,
+    )
+
+    assert returned_process is process
+    assert outcome["runtime_poisoned"] is True
+    assert outcome["error"].startswith("ExternalTaskTimeout:")
+    assert json.loads((task_dir / "result.json").read_text()) == outcome
+    assert json.loads((task_dir / "trace.json").read_text())[-1]["status"] == "fail"
+
+
+def test_dead_browser_restarts_before_next_task_without_replaying_result(
+    tmp_path: Path,
+) -> None:
+    process = _FakeBrowserProcess()
+    calls: list[int] = []
+    restarts: list[_FakeBrowserProcess] = []
+
+    def restart(current: _FakeBrowserProcess) -> _FakeBrowserProcess:
+        restarts.append(current)
+        return _FakeBrowserProcess()
+
+    def first_task() -> dict:
+        calls.append(1)
+        process.returncode = 1
+        return {"task_id": 1, "status": "fail"}
+
+    first, process = _run_with_setup_recovery(
+        task_id=1,
+        task_dir=tmp_path / "task-1",
+        process=process,
+        restart_browser=restart,
+        run_task=first_task,
+    )
+    second, process = _run_with_setup_recovery(
+        task_id=2,
+        task_dir=tmp_path / "task-2",
+        process=process,
+        restart_browser=restart,
+        run_task=lambda: calls.append(2) or {"task_id": 2, "status": "pass"},
+    )
+
+    assert first["status"] == "fail"
+    assert second["status"] == "pass"
+    assert calls == [1, 2]
+    assert len(restarts) == 1
+    assert process.poll() is None
+
+
+def test_setup_disconnect_restarts_and_executes_candidate_once(tmp_path: Path) -> None:
+    process = _FakeBrowserProcess()
+    calls = 0
+    restarts = 0
+
+    def restart(current: _FakeBrowserProcess) -> _FakeBrowserProcess:
+        nonlocal restarts
+        restarts += 1
+        return _FakeBrowserProcess()
+
+    def run_task() -> dict:
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            process.returncode = 1
+            raise ConnectionError("browser closed before session setup")
+        return {"task_id": 7, "status": "fail"}
+
+    outcome, _ = _run_with_setup_recovery(
+        task_id=7,
+        task_dir=tmp_path / "task-7",
+        process=process,
+        restart_browser=restart,
+        run_task=run_task,
+    )
+
+    assert outcome == {"task_id": 7, "status": "fail"}
+    assert calls == 2
+    assert restarts == 1
+
+
+def test_existing_task_evidence_prevents_setup_retry(tmp_path: Path) -> None:
+    task_dir = tmp_path / "task-9"
+    process = _FakeBrowserProcess()
+    calls = 0
+    restarts = 0
+
+    def restart(current: _FakeBrowserProcess) -> _FakeBrowserProcess:
+        nonlocal restarts
+        restarts += 1
+        return _FakeBrowserProcess()
+
+    def run_task() -> dict:
+        nonlocal calls
+        calls += 1
+        task_dir.mkdir()
+        raise RuntimeError("unexpected failure after evidence began")
+
+    outcome, _ = _run_with_setup_recovery(
+        task_id=9,
+        task_dir=task_dir,
+        process=process,
+        restart_browser=restart,
+        run_task=run_task,
+    )
+
+    assert outcome["status"] == "candidate_setup_failure"
+    assert "unexpected failure" in outcome["error"]
+    assert calls == 1
+    assert restarts == 0
+
+
+def _write(path: Path, value: object) -> str:
+    path.write_text(json.dumps(value) + "\n", encoding="utf-8")
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _amendment(task_id: int, action_index: int, action: dict, updates: dict) -> dict:
+    return {
+        "task_id": task_id,
+        "action_index": action_index,
+        "precondition_action_sha256": _json_sha256(action),
+        "set": updates,
+    }
+
+
+def test_action_overrides_apply_to_portable_and_direct_source_actions() -> None:
+    evaluate = {"action": "evaluate", "script": "return true;"}
+    fill = {"action": "fill", "target": {"by": "css", "value": "input"}, "text": "x"}
+    portable = apply_action_overrides(
+        [fill],
+        [_amendment(7, 1, fill, {"wait_for_selector_hidden": ".loading-mask"})],
+        task_id=7,
+    )
+    direct = apply_action_overrides(
+        [evaluate],
+        [
+            _amendment(
+                8,
+                1,
+                evaluate,
+                {"script": "return false;", "script_mode": "function_body"},
+            )
+        ],
+        task_id=8,
+    )
+    assert portable[0]["wait_for_selector_hidden"] == ".loading-mask"
+    assert direct[0]["script_mode"] == "function_body"
+    assert direct[0]["script"] == "return false;"
+    assert "wait_for_selector_hidden" not in fill
+    assert "script_mode" not in evaluate
+
+
+def test_action_overrides_support_explicit_compiled_ref_readiness() -> None:
+    click = {
+        "action": "click",
+        "selector": "button:visible",
+        "replay_mode": "snapshot_ref_semantics",
+    }
+    fill = {
+        "action": "fill",
+        "selector": "input:visible",
+        "text": "value",
+        "replay_mode": "snapshot_ref_semantics",
+    }
+
+    actions = apply_action_overrides(
+        [click, fill],
+        [
+            _amendment(7, 1, click, {"wait_for_enabled": True}),
+            _amendment(7, 2, fill, {"selector": "input", "force": True}),
+        ],
+        task_id=7,
+    )
+
+    assert actions[0]["wait_for_enabled"] is True
+    assert actions[1]["selector"] == "input"
+    assert actions[1]["force"] is True
+
+
+def test_action_overrides_support_non_geometric_selector_wait_state() -> None:
+    wait = {
+        "action": "wait",
+        "selector": "div.message-success:visible",
+        "timeout": 30_000,
+    }
+
+    actions = apply_action_overrides(
+        [wait],
+        [
+            _amendment(
+                464,
+                1,
+                wait,
+                {"selector": "div.message-success", "wait_state": "attached"},
+            )
+        ],
+        task_id=464,
+    )
+
+    assert actions[0]["selector"] == "div.message-success"
+    assert actions[0]["wait_state"] == "attached"
+
+
+def test_action_overrides_reject_wrong_hash_unknown_field_and_bad_index() -> None:
+    action = {"action": "evaluate", "script": "return true;"}
+    wrong_hash = _amendment(9, 1, action, {"script_mode": "function_body"})
+    wrong_hash["precondition_action_sha256"] = "0" * 64
+    with pytest.raises(ValueError, match="precondition"):
+        apply_action_overrides([action], [wrong_hash], task_id=9)
+    with pytest.raises(ValueError, match="invalid"):
+        apply_action_overrides(
+            [action], [_amendment(9, 1, action, {"task_specific_delay": 2})], task_id=9
+        )
+    with pytest.raises(ValueError, match="outside"):
+        apply_action_overrides(
+            [action], [_amendment(9, 2, action, {"script_mode": "function_body"})], task_id=9
+        )
+    fill = {"action": "fill", "text": "x"}
+    with pytest.raises(ValueError, match="script fields require an evaluate action"):
+        apply_action_overrides(
+            [fill], [_amendment(9, 1, fill, {"script": "return true;"})], task_id=9
+        )
+
+
+@pytest.mark.parametrize("script", ["", "   ", "x" * (64 * 1024 + 1)])
+def test_action_overrides_reject_invalid_replacement_scripts(script: str) -> None:
+    action = {"action": "evaluate", "script": "return true;"}
+    with pytest.raises(ValueError, match="script is invalid"):
+        apply_action_overrides(
+            [action], [_amendment(9, 1, action, {"script": script})], task_id=9
+        )
+
+
+@pytest.mark.parametrize(
+    ("catalog", "row_source_sha256"),
+    [
+        ({}, None),
+        ({"action_overrides_sha256": None}, None),
+        ({"action_overrides_sha256": "invalid"}, "invalid"),
+        ({"action_overrides_sha256": "A" * 64}, "A" * 64),
+    ],
+)
+def test_row_action_overrides_require_valid_catalog_source_hash(
+    catalog: dict, row_source_sha256: object
+) -> None:
+    action = {"action": "evaluate", "script": "return true;"}
+    amendments = [_amendment(9, 1, action, {"script_mode": "function_body"})]
+    row = {
+        "task_id": 9,
+        "action_overrides": {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "source_sha256": row_source_sha256,
+            "amendments_sha256": _json_sha256(amendments),
+            "amendments": amendments,
+        },
+    }
+    with pytest.raises(ValueError, match="provenance is invalid"):
+        catalog_action_overrides(catalog, row)
+
+
+def test_row_action_overrides_reject_empty_amendment_binding() -> None:
+    source_sha256 = "a" * 64
+    row = {
+        "task_id": 9,
+        "action_overrides": {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "source_sha256": source_sha256,
+            "amendments_sha256": _json_sha256([]),
+            "amendments": [],
+        },
+    }
+    with pytest.raises(ValueError, match="provenance changed"):
+        catalog_action_overrides({"action_overrides_sha256": source_sha256}, row)
+
+
+def test_external_action_overrides_reject_empty_amendment_binding(tmp_path: Path) -> None:
+    path = tmp_path / "empty-overrides.json"
+    _write(
+        path,
+        {"schema": "lex_webarena_replay_action_overrides/1", "amendments": []},
+    )
+    with pytest.raises(ValueError, match="at least one amendment"):
+        load_action_overrides(path)
+
+
+def test_catalog_requires_one_hash_bound_row_in_declared_order(tmp_path: Path) -> None:
+    sites_path = tmp_path / "sites.json"
+    sites_hash = _write(
+        sites_path,
+        {
+            "schema": "lex_webarena_remote_sites/1",
+            "sites": {
+                "shopping": "https://shopping.example.test",
+                "shopping_admin": "https://admin.example.test/admin",
+                "reddit": "https://reddit.example.test",
+                "gitlab": "https://gitlab.example.test",
+            },
+        },
+    )
+    action = {"action": "navigate", "url": "https://example.test", "save_as": "page"}
+    suite_path = tmp_path / "portable.json"
+    suite_hash = _write(
+        suite_path,
+        {
+            "schema": "lex_browser_use_tasks/4",
+            "tasks": [
+                {
+                    "id": "webarena-0",
+                    "title": "Open page",
+                    "difficulty": "simple",
+                    "tags": ["webarena"],
+                    "trajectory": {
+                        "schema": "lex_browser_trajectory/1",
+                        "actions": [action],
+                    },
+                    "assertions": [],
+                    "evaluator": {
+                        "kind": "webarena_verified",
+                        "task_id": 0,
+                        "revision": 2,
+                    },
+                    "environment": {"kind": "webarena_verified"},
+                    "ground_truth": {
+                        "oracle": "webarena_verified",
+                        "expected_actions": ["navigate"],
+                    },
+                    "source": {"url": "https://example.test"},
+                }
+            ],
+        },
+    )
+    rows = [
+        {
+            "task_id": task_id,
+            "mode": "portable" if task_id == 0 else "direct",
+            "bound_calls_sha256": _json_sha256([action]),
+            **({} if task_id == 0 else {"source_actions": [action]}),
+        }
+        for task_id in range(3)
+    ]
+    catalog_path = tmp_path / "catalog.json"
+    catalog = {
+        "schema": "lex_webarena_full_replay_catalog/1",
+        "task_ids": list(range(3)),
+        "site_map_sha256": sites_hash,
+        "portable_suite_sha256": suite_hash,
+        "rows": rows,
+    }
+    _write(catalog_path, catalog)
+    loaded, _sites, tasks = load_catalog_inputs(catalog_path, suite_path, sites_path)
+    assert len(loaded["rows"]) == 3
+    assert set(tasks) == {0}
+
+    catalog["rows"][2]["recorded_response"] = {"status": "SUCCESS"}
+    _write(catalog_path, catalog)
+    with pytest.raises(ValueError, match="recorded answers"):
+        load_catalog_inputs(catalog_path, suite_path, sites_path)
+
+
+def test_catalog_binding_embeds_override_provenance_and_hashes_amended_actions(
+    tmp_path: Path,
+) -> None:
+    sites_path = tmp_path / "sites.json"
+    sites_hash = _write(
+        sites_path,
+        {
+            "schema": "lex_webarena_remote_sites/1",
+            "sites": {
+                "shopping": "https://shopping.example.test",
+                "shopping_admin": "https://admin.example.test/admin",
+                "reddit": "https://reddit.example.test",
+                "gitlab": "https://gitlab.example.test",
+            },
+        },
+    )
+    portable_action = {"action": "fill", "target": {"by": "css", "value": "input"}, "text": "x"}
+    direct_action = {"action": "evaluate", "script": "return true;"}
+    suite_path = tmp_path / "suite.json"
+    suite_hash = _write(
+        suite_path,
+        {
+            "schema": "lex_browser_use_tasks/4",
+            "tasks": [
+                {
+                    "id": "webarena-0",
+                    "title": "Portable",
+                    "difficulty": "simple",
+                    "tags": ["webarena"],
+                    "trajectory": {
+                        "schema": "lex_browser_trajectory/1",
+                        "actions": [portable_action],
+                    },
+                    "assertions": [],
+                    "evaluator": {"kind": "webarena_verified", "task_id": 0, "revision": 2},
+                    "environment": {"kind": "webarena_verified"},
+                    "ground_truth": {
+                        "oracle": "webarena_verified",
+                        "expected_actions": ["fill"],
+                    },
+                    "source": {"url": "https://example.test"},
+                }
+            ],
+        },
+    )
+    catalog_path = tmp_path / "catalog.json"
+    _write(
+        catalog_path,
+        {
+            "schema": "lex_webarena_replay_catalog/1",
+            "task_ids": [0, 1],
+            "site_map_sha256": sites_hash,
+            "portable_suite_sha256": suite_hash,
+            "rows": [
+                {
+                    "task_id": 0,
+                    "mode": "portable",
+                    "bound_calls_sha256": _json_sha256([portable_action]),
+                },
+                {
+                    "task_id": 1,
+                    "mode": "direct",
+                    "source_actions": [direct_action],
+                    "bound_calls_sha256": _json_sha256([direct_action]),
+                },
+            ],
+        },
+    )
+    overrides_path = tmp_path / "overrides.json"
+    _write(
+        overrides_path,
+        {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "amendments": [
+                _amendment(0, 1, portable_action, {"wait_for_selector_hidden": ".loading-mask"}),
+                _amendment(1, 1, direct_action, {"script_mode": "function_body"}),
+            ],
+        },
+    )
+
+    rebound = rebind_catalog(
+        catalog_path, suite_path, sites_path, action_overrides_path=overrides_path
+    )
+    rows = {row["task_id"]: row for row in rebound["rows"]}
+    assert rebound["action_overrides_sha256"] == file_sha256(overrides_path)
+    assert rows[0]["bound_calls_sha256"] == _json_sha256(
+        [{**portable_action, "wait_for_selector_hidden": ".loading-mask"}]
+    )
+    assert rows[1]["bound_calls_sha256"] == _json_sha256(
+        [{**direct_action, "script_mode": "function_body"}]
+    )
+    assert rows[0]["action_overrides"]["source_sha256"] == file_sha256(overrides_path)
+
+    rebound_path = tmp_path / "rebound.json"
+    _write(rebound_path, rebound)
+    loaded, _, _ = load_catalog_inputs(
+        rebound_path, suite_path, sites_path, action_overrides_path=overrides_path
+    )
+    assert loaded["rows"] == rebound["rows"]
+
+    for top_hash in ("missing", None, "invalid"):
+        changed = json.loads(json.dumps(rebound))
+        if top_hash == "missing":
+            changed.pop("action_overrides_sha256")
+        else:
+            changed["action_overrides_sha256"] = top_hash
+        _write(rebound_path, changed)
+        with pytest.raises(ValueError, match="external action overrides source"):
+            load_catalog_inputs(rebound_path, suite_path, sites_path)
+        with pytest.raises(ValueError, match="not bound to the action overrides"):
+            load_catalog_inputs(
+                rebound_path,
+                suite_path,
+                sites_path,
+                action_overrides_path=overrides_path,
+            )
+
+    _write(rebound_path, rebound)
+    with pytest.raises(ValueError, match="rebind requires its external action overrides"):
+        rebind_catalog(rebound_path, suite_path, sites_path)
+
+    revised_overrides_path = tmp_path / "revised-overrides.json"
+    _write(
+        revised_overrides_path,
+        {
+            "schema": "lex_webarena_replay_action_overrides/1",
+            "amendments": [
+                _amendment(0, 1, portable_action, {"wait_for_selector_hidden": ".loading-mask"}),
+                _amendment(1, 1, direct_action, {"script": "return false;"}),
+            ],
+        },
+    )
+    with pytest.raises(ValueError, match="differ from the predecessor"):
+        rebind_catalog(
+            rebound_path,
+            suite_path,
+            sites_path,
+            action_overrides_path=revised_overrides_path,
+        )
+    revised = rebind_catalog(
+        rebound_path,
+        suite_path,
+        sites_path,
+        action_overrides_path=revised_overrides_path,
+        revise_action_overrides=True,
+    )
+    assert revised["predecessor_catalog_sha256"] == file_sha256(rebound_path)
+    assert revised["action_overrides_sha256"] == file_sha256(revised_overrides_path)
+    assert revised["repairs"][-1] == {
+        "kind": "action_override_revision",
+        "predecessor_action_overrides_sha256": file_sha256(overrides_path),
+        "action_overrides_sha256": file_sha256(revised_overrides_path),
+        "task_ids": [0, 1],
+    }
+    revised_path = tmp_path / "revised-catalog.json"
+    _write(revised_path, revised)
+    load_catalog_inputs(
+        revised_path,
+        suite_path,
+        sites_path,
+        action_overrides_path=revised_overrides_path,
+    )
+
+    wrong_overrides_path = tmp_path / "wrong-overrides.json"
+    wrong_overrides = json.loads(overrides_path.read_text(encoding="utf-8"))
+    wrong_overrides["amendments"].reverse()
+    _write(wrong_overrides_path, wrong_overrides)
+    with pytest.raises(ValueError, match="not bound to the action overrides"):
+        load_catalog_inputs(
+            rebound_path,
+            suite_path,
+            sites_path,
+            action_overrides_path=wrong_overrides_path,
+        )
+
+
+@pytest.mark.parametrize("top_hash", ["missing", None])
+def test_runner_rejects_row_overrides_without_external_version_binding(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, top_hash: object
+) -> None:
+    catalog = {"rows": [{"task_id": 9, "action_overrides": {"amendments": []}}]}
+    if top_hash != "missing":
+        catalog["action_overrides_sha256"] = top_hash
+    monkeypatch.setattr(catalog_runner, "verify_webarena_runtime_source", lambda: None)
+    monkeypatch.setattr(
+        catalog_runner,
+        "load_catalog_inputs",
+        lambda *args, **kwargs: (catalog, {}, {}),
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "run",
+            "--catalog",
+            str(tmp_path / "catalog.json"),
+            "--portable-suite",
+            str(tmp_path / "suite.json"),
+            "--site-map",
+            str(tmp_path / "sites.json"),
+            "--site-reset-receipt",
+            str(tmp_path / "receipt.json"),
+            "--environment-profile",
+            str(tmp_path / "environment.json"),
+            "--binary",
+            str(tmp_path / "browser"),
+            "--backend",
+            "playwright",
+            "--output",
+            str(tmp_path / "output"),
+        ],
+    )
+    with pytest.raises(ValueError, match="requires its versioned action overrides"):
+        catalog_runner.main()
+
+
+def test_rows_can_rebind_accepted_actions_from_distinct_source_deployments() -> None:
+    from webarena_verified.types.task import WebArenaSite
+
+    target = {WebArenaSite.SHOPPING: "https://new-shop.example.test"}
+    actions = [{"action": "navigate", "url": "https://old-shop-a.example.test/product"}]
+    row = {"source_site_urls": {"shopping": "https://old-shop-a.example.test"}}
+    rebound = rebind_remote_calls(
+        actions, target, set(TRUSTED_RECORDED_HOSTS), row_source_sites({}, row)
+    )
+    assert rebound[0]["url"] == "https://new-shop.example.test/product"
+
+    actions_b = [{"action": "navigate", "url": "https://old-shop-b.example.test/product"}]
+    row_b = {"source_site_urls": {"shopping": "https://old-shop-b.example.test"}}
+    rebound_b = rebind_remote_calls(
+        actions_b, target, set(TRUSTED_RECORDED_HOSTS), row_source_sites({}, row_b)
+    )
+    assert rebound_b == rebound
+
+
+def test_remote_task_passes_host_overrides_to_playwright_tool(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    from harness.trajectory_replay import playwright_tool, remote_execution
+
+    observed = {}
+
+    class StopConstruction(Exception):
+        pass
+
+    class Tool:
+        def __init__(self, endpoint, **kwargs):
+            observed.update({"endpoint": endpoint, **kwargs})
+            raise StopConstruction
+
+    class Benchmark:
+        def get_task(self, task_id):
+            assert task_id == 281
+            return object()
+
+    class Catalog:
+        records = {}
+
+    monkeypatch.setattr(playwright_tool, "PlaywrightReplayTool", Tool)
+    overrides = [{"pattern": "*.dev.lexmount.net", "address": "10.2.62.142"}]
+    with pytest.raises(StopConstruction):
+        remote_execution._run_task(
+            task_id=281,
+            row={},
+            benchmark=Benchmark(),
+            endpoint=_REMOTE_ENDPOINT,
+            output=tmp_path,
+            run_id="host-override",
+            origins=set(),
+            sites={},
+            catalog=Catalog(),
+            backend="playwright",
+            engine="kitesurf",
+            cdp_headers={"Authorization": "Bearer secret"},
+            request_bridge_origins={"http://shop.test"},
+            request_bridge_host_overrides=overrides,
+        )
+
+    assert observed == {
+        "endpoint": _REMOTE_ENDPOINT,
+        "engine": "kitesurf",
+        "headers": {"Authorization": "Bearer secret"},
+        "request_bridge_origins": {"http://shop.test"},
+        "request_bridge_host_overrides": overrides,
+    }
+
+
+def test_progress_must_be_a_strict_execution_prefix(tmp_path: Path) -> None:
+    progress = tmp_path / "progress.jsonl"
+    progress.write_text('{"task_id":0}\n{"task_id":2}\n', encoding="utf-8")
+    with pytest.raises(ValueError, match="strict prefix"):
+        completed_task_ids(progress, [0, 1, 2])
+
+
+def test_resume_commits_finished_orphan_without_reexecuting(tmp_path: Path) -> None:
+    progress = tmp_path / "progress.jsonl"
+    task = tmp_path / "task-0"
+    task.mkdir()
+    trace = task / "trace.json"
+    trace.write_text("[]\n", encoding="utf-8")
+    row = {
+        "task_id": 0,
+        "status": "fail",
+        "action_count": 1,
+        "evidence_sha256": {"trace": hashlib.sha256(trace.read_bytes()).hexdigest()},
+    }
+    (task / "result.json").write_text(json.dumps(row), encoding="utf-8")
+
+    assert _recover_completed_attempts(tmp_path, progress, [0, 1]) == {0}
+    assert json.loads(progress.read_text(encoding="utf-8"))["task_id"] == 0
+
+
+def test_resume_rejects_partial_orphan_instead_of_rerunning(tmp_path: Path) -> None:
+    (tmp_path / "task-0").mkdir()
+    with pytest.raises(ValueError, match="interrupted evidence"):
+        _recover_completed_attempts(tmp_path, tmp_path / "progress.jsonl", [0])
diff --git a/tests/trajectory_replay/test_webarena_corrections.py b/tests/trajectory_replay/test_webarena_corrections.py
index 6242c6e83..b519b9126 100644
--- a/tests/trajectory_replay/test_webarena_corrections.py
+++ b/tests/trajectory_replay/test_webarena_corrections.py
@@ -4,13 +4,18 @@
 import hashlib
 import json
 from pathlib import Path
+from urllib.parse import urlencode
 
 import pytest
+from webarena_verified.types.task import WebArenaSite
 
 from harness.browser_use.runner import _configure_benchmark
+from harness.trajectory_replay.remote_runtime import remote_benchmark
 from harness.trajectory_replay.webarena_corrections import (
+    DEFAULT_CORRECTIONS_PATH,
     CorrectedReplayContract,
     apply_correction,
+    check_replay_assertions,
     corrected_task_ids,
     evaluate_corrected_contract,
     load_corrections,
@@ -28,19 +33,320 @@ def catalog():
     return load_corrections()
 
 
+@pytest.fixture(scope="module")
+def route_benchmark():
+    return remote_benchmark(
+        {
+            WebArenaSite.SHOPPING: "http://shop.test",
+            WebArenaSite.SHOPPING_ADMIN: "http://admin.test",
+            WebArenaSite.REDDIT: "http://reddit.test",
+            WebArenaSite.GITLAB: "http://git.test",
+            WebArenaSite.MAP: "http://map.test",
+            WebArenaSite.WIKIPEDIA: "http://wiki.test",
+        }
+    )
+
+
 def _dump_eval(contract: CorrectedReplayContract, index: int) -> dict:
     return contract.task.eval[index].model_dump(mode="json")
 
 
+def _route_har(path: Path, url: str) -> Path:
+    engine = "bicycle" if "routed-bike" in url else "car"
+    path.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "version": "1.2",
+                    "creator": {"name": "route-order-control", "version": "1"},
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "GET",
+                                "url": "http://map.test/directions",
+                                "headers": [
+                                    {"name": "accept", "value": "text/html"},
+                                    {"name": "sec-fetch-dest", "value": "document"},
+                                    {"name": "sec-fetch-mode", "value": "navigate"},
+                                    {"name": "sec-fetch-user", "value": "?1"},
+                                ],
+                            },
+                            "response": {
+                                "status": 200,
+                                "headers": [],
+                                "cookies": [],
+                                "content": {"mimeType": "text/html", "text": ""},
+                            },
+                        },
+                        {
+                            "request": {
+                                "method": "GET",
+                                "url": url,
+                                "headers": [
+                                    {
+                                        "name": "Cookie",
+                                        "value": f"_osm_directions_engine=fossgis_osrm_{engine}",
+                                    }
+                                ],
+                            },
+                            "response": {
+                                "status": 200,
+                                "headers": [],
+                                "cookies": [],
+                                "content": {"mimeType": "application/json", "text": "{}"},
+                            },
+                        }
+                    ],
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def _empty_har(path: Path) -> Path:
+    return _route_har(path, "http://map.test/unrelated")
+
+
+@pytest.mark.parametrize(
+    "task_id",
+    [9, 18, 32, 34, 35, 61, 90, 252, 363, 367, 383],
+)
+def test_live_map_reference_corrections_accept_bound_observation_and_reject_neighbor(
+    tmp_path, route_benchmark, catalog, task_id
+) -> None:
+    contract = apply_correction(route_benchmark.get_task(task_id), catalog)
+    record = catalog.records[task_id]
+    receipt_path = Path(__file__).resolve().parents[2] / record["reference_receipt"]["path"]
+    receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+    assert record["regression_ids"] == [
+        f"tr-{task_id}-accept-live-map-reference",
+        f"tr-{task_id}-reject-nearby-wrong-map-reference",
+    ]
+    response = receipt["observed_response"]
+    positive_har = _empty_har(tmp_path / "positive.har")
+    positive = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=response,
+        network_trace=positive_har,
+    )
+    assert float(positive.score) == 1.0
+
+    negative_har = _empty_har(tmp_path / "negative.har")
+    negative_response = copy.deepcopy(response)
+    negative_response["retrieved_data"] = ["definitely wrong"]
+    negative = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=negative_response,
+        network_trace=negative_har,
+    )
+    assert float(negative.score) == 0.0
+
+
+def test_live_gitlab_issue_state_accepts_observed_boolean_and_rejects_opposite(
+    tmp_path, route_benchmark, catalog
+) -> None:
+    contract = apply_correction(route_benchmark.get_task(179), catalog)
+    record = catalog.records[179]
+    receipt_path = Path(__file__).resolve().parents[2] / record["reference_receipt"]["path"]
+    receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+    assert record["regression_ids"] == [
+        "tr-179-accept-live-authored-issue-state",
+        "tr-179-reject-opposite-issue-state",
+    ]
+    assert receipt["selected_issue"]["author_id"] == receipt["query"]["authenticated_user_id"]
+    assert receipt["selected_issue"]["state"] == "opened"
+
+    positive = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=receipt["observed_response"],
+        network_trace=_empty_har(tmp_path / "positive.har"),
+    )
+    negative_response = copy.deepcopy(receipt["observed_response"])
+    negative_response["retrieved_data"] = [True]
+    negative = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=negative_response,
+        network_trace=_empty_har(tmp_path / "negative.har"),
+    )
+    assert float(positive.score) == 1.0
+    assert float(negative.score) == 0.0
+
+
+@pytest.mark.parametrize("task_id", [682, 683])
+def test_live_gitlab_repository_post_accepts_observed_project_and_rejects_frozen_value(
+    tmp_path, route_benchmark, catalog, task_id
+) -> None:
+    contract = apply_correction(route_benchmark.get_task(task_id), catalog)
+    record = catalog.records[task_id]
+    receipt_path = Path(__file__).resolve().parents[2] / record["reference_receipt"]["path"]
+    receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
+    assert record["regression_ids"] == [
+        f"tr-{task_id}-accept-live-repository-post",
+        f"tr-{task_id}-reject-frozen-repository-post",
+    ]
+    assert receipt["query"]["result_count"] == 1
+    assert sum(receipt["commit_page_sizes"]) == receipt["commit_count"]
+
+    corrected_post = _dump_eval(contract, 1)["expected"]["post_data"]
+    positive_fields = {
+        "submission[mediaType]": "url",
+        "submission[url]": corrected_post["submission[url]"].replace(
+            "__GITLAB__", "http://git.test"
+        ),
+        "submission[title]": corrected_post["submission[title]"],
+        "submission[body]": corrected_post["submission[body]"],
+        "submission[forum]": "10043",
+    }
+
+    def score(fields: dict[str, str], name: str) -> float:
+        har = tmp_path / f"{name}.har"
+        har.write_text(
+            json.dumps(
+                {
+                    "log": {
+                        "version": "1.2",
+                        "creator": {"name": "gitlab-repository-control", "version": "1"},
+                        "entries": [
+                            {
+                                "request": {
+                                    "method": "POST",
+                                    "url": "http://reddit.test/submit/",
+                                    "headers": [],
+                                    "postData": {
+                                        "mimeType": "application/x-www-form-urlencoded",
+                                        "text": urlencode(fields),
+                                    },
+                                },
+                                "response": {
+                                    "status": 302,
+                                    "headers": [],
+                                    "cookies": [],
+                                    "content": {"mimeType": "text/html", "text": ""},
+                                },
+                            }
+                        ],
+                    }
+                }
+            ),
+            encoding="utf-8",
+        )
+        response = {"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None}
+        return float(
+            evaluate_corrected_contract(
+                route_benchmark,
+                contract,
+                agent_response=response,
+                network_trace=har,
+            ).score
+        )
+
+    assert score(positive_fields, "positive") == 1.0
+    negative_fields = dict(positive_fields)
+    first_change = record["preconditions"][0]
+    field = first_change["path"].rsplit("/", 1)[-1]
+    negative_fields[field] = first_change["equals"].replace(
+        "__GITLAB__", "http://git.test"
+    )
+    assert score(negative_fields, "negative") == 0.0
+
+
+def test_reddit_forum_slug_correction_accepts_deployed_route_and_rejects_label_url(
+    tmp_path, route_benchmark, catalog
+) -> None:
+    contract = apply_correction(route_benchmark.get_task(599), catalog)
+    response = {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": None}
+
+    def write_har(path: Path, forum_path: str) -> Path:
+        post = f"http://reddit.test/f/{forum_path}/120613/d-yan-lecun-s-recent-recommendations"
+        payload = {
+            "log": {
+                "entries": [
+                    {
+                        "request": {
+                            "method": "POST",
+                            "url": f"http://reddit.test/f/{forum_path}/subscribe.json",
+                            "headers": [{"name": "referer", "value": post}],
+                        },
+                        "response": {
+                            "status": 200,
+                            "headers": [],
+                            "content": {
+                                "mimeType": "application/json",
+                                "text": json.dumps({"subscribed": True}),
+                            },
+                        },
+                    }
+                ]
+            }
+        }
+        path.write_text(json.dumps(payload), encoding="utf-8")
+        return path
+
+    positive = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=response,
+        network_trace=write_har(tmp_path / "positive.har", "MachineLearning"),
+    )
+    negative = evaluate_corrected_contract(
+        route_benchmark,
+        contract,
+        agent_response=response,
+        network_trace=write_har(tmp_path / "negative.har", "machine%20learning"),
+    )
+    assert float(positive.score) == 1.0
+    assert float(negative.score) == 0.0
+
+
 def test_catalog_is_replay_only_and_has_exact_reviewed_scope(catalog) -> None:
     payload = json.loads(catalog.source_path.read_text(encoding="utf-8"))
     assert payload["historical_browser_use_scores_immutable"] is True
     assert "Replay" in payload["scope"]
-    assert catalog.upstream["package_version"] == "1.2.3"
+    assert catalog.upstream["repository"] == "https://github.com/lexmount/webarena-verified.git"
+    assert catalog.upstream["branch"] == "lex-main"
+    assert len(catalog.upstream["revision"]) == 40
     assert len(catalog.upstream["dataset_sha256"]) == 64
     assert set(catalog.record_sha256s) == set(catalog.records)
     assert len(set(catalog.record_sha256s.values())) == len(catalog.records)
-    assert corrected_task_ids(catalog) == (225, 279, 448, 510, 528, 547, 699)
+    assert corrected_task_ids(catalog) == (
+        9,
+        18,
+        32,
+        34,
+        35,
+        61,
+        90,
+        179,
+        225,
+        252,
+        279,
+        280,
+        282,
+        363,
+        367,
+        383,
+        429,
+        430,
+        448,
+        510,
+        528,
+        529,
+        530,
+        599,
+        682,
+        683,
+        689,
+        690,
+        691,
+        692,
+        693,
+    )
     assert catalog.records[448]["regression_ids"] == [
         "tr-448-302-success",
         "tr-448-200-json-success",
@@ -52,9 +358,23 @@ def test_catalog_is_replay_only_and_has_exact_reviewed_scope(catalog) -> None:
     ]
 
 
+def test_current_catalog_rejects_a_different_fork_revision(tmp_path) -> None:
+    payload = json.loads(DEFAULT_CORRECTIONS_PATH.read_text(encoding="utf-8"))
+    payload["upstream"]["revision"] = "b" * 40
+    changed = tmp_path / "task-corrections.json"
+    changed.write_text(json.dumps(payload), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="different WebArena fork revision"):
+        load_corrections(changed)
+
+
 def test_448_scoring_revision_is_bound_to_correction_and_regressions(catalog) -> None:
     root = Path(__file__).resolve().parents[2]
-    catalog = load_corrections(root / "evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json")
+    catalog = load_corrections(
+        root
+        / "evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json",
+        require_current_upstream=False,
+    )
     receipt = json.loads(
         (root / "research/trajectory-replay-task448-scoring-revision.json").read_text(
             encoding="utf-8"
@@ -98,11 +418,17 @@ def test_apply_is_pure_traceable_and_fails_closed(benchmark, catalog) -> None:
 
 def test_final_event_revision_binds_current_and_historical_catalogs(catalog):
     root = Path(__file__).resolve().parents[2]
-    receipt = json.loads((root / "research/trajectory-replay-task448-final-event-revision.json").read_text())
-    predecessor = load_corrections(root / receipt["predecessor_catalog_path"])
+    receipt = json.loads(
+        (root / "research/trajectory-replay-task448-final-event-revision.json").read_text()
+    )
+    predecessor = load_corrections(
+        root / receipt["predecessor_catalog_path"], require_current_upstream=False
+    )
     assert receipt["predecessor_correction_catalog_sha256"] == predecessor.source_sha256
     assert receipt["predecessor_correction_record_sha256"] == predecessor.record_sha256s[448]
-    revised = load_corrections(root / receipt["revised_catalog_path"])
+    revised = load_corrections(
+        root / receipt["revised_catalog_path"], require_current_upstream=False
+    )
     assert receipt["revised_correction_catalog_sha256"] == revised.source_sha256
     assert receipt["revised_correction_record_sha256"] == catalog.record_sha256s[448]
     assert receipt["regression_ids"] == catalog.records[448]["regression_ids"]
@@ -111,11 +437,13 @@ def test_final_event_revision_binds_current_and_historical_catalogs(catalog):
 
 def test_archived_catalog_reproduces_all_unchanged_task_contracts(benchmark, catalog):
     archived = load_corrections(
-        Path("evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json"))
-    for task_id in corrected_task_ids(archived):
+        Path("evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json"),
+        require_current_upstream=False,
+    )
+    for task_id in sorted(set(corrected_task_ids(archived)) & set(catalog.records)):
         previous = apply_correction(benchmark.get_task(task_id), archived)
         current = apply_correction(benchmark.get_task(task_id), catalog)
-        if task_id in {448, 510}:
+        if task_id in {280, 282, 448, 510}:
             assert previous.corrected_task_sha256 != current.corrected_task_sha256
         else:
             assert previous.corrected_task_sha256 == current.corrected_task_sha256
@@ -179,6 +507,63 @@ def test_279_complete_catalog_scores_both_directions(tmp_path, benchmark, catalo
     assert float(result.score) == score
 
 
+@pytest.mark.parametrize(
+    "task_id,expected_count,expected_min,expected_max",
+    [(280, 17, 8.99, 59.99), (282, 9, 36.86, 90.65)],
+)
+def test_current_catalog_price_ranges_are_complete(
+    benchmark, catalog, task_id, expected_count, expected_min, expected_max
+):
+    contract = apply_correction(benchmark.get_task(task_id), catalog)
+    result = _dump_eval(contract, 0)["expected"]["retrieved_data"][0]
+    assert len(result["names"]) == expected_count
+    assert result["min"] == expected_min
+    assert result["max"] == expected_max
+
+
+@pytest.mark.parametrize(
+    "task_id,mutation,score",
+    [
+        (280, "complete", 1.0),
+        (280, "omitted", 0.0),
+        (280, "unqualified", 0.0),
+        (282, "complete", 1.0),
+        (282, "retired", 0.0),
+        (282, "unqualified", 0.0),
+    ],
+    ids=[
+        "tr-280-accept-complete-seventeen-products",
+        "tr-280-reject-frozen-omitted-products",
+        "tr-280-reject-non-charger-anker-product",
+        "tr-282-accept-current-nine-slides",
+        "tr-282-reject-retired-benassi-product",
+        "tr-282-reject-non-slide-nike-sandal",
+    ],
+)
+def test_current_catalog_price_ranges_score_both_directions(
+    tmp_path, benchmark, catalog, task_id, mutation, score
+):
+    contract = apply_correction(benchmark.get_task(task_id), catalog)
+    response = copy.deepcopy(_dump_eval(contract, 0)["expected"])
+    values = response["retrieved_data"][0]
+    if mutation == "omitted":
+        values["names"].pop()
+    elif mutation == "retired":
+        values["names"][-1] = "Nike womens Benassi Just Do It"
+        values["min"] = 27.6
+    elif mutation == "unqualified":
+        values["names"].append(
+            "Anker Soundcore Mini 3 Bluetooth Speaker"
+            if task_id == 280
+            else "Nike Women's Tanjun Sandal"
+        )
+    trace = _route_har(tmp_path / f"catalog-{task_id}.har", "http://shop.test/")
+    result = evaluate_corrected_contract(
+        benchmark, contract, agent_response=response, network_trace=trace
+    )
+    assert float(result.score) == score
+
+
 def test_225_corrects_unverifiable_brand_scope_from_independent_catalog(benchmark, catalog) -> None:
     original = benchmark.get_task(225)
     corrected = apply_correction(original, catalog)
@@ -289,7 +674,9 @@ def test_448_official_evaluator_rejects_overwritten_correct_save(
 ) -> None:
     corrected = apply_correction(benchmark.get_task(448), catalog)
     predecessor = apply_correction(benchmark.get_task(448), load_corrections(
-        Path("evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json")))
+        Path("evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json"),
+        require_current_upstream=False,
+    ))
     target = "http://host.docker.internal:8023/-/profile"
 
     def entry(value: str) -> dict:
@@ -419,8 +806,11 @@ def test_510_freezes_selection_order_and_product(benchmark, catalog) -> None:
     assert _dump_eval(applied, 2)["expected"]["response_content"]["items_qty"] == 1
 
 
-def test_528_uses_final_dom_state_and_forbids_real_submission(benchmark, catalog) -> None:
-    applied = apply_correction(benchmark.get_task(528), catalog)
+@pytest.mark.parametrize("task_id", [528, 529, 530])
+def test_refund_forms_use_final_dom_state_and_forbid_real_submission(
+    benchmark, catalog, task_id
+) -> None:
+    applied = apply_correction(benchmark.get_task(task_id), catalog)
     network = [_dump_eval(applied, index) for index in range(1, len(applied.task.eval))]
     assert len(network) == 1
     assert network[0]["expected"]["url"] == "__SHOPPING__/contact/index/post"
@@ -431,15 +821,100 @@ def test_528_uses_final_dom_state_and_forbids_real_submission(benchmark, catalog
     assert assertion["forbid_network_event"]["http_method"] == "POST"
 
 
-def test_547_removes_redundant_attribute_save_but_keeps_variation_save(benchmark, catalog) -> None:
-    applied = apply_correction(benchmark.get_task(547), catalog)
+@pytest.mark.parametrize("task_id", [689, 690, 691, 692, 693])
+def test_coupon_requests_use_exact_final_dom_state_without_submission(
+    benchmark, catalog, tmp_path, task_id
+) -> None:
+    applied = apply_correction(benchmark.get_task(task_id), catalog)
+    network = [_dump_eval(applied, index) for index in range(1, len(applied.task.eval))]
+    assert len(network) == 1
+    assert network[0]["expected"]["url"] == "__SHOPPING__/contact/index/post"
+    assert network[0]["should_not_exist"] is True
+
+    assertion = applied.replay_assertions[0]
+    expected = assertion["fields"]
+    passed = check_replay_assertions(
+        applied,
+        page_url="http://shop.test/contact/",
+        page_text="Contact Us",
+        field_values=expected,
+    )
+    assert passed == (
+        {
+            "kind": "final_dom_form_state",
+            "status": "pass",
+            "expected": expected,
+            "actual": expected,
+        },
+    )
+
+    wrong = dict(expected)
+    wrong["textarea[name='comment']"] = "a different coupon reason"
+    assert check_replay_assertions(
+        applied,
+        page_url="http://shop.test/contact/",
+        page_text="Contact Us",
+        field_values=wrong,
+    )[0]["status"] == "fail"
+
+    empty_har = tmp_path / f"coupon-{task_id}.har"
+    empty_har.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "version": "1.2",
+                    "creator": {"name": "coupon-form-control", "version": "1"},
+                    "entries": [
+                        {
+                            "request": {
+                                "method": "GET",
+                                "url": "http://shop.test/contact/",
+                                "headers": [],
+                            },
+                            "response": {
+                                "status": 200,
+                                "headers": [],
+                                "cookies": [],
+                                "content": {"mimeType": "text/html", "text": ""},
+                            },
+                        }
+                    ],
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+    response = {
+        "task_type": "mutate",
+        "status": "SUCCESS",
+        "retrieved_data": None,
+        "error_details": None,
+    }
+    assert evaluate_corrected_contract(
+        benchmark, applied, agent_response=response, network_trace=empty_har
+    ).score == 1
+
+
+def test_547_upstream_keeps_only_the_variation_save(benchmark, catalog) -> None:
+    assert 547 not in catalog.records
+    task = benchmark.get_task(547)
     network_urls = [
         item["expected"]["url"]
-        for item in (_dump_eval(applied, index) for index in range(1, len(applied.task.eval)))
+        for item in (item.model_dump(mode="json") for item in task.eval[1:])
     ]
     assert not any("product_attribute/save" in url for url in network_urls)
     assert any("catalog/product/save/id/1130" in url for url in network_urls)
-    assert applied.replay_assertions == ()
+
+
+def test_566_commit_path_matches_the_project_creation_contract(benchmark, catalog) -> None:
+    assert 566 not in catalog.records
+    task = benchmark.get_task(566)
+    project = task.eval[1].model_dump(mode="json")
+    commit = task.eval[2].model_dump(mode="json")
+    assert project["expected"]["post_data"]["project[path]"] == "do-it-myself"
+    assert commit["expected"]["url"] == (
+        "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits"
+    )
 
 
 def test_676_uses_upstream_filter_xhr_contract(benchmark, catalog) -> None:
@@ -453,8 +928,8 @@ def test_676_uses_upstream_filter_xhr_contract(benchmark, catalog) -> None:
 def test_699_checks_browser_form_array_fields_and_all_registered_customer_groups(
     benchmark, catalog
 ) -> None:
-    applied = apply_correction(benchmark.get_task(699), catalog)
-    evaluator = _dump_eval(applied, 1)
+    assert 699 not in catalog.records
+    evaluator = benchmark.get_task(699).eval[1].model_dump(mode="json")
     assert evaluator["post_data_schema"] is None
     assert evaluator["expected"]["post_data"] == {
         "name": "spring sale",
diff --git a/tests/trajectory_replay/test_webarena_replay.py b/tests/trajectory_replay/test_webarena_replay.py
index d00a0beaa..363fa9d1c 100644
--- a/tests/trajectory_replay/test_webarena_replay.py
+++ b/tests/trajectory_replay/test_webarena_replay.py
@@ -10,6 +10,7 @@
     RECEIPT_SCHEMA,
     _attempt_needs_retry,
     _capture_failure_ui_state,
+    _clear_browser_state,
     _click_was_dispatched_before_navigation_timeout,
     _embedded_observation_error,
     _execute_replay_call,
@@ -17,6 +18,7 @@
     _origin,
     _rebind_url,
     _reject_partial_method_resume,
+    _replay_step_status,
     _run_one,
     _validated_resume,
     run_replay_suite,
@@ -26,6 +28,172 @@
 )
 
 
+def test_clear_browser_state_uses_page_storage_for_sessions_without_storage_cdp() -> None:
+    events = []
+
+    class Session:
+        def send(self, method, params=None):
+            events.append(("cdp", method, params))
+
+        def detach(self):
+            events.append(("detach",))
+
+    class Page:
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, script):
+            events.append(("evaluate", script))
+
+    page = Page()
+    context = type(
+        "Context",
+        (),
+        {
+            "clear_cookies": lambda self: events.append(("clear_cookies",)),
+            "clear_permissions": lambda self: events.append(("clear_permissions",)),
+            "new_cdp_session": lambda self, selected: Session(),
+        },
+    )()
+    service = type(
+        "Service",
+        (),
+        {
+            "_context": context,
+            "_page": page,
+            "_supports_storage_cdp": False,
+            "_submit": lambda self, callback: callback(),
+        },
+    )()
+
+    _clear_browser_state(service, {"http://b.test", "http://a.test"})
+
+    assert ("cdp", "Storage.clearDataForOrigin", None) not in events
+    assert [event for event in events if event[0] == "goto"] == [
+        ("goto", "http://a.test"),
+        ("goto", "http://b.test"),
+        ("goto", "about:blank"),
+    ]
+    assert sum(event[0] == "evaluate" for event in events) == 2
+
+
+def test_clear_browser_state_accepts_a_fresh_per_task_context_without_revisiting_sites() -> None:
+    class Context:
+        def clear_cookies(self):
+            raise AssertionError("a new context must not repeat storage cleanup")
+
+        def clear_permissions(self):
+            raise AssertionError("a new context must not repeat permission cleanup")
+
+    class Page:
+        def goto(self, *_args, **_kwargs):
+            raise AssertionError("a new context must not navigate through site origins")
+
+    service = type(
+        "Service",
+        (),
+        {
+            "_context": Context(),
+            "_page": Page(),
+            "_fresh_context_per_task": True,
+            "_submit": lambda self, callback: callback(),
+        },
+    )()
+
+    _clear_browser_state(service, {"http://example.test"})
+
+
+def test_clear_browser_state_avoids_unsupported_obscura_cdp_domains() -> None:
+    events = []
+
+    class Page:
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, script):
+            events.append(("evaluate", script))
+
+    class Context:
+        def clear_cookies(self):
+            events.append(("clear_cookies",))
+
+        def clear_permissions(self):
+            events.append(("clear_permissions",))
+
+        def new_cdp_session(self, _page):
+            raise AssertionError("Obscura state reset must not open a CDP session")
+
+    service = type(
+        "Service",
+        (),
+        {
+            "_context": Context(),
+            "_page": Page(),
+            "_supports_network_reset_cdp": False,
+            "_supports_storage_cdp": False,
+            "_submit": lambda self, callback: callback(),
+        },
+    )()
+
+    _clear_browser_state(service, {"http://example.test"})
+
+    assert events[0:2] == [("clear_cookies",), ("clear_permissions",)]
+    assert [event for event in events if event[0] == "goto"] == [
+        ("goto", "http://example.test"),
+        ("goto", "about:blank"),
+    ]
+    assert sum(event[0] == "evaluate" for event in events) == 1
+
+
+def test_clear_browser_state_retries_storage_after_navigation_settles() -> None:
+    events = []
+
+    class Page:
+        def goto(self, url, **_kwargs):
+            events.append(("goto", url))
+
+        def evaluate(self, script):
+            events.append(("evaluate", script))
+            if sum(event[0] == "evaluate" for event in events) == 1:
+                raise RuntimeError(
+                    "Execution context was destroyed, most likely because of a navigation"
+                )
+
+        def wait_for_load_state(self, state, **kwargs):
+            events.append(("load_state", state, kwargs))
+
+        def wait_for_timeout(self, milliseconds):
+            events.append(("wait", milliseconds))
+
+    class Context:
+        def clear_cookies(self):
+            events.append(("clear_cookies",))
+
+        def clear_permissions(self):
+            events.append(("clear_permissions",))
+
+        def new_cdp_session(self, _page):
+            raise AssertionError("state reset must not open a CDP session")
+
+    service = type(
+        "Service",
+        (),
+        {
+            "_context": Context(),
+            "_page": Page(),
+            "_supports_network_reset_cdp": False,
+            "_supports_storage_cdp": False,
+            "_submit": lambda self, callback: callback(),
+        },
+    )()
+
+    _clear_browser_state(service, {"http://example.test"})
+
+    assert sum(event[0] == "evaluate" for event in events) == 2
+    assert any(event[:2] == ("load_state", "domcontentloaded") for event in events)
+    assert any(event[0] == "wait" for event in events)
+
+
 def test_method_scoped_site_cohort_rejects_partial_resume() -> None:
     with pytest.raises(ValueError, match="cannot resume after a partial method"):
         _reject_partial_method_resume([("moli", 1)], ["moli"], [1, 2])
@@ -325,6 +493,15 @@ def test_snapshot_payload_error_is_exposed_as_observation_error() -> None:
         is None
     )
     assert _embedded_observation_error({"action": "snapshot"}, "success", "Page: stable") is None
+    assert _replay_step_status(
+        {"action": "snapshot"},
+        "success",
+        "[Snapshot error: context destroyed]",
+        dispatched=False,
+    ) == ("error", "context destroyed")
+    assert _replay_step_status(
+        {"action": "click"}, "error", "navigation timeout", dispatched=True
+    ) == ("success", None)
     assert _attempt_needs_retry(
         {
             "status": "fail",
@@ -362,7 +539,9 @@ def test_completed_click_is_not_replayed_when_only_navigation_wait_times_out() -
 
 @pytest.mark.parametrize("collect_diagnostics", [False, True])
 @pytest.mark.parametrize("disabled", [False, True])
-def test_compiled_snapshot_ref_click_preserves_dom_click_semantics(collect_diagnostics, disabled) -> None:
+def test_compiled_snapshot_ref_click_preserves_dom_click_semantics(
+    collect_diagnostics, disabled
+) -> None:
     events = []
 
     class Locator:
@@ -371,6 +550,9 @@ class Locator:
         def __init__(self) -> None:
             self.first = self
 
+        def count(self):
+            return 1
+
         def evaluate(self, script, *args):
             events.append(("evaluate", script, args))
             return {"connected": True, "disabled": disabled,
@@ -387,6 +569,10 @@ def wait_for_timeout(self, timeout):
     class Service:
         _page = Page()
 
+        @property
+        def layout_observer(self):
+            return lambda page, timeout: events.append(("layout", page, timeout))
+
         def _submit(self, callback):
             return callback()
 
@@ -409,9 +595,14 @@ def execute(self, _arguments):
     )
 
     assert status == "success"
-    assert events[0] == ("locator", "a[href='/next']:visible")
+    assert events[0][:2] == ("layout", Service._page)
+    assert 0 < events[0][2] <= 60_000
+    assert ("locator", "a[href='/next']") in events
     assert events[-1] == ("wait", 500)
-    evaluations = [event for event in events if event[0] == "evaluate"]
+    evaluations = [
+        event for event in events
+        if event[0] == "evaluate" and "getComputedStyle" not in event[1]
+    ]
     assert len(evaluations) == 1
     assert "element.click()" in evaluations[0][1]
     if collect_diagnostics:
@@ -419,6 +610,198 @@ def execute(self, _arguments):
         assert evaluations[0][1].index("disabled:") < evaluations[0][1].index("element.click()")
 
 
+def test_moli_compiled_ref_waits_for_delayed_visible_target() -> None:
+    visibility = iter((0, 1))
+    events = []
+
+    class Locator:
+        first = None
+
+        def __init__(self, selector) -> None:
+            self.first = self
+            self.selector = selector
+
+        def count(self):
+            value = next(visibility) if self.selector.endswith(":visible") else 1
+            events.append(("count", self.selector, value))
+            return value
+
+        def evaluate(self, script, *args):
+            events.append(("action", script, args))
+            return {"before": "12", "after": args[0], "selectedText": "Illinois"}
+
+    class Page:
+        def locator(self, selector):
+            events.append(("locator", selector))
+            return Locator(selector)
+
+        def wait_for_timeout(self, timeout):
+            events.append(("wait", timeout))
+
+    class Service:
+        _page = Page()
+
+        @property
+        def layout_observer(self):
+            return lambda page, timeout: events.append(("layout", page, timeout))
+
+        def _submit(self, callback):
+            return callback()
+
+    class Tool:
+        def _get_service(self):
+            return Service()
+
+    status, _ = _execute_replay_call(
+        Tool(),
+        {
+            "action": "select",
+            "selector": "select[name='region_id']:visible",
+            "value": "23",
+            "replay_mode": "snapshot_ref_semantics",
+        },
+    )
+
+    assert status == "success"
+    assert events[0][:2] == ("layout", Service._page)
+    assert 0 < events[0][2] <= 60_000
+    assert ("wait", 100) in events
+    assert [event for event in events if event[0] == "count" and event[1].endswith(":visible")] == [
+        ("count", "select[name='region_id']:visible", 0),
+        ("count", "select[name='region_id']:visible", 1),
+    ]
+    assert events[-1][0] == "action"
+
+
+def test_moli_compiled_ref_force_uses_unique_stable_target_without_geometry_wait() -> None:
+    events = []
+
+    class Locator:
+        first = None
+
+        def __init__(self, selector) -> None:
+            self.first = self
+            self.selector = selector
+
+        def count(self):
+            events.append(("count", self.selector))
+            return 1 if not self.selector.endswith(":visible") else 0
+
+        def evaluate(self, script, *args):
+            events.append(("evaluate", self.selector, script, args))
+            return "Linux"
+
+    class Keyboard:
+        def press(self, key):
+            events.append(("press", key))
+
+        def insert_text(self, text):
+            events.append(("insert_text", text))
+
+    class Page:
+        keyboard = Keyboard()
+
+        def locator(self, selector):
+            return Locator(selector)
+
+    class Service:
+        _page = Page()
+        layout_observer = staticmethod(lambda _page, _timeout: events.append(("layout",)))
+
+        def _submit(self, callback):
+            return callback()
+
+    class Tool:
+        def _get_service(self):
+            return Service()
+
+    status, _ = _execute_replay_call(
+        Tool(),
+        {
+            "action": "fill",
+            "selector": "input[name='created_at[from]']:visible",
+            "text": "01/01/2023",
+            "force": True,
+            "replay_mode": "snapshot_ref_semantics",
+        },
+    )
+
+    assert status == "success"
+    assert ("count", "input[name='created_at[from]']:visible") not in events
+    assert ("insert_text", "01/01/2023") not in events
+    dispatches = [event for event in events if event[0] == "evaluate" and event[3]]
+    assert len(dispatches) == 1
+    assert "setter.call(element, expected)" in dispatches[0][2]
+    assert "dispatchEvent(new Event('input'" in dispatches[0][2]
+    assert dispatches[0][3] == ("01/01/2023",)
+
+
+def test_compiled_ref_wait_for_enabled_delays_single_click_dispatch() -> None:
+    states = iter(
+        (
+            {
+                "connected": True,
+                "disabled": True,
+                "effectivelyDisabled": True,
+                "ariaDisabled": None,
+            },
+            {
+                "connected": True,
+                "disabled": False,
+                "effectivelyDisabled": False,
+                "ariaDisabled": None,
+            },
+        )
+    )
+    events = []
+
+    class Locator:
+        first = None
+
+        def __init__(self) -> None:
+            self.first = self
+
+        def evaluate(self, script, *args):
+            if "element.click()" in script:
+                events.append("click")
+                return {}
+            events.append("probe")
+            return next(states)
+
+    class Page:
+        def locator(self, _selector):
+            return Locator()
+
+        def wait_for_timeout(self, timeout):
+            events.append(("wait", timeout))
+
+    class Service:
+        _page = Page()
+        layout_observer = None
+
+        def _submit(self, callback):
+            return callback()
+
+    class Tool:
+        def _get_service(self):
+            return Service()
+
+    status, _ = _execute_replay_call(
+        Tool(),
+        {
+            "action": "click",
+            "selector": "button.add-to-cart",
+            "wait_for_enabled": True,
+            "replay_mode": "snapshot_ref_semantics",
+        },
+    )
+
+    assert status == "success"
+    assert events.count("probe") == 2
+    assert events.count("click") == 1
+    assert ("wait", 100) in events
+
+
 @pytest.mark.parametrize(
     ("platform", "modifier"),
     [("Win32", "Control"), ("MacIntel", "Meta"), ("Linux x86_64", "Control")],
@@ -434,9 +817,14 @@ class Locator:
         def __init__(self) -> None:
             self.first = self
 
+        def count(self):
+            return 1
+
         def evaluate(self, script, *args):
             events.append(("evaluate", script, args))
-            return platform if "navigator.platform" in script else None
+            if "navigator.platform" in script:
+                return platform
+            return {"before": "closed", "after": args[0], "selectedText": "Open"}
 
     class Keyboard:
         def press(self, key):
@@ -455,6 +843,10 @@ def locator(self, selector):
     class Service:
         _page = Page()
 
+        @property
+        def layout_observer(self):
+            return lambda page, timeout: events.append(("layout", page, timeout))
+
         def _submit(self, callback):
             return callback()
 
@@ -471,6 +863,7 @@ def _get_service(self):
             "replay_mode": "snapshot_ref_semantics",
         },
     )
+    diagnostics = {}
     select_status, _ = _execute_replay_call(
         Tool(),
         {
@@ -479,13 +872,26 @@ def _get_service(self):
             "value": "open",
             "replay_mode": "snapshot_ref_semantics",
         },
+        diagnostics=diagnostics,
     )
 
     assert fill_status == select_status == "success"
+    assert events[0][:2] == ("layout", Service._page)
+    layouts = [event for event in events if event[:2] == ("layout", Service._page)]
+    assert len(layouts) == 2
+    assert all(0 < event[2] <= 60_000 for event in layouts)
+    assert ("locator", "input[name='query']") in events
+    assert ("locator", "select[name='status']") in events
     assert events.index(("press", f"{modifier}+A")) < events.index(("press", "Backspace"))
     assert events.index(("press", "Backspace")) < events.index(("insert_text", "needle"))
     assert not any(event[0] == "evaluate" and "element.value = ''" in event[1] for event in events)
     assert any(event[0] == "evaluate" and event[2] == ("open",) for event in events)
+    assert diagnostics["select_dispatch_state"]["after"] == "open"
+    select_script = next(
+        event[1] for event in events if event[0] == "evaluate" and event[2] == ("open",)
+    )
+    assert select_script.index("new Event('input'") < select_script.index("new Event('change'")
+    assert "element.value !== value" in select_script
 
 
 def test_declared_wait_seconds_are_dispatched_as_browser_tool_milliseconds() -> None:
diff --git a/tests/trajectory_replay/test_webarena_report.py b/tests/trajectory_replay/test_webarena_report.py
index 0730f86cb..49a078a35 100644
--- a/tests/trajectory_replay/test_webarena_report.py
+++ b/tests/trajectory_replay/test_webarena_report.py
@@ -11,7 +11,6 @@
 from harness.trajectory_replay.webarena_report import (
     LEGACY_REPLAY_VALIDITY_INPUTS,
     _assets,
-    build_webarena_replay_report,
     _clean,
     _copy_failure_screenshot,
     _detail_page,
@@ -25,24 +24,30 @@
     _planned_actions_digest,
     _public_evidence,
     _reference_validity_by_task,
-    require_all_reference_validity,
+    _resource_table,
     _shell,
     _successful_distribution_table,
     _task_page,
     _unique_winners,
     _validated_planned_actions,
     _verdict_label,
+    build_webarena_replay_report,
+    require_all_reference_validity,
 )
+from scripts.build_webarena_replay_report import _bound_dataset_path
 from scripts.check_webarena_replay_report import (
-    check as check_report,
     _check_affected36_comparison,
     _check_core_comparison,
+    _check_english_localization,
+    _check_formal_execution_scope,
     _check_legacy_retrieval_audit,
     _check_result_detail_semantics,
-    _check_english_localization,
+    _contains_local_path_outside_web_url,
     _semantic_page_inventory,
 )
-from scripts.build_webarena_replay_report import _bound_dataset_path
+from scripts.check_webarena_replay_report import (
+    check as check_report,
+)
 
 
 def test_report_build_requires_hash_bound_corrected_dataset(tmp_path: Path) -> None:
@@ -121,13 +126,42 @@ def test_complete_three_method_report_builds_one_unified_matrix(tmp_path: Path,
         review_path=files["review.json"], profile_path=files["profile.json"],
         output_dir=output, include_english=True,
     )
+    invalid_profile = tmp_path / "reset-every-task-profile.json"
+    invalid_profile.write_text(
+        json.dumps(
+            {
+                **profile,
+                "execution": {
+                    **profile["execution"],
+                    "reset_before_every_task": True,
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError, match="common execution protocol"):
+        build_webarena_replay_report(
+            results_path=files["results.json"],
+            dataset_path=files["dataset.json"],
+            review_path=files["review.json"],
+            profile_path=invalid_profile,
+            output_dir=tmp_path / "invalid-report",
+        )
     assert manifest["participants"] == [item["id"] for item in methods]
+    assert manifest["browser_count"] == 2
+    assert manifest["method_count"] == len(methods)
     assert manifest["logical_results"] == len(methods)
     assert manifest["routes"] == 20 + 2 * extra_methods
     assert "36/36" not in (output / "index.html").read_text()
     assert "fixes.html" not in (output / "index.html").read_text()
-    assert "BrowserTool 与浏览器的组合" in (output / "index.html").read_text()
-    assert "BrowserTool–browser combinations" in (output / "en/index.html").read_text()
+    chinese_home = (output / "index.html").read_text()
+    english_home = (output / "en/index.html").read_text()
+    assert "Playwright 重放器与浏览器的组合" in chinese_home
+    assert "Playwright-adapter–browser configurations" in english_home
+    assert "没有调用 CowAgent BrowserTool" in chinese_home
+    assert "本轮没有记录进程级 CPU、内存和网络采样" in chinese_home
+    assert "CPU 时间" not in chinese_home
+    assert "平均内存" not in chinese_home
     assert "参考答案均已核验" not in (output / "index.html").read_text()
     assert "with verified references" not in (output / "en/index.html").read_text()
     assert "独立复核范围见复核说明" in (output / "index.html").read_text()
@@ -146,6 +180,14 @@ def test_complete_three_method_report_builds_one_unified_matrix(tmp_path: Path,
         environment = (output / route).read_text()
         assert 'data-topology-fallback' in environment and 'class="mermaid"' in environment
         assert 'Rainbond WebArena' in environment
+    assert "方法身份与修复序列" in (output / "environment.html").read_text()
+    assert "Method identity and repair sequence" in (output / "en/environment.html").read_text()
+    environment_path = output / "environment.html"
+    original_environment = environment_path.read_text()
+    environment_path.write_text(original_environment.replace("方法身份与修复序列", ""))
+    with pytest.raises(ValueError, match="method identity"):
+        check_report(output)
+    environment_path.write_text(original_environment)
     assert not (output / "diagnostics.html").exists()
     assert check_report(output)["html_pages"] == 20 + 2 * extra_methods
     home = output / "index.html"
@@ -169,6 +211,14 @@ def test_complete_three_method_report_builds_one_unified_matrix(tmp_path: Path,
     assert not (output / "en/results.html").exists()
     assert 'class="language" href="../en/index.html"' in (output / "result/chromium-29.html").read_text()
     assert check_report(output)["html_pages"] == 11 + extra_methods
+    manifest_path = output / "report-manifest.json"
+    changed = json.loads(manifest_path.read_text())
+    changed["browser_count"] += 1
+    manifest_path.write_text(json.dumps(changed))
+    with pytest.raises(ValueError, match="complete manifest-declared"):
+        check_report(output)
+    changed["browser_count"] -= 1
+    manifest_path.write_text(json.dumps(changed))
     stale = output / "en/results.html"
     stale.write_text("stale English report")
     with pytest.raises(ValueError, match="stale English"):
@@ -244,6 +294,24 @@ def test_formal_reference_validity_gate_requires_all_selected_tasks_for_all_meth
         require_all_reference_validity(profile, results, dataset)
 
 
+def test_formal_release_scope_accepts_any_complete_declared_split_size() -> None:
+    _check_formal_execution_scope(
+        {
+            "formal_reference_validity": True,
+            "task_count": 812,
+            "site_reset_between_tasks": False,
+        }
+    )
+    with pytest.raises(ValueError, match="method-reset task cohort"):
+        _check_formal_execution_scope(
+            {
+                "formal_reference_validity": True,
+                "task_count": 812,
+                "site_reset_between_tasks": True,
+            }
+        )
+
+
 def test_core_three_method_comparison_rejects_wrong_score_and_tie_highlight(tmp_path: Path) -> None:
     participants = ["chromium", "moli-old", "moli-fixed"]
     results = [
@@ -297,6 +365,30 @@ def test_result_detail_gate_rejects_duplicate_summary_and_missing_failure_log()
         _check_result_detail_semantics(
             html, {"programmatic_failure": {"stage": "official_evaluation", "message": "Exact timeout"}}, "fail",
         )
+    _check_result_detail_semantics(
+        html,
+        {
+            "programmatic_failure": {
+                "stage": "browser_tool_execution",
+                "step": None,
+                "message": "Exact timeout",
+            },
+            "trajectory": [],
+        },
+        "fail",
+    )
+    with pytest.raises(ValueError, match="evaluation logs"):
+        _check_result_detail_semantics(
+            html,
+            {
+                "programmatic_failure": {
+                    "stage": "browser_tool_execution",
+                    "step": None,
+                    "message": "Exact timeout",
+                }
+            },
+            "fail",
+        )
 
 
 def test_affected36_comparison_rejects_wrong_score_and_tied_highlight(tmp_path: Path) -> None:
@@ -361,6 +453,18 @@ def test_core_metrics_are_registry_driven_and_mark_only_unique_winners() -> None
     assert "median" not in rendered.lower()
 
 
+def test_unavailable_resource_metrics_are_not_rendered_as_empty_comparisons() -> None:
+    methods = [_method("Browser A", 1.0, 3.0), _method("Browser B", 0.8, 2.0)]
+    for method in methods:
+        for key in ("cpu", "memory", "memory_peak", "network_rx", "network_tx", "cpu_utilization"):
+            method[key] = {"n": 0, "mean": None, "p50": None, "p95": None}
+    rendered = _metrics_table(methods, "zh")
+    assert "CPU 时间" not in rendered
+    assert "平均内存" not in rendered
+    assert "端到端耗时" in rendered and "单次动作耗时" in rendered
+    assert _resource_table(methods, "zh") == ""
+
+
 def test_legacy_retrieval_provenance_binds_every_browser_to_saved_answer(tmp_path) -> None:
     answer = {"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": ["value"]}
     seed = {"tasks": [{"task_id": 243, "canonical_candidate_id": "candidate",
@@ -511,7 +615,9 @@ def test_own_success_distributions_do_not_select_shared_pass_tasks() -> None:
     assert methods[1]["successful"]["duration"]["mean"] == 2
     for lang in ("zh", "en"):
         rendered = _successful_distribution_table(methods, lang)
-        assert "n=1/2" in rendered and "n=0/1" in rendered
+        assert "n=1/2" in rendered and "n=1/1" in rendered
+        assert "CPU" not in rendered
+        assert ("接收数据" if lang == "zh" else "Network received") not in rendered
         assert 'class="winner"' not in rendered
 
 
@@ -798,6 +904,58 @@ def test_replay_task_and_result_have_connected_drilldown() -> None:
     assert 'href="../evidence-viewer.html?file=data/evidence/moli-29.json"' in detail
 
 
+def test_detail_page_preserves_task_timeout_with_unknown_active_action() -> None:
+    task = {
+        "task_id": 90,
+        "task": {"intent": "Open the route", "sites": ["map"]},
+        "tool_calls": [{"action": "navigate", "url": "https://example.test"}],
+    }
+    attempt = {
+        "engine": "obscura",
+        "participant_id": "obscura-0.2.3",
+        "task_id": 90,
+        "status": "fail",
+        "completed_action_count": 0,
+        "action_count": 1,
+        "duration_ms": 590_000,
+        "resource": {},
+        "planned_actions": task["tool_calls"],
+    }
+    evidence = {
+        "programmatic_failure": {
+            "stage": "browser_tool_execution",
+            "step": None,
+            "trigger_step": None,
+            "message": "browser stopped responding",
+        },
+        "review": {
+            "label_zh": "浏览器运行时无响应",
+            "label_en": "Browser runtime became unresponsive",
+            "reason_zh": "活动动作边界未记录",
+            "reason_en": "The active action boundary was not recorded",
+            "remediation_owner": "Obscura",
+            "remediation_status": "recorded",
+        },
+        "trajectory": [
+            {
+                "index": None,
+                "action": "task_timeout",
+                "arguments": {},
+                "status": "fail",
+                "duration_ms": 590_000,
+                "error": "browser stopped responding",
+            }
+        ],
+        "screenshot": None,
+    }
+
+    detail = _detail_page("zh", attempt, task, {"label": "Obscura 0.2.3"}, evidence, [])
+
+    assert "活动动作未记录" in detail
+    assert "查看已保存轨迹证据" in detail
+    assert 'id="step-1"' in detail
+
+
 def test_unexecuted_qualified_step_stays_visible_and_is_not_called_an_action_error() -> None:
     task = {
         "task_id": 29,
@@ -848,6 +1006,81 @@ def test_report_removes_macos_and_linux_local_paths() -> None:
         assert _clean(f"failure at {path}") == "failure at [local path removed]"
 
 
+def test_public_evidence_return_value_is_the_same_sanitized_value_written_to_disk(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    trace = tmp_path / "trace.json"
+    trace.write_text('[{"index": 1, "status": "fail"}]')
+    attempt = {
+        "participant_id": "browser",
+        "engine": "browser",
+        "task_id": 1,
+        "status": "fail",
+        "action_count": 1,
+        "completed_action_count": 0,
+        "duration_ms": 1,
+        "error": "failure at /Users/example/private/run.py",
+        "evidence": {
+            "trace": {
+                "path": str(trace),
+                "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(),
+            }
+        },
+    }
+    monkeypatch.setattr(
+        "harness.trajectory_replay.webarena_report.programmatic_failure",
+        lambda _: {
+            "stage": "browser_tool_execution",
+            "step": 1,
+            "message": "failure at /Users/example/private/run.py",
+        },
+    )
+
+    public = _public_evidence(attempt, None, tmp_path)
+    written = json.loads(
+        (tmp_path / "data/evidence/browser-1.json").read_text(encoding="utf-8")
+    )
+
+    assert public == written
+    assert public["programmatic_failure"]["message"] == "failure at [local path removed]"
+    assert public["programmatic_failure"]["step"] == 1
+
+
+def test_report_preserves_web_routes_that_resemble_linux_paths() -> None:
+    actions = [
+        {
+            "action": "navigate",
+            "url": "https://gitlab.example.test/root/project/-/issues",
+        }
+    ]
+    attempt = {
+        "participant_id": "moli",
+        "task_id": 317,
+        "action_count": 1,
+        "planned_actions": actions,
+        "planned_actions_sha256": _planned_actions_digest(actions),
+    }
+
+    assert _clean(actions) == actions
+    assert _validated_planned_actions(attempt) == actions
+
+
+def test_release_path_gate_ignores_web_routes_but_detects_machine_paths() -> None:
+    assert not _contains_local_path_outside_web_url(
+        "https://gitlab.example.test/root/project/-/issues"
+    )
+    assert _contains_local_path_outside_web_url(
+        "opened /root/private/result.json from https://example.test/root/public"
+    )
+
+
+def test_report_does_not_redact_secret_like_substrings_inside_words() -> None:
+    product = "https://shop.example.test/task-chairhome-office-chair.html"
+
+    assert _clean(product) == product
+    assert _clean("token=sk-secret_value_123456") == "token=[secret removed]"
+
+
 def test_report_rejects_symlinked_evidence_parent(tmp_path: Path) -> None:
     evidence = tmp_path / "evidence.json"
     evidence.write_text("{}", encoding="utf-8")
diff --git a/tests/trajectory_replay/test_webarena_review.py b/tests/trajectory_replay/test_webarena_review.py
index e2bf7067a..6534b025f 100644
--- a/tests/trajectory_replay/test_webarena_review.py
+++ b/tests/trajectory_replay/test_webarena_review.py
@@ -45,6 +45,41 @@ def test_programmatic_failure_keeps_location_separate_from_root_cause() -> None:
     }
 
 
+def test_programmatic_failure_preserves_browser_session_setup_stage() -> None:
+    result = programmatic_failure(
+        {
+            "process_status": "fail",
+            "failure_stage": "browser_session_setup",
+            "error": "Traceback: _login_context failed before trajectory execution",
+        }
+    )
+    assert result == {
+        "stage": "browser_session_setup",
+        "step": None,
+        "message": "Traceback: _login_context failed before trajectory execution",
+    }
+
+
+def test_programmatic_failure_omits_mutable_traceback_source_excerpt() -> None:
+    result = programmatic_failure(
+        {
+            "process_status": "fail",
+            "failure_stage": "browser_session_setup",
+            "error": (
+                "Traceback (most recent call last):\n"
+                '  File "remote_execution.py", line 322, in _run_task\n'
+                '    source text from another checkout revision\n'
+                "playwright._impl._errors.TimeoutError: Locator.click timed out\n"
+                "Call log:\n  - element is outside of the viewport\n"
+            ),
+        }
+    )
+    assert result["message"] == (
+        "playwright._impl._errors.TimeoutError: Locator.click timed out\n"
+        "Call log:\n  - element is outside of the viewport"
+    )
+
+
 @pytest.mark.parametrize("tamper", ["row", "digest", "symlink", "new-result"])
 def test_append_methods_keeps_old_diagnostics_bound_to_unchanged_results(tmp_path: Path, tamper) -> None:
     old_row = {"participant_id": "moli", "engine": "moli", "task_id": 1,
diff --git a/tests/trajectory_replay/test_webarena_runtime_source.py b/tests/trajectory_replay/test_webarena_runtime_source.py
new file mode 100644
index 000000000..79a7b7eb9
--- /dev/null
+++ b/tests/trajectory_replay/test_webarena_runtime_source.py
@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import importlib.machinery
+import json
+from pathlib import Path
+
+import pytest
+
+from harness.trajectory_replay import (
+    catalog_acceptance,
+    catalog_consolidation,
+    catalog_release,
+    catalog_runner,
+)
+from harness.trajectory_replay import webarena_runtime_source as runtime_source
+
+REVISION = runtime_source.WEBARENA_VERIFIED_REVISION
+FORK = runtime_source.WEBARENA_VERIFIED_FORK_URL
+
+
+class _Distribution:
+    def __init__(self, root: Path, direct_url: object, *, owns_package: bool = True):
+        self.root = root
+        self.direct_url = direct_url
+        self.files = (
+            [Path("webarena_verified/__init__.py")] if owns_package else [Path("other.py")]
+        )
+
+    def read_text(self, name: str):
+        assert name == "direct_url.json"
+        if isinstance(self.direct_url, str):
+            return self.direct_url
+        if self.direct_url is None:
+            return None
+        return json.dumps(self.direct_url)
+
+    def locate_file(self, path):
+        return self.root / path
+
+
+def _direct_url(*, url=FORK, revision=REVISION, requested_revision=REVISION):
+    return {
+        "url": url,
+        "vcs_info": {
+            "vcs": "git",
+            "commit_id": revision,
+            "requested_revision": requested_revision,
+        },
+    }
+
+
+def _install(monkeypatch, tmp_path, direct_url, *, module_root=None, owns_package=True):
+    distribution = _Distribution(tmp_path, direct_url, owns_package=owns_package)
+    expected_root = tmp_path / "webarena_verified"
+    expected_root.mkdir(parents=True)
+    (expected_root / "__init__.py").write_text("", encoding="utf-8")
+    selected_root = module_root or expected_root
+    spec = importlib.machinery.ModuleSpec(
+        "webarena_verified",
+        loader=None,
+        origin=str(selected_root / "__init__.py"),
+        is_package=True,
+    )
+    spec.submodule_search_locations = [str(selected_root)]
+    monkeypatch.setattr(
+        runtime_source.importlib.metadata, "distribution", lambda _name: distribution
+    )
+    monkeypatch.setattr(runtime_source.importlib.util, "find_spec", lambda _name: spec)
+    return expected_root
+
+
+def test_accepts_exact_lexmount_fork_revision_and_owned_module(monkeypatch, tmp_path):
+    expected_root = _install(monkeypatch, tmp_path, _direct_url())
+    identity = runtime_source.verify_webarena_runtime_source()
+    assert identity.repository_url == FORK
+    assert identity.revision == REVISION
+    assert identity.module_root == expected_root.resolve()
+
+
+def test_rejects_servicenow_repository(monkeypatch, tmp_path):
+    _install(monkeypatch, tmp_path, _direct_url(url="https://github.com/ServiceNow/webarena-verified.git"))
+    with pytest.raises(RuntimeError, match="Lexmount fork"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+@pytest.mark.parametrize("direct_url", [None, {"archive_info": {}}, {"url": "pypi"}])
+def test_rejects_pypi_or_missing_direct_url(monkeypatch, tmp_path, direct_url):
+    _install(monkeypatch, tmp_path, direct_url)
+    with pytest.raises(RuntimeError, match="direct_url"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+def test_rejects_wrong_commit(monkeypatch, tmp_path):
+    _install(monkeypatch, tmp_path, _direct_url(revision="b" * 40))
+    with pytest.raises(RuntimeError, match="revision"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+@pytest.mark.parametrize(
+    "direct_url",
+    ["{", [], {"url": FORK, "vcs_info": "git"}, _direct_url(requested_revision=1)],
+)
+def test_rejects_malformed_metadata(monkeypatch, tmp_path, direct_url):
+    _install(monkeypatch, tmp_path, direct_url)
+    with pytest.raises(RuntimeError, match="metadata|revision"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+def test_rejects_module_path_injection(monkeypatch, tmp_path):
+    injected = tmp_path / "injected" / "webarena_verified"
+    _install(monkeypatch, tmp_path / "distribution", _direct_url(), module_root=injected)
+    with pytest.raises(RuntimeError, match="import path"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+def test_rejects_distribution_without_package_ownership(monkeypatch, tmp_path):
+    _install(monkeypatch, tmp_path, _direct_url(), owns_package=False)
+    with pytest.raises(RuntimeError, match="does not own"):
+        runtime_source.verify_webarena_runtime_source()
+
+
+def test_acceptance_checks_source_before_reading_evidence(monkeypatch, tmp_path):
+    monkeypatch.setattr(
+        catalog_acceptance,
+        "verify_webarena_runtime_source",
+        lambda: (_ for _ in ()).throw(RuntimeError("wrong source")),
+    )
+    with pytest.raises(RuntimeError, match="wrong source"):
+        catalog_acceptance.audit(tmp_path / "missing", [], [])
+
+
+def test_consolidation_checks_source_before_reading_evidence(monkeypatch, tmp_path):
+    monkeypatch.setattr(
+        catalog_consolidation,
+        "verify_webarena_runtime_source",
+        lambda: (_ for _ in ()).throw(RuntimeError("wrong source")),
+    )
+    with pytest.raises(RuntimeError, match="wrong source"):
+        catalog_consolidation.consolidate(
+            tmp_path / "missing", tmp_path, [], tmp_path / "sites"
+        )
+
+
+def test_runner_checks_source_before_browser_or_input_access(monkeypatch, tmp_path):
+    monkeypatch.setattr(
+        catalog_runner,
+        "verify_webarena_runtime_source",
+        lambda: (_ for _ in ()).throw(RuntimeError("wrong source")),
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "run",
+            "--catalog",
+            str(tmp_path / "catalog"),
+            "--portable-suite",
+            str(tmp_path / "suite"),
+            "--site-map",
+            str(tmp_path / "sites"),
+            "--site-reset-receipt",
+            str(tmp_path / "receipt"),
+            "--environment-profile",
+            str(tmp_path / "environment"),
+            "--binary",
+            str(tmp_path / "browser"),
+            "--output",
+            str(tmp_path / "output"),
+        ],
+    )
+    with pytest.raises(RuntimeError, match="wrong source"):
+        catalog_runner.main()
+
+
+def test_release_checks_source_before_profile_access(monkeypatch):
+    monkeypatch.setattr(
+        catalog_release,
+        "verify_webarena_runtime_source",
+        lambda: (_ for _ in ()).throw(RuntimeError("wrong source")),
+    )
+    monkeypatch.setattr("sys.argv", ["release-check"])
+    with pytest.raises(RuntimeError, match="wrong source"):
+        catalog_release.main()
diff --git a/uv.lock b/uv.lock
index 1be411dea..a163a603c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -369,7 +369,7 @@ requires-dist = [
     { name = "requests", marker = "extra == 'reference-audit'", specifier = ">=2.32,<3" },
     { name = "ruff", marker = "extra == 'test'", specifier = ">=0.9,<1" },
     { name = "tiktoken", marker = "extra == 'browser-use-eval'", specifier = ">=0.7,<1" },
-    { name = "webarena-verified", marker = "extra == 'browser-use-eval'", git = "https://github.com/lexmount/webarena-verified.git?rev=7d060ec92b611d66b48d667acb9d92c35cfbdbda" },
+    { name = "webarena-verified", marker = "extra == 'browser-use-eval'", git = "https://github.com/lexmount/webarena-verified.git?rev=da7b992eb10de78000f11a7f34fd4590c171fbd6" },
 ]
 provides-extras = ["headless-eval", "reference-audit", "webmainbench-eval", "browser-use-eval", "test", "oss-eval"]
 
@@ -4423,8 +4423,8 @@ wheels = [
 
 [[package]]
 name = "webarena-verified"
-version = "1.2.3.post1.dev6"
-source = { git = "https://github.com/lexmount/webarena-verified.git?rev=7d060ec92b611d66b48d667acb9d92c35cfbdbda#7d060ec92b611d66b48d667acb9d92c35cfbdbda" }
+version = "1.2.3.post1.dev16"
+source = { git = "https://github.com/lexmount/webarena-verified.git?rev=da7b992eb10de78000f11a7f34fd4590c171fbd6#da7b992eb10de78000f11a7f34fd4590c171fbd6" }
 dependencies = [
     { name = "beautifulsoup4" },
     { name = "compact-json" },