diff --git a/.agents/skills/benchmark-editorial/references/html-delivery.md b/.agents/skills/benchmark-editorial/references/html-delivery.md index deebbbd00..fcb2a1bc0 100644 --- a/.agents/skills/benchmark-editorial/references/html-delivery.md +++ b/.agents/skills/benchmark-editorial/references/html-delivery.md @@ -2,6 +2,8 @@ The dashboard reading page and standalone HTML are two presentations of the same authored article. Use the current report renderer and preserve its integrity, safety and localization contracts. +Generate Chinese by default. English prose, illustrations, rendering and language-specific checks run only on an explicit manual request. An absent English edition gets a not-generated notice and a return link, not stale translated pages. Preserve translation sources without distributing unused English assets. Apply cross-language parity checks only when both editions are generated; always test the notice in a Chinese-only release. + ## Reading layout Choose a clear hierarchy for title, introduction, sections, body and captions. Align headings and their supporting figures deliberately, keep body lines readable, and check the result at actual display sizes. Do not create an oversized heading from an editing instruction or shrink a necessary qualification into unreadable text. diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md index 0c3e0c437..a9e57288a 100644 --- a/.agents/skills/benchmark-report/SKILL.md +++ b/.agents/skills/benchmark-report/SKILL.md @@ -11,6 +11,8 @@ Read the repository's `AGENTS.md` first. Project rules override this general ski ## Working contract +Repository reports default to Chinese-only output and checks. English generation (including translated illustrations) requires an explicit manual request. Bind generated languages in the report manifest; English navigation opens a not-generated notice when absent. Do not publish stale English pages. Run English-specific validation only for an explicitly generated English edition, retaining full parity checks when both editions exist. + 1. Establish the decision being supported, the measured unit, cohort, versions, execution location, freshness policy, and intended deliverable. Inspect the actual dataset, runner, outputs, scorer, prior review evidence, and current report. Do not infer them from screenshots or prose. 2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. For source-dependent tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release. 3. Let each method use its strongest documented route to produce the required output. Equalize the task, inputs, output contract, call budget, and scoring. Do not require identical internals. Record material asymmetries such as local versus remote execution, cold versus warm start, bundled conversion, network transit, account tier, and billable time. diff --git a/.agents/skills/benchmark-report/references/report-and-release.md b/.agents/skills/benchmark-report/references/report-and-release.md index 90593f87b..56377ce48 100644 --- a/.agents/skills/benchmark-report/references/report-and-release.md +++ b/.agents/skills/benchmark-report/references/report-and-release.md @@ -1,5 +1,11 @@ # Report and release +## Language generation + +All tracks generate and validate Chinese by default. English requires an explicit manual trigger; do not automatically translate prose, generate English illustrations, render English routes or run English-specific checks. A manifest records which languages were generated. When English is absent, retain only a clear not-generated notice with a return link, and test that control. Remove stale English output and unused translated assets from the new bundle while preserving source translations. Parity and English localization checks apply when the English edition is explicitly generated; Chinese-only releases retain all Chinese evidence, semantics, links and visual gates. + +For the current trajectory report, `make trajectory-replay-report EVIDENCE=/absolute/path/results.json` builds Chinese only. Explicit `ENGLISH=1` (or the build script's `--include-english`) enables English. Report, visual and packaging checks use the generated manifest, so they do not regenerate an absent edition. + ## Reader-facing structure Organize around reader questions, not storage tables. A useful report usually makes these answers easy to find: diff --git a/AGENTS.md b/AGENTS.md index dfe4edfb5..78165bfed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,8 @@ # Report maintenance -- Keep report-specific illustrations and review payloads outside Git, alongside frozen evidence. Commit only their compact path/hash bindings. WCXB builds read the profile-bound files from `/report-assets/` and preserve the same relative paths in the public package; restore missing files from the profile's pinned Artifact Site version, never from an unversioned latest report. Do not silently fall back to a repository copy. -- Remove retired report-specific generators and their exclusive tests when no supported entry point calls them. Preserve published reports and historical evidence. Keep behavioral tests for the supported build, package, and publication boundaries; do not replace them with assertions about private helper structure. +- Keep report-specific prose, illustrations and review payloads outside Git, alongside frozen evidence. Commit compact path/hash bindings instead. The replay build reads `research/trajectory-replay-content.json` from the `--content` directory (default: the data home's `report-assets/trajectory-replay/`); restore missing content from the pinned Artifact Site version, not an unversioned latest report. Do not silently fall back to a repository copy. Compact task definitions, acceptance contracts and provenance annotations remain versioned inputs. +- Remove retired report-specific generators and their exclusive tests when no supported entry point calls them. Preserve published reports and historical evidence. Keep behavioral tests for supported build, package and publication boundaries; do not replace them with assertions about private helper structure. +- WCXB builds read profile-bound files from `/report-assets/` and preserve the same relative paths in the public package; restore missing files from the profile's pinned Artifact Site version, never from an unversioned latest report. Do not silently fall back to a repository copy. ## Common report presentation @@ -28,7 +29,7 @@ These requirements apply to **all** report tracks; browser-specific rules below - A reference, scoring or acceptance correction must fix the shared cause, not patch stored verdicts or one provider. Add paired executable controls to `evaluation/scoring-regressions.json` where both error directions are plausible, run them through the public `score_attempt` and task predicate, and bind the root cause plus regression IDs in every new reference/scoring revision receipt. A revision without this provenance fails before evidence is changed. - A heterogeneous same-window latency cohort keeps every predeclared method-task call. Report the common-denominator all-call distribution separately from each method's conditional successful-call distribution; an all-method common-pass intersection is diagnostic only and must never drive post-outcome task selection or a ranking. - Evaluation credentials come only from the repository `.env`. Artifact Site publishing is the sole user-scoped exception: read explicit `ARTIFACT_SITE_TOKEN`, then `~/.config/artifact-site/tokens/`; the previous `ARTIFACT_HUB_TOKEN` and `~/.config/artifact-hub/` locations are read-only migration fallbacks. Never load `.env` for publishing. Save new authorizations per target host without overwriting another deployment's token. Keep `.env.example` current and publish only account requirements, official acquisition links and variable names. -- Chinese and English are two localized views of one report model. Their HTML route inventories, section order, tables, values, filters and evidence links must match one-to-one. Localize interface and explanatory prose for each language; keep task names, IDs, URLs, references, provider output and other measured evidence unchanged. +- All report tracks default to Chinese-only generation and checks. English prose, illustrations, rendering and validation require an explicit manual request. When English is absent, its navigation control must open a not-generated notice with a return link; do not ship stale English pages or unused English-only assets. Preserve translation sources for manual use. Record generated languages in the report manifest. Run release/visual checks for those languages and the not-generated notice. When both editions are generated, their HTML route inventories, section order, tables, values, filters and evidence links must match one-to-one. Keep measured evidence unchanged. - Web, dynamic, popular-site and document text use shared `score_attempt` orchestration and one task-result predicate. Task-specific requirements live in case configuration, not provider- or track-specific scoring branches. Binary acceptance and text diagnostics remain separate; missing reference coverage is unavailable, not zero or perfect. - Dynamic-content reference capture follows `harness/web_fetch/README.md`: retain paired initial HTTP bytes and independently rendered visible text, bound completion and the full declared scope. Distinguish embedded state from follow-up fetching; never require JavaScript as the only valid route. - Popular-site stable text must be independently observed and task-relevant. Do not freeze volatile ranks, prices, member counts or current list membership. Title diagnostics join returned entities to independent names by detail URL and never substitute for body quality, recall, freshness or task acceptance. @@ -57,5 +58,5 @@ These requirements apply to **all** report tracks; browser-specific rules below - Split environment preparation, browser-session setup, Agent run, BrowserTool wall time, evidence finalization and evaluator time at collection. Publish the self-hosted deployment topology and image identity; claim per-execution reset/readiness only when a preparation receipt binds reset, readiness and application health. - Formal report prose is reader-facing: no prompts, reasoning, internal work narration, conversational rebuttals or references to parallel sessions. Review pages contain review rounds and corrections, not per-case audit tables; release notes contain reader-visible effects. - English routes may preserve measured task names, IDs and evidence verbatim, but all report-authored interface and explanatory prose must be localized. Release checks must reject known untranslated interface markers and untranslated accessibility attributes. -- Before release, inspect every core route class in Chinese and English at desktop and narrow widths in both themes. Prevent vertical table labels, clipped controls and unreadable evidence; use ImageGen raster assets or Mermaid rather than SVG for Browser Use diagrams. +- Before release, inspect every core route class in each explicitly generated language at desktop and narrow widths in both themes, plus the English not-generated notice when applicable. Prevent vertical table labels, clipped controls and unreadable evidence; use ImageGen raster assets or Mermaid rather than SVG for Browser Use diagrams. - Keep routine report checks fast and deterministic; they cover structure, evidence, links, terminology, and static readability guards without launching a browser. Run the separate full release check before packaging or publishing; it adds the complete cross-language, cross-theme, desktop and narrow-width visual gate. diff --git a/Makefile b/Makefile index 77816bef3..fabc491b4 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ ci: naming-check test fixture-smoke report: runtime-check @test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path is required' >&2; exit 2) - $(PY) scripts/build_report.py --evidence "$(EVIDENCE)" --output reports/webfetch + $(PY) scripts/build_report.py --evidence "$(EVIDENCE)" --output reports/webfetch $(if $(filter 1,$(ENGLISH)),--include-english,) report-check: runtime-check $(PY) scripts/check_report.py reports/webfetch @@ -54,7 +54,7 @@ wcxb-release-check: runtime-check browser-use-report: runtime-check @test -n "$(EVIDENCE)" || (echo 'EVIDENCE="/absolute/path [/absolute/path...]" is required' >&2; exit 2) - $(PY) scripts/build_browser_use_report.py $(foreach path,$(EVIDENCE),--evidence "$(path)") --output reports/browser-use + $(PY) scripts/build_browser_use_report.py $(foreach path,$(EVIDENCE),--evidence "$(path)") --output reports/browser-use $(if $(filter 1,$(ENGLISH)),--include-english,) browser-use-review: runtime-check @test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path is required' >&2; exit 2) @@ -73,7 +73,7 @@ browser-use-release-check: browser-use-report-check trajectory-replay-report: runtime-check @test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path is required' >&2; exit 2) - $(PY) scripts/build_webarena_replay_report.py --results "$(EVIDENCE)" + $(PY) scripts/build_webarena_replay_report.py --results "$(EVIDENCE)" $(if $(filter 1,$(ENGLISH)),--include-english,) trajectory-replay-report-check: runtime-check $(PY) scripts/check_webarena_replay_report.py reports/trajectory-replay @@ -106,7 +106,7 @@ resource-image: public-benchmark-report: runtime-check @test -n "$(EVIDENCE)" || (echo 'EVIDENCE=/absolute/path/to/reviewed-public-benchmark-evidence is required' >&2; exit 2) - $(PY) scripts/build_report.py --evidence "$(EVIDENCE)" --output reports/webfetch-public-benchmarks + $(PY) scripts/build_report.py --evidence "$(EVIDENCE)" --output reports/webfetch-public-benchmarks $(if $(filter 1,$(ENGLISH)),--include-english,) $(PY) scripts/check_report.py reports/webfetch-public-benchmarks $(PY) scripts/check_report_links.py reports/webfetch-public-benchmarks diff --git a/evaluation/browser-use/current-report.json b/evaluation/browser-use/current-report.json index 294992029..7ff100772 100644 --- a/evaluation/browser-use/current-report.json +++ b/evaluation/browser-use/current-report.json @@ -35,6 +35,10 @@ "source_path": "research/browser-use-failure-cause-review.json", "source_sha256": "a11134919f5e7b9a65c601374211e20e7e940f82e6f363ed025c49b52336e721" }, + "trajectory_replay_bridge": { + "source_path": "research/browser-use-trajectory-replay-bridge.json", + "source_sha256": "731c04735e6b58d271d4ed5d43514300060c3ecfca0f54d7bc826c1300dd2ac5" + }, "publication": { "base_url": "https://artifact-hub.app.lexmount.com", "site_slug": "7l77EeXPRfml", diff --git a/evaluation/trajectory-replay/webarena-verified/README.md b/evaluation/trajectory-replay/webarena-verified/README.md index a724d207f..23b706ed3 100644 --- a/evaluation/trajectory-replay/webarena-verified/README.md +++ b/evaluation/trajectory-replay/webarena-verified/README.md @@ -1,10 +1,26 @@ # WebArena-Verified verified trajectories -`trajectories.json` is the compact, versioned trajectory portion of the benchmark dataset. -Each trajectory is one verified feasible BrowserTool path, not the task's only correct path. +`current-run.json.dataset.path` and its SHA-256 identify the authoritative trajectory +input for a formal build; currently this is `trajectories-corrected.json`. +`trajectories.json` is the retained predecessor used by qualification and correction +provenance. It is not a second current dataset. Each trajectory is one verified feasible +BrowserTool path, not the task's only correct path. + +WebArena task definitions and shared evaluator fixes belong in the +`lexmount/webarena-verified` fork. BrowserTool actions, selector compilation, local +qualification and method comparisons belong here: the upstream task dataset does not +define those browser-tool trajectories. Historical correction catalogs in `revisions/` +are immutable inputs bound by retained cohorts, not copies of report output; removing +them must not make historical contracts impossible to resolve. + +Authored report content and illustrations live outside Git. The report command takes +`--content` (default `$BROWSER_EVAL_DATA_HOME/report-assets/trajectory-replay`, with the +data home defaulting to `~/.browser-eval`). `research/trajectory-replay-content.json` +binds every consumed file. The prior Git revision retains migration recovery copies; +published report versions and physical execution evidence remain unchanged. `current-run.json` is the run authority for the next formal replay. It freezes the same 90-task -selection used by the current Browser Use evaluation, the two registered browser methods, resource +selection used by the current Browser Use evaluation, the registered browser methods, resource limits, and the isolation schedule. The private ready-to-run seed manifest must contain exactly one canonical candidate for each of those 90 tasks. The current inventory is assembled from 60 clean autonomous successes and 30 reviewed repairs; successful executions from different methods are @@ -34,23 +50,33 @@ candidate evidence, not additional tasks. `BROWSER_EVAL_DATA_HOME`; this directory stores deterministic actions and provenance hashes. - The current self-hosted WebArena deployment has one fixed-port site stack, so a formal suite owns the physical environment exclusively. It runs every task for one browser before starting the next - browser, restores the official fixture before every task, and tears down both the browser and site - containers between browsers. Parallel methods are valid only after separate site stacks also have - separate containers, data stores, ports, gateways, and lock namespaces; multiple browser + browser, restores the site stack once before each browser method, and tears down the browser and site + containers between methods. It does not reset the site between tasks. Parallel methods are valid + only after separate site stacks also have separate containers, data stores, ports, gateways, and + lock namespaces; multiple browser containers against one site stack are not independent environments. -- Formal execution requires clean browser-eval and CowAgent checkouts and binds their commits, the - resolved browser and site image identities, and the frozen CPU, memory, and sampling limits into - the resumable receipt. A resume with different runtime identity is rejected. Browser storage, - cache, cookies, permissions, service workers, and IndexedDB are cleared before every task; failed - teardown aborts the suite. Any pre-action context recovery writes a new physical-evidence - directory instead of overwriting the first one. +- Formal execution binds each method to frozen actions, a browser-binary hash, the site map, and a + method-scoped site reset receipt. A fresh BrowserTool context is created for each task; browser + storage does not carry between tasks, while site data is not reset between tasks. Every physical + execution and its evidence is retained separately. - More than one trajectory may be retained for a task when paths are materially different and independently satisfy the same promotion gate. -Run the frozen suite with `scripts/run_webarena_trajectory_replay.py`. Diagnostic subsets are -explicitly non-promotable. `--qualification-engine chromium` runs the complete frozen task set on -the reference browser and is the only scope that may promote the standard trajectory dataset. -After all 90 qualify, promote that receipt with `scripts/promote_webarena_trajectories.py`, then run -the measured method matrix with `--qualified-dataset` and no scope option. The runner rejects formal -execution when that dataset is missing, incomplete, or differs from the frozen profile, seeds, or -canonical actions. Never edit a verdict or verification claim by hand. +Freeze each method's exact 90-task inputs with `scripts/freeze_webarena_three_method_inputs.py`, +run each complete method cohort with `scripts/run_remote_three_method_cohort.py`, and bind the +certified cohorts with `scripts/aggregate_webarena_three_method_cohorts.py`. Each method +receives one site reset before its first task. The runner rejects a changed binary, task set, +action sequence, or reset policy. Diagnostic subsets never enter the formal matrix. Never edit a +verdict or verification claim by hand. + +The report presents every registered method with its recorded protocol. Diagnostic-only cohorts cannot +enter its score-bearing matrix or add a separate result appendix. + +Adding a browser version appends a method and its complete cohort; existing results remain +unchanged. Historical failure diagnostics may retain their original cohort binding through +`inherited_result_cohorts` in the review record. Each inherited cohort must be hash-bound, use +the same dataset, and have every logical result preserved exactly in the expanded matrix. +When appended cohorts use a corrected evaluator, declare the exact method-specific correction ID +and corrected task hash under `acceptance_revisions`. The task page retains both acceptance +versions, and measurement notes identify the affected task IDs. Identical frozen trajectory +hashes do not prove identical evaluator revisions. Historical verdicts are not rewritten. diff --git a/evaluation/trajectory-replay/webarena-verified/cohort-diagnostic-variant.json b/evaluation/trajectory-replay/webarena-verified/cohort-diagnostic-variant.json new file mode 100644 index 000000000..d5b776005 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/cohort-diagnostic-variant.json @@ -0,0 +1,19 @@ +{ + "schema": "webarena_original36_diagnostic_variant/1", + "promotable": false, + "overrides": [ + { + "task_id": 108, + "kind": "retrieval_candidate", + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-ajax-proposal.json", + "sha256": "aae7171e90cb3f309400fa7c5289ae3e6a13254c8c075b0cf591f6468e81514a" + }, + { + "task_id": 464, + "kind": "omit_trailing_diagnostic_observer", + "original_action_count": 8, + "trailing_action_sha256": "b4075840ad089ef703a88aa60ee799a3e5d876bd371358e168dee370ca7b89e7", + "action_count_after": 7 + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/cohort-step-roles.json b/evaluation/trajectory-replay/webarena-verified/cohort-step-roles.json new file mode 100644 index 000000000..9ae31dde0 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/cohort-step-roles.json @@ -0,0 +1,100 @@ +{ + "schema": "webarena_qualified_cohort_step_roles/1", + "site_readiness": [ + { + "site": "reddit", + "path": "/f/DIY/119019/how-can-i-bring-an-hdmi-cable-from-my-pc-downstairs-to-my-tv", + "max_stable_ms": 5000 + }, + { + "site": "gitlab", + "path": "/primer/design/-/merge_requests/450", + "max_stable_ms": 5000 + } + ], + "candidate_overrides": [ + { + "task_id": 66, + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66-budget.json", + "sha256": "8066148fbbc8acd2d30e4488f384b053dbed7988a3605c28e5c6dd5033ded206", + "reason": "allow the declared ten sequential post fetches to complete within the BrowserTool action budget" + }, + { + "task_id": 108, + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-stabilized.json", + "sha256": "b45982774cf2e89f2ae1d6df56024501185efa6a6f420aa6fda731b67ed446af", + "action_count": 16, + "stabilization_parent_sha256": "aae7171e90cb3f309400fa7c5289ae3e6a13254c8c075b0cf591f6468e81514a", + "paired_qualification": { + "chromium_receipt_sha256": "0305006ef4c3da5d5fd095e20f8c1fab746379876f3402a094d47681e147ab61", + "moli_receipt_sha256": "71c768f715d400af242801af069d35acac043e703763e296589c21d00f147eab" + }, + "reason": "commit both date-filter inputs and await the matching Magento UI provider state before reading the full order grid" + }, + { + "task_id": 163, + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-163.json", + "sha256": "785e3827b17ac0cf8ae428cbf1343c5df7ddba5eafb7505a281e11cfab8d5b6d", + "reason": "replace local-Docker origin guard with same-origin review endpoint validation" + }, + { + "task_id": 243, + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-243.json", + "sha256": "6bbd9124f65221a096d90127483f523076046dc936d24cd680768d48640597f5", + "action_count": 9, + "reason": "read the nickname on the unique lowest-rated product review from the complete current review population, without inferring a customer email" + }, + { + "task_id": 313, + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-313.json", + "sha256": "2ed9685a48cbd674e424df631f5133fdea800b7ada08dc3e60517f080838ad08", + "paired_qualification": { + "chromium_receipt_sha256": "2ac325728dcf07cb68e6573c7542128cd01d588b819582ff1823066b83e6d82e", + "moli_receipt_sha256": "983fc521bffbbf4439d129cfa8078665f33f2c6ef9ff20b6a5a9e7a8c1b19eac" + }, + "reason": "bind the retrieval contract to its source actions, including the same-site HTTP/HTTPS link selector, instead of the older remote absolute-link trace" + }, + { + "task_id": 404, + "path": "evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-404.json", + "sha256": "33338a5dd27b991e73f1166bc394661f199420631ef7701a390e305ec5e6f93e", + "reason": "wait for the newly opened forum page to finish loading its Stimulus vote controller and bind the click to post 124260" + }, + { + "task_id": 464, + "path": "evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-464.json", + "sha256": "06dc0afaa7d3a2aef1c5e6777520069dc7add2b73e61825ce98692cd5d093746", + "reason": "submit the Magento product form through its visible Save control and await the completion message instead of an arbitrary short delay" + }, + { + "task_id": 714, + "path": "evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-714.json", + "sha256": "fe4386049a47f14c22b67ac3fe285b4daf83750efd12bbec23b0964349352767", + "reason": "stabilize the time-sorted page and bind the vote click to post 19459 rather than a page-wide button ordinal" + } + ], + "exceptions": [ + { + "task_id": 464, + "trace_index": "before-1", + "role": "readiness", + "before_action": 1 + }, + { + "task_id": 464, + "trace_index": 8, + "role": "diagnostic_post_observer" + }, + { + "task_id": 465, + "trace_index": "4a", + "role": "readiness", + "before_action": 5 + }, + { + "task_id": 465, + "trace_index": 9, + "role": "diagnostic_post_observer" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/current-run.json b/evaluation/trajectory-replay/webarena-verified/current-run.json index 16d1c274a..98b536323 100644 --- a/evaluation/trajectory-replay/webarena-verified/current-run.json +++ b/evaluation/trajectory-replay/webarena-verified/current-run.json @@ -1,29 +1,57 @@ { "schema": "lex_webarena_trajectory_replay_profile/1", "benchmark": "WebArena-Verified", + "measurement_notes": { + "zh": "原三组结果保留，两组 1.1.9 方法为后续追加；新 PR 使用独立重编译的 release 构建。各版本没有在同一时间窗口内逐题交替运行，且 1.1.9 正式版开头存在本机编译和其他任务并行。耗时统计描述实际调用，不用于推断源码提速。 第 448、510 题的新两组分别恢复最后一次保存检查和商品数量检查；旧三组保留历史验收与成绩。这两题的规则版本在任务详情中分别列出，不作为纯浏览器差异。", + "en": "The original three cohorts are retained and two 1.1.9 methods are appended. The new PR uses a separately rebuilt release binary. Builds were not alternated per task in one time window, and the initial official 1.1.9 calls overlapped local compilation and other work. Timings describe observed calls, not code-induced speedup. For tasks 448 and 510, the appended methods restore final-save and item-quantity checks respectively. Original cohorts retain historical acceptance and scores. Task pages list both rule versions; these two tasks are not pure browser comparisons." + }, "benchmark_version": "1.2.3", "task_selection": { - "path": "evaluation/browser-use/webarena-verified-pilot.json", - "sha256": "2cb716552e36df7535e83ed6d2cf3f84283e3aca86072817cd6e039a38547c7b", + "path": "evaluation/trajectory-replay/webarena-verified/selection.json", + "sha256": "06d8d999164c7c2de6cb21d15ec12f5260bfc2a58ca81b098a8c7779d4d2d91a", "task_count": 90 }, "seed_inventory": { "base_kind": "successful Browser Use executions", "expert_repairs_path": "evaluation/trajectory-replay/webarena-verified/expert-repairs.json", - "expert_repairs_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "expert_repairs_sha256": "4eb4692ff8aefc39d3c7345e21f819fd9248a9b6ebe2e70c1a4c878a2d911a02", "required_ready_tasks": 90, "canonical_trajectories_per_task": 1 }, + "dataset": { + "path": "evaluation/trajectory-replay/webarena-verified/trajectories-corrected.json", + "sha256": "624a0ea83c6e4c3533714e961656d7e4159189a9e4941017b06a5299e4dc31d5" + }, "methods": [ { "id": "chromium", "engine": "chromium", - "label": "Chromium" + "label": "Chromium", + "binary_sha256": "d999cf00c6244a84279785c48eb78f97764a8a13ea3a1a7cc5030bf2f4ca481e" + }, + { + "id": "moli-1.1.8", + "engine": "moli", + "label": "Moli 1.1.8", + "binary_sha256": "522f752f30397f3b77910736c4ff76f2bdf4cd1a62a2bd14332bbccc97ce26dc" + }, + { + "id": "moli-1.1.8-pr-39d75b69", + "engine": "moli", + "label": "Moli 1.1.8 · PR 640 (39d75b69)", + "binary_sha256": "1c4dbbe67e4b97c903b3916fe65f8c5676c21810a1f4a9095cf6dfbd9d9286d4" + }, + { + "id": "moli-1.1.9", + "engine": "moli", + "label": "Moli 1.1.9", + "binary_sha256": "79699e50476118a7a5d1a36775f52d4106801fd5bc803727fd317501faa840cd" }, { - "id": "moli-1.1.3", + "id": "moli-1.1.9-pr-f315ff26", "engine": "moli", - "label": "Moli 1.1.3" + "label": "Moli 1.1.9 · PR 640 (f315ff26)", + "binary_sha256": "9b3f3dd6e550a71cddfb93ec3490317b0dfa5db407ebc2d5fb0a9748f58dd226" } ], "qualification": { @@ -32,8 +60,9 @@ }, "runs_per_task": 1, "resources": { - "cpus": 2.0, - "memory": "2g", + "browser_host": "macOS arm64 host", + "browser_cpu_limit": null, + "browser_memory_limit": null, "sample_interval_ms": 100 }, "execution": { @@ -41,25 +70,66 @@ "method_parallelism": 1, "task_parallelism": 1, "schedule": "complete one method before starting the next", - "reset_before_every_task": true, + "reset_before_every_task": false, + "reset_before_every_method": true, "teardown_between_methods": true, + "retry_failed_task": false, "trusted_source_site_hosts": [ "192.168.5.2", "host.docker.internal" ], - "site_stack": "one fixed-port self-hosted WebArena environment" + "site_stack": "one self-hosted Rainbond WebArena app slice, restored once before each browser method" }, "scoring": { "authority": "official programmatic evaluator or a hash-bound replay correction", + "require_all_reference_validity": true, "promotion_gate": "the canonical trajectory passes execution and official evaluation on the frozen reference browser before any measured browser comparison", "model_calls": 0 }, "publication": { - "base_url": "https://artifact-hub.app.lexmount.com", - "title": "WebArena-Verified 浏览器轨迹重放评测", - "state": "verified", - "site_slug": "7dF7uNK5SKIx", - "share_url": "https://artifact-hub.app.lexmount.com/v/hkZt38QFBpNIKtThKnqjpnBQk8At19PM", - "expected_version": "ver_fcd443ac19fe403f8baa3442262246e1" + "base_url": "https://artifact-site.lexmount.cn", + "title": "浏览器轨迹重放：Chromium 与 Moli 五种方法对比（90 题）", + "site_slug": "KoYm0UCog5th", + "share_url": "https://artifact-site.lexmount.cn/v/coY1pMVPbD6Z8cVLaUi_ZbpSnalfKpxY", + "expected_version": "ver_e40a44044966438192bea4cb1dc96e56", + "state": "verified" + }, + "acceptance_revisions": { + "448": { + "chromium": { + "correction_id": "wv-448-profile-save-success-variants-20260918", + "corrected_task_sha256": "f6330b0eef45ab950be13b2db63f51ee3eab99560ab2527cde22131440959f10" + }, + "moli-1.1.8-pr-39d75b69": { + "correction_id": "wv-448-profile-save-success-variants-20260918", + "corrected_task_sha256": "f6330b0eef45ab950be13b2db63f51ee3eab99560ab2527cde22131440959f10" + }, + "moli-1.1.9-pr-f315ff26": { + "correction_id": "wv-448-final-profile-save-success-variants-20260920", + "corrected_task_sha256": "6543e072d795f6446d2393d1e79a6075c2e06e600c73c418acb931dad3cc4f9e" + } + }, + "510": { + "chromium": { + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "corrected_task_sha256": "8b59be6dc204f64a6a3d92aff6f780fb91508894c9b75586d2fa79202e2c86cd" + }, + "moli-1.1.8": { + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "corrected_task_sha256": "8b59be6dc204f64a6a3d92aff6f780fb91508894c9b75586d2fa79202e2c86cd" + }, + "moli-1.1.8-pr-39d75b69": { + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "corrected_task_sha256": "8b59be6dc204f64a6a3d92aff6f780fb91508894c9b75586d2fa79202e2c86cd" + }, + "moli-1.1.9": { + "correction_id": "wv-510-restore-cart-quantity-contract-20260920", + "corrected_task_sha256": "f94a3014dbda8cd6d5ffafe5816a3f249df6a9ea7b00a027c9e2710eb0af2a96" + }, + "moli-1.1.9-pr-f315ff26": { + "correction_id": "wv-510-restore-cart-quantity-contract-20260920", + "corrected_task_sha256": "f94a3014dbda8cd6d5ffafe5816a3f249df6a9ea7b00a027c9e2710eb0af2a96" + } + } } } diff --git a/evaluation/trajectory-replay/webarena-verified/dataset-67-225-243-corrections.json b/evaluation/trajectory-replay/webarena-verified/dataset-67-225-243-corrections.json new file mode 100644 index 000000000..05dcca346 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/dataset-67-225-243-corrections.json @@ -0,0 +1,260 @@ +{ + "schema": "lex_webarena_replay_task_corrections/1", + "scope": "Browser Trajectory Replay standard-trajectory validation only", + "historical_browser_use_scores_immutable": true, + "upstream": { + "benchmark": "WebArena-Verified", + "package_version": "1.2.3", + "dataset_sha256": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30" + }, + "corrections": [ + { + "task_id": 225, + "official_revision": 2, + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + }, + { + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." + }, + { + "op": "replace", + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + }, + { + "op": "replace", + "path": "/instantiation_dict", + "value": { + "brand_text": "Sephora", + "product_text": "Brush" + } + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-reference-title-drift-20260911", + "rationale": "The current seeded catalog title contains two closing parentheses; the frozen reference contains one.", + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + } + ] + }, + { + "task_id": 448, + "official_revision": 2, + "correction_id": "wv-448-profile-save-success-variants-20260918", + "rationale": "GitLab can persist this profile form through either a 302 redirect or a 200 JSON success response. Match the submitted website URL before event recency and require an explicit success response for either form.", + "root_cause": "A single expected HTTP 302 falsely rejects a successful 200 JSON profile update when the redirect request is canceled; an unqualified 200 would also admit validation errors.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__GITLAB__/-/profile" + }, + { + "path": "/eval/1/expected/post_data/user[website_url]", + "equals": "https://egg.tart.com" + }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, + { + "path": "/eval/1/last_event_only", + "equals": true + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/last_event_only", + "value": false + } + ], + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] + } + ] + }, + { + "task_id": 510, + "official_revision": 2, + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "rationale": "The prose combines rating and price without an ordering rule, and the cart-totals response body is not available through every browser evidence interface. Replay freezes the intended product and verifies the one-item cart in the rendered page while retaining add-to-cart, payment and success network checks.", + "preconditions": [ + { + "path": "/intent", + "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + }, + { + "path": "/eval/1/expected/post_data/product", + "equals": "75640" + }, + { + "path": "/eval/2/expected/response_content/items_qty", + "equals": 1 + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." + }, + { + "op": "add", + "path": "/instantiation_dict/selection_policy", + "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" + }, + { + "op": "add", + "path": "/instantiation_dict/target_product_id", + "value": "75640" + }, + { + "op": "remove", + "path": "/eval/2" + } + ] + }, + { + "task_id": 528, + "official_revision": 2, + "correction_id": "wv-528-dom-state-without-submit-20260911", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 547, + "official_revision": 2, + "correction_id": "wv-547-existing-brown-option-20260912", + "rationale": "Brown already exists as global color option 51 in the seeded fixture. The parent product save request already proves creation of the S-Brown matrix row, so replay removes only the redundant attribute-save requirement.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93" + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ] + }, + { + "task_id": 699, + "official_revision": 3, + "correction_id": "wv-699-all-customer-groups-20260918", + "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", + "preconditions": [ + { + "path": "/eval/1/expected/post_data/customer_group_ids[0]", + "equals": "1" + }, + { + "path": "/eval/1/post_data_schema", + "equals": null + } + ], + "operations": [ + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[1]", + "value": "2" + }, + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[2]", + "value": "3" + } + ] + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/expert-repairs.json b/evaluation/trajectory-replay/webarena-verified/expert-repairs.json index b460bf76a..c88302534 100644 --- a/evaluation/trajectory-replay/webarena-verified/expert-repairs.json +++ b/evaluation/trajectory-replay/webarena-verified/expert-repairs.json @@ -31,8 +31,7 @@ "task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": [ - "The Hobbit", - "A Christmas Carol" + "The Hobbit" ], "error_details": null } diff --git a/evaluation/trajectory-replay/webarena-verified/mutation-completion-candidates.json b/evaluation/trajectory-replay/webarena-verified/mutation-completion-candidates.json new file mode 100644 index 000000000..71860a785 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/mutation-completion-candidates.json @@ -0,0 +1,542 @@ +{ + "schema": "webarena_mutation_completion_candidates/1", + "purpose": "Diagnostic qualification only; does not replace frozen actions, official evaluator, or scores", + "observation_protocol": { + "start_after": "last frozen action", + "timeout_ms": 20000, + "poll_interval_ms": 100, + "method": "GET only, same authenticated origin, HTTP 200, no redirect, no retry of the mutating action", + "require_independent_official_evaluation": true, + "physical_executions_per_task": 1, + "score_effect": "none; diagnostic evidence only" + }, + "official_evaluator": { + "package": "webarena-verified", + "version": "1.2.3", + "invocation": "WebArenaVerified.evaluate_task(task_id, agent_response, network_trace)", + "agent_response": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "network_trace": "fresh same-execution HAR captured from before the trigger action through observer completion", + "required_score": 1.0 + }, + "rules": [ + { + "task_id": 399, + "action_count": 14, + "observer": { + "kind": "same_origin_get_html", + "path": "/user/MarvelsGrantMan136/edit_biography", + "selector": "textarea[name=\"user_biography[biography]\"]", + "property": "value", + "equals": "I am a robot", + "require_authenticated_user": "MarvelsGrantMan136" + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/user/MarvelsGrantMan136/edit_biography", + "post_data": {"user_biography[biography]": "I am a robot"}, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 404, + "action_count": 11, + "observer": { + "kind": "pending_live_dom_contract", + "path": "/f/books/new", + "target_post_id": 124260, + "reason": "The old snapshot omits vote-state attributes. POST JSON netScore=1 is a response, not a read-only persisted-state observation. Inspect the authenticated rendered post under the shared lock before specifying the exact state predicate." + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/sv/124260.json", + "post_data": {"choice": "1"} + }, + "qualification": "blocked until observer has a tested persisted-state predicate; official evaluator remains mandatory" + }, + { + "task_id": 415, + "action_count": 6, + "observer": { + "kind": "same_origin_get_json_discussions", + "path": "/byteblaze/a11y-webring.club/-/merge_requests/40/discussions.json?per_page=20", + "page_size": 20, + "require_not_full_page": true, + "noteable_id": 138783, + "noteable_type": "MergeRequest", + "noteable_iid": 40, + "author_username": "byteblaze", + "note_equals": "@davepgreene", + "require_exactly_one_matching_note": true + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/byteblaze/a11y-webring.club/notes?target_id=138783&target_type=merge_request", + "post_data": { + "$.note.noteable_type": "MergeRequest", + "$.note.note": "^.*@davepgreene.*$" + } + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 418, + "action_count": 8, + "observer": { + "kind": "same_origin_get_json", + "path": "/api/v4/user/status", + "json_key": "availability", + "equals": "busy", + "require_authenticated_user": "byteblaze" + }, + "official_network_event": { + "method": "PUT", + "url_suffix": "/api/v4/user/status", + "post_data": {"availability": "busy", "clear_status_after": null} + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 441, + "action_count": 16, + "pre_commit_observer": { + "kind": "current_dom_property", + "position_before_tool_call_index": 13, + "selector": "textarea[aria-label^=\"Editor content\"]", + "property": "value", + "expected_from_tool_call_index": 10, + "expected_text_sha256": "1685f452f0aa0e0430b8454c746ef7496b859121298490b60ce2ef2841d15b6f", + "compare": "exact text after CRLF normalization", + "read_only": true + }, + "observer": { + "kind": "same_origin_get_text", + "path": "/api/v4/projects/byteblaze%2Fgimmiethat.space/repository/files/index.html/raw?ref=main", + "body_kind": "repository_file_source", + "require_html_title": "GIVE ME SPACE", + "require_no_other_title": true, + "expected_from_tool_call_index": 10, + "compare": "exact text after CRLF normalization" + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/byteblaze/gimmiethat.space/-/update/main/index.html", + "post_data": { + "_method": "put", + "file_path": "index.html", + "branch_name": "main", + "original_branch": "main", + "content": "^<\\!doctype html>.*.*GIVE ME SPACE.*.*$" + }, + "response_status": 302 + }, + "qualification": "pre-commit observer, post-commit observer and official evaluator must all pass" + }, + { + "task_id": 448, + "action_count": 8, + "observer": { + "kind": "same_origin_get_json", + "path": "/api/v4/user", + "require_authenticated_user": "byteblaze", + "json_key": "website_url", + "equals": "https://egg.tart.com" + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/-/profile", + "post_data": {"user[website_url]": "https://egg.tart.com"}, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 465, + "action_count": 8, + "pre_action_readiness": { + "kind": "amd_deferred_ready", + "before_original_action": 5, + "module": "Magento_Customer/js/customer-data", + "deferred_method": "getInitCustomerData", + "timeout_ms": 15000 + }, + "observer": { + "kind": "same_origin_get_html", + "path": "/wishlist/index/index/wishlist_id/5/", + "href_origin": "response_url_origin", + "within": "li", + "require_authenticated_sign_out_path": "/customer/account/logout/", + "require_same_origin_sign_out_href": true, + "product_name": "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count", + "edit_href_path_pattern": "^/wishlist/index/configure/id/[0-9]+/product_id/71337/?$", + "require_same_origin_edit_href": true, + "require_product_name_and_edit_link_in_same_item": true, + "require_exactly_one_matching_item": true + }, + "observer_compiler": "same_origin_authenticated_item_html/1", + "observer_timeout_ms": 20000, + "expected_completion": "persisted_wishlist_item_observed", + "observer_basis": { + "historical_chromium_trace_sha256": "a5ea0a4d6b365ce221dafed1098c0748695c9bde39b1946de47928df36434938", + "historical_chromium_har_sha256": "20db92ae2d20944e60e20bf860345b1c3ee8a13d31ac8d2a7edec5cf172a47e5", + "historical_moli_har_sha256": "60d59efd6139020711a617bd9f3704034dd8be9d15e9fd94cefda27b0917fe2f", + "evidence": "Chromium final authenticated wishlist snapshot shows the named item with an Edit URL under /wishlist/index/configure/id/3/product_id/713..., while the same-run HAR confirms POST product=71337, wishlist GET 200, and a subsequent GET customer-section JSON item with product_id=71337 and the exact name. The snapshot URL is truncated, so the complete Edit URL remains a proposed live GET predicate, not a historical assertion. Formal Moli HAR has no wishlist add or wishlist page request." + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/wishlist/index/add/", + "post_data": {"product": "71337", "qty": null}, + "response_status": 302 + }, + "qualification": "same-run authenticated wishlist GET must expose exact product identity and official evaluator must pass; offline observer candidate is not a new result" + }, + { + "task_id": 506, + "action_count": 18, + "observer": { + "kind": "same_origin_get_order_history_detail", + "order_number_source": "current_success_page_text_only_as_lookup_key", + "order_number_pattern": "Your order number is:\\s*([0-9]+)", + "require_unique_order_number": true, + "history_path": "/sales/order/history/?limit=50", + "require_authenticated_sign_out_path": "/customer/account/logout/", + "require_same_origin_sign_out_href": true, + "history_detail_href_path_pattern": "^/sales/order/view/order_id/[0-9]+/$", + "require_same_origin_detail_href": true, + "require_exactly_one_history_row_for_order_number": true, + "detail_heading_prefix": "Order # ", + "ordered_item_name": "Beyond Meat Beef Beefy Crumble, 5 Pound -- 2 per case.", + "ordered_item_quantity": 1, + "require_exactly_one_matching_order_item": true, + "read_only_get_count": 2, + "target_product_id_for_official_cart_add_only": 22490 + }, + "observer_basis": { + "historical_chromium_506_trace_sha256": "1f76d49e8a80d27611ce3b7fc022c4dd53a9d8330f66c6cb65d287ed075b4937", + "historical_chromium_506_har_sha256": "3771aad8f66d38e91e2f0cd02ef7f85e0e0efc7efc7925acc9506a9e6e01cfa3", + "historical_order_history_trace_sha256": "53c0cc449ea0d4fdff00898ec061628e03d6de63c0404de0ef84c8605bdbec80", + "historical_order_detail_trace_sha256": "e7fd92654d552d51dee4e7c3b0fbdd6ea2bfbb237a824742bda7d1a29e191065", + "evidence": "Chromium task 506 ends on the purchase success page with order number 000000190; its HAR contains all four official events, including the exact cart item name, but no subsequent order-detail GET. Other authenticated Shopping traces establish that order history lists order numbers with View Order links to /sales/order/view/order_id// and detail pages expose an Order # heading and Items Ordered table. The order number is only a lookup key; the new same-run history and detail GETs must independently bind that number to the exact ordered item. This is a predeclared observer, not a historically qualified 506 order-detail result." + }, + "official_network_evaluator_count": 4, + "qualification": "current success order number, authenticated history GET, exact order-detail GET item and all four official network evaluators must pass; candidate requires live GET qualification" + }, + { + "task_id": 571, + "action_count": 13, + "observer": { + "kind": "same_origin_get_html", + "path": "/customer/address/index/", + "require_authenticated_customer": "Emma Lopez", + "address_selectors": [ + ".box-address-billing address", + ".box-address-shipping address" + ], + "address_lines": [ + "Emma Lopez", + "231 Willow Way", + "Suite 100", + "Chicago, Illinois, 60601", + "United States" + ] + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/customer/address/formPost/id/26/", + "post_data": { + "firstname": "Emma", + "lastname": "Lopez", + "street[0]": "231 Willow Way", + "street[1]": "Suite 100", + "country_id": "US", + "city": "Chicago", + "postcode": "60601" + }, + "response_status": 302 + }, + "qualification": "persisted address-book text and official evaluator must both pass; the edit form's region select is populated by JavaScript and is not a valid static-HTML persistence observer" + }, + { + "task_id": 576, + "action_count": 13, + "observer": { + "kind": "same_origin_get_json_pair", + "paths": [ + "/api/v4/projects/179/members/5", + "/api/v4/projects/179/members/1842" + ], + "require_user_ids": [5, 1842], + "require_access_level": 30, + "require_direct_project_membership": true + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/api/v4/projects/179/invitations", + "post_data": {"user_id": "5,1842", "access_level": 30}, + "response_status": 201 + }, + "qualification": "both direct membership records and official evaluator must pass" + }, + { + "task_id": 585, + "action_count": 11, + "observer": { + "kind": "pending_read_only_review_contract", + "target_product_id": 73063, + "reason": "Submitted reviews may await moderation. No stable authenticated read-only review record with product, nickname, title, text and five-star rating has been qualified; the old action-only HAR lacks the required POST." + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/review/product/post/id/73063/", + "post_data": { + "ratings[4]": "20", + "nickname": "Emma Lopez", + "title": "Good purchase", + "detail": "I like it" + }, + "response_status": 302 + }, + "qualification": "blocked until a read-only review-record identity check is qualified; official evaluator remains mandatory" + }, + { + "task_id": 590, + "action_count": 12, + "observer": { + "kind": "same_origin_get_json_collection", + "path": "/api/v4/projects/primer%2Fdesign/milestones?state=all&per_page=100", + "require_single_page": true, + "require_exactly_one_match": true, + "fields": { + "title": "product launch", + "start_date": "2023-01-16", + "due_date": "2023-01-30" + } + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/primer/design/-/milestones", + "post_data": { + "milestone[title]": "product launch", + "milestone[start_date]": "2023-01-16", + "milestone[due_date]": "2023-01-30" + }, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 595, + "action_count": 15, + "observer": { + "kind": "same_origin_get_html", + "path": "/f/space/69581/the-moon-saturn-and-jupiter-through-my-4-telescope-and", + "selector": "button.subscribe-button.subscribe-button--unsubscribe", + "require_exactly_one": true, + "require_authenticated_user": "MarvelsGrantMan136" + }, + "official_network_event": { + "method": "POST", + "url_suffix": ["/f/space/subscribe.json", "/f/space/unsubscribe.json"], + "referer_suffix": "/f/space/69581/the-moon-saturn-and-jupiter-through-my-4-telescope-and", + "response_content": {"subscribed": true} + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 464, + "action_count": 7, + "pre_action_readiness": { + "kind": "same_origin_magento_product_type", + "before_original_action": 1, + "path": "/admin/catalog/product/edit/id/1796/", + "product_id": 1796, + "expected_type": "configurable", + "timeout_ms": 15000 + }, + "observer": { + "kind": "same_origin_get_html", + "path": "/admin/catalog/product/edit/id/1796/", + "selector": "textarea[name=\"product[short_description]\"]", + "require_product_id": 1796, + "html_value": "

1 customer(s) love it!

", + "require_exactly_one": true + }, + "official_network_event": { + "method": "POST", + "url_pattern": "^__SHOPPING_ADMIN__/catalog/product/save/id/1796/type/configurable/store/0/set/\\d+/back/edit$", + "post_data": {"product[short_description]": "

1 customer(s) love it!

"}, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 620, + "action_count": 13, + "observer": { + "kind": "current_redirect_get_html", + "path_pattern": "^/f/relationship_advice/[0-9]+/[^/]+$", + "forum_href": "/f/relationship_advice", + "title": "deal with long-distance relationships", + "body": "Please help", + "author_href": "/user/MarvelsGrantMan136", + "require_unique_numeric_post_id": true + }, + "official_network_event": { + "method": "POST", + "url_suffix": ["/submit", "/submit/relationship_advice"], + "post_data": { + "submission[title]": "deal with long-distance relationships", + "submission[body]": "Please help", + "submission[forum]": "10075" + }, + "response_status": 302 + }, + "qualification": "fresh redirected post page and official evaluator must both pass" + }, + { + "task_id": 640, + "action_count": 13, + "observer": { + "kind": "current_redirect_get_html", + "path_pattern": "^/f/books/[0-9]+/[^/]+$", + "forum_href": "/f/books", + "title": "book reading Meet up!", + "body": "virtual meetup for book reading on March 15th", + "author_href": "/user/MarvelsGrantMan136", + "require_unique_numeric_post_id": true + }, + "official_network_event": { + "method": "POST", + "url_suffix": ["/submit", "/submit/books"], + "post_data": { + "submission[title]": "book reading Meet up!", + "submission[body]": "virtual meetup for book reading on March 15th", + "submission[forum]": "10037" + }, + "response_status": 302 + }, + "qualification": "fresh redirected post page and official evaluator must both pass" + }, + { + "task_id": 645, + "action_count": 12, + "observer": { + "kind": "current_redirect_get_html", + "path_pattern": "^/f/books/[0-9]+/[^/]+$", + "forum_href": "/f/books", + "title": "What could machine learning help the books field?", + "author_href": "/user/MarvelsGrantMan136", + "require_unique_numeric_post_id": true + }, + "official_network_event": { + "method": "POST", + "url_suffix": ["/submit", "/submit/books"], + "post_data": { + "submission[title]": "What could machine learning help the books field?", + "submission[forum]": "10037" + }, + "response_status": 302 + }, + "qualification": "fresh redirected post page and official evaluator must both pass" + }, + { + "task_id": 658, + "action_count": 13, + "observer": { + "kind": "same_origin_get_json_collection", + "path": "/api/v4/projects/a11yproject%2Fa11yproject.com/issues?search=401%20bad%20gateway&per_page=100", + "require_single_page": true, + "require_exactly_one_match": true, + "fields": {"title": "401 bad gateway", "due_date": "2030-12-31"}, + "assignee_id": 2264, + "require_project_path": "a11yproject/a11yproject.com" + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/a11yproject/a11yproject.com/-/issues", + "post_data": { + "issue[title]": "401 bad gateway", + "issue[assignee_ids][]": "2264", + "issue[due_date]": "2030-12-31" + }, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + }, + { + "task_id": 666, + "action_count": 20, + "input_gate": { + "kind": "recompiled_seed_differs_from_qualified_trajectory", + "tool_call_index": 7, + "qualified_selector": ":nth-match(a[href=\"#\"]:visible, 11)", + "recompiled_selector": "a[href=\"#\"]:visible:has-text(\"dialog-component\")", + "recompiled_seed_sha256": "fd756fa1801d1daa704f851979e29d618e0eebacecc77c6fda5c121bb736f86f" + }, + "observer": { + "kind": "pending_requalified_trajectory", + "proposed_read_only_path": "/api/v4/projects/180/merge_requests?state=all&source_branch=dialog-component&target_branch=dialog", + "proposed_required_reviewer_id": 2367, + "reason": "The latest recompiled seed changes frozen click index 7 from positional a[href=#] match 11 to the dialog-component text selector. The original qualified 20-action trajectory therefore is not byte-identical to the current seed and needs new Chromium qualification before a latest-seed Moli run." + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/primer/design/-/merge_requests", + "post_data": { + "merge_request[target_branch]": "dialog", + "merge_request[source_branch]": "dialog-component", + "merge_request[target_project_id]": "180", + "merge_request[source_project_id]": "180", + "merge_request[assignee_ids][]": "0", + "merge_request[reviewer_ids][]": "2367" + }, + "response_status": 302 + }, + "qualification": "blocked until latest-seed selector is independently Chromium-qualified; official evaluator remains mandatory" + }, + { + "task_id": 714, + "action_count": 14, + "observer": { + "kind": "pending_read_only_vote_state_contract", + "target_post_id": 19459, + "reason": "The old snapshot omits authenticated downvote-state attributes. A successful click or POST response does not independently prove the persisted vote; inspect the rendered state under the shared lock before defining a read-only predicate." + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/sv/19459.json", + "post_data": {"choice": "-1"} + }, + "qualification": "blocked until authenticated persisted downvote state can be read; official evaluator remains mandatory" + }, + { + "task_id": 731, + "action_count": 11, + "observer": { + "kind": "same_origin_get_html", + "path": "/f/MachineLearning/1/nvidia-rtx-4090", + "body_selector": "main article p", + "exclude_ancestor": "header", + "title": "Nvidia RTX 4090", + "body_exact_lines": ["Crazy device for ML!", "EDIT: This news aged well"], + "author_href": "/user/MarvelsGrantMan136", + "require_existing_post_id": 1 + }, + "official_network_event": { + "method": "POST", + "url_suffix": "/f/MachineLearning/1/-/edit", + "post_data": { + "submission[title]": "Nvidia RTX 4090", + "submission[body]": "^\\s*Crazy device for ML!\\s*EDIT: This news aged well\\s*$" + }, + "response_status": 302 + }, + "qualification": "observer and official evaluator must both pass" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/mutation-completion-review.json b/evaluation/trajectory-replay/webarena-verified/mutation-completion-review.json new file mode 100644 index 000000000..55d28b3d1 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/mutation-completion-review.json @@ -0,0 +1,27 @@ +{ + "schema": "webarena_mutation_completion_review/1", + "task_id": 441, + "evidence_class": "old non-promotable Moli diagnostic request, not a latest-source result", + "old_receipt_sha256": "e31f8b38ad2bdc8268d6823618eb3fadb6fe8ee1cee38f5f9abf26ba539b9839", + "old_network_har_sha256": "e1940a859f9bbe2dcc5fb801f011403ce91e71493544edd115368b3b1cc93aa8", + "old_moli_binary_sha256": "fa7779c6d01184a4dba950ba40c1f0794a64242b54aef27c8d3444c0c0de60f2", + "old_receipt_evaluator_status": "not_run", + "request_content_observation": { + "doctype_count": 2, + "head_end_count": 2, + "title_count": 2, + "target_title_present": true, + "single_document_predicate": false + }, + "offline_official_evaluator_check": { + "package": "webarena-verified", + "version": "1.2.3", + "response_input": "frozen task 441 MUTATE/SUCCESS response, not a new agent call", + "network_input": "old hash-bound HAR request", + "score": 1.0, + "status": "success", + "interpretation": "The official network-content regular expression accepts a request containing the target title inside duplicated HTML. It does not establish that the committed file is one valid document." + }, + "qualification_effect": "none; a latest-source run needs a fresh read-only main-branch file observation and its own official evaluator result", + "published_score_effect": "none" +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/README.md b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/README.md new file mode 100644 index 000000000..8def5aa04 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/README.md @@ -0,0 +1,24 @@ +# Current-retrieval contract candidates + +This directory contains bounded, non-score-bearing current-source observation candidates for +21 of the 37 WebArena-Verified retrieval tasks. `recompiled-seed-bindings.json` freezes each +task's canonical action count and digest from the recompiled 90-task seed. Preserving every +canonical action exactly is necessary for an observation-tail candidate, but not sufficient for +qualification. The public contract validator binds JSON observation steps and action hashes; +it does not prove that an `evaluate` script or navigation is side-effect-free. No candidate in +this directory claims qualified read-only tail semantics. Matching a final answer is not +evidence that the action boundary or observation scope holds. +`qualification-matrix.json` separately records prefix compatibility, contract validity, +current observation acceptance, diagnostic official evaluation, and remaining blockers. + +These definitions are **not** bound by `current-run.json`, are not a qualified 90-task dataset, +and do not revise the published replay scores. A diagnostic score of 1 is not a new formal +benchmark result. Task 108 has current-source acceptance without an official evaluator score. +The additional nine-task batch is blocked: 50, 94, 188, and 231 replace frozen actions; 198 +and 320 preserve the prefix but their offline observation scope remains unqualified; 77, 183, +and 386 lack an independent current-page observer. Task 243's former customer-email task was +replaced with a review-nickname task in the corrected fork; its new candidate is pending +current-browser qualification. Task 313 has a diagnostic-only current +browser observation of four declared support pages; its bounded `NOT_FOUND_ERROR` is not +full-site absence proof or a qualified benchmark result. No blocked task has a promoted +candidate definition here. diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/qualification-matrix.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/qualification-matrix.json new file mode 100644 index 000000000..7ff54c4fc --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/qualification-matrix.json @@ -0,0 +1,618 @@ +{ + "schema": "lex_replay_retrieval_candidate_matrix/1", + "score_bearing": false, + "profile_bound": false, + "official_scores_modified": false, + "source_protocol": "current public validate_contract", + "population_task_ids": [ + 29, + 50, + 66, + 77, + 94, + 108, + 163, + 173, + 183, + 188, + 196, + 198, + 204, + 231, + 243, + 313, + 314, + 320, + 349, + 386, + 784 + ], + "rows": [ + { + "task_id": 29, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-29.json", + "candidate_sha256": "eb3525342e22372a6241a7a19bfebfd3f491cc5b8ee55d95f4a4ddfdeec5d0fe", + "receipt_sha256": "fa2cca44ca7507c48c0ba675ee7327efa3ad46c712c4c2f4f1e4ba1cc491f06b", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Frozen canonical actions are preserved, but the observation tail lacks a production proof of read-only behavior and full reference-browser qualification.", + "contract_sha256": "30d64d49b174637d7657f3418f2e9ba31dc179aaaca08a1f0cdca2e438a47f4c", + "calls_sha256": "4de8202bcc49843884cdab163b4d2e3cdf3a679fb23f1756a0bdf2c093558488", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "49f4ca63a40b295c6ad566153530514a091c9a2e575abea6858f6223edd0cabe", + "task_intent_sha256": "3db3a1ce0087e7924c000225ab9c1278ef04b3747f13ae63beb986d496795795", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 8, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 50, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Offline observer changes a frozen canonical action before its observation tail; no current-page execution or qualification exists.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "b2cd211239d1d6b6c2ba180f35eeecfe8383a796abcac495e365be3ce4080a3f", + "task_intent_sha256": "9827622d68dc3914a6002a7df25c74a6eaf6509d9b25ee59bc47f15a6728d13a", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": "20208dc07a398e8f6150d53d0a0f9223a6c7f5fb9bc5d3815921bd11379f6cc2", + "proposed_contract_sha256": "e0bea497a1e7872d78212703b3147e3a135691352f7f34bb5b9fd683b2665e92", + "proposed_calls_sha256": "319690fd190b417356c0a9f32cde7d1b02dbab32745559e7cde1c1b8d36fd2af", + "diagnostic_stage": null, + "canonical_action_count": 5, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 1 + }, + { + "task_id": 66, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66.json", + "candidate_sha256": "352bc6f546170e8f7201b0667b101307e0be0a4bf087d24a3384624209bf29c2", + "receipt_sha256": "a2e61fb4d1143065e337e337251cd49984ad6ae98a4a992a83f731654aacb7bb", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Frozen canonical actions are preserved, but the observation tail lacks a production proof of read-only behavior and full reference-browser qualification.", + "contract_sha256": "4479b83e98a8df2026eed76ed5717f02da9a80dded5010f72bb86314f9d152b5", + "calls_sha256": "6f26e505ed350c7ab7b73cd46fb523002f362bbb8ccb9e20f7906096480bf661", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "5f6735f1058724eec0a2b9ff001e18cb137135fdf47237efebfbf6af7682eee9", + "task_intent_sha256": "253c889b8a843266be76fa90a7eb32ba2355c83f6e768dcfb7ec71e11d1df6b2", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 9, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "navigate", + "wait", + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 77, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "No independent current-page structured observation and completeness proof for this frozen task.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "12581dfb1d8e7eb8e8eb6f16c0d54fc006c1fcfe337a99c8a23a9a7913d23633", + "task_intent_sha256": "c58e624f93d0b19890e14b14c81fb95b2120c860b75381b4a0083fccfdeb8ea7", + "eligible_for_qualification": false, + "seed_binding": "no_observer", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": null, + "canonical_action_count": null, + "canonical_prefix_preserved": null, + "tail_action_types": [], + "first_differing_action": null + }, + { + "task_id": 94, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Offline observer changes a frozen canonical action before its observation tail; no current-page execution or qualification exists.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "0c6a0d744f3e8f4d9b5d87f4dbe3a68caef47a969847d541976344cfe60925d9", + "task_intent_sha256": "c70b55f52a6c4f4be4985026202c21419d893e133ccb91d7a04e621d95925112", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": "0080701793159ffa0651f8a104b181f585892e3c56486e21d84b64032d5dd2e6", + "proposed_contract_sha256": "2ad6892084595e757b37ecbf0e1272662d73f12accf323411653b0d98ad83aba", + "proposed_calls_sha256": "372eaa6b861dba1849b68f83c63172c2c12ab0aaa4ab82c6278c6455ec118b6a", + "diagnostic_stage": null, + "canonical_action_count": 7, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 1 + }, + { + "task_id": 108, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-ajax-proposal.json", + "candidate_sha256": "aae7171e90cb3f309400fa7c5289ae3e6a13254c8c075b0cf591f6468e81514a", + "receipt_sha256": null, + "historical_receipt": { + "candidate_sha256": "eb6d02e4d96615e79fc06f88c541cc4336b88bc92cc502c14d9f2e79946f75bb", + "contract_sha256": "190d01a803fdf43b6c539a4444cafbbb3ca260750f40ac20597a6b7ba695a515", + "calls_sha256": "3427704c619e57d680343f31befd35864d22f0927a57d4067a1634bf0b5468fe", + "receipt_sha256": "132a1766dcd2c77bb268969ab8f3720a4c7f539a6b575f3118c00318a49b261f" + }, + "previous_affected_task_qualification": { + "plan_sha256": "c8a13f13dacfeccfd3e90d663438af77391c685433de1e0b9896f5a694a4dfd2", + "candidate_sha256": "13a7da24df97febfa7d08708c4178f0f4ff67b143f342516a4cbc57ee5bc8b79", + "chromium_receipt_sha256": "980454b7526c0b86df5ddd9d9d41cfe069e52107fc5df71a4339192844c26dac", + "moli_receipt_sha256": "f62e09b00d679861588ab23a02d1de085654c5a1726d0f9f3bf783ad459a3097" + }, + "affected_task_qualification": { + "chromium_plan_sha256": "24a4ee5b1961b60be34384a51203280d025c37067ee71792b5bfc10a8eb89d5d", + "moli_plan_sha256": "ac46cfbbd0f065b54bea1540bc2e25a9ce53b7f31ed037052c9d1573e2b06997", + "candidate_sha256": "aae7171e90cb3f309400fa7c5289ae3e6a13254c8c075b0cf591f6468e81514a", + "chromium_receipt_sha256": "0305006ef4c3da5d5fd095e20f8c1fab746379876f3402a094d47681e147ab61", + "moli_receipt_sha256": "71c768f715d400af242801af069d35acac043e703763e296589c21d00f147eab", + "executed_actions_sha256": "9a090def66045e40d31d647fd817bc9df0d3bafe61dc7da0569391542d306430", + "site_map_sha256": "78c6c9498a07db4cd2b0b1ea3e573ad2ab92c4a9275b24c6dca054a641221ced", + "action_count_per_browser": 16, + "retrieval_status": "supported", + "chromium_official_score": 1, + "moli_official_score": 1 + }, + "level": "diagnostic_only", + "official_score": null, + "blocker": "The request-corrected 16-action input passed current-run qualification in both browsers. Historical benchmark scores remain unchanged; formal inclusion requires a preregistered full-scope rerun.", + "contract_sha256": "6c963101d3677aa958b32df199bc85bb16144e04f7b370f56d81936e42119439", + "calls_sha256": "232669e9374158b513a061bb6cc0c36e3e6d695996282ffe78a78a81d471861c", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "e19497a8b775e0d55adbb9bbc2e1cbfb5815e58dd3df56c39c3a482be1e7d561", + "task_intent_sha256": "2a3337dd851523f621e25a91fc77f5d5641d60cdd6268154880a6d7be40ebdb9", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "affected_task_qualified", + "canonical_action_count": 14, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 2 + }, + { + "task_id": 163, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-163.json", + "candidate_sha256": "785e3827b17ac0cf8ae428cbf1343c5df7ddba5eafb7505a281e11cfab8d5b6d", + "receipt_sha256": "11d8bf7fc1d40c2c0c0cbef1c171e29bde139c6b066701fc3f1aacead9e6f2bc", + "historical_receipt": { + "candidate_sha256": "02d7fed6ca030f0c2430d71905e8089b6ef2d4c7bf04fe7f531be6c71529b9b6", + "receipt_sha256": "0b3236ae8870cb91c539570e20ca362e432d07194d9238b0441d6485d82493f1" + }, + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Same-origin portable candidate passed one remote Moli diagnostic, but a newly declared complete cohort with reference checks is still required.", + "contract_sha256": "61bc899f9e580df5098954bb692c6af27b3a5315b259f7c5b95cc3610d8d0559", + "calls_sha256": "ca81826179ccce875f9d18637d6e67949b0c5a7cbaaf31c706371eb50fb28b59", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "97ea26eae2ae20621b4c64fb0c4923f4234ea2ee01f44a37a0bb185fa1b44a29", + "task_intent_sha256": "d9ff84a6324844e40aaa88a08713fa2a3dc30f57346ee43842f7b8802efcb705", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 5, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 1 + }, + { + "task_id": 173, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-173.json", + "candidate_sha256": "ada5dd80d638b05498d7ace62f7367520efbcb5805c0cba72837ae293ca75123", + "receipt_sha256": "20f62faf18d914cb05a809a83b402953428434ecee7fd6937521e623a4542bb2", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Diagnostic candidate changes a frozen canonical action before its observation tail; requalification on a newly declared trajectory is required.", + "contract_sha256": "09afa9e84b0cf8a0f97c4ec722486ad12f8955571f4a97083a7bbfad1599fd25", + "calls_sha256": "9f278b55f0847c5a26970ea6cdcfdee81a8c4791986585abcde26344089c3dd5", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "82051decd9ba5e3edc5b29845f23e7d30fb696fb079915df35a5990a9d667809", + "task_intent_sha256": "7d982bcf9dc1de2a2ef073bc424741a44d88b7063f6c9dbc25c8a9095b9bf063", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 8, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 4 + }, + { + "task_id": 183, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "No independent current-page structured observation and completeness proof for this frozen task.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "35950dde0d474d233f1656853e850d3e27093e0cbf8359aa99d27c105c823fd8", + "task_intent_sha256": "99c9c7d13ca2b4167c1fb84d5e2a685857843d1bcca8ea38b61d210c9c1de57b", + "eligible_for_qualification": false, + "seed_binding": "no_observer", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": null, + "canonical_action_count": null, + "canonical_prefix_preserved": null, + "tail_action_types": [], + "first_differing_action": null + }, + { + "task_id": 188, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Offline observer changes a frozen canonical action before its observation tail; no current-page execution or qualification exists.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "504cb09218b406ff19a0586cbe57742e3ac6e4b41383ea68e7082f61a15bec8b", + "task_intent_sha256": "65405e7f310934ad878e6a736075dd3e66718841df2bab76b36a3c4b6be50dba", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": "bdd682b5c15e3a19be964efceae8d2cf1c95aed2108bb6a22091c168f63408a9", + "proposed_contract_sha256": "f1e9203c123a654519e85ee29e0788b4ae4507eed439e46df0360c30d3d488ab", + "proposed_calls_sha256": "ed263082775c2f2379c12f59f287df662a0c90fd1ad3bb028326ee2ca3427ab8", + "diagnostic_stage": null, + "canonical_action_count": 7, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 1 + }, + { + "task_id": 196, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-196.json", + "candidate_sha256": "1bf312ea7e6e75921d225517364ad784c128bc731904267682c1bf52a28c65c6", + "receipt_sha256": null, + "level": "diagnostic_only", + "official_score": null, + "blocker": "The current grid observer waits for loaded data and enumerates unique record IDs. Historical diagnostic scores apply only to the superseded input; qualification requires a new complete cohort.", + "contract_sha256": "438ea0f89d62bed6cd92e22300ec51087ff4f3336d6c165ad26ae9f3a8efd8df", + "calls_sha256": "a0413e6b6e28d2f540e4db75ea42337f671cdf65a284f95ede14573d579905ff", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "e0801082043fd71f8a86ec0636c62023ede890a7ae6e4b21fe4c3bb3a53fa158", + "task_intent_sha256": "a831964838291a142eec1243a9460ab0e20bd1b2bc13bd44332f3ce70b62605b", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "revised_input_pending_qualification", + "canonical_action_count": 9, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 2, + "historical_receipt": { + "candidate_sha256": "eff2bd31c203fa0303f8c139af20e75514305fe8c8b5097b7ca9be22a2149888", + "receipt_sha256": "e908d39aedb386f20b1add720028f225a8d7cc4d17d957f43b5555af5de1629c", + "official_score": 1, + "contract_sha256": "c769b8c2533a7112c298c93092db94016e010f33038301cff92a5572a1714845", + "calls_sha256": "429b63b185f781af6c6f966e3d153dd3ac5a285caa035156343acdd1269df9ad", + "independent_downstream_audit_sha256": "fa73042fd3ea491e70d7f813ccb3deb88f160db827716753ace8022b7b7e311c" + } + }, + { + "task_id": 198, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Frozen actions are preserved, but the offline tail navigates to a preselected order; it does not independently prove the latest cancelled order or customer identity.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "5cedcf4aa14c31ed31408ba0c573e2208afc7a435e74faee462fbfe60c5dc7ce", + "task_intent_sha256": "561cf2c0df047022ab8f52f36c01f772da19e8ee7efa79949be4523f5d052a70", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": "25fbea7c1f47024f8b407ccd9efa095c8aa87036e257f805e95f6b46c68048fd", + "proposed_contract_sha256": "a8d2cf83a6e09b7963506b78ad18deb6cb156c1f7a6b00033d3fab9c5e68a91e", + "proposed_calls_sha256": "b0dbde0cd9fa9603373785d1877c37b679602a8b459417f7cd0115a84b19f02f", + "diagnostic_stage": null, + "canonical_action_count": 3, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "navigate", + "wait", + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 204, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-204.json", + "candidate_sha256": "729ece693e0ec1105c1239cb993c9875caacb675668bead4aea00e86f61e358f", + "receipt_sha256": null, + "level": "diagnostic_only", + "official_score": null, + "blocker": "The current grid observer waits for loaded data and enumerates unique record IDs. Historical diagnostic scores apply only to the superseded input; qualification requires a new complete cohort.", + "contract_sha256": "b96e6d577b4f2aa20588f5a625d9473f1aace8f627309ffad84c24b2bfdcb12b", + "calls_sha256": "614431b974142f756bb5aeecc29f190cf0a76b6e93ff4ce7f7a15ee0e0ad4276", + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "3b50b31b06da571f40b0bc27a23c4e896ac51b6314d36410f9869813f7b42cc4", + "task_intent_sha256": "d7460c7441d80f646291f76fe7a4fa3cedd596e9ea43f8de612db215b5b2cba0", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "revised_input_pending_qualification", + "canonical_action_count": 7, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 2, + "historical_receipt": { + "candidate_sha256": "cded742171d7d4b58444e270dd4c3b95a7ba25c7bed75d8c29cfa5ee5ff6757f", + "receipt_sha256": "804a52f5ff94e405cf8ac8137d8b55a96a72dd8b9576896a97469ccc24eb6b65", + "official_score": 1, + "contract_sha256": "a452816d4bbb459a57319bb1dc55c986c3bf4fe6651e05936c8e9e02f1942fd8", + "calls_sha256": "ec6d93eec0fb1d21d14d9d8a8ea43d4bae997ff9397dd3b58f398d2be416dd6e", + "independent_downstream_audit_sha256": "9415eaf0343ff0cf99884ef5e57478a5fe476c01eaa4088ae41d9d9a4af928e2" + } + }, + { + "task_id": 231, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Offline observer changes a frozen canonical action before its observation tail; no current-page execution or qualification exists.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "c32727206f330b58d54a67c2d132870792e100820db76bfbc742675e8033d7d9", + "task_intent_sha256": "89f53bd43b471a7fbbe096ac5c9944b5f8ffd0170d64ec588e92a0d51959eee1", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_modified", + "proposal_sha256": "6ba6fabfa6287ab8c71ab66cc1253a9a321ed4728a0eea290aaba8722a6266e5", + "proposed_contract_sha256": "40c0d00d643ba949e8ccca52e499b187acea66330ebbc09baf0fe17ee6270f9f", + "proposed_calls_sha256": "4bc12b65fe5d46027c0564910a197fcbc2c9d752fe567762274752471db8bed5", + "diagnostic_stage": null, + "canonical_action_count": 5, + "canonical_prefix_preserved": false, + "tail_action_types": [], + "first_differing_action": 1 + }, + { + "task_id": 243, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": "b8303586579a7fa60e8fda8899ca8385342dc8f5fa67f9143c7e16833e4fa263", + "level": "blocked", + "official_score": null, + "blocker": "Candidate's customer identity/scope contract is not independently qualified by current public acceptance; retrospective downstream audit is not qualification.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": "9f96fce273ead1b1a28010705be6204078d98909d1f381917e703486f4cc4ef9", + "canonical_seed_calls_sha256": "3f45855646193068a64d9e5ec497b0d8b69e4169e044c8a55532f49c7114b14c", + "task_intent_sha256": "6e9d0db02cc4ef1415e6c020664b4b211e842a0a1ce5832f8b63495dd0c0da50", + "eligible_for_qualification": false, + "seed_binding": "no_observer", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": null, + "canonical_action_count": null, + "canonical_prefix_preserved": null, + "tail_action_types": [], + "first_differing_action": null + }, + { + "task_id": 313, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-313.json", + "candidate_sha256": "2ed9685a48cbd674e424df631f5133fdea800b7ada08dc3e60517f080838ad08", + "receipt_sha256": "0e24653cbc0bcc673f6c6b5de318699cd73985c9270e1abbdfc368c10984efe8", + "affected_task_qualification": { + "chromium_plan_sha256": "e0e26912d71e11b72526c47f19942f7fe70c05ef6dd85c9d137710e2d5e27648", + "moli_plan_sha256": "f07fa1395ede7de3794abfe14d1096fb1459e49393a31945bb07fba965a01ffd", + "candidate_sha256": "2ed9685a48cbd674e424df631f5133fdea800b7ada08dc3e60517f080838ad08", + "chromium_receipt_sha256": "2ac325728dcf07cb68e6573c7542128cd01d588b819582ff1823066b83e6d82e", + "moli_receipt_sha256": "983fc521bffbbf4439d129cfa8078665f33f2c6ef9ff20b6a5a9e7a8c1b19eac", + "executed_actions_sha256": "2d38517c5eb57c2d8516bad7745e2dd791fa68c1208416b38d6dbd445c7a51c5", + "site_map_sha256": "78c6c9498a07db4cd2b0b1ea3e573ad2ab92c4a9275b24c6dca054a641221ced", + "action_count_per_browser": 9, + "retrieval_status": "supported", + "chromium_official_score": 1, + "moli_official_score": 1 + }, + "level": "diagnostic_only", + "official_score": 1, + "blocker": "The same nine actions and bounded store-support observation passed both browsers. This is affected-task diagnostic qualification; historical scores remain unchanged and full-scope inclusion requires a frozen rerun.", + "contract_sha256": "ae4420b6c7bbcb5f80beb705446a160429ef34eb7a85b463d1c4f3a5636a5f97", + "calls_sha256": "e476c757200b709df685f32fa2d423c4c5fb505a0f1c12b3eef85a4d0b928ed7", + "independent_downstream_audit_sha256": "7836ff23fe2cc60c3404cb0eb53a2931c7864289b6d22d3048c6158205e8d1dc", + "canonical_seed_calls_sha256": "7daa31ede49e5f0031843d71f4cbe63f0fc15ba2185083f46babf181079850da", + "task_intent_sha256": "da870951505250a9ecabf5b73dc01ab83cbe72f11642cae51959ebf081832f82", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "affected_task_qualified", + "canonical_action_count": 8, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 314, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-314.json", + "candidate_sha256": "36341f8f8bd878ca57f91d570b51b249894e07c11ed0ba3eed3be41d0478b456", + "receipt_sha256": "37e488388a08904735dd26f7a60a2b6c3ebb6cb2b9086f4c3d04c270062e2935", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Frozen canonical actions are preserved, but the observation tail lacks a production proof of read-only behavior and full reference-browser qualification.", + "contract_sha256": "8ae5b3b4f9c4967cdb34873bc4a66917f3a76e21ca8458126d723289737dbaf0", + "calls_sha256": "7d8b2c3d376ef62b5dcd2e5e0f7f0fe04ea22adfdb637cd2c2852219df413751", + "independent_downstream_audit_sha256": "714d29fddce221a92cd66e739942d4405c8cd7d3359d60c9e64486f66cab5b36", + "canonical_seed_calls_sha256": "6841da84c9855b096736474fc42ec8fcf98135292969d3b09a995dad5c0746d1", + "task_intent_sha256": "7e0cee546d04be798858a9b694883310ef463bd8d183cf716b4e0c513717c45f", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 7, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 320, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "Frozen actions are preserved, but the offline tail inspects preselected orders; it does not independently prove the complete February cancelled-order population or refund scope.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "de0c74364402cac6d691ce0589acfcad521b243849ffa973c29c955d1bb4f950", + "task_intent_sha256": "a0ef7f9c5bd9573c968dc9807631d82f6b2fcef52afc379d0dbdb5a9fa52b961", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": "7d3056e6ba1ddfbb512c772ab8ddbe9f55df26a634e5e69ddd5a4241a0b109b1", + "proposed_contract_sha256": "668fc3abb30ba3651de6b1a9f3fc6f940d27128a6414400e51e44e9caad8e5ff", + "proposed_calls_sha256": "4a9cfca49ce64a8cba7906fff7b61ad338e2ba7cf86c5ba7309ac55190163f7d", + "diagnostic_stage": null, + "canonical_action_count": 6, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "navigate", + "wait", + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 349, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-349.json", + "candidate_sha256": "33bb5bb5540970507b518d9d0097a7c378fe00284eaec8b5bb4d03e6b33f869b", + "receipt_sha256": "08676f1fe8b7c4b419098eb9941ecb0a4ee3d06c8934558a8d2884948a23085d", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Frozen canonical actions are preserved, but the observation tail lacks a production proof of read-only behavior and full reference-browser qualification.", + "contract_sha256": "a79b1432c70bdaab680a1da0663395f2ff440a900a2d19d8d5987441fbf52a00", + "calls_sha256": "0a38ac087eaaf0f4aa73ff8cbf7a2ebe9b8b5d3963f015aa5a87724289eededf", + "independent_downstream_audit_sha256": "050e42d796a794d442d1668b11fac44ad3ed71b6267f64145f4c2a8bdf0c8142", + "canonical_seed_calls_sha256": "29e4a3c16ac017063d56c1795a9e6cff221d923a2e30d3b462d3e043a732bc26", + "task_intent_sha256": "916bafaa60bf29ddb2eb180bdd387139dcb52b50595fb04d4a48a6919e91f40f", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 4, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "evaluate" + ], + "first_differing_action": null + }, + { + "task_id": 386, + "candidate_path": null, + "candidate_sha256": null, + "receipt_sha256": null, + "level": "blocked", + "official_score": null, + "blocker": "No independent current-page structured observation and completeness proof for this frozen task.", + "contract_sha256": null, + "calls_sha256": null, + "independent_downstream_audit_sha256": null, + "canonical_seed_calls_sha256": "33d51f1af9ce36d501d4c63b9ad22fff1c045ccc6bde621c3bc30e4eff7167f1", + "task_intent_sha256": "de65f64bdfdafa236dd6929dd5fa31e6a5f8b7928b8605bf3d2cfa5e92b352d9", + "eligible_for_qualification": false, + "seed_binding": "no_observer", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": null, + "canonical_action_count": null, + "canonical_prefix_preserved": null, + "tail_action_types": [], + "first_differing_action": null + }, + { + "task_id": 784, + "candidate_path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-784.json", + "candidate_sha256": "c926ed0c2617186bf70431006c2719aaf0ea1fda6f70224e9c10e114e24c7c63", + "receipt_sha256": "fe5e404b9d3e46bcfc58e640e9b9fda53ade1dc364c6626900b4b7f83cb32139", + "level": "diagnostic_only", + "official_score": 1, + "blocker": "Frozen canonical actions are preserved, but the observation tail lacks a production proof of read-only behavior and full reference-browser qualification.", + "contract_sha256": "834b273865187ba47789c0d88718424b04c935aab334c8a3744c98215317b0ab", + "calls_sha256": "c19a76b510700ab55ec51f4e29b64007bc45e9006a5efc270d754f5841dd1739", + "independent_downstream_audit_sha256": "d9e60cfe7d7d10554672e219728a6178bb037d4df811f8c6efa1df4b99ffef43", + "canonical_seed_calls_sha256": "7ac27e2c9e58406897c07070b8c411e55653361bb82f428cfc94c2f0a78c71d0", + "task_intent_sha256": "9b49257d75f27fabe5c7000ce05c5959bd4682eb173da64cc17c99dd4e8ae9c2", + "eligible_for_qualification": false, + "seed_binding": "frozen_prefix_preserved", + "proposal_sha256": null, + "proposed_contract_sha256": null, + "proposed_calls_sha256": null, + "diagnostic_stage": "official_evaluator", + "canonical_action_count": 9, + "canonical_prefix_preserved": true, + "tail_action_types": [ + "evaluate" + ], + "first_differing_action": null + } + ], + "ready_90_sha256": "fd756fa1801d1daa704f851979e29d618e0eebacecc77c6fda5c121bb736f86f", + "seed_bindings_sha256": "e08946d65358d0e8ce2ac389e1c07f49fd7ed60968a638af1387fb345f1a4d52" +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/recompiled-seed-bindings.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/recompiled-seed-bindings.json new file mode 100644 index 000000000..3288a6f3a --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/recompiled-seed-bindings.json @@ -0,0 +1,266 @@ +{ + "schema": "lex_replay_recompiled_retrieval_seed_calls/1", + "ready_90_sha256": "fd756fa1801d1daa704f851979e29d618e0eebacecc77c6fda5c121bb736f86f", + "task_count": 37, + "rows": [ + { + "task_id": 28, + "canonical_candidate_id": "expert__task-28__latest-worcester-post", + "canonical_calls_sha256": "3d64afd95dd8ec26dcbea654df8f82f8a178bdca70a0fbb35d29ddb11161a740", + "intent_sha256": "001e46991c833c9b1bbeb7ceb764c4bb185f8b3129bfd7ce7df40ceceb6daac8", + "canonical_action_count": 5 + }, + { + "task_id": 29, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-29", + "canonical_calls_sha256": "49f4ca63a40b295c6ad566153530514a091c9a2e575abea6858f6223edd0cabe", + "intent_sha256": "3db3a1ce0087e7924c000225ab9c1278ef04b3747f13ae63beb986d496795795", + "canonical_action_count": 8 + }, + { + "task_id": 41, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-41", + "canonical_calls_sha256": "c89e6b139fbeb9c53f4ad3116d595adcdddd7f5a035ebc77c38c51ca3b3d1530", + "intent_sha256": "c707527241fa29e2f0a2a5590b7e26b55f4d1c5680e12c34c1aa1f6a95211af5", + "canonical_action_count": 1 + }, + { + "task_id": 50, + "canonical_candidate_id": "expert__task-50__annual-complete-orders", + "canonical_calls_sha256": "b2cd211239d1d6b6c2ba180f35eeecfe8383a796abcac495e365be3ce4080a3f", + "intent_sha256": "9827622d68dc3914a6002a7df25c74a6eaf6509d9b25ee59bc47f15a6728d13a", + "canonical_action_count": 5 + }, + { + "task_id": 66, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-66", + "canonical_calls_sha256": "5f6735f1058724eec0a2b9ff001e18cb137135fdf47237efebfbf6af7682eee9", + "intent_sha256": "253c889b8a843266be76fa90a7eb32ba2355c83f6e768dcfb7ec71e11d1df6b2", + "canonical_action_count": 9 + }, + { + "task_id": 67, + "canonical_candidate_id": "expert__task-67__books-hot-v2", + "canonical_calls_sha256": "1ca584eacde3c4b3df0fae77015b7b67ddd91e5fe673d8df86f30fbf134f8d91", + "intent_sha256": "bbded4eaa64bf95dc5ce13983651afd3a5e4c50448038f424b110ffe30982268", + "canonical_action_count": 3 + }, + { + "task_id": 77, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-77", + "canonical_calls_sha256": "12581dfb1d8e7eb8e8eb6f16c0d54fc006c1fcfe337a99c8a23a9a7913d23633", + "intent_sha256": "c58e624f93d0b19890e14b14c81fb95b2120c860b75381b4a0083fccfdeb8ea7", + "canonical_action_count": 5 + }, + { + "task_id": 94, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-94", + "canonical_calls_sha256": "0c6a0d744f3e8f4d9b5d87f4dbe3a68caef47a969847d541976344cfe60925d9", + "intent_sha256": "c70b55f52a6c4f4be4985026202c21419d893e133ccb91d7a04e621d95925112", + "canonical_action_count": 7 + }, + { + "task_id": 108, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-108", + "canonical_calls_sha256": "e19497a8b775e0d55adbb9bbc2e1cbfb5815e58dd3df56c39c3a482be1e7d561", + "intent_sha256": "2a3337dd851523f621e25a91fc77f5d5641d60cdd6268154880a6d7be40ebdb9", + "canonical_action_count": 14 + }, + { + "task_id": 110, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-110", + "canonical_calls_sha256": "9f8f882f416805aadeadf204a2f389a441a60d5c6983cb44a074e75c92b43e09", + "intent_sha256": "5e3dbd1e03d5c4f5d2f1fd492204b400ecc0952f18efa5c27ac3d1e10aa72db5", + "canonical_action_count": 14 + }, + { + "task_id": 117, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-117", + "canonical_calls_sha256": "68fdaf692f3f2a4b4f031c66f17be06d76d051530057552b3db26e7e969ab32e", + "intent_sha256": "675d0f1ffc001588f9d0c49c7872668fa701484bfd5a909f464a949739f3b4f8", + "canonical_action_count": 7 + }, + { + "task_id": 124, + "canonical_candidate_id": "expert-repair-a__task-124", + "canonical_calls_sha256": "b49b388278fbca7a306d8e7cbb9b77d0bd1d1c33055849afcb0e2b52ec283c26", + "intent_sha256": "7db62690f0b1b9cc41c8597934cf25183185e011e7d06b40af23176d4aa455a4", + "canonical_action_count": 4 + }, + { + "task_id": 127, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-127", + "canonical_calls_sha256": "54f6ffe55389bfc77448b96060e18779d27dadf8d2a80cf38b45a132eda0825c", + "intent_sha256": "4c8012dc00488c00bb97e07f97976d3209030bff942c499dedbc044f20d8ba32", + "canonical_action_count": 8 + }, + { + "task_id": 132, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-132", + "canonical_calls_sha256": "e9dbfca24c176b8284d39972a85d947aa340eecc25cc827efc739822dfde01d6", + "intent_sha256": "df1989bf3a8754b870fd2298b19c63b8d2b50f261c1e48d7844f9e0cf9a95bc0", + "canonical_action_count": 2 + }, + { + "task_id": 163, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-163", + "canonical_calls_sha256": "97ea26eae2ae20621b4c64fb0c4923f4234ea2ee01f44a37a0bb185fa1b44a29", + "intent_sha256": "d9ff84a6324844e40aaa88a08713fa2a3dc30f57346ee43842f7b8802efcb705", + "canonical_action_count": 5 + }, + { + "task_id": 168, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-168", + "canonical_calls_sha256": "1b831f87eb5f6bf380909e733daa091823a00aae1b6bb75ee7fdaa4e98c99689", + "intent_sha256": "566d3609bc7025fb9e0f74e29293fcc2527e3ddb943324314f8a7b54e7852dec", + "canonical_action_count": 2 + }, + { + "task_id": 173, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-173", + "canonical_calls_sha256": "82051decd9ba5e3edc5b29845f23e7d30fb696fb079915df35a5990a9d667809", + "intent_sha256": "7d982bcf9dc1de2a2ef073bc424741a44d88b7063f6c9dbc25c8a9095b9bf063", + "canonical_action_count": 8 + }, + { + "task_id": 183, + "canonical_candidate_id": "expert__task-183__quantity-ten", + "canonical_calls_sha256": "35950dde0d474d233f1656853e850d3e27093e0cbf8359aa99d27c105c823fd8", + "intent_sha256": "99c9c7d13ca2b4167c1fb84d5e2a685857843d1bcca8ea38b61d210c9c1de57b", + "canonical_action_count": 7 + }, + { + "task_id": 184, + "canonical_candidate_id": "expert__task-184__zero-simple-products", + "canonical_calls_sha256": "53f41746ccdf338e4db4c0b11f935a8e8911652c1c10d09530b7e9fdcfc21579", + "intent_sha256": "40110c86c139cc638d34eb51d3f9be5d24d94d26281ba1aa13b4430b07423ce8", + "canonical_action_count": 7 + }, + { + "task_id": 188, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-188", + "canonical_calls_sha256": "504cb09218b406ff19a0586cbe57742e3ac6e4b41383ea68e7082f61a15bec8b", + "intent_sha256": "65405e7f310934ad878e6a736075dd3e66718841df2bab76b36a3c4b6be50dba", + "canonical_action_count": 7 + }, + { + "task_id": 193, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-193", + "canonical_calls_sha256": "ce490a51db011dfa14b2001db12ceb8debe2b1c896f563068353abef6679e12e", + "intent_sha256": "7d5446703a3e187583bda80e23b1d9f5f7da77b35abdba622ac20c8c0eea3815", + "canonical_action_count": 5 + }, + { + "task_id": 196, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-196", + "canonical_calls_sha256": "e0801082043fd71f8a86ec0636c62023ede890a7ae6e4b21fe4c3bb3a53fa158", + "intent_sha256": "a831964838291a142eec1243a9460ab0e20bd1b2bc13bd44332f3ce70b62605b", + "canonical_action_count": 9 + }, + { + "task_id": 198, + "canonical_candidate_id": "expert__task-198__order-136-v2", + "canonical_calls_sha256": "5cedcf4aa14c31ed31408ba0c573e2208afc7a435e74faee462fbfe60c5dc7ce", + "intent_sha256": "561cf2c0df047022ab8f52f36c01f772da19e8ee7efa79949be4523f5d052a70", + "canonical_action_count": 3 + }, + { + "task_id": 204, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-204", + "canonical_calls_sha256": "3b50b31b06da571f40b0bc27a23c4e896ac51b6314d36410f9869813f7b42cc4", + "intent_sha256": "d7460c7441d80f646291f76fe7a4fa3cedd596e9ea43f8de612db215b5b2cba0", + "canonical_action_count": 7 + }, + { + "task_id": 225, + "canonical_candidate_id": "expert-repair-a__task-225", + "canonical_calls_sha256": "7f98391d517646ffa3d828afedc69b9aa95d1f345f5a09413884df43f99255cc", + "intent_sha256": "91e5a99677b265b187d30cf9c91063e64a485bafd9556f58fcf15a08a5a7f39a", + "canonical_action_count": 4 + }, + { + "task_id": 231, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-231", + "canonical_calls_sha256": "c32727206f330b58d54a67c2d132870792e100820db76bfbc742675e8033d7d9", + "intent_sha256": "89f53bd43b471a7fbbe096ac5c9944b5f8ffd0170d64ec588e92a0d51959eee1", + "canonical_action_count": 5 + }, + { + "task_id": 243, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-243", + "canonical_calls_sha256": "3f45855646193068a64d9e5ec497b0d8b69e4169e044c8a55532f49c7114b14c", + "intent_sha256": "6e9d0db02cc4ef1415e6c020664b4b211e842a0a1ce5832f8b63495dd0c0da50", + "canonical_action_count": 14 + }, + { + "task_id": 279, + "canonical_candidate_id": "expert-repair-a__task-279", + "canonical_calls_sha256": "aee97fafb52375d1f71435d1e0ec961112dfcb51d909c3fd08a2102e32e88b8c", + "intent_sha256": "6157be49b4de08a776b46ed9e8f7b818c36558ec5a7631beaae089cc3c36d687", + "canonical_action_count": 2 + }, + { + "task_id": 288, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-288", + "canonical_calls_sha256": "5f0f5ecf99c3c393c4d61b164ed63b3cb08723e856714dde0709435286c44dde", + "intent_sha256": "e66bf14a81eddff338422fe87b86db46bbe41957a5d9cb4e41ffd0c3dfdc78d8", + "canonical_action_count": 10 + }, + { + "task_id": 303, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-303", + "canonical_calls_sha256": "a89d802e46d15b3e58c329108d5b430d20ad04b4c7c44dc3f1cb622967ef6f08", + "intent_sha256": "70f26781f870c58362b833e8781fe6ffbed491b40b4ba20ebac4364eaa1697db", + "canonical_action_count": 3 + }, + { + "task_id": 313, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-313", + "canonical_calls_sha256": "7daa31ede49e5f0031843d71f4cbe63f0fc15ba2185083f46babf181079850da", + "intent_sha256": "da870951505250a9ecabf5b73dc01ab83cbe72f11642cae51959ebf081832f82", + "canonical_action_count": 8 + }, + { + "task_id": 314, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-314", + "canonical_calls_sha256": "6841da84c9855b096736474fc42ec8fcf98135292969d3b09a995dad5c0746d1", + "intent_sha256": "7e0cee546d04be798858a9b694883310ef463bd8d183cf716b4e0c513717c45f", + "canonical_action_count": 7 + }, + { + "task_id": 320, + "canonical_candidate_id": "expert__task-320__february-refunds", + "canonical_calls_sha256": "de0c74364402cac6d691ce0589acfcad521b243849ffa973c29c955d1bb4f950", + "intent_sha256": "a0ef7f9c5bd9573c968dc9807631d82f6b2fcef52afc379d0dbdb5a9fa52b961", + "canonical_action_count": 6 + }, + { + "task_id": 334, + "canonical_candidate_id": "expert__task-334__last-cornbread-order", + "canonical_calls_sha256": "1c33b18e1da46cf06a6937148c307c79b5cf1aa89a9a18aa99e4905708b5271d", + "intent_sha256": "485022b363427222ce806d8f16309c55a6f4a3755c1efcc43cf44443929a80e3", + "canonical_action_count": 3 + }, + { + "task_id": 349, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-349", + "canonical_calls_sha256": "29e4a3c16ac017063d56c1795a9e6cff221d923a2e30d3b462d3e043a732bc26", + "intent_sha256": "916bafaa60bf29ddb2eb180bdd387139dcb52b50595fb04d4a48a6919e91f40f", + "canonical_action_count": 4 + }, + { + "task_id": 386, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-386", + "canonical_calls_sha256": "33d51f1af9ce36d501d4c63b9ad22fff1c045ccc6bde621c3bc30e4eff7167f1", + "intent_sha256": "de65f64bdfdafa236dd6929dd5fa31e6a5f8b7928b8605bf3d2cfa5e92b352d9", + "canonical_action_count": 9 + }, + { + "task_id": 784, + "canonical_candidate_id": "gpt-5.6-sol__chromium__task-784", + "canonical_calls_sha256": "7ac27e2c9e58406897c07070b8c411e55653361bb82f428cfc94c2f0a78c71d0", + "intent_sha256": "9b49257d75f27fabe5c7000ce05c5959bd4682eb173da64cc17c99dd4e8ae9c2", + "canonical_action_count": 9 + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-ajax-proposal.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-ajax-proposal.json new file mode 100644 index 000000000..6b017dc53 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-ajax-proposal.json @@ -0,0 +1,118 @@ +{ + "task_id": 108, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "01/01/2023", + "selector": "input[name=\"created_at[from]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "05/31/2023", + "selector": "input[name=\"created_at[to]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "complete", + "selector": "select[name=\"status\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Tab" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Apply Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\nconst cfg={\"namespace\": \"sales_order_grid\", \"filters\": {\"filters[status]\": \"complete\", \"filters[created_at][from]\": \"1/01/2023\", \"filters[created_at][to]\": \"5/31/2023\"}, \"fields\": [\"created_at\", \"status\"]};\n// cfg is frozen task configuration, never an expected answer.\nconst providers=[];\nrequire('uiRegistry').get(c=>{if(c.component==='Magento_Ui/js/grid/provider'&&c.params?.namespace===cfg.namespace) providers.push(c);});\nif(providers.length!==1) throw new Error('unique current UI grid provider unavailable');\nconst provider=providers[0];\nconst endpoint=new URL(provider.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/') throw new Error('unexpected current grid endpoint');\nconst flatten=(value,prefix)=>Object.entries(value).flatMap(([key,item])=>{const name=prefix?prefix+'['+key+']':key;return item!==null&&typeof item==='object'?flatten(item,name):[[name,String(item)]];});\nfor(const [key,value] of flatten(provider.params,'')) endpoint.searchParams.set(key,value);\n// Magento grid render requires AJAX mode even for read-only pagination.\nendpoint.searchParams.set('isAjax','true');\nconst actualFilters=Object.fromEntries([...endpoint.searchParams].filter(([k])=>k.startsWith('filters[')&&k!=='filters[placeholder]'));\nconst normalize=v=>String(v).replace(/\\b0(?=\\d)/g,'');\nconst scope=Object.keys(actualFilters).length===Object.keys(cfg.filters).length && Object.entries(cfg.filters).every(([k,v])=>normalize(actualFilters[k])===normalize(v));\nif(!scope) throw new Error('last UI grid request has wrong filters');\nconst pageSize=Number(endpoint.searchParams.get('paging[pageSize]'));\nif(!Number.isInteger(pageSize)||pageSize<1||pageSize>1000) throw new Error('UI page size unavailable');\nlet total=null;\nconst rows=[],pages=[];\nfor(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n let data;\n if(page===Number(provider.params.paging.current)) data=provider.data;\n else {\n const response=await fetch(endpoint.href,{credentials:'same-origin'});\n if(!response.ok||response.redirected) throw new Error('grid request failed or redirected');\n data=await response.json();\n }\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('grid total or rows unavailable');\n if(total===null) total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/pageSize)>100) throw new Error('unstable or oversized grid');\n const expectedCount=Math.min(pageSize,Math.max(0,total-(page-1)*pageSize));\n if(data.items.length!==expectedCount) throw new Error('partial or ignored grid page');\n for(const row of data.items) {\n if(row.entity_id==null||cfg.fields.some(k=>row[k]==null)) throw new Error('required grid field unavailable');\n rows.push(Object.fromEntries(['entity_id',...cfg.fields].map(k=>[k,row[k]])));\n }\n pages.push({page,total: data.totalRecords,count:data.items.length});\n if(page>=Math.max(1,Math.ceil(total/pageSize))) break;\n}\nconst complete=rows.length===total && new Set(rows.map(r=>String(r.entity_id))).size===rows.length;\nif(!complete) throw new Error('grid enumeration not complete or unique');\nconst monthNames=['January','February','March','April','May','June','July','August','September','October','November','December'];\nconst counts=Array(5).fill(0);\nfor(const row of rows) {\n const match=String(row.created_at).match(/^2023-(0[1-5])-\\d{2}(?: |T)/);\n if(row.status!=='complete'||!match) throw new Error('returned order violates requested range/status');\n counts[Number(match[1])-1]++;\n}\nreturn {scope,complete,pages,filters:actualFilters,rows,answer:counts.map((count,i)=>({month:monthNames[i],count}))};\n\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 108, + "canonical_calls_sha256": "232669e9374158b513a061bb6cc0c36e3e6d695996282ffe78a78a81d471861c", + "sources": [ + { + "name": "observation", + "step": 16, + "action_sha256": "59dda026d602b35368edf628f0abe85f7e2855a2145ece54e356b1dac503755d", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "map": { + "input": { + "source": "observation", + "pointer": "/answer" + }, + "projection": { + "object": { + "month": { + "source": "$item", + "pointer": "/month" + }, + "count": { + "source": "$item", + "pointer": "/count" + } + } + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-stabilized.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-stabilized.json new file mode 100644 index 000000000..315a88354 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108-stabilized.json @@ -0,0 +1,119 @@ +{ + "task_id": 108, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "01/01/2023", + "selector": "input[name=\"created_at[from]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "05/31/2023", + "selector": "input[name=\"created_at[to]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "complete", + "selector": "select[name=\"status\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async () => { const names=['created_at[from]','created_at[to]']; const expected=['01/01/2023','05/31/2023']; for(let i=0;isetTimeout(resolve,300)); return {date_inputs_committed:true}; })()" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Apply Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "evaluate", + "script": "(async () => { const expected={'filters[status]':'complete','filters[created_at][from]':'1/01/2023','filters[created_at][to]':'5/31/2023'}; const normalize=value=>String(value).replace(/\\b0(?=\\d)/g,''); const flatten=(value,prefix)=>Object.entries(value).flatMap(([key,item])=>{const name=prefix?prefix+'['+key+']':key;return item!==null&&typeof item==='object'?flatten(item,name):[[name,String(item)]];}); const deadline=Date.now()+15000; while(Date.now(){if(component.component==='Magento_Ui/js/grid/provider'&&component.params?.namespace==='sales_order_grid') providers.push(component);}); if(providers.length===1){const provider=providers[0], flat=Object.fromEntries(flatten(provider.params,'')); const actual=Object.fromEntries(Object.entries(flat).filter(([key])=>key.startsWith('filters[')&&key!=='filters[placeholder]')); if(Object.keys(actual).length===3&&Object.entries(expected).every(([key,value])=>normalize(actual[key])===normalize(value))&&Number.isInteger(provider.data?.totalRecords)&&Array.isArray(provider.data?.items)) return {grid_filters_ready:true,filters:actual,total:provider.data.totalRecords};} await new Promise(resolve=>setTimeout(resolve,250));} throw new Error('applied_order_grid_filters_not_ready'); })()" + }, + { + "action": "evaluate", + "script": "(async () => {\nconst cfg={\"namespace\": \"sales_order_grid\", \"filters\": {\"filters[status]\": \"complete\", \"filters[created_at][from]\": \"1/01/2023\", \"filters[created_at][to]\": \"5/31/2023\"}, \"fields\": [\"created_at\", \"status\"]};\n// cfg is frozen task configuration, never an expected answer.\nconst providers=[];\nrequire('uiRegistry').get(c=>{if(c.component==='Magento_Ui/js/grid/provider'&&c.params?.namespace===cfg.namespace) providers.push(c);});\nif(providers.length!==1) throw new Error('unique current UI grid provider unavailable');\nconst provider=providers[0];\nconst endpoint=new URL(provider.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/') throw new Error('unexpected current grid endpoint');\nconst flatten=(value,prefix)=>Object.entries(value).flatMap(([key,item])=>{const name=prefix?prefix+'['+key+']':key;return item!==null&&typeof item==='object'?flatten(item,name):[[name,String(item)]];});\nfor(const [key,value] of flatten(provider.params,'')) endpoint.searchParams.set(key,value);\n// Magento grid render requires AJAX mode even for read-only pagination.\nendpoint.searchParams.set('isAjax','true');\nconst actualFilters=Object.fromEntries([...endpoint.searchParams].filter(([k])=>k.startsWith('filters[')&&k!=='filters[placeholder]'));\nconst normalize=v=>String(v).replace(/\\b0(?=\\d)/g,'');\nconst scope=Object.keys(actualFilters).length===Object.keys(cfg.filters).length && Object.entries(cfg.filters).every(([k,v])=>normalize(actualFilters[k])===normalize(v));\nif(!scope) throw new Error('last UI grid request has wrong filters');\nconst pageSize=Number(endpoint.searchParams.get('paging[pageSize]'));\nif(!Number.isInteger(pageSize)||pageSize<1||pageSize>1000) throw new Error('UI page size unavailable');\nlet total=null;\nconst rows=[],pages=[];\nfor(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n let data;\n if(page===Number(provider.params.paging.current)) data=provider.data;\n else {\n const response=await fetch(endpoint.href,{credentials:'same-origin'});\n if(!response.ok||response.redirected) throw new Error('grid request failed or redirected');\n data=await response.json();\n }\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('grid total or rows unavailable');\n if(total===null) total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/pageSize)>100) throw new Error('unstable or oversized grid');\n const expectedCount=Math.min(pageSize,Math.max(0,total-(page-1)*pageSize));\n if(data.items.length!==expectedCount) throw new Error('partial or ignored grid page');\n for(const row of data.items) {\n if(row.entity_id==null||cfg.fields.some(k=>row[k]==null)) throw new Error('required grid field unavailable');\n rows.push(Object.fromEntries(['entity_id',...cfg.fields].map(k=>[k,row[k]])));\n }\n pages.push({page,total: data.totalRecords,count:data.items.length});\n if(page>=Math.max(1,Math.ceil(total/pageSize))) break;\n}\nconst complete=rows.length===total && new Set(rows.map(r=>String(r.entity_id))).size===rows.length;\nif(!complete) throw new Error('grid enumeration not complete or unique');\nconst monthNames=['January','February','March','April','May','June','July','August','September','October','November','December'];\nconst counts=Array(5).fill(0);\nfor(const row of rows) {\n const match=String(row.created_at).match(/^2023-(0[1-5])-\\d{2}(?: |T)/);\n if(row.status!=='complete'||!match) throw new Error('returned order violates requested range/status');\n counts[Number(match[1])-1]++;\n}\nreturn {scope,complete,pages,filters:actualFilters,rows,answer:counts.map((count,i)=>({month:monthNames[i],count}))};\n\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 108, + "canonical_calls_sha256": "0e2f8a8377798335f4425f604001574c66a22bda6305e9ab9ace85ac13bd4c74", + "sources": [ + { + "name": "observation", + "step": 16, + "action_sha256": "59dda026d602b35368edf628f0abe85f7e2855a2145ece54e356b1dac503755d", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "map": { + "input": { + "source": "observation", + "pointer": "/answer" + }, + "projection": { + "object": { + "month": { + "source": "$item", + "pointer": "/month" + }, + "count": { + "source": "$item", + "pointer": "/count" + } + } + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108.json new file mode 100644 index 000000000..eb13cece5 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-108.json @@ -0,0 +1,118 @@ +{ + "task_id": 108, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "01/01/2023", + "selector": "input[name=\"created_at[from]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "05/31/2023", + "selector": "input[name=\"created_at[to]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "complete", + "selector": "select[name=\"status\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Tab" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Apply Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\nconst cfg={\"namespace\": \"sales_order_grid\", \"filters\": {\"filters[status]\": \"complete\", \"filters[created_at][from]\": \"1/01/2023\", \"filters[created_at][to]\": \"5/31/2023\"}, \"fields\": [\"created_at\", \"status\"]};\n// cfg is frozen task configuration, never an expected answer.\nconst providers=[];\nrequire('uiRegistry').get(c=>{if(c.component==='Magento_Ui/js/grid/provider'&&c.params?.namespace===cfg.namespace) providers.push(c);});\nif(providers.length!==1) throw new Error('unique current UI grid provider unavailable');\nconst provider=providers[0];\nconst endpoint=new URL(provider.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/') throw new Error('unexpected current grid endpoint');\nconst flatten=(value,prefix)=>Object.entries(value).flatMap(([key,item])=>{const name=prefix?prefix+'['+key+']':key;return item!==null&&typeof item==='object'?flatten(item,name):[[name,String(item)]];});\nfor(const [key,value] of flatten(provider.params,'')) endpoint.searchParams.set(key,value);\nconst actualFilters=Object.fromEntries([...endpoint.searchParams].filter(([k])=>k.startsWith('filters[')&&k!=='filters[placeholder]'));\nconst normalize=v=>String(v).replace(/\\b0(?=\\d)/g,'');\nconst scope=Object.keys(actualFilters).length===Object.keys(cfg.filters).length && Object.entries(cfg.filters).every(([k,v])=>normalize(actualFilters[k])===normalize(v));\nif(!scope) throw new Error('last UI grid request has wrong filters');\nconst pageSize=Number(endpoint.searchParams.get('paging[pageSize]'));\nif(!Number.isInteger(pageSize)||pageSize<1||pageSize>1000) throw new Error('UI page size unavailable');\nlet total=null;\nconst rows=[],pages=[];\nfor(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n let data;\n if(page===Number(provider.params.paging.current)) data=provider.data;\n else {\n const response=await fetch(endpoint.href,{credentials:'same-origin'});\n if(!response.ok||response.redirected) throw new Error('grid request failed or redirected');\n data=await response.json();\n }\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('grid total or rows unavailable');\n if(total===null) total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/pageSize)>100) throw new Error('unstable or oversized grid');\n const expectedCount=Math.min(pageSize,Math.max(0,total-(page-1)*pageSize));\n if(data.items.length!==expectedCount) throw new Error('partial or ignored grid page');\n for(const row of data.items) {\n if(row.entity_id==null||cfg.fields.some(k=>row[k]==null)) throw new Error('required grid field unavailable');\n rows.push(Object.fromEntries(['entity_id',...cfg.fields].map(k=>[k,row[k]])));\n }\n pages.push({page,total: data.totalRecords,count:data.items.length});\n if(page>=Math.max(1,Math.ceil(total/pageSize))) break;\n}\nconst complete=rows.length===total && new Set(rows.map(r=>String(r.entity_id))).size===rows.length;\nif(!complete) throw new Error('grid enumeration not complete or unique');\nconst monthNames=['January','February','March','April','May','June','July','August','September','October','November','December'];\nconst counts=Array(5).fill(0);\nfor(const row of rows) {\n const match=String(row.created_at).match(/^2023-(0[1-5])-\\d{2}(?: |T)/);\n if(row.status!=='complete'||!match) throw new Error('returned order violates requested range/status');\n counts[Number(match[1])-1]++;\n}\nreturn {scope,complete,pages,filters:actualFilters,rows,answer:counts.map((count,i)=>({month:monthNames[i],count}))};\n\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 108, + "canonical_calls_sha256": "07d6c1098f9409fd61c5315f57ddfac3c6bfa119bcace58e0ac5811e1df67d18", + "sources": [ + { + "name": "observation", + "step": 16, + "action_sha256": "eddf9002825bc5b53dc49dd17f874ff3cb486d9def23b24e181c0cfe7f0bf8e7", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "map": { + "input": { + "source": "observation", + "pointer": "/answer" + }, + "projection": { + "object": { + "month": { + "source": "$item", + "pointer": "/month" + }, + "count": { + "source": "$item", + "pointer": "/count" + } + } + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-110.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-110.json new file mode 100644 index 000000000..d9b2512f3 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-110.json @@ -0,0 +1,40 @@ +{ + "task_id": 110, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;i=1&&month<=11)counts[month-1]++;}}\n return {scope:true,complete:true,pages,total,answer:months.map((month,i)=>({month,count:counts[i]}))};\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 110, + "canonical_calls_sha256": "97f90a56707dddb87025f4945ef1a13b8ed50cab42ce3f21215a446b48cbba55", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "3a28b1427e97104828c43b22e97d1d3ebf85181246a14fe03cef412af12e0cb6", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-117.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-117.json new file mode 100644 index 000000000..c156b7d43 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-117.json @@ -0,0 +1,40 @@ +{ + "task_id": 117, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].date{const rows=[...document.querySelectorAll('.product-item')].map(e=>({name:e.querySelector('.product-item-link')?.textContent.trim(),price:e.querySelector('[data-price-type=finalPrice] .price, .price-box .price')?.textContent.trim()})).filter(x=>x.name&&x.price);if(!rows.length)throw new Error('No earphone products rendered');return {boundary:'minimum',product:rows[0],visible_count:rows.length};})()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sony-wi-1000xm2-industry-leading-noise-canceling-wireless-behind-neck-in-ear-headset-headphones-with-mic-for-phone-call-with-alexa-voice-control-black.html" + }, + { + "action": "evaluate", + "script": "(()=>{const product={name:document.querySelector('h1.page-title')?.textContent.trim(),price:document.querySelector('[data-price-type=finalPrice] .price, .product-info-price .price')?.textContent.trim(),description:document.querySelector('.product.attribute.description')?.textContent.replace(/\\s+/g,' ').trim()};if(!product.name||!product.price||!/wireless/i.test(product.description||'')||!/(in-ear|in ear|behind-neck)/i.test(product.name+' '+(product.description||'')))throw new Error('Upper-bound product is not a wireless in-ear model');return {boundary:'maximum',product};})()" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(!location.pathname.endsWith('.html'))throw Error('wrong current product page');\nconst records=new Map(),coverage=[];for(const term of ['ear','earphone','earbud','neckband']){const rows=[],pages=[],seen=new Set();let total=null;\nfor(let page=1;page<=100;page++){\n const query='query {products(filter:{name:{match:'+JSON.stringify(term)+'}},pageSize:100,currentPage:'+page+'){total_count page_info{current_page total_pages} items{sku name url_key description{html} rating_summary price_range{minimum_price{final_price{value currency}}}}}}';\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog response unavailable');\n const payload=await response.json(),data=payload.data?.products;\n if(payload.errors?.length||!data||!Number.isSafeInteger(data.total_count)||data.total_count<0||!Array.isArray(data.items))throw Error('invalid catalog response');\n if(total===null)total=data.total_count;\n const count=Math.max(1,Math.ceil(total/100));\n if(total!==data.total_count||count>100||data.page_info.current_page!==page||data.page_info.total_pages!==Math.ceil(total/100)||data.items.length!==Math.min(100,Math.max(0,total-(page-1)*100)))throw Error('incomplete or changed catalog');\n for(const row of data.items){\n if(typeof row.sku!=='string'||!row.sku||seen.has(row.sku)||typeof row.name!=='string'||!row.name||typeof row.url_key!=='string'||!row.url_key||row.url_key.includes('/'))throw Error('catalog identity missing or duplicated');\n seen.add(row.sku);\n const doc=new DOMParser().parseFromString(row.description?.html||'','text/html');\n for(const n of doc.querySelectorAll('script,style,noscript,template'))n.remove();\n const price=row.price_range?.minimum_price?.final_price;\n if(price?.currency!=='USD'||typeof price.value!=='number'||!Number.isFinite(price.value)||price.value<0)throw Error('invalid USD catalog price');\n rows.push({sku:row.sku,name:row.name,url:new URL('/'+row.url_key+'.html',location.origin).href,description:doc.body.textContent.replace(/\\s+/g,' ').trim(),rating:row.rating_summary,cents:Math.round(price.value*100)});\n }\n pages.push({page,rows:data.items.length,total,ids:data.items.map(r=>r.sku)});\n if(page===count)break;\n}\nif(rows.length!==total)throw Error('catalog population incomplete');\n\nfor(const r of rows){const prior=records.get(r.sku);if(prior&&(prior.name!==r.name||prior.cents!==r.cents))throw Error('catalog changed between searches');records.set(r.sku,r)}\ncoverage.push({term,total,pages});\n}\nconst rows=[...records.values()];\nconst classified=rows.map(r=>{\n const n=r.name;\n const earForm=/earphones?|earbuds?|in[- ]ear|behind[- ]neck|neckband/i.test(n);\n const wireless=/wireless|bluetooth/i.test(n+' '+r.description);\n const unrelated=/\\b(smartphone|cell phone|tablet|laptop|smartwatch|television|smart tv|sunglasses|goggles|amplifier)\\b/i.test(n);\n const accessory=/earphone accessories|earbuds? cover|replacement ear|ear tips|earpads|ear cushions|protective earhooks|charging case only|case for|cover for/i.test(n);\n const overEar=/\\b(over[- ]ear|on[- ]ear)\\b/i.test(n)&&!/\\bin[- ]ear\\b/i.test(n);\n return {...r,qualified:earForm&&wireless&&!unrelated&&!accessory&&!overEar};\n});\nconst selected=classified.filter(r=>r.qualified);\nif(!selected.length)throw Error('no wireless earphones in complete catalog');\nconst prices=selected.map(r=>r.cents);\nreturn {scope:true,complete:true,coverage,products:classified,answer:{min:Math.min(...prices)/100,max:Math.max(...prices)/100}};\n}finally{controller.abort();clearTimeout(timer)}})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 124, + "canonical_calls_sha256": "8602b461bd3ef93d7639129f9bbc524de84f1848d2a79b04ae4dab546a5f5585", + "sources": [ + { + "name": "observation", + "step": 5, + "action_sha256": "9bcf063e09f3bd7a88dd3e3e13033641e9788534d3555ceba25b216591d19dfc", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-127.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-127.json new file mode 100644 index 000000000..6ca85bbe2 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-127.json @@ -0,0 +1,40 @@ +{ + "task_id": 127, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/search/term/report/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nconst gridId=\"searchReportGrid\",identity=\"query_id\";\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/search/term/report\")throw Error('wrong legacy grid page');\nconst number=(text)=>{const s=String(text??'').trim();if(!/^\\d+$/.test(s)||!Number.isSafeInteger(Number(s)))throw Error('invalid grid integer');return Number(s)};\nconst collect=(doc,index)=>{\n const root=doc.getElementById(gridId),table=doc.getElementById(gridId+'_table');\n if(!root||!table||!root.contains(table))throw Error('legacy grid missing');\n const total=number(doc.getElementById(gridId+'-total-count')?.textContent);\n const current=number(root.querySelector('input[name=\"page\"]')?.value),size=number(root.querySelector('select[name=\"limit\"]')?.value);\n if(current!==index||size<1||size>1000||Math.ceil(total/size)>100)throw Error('legacy pagination invalid');\n const active=[...table.querySelectorAll('thead input[name],thead select[name]')].filter(n=>!(n.type==='hidden'&&/\\[locale\\]$/.test(n.name))&&n.value!==''&&!(['store_id','massaction'].includes(n.name)&&n.value==='0')&&!(n.name==='visible_in'&&n.value==='1'));\n if(active.length)throw Error('filtered legacy grid');\n const rows=[...table.querySelectorAll('tbody > tr')].filter(tr=>!tr.querySelector('td.empty')).map(tr=>{\n const fields=identity==='review_id'?['review_id']:['query_id','query_text','num_results','popularity'];\n return Object.fromEntries(fields.map(field=>{const cells=tr.querySelectorAll('td.col-'+field);if(cells.length!==1)throw Error('legacy field missing or ambiguous');return [field,cells[0].textContent.replace(/\\s+/g,' ').trim()]}));\n });\n if(rows.length!==Math.min(size,Math.max(0,total-(index-1)*size)))throw Error('incomplete legacy page');\n for(const row of rows)number(row[identity]);\n return {rows,total,size};\n};\nconst first=collect(document,1),total=first.total,rows=[...first.rows],pages=[{page:1,rows:first.rows.length,total}],params={namespace:gridId};\nfor(let page=2;page<=Math.ceil(total/first.size);page++){\n const url=new URL(location.href);url.searchParams.set('page',String(page));url.searchParams.set('limit',String(first.size));\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal});\n if(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('legacy page fetch failed');\n const data=collect(new DOMParser().parseFromString(await response.text(),'text/html'),page);\n if(data.total!==total||data.size!==first.size)throw Error('legacy grid changed');\n rows.push(...data.rows);pages.push({page,rows:data.rows.length,total});\n}\nif(rows.length!==total||new Set(rows.map(r=>r[identity])).size!==total)throw Error('incomplete or duplicate legacy identities');\nconst field=(r,names)=>{const found=names.filter(k=>Object.hasOwn(r,k));if(found.length!==1)throw Error('ambiguous search field');return r[found[0]]};\nconst mapped=rows.map(r=>({term:String(field(r,['search_query','query_text'])).trim(),hits:Number(field(r,['popularity','uses'])),results:Number(field(r,['num_results','results']))}));\nif(mapped.some(r=>!r.term||!Number.isSafeInteger(r.hits)||r.hits<0||!Number.isSafeInteger(r.results)||r.results<0))throw Error('invalid search terms');\nconst eligible=mapped.filter(r=>r.results>0).sort((a,b)=>b.hits-a.hits||a.term.localeCompare(b.term));\nif(eligible.length<3||eligible[3]?.hits===eligible[2].hits)throw Error('top three incomplete or tied');\nreturn {scope:true,complete:true,namespace:params.namespace,pages,total,\nanswer:eligible.slice(0,3).map(r=>r.term)};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 127, + "canonical_calls_sha256": "c2da0bcabdcad6f7da14a23c46bd40072a26c86270397fbb28c2cab32fa2e1c8", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "e098c2bcdea2de399ce97c411f6ade585b775c643acccf251bb6cb399a410670", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-132.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-132.json new file mode 100644 index 000000000..56aeec57e --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-132.json @@ -0,0 +1,44 @@ +{ + "task_id": 132, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/commits/main?author=kilian" + }, + { + "action": "evaluate", + "script": "(async()=>{\n const ctl=new AbortController(),timer=setTimeout(()=>ctl.abort(),30000);\n try {\n const api=location.origin+'/api/v4',path='a11yproject/a11yproject.com';\n const range=[\"2023-03-05T00:00:00Z\",\"2023-03-05T23:59:59Z\"];\n async function read(url){\n const r=await fetch(url,{credentials:'same-origin',redirect:'error',signal:ctl.signal,headers:{Accept:'application/json'}});\n if(r.status!==200||r.redirected||!/^application\\/json\\b/i.test(r.headers.get('content-type')||''))throw Error('GitLab JSON read failed');\n return {r,body:await r.json()};\n }\n const {body:project}=await read(api+'/projects/'+encodeURIComponent(path));\n if(!Number.isInteger(project.id)||project.path_with_namespace!==path||project.default_branch!=='main')throw Error('wrong project or branch');\n const rows=[],pages=[],seen=new Set(),size=100;\n for(let page=1;page<=100;page++){\n const url=new URL(api+'/projects/'+project.id+'/repository/commits');\n for(const [k,v] of Object.entries({ref_name:'main',since:range[0],until:range[1],per_page:String(size),page:String(page)}))url.searchParams.set(k,v);\n const {r,body}=await read(url.href);\n const current=r.headers.get('x-page'),next=r.headers.get('x-next-page');\n if(current!==String(page)||next===null||!Array.isArray(body)||body.length>size)throw Error('incomplete commit page');\n for(const c of body){\n const time=Date.parse(c.committed_date);\n if(!/^[0-9a-f]{40}$/.test(c.id)||seen.has(c.id)||typeof c.author_name!=='string'||!Number.isFinite(time)||timeDate.parse(range[1]))throw Error('commit identity or date ambiguity');\n seen.add(c.id);rows.push({id:c.id,author_name:c.author_name,committed_date:c.committed_date});\n }\n pages.push({page,rows:body.length,next});\n if(next==='')break;\n if(next!==String(page+1)||page===100)throw Error('commit pagination gap or safety bound');\n }\n if(!pages.length||pages.at(-1).next!==''||pages.reduce((n,p)=>n+p.rows,0)!==rows.length)throw Error('incomplete commit population');\n const commits=rows.filter(c=>/^kilian(?:\\s|$)/i.test(c.author_name));\n return {scope:true,complete:true,project:{id:project.id,path_with_namespace:project.path_with_namespace,branch:project.default_branch},range,pages,observed_commits:rows,commits,answer:commits.length};\n }finally{ctl.abort();clearTimeout(timer)}\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 132, + "canonical_calls_sha256": "7c3455f4de71b7780e242804db462eb23c9bd8fc4658fc11c34cd4018c5cc7f0", + "sources": [ + { + "name": "observation", + "step": 3, + "action_sha256": "6452cdd1ca7405626cc8a701c116ae021d28c2a855204c83ba7d55039be3c214", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "source": "observation", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-163.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-163.json new file mode 100644 index 000000000..ad97fc70f --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-163.json @@ -0,0 +1,149 @@ +{ + "task_id": 163, + "tool_calls": [ + { + "action": "navigate", + "url": "http://127.0.0.1:7770/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html" + }, + { + "action": "evaluate", + "script": "(async () => {\n const started = Date.now();\n const timeoutMs = 15000;\n while (Date.now() - started < timeoutMs) {\n const jq = window.jQuery;\n const root = document.querySelector('.product.data.items');\n const title = document.querySelector('#tab-label-reviews');\n if (typeof jq === 'function' && root && title &&\n typeof jq.fn.tabs === 'function' && typeof jq.fn.collapsible === 'function') {\n const tabs = jq(root).tabs('instance');\n const collapsible = jq(title).collapsible('instance');\n if (tabs && tabs.widgetFullName === 'mage-tabs' &&\n collapsible && collapsible.widgetFullName === 'mage-collapsible' &&\n collapsible.options.disabled !== true) {\n return {ready:true, waitedMs:Date.now()-started,\n tabs:tabs.widgetFullName, collapsible:collapsible.widgetFullName};\n }\n }\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n throw new Error('Magento Reviews widgets not ready within 15000ms');\n})()\n" + }, + { + "action": "click", + "selector": "a[href=\"#reviews\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/review/product/listAjax/id/38805/?p=2\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async (expectedPath) => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 20000);\n try {\n const one = (root, selector) => {\n const nodes = [...root.querySelectorAll(selector)];\n if (nodes.length !== 1) throw Error(`Expected one ${selector}, found ${nodes.length}`);\n return nodes[0];\n };\n const natural = text => {\n if (!/^(0|[1-9][0-9]*)$/.test(text)) throw Error('Invalid natural number');\n const value = Number(text);\n if (!Number.isSafeInteger(value)) throw Error('Unsafe number');\n return value;\n };\n const initial = new URL(location.href);\n if (initial.pathname !== expectedPath || initial.search) throw Error('Wrong current product');\n const productId = one(document, '#product_addtocart_form input[name=\"product\"]').value;\n if (!/^[1-9][0-9]*$/.test(productId)) throw Error('Invalid product identity');\n const countSelector = '.product-reviews-summary [itemprop=\"reviewCount\"]';\n const total = natural(one(document, countSelector).textContent.trim());\n // Read configuration from the response layer where Magento publishes it.\n // mage/apply/scripts consumes and removes these nodes before widget readiness.\n const productResponse = await fetch(initial.href, {credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\n if (productResponse.status !== 200 || productResponse.redirected || productResponse.url !== initial.href ||\n !/^text\\/html\\b/i.test(productResponse.headers.get('content-type') || ''))\n throw Error('Current product HTML response identity mismatch');\n const productHTML = await productResponse.text();\n const productDoc = new DOMParser().parseFromString(productHTML, 'text/html');\n if (one(productDoc, '#product_addtocart_form input[name=\"product\"]').value !== productId ||\n natural(one(productDoc, countSelector).textContent.trim()) !== total)\n throw Error('Current product HTML identity or review total differs from rendered product');\n const configs = [...productDoc.querySelectorAll('script[type=\"text/x-magento-init\"]')]\n .map(node => JSON.parse(node.textContent))\n .flatMap(config => Object.values(config))\n .map(config => config['Magento_Review/js/process-reviews']).filter(Boolean);\n if (configs.length !== 1) throw Error(`Expected one initial-HTML review endpoint, found ${configs.length}`);\n // The review endpoint must belong to the page's active shopping origin.\n const siteURL = (value, base) => {\n const configured = new URL(value, base);\n if (configured.username || configured.password || configured.hash ||\n configured.origin !== initial.origin) throw Error('Foreign shopping origin');\n return {configured, requested: configured};\n };\n const endpointBinding = siteURL(configs[0].productReviewUrl, initial);\n const endpoint = endpointBinding.requested;\n if (endpoint.origin !== initial.origin || endpoint.username || endpoint.password ||\n endpoint.hash || endpoint.search ||\n endpoint.pathname !== `/review/product/listAjax/id/${productId}/`)\n throw Error('Review endpoint is not bound to current product');\n const pageUrl = value => {\n const {requested: url} = siteURL(value, endpoint);\n if (url.origin !== endpoint.origin || url.pathname !== endpoint.pathname ||\n url.username || url.password || url.hash) throw Error('Foreign review page');\n const pairs = [...url.searchParams];\n if (pairs.length > 1 || pairs.some(([key, value]) => key !== 'p' || !/^[1-9][0-9]*$/.test(value)))\n throw Error('Unexpected review query');\n return {url, page: natural(url.searchParams.get('p') || '1')};\n };\n const rows = [], seen = new Set(), pages = [];\n let next = endpoint.href, expectedPage = 1;\n while (next) {\n if (pages.length >= 100) throw Error('Review pagination exceeds bounded scope');\n const {url, page} = pageUrl(next);\n if (page !== expectedPage) throw Error('Nonsequential review pagination');\n const response = await fetch(url.href, {credentials: 'same-origin', redirect: 'error', signal:controller.signal});\n if (!response.ok || response.redirected || response.url !== url.href)\n throw Error('Review response identity mismatch');\n const html = await response.text();\n const doc = new DOMParser().parseFromString(html, 'text/html');\n const items = [...doc.querySelectorAll('.review-items > .review-item')];\n if (!items.length && total !== 0) throw Error('Missing review rows');\n for (const item of items) {\n const title = one(item, '.review-title').textContent.trim();\n const rating = one(item, '.review-ratings .rating-result');\n const match = /^review_([1-9][0-9]*)_vote_([1-9][0-9]*)$/.exec(rating.id);\n if (!match || seen.has(match[1]) || !title) throw Error('Missing or duplicate review identity');\n if (one(rating, '[itemprop=\"bestRating\"]').getAttribute('content') !== '100' ||\n one(rating, '[itemprop=\"worstRating\"]').getAttribute('content') !== '1')\n throw Error('Unsupported review rating scale');\n const display = one(rating, '[itemprop=\"ratingValue\"]').textContent.trim();\n if (!/^(20|40|60|80|100)%$/.test(display) || rating.title !== display)\n throw Error('Inconsistent individual review rating');\n seen.add(match[1]);\n rows.push({review_id: match[1], title, stars: Number(display.slice(0, -1)) / 20});\n }\n const current = [...doc.querySelectorAll('.review-toolbar .item.current strong.page')]\n .map(node => natural(one(node, 'span:not(.label)').textContent.trim()));\n if (current.some(value => value !== page) || (!current.length && page !== 1))\n throw Error('Review current-page marker mismatch');\n const links = [...new Set([...doc.querySelectorAll('.review-toolbar .pages-item-next a[href]')]\n .map(node => pageUrl(node.getAttribute('href')).url.href))];\n if (links.length > 1) throw Error('Inconsistent next-page links');\n pages.push({url: url.href, page, count: items.length});\n next = links[0] || null;\n expectedPage++;\n if (rows.length > total) throw Error('Review count exceeds product total');\n }\n if (rows.length !== total || location.href !== initial.href ||\n one(document, '#product_addtocart_form input[name=\"product\"]').value !== productId ||\n natural(one(document, countSelector).textContent.trim()) !== total)\n throw Error('Incomplete or changed product review scope');\n return JSON.stringify({product_id: productId, product_path: expectedPath,\n scope: true, complete: true, total, pages, rows,\n product_source: {configured:initial.href, requested:productResponse.url, product_id:productId, total},\n review_endpoint: {configured:endpointBinding.configured.href, requested:endpoint.href},\n answer: rows.filter(row => row.stars <= 2).map(row => row.title)});\n } finally { clearTimeout(deadlineTimer); }\n})(\"/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html\")" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 163, + "canonical_calls_sha256": "ca81826179ccce875f9d18637d6e67949b0c5a7cbaaf31c706371eb50fb28b59", + "sources": [ + { + "name": "reviews", + "step": 7, + "action_sha256": "d54ef91ac88f66a7d5535be3d1c360a182a555232cffbb319e2c5913fddbfb6e", + "decode": "json" + } + ], + "projection": { + "distinct_exact_strings": { + "source": "reviews", + "pointer": "/answer" + } + }, + "requirements": { + "scope": { + "source": "reviews", + "pointer": "/scope" + }, + "completeness": { + "source": "reviews", + "pointer": "/complete" + } + }, + "empty_result": "NOT_FOUND_ERROR", + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "Codex task-163 remote-origin contract audit" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "task_id": 163, + "sources_sha256": "3c597797f498bffe12ad9e7597f757580fdc7b08737a8383b5f04a77f3189152", + "projection_sha256": "4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "Codex task-163 remote-origin contract audit" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "task_id": 163, + "sources_sha256": "3c597797f498bffe12ad9e7597f757580fdc7b08737a8383b5f04a77f3189152", + "projection_sha256": "4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8", + "decision": "Carry forward the reviewed exact-title value-set interpretation after replacing only the deployment-specific origin guard.", + "rationale": "The observation still verifies the same product ID, complete review pagination, rating scale and title projection. Its endpoint now must be the page's active origin rather than an enumerated local Docker host and port.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"retrieval_pair_runner agent, parent-authorized contract revision\"\n },\n \"scope\": \"template 136 review-title value set; exact strings, not review-record multiplicity\",\n \"task_id\": 163,\n \"sources_sha256\": \"fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6\",\n \"projection_sha256\": \"4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8\",\n \"decision\": \"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\",\n \"rationale\": \"Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator.\"\n}", + "sha256": "8e5429f1fa3fdaefb2311ab0d103063b25e1064b3f974fd76f10f9e6e3584f45", + "content": { + "reviewer": { + "kind": "agent", + "identity": "retrieval_pair_runner agent, parent-authorized contract revision" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "task_id": 163, + "sources_sha256": "fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6", + "projection_sha256": "4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "rationale": "Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator." + } + } + ] + }, + "source_review_sha256": "fa2ddebd9d7e8c0b19865a75a4074c2f28008828ff3caf3ea791327a4c01e1c9", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex task-163 remote-origin contract audit\"\n },\n \"scope\": \"template 136 review-title value set; exact strings, not review-record multiplicity\",\n \"task_id\": 163,\n \"sources_sha256\": \"3c597797f498bffe12ad9e7597f757580fdc7b08737a8383b5f04a77f3189152\",\n \"projection_sha256\": \"4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8\",\n \"decision\": \"Carry forward the reviewed exact-title value-set interpretation after replacing only the deployment-specific origin guard.\",\n \"rationale\": \"The observation still verifies the same product ID, complete review pagination, rating scale and title projection. Its endpoint now must be the page's active origin rather than an enumerated local Docker host and port.\",\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"retrieval_pair_runner agent, parent-authorized contract revision\\\"\\n },\\n \\\"scope\\\": \\\"template 136 review-title value set; exact strings, not review-record multiplicity\\\",\\n \\\"task_id\\\": 163,\\n \\\"sources_sha256\\\": \\\"fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6\\\",\\n \\\"projection_sha256\\\": \\\"4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8\\\",\\n \\\"decision\\\": \\\"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\\\",\\n \\\"rationale\\\": \\\"Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator.\\\"\\n}\",\n \"sha256\": \"8e5429f1fa3fdaefb2311ab0d103063b25e1064b3f974fd76f10f9e6e3584f45\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"retrieval_pair_runner agent, parent-authorized contract revision\"\n },\n \"scope\": \"template 136 review-title value set; exact strings, not review-record multiplicity\",\n \"task_id\": 163,\n \"sources_sha256\": \"fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6\",\n \"projection_sha256\": \"4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8\",\n \"decision\": \"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\",\n \"rationale\": \"Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator.\"\n }\n }\n ]\n}", + "sha256": "dcf68ab1ab400c99b48735cbfd3cf903062eb25320c8b2c9a311909275aeb2b5", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex task-163 remote-origin contract audit" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "task_id": 163, + "sources_sha256": "3c597797f498bffe12ad9e7597f757580fdc7b08737a8383b5f04a77f3189152", + "projection_sha256": "4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8", + "decision": "Carry forward the reviewed exact-title value-set interpretation after replacing only the deployment-specific origin guard.", + "rationale": "The observation still verifies the same product ID, complete review pagination, rating scale and title projection. Its endpoint now must be the page's active origin rather than an enumerated local Docker host and port.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"retrieval_pair_runner agent, parent-authorized contract revision\"\n },\n \"scope\": \"template 136 review-title value set; exact strings, not review-record multiplicity\",\n \"task_id\": 163,\n \"sources_sha256\": \"fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6\",\n \"projection_sha256\": \"4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8\",\n \"decision\": \"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\",\n \"rationale\": \"Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator.\"\n}", + "sha256": "8e5429f1fa3fdaefb2311ab0d103063b25e1064b3f974fd76f10f9e6e3584f45", + "content": { + "reviewer": { + "kind": "agent", + "identity": "retrieval_pair_runner agent, parent-authorized contract revision" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "task_id": 163, + "sources_sha256": "fefa3a82138b0758b6e2e69ec65201e66ff03eb679b40ee79167a436e2993aa6", + "projection_sha256": "4b9726268258709613df0398ac930362ea2f5c9d0bf73c69fc186489d58600f8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "rationale": "Verified revision 2 expects title values, but array schema does not specify multiplicity. Explicit reviewed value-set declaration resolves this ambiguity without changing ordinary arrays or official comparator." + } + } + ] + } + } + }, + "semantic_reference": { + "kind": "reviewed_value_set", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "Codex task-163 remote-origin contract audit" + }, + "scope": "template 136 review-title value set; exact strings, not review-record multiplicity", + "evidence_sha256": "d2ecfb903b2ba64d6a4ff2364885fcb0d739b543ffc646c11c0b42e7711a05b4" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-168.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-168.json new file mode 100644 index 000000000..d1f44dc39 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-168.json @@ -0,0 +1,197 @@ +{ + "task_id": 168, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/?personal=true&sort=stars_desc" + }, + { + "action": "evaluate", + "script": "(async()=>{\n const ctl=new AbortController(),timer=setTimeout(()=>ctl.abort(),30000);\n try{\n const api=location.origin+'/api/v4';\n async function read(url){\n const target=new URL(url);\n if(target.origin!==location.origin||!target.pathname.startsWith('/api/v4/'))throw Error('foreign GitLab API');\n const response=await fetch(target.href,{credentials:'same-origin',redirect:'error',cache:'no-store',signal:ctl.signal,headers:{Accept:'application/json'}});\n if(response.status!==200||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('authenticated JSON read failed');\n return {response,body:await response.json()};\n }\n const {body:user}=await read(api+'/user');\n if(!Number.isInteger(user.id)||user.username!=='byteblaze')throw Error('wrong current-user identity');\n const records=[],pages=[],seen=new Set(),size=100;\n let total=null;\n for(let page=1;page<=100;page++){\n const url=new URL(api+'/projects');\n for(const [key,value] of Object.entries({owned:'true',per_page:String(size),page:String(page)}))url.searchParams.set(key,value);\n const {response,body}=await read(url.href);\n const current=response.headers.get('x-page'),next=response.headers.get('x-next-page'),reported=Number(response.headers.get('x-total'));\n if(current!==String(page)||next===null||!Number.isSafeInteger(reported)||reported<0||!Array.isArray(body)||body.length>size||total!==null&&reported!==total)throw Error('owned-project pagination invalid');\n total=reported;\n const ids=[];\n for(const item of body){\n if(!Number.isInteger(item.id)||seen.has(item.id)||item.namespace?.kind!=='user'||item.namespace?.path!==user.username||item.owner?.id!==user.id||!Number.isInteger(item.star_count)||item.star_count<0)throw Error('not a unique owned personal project');\n seen.add(item.id);ids.push(String(item.id));\n records.push({id:item.id,star_count:item.star_count,namespace_kind:item.namespace.kind,owner_id:item.owner.id,path_with_namespace:item.path_with_namespace,visibility:item.visibility});\n }\n pages.push({page,rows:body.length,total,has_next:next!=='',ids});\n if(next==='')break;\n if(next!==String(page+1)||page===100)throw Error('owned-project pagination gap or safety bound');\n }\n if(!pages.length||pages.at(-1).has_next||records.length!==total||pages.reduce((n,p)=>n+p.rows,0)!==total||new Set(records.map(x=>x.id)).size!==total)throw Error('owned personal project set incomplete');\n const answer=records.filter(x=>x.star_count>100).map(x=>String(x.id));\n return {scope:{owner_username:user.username,namespace_kind:'user'},scope_ok:true,complete:true,source:'authenticated GitLab owned-project API',page_size:size,total,pages,records,match_count:answer.length,answer};\n }finally{ctl.abort();clearTimeout(timer)}\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 168, + "canonical_calls_sha256": "83b256dcae4d80d544e2c169b09919bcc8b0ba109ccde731d424d6caf656f4f7", + "sources": [ + { + "name": "owned", + "step": 3, + "action_sha256": "e0720548e98e541be6a18f9f25c5f0402bc0e54d60702c1967800fae61a4b03a", + "decode": "json" + } + ], + "semantic_reference": { + "kind": "reviewed_value_set", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "evidence_sha256": "7afe0dc21e6d9566d518e25a00332a8867eed4147cef59bc11879445737a55d8" + }, + "projection": { + "distinct_exact_strings": { + "source": "owned", + "pointer": "/answer" + } + }, + "requirements": { + "scope": { + "source": "owned", + "pointer": "/scope_ok" + }, + "completeness": { + "source": "owned", + "pointer": "/complete" + } + }, + "empty_result": "NOT_FOUND_ERROR", + "absence_coverage": { + "kind": "exhaustive_records/1", + "source": "owned", + "scope_pointer": "/scope", + "expected_scope": { + "owner_username": "byteblaze", + "namespace_kind": "user" + }, + "records_pointer": "/records", + "pages_pointer": "/pages", + "match_count_pointer": "/match_count", + "page_size": 100, + "id_pointer": "/id", + "required_fields": [ + "/id", + "/star_count", + "/namespace_kind", + "/owner_id", + "/path_with_namespace", + "/visibility" + ], + "predicate": { + "field_pointer": "/star_count", + "type": "integer", + "operator": "gt", + "value": 100 + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "task_id": 168, + "sources_sha256": "a8cb987d130f86fcc0dd1af7529159d273e32c327f092c5a77ae876e5291dbde", + "projection_sha256": "9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "task_id": 168, + "sources_sha256": "a8cb987d130f86fcc0dd1af7529159d273e32c327f092c5a77ae876e5291dbde", + "projection_sha256": "9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7", + "decision": "accept", + "rationale": "The owned-project API identifies ownership through owner.id and personal namespace.path. The current user has user ID 2330 and namespace ID 2505; these are different database identity domains. Check owner.id against user.id and namespace.path against user.username, retaining exhaustive pagination and the original star threshold.", + "observation": { + "current_user": "byteblaze", + "total": 12, + "private_count": 3, + "max_star_count": 6, + "matching_count": 0, + "endpoint": "/api/v4/projects?owned=true&per_page=100&page=1", + "pagination": "X-Total=12; X-Next-Page empty" + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current GitLab source investigation\"\n },\n \"scope\": \"Authenticated complete personal-project set for current byteblaze user; stars greater than 100\",\n \"task_id\": 168,\n \"sources_sha256\": \"156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a\",\n \"projection_sha256\": \"9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7\",\n \"decision\": \"accept\",\n \"rationale\": \"The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.\",\n \"observation\": {\n \"current_user\": \"byteblaze\",\n \"total\": 12,\n \"private_count\": 3,\n \"max_star_count\": 6,\n \"matching_count\": 0,\n \"endpoint\": \"/api/v4/projects?owned=true&per_page=100&page=1\",\n \"pagination\": \"X-Total=12; X-Next-Page empty\"\n }\n}", + "sha256": "f9512cfaf465d123ffabf2a30bb3dbd221c431381613e84308c359f943889891", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "task_id": 168, + "sources_sha256": "156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a", + "projection_sha256": "9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7", + "decision": "accept", + "rationale": "The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.", + "observation": { + "current_user": "byteblaze", + "total": 12, + "private_count": 3, + "max_star_count": 6, + "matching_count": 0, + "endpoint": "/api/v4/projects?owned=true&per_page=100&page=1", + "pagination": "X-Total=12; X-Next-Page empty" + } + } + } + ] + }, + "source_review_sha256": "19a5487d8418ed151a7501e36c9fdcdfa21430ec35c47fbb52256c6d35856777", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current GitLab source investigation\"\n },\n \"scope\": \"Authenticated complete personal-project set for current byteblaze user; stars greater than 100\",\n \"task_id\": 168,\n \"sources_sha256\": \"a8cb987d130f86fcc0dd1af7529159d273e32c327f092c5a77ae876e5291dbde\",\n \"projection_sha256\": \"9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7\",\n \"decision\": \"accept\",\n \"rationale\": \"The owned-project API identifies ownership through owner.id and personal namespace.path. The current user has user ID 2330 and namespace ID 2505; these are different database identity domains. Check owner.id against user.id and namespace.path against user.username, retaining exhaustive pagination and the original star threshold.\",\n \"observation\": {\n \"current_user\": \"byteblaze\",\n \"total\": 12,\n \"private_count\": 3,\n \"max_star_count\": 6,\n \"matching_count\": 0,\n \"endpoint\": \"/api/v4/projects?owned=true&per_page=100&page=1\",\n \"pagination\": \"X-Total=12; X-Next-Page empty\"\n },\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current GitLab source investigation\\\"\\n },\\n \\\"scope\\\": \\\"Authenticated complete personal-project set for current byteblaze user; stars greater than 100\\\",\\n \\\"task_id\\\": 168,\\n \\\"sources_sha256\\\": \\\"156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a\\\",\\n \\\"projection_sha256\\\": \\\"9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.\\\",\\n \\\"observation\\\": {\\n \\\"current_user\\\": \\\"byteblaze\\\",\\n \\\"total\\\": 12,\\n \\\"private_count\\\": 3,\\n \\\"max_star_count\\\": 6,\\n \\\"matching_count\\\": 0,\\n \\\"endpoint\\\": \\\"/api/v4/projects?owned=true&per_page=100&page=1\\\",\\n \\\"pagination\\\": \\\"X-Total=12; X-Next-Page empty\\\"\\n }\\n}\",\n \"sha256\": \"f9512cfaf465d123ffabf2a30bb3dbd221c431381613e84308c359f943889891\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current GitLab source investigation\"\n },\n \"scope\": \"Authenticated complete personal-project set for current byteblaze user; stars greater than 100\",\n \"task_id\": 168,\n \"sources_sha256\": \"156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a\",\n \"projection_sha256\": \"9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7\",\n \"decision\": \"accept\",\n \"rationale\": \"The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.\",\n \"observation\": {\n \"current_user\": \"byteblaze\",\n \"total\": 12,\n \"private_count\": 3,\n \"max_star_count\": 6,\n \"matching_count\": 0,\n \"endpoint\": \"/api/v4/projects?owned=true&per_page=100&page=1\",\n \"pagination\": \"X-Total=12; X-Next-Page empty\"\n }\n }\n }\n ]\n}\n", + "sha256": "d3334db56ae272fd492306fcc4a4b7265cf8c3c14d34d5650b07148c7cda9b68", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "task_id": 168, + "sources_sha256": "a8cb987d130f86fcc0dd1af7529159d273e32c327f092c5a77ae876e5291dbde", + "projection_sha256": "9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7", + "decision": "accept", + "rationale": "The owned-project API identifies ownership through owner.id and personal namespace.path. The current user has user ID 2330 and namespace ID 2505; these are different database identity domains. Check owner.id against user.id and namespace.path against user.username, retaining exhaustive pagination and the original star threshold.", + "observation": { + "current_user": "byteblaze", + "total": 12, + "private_count": 3, + "max_star_count": 6, + "matching_count": 0, + "endpoint": "/api/v4/projects?owned=true&per_page=100&page=1", + "pagination": "X-Total=12; X-Next-Page empty" + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current GitLab source investigation\"\n },\n \"scope\": \"Authenticated complete personal-project set for current byteblaze user; stars greater than 100\",\n \"task_id\": 168,\n \"sources_sha256\": \"156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a\",\n \"projection_sha256\": \"9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7\",\n \"decision\": \"accept\",\n \"rationale\": \"The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.\",\n \"observation\": {\n \"current_user\": \"byteblaze\",\n \"total\": 12,\n \"private_count\": 3,\n \"max_star_count\": 6,\n \"matching_count\": 0,\n \"endpoint\": \"/api/v4/projects?owned=true&per_page=100&page=1\",\n \"pagination\": \"X-Total=12; X-Next-Page empty\"\n }\n}", + "sha256": "f9512cfaf465d123ffabf2a30bb3dbd221c431381613e84308c359f943889891", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current GitLab source investigation" + }, + "scope": "Authenticated complete personal-project set for current byteblaze user; stars greater than 100", + "task_id": 168, + "sources_sha256": "156e43a46be84d605bd4fd5004a481835cd4d29e2e2661ccd4c2442dd2f3895a", + "projection_sha256": "9f0876f2ea3b3be0ccbd766aaf91d3f8ac12ef7f8a84202b7609f04217e34ce7", + "decision": "accept", + "rationale": "The authenticated current-user API and owned-project endpoint returned one terminal page with all 12 distinct projects, including three private projects. Each project is owned by the current user's personal namespace; the greatest observed star count was 6. No project exceeds 100 stars.", + "observation": { + "current_user": "byteblaze", + "total": 12, + "private_count": 3, + "max_star_count": 6, + "matching_count": 0, + "endpoint": "/api/v4/projects?owned=true&per_page=100&page=1", + "pagination": "X-Total=12; X-Next-Page empty" + } + } + } + ] + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-173.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-173.json new file mode 100644 index 000000000..a35665406 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-173.json @@ -0,0 +1,52 @@ +{ + "task_id": 173, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/dashboard/issues?assignee_username=byteblaze\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 20000);\n try {\n // DECISION_START: identical pure logic is exercised by offline controls.\n function decide(observed) {\n const fail = reason => { throw new Error(reason); };\n const owner = new URL(observed.issuesNavigation, observed.origin).searchParams.get('assignee_username');\n if (!owner || new URL(observed.issuesNavigation, observed.origin).origin !== observed.origin || new URL(observed.issuesNavigation, observed.origin).pathname !== '/dashboard/issues') fail('missing current-user Issues navigation');\n if (!observed.pages.length) fail('missing pages');\n const count = observed.pages[0].total;\n if (!Number.isSafeInteger(count) || count < 0) fail('invalid All counter');\n const rows = [], identities = new Set();\n for (let i = 0; i < observed.pages.length; i++) {\n const page = observed.pages[i];\n for (const address of [page.url, page.allHref]) {\n const url = new URL(address, observed.origin);\n if (url.origin !== observed.origin || url.pathname !== '/dashboard/issues' ||\n url.searchParams.get('assignee_username') !== owner || url.searchParams.has('author_username') ||\n url.searchParams.get('state') !== 'all' || url.searchParams.get('search') !== 'better' ||\n url.searchParams.get('sort') !== 'updated_desc') fail('wrong issue scope');\n }\n if (page.status !== 200 || page.total !== count || page.owner !== owner) fail('response or scope drift');\n if (Number(new URL(page.url).searchParams.get('page') || 1) !== i + 1) fail('missing page');\n if (Boolean(page.next) !== (i + 1 < observed.pages.length)) fail('incomplete pagination');\n if (page.next && new URL(page.next, observed.origin).href !== observed.pages[i + 1].url) fail('pagination chain mismatch');\n for (const row of page.rows) {\n const url = new URL(row.href, observed.origin);\n if (url.origin !== observed.origin || !/\\/-\\/issues\\/\\d+$/.test(url.pathname) || url.search || url.hash || identities.has(url.href)) fail('invalid or duplicate issue');\n if (!row.title || !row.assignees.includes(owner) || !/^\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(?:\\.\\d+)?Z$/.test(row.updatedAt) || !Number.isFinite(Date.parse(row.updatedAt))) fail('missing row identity, ownership or updated time');\n identities.add(url.href); rows.push({...row, href:url.href});\n }\n }\n if (rows.length !== count) fail('All count differs from complete unique rows');\n const matches = rows.filter(row => row.title.toLowerCase().includes('better')).sort((a,b) => Date.parse(b.updatedAt)-Date.parse(a.updatedAt));\n if (!matches.length) fail('no title match; absence is not false');\n if (matches[1] && Date.parse(matches[0].updatedAt) === Date.parse(matches[1].updatedAt)) fail('latest updated identity ambiguous');\n const selected = matches[0];\n if (!observed.detail) return selected;\n const detail = observed.detail;\n if (detail.status !== 200 || detail.url !== selected.href || detail.title !== selected.title || !['Open','Closed'].includes(detail.state)) fail('detail identity or explicit state mismatch');\n return {scope:true, completeness:true, answer:detail.state === 'Closed', selected,\n observed:{issuesNavigation:observed.issuesNavigation,pages:observed.pages,detail}};\n }\n // DECISION_END\n const normalize = value => value.replace(/\\s+/g,' ').trim();\n const one = (doc, selector) => {\n const nodes = [...doc.querySelectorAll(selector)];\n if (nodes.length !== 1) throw new Error('expected one ' + selector);\n return nodes[0];\n };\n const navigation = one(document,'a[aria-label=\"Issues\"]');\n const start = new URL(navigation.getAttribute('href'),location.href);\n if (start.origin !== location.origin || start.pathname !== '/dashboard/issues' || !start.searchParams.get('assignee_username')) throw new Error('not authenticated Issues navigation');\n const observed = {origin:location.origin,issuesNavigation:start.href,pages:[]};\n start.searchParams.set('state','all');start.searchParams.set('search','better');start.searchParams.set('sort','updated_desc');start.searchParams.delete('page');\n const read = async url => {\n const target = new URL(url,location.href);\n if (target.origin !== location.origin || !(target.pathname === '/dashboard/issues' || /\\/-\\/issues\\/\\d+$/.test(target.pathname))) throw new Error('out-of-scope read URL');\n const response = await fetch(url,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\n if (response.status !== 200 || !/^text\\/html\\b/i.test(response.headers.get('content-type') || '')) throw new Error('not an HTML 200 observation');\n return {url:response.url,status:response.status,doc:new DOMParser().parseFromString(await response.text(),'text/html')};\n };\n let address = start.href;\n const visited = new Set();\n while (address) {\n if (visited.has(address) || visited.size >= 100) throw new Error('pagination loop or bound exceeded');\n visited.add(address);\n const page = await read(address), doc = page.doc;\n const all = one(doc,'a#state-all');\n const countMatch = normalize(all.textContent).match(/^All\\s*([\\d,]+)$/);\n if (!countMatch) throw new Error('missing All counter');\n const currentNav = new URL(one(doc,'a[aria-label=\"Issues\"]').getAttribute('href'),page.url);\n const nextNodes = [...doc.querySelectorAll('a')].filter(a => normalize(a.textContent)==='Next' && a.getAttribute('href') && a.getAttribute('href') !== '#');\n if (nextNodes.length > 1) throw new Error('ambiguous next page');\n const next = nextNodes.length ? new URL(nextNodes[0].getAttribute('href'),page.url).href : null;\n const rows = [...doc.querySelectorAll('li.issue')].map(row => {\n const title = one(row,'.issue-title-text a');\n const times = [...row.querySelectorAll('time[datetime]')].filter(t => /^updated\\b/i.test(normalize(t.parentElement.textContent)) && t.parentElement.querySelectorAll('time').length === 1);\n if (times.length !== 1) throw new Error('missing explicit updated time');\n const assignees = [...row.querySelectorAll('a[data-qa-selector=\"assignee_link\"]')].map(a => new URL(a.getAttribute('href'),page.url).pathname.slice(1));\n return {href:new URL(title.getAttribute('href'),page.url).href,title:normalize(title.textContent),updatedAt:times[0].getAttribute('datetime'),assignees,\n evidence:{title:title.outerHTML,updated:times[0].parentElement.outerHTML}};\n });\n observed.pages.push({url:page.url,status:page.status,owner:currentNav.searchParams.get('assignee_username'),allHref:new URL(all.getAttribute('href'),page.url).href,total:Number(countMatch[1].replaceAll(',','')),rows,next});\n address = next;\n }\n const selected = decide(observed);\n const detail = await read(selected.href);\n const badges = [...detail.doc.querySelectorAll('.issuable-status-badge')].filter(b => !b.classList.contains('hidden') && !b.hasAttribute('hidden') && !/display\\s*:\\s*none/.test(b.getAttribute('style') || ''));\n if (badges.length !== 1) throw new Error('missing or ambiguous explicit issue state');\n observed.detail = {url:detail.url,status:detail.status,title:normalize(one(detail.doc,'h1.title').textContent),state:normalize(badges[0].textContent),evidence:badges[0].outerHTML};\n return decide(observed);\n } finally { clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 173, + "canonical_calls_sha256": "9f278b55f0847c5a26970ea6cdcfdee81a8c4791986585abcde26344089c3dd5", + "sources": [ + { + "name": "current", + "step": 4, + "action_sha256": "e40c0656b121fffdfd115bebd2af975e5f3bd0d4b2085305474f8e7cc43cb93a", + "decode": "json" + } + ], + "projection": { + "array": [ + { + "source": "current", + "pointer": "/answer" + } + ] + }, + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/completeness" + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-183.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-183.json new file mode 100644 index 000000000..1ec2f1a02 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-183.json @@ -0,0 +1,333 @@ +{ + "task_id": 183, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(), timer=setTimeout(()=>controller.abort(),120000);try{\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/catalog/product\")throw Error('wrong current page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='product_listing'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\nconst params={...p.params,sorting:{field:'entity_id',direction:'asc'},search:'',filters:{placeholder:true},paging:{current:1,pageSize:200}},size=params.paging.pageSize;\nif(!Number.isInteger(size)||size<1||size>1000)throw Error('invalid page size');\nconst endpoint=new URL(p.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/')throw Error('wrong grid endpoint');\nconst flatten=(v,prefix='')=>Object.entries(v).flatMap(([k,x])=>{const key=prefix?`${prefix}[${k}]`:k;\nreturn x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]]});\nfor(const [k,v] of flatten(params))endpoint.searchParams.set(k,v);\nlet total=null;const rows=[],pages=[];\nfor(let page=1;page<=100;page++){endpoint.searchParams.set('paging[current]',String(page));\nconst response=await fetch(endpoint.href,{credentials:'same-origin',cache:'no-store',redirect:'error',\nsignal:controller.signal,headers:{Accept:'application/json'}});\nif(!response.ok||response.redirected||!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||''))throw Error('grid fetch failed');\nconst data=await response.json();\nif(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\nif(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items))throw Error('invalid grid JSON');\nif(total===null)total=data.totalRecords;\nif(total!==data.totalRecords||Math.ceil(total/size)>100||\ndata.items.length!==Math.min(size,Math.max(0,total-(page-1)*size)))throw Error('incomplete or changed page');\nrows.push(...data.items);pages.push({page,rows:data.items.length,total});\nif(page>=Math.max(1,Math.ceil(total/size)))break;\n}\nif(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total||\nrows.some(r=>r.entity_id===null||r.entity_id===undefined))throw Error('incomplete or duplicate grid identities');\nif(!/product/i.test(params.namespace))throw Error('wrong product grid');\nif(Object.entries(params.filters||{}).some(([k,v])=>k!=='placeholder'&&v!=null&&v!==''))throw Error('filtered product grid');\nconst selected=[];for(const r of rows){const qty=r.qty;if(qty==null||qty===''||!/^\\d+(?:\\.\\d+)?$/.test(String(qty)))throw Error('missing product quantity');\nconst n=Number(qty);if(!Number.isFinite(n)||n<0)throw Error('invalid quantity');\nif(n===10){if(typeof r.sku!=='string'||!r.sku.trim())throw Error('matching product SKU missing');selected.push({entity_id:String(r.entity_id),sku:r.sku.trim(),qty:n});}}\nif(new Set(selected.map(r=>r.sku)).size!==selected.length)throw Error('duplicate matching SKU');\nreturn {scope:true,complete:true,namespace:params.namespace,pages,total,selected,inventory_scope:{namespace:params.namespace},records:rows.map(r=>{const quantity=Number(r.qty)*10000;if(!Number.isSafeInteger(quantity)||typeof r.sku!=='string'||!r.sku.trim())throw Error('invalid inventory record');return {id:String(r.entity_id),sku:r.sku,quantity}}),coverage_pages:pages.map(p=>({...p,has_next:p.pageString(r.entity_id))})),match_count:selected.length,answer:selected.map(r=>r.sku)};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 183, + "canonical_calls_sha256": "099b1d24d9a70c1e615917db29a655b9d4bcac460e834875c288384094b581ee", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "09efe353e8951805e8a922ab612a2c0fe5bcea4798229f9b877ea6f30e346f7c", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "distinct_exact_strings": { + "source": "current", + "pointer": "/answer" + } + }, + "empty_result": "NOT_FOUND_ERROR", + "absence_coverage": { + "kind": "exhaustive_records/1", + "source": "current", + "scope_pointer": "/inventory_scope", + "expected_scope": { + "namespace": "product_listing" + }, + "records_pointer": "/records", + "pages_pointer": "/coverage_pages", + "match_count_pointer": "/match_count", + "page_size": 200, + "id_pointer": "/id", + "required_fields": [ + "/id", + "/sku", + "/quantity" + ], + "predicate": { + "field_pointer": "/quantity", + "type": "integer", + "operator": "eq", + "value": 100000 + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "9b44863eb8599084055cd7b2a9b5a844c37bb0b0338641065b5856916c01dd26", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "9b44863eb8599084055cd7b2a9b5a844c37bb0b0338641065b5856916c01dd26", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged. Pagination is now explicitly ordered by unique entity_id rather than inherited non-unique quantity sorting. Full record coverage and uniqueness remain mandatory.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n },\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current Magento inventory investigation\\\"\\n },\\n \\\"scope\\\": \\\"Complete unfiltered product inventory; quantity equals ten units\\\",\\n \\\"task_id\\\": 183,\\n \\\"sources_sha256\\\": \\\"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\\\",\\n \\\"projection_sha256\\\": \\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\\\",\\n \\\"observation\\\": {\\n \\\"total\\\": 2040,\\n \\\"matching_records\\\": [],\\n \\\"pages\\\": [\\n {\\n \\\"page\\\": 1,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 2,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 3,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 40\\n }\\n ],\\n \\\"source_observation_sha256\\\": \\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\",\\n \\\"endpoint\\\": \\\"/admin/mui/index/render/?namespace=product_listing\\\",\\n \\\"filters\\\": {}\\n }\\n}\\n\",\n \"sha256\": \"8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n }\n }\n }\n ]\n}\n", + "sha256": "ce3a68382077260c83ac91087079b8af8999c4b758c2105c3ef0966c6396b75d", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n }\n}\n", + "sha256": "8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + } + } + } + ] + } + } + ] + }, + "source_review_sha256": "35150737c4a64cce135360cc6b16a89baf2a1f62be322014605db0c921b73cd8", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"9b44863eb8599084055cd7b2a9b5a844c37bb0b0338641065b5856916c01dd26\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged. Pagination is now explicitly ordered by unique entity_id rather than inherited non-unique quantity sorting. Full record coverage and uniqueness remain mandatory.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n },\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current Magento inventory investigation\\\"\\n },\\n \\\"scope\\\": \\\"Complete unfiltered product inventory; quantity equals ten units\\\",\\n \\\"task_id\\\": 183,\\n \\\"sources_sha256\\\": \\\"7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27\\\",\\n \\\"projection_sha256\\\": \\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.\\\",\\n \\\"observation\\\": {\\n \\\"total\\\": 2040,\\n \\\"matching_records\\\": [],\\n \\\"pages\\\": [\\n {\\n \\\"page\\\": 1,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 2,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 3,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 40\\n }\\n ],\\n \\\"source_observation_sha256\\\": \\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\",\\n \\\"endpoint\\\": \\\"/admin/mui/index/render/?namespace=product_listing\\\",\\n \\\"filters\\\": {}\\n },\\n \\\"predecessor_artifacts\\\": [\\n {\\n \\\"utf8\\\": \\\"{\\\\n \\\\\\\"reviewer\\\\\\\": {\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"agent\\\\\\\",\\\\n \\\\\\\"identity\\\\\\\": \\\\\\\"Codex current Magento inventory investigation\\\\\\\"\\\\n },\\\\n \\\\\\\"scope\\\\\\\": \\\\\\\"Complete unfiltered product inventory; quantity equals ten units\\\\\\\",\\\\n \\\\\\\"task_id\\\\\\\": 183,\\\\n \\\\\\\"sources_sha256\\\\\\\": \\\\\\\"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\\\\\\\",\\\\n \\\\\\\"projection_sha256\\\\\\\": \\\\\\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\\\\\",\\\\n \\\\\\\"decision\\\\\\\": \\\\\\\"accept\\\\\\\",\\\\n \\\\\\\"rationale\\\\\\\": \\\\\\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\\\\\\\",\\\\n \\\\\\\"observation\\\\\\\": {\\\\n \\\\\\\"total\\\\\\\": 2040,\\\\n \\\\\\\"matching_records\\\\\\\": [],\\\\n \\\\\\\"pages\\\\\\\": [\\\\n {\\\\n \\\\\\\"page\\\\\\\": 1,\\\\n \\\\\\\"total\\\\\\\": 2040,\\\\n \\\\\\\"rows\\\\\\\": 1000\\\\n },\\\\n {\\\\n \\\\\\\"page\\\\\\\": 2,\\\\n \\\\\\\"total\\\\\\\": 2040,\\\\n \\\\\\\"rows\\\\\\\": 1000\\\\n },\\\\n {\\\\n \\\\\\\"page\\\\\\\": 3,\\\\n \\\\\\\"total\\\\\\\": 2040,\\\\n \\\\\\\"rows\\\\\\\": 40\\\\n }\\\\n ],\\\\n \\\\\\\"source_observation_sha256\\\\\\\": \\\\\\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\\\\\",\\\\n \\\\\\\"endpoint\\\\\\\": \\\\\\\"/admin/mui/index/render/?namespace=product_listing\\\\\\\",\\\\n \\\\\\\"filters\\\\\\\": {}\\\\n }\\\\n}\\\\n\\\",\\n \\\"sha256\\\": \\\"8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3\\\",\\n \\\"content\\\": {\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current Magento inventory investigation\\\"\\n },\\n \\\"scope\\\": \\\"Complete unfiltered product inventory; quantity equals ten units\\\",\\n \\\"task_id\\\": 183,\\n \\\"sources_sha256\\\": \\\"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\\\",\\n \\\"projection_sha256\\\": \\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\\\",\\n \\\"observation\\\": {\\n \\\"total\\\": 2040,\\n \\\"matching_records\\\": [],\\n \\\"pages\\\": [\\n {\\n \\\"page\\\": 1,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 2,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 3,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 40\\n }\\n ],\\n \\\"source_observation_sha256\\\": \\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\",\\n \\\"endpoint\\\": \\\"/admin/mui/index/render/?namespace=product_listing\\\",\\n \\\"filters\\\": {}\\n }\\n }\\n }\\n ]\\n}\\n\",\n \"sha256\": \"ce3a68382077260c83ac91087079b8af8999c4b758c2105c3ef0966c6396b75d\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n },\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current Magento inventory investigation\\\"\\n },\\n \\\"scope\\\": \\\"Complete unfiltered product inventory; quantity equals ten units\\\",\\n \\\"task_id\\\": 183,\\n \\\"sources_sha256\\\": \\\"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\\\",\\n \\\"projection_sha256\\\": \\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\\\",\\n \\\"observation\\\": {\\n \\\"total\\\": 2040,\\n \\\"matching_records\\\": [],\\n \\\"pages\\\": [\\n {\\n \\\"page\\\": 1,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 2,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 3,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 40\\n }\\n ],\\n \\\"source_observation_sha256\\\": \\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\",\\n \\\"endpoint\\\": \\\"/admin/mui/index/render/?namespace=product_listing\\\",\\n \\\"filters\\\": {}\\n }\\n}\\n\",\n \"sha256\": \"8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n }\n }\n }\n ]\n }\n }\n ]\n}\n", + "sha256": "7d116462b87e5e255ceb1416505c6ef2004eb8bd1bccd4c0e96659b312bc65ed", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "9b44863eb8599084055cd7b2a9b5a844c37bb0b0338641065b5856916c01dd26", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged. Pagination is now explicitly ordered by unique entity_id rather than inherited non-unique quantity sorting. Full record coverage and uniqueness remain mandatory.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n },\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"reviewer\\\": {\\n \\\"kind\\\": \\\"agent\\\",\\n \\\"identity\\\": \\\"Codex current Magento inventory investigation\\\"\\n },\\n \\\"scope\\\": \\\"Complete unfiltered product inventory; quantity equals ten units\\\",\\n \\\"task_id\\\": 183,\\n \\\"sources_sha256\\\": \\\"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\\\",\\n \\\"projection_sha256\\\": \\\"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\\\",\\n \\\"decision\\\": \\\"accept\\\",\\n \\\"rationale\\\": \\\"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\\\",\\n \\\"observation\\\": {\\n \\\"total\\\": 2040,\\n \\\"matching_records\\\": [],\\n \\\"pages\\\": [\\n {\\n \\\"page\\\": 1,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 2,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 1000\\n },\\n {\\n \\\"page\\\": 3,\\n \\\"total\\\": 2040,\\n \\\"rows\\\": 40\\n }\\n ],\\n \\\"source_observation_sha256\\\": \\\"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\\\",\\n \\\"endpoint\\\": \\\"/admin/mui/index/render/?namespace=product_listing\\\",\\n \\\"filters\\\": {}\\n }\\n}\\n\",\n \"sha256\": \"8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3\",\n \"content\": {\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n }\n }\n }\n ]\n}\n", + "sha256": "ce3a68382077260c83ac91087079b8af8999c4b758c2105c3ef0966c6396b75d", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "7ce350c2006993925ce19b3a99d28d72a0a55b616ae31263399625683d4ffc27", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected. The observer now waits for the current product grid request to finish before reading; inventory scope, observed record requirements and answer projection are unchanged.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + }, + "predecessor_artifacts": [ + { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento inventory investigation\"\n },\n \"scope\": \"Complete unfiltered product inventory; quantity equals ten units\",\n \"task_id\": 183,\n \"sources_sha256\": \"0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc\",\n \"projection_sha256\": \"bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90\",\n \"decision\": \"accept\",\n \"rationale\": \"The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.\",\n \"observation\": {\n \"total\": 2040,\n \"matching_records\": [],\n \"pages\": [\n {\n \"page\": 1,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 2,\n \"total\": 2040,\n \"rows\": 1000\n },\n {\n \"page\": 3,\n \"total\": 2040,\n \"rows\": 40\n }\n ],\n \"source_observation_sha256\": \"fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0\",\n \"endpoint\": \"/admin/mui/index/render/?namespace=product_listing\",\n \"filters\": {}\n }\n}\n", + "sha256": "8405ae660e2f37f1c5046b64e5e7168275f080cd91bbc585f67fddc83ce781f3", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "task_id": 183, + "sources_sha256": "0be66b48432e3e00de43c3e4f54f1a86f2106a024db1155c361324db045893dc", + "projection_sha256": "bdeced597fe32ddbe7cbf2dc56ebc3d8b650e4f2f4c8b5cedebec84749ea7d90", + "decision": "accept", + "rationale": "The observed product listing has 2040 distinct inventory records and no record with quantity ten. Current-run output must supply the complete independent-checkable record population and page identities; a bare empty answer is rejected.", + "observation": { + "total": 2040, + "matching_records": [], + "pages": [ + { + "page": 1, + "total": 2040, + "rows": 1000 + }, + { + "page": 2, + "total": 2040, + "rows": 1000 + }, + { + "page": 3, + "total": 2040, + "rows": 40 + } + ], + "source_observation_sha256": "fdfee7e6215175911a3aa4d3884a670cbb53298259289c9ed83c1332abded2d0", + "endpoint": "/admin/mui/index/render/?namespace=product_listing", + "filters": {} + } + } + } + ] + } + } + ] + } + } + }, + "semantic_reference": { + "kind": "reviewed_value_set", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento inventory investigation" + }, + "scope": "Complete unfiltered product inventory; quantity equals ten units", + "evidence_sha256": "237b04469a3347150b71a6748a62473a2033c8ec666a8a98926254b602fa8fb0" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-184.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-184.json new file mode 100644 index 000000000..29b60d853 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-184.json @@ -0,0 +1,40 @@ +{ + "task_id": 184, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(), timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/catalog/product\")throw Error('wrong current page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='product_listing'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\nconst params={...p.params,sorting:{field:'entity_id',direction:'asc'},search:'',filters:{placeholder:true,qty:{from:'0',to:'0'},type_id:'simple'},paging:{current:1,pageSize:200}},size=params.paging.pageSize;\nif(!Number.isInteger(size)||size<1||size>1000)throw Error('invalid page size');\nconst endpoint=new URL(p.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/')throw Error('wrong grid endpoint');\nconst flatten=(v,prefix='')=>Object.entries(v).flatMap(([k,x])=>{const key=prefix?`${prefix}[${k}]`:k;\nreturn x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]]});\nfor(const [k,v] of flatten(params))endpoint.searchParams.set(k,v);\nlet total=null;const rows=[],pages=[];\nfor(let page=1;page<=100;page++){endpoint.searchParams.set('paging[current]',String(page));\nconst response=await fetch(endpoint.href,{credentials:'same-origin',cache:'no-store',redirect:'error',\nsignal:controller.signal,headers:{Accept:'application/json'}});\nif(!response.ok||response.redirected||!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||''))throw Error('grid fetch failed');\nconst data=await response.json();\nif(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\nif(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items))throw Error('invalid grid JSON');\nif(total===null)total=data.totalRecords;\nif(total!==data.totalRecords||Math.ceil(total/size)>100||\ndata.items.length!==Math.min(size,Math.max(0,total-(page-1)*size)))throw Error('incomplete or changed page');\nrows.push(...data.items);pages.push({page,rows:data.items.length,total});\nif(page>=Math.max(1,Math.ceil(total/size)))break;\n}\nif(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total||\nrows.some(r=>r.entity_id===null||r.entity_id===undefined))throw Error('incomplete or duplicate grid identities');\nif(!/product/i.test(params.namespace))throw Error('wrong product grid');\nconst colorColumn=registry.get('product_listing.product_listing.product_columns.color');\nif(!Array.isArray(colorColumn?.options))throw Error('color label dictionary missing');\nconst colors=new Map();for(const option of colorColumn.options){const key=String(option.value);if(colors.has(key)||typeof option.label!=='string')throw Error('ambiguous color dictionary');colors.set(key,option.label.trim());}\nconst selected=[];for(const r of rows){const qty=r.qty;if(qty==null||qty===''||!/^\\d+(?:\\.\\d+)?$/.test(String(qty)))throw Error('missing product quantity');\nconst n=Number(qty);if(!Number.isFinite(n)||n<0)throw Error('invalid quantity');\nif(n!==0)throw Error('quantity filter mismatch');if(n===0){if(r.type_id!=='simple')throw Error('non-stock product in inventory scope');const name=r.name,color=colors.get(String(r.color));\nif(typeof name!=='string'||!name.trim()||typeof color!=='string'||!color.trim()||/^\\d+$/.test(color.trim()))throw Error('matching product name or color label missing');\nselected.push({entity_id:String(r.entity_id),name:name.trim(),color:color.trim()});}}\nif(new Set(selected.map(r=>r.entity_id)).size!==selected.length)throw Error('duplicate matching product');\nreturn {scope:true,complete:true,namespace:params.namespace,pages,total,selected,answer:selected.map(({name,color})=>({name,color}))};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 184, + "canonical_calls_sha256": "753276e8ce685d39c313d5fcabf5d5d64a70cf0733df080eaaab705ac755246b", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "9950a916fa0d6a5c02b2497218fa51ed7a718e1f1342aa27ecc0f5ce86f3c584", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-188.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-188.json new file mode 100644 index 000000000..fd6829f1d --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-188.json @@ -0,0 +1,40 @@ +{ + "task_id": 188, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].dater.status==='Canceled');if(!match)throw Error('no canceled order');\nif(rows.some(r=>r.status==='Cancelled'))throw Error('ambiguous canceled spelling');\nreturn {scope:true,complete:true,pages,total,selected:match,answer:match.cents/100};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 188, + "canonical_calls_sha256": "89831fdc648a18ffe4bce09dbaf5cada4937e3ee2f2099628a8889f942101e4d", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "5564b66d0522411658a9a69f7869f10ec5841940d783498ab11c08af67c8c4f4", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-193.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-193.json new file mode 100644 index 000000000..cbbe3de7e --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-193.json @@ -0,0 +1,40 @@ +{ + "task_id": 193, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;ir.status==='complete');\n if(completeOrders.length<2||completeOrders[2]?.created_at===completeOrders[1].created_at)throw new Error('last-two completed orders are incomplete or tied');\n const selected=completeOrders.slice(0,2).map(r=>{if(r.order_currency_code!=='USD'||typeof r.grand_total!=='string'||!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(r.grand_total))throw new Error('invalid USD grand total');const cents=Number(r.grand_total.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw new Error('unsafe amount');return {entity_id:String(r.entity_id),created_at:r.created_at,cents};});\n const cents=selected.reduce((n,r)=>n+r.cents,0);if(!Number.isSafeInteger(cents))throw new Error('unsafe sum');\n return {scope:true,complete:true,pages,total,selected,answer:cents/100};\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 193, + "canonical_calls_sha256": "f72f5a16844e932309d83b8c2d8e468369971e0c90b551b477688b949f838bfd", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "44b2e9bfaaa09bd4db70242cec5ea315790b9b38cbf95e02adbdd035696085c8", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-196.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-196.json new file mode 100644 index 000000000..910ec3eb4 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-196.json @@ -0,0 +1,76 @@ +{ + "task_id": 196, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "evaluate", + "script": "(async()=>{const deadline=Date.now()+15000;while(Date.now()setTimeout(r,50));}throw new Error('globalNavigation instance not ready within15s');})()" + }, + { + "action": "click", + "selector": "a[href=\"#\"]:visible:has-text(\"Sales\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const timeoutMs = 60000;\n const deadline = Date.now() + timeoutMs;\n const sleep = () => new Promise(resolve => setTimeout(resolve, 50));\n const meaningful = value => {\n if (value === null || value === undefined || value === '') return false;\n if (typeof value === 'object') return Object.values(value).some(meaningful);\n return true;\n };\n const activeFilters = params => Object.entries(params.filters || {})\n .filter(([key, value]) => key !== 'placeholder' && meaningful(value));\n const visible = node => node.isConnected && node.getClientRects().length > 0 &&\n !['hidden', 'collapse'].includes(getComputedStyle(node).visibility) &&\n getComputedStyle(node).display !== 'none';\n const observe = () => {\n if (typeof requirejs !== 'function' || typeof requirejs.defined !== 'function' ||\n !requirejs.defined('uiRegistry')) return null;\n const providers = [];\n require('uiRegistry').get(component => {\n if (component.component === 'Magento_Ui/js/grid/provider' &&\n component.params?.namespace === 'sales_order_grid') providers.push(component);\n });\n if (providers.length !== 1) return null;\n const provider = providers[0];\n if (provider.firstLoad !== false || provider.lastError !== false || provider.triggerDataReload) return null;\n const storage = require('uiRegistry').get(provider.storageConfig?.name);\n if (!storage || typeof storage.getRequest !== 'function') return null;\n const completed = storage.getRequest(provider.params);\n if (!completed || completed.errorMessage || !Array.isArray(completed.ids)) return null;\n const rawCount=provider.data?.totalRecords;\n const count=/^\\d+$/.test(String(rawCount))?Number(rawCount):NaN;\n const items = provider.data?.items;\n if (!Number.isInteger(count) || count < 0 || !Array.isArray(items) ||\n (count > 0 && items.length === 0)) return null;\n if ((!/^\\d+$/.test(String(completed.totalRecords))||Number(completed.totalRecords)!==count) || completed.ids.length !== items.length) return null;\n return {provider, count, active: activeFilters(provider.params),\n buttons: [...document.querySelectorAll('button')]\n .filter(node => visible(node) && node.textContent.trim() === 'Clear all')};\n };\n let initial;\n while (Date.now() < deadline) {\n initial = observe();\n if (initial) break;\n await sleep();\n }\n if (!initial) throw new Error('Order grid never provided complete initial filter state');\n if (!initial.active.length) {\n if (initial.buttons.length) throw new Error('Filter state and visible clear control disagree');\n return {operation: 'already-clear', namespace: 'sales_order_grid', activeFilterCount: 0};\n }\n if (initial.buttons.length !== 1) throw new Error('Active filters lack one visible clear control');\n initial.buttons[0].click();\n while (Date.now() < deadline) {\n const current = observe();\n if (current && !current.active.length && !current.buttons.length)\n return {operation: 'cleared', namespace: 'sales_order_grid', activeFilterCount: 0};\n await sleep();\n }\n throw new Error('Filter clear did not reach an empty applied state');\n})()\n" + }, + { + "action": "wait", + "timeout": 1500 + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;i{\n const filtered=rows.filter(r=>r.status===status);\n if(filtered.length<4)throw new Error('fewer than four matching orders');\n if(filtered[4]?.created_at===filtered[3].created_at)throw new Error('ambiguous fourth-order timestamp tie');\n return filtered.slice(0,4).map(r=>{\n if(r.order_currency_code!=='USD'||typeof r.grand_total!=='string'||!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(r.grand_total))throw new Error('unsupported or missing purchased amount/currency');\n const cents=Number(r.grand_total.replace(/[$,.]/g,''));\n if(!Number.isSafeInteger(cents))throw new Error('unsafe amount');\n return {entity_id:String(r.entity_id),created_at:r.created_at,status:r.status,currency:r.order_currency_code,cents};\n });\n };\n const canceled=select('canceled'),complete=select('complete');\n const sum=rs=>rs.reduce((n,r)=>n+r.cents,0);\n const canceled_cents=sum(canceled),complete_cents=sum(complete);\n if(!Number.isSafeInteger(canceled_cents)||!Number.isSafeInteger(complete_cents))throw new Error('unsafe total');\n return {scope:true,complete:true,pages,total,selected:{canceled,complete},canceled_cents,complete_cents,\n interpretation:'latest by purchase date; purchased grand totals; canceled minus complete',answer:[(canceled_cents-complete_cents)/100]};\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 196, + "canonical_calls_sha256": "a0413e6b6e28d2f540e4db75ea42337f671cdf65a284f95ede14573d579905ff", + "sources": [ + { + "name": "orders", + "step": 11, + "action_sha256": "e532f06554a020e5efad68cfb596c522ee837f64e767036a1e920ac230cd0693", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "orders", + "pointer": "/scope" + }, + "completeness": { + "source": "orders", + "pointer": "/complete" + } + }, + "projection": { + "source": "orders", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-198.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-198.json new file mode 100644 index 000000000..59a4b22a7 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-198.json @@ -0,0 +1,40 @@ +{ + "task_id": 198, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;ir.status==='canceled');\n if(!canceled.length||canceled[1]?.created_at===canceled[0].created_at)throw new Error('latest canceled order missing or tied');\n const chosen=canceled[0],email=chosen.customer_email;\n if(typeof email!=='string'||!/^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$/.test(email))throw new Error('customer email absent from complete source');\n return {scope:true,complete:true,pages,total,selected:{entity_id:String(chosen.entity_id),created_at:chosen.created_at,email},answer:email};\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 198, + "canonical_calls_sha256": "aaae1dd68fe445ec55ca6184bd95100ff12a55a2b9599f4cdf2fe0890bd63fab", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "827b192aa1689070ce839e5233b62bb6b0fddaaf65e0af5f93a34185f04ac69f", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-204.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-204.json new file mode 100644 index 000000000..b6749aa9c --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-204.json @@ -0,0 +1,64 @@ +{ + "task_id": 204, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "evaluate", + "script": "(async()=>{const deadline=Date.now()+15000;while(Date.now()setTimeout(r,50));}throw new Error('globalNavigation instance not ready within15s');})()" + }, + { + "action": "click", + "selector": "a[href=\"#\"]:visible:has-text(\"Sales\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const timeoutMs = 60000;\n const deadline = Date.now() + timeoutMs;\n const sleep = () => new Promise(resolve => setTimeout(resolve, 50));\n const meaningful = value => {\n if (value === null || value === undefined || value === '') return false;\n if (typeof value === 'object') return Object.values(value).some(meaningful);\n return true;\n };\n const activeFilters = params => Object.entries(params.filters || {})\n .filter(([key, value]) => key !== 'placeholder' && meaningful(value));\n const visible = node => node.isConnected && node.getClientRects().length > 0 &&\n !['hidden', 'collapse'].includes(getComputedStyle(node).visibility) &&\n getComputedStyle(node).display !== 'none';\n const observe = () => {\n if (typeof requirejs !== 'function' || typeof requirejs.defined !== 'function' ||\n !requirejs.defined('uiRegistry')) return null;\n const providers = [];\n require('uiRegistry').get(component => {\n if (component.component === 'Magento_Ui/js/grid/provider' &&\n component.params?.namespace === 'sales_order_grid') providers.push(component);\n });\n if (providers.length !== 1) return null;\n const provider = providers[0];\n if (provider.firstLoad !== false || provider.lastError !== false || provider.triggerDataReload) return null;\n const storage = require('uiRegistry').get(provider.storageConfig?.name);\n if (!storage || typeof storage.getRequest !== 'function') return null;\n const completed = storage.getRequest(provider.params);\n if (!completed || completed.errorMessage || !Array.isArray(completed.ids)) return null;\n const rawCount=provider.data?.totalRecords;\n const count=/^\\d+$/.test(String(rawCount))?Number(rawCount):NaN;\n const items = provider.data?.items;\n if (!Number.isInteger(count) || count < 0 || !Array.isArray(items) ||\n (count > 0 && items.length === 0)) return null;\n if ((!/^\\d+$/.test(String(completed.totalRecords))||Number(completed.totalRecords)!==count) || completed.ids.length !== items.length) return null;\n return {provider, count, active: activeFilters(provider.params),\n buttons: [...document.querySelectorAll('button')]\n .filter(node => visible(node) && node.textContent.trim() === 'Clear all')};\n };\n let initial;\n while (Date.now() < deadline) {\n initial = observe();\n if (initial) break;\n await sleep();\n }\n if (!initial) throw new Error('Order grid never provided complete initial filter state');\n if (!initial.active.length) {\n if (initial.buttons.length) throw new Error('Filter state and visible clear control disagree');\n return {operation: 'already-clear', namespace: 'sales_order_grid', activeFilterCount: 0};\n }\n if (initial.buttons.length !== 1) throw new Error('Active filters lack one visible clear control');\n initial.buttons[0].click();\n while (Date.now() < deadline) {\n const current = observe();\n if (current && !current.active.length && !current.buttons.length)\n return {operation: 'cleared', namespace: 'sales_order_grid', activeFilterCount: 0};\n await sleep();\n }\n throw new Error('Filter clear did not reach an empty applied state');\n})()\n" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\nfunction decide(observed) {\n const {rows,pages,total,detail}=observed;\n const fail = message => {throw new Error(message);};\n if (!observed.scope || !observed.complete || !Number.isInteger(total) || rows.length!==total ||\n new Set(rows.map(r=>String(r.entity_id))).size!==total || !pages.length ||\n pages.reduce((sum,p)=>sum+p.rows,0)!==total ||\n pages.some((p,i)=>p.page!==i+1 || p.total!==total)) fail('incomplete full order grid');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;ir.status==='complete');\n if(!matches.length || matches[1]?.created_at===matches[0].created_at) fail('missing or tied latest complete order');\n const selected=matches[0];\n if(selected.order_currency_code!=='USD' || !/^\\d+$/.test(selected.increment_id)) fail('unsupported order currency/identity');\n if(!detail) return selected;\n if(detail.entity_id!==String(selected.entity_id) || detail.increment_id!==selected.increment_id ||\n detail.status!=='Complete' || detail.http_status!==200 || detail.next ||\n !Number.isInteger(detail.body_count) || detail.body_count<1 || detail.items.length!==detail.body_count ||\n new Set(detail.items.map(i=>i.id)).size!==detail.body_count) fail('wrong order or incomplete item bodies');\n for(const item of detail.items) {\n if(!/^order_item_\\d+$/.test(item.id) || !item.name || !Number.isSafeInteger(item.qty) || item.qty<1 ||\n ['unit','original','subtotal','discount','tax','rowtotal'].some(k=>!Number.isSafeInteger(item[k])||item[k]<0))\n fail('missing item identity/amount/quantity');\n if(item.discount!==0 || item.tax!==0) fail('final-price interpretation ambiguous with row tax/discount');\n if(item.unit*item.qty!==item.subtotal || item.rowtotal!==item.subtotal) fail('unit and row arithmetic disagree');\n }\n if(typeof selected.subtotal!=='string' || !/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(selected.subtotal) ||\n detail.items.reduce((sum,item)=>sum+item.subtotal,0)!==Number(selected.subtotal.replace(/[$,.]/g,'')))\n fail('detail item subtotals do not cover order subtotal');\n return {scope:true,complete:true,selected:{entity_id:String(selected.entity_id),increment_id:selected.increment_id,\n created_at:selected.created_at,status:selected.status},pages,total,detail,\n interpretation:'Price column: per-unit purchased price, not Original Price or Row Total; zero row discount/tax required',\n answer:[...detail.items].sort((a,b)=>a.unit-b.unit||a.id.localeCompare(b.id)).map(i=>({name:i.name,price:i.unit/100}))};\n}\n\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;it.replace(/\\s+/g,' ').trim();\n const one=(node,selector)=>{const ns=[...node.querySelectorAll(selector)];if(ns.length!==1)throw new Error('expected one '+selector);return ns[0];};\n const money=node=>{\n const prices=[...node.querySelectorAll('.price')];\n if(prices.length!==1)throw new Error('missing or multiple currency prices');\n const text=norm(prices[0].textContent);\n if(!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(text))throw new Error('invalid USD price');\n const cents=Number(text.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw new Error('unsafe price');return cents;\n };\n const table=one(doc,'table.edit-order-table');\n const headers={'product':'Product','price-original':'Original Price','price':'Price','ordered-qty':'Qty',\n 'subtotal':'Subtotal','tax-amount':'Tax Amount','discont':'Discount Amount','total':'Row Total'};\n for(const [key,label] of Object.entries(headers)) if(norm(one(table,`thead th.col-${key}`).textContent)!==label)throw new Error('wrong item column semantics');\n const bodies=[...table.children].filter(n=>n.tagName==='TBODY');\n const items=bodies.map(body=>{\n const rows=[...body.children].filter(n=>n.tagName==='TR');\n if(rows.length!==1)throw new Error('unsupported multi-row item renderer');\n const row=rows[0],title=one(row,'.col-product .product-title');\n const id=title.id.replace(/_title$/,'');\n if(title.id!==id+'_title' || !/^order_item_\\d+$/.test(id))throw new Error('missing item id');\n const priceCell=one(row,'td.col-price');\n if(priceCell.querySelector('.price-incl-tax') || !priceCell.querySelector('.price-excl-tax'))throw new Error('unsupported tax display');\n const qtyRows=[...row.querySelectorAll('td.col-ordered-qty table.qty-table tr')].filter(r=>norm(one(r,'th').textContent)==='Ordered');\n if(qtyRows.length!==1)throw new Error('missing Ordered quantity');\n const qtyText=norm(one(qtyRows[0],'td').textContent);\n if(!/^\\d+$/.test(qtyText))throw new Error('unsupported fractional quantity');\n return {id,name:norm(title.textContent),qty:Number(qtyText),unit:money(priceCell),\n original:money(one(row,'td.col-price-original')),subtotal:money(one(row,'td.col-subtotal')),\n tax:money(one(row,'td.col-tax-amount')),discount:money(one(row,'td.col-discont')),rowtotal:money(one(row,'td.col-total'))};\n });\n const ids=[...doc.querySelectorAll('input[name=\"order_id\"]')].map(n=>n.value);\n if(!ids.length || new Set(ids).size!==1)throw new Error('missing consistent detail order id');\n const section=table.closest('section');\n if(!section || ![...section.querySelectorAll('h2,.title')].some(n=>norm(n.textContent)==='Items Ordered'))throw new Error('missing complete Items Ordered section');\n observed.detail={url:response.url,http_status:response.status,entity_id:ids[0],\n increment_id:norm(one(doc,'h1.page-title').textContent).replace(/^#/,''),status:norm(one(doc,'#order_status').textContent),\n body_count:bodies.length,next:!!section.querySelector('[rel=\"next\"],.pages,.admin__data-grid-pager'),items};\n return decide(observed);\n\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 204, + "canonical_calls_sha256": "614431b974142f756bb5aeecc29f190cf0a76b6e93ff4ce7f7a15ee0e0ad4276", + "sources": [ + { + "name": "current", + "step": 8, + "action_sha256": "8fde3feddf74271492ab58335b6f242d9eeaabed36e55e9f78eda62cff42a126", + "decode": "json" + } + ], + "projection": { + "source": "current", + "pointer": "/answer" + }, + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-225.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-225.json new file mode 100644 index 000000000..ea5e72051 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-225.json @@ -0,0 +1,167 @@ +{ + "task_id": 225, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sephora-collection-skin-perfecting-brush-set.html" + }, + { + "action": "evaluate", + "script": "(()=>({product:document.querySelector('h1.page-title')?.textContent.trim(),reviews:[...document.querySelectorAll('.review-item')].map(e=>({title:e.querySelector('.review-title')?.textContent.trim(),rating:e.querySelector('.rating-result')?.getAttribute('title')}))}))()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sephora-collection-pro-featherweight-powder-brush-91.html" + }, + { + "action": "evaluate", + "script": "(()=>({product:document.querySelector('h1.page-title')?.textContent.trim(),reviews:[...document.querySelectorAll('.review-item')].map(e=>({title:e.querySelector('.review-title')?.textContent.trim(),rating:e.querySelector('.rating-result')?.getAttribute('title')}))}))()" + }, + { + "action": "evaluate", + "script": "(async()=>{\n const ctl=new AbortController(),timer=setTimeout(()=>ctl.abort(),30000);\n try{\n const query='{ products(filter: { name: { match: \"Sephora\" } }, pageSize: 100) { total_count items { sku name review_count reviews(pageSize: 100) { items { summary average_rating } page_info { current_page total_pages page_size } } } } }';\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',cache:'no-store',signal:ctl.signal,headers:{Accept:'application/json'}});\n if(response.status!==200||response.redirected||new URL(response.url).origin!==location.origin||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog GraphQL read failed');\n const payload=await response.json();\n if(payload.errors?.length||!payload.data?.products)throw Error('catalog GraphQL errors');\n const source=payload.data.products,all=source.items;\n if(!Number.isInteger(source.total_count)||source.total_count<0||source.total_count>100||!Array.isArray(all)||all.length!==source.total_count||!all.length)throw Error('Sephora title search is incomplete');\n const seen=new Set(),products=[],answer=[];\n for(const product of all){\n if(typeof product.sku!=='string'||!product.sku||seen.has(product.sku)||typeof product.name!=='string'||!/sephora/i.test(product.name))throw Error('catalog product identity mismatch');\n seen.add(product.sku);\n if(!/brush/i.test(product.name))continue;\n const reviews=product.reviews,info=reviews?.page_info,items=reviews?.items;\n if(!Number.isInteger(product.review_count)||product.review_count<0||product.review_count>100||!Array.isArray(items)||items.length!==product.review_count||info?.current_page!==1||info?.page_size!==100||info?.total_pages!==(product.review_count?1:0))throw Error('brush review pagination incomplete');\n let low=0;\n for(const review of items){\n if(typeof review.summary!=='string'||!review.summary.trim()||!Number.isInteger(review.average_rating)||review.average_rating<0||review.average_rating>100||review.average_rating%20!==0)throw Error('review title or star scale invalid');\n if(review.average_rating<=60){low++;answer.push(review.summary.trim())}\n }\n products.push({id:product.sku,name:product.name,review_count:product.review_count,low_rating_count:low,review_titles:items.map(x=>x.summary)});\n }\n if(!products.length||new Set(products.map(x=>x.id)).size!==products.length)throw Error('no uniquely linked Sephora brush products');\n const pages=[{page:1,rows:products.length,total:products.length,has_next:false,ids:products.map(x=>x.id)}];\n return {scope:{catalog_filter:'name match Sephora',product_selection:'title contains Brush'},scope_ok:true,complete:true,graphql_url:url.href,source_total:source.total_count,source_skus:all.map(x=>x.sku),records:products,pages,match_count:products.reduce((n,x)=>n+x.low_rating_count,0),answer};\n }finally{ctl.abort();clearTimeout(timer)}\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 225, + "canonical_calls_sha256": "05c27f12cf05f83f900a98fb96698eaffe6c63a61e9227065c8dc2cc69650a88", + "sources": [ + { + "name": "catalog", + "step": 5, + "action_sha256": "ad4c934a93778227c96e49646a5d2761ea9780d6e37fa7292060cc2b2ae46c9d", + "decode": "json" + } + ], + "semantic_reference": { + "kind": "reviewed_value_set", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento source investigation" + }, + "scope": "Complete storefront title-matched Sephora brush products and their published reviews", + "evidence_sha256": "d24a046a4378cd90509e79d6206b3f35877697ab6d5307575409632b8168f21b" + }, + "projection": { + "distinct_exact_strings": { + "source": "catalog", + "pointer": "/answer" + } + }, + "requirements": { + "scope": { + "source": "catalog", + "pointer": "/scope_ok" + }, + "completeness": { + "source": "catalog", + "pointer": "/complete" + } + }, + "empty_result": "NOT_FOUND_ERROR", + "absence_coverage": { + "kind": "exhaustive_records/1", + "source": "catalog", + "scope_pointer": "/scope", + "expected_scope": { + "catalog_filter": "name match Sephora", + "product_selection": "title contains Brush" + }, + "records_pointer": "/records", + "pages_pointer": "/pages", + "match_count_pointer": "/match_count", + "page_size": 100, + "id_pointer": "/id", + "required_fields": [ + "/id", + "/name", + "/review_count", + "/low_rating_count" + ], + "predicate": { + "field_pointer": "/low_rating_count", + "type": "integer", + "operator": "gt", + "value": 0 + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento source investigation" + }, + "scope": "Complete storefront title-matched Sephora brush products and their published reviews", + "task_id": 225, + "sources_sha256": "c42b23cf49120f5eda0d60f583e361c0efbbc9d01e95146d9dbbc4668adbedfb", + "projection_sha256": "55e4d5867c5d2850d54fc1d88240d8ca9b680284d1e37d03a45e29b87d7a7275", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento source investigation" + }, + "scope": "Complete storefront title-matched Sephora brush products and their published reviews", + "task_id": 225, + "sources_sha256": "c42b23cf49120f5eda0d60f583e361c0efbbc9d01e95146d9dbbc4668adbedfb", + "projection_sha256": "55e4d5867c5d2850d54fc1d88240d8ca9b680284d1e37d03a45e29b87d7a7275", + "decision": "accept", + "rationale": "The public GraphQL catalog query returned all six products whose titles match Sephora; exactly two titles also contain Brush. The catalog DB independently confirms those two product records and zero review rows for each. Manufacturer EAV is unpopulated, so this source review supports a title-based product scope only.", + "observation": { + "graphql_query": "products(filter:{name:{match:\"Sephora\"}},pageSize:100)", + "source_total": 6, + "title_matched_products": [ + { + "sku": "B08YP1ZGNJ", + "name": "Sephora Collection Skin Perfecting Brush Set", + "review_count": 0 + }, + { + "sku": "B01N77Y4ZH", + "name": "SEPHORA COLLECTION PRO Featherweight Powder Brush #91", + "review_count": 0 + } + ], + "manufacturer_eav_rows": 0, + "matching_low_rating_reviews": 0 + } + }, + "source_review_sha256": "5d7cef62baca89f31e081bb46a11c7fe8158064931c24e4833c98dda26d79166", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"Codex current Magento source investigation\"\n },\n \"scope\": \"Complete storefront title-matched Sephora brush products and their published reviews\",\n \"task_id\": 225,\n \"sources_sha256\": \"c42b23cf49120f5eda0d60f583e361c0efbbc9d01e95146d9dbbc4668adbedfb\",\n \"projection_sha256\": \"55e4d5867c5d2850d54fc1d88240d8ca9b680284d1e37d03a45e29b87d7a7275\",\n \"decision\": \"accept\",\n \"rationale\": \"The public GraphQL catalog query returned all six products whose titles match Sephora; exactly two titles also contain Brush. The catalog DB independently confirms those two product records and zero review rows for each. Manufacturer EAV is unpopulated, so this source review supports a title-based product scope only.\",\n \"observation\": {\n \"graphql_query\": \"products(filter:{name:{match:\\\"Sephora\\\"}},pageSize:100)\",\n \"source_total\": 6,\n \"title_matched_products\": [\n {\n \"sku\": \"B08YP1ZGNJ\",\n \"name\": \"Sephora Collection Skin Perfecting Brush Set\",\n \"review_count\": 0\n },\n {\n \"sku\": \"B01N77Y4ZH\",\n \"name\": \"SEPHORA COLLECTION PRO Featherweight Powder Brush #91\",\n \"review_count\": 0\n }\n ],\n \"manufacturer_eav_rows\": 0,\n \"matching_low_rating_reviews\": 0\n }\n}", + "sha256": "5f31bfaa6edf56d3d96e07811d8a54429534a5f2e6831129ea8b1c41914d903d", + "content": { + "reviewer": { + "kind": "agent", + "identity": "Codex current Magento source investigation" + }, + "scope": "Complete storefront title-matched Sephora brush products and their published reviews", + "task_id": 225, + "sources_sha256": "c42b23cf49120f5eda0d60f583e361c0efbbc9d01e95146d9dbbc4668adbedfb", + "projection_sha256": "55e4d5867c5d2850d54fc1d88240d8ca9b680284d1e37d03a45e29b87d7a7275", + "decision": "accept", + "rationale": "The public GraphQL catalog query returned all six products whose titles match Sephora; exactly two titles also contain Brush. The catalog DB independently confirms those two product records and zero review rows for each. Manufacturer EAV is unpopulated, so this source review supports a title-based product scope only.", + "observation": { + "graphql_query": "products(filter:{name:{match:\"Sephora\"}},pageSize:100)", + "source_total": 6, + "title_matched_products": [ + { + "sku": "B08YP1ZGNJ", + "name": "Sephora Collection Skin Perfecting Brush Set", + "review_count": 0 + }, + { + "sku": "B01N77Y4ZH", + "name": "SEPHORA COLLECTION PRO Featherweight Powder Brush #91", + "review_count": 0 + } + ], + "manufacturer_eav_rows": 0, + "matching_low_rating_reviews": 0 + } + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-231.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-231.json new file mode 100644 index 000000000..49528206c --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-231.json @@ -0,0 +1,40 @@ +{ + "task_id": 231, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].dater.status==='Canceled');if(!match)throw Error('no canceled order');\nif(rows.some(r=>r.status==='Cancelled'))throw Error('ambiguous canceled spelling');\nreturn {scope:true,complete:true,pages,total,selected:match,answer:match.number};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 231, + "canonical_calls_sha256": "9af0627850b3f3ac1aa3e31cddae0fbb59603e4fcfa98f99ef418a066ad8a09f", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "019284c832ca880164a3a767e30a320d2faf1728684cdcb52d18bf8dadab5af8", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-243.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-243.json new file mode 100644 index 000000000..545dbc58d --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-243.json @@ -0,0 +1,72 @@ +{ + "schema": "webarena_task243_review_nickname_candidate/1", + "task_id": 243, + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-shopping-admin.dev.lexmount.net/admin" + }, + { + "action": "click", + "selector": "a[href=\"#\"]:visible:has-text(\"Marketing\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href$=\"/admin/review/product/index/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "Circe Hooded Ice Fleece", + "selector": "input[name=\"name\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Search\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const product = 'Circe Hooded Ice Fleece';\n const normalize = value => String(value ?? '').replace(/\\s+/g, ' ').trim();\n const fail = message => { throw new Error(message); };\n const origin = location.origin;\n if (!location.pathname.startsWith('/admin/review/product/index/')) fail('not_on_review_grid');\n const deadline = Date.now() + 90000;\n async function read(path) {\n const url = new URL(path, origin);\n if (url.origin !== origin || Date.now() >= deadline) fail('source_scope_or_deadline');\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), Math.max(1, deadline - Date.now()));\n try {\n const response = await fetch(url, {credentials: 'same-origin', redirect: 'error', signal: controller.signal});\n if (!response.ok || response.redirected) fail('source_http_' + response.status);\n return new DOMParser().parseFromString(await response.text(), 'text/html');\n } finally { clearTimeout(timer); }\n }\n function parseGrid(doc, expectedPage) {\n const page = Number(doc.querySelector('input[name=\"page\"]')?.value);\n const size = Number(doc.querySelector('select[name=\"limit\"]')?.value);\n const count = normalize(doc.body?.textContent).match(/\\b(\\d+)\\s+records found\\b/i);\n if (page !== expectedPage || !Number.isInteger(size) || size < 1 || !count) fail('review_grid_incomplete');\n const total = Number(count[1]);\n const allRows = [...doc.querySelectorAll('input[name=\"reviews\"]')];\n const rows = allRows.map(input => {\n const tr = input.closest('tr');\n const link = tr?.querySelector('a[href*=\"/review/product/edit/id/\"]');\n const match = /\\/review\\/product\\/edit\\/id\\/(\\d+)\\//.exec(link?.getAttribute('href') || '');\n if (!match || Number(match[1]) !== Number(input.value)) fail('review_row_id_mismatch');\n if (![...tr.querySelectorAll('td')].some(td => normalize(td.textContent) === product)) return null;\n return {id: Number(match[1]), url: link.getAttribute('href')};\n }).filter(Boolean);\n return {page, size, total, rowCount: allRows.length, rows};\n }\n function parseDetail(doc, id) {\n const productLink = [...doc.querySelectorAll('a[href*=\"/catalog/product/edit/id/\"]')]\n .find(a => normalize(a.textContent) === product);\n const nickname = normalize(doc.querySelector('[name=\"nickname\"]')?.value);\n const ratings = [...doc.querySelectorAll('input[type=\"radio\"][name^=\"ratings[\"][checked]')]\n .map(input => Number(/_(\\d+)$/.exec(input.id)?.[1]));\n if (!productLink || !nickname || ratings.length !== 1 || !Number.isInteger(ratings[0]) ||\n ratings[0] < 1 || ratings[0] > 5) fail('review_detail_incomplete');\n return {id, nickname, rating: ratings[0]};\n }\n const first = parseGrid(await read('/admin/review/product/index/'), 1);\n const pages = Math.max(1, Math.ceil(first.total / first.size));\n if (pages > 100) fail('review_page_resource_limit');\n const reviewRows = [...first.rows];\n let observedRows = first.rowCount;\n for (let page = 2; page <= pages; page++) {\n const part = parseGrid(await read('/admin/review/product/index/page/' + page + '/'), page);\n if (part.total !== first.total || part.size !== first.size) fail('review_population_changed');\n observedRows += part.rowCount;\n reviewRows.push(...part.rows);\n }\n if (observedRows !== first.total || reviewRows.length < 2 ||\n new Set(reviewRows.map(row => row.id)).size !== reviewRows.length) fail('review_population_incomplete');\n const reviews = [];\n for (const row of reviewRows) reviews.push(parseDetail(await read(row.url), row.id));\n const lowest = Math.min(...reviews.map(row => row.rating));\n const matches = reviews.filter(row => row.rating === lowest);\n if (matches.length !== 1) fail('lowest_rating_not_unique');\n return {scope: true, complete: true, answer: [matches[0].nickname],\n review_count: reviews.length, lowest_rating: lowest, review_id: matches[0].id};\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 243, + "canonical_calls_sha256": "c7152e71effa2b529e31e173b4e4958bef110315c62dd7f350d92c31b71afc7b", + "sources": [ + { + "name": "observation", + "step": 9, + "action_sha256": "3974d5d472b68a65a829c1a5c38897dcbb0c7131fd94ebb232283ef36629e573", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "source": "observation", + "pointer": "/answer" + } + }, + "rule": "Read the unique lowest-rated review nickname from the complete current product review population; never infer guest authorship or customer email." +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-279.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-279.json new file mode 100644 index 000000000..096e78c55 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-279.json @@ -0,0 +1,50 @@ +{ + "task_id": 279, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/catalogsearch/result/?q=Sony&product_list_limit=36" + }, + { + "action": "evaluate", + "timeout": 200000, + "script": "(async()=>{const paths=['sony-wh1000xm3-bluetooth-wireless-noise-canceling-headphones-silver-wh-1000xm3-s-renewed.html','sony-wh-ch710n-h-wireless-bluetooth-noise-cancelling-headphones.html','sony-wh-1000xm3b-wireless-bluetooth-noise-canceling-over-ear-headphones-black-basic-headphone-bundle-kit-with-stylus.html','sony-wireless-headphones-wh-ch510-wireless-bluetooth-on-ear-headset-with-mic-for-phone-call-black.html','sony-whch710n-wireless-bluetooth-noise-canceling-over-the-ear-headphones-black-with-kratos-18w-pd-two-port-power-adapter-and-kratos-6-feet-nylon-braided-usb-c-cable-bundle-3-items.html','sony-wi-sp500-wireless-in-ear-sports-headphones-white-wisp500-w.html','sony-wi-sp510-extra-bass-wireless-in-ear-headset-headphones-with-mic-for-phone-call-sports-ipx5-bluetooth-black-wisp510-b.html','sony-mdras600bt-active-sports-bluetooth-headset-black.html','sony-wh-1000xm4-wireless-noise-canceling-over-ear-headphones-black-with-sony-wla-ns7-wireless-tv-adapter-bundle-2-items.html','sony-wi-c300-wireless-in-ear-headphones-red-wic300-r.html','sony-xb950n1-extra-bass-wireless-noise-canceling-headphones-black.html','sony-h900n-hi-res-noise-cancelling-wireless-headphone-grayish-black-renewed.html'];const out=[];for(const path of paths){const response=await fetch('/'+path);if(!response.ok)throw new Error(path+': HTTP '+response.status);const doc=new DOMParser().parseFromString(await response.text(),'text/html');const name=doc.querySelector('h1.page-title')?.textContent.trim();const price=doc.querySelector('[data-price-type=finalPrice] .price, .product-info-price .price')?.textContent.trim();const stock=doc.querySelector('.stock')?.textContent.trim();if(!name||!price)throw new Error(path+': product facts missing');out.push({name,price,stock});}return out;})()" + }, + { + "action": "evaluate", + "timeout": 200000, + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(!location.pathname.startsWith('/catalogsearch/result')||new URL(location.href).searchParams.get('q')!=='Sony')throw Error('wrong current catalog scope');\nconst rows=[],pages=[],seen=new Set();let total=null;\nfor(let page=1;page<=100;page++){\n const query='query {products(filter:{name:{match:\"Sony\"}},pageSize:100,currentPage:'+page+'){total_count page_info{current_page total_pages} items{sku name url_key description{html} rating_summary price_range{minimum_price{final_price{value currency}}}}}}';\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog response unavailable');\n const payload=await response.json(),data=payload.data?.products;\n if(payload.errors?.length||!data||!Number.isSafeInteger(data.total_count)||data.total_count<0||!Array.isArray(data.items))throw Error('invalid catalog response');\n if(total===null)total=data.total_count;\n const count=Math.max(1,Math.ceil(total/100));\n if(total!==data.total_count||count>100||data.page_info.current_page!==page||data.page_info.total_pages!==Math.ceil(total/100)||data.items.length!==Math.min(100,Math.max(0,total-(page-1)*100)))throw Error('incomplete or changed catalog');\n for(const row of data.items){\n if(typeof row.sku!=='string'||!row.sku||seen.has(row.sku)||typeof row.name!=='string'||!row.name||typeof row.url_key!=='string'||!row.url_key||row.url_key.includes('/'))throw Error('catalog identity missing or duplicated');\n seen.add(row.sku);\n const doc=new DOMParser().parseFromString(row.description?.html||'','text/html');\n for(const n of doc.querySelectorAll('script,style,noscript,template'))n.remove();\n const price=row.price_range?.minimum_price?.final_price;\n if(price?.currency!=='USD'||typeof price.value!=='number'||!Number.isFinite(price.value)||price.value<0)throw Error('invalid USD catalog price');\n rows.push({sku:row.sku,name:row.name,url:new URL('/'+row.url_key+'.html',location.origin).href,description:doc.body.textContent.replace(/\\s+/g,' ').trim(),rating:row.rating_summary,cents:Math.round(price.value*100)});\n }\n pages.push({page,rows:data.items.length,total,ids:data.items.map(r=>r.sku)});\n if(page===count)break;\n}\nif(rows.length!==total)throw Error('catalog population incomplete');\nconst details=rows.filter(r=>/^sony\\b/i.test(r.name)&&/headphone|headset|earphone/i.test(r.name)&&!/boombox|carrying case/i.test(r.name));\nconst capabilitySources=[{model:'WI-1000XM2',url:'https://www.sony.com/electronics/support/wireless-headphones-bluetooth-headphones/wi-1000xm2/specifications'},{model:'WH-H900N',url:'https://www.sony.com/electronics/support/wireless-headphones-bluetooth-headphones/wh-h900n/specifications'}];\nconst qualified=details.map(r=>({...r,bluetooth:/bluetooth/i.test(r.name+' '+r.description)||/\\bWI-1000XM2\\b|\\b(?:WH-)?H900N\\b/i.test(r.name)}));\nif(qualified.some(r=>/wireless/i.test(r.name)&&!r.bluetooth))throw Error('wireless model needs independent Bluetooth capability review');\nconst matched=qualified.filter(r=>r.bluetooth);\nif(!matched.length||new Set(matched.map(r=>r.name)).size!==matched.length)throw Error('headphone names absent or duplicated');\nconst prices=matched.map(r=>r.cents);\nreturn {scope:true,complete:true,pages,total,details:qualified,capabilitySources,answer:{names:matched.map(r=>r.name),min:Math.min(...prices)/100,max:Math.max(...prices)/100}};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 279, + "canonical_calls_sha256": "c54bdc5730b9ba0862dc897d1dd6363c666875b4b1a08cf49ef08f432ca83976", + "sources": [ + { + "name": "observation", + "step": 3, + "action_sha256": "0d5f3cb7ddecc8d91f635478fff1adb5b0be138747e0d9c9c64dcf5f5e488969", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-28.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-28.json new file mode 100644 index 000000000..4a10fcbf4 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-28.json @@ -0,0 +1,19 @@ +{ + "task_id": 28, + "tool_calls": [ + {"action": "navigate", "url": "http://host.docker.internal:9999/f/WorcesterMA/new"}, + {"action": "wait", "seconds": 2}, + {"action": "evaluate", "script": "const link=document.querySelector('main article h1 a, main article h2 a, article a[href*=\"/post/\"]'); if(!link) throw new Error('newest WorcesterMA post link not found'); link.click(); return {href:link.href,text:link.textContent.trim()};"}, + {"action": "wait", "seconds": 2}, + {"action": "get_text", "selector": "main"}, + {"action": "evaluate", "script": "(async()=>{const norm=t=>t.replace(/\\s+/g,' ').trim(),one=(root,selector)=>{const xs=[...root.querySelectorAll(selector)];if(xs.length!==1)throw Error('expected one '+selector);return xs[0]},abs=(href,base)=>new URL(href,base).href,int=t=>{if(!/^-?\\d+$/.test(t||''))throw Error('missing integer');const n=Number(t);if(!Number.isSafeInteger(n))throw Error('unsafe integer');return n};const ctl=new AbortController(),timer=setTimeout(()=>ctl.abort(),30000);try{async function read(url){const u=new URL(url);if(u.origin!==location.origin||!u.pathname.startsWith('/f/WorcesterMA/'))throw Error('out-of-scope forum read');const r=await fetch(u.href,{credentials:'same-origin',redirect:'error',cache:'no-store',signal:ctl.signal});if(r.status!==200||r.redirected||!/^text\\/html\\b/i.test(r.headers.get('content-type')||''))throw Error('missing forum HTML');return {url:r.url,doc:new DOMParser().parseFromString(await r.text(),'text/html')}}const listing=await read(location.origin+'/f/WorcesterMA/new'),articles=[...listing.doc.querySelectorAll('main article.submission')];if(!articles.length||articles.some(x=>x.classList.contains('submission--sticky'))||!listing.doc.querySelector('a.menu-item--active[href=\"/f/WorcesterMA/new\"]'))throw Error('newest listing is unavailable');const first=articles[0],author=one(first,'.submission__info a[href^=\"/user/\"]'),link=one(first,'.submission__nav a[href^=\"/f/WorcesterMA/\"]'),post={url:abs(link.getAttribute('href'),listing.url),title:norm(one(first,'.submission__title').textContent),username:norm(author.textContent),author:abs(author.getAttribute('href'),listing.url)};if(!post.title||!post.username||!/^\\/f\\/WorcesterMA\\/\\d+\\/[^/]+$/.test(new URL(post.url).pathname))throw Error('missing first-post identity');const pages=[],seen=new Set();let address=post.url,total=null,count=0;while(address){if(pages.length>=100||pages.some(p=>p.url===address))throw Error('pagination loop or bound');const {url,doc}=await read(address),article=one(doc,'main article.submission'),who=one(article,'.submission__info a[href^=\"/user/\"]'),canonical=abs(one(doc,'link[rel=\"canonical\"]').getAttribute('href'),url),counter=one(doc,'.js-update-comment-count[data-comment-count][data-submission-id]');if(canonical!==post.url||norm(one(article,'.submission__title').textContent)!==post.title||abs(who.getAttribute('href'),url)!==post.author||new URL(canonical).pathname.split('/')[3]!==counter.getAttribute('data-submission-id'))throw Error('comment page belongs to another post');const pageTotal=int(counter.getAttribute('data-comment-count'));if(total===null)total=pageTotal;if(pageTotal!==total)throw Error('comment count changed during read');const nexts=[...new Set([...doc.querySelectorAll('link[rel=\"next\"],nav.pagination a[rel=\"next\"]')].map(x=>abs(x.getAttribute('href'),url)))];if(nexts.length>1)throw Error('ambiguous next page');const items=[...doc.querySelectorAll('article.comment')];if(items.some(x=>!x.classList.contains('comment--visibility-visible')))throw Error('hidden comment identity unavailable');for(const item of items){const own=selector=>[...item.querySelectorAll(selector)].filter(x=>x.closest('article.comment')===item),users=own('.comment__info a[href^=\"/user/\"]'),votes=own('form[data-vote-route-value=\"comment_vote\"][data-vote-score-value]');if(users.length!==1||votes.length!==1||item.id!=='comment_'+votes[0].getAttribute('data-vote-id-value')||seen.has(item.id))throw Error('invalid or duplicate comment');seen.add(item.id);if(abs(users[0].getAttribute('href'),url)!==post.author&&int(votes[0].getAttribute('data-vote-score-value'))<0)count++}pages.push({url,rows:items.length,total,next:nexts[0]||null});address=nexts[0]||null}if(total===null||seen.size!==total||pages.reduce((n,p)=>n+p.rows,0)!==total)throw Error('incomplete comments');return {scope:true,complete:true,listing:{url:listing.url,first:post.url},post,pages,total,answer:{username:post.username,post_title:post.title,count}}}finally{ctl.abort();clearTimeout(timer)}})()"} + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 28, + "canonical_calls_sha256": "aa19ff925c165ff79be0e76bb622e7870cd0fb9900a48b1d829f2d691945bc15", + "sources": [{"name": "observation", "step": 6, "action_sha256": "7851532a05e9445b6a379a8289531462e1cd14765bcc00a095464db3a36c2502", "decode": "json"}], + "requirements": {"scope": {"source": "observation", "pointer": "/scope"}, "completeness": {"source": "observation", "pointer": "/complete"}}, + "projection": {"array": [{"source": "observation", "pointer": "/answer"}]} + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-288.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-288.json new file mode 100644 index 000000000..70752ca81 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-288.json @@ -0,0 +1,40 @@ +{ + "task_id": 288, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 60000);\n try {\n if (location.pathname.replace(/\\/$/, '') !== '/admin/sales/order') throw new Error('wrong grid page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\n const params={...p.params,search:'',filters:{placeholder:true},sorting:{field:'entity_id',direction:'asc'},paging:{current:1,pageSize:200}};\n const endpoint = new URL(p.update_url,location.href);\n if(endpoint.origin !== location.origin || endpoint.pathname !== '/admin/mui/index/render/') throw new Error('wrong endpoint');\n const flatten = (v,prefix='') => Object.entries(v).flatMap(([k,x])=>{\n const key=prefix?`${prefix}[${k}]`:k;\n return x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]];\n });\n for(const [k,v] of flatten(params)) endpoint.searchParams.set(k,v);\n const size=params.paging.pageSize;\n if(!Number.isInteger(size)||size<1||size>1000) throw new Error('invalid page size');\n let total=null;const rows=[],pages=[];\n for(let page=1;page<=100;page++) {\n endpoint.searchParams.set('paging[current]',String(page));\n const response=await fetch(endpoint.href, {credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected) throw new Error('grid request failed');\n if(!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||'')) throw new Error('grid response must be application/json');\n const data=await response.json();\n if(data===null||typeof data!=='object'||Array.isArray(data)) throw new Error('grid JSON must be an object');\n if(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\n if(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items)) throw new Error('invalid response');\n if(total===null)total=data.totalRecords;\n if(total!==data.totalRecords||Math.ceil(total/size)>100||data.items.length!==Math.min(size,Math.max(0,total-(page-1)*size))) throw new Error('incomplete page');\n rows.push(...data.items);pages.push({page,rows:data.items.length,total});\n if(page>=Math.max(1,Math.ceil(total/size)))break;\n }\n if(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total) throw new Error('incomplete or duplicate identities');\n rows.sort((a,b)=>String(b.created_at).localeCompare(String(a.created_at))||Number(b.entity_id)-Number(a.entity_id));\n for(let i=0;ir.status==='canceled');\n if(!canceled.length)throw new Error('no canceled orders');\n const counts=new Map();for(const row of canceled){const email=row.customer_email;\n if(typeof email!=='string'||!/^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$/.test(email))throw new Error('canceled order missing customer email');\n counts.set(email,(counts.get(email)||0)+1);}\n const ranked=[...counts].sort((a,b)=>b[1]-a[1]||a[0].localeCompare(b[0]));\n if(ranked.length>1&&ranked[0][1]===ranked[1][1])throw new Error('most cancellations are tied');\n return {scope:true,complete:true,pages,total,canceled:canceled.length,ranked,answer:ranked[0][0]};\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 288, + "canonical_calls_sha256": "4db74abefe26b2e8a36487e00f5b38ea4397f25b400ffa362455ed59c6bfc881", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "f27d1e23e1954ffca764d0352696f225dac44d867c674cff67d85cea8ed26cd5", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-29.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-29.json new file mode 100644 index 000000000..41dff5275 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-29.json @@ -0,0 +1,72 @@ +{ + "task_id": 29, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:9999/f/DIY" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/DIY/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n function decide(observed) {\n const fail = message => { throw new Error(message); };\n const list = new URL(observed.listing.url);\n if (list.origin !== observed.origin || list.pathname !== '/f/DIY/new' || list.search ||\n observed.listing.status !== 200 || !observed.listing.newActive || observed.listing.sticky)\n fail('unproven newest DIY first post');\n const post = observed.listing.first;\n const postURL = new URL(post.url);\n if (postURL.origin !== observed.origin || !/^\\/f\\/DIY\\/\\d+\\/[^/]+$/.test(postURL.pathname) ||\n postURL.search || postURL.hash || !post.title || !post.username ||\n new URL(post.author).origin !== observed.origin ||\n !/^\\/user\\/[^/]+$/.test(new URL(post.author).pathname)) fail('missing first post identity');\n const seen = new Set(), pages = observed.pages;\n if (!pages.length) fail('missing comment pages');\n const total = pages[0].total;\n if (!Number.isSafeInteger(total) || total < 0) fail('missing server comment count');\n let count = 0;\n for (let i = 0; i < pages.length; i++) {\n const page = pages[i], url = new URL(page.url);\n if (url.origin !== observed.origin || url.pathname !== postURL.pathname || page.status !== 200 ||\n page.postURL !== post.url || page.title !== post.title || page.author !== post.author ||\n page.total !== total) fail('comment page identity or count drift');\n if (Boolean(page.next) !== (i + 1 < pages.length) ||\n (page.next && page.next !== pages[i + 1].url)) fail('incomplete comment pagination');\n for (const comment of page.comments) {\n if (!/^comment_\\d+$/.test(comment.id) || seen.has(comment.id)) fail('duplicate comment');\n seen.add(comment.id);\n if (!comment.author || new URL(comment.author).origin !== observed.origin ||\n !/^\\/user\\/[^/]+$/.test(new URL(comment.author).pathname) ||\n !Number.isSafeInteger(comment.score)) fail('missing comment author or net score');\n if (comment.author !== post.author && comment.score < 0) count++;\n }\n }\n if (seen.size !== total) fail('visible unique comment count is incomplete');\n return {scope:true, completeness:true,\n answer:{username:post.username, post_title:post.title, count}, observed};\n }\n const norm = text => text.replace(/\\s+/g, ' ').trim();\n const one = (node, selector) => {\n const nodes = [...node.querySelectorAll(selector)];\n if (nodes.length !== 1) throw new Error('expected one ' + selector);\n return nodes[0];\n };\n const absolute = (href, base) => new URL(href, base).href;\n const read = async url => {\n const target = new URL(url);\n if (target.origin !== location.origin || !target.pathname.startsWith('/f/DIY/'))\n throw new Error('out-of-scope read');\n const response = await fetch(url, {credentials:'same-origin',cache:'no-store',redirect:'error'});\n if (response.status !== 200 || !/^text\\/html\\b/i.test(response.headers.get('content-type') || ''))\n throw new Error('missing HTML 200');\n return {url:response.url,status:response.status,\n doc:new DOMParser().parseFromString(await response.text(),'text/html')};\n };\n const integer = value => {\n if (!/^-?\\d+$/.test(value || '')) throw new Error('missing integer');\n const n = Number(value); if (!Number.isSafeInteger(n)) throw new Error('unsafe integer');\n return n;\n };\n // Discover newest from the actual current UI, not a recorded post or answer.\n const newest = [...document.querySelectorAll('a[href]')].find(a =>\n new URL(a.getAttribute('href'),location.href).pathname === '/f/DIY/new');\n const listingURL = location.pathname === '/f/DIY/new' && !location.search ? location.href :\n newest ? absolute(newest.getAttribute('href'),location.href) : null;\n if (!listingURL) throw new Error('missing current DIY newest navigation');\n const listing = await read(listingURL);\n const articles = [...listing.doc.querySelectorAll('main article.submission')];\n if (!articles.length) throw new Error('empty newest listing');\n const first = articles[0];\n const author = one(first,'.submission__info a[href^=\"/user/\"]');\n const commentLink = one(first,'.submission__nav a[href^=\"/f/DIY/\"]');\n const observed = {origin:location.origin,listing:{url:listing.url,status:listing.status,\n newActive:!!listing.doc.querySelector('a.menu-item--active[href=\"/f/DIY/new\"]'),\n sticky:articles.some(a => a.classList.contains('submission--sticky')),\n first:{url:absolute(commentLink.getAttribute('href'),listing.url),\n title:norm(one(first,'.submission__title').textContent), username:norm(author.textContent),\n author:absolute(author.getAttribute('href'),listing.url)}},pages:[]};\n let address = observed.listing.first.url;\n const visited = new Set();\n while (address) {\n if (visited.has(address) || visited.size >= 100) throw new Error('pagination loop or bound');\n visited.add(address);\n const page = await read(address), doc = page.doc;\n const post = one(doc,'main article.submission');\n const who = one(post,'.submission__info a[href^=\"/user/\"]');\n const canonical = absolute(one(doc,'link[rel=\"canonical\"]').getAttribute('href'),page.url);\n const counter = one(doc,'.js-update-comment-count[data-comment-count][data-submission-id]');\n if (new URL(canonical).pathname.split('/')[3] !== counter.getAttribute('data-submission-id'))\n throw new Error('counter belongs to another submission');\n const nextLinks = [...doc.querySelectorAll('link[rel=\"next\"], nav.pagination a[rel=\"next\"]')];\n const nextURLs = [...new Set(nextLinks.map(a => absolute(a.getAttribute('href'),page.url)))];\n if (nextURLs.length > 1) throw new Error('ambiguous next page');\n if ([...doc.querySelectorAll('article.comment')].some(n => !n.classList.contains('comment--visibility-visible')))\n throw new Error('deleted or moderated comment identity is unavailable');\n const comments = [...doc.querySelectorAll('article.comment--visibility-visible')].map(comment => {\n // Descendant selectors must never borrow a nested reply's author/score.\n const own = selector => [...comment.querySelectorAll(selector)].filter(n => n.closest('article.comment') === comment);\n const users = own('.comment__info a[href^=\"/user/\"]');\n const votes = own('form[data-vote-route-value=\"comment_vote\"][data-vote-score-value]');\n if (users.length !== 1 || votes.length !== 1 ||\n comment.id !== 'comment_' + votes[0].getAttribute('data-vote-id-value'))\n throw new Error('missing own comment identity');\n return {id:comment.id,author:absolute(users[0].getAttribute('href'),page.url),\n score:integer(votes[0].getAttribute('data-vote-score-value'))};\n });\n observed.pages.push({url:page.url,status:page.status,postURL:canonical,\n title:norm(one(post,'.submission__title').textContent),author:absolute(who.getAttribute('href'),page.url),\n total:integer(counter.getAttribute('data-comment-count')),next:nextURLs[0] || null,comments});\n address = nextURLs[0] || null;\n }\n return decide(observed);\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 29, + "canonical_calls_sha256": "4de8202bcc49843884cdab163b4d2e3cdf3a679fb23f1756a0bdf2c093558488", + "sources": [ + { + "name": "current", + "step": 9, + "action_sha256": "3dfdae8a3c59197c42e9b5ff6300c2a6baf96638b623a1431ccb289d407f30a6", + "decode": "json" + } + ], + "projection": { + "array": [ + { + "source": "current", + "pointer": "/answer" + } + ] + }, + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/completeness" + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-303.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-303.json new file mode 100644 index 000000000..a8c18a3f0 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-303.json @@ -0,0 +1,48 @@ +{ + "task_id": 303, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/graphs/main/charts?from=2023-01-01&to=2023-12-31" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/commits/main?author=Kilian&since=2023-01-01T00%3A00%3A00Z&until=2023-12-31T23%3A59%3A59Z" + }, + { + "action": "evaluate", + "script": "(async()=>{\n const ctl=new AbortController(),timer=setTimeout(()=>ctl.abort(),30000);\n try {\n const api=location.origin+'/api/v4',path='a11yproject/a11yproject.com';\n const range=[\"2023-01-01T00:00:00Z\",\"2023-12-31T23:59:59Z\"];\n async function read(url){\n const r=await fetch(url,{credentials:'same-origin',redirect:'error',signal:ctl.signal,headers:{Accept:'application/json'}});\n if(r.status!==200||r.redirected||!/^application\\/json\\b/i.test(r.headers.get('content-type')||''))throw Error('GitLab JSON read failed');\n return {r,body:await r.json()};\n }\n const {body:project}=await read(api+'/projects/'+encodeURIComponent(path));\n if(!Number.isInteger(project.id)||project.path_with_namespace!==path||project.default_branch!=='main')throw Error('wrong project or branch');\n const rows=[],pages=[],seen=new Set(),size=100;\n for(let page=1;page<=100;page++){\n const url=new URL(api+'/projects/'+project.id+'/repository/commits');\n for(const [k,v] of Object.entries({ref_name:'main',since:range[0],until:range[1],per_page:String(size),page:String(page)}))url.searchParams.set(k,v);\n const {r,body}=await read(url.href);\n const current=r.headers.get('x-page'),next=r.headers.get('x-next-page');\n if(current!==String(page)||next===null||!Array.isArray(body)||body.length>size)throw Error('incomplete commit page');\n for(const c of body){\n const time=Date.parse(c.committed_date);\n if(!/^[0-9a-f]{40}$/.test(c.id)||seen.has(c.id)||typeof c.author_name!=='string'||!Number.isFinite(time)||timeDate.parse(range[1]))throw Error('commit identity or date ambiguity');\n seen.add(c.id);rows.push({id:c.id,author_name:c.author_name,committed_date:c.committed_date});\n }\n pages.push({page,rows:body.length,next});\n if(next==='')break;\n if(next!==String(page+1)||page===100)throw Error('commit pagination gap or safety bound');\n }\n if(!pages.length||pages.at(-1).next!==''||pages.reduce((n,p)=>n+p.rows,0)!==rows.length)throw Error('incomplete commit population');\n const commits=rows.filter(c=>/^kilian(?:\\s|$)/i.test(c.author_name));\n return {scope:true,complete:true,project:{id:project.id,path_with_namespace:project.path_with_namespace,branch:project.default_branch},range,pages,observed_commits:rows,commits,answer:commits.length};\n }finally{ctl.abort();clearTimeout(timer)}\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 303, + "canonical_calls_sha256": "dae8c6e1c000d0abe63ef16ad61e9b537fa023cafd38482c7f5dff5dc5adbeea", + "sources": [ + { + "name": "observation", + "step": 4, + "action_sha256": "540edf4574592fee3961d0a1d752121544a81d0b40768c1f59210a3b88b795b9", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "source": "observation", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-313.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-313.json new file mode 100644 index 000000000..64b597ad2 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-313.json @@ -0,0 +1,80 @@ +{ + "task_id": 313, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/contact/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "customer service phone number", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const paths = ['/', '/contact/', '/privacy-policy-cookie-restriction-mode', '/enable-cookies'];\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), 20000);\n try {\n const pages = [];\n for (const path of paths) {\n const url = new URL(path, location.origin);\n const response = await fetch(url.href, {\n credentials: 'same-origin', redirect: 'error', cache: 'no-store',\n signal: controller.signal\n });\n if (response.status !== 200 || !/^text\\/html\\b/i.test(response.headers.get('content-type') || '') ||\n new URL(response.url).origin !== location.origin)\n throw new Error('support page request failed: ' + path);\n const doc = new DOMParser().parseFromString(await response.text(), 'text/html');\n for (const node of doc.querySelectorAll('script,style,noscript,template')) node.remove();\n const text = (doc.body?.textContent || '').replace(/\\s+/g, ' ').trim();\n if (!text) throw new Error('support page has no body text: ' + path);\n const tel = [...doc.querySelectorAll('a[href^=\"tel:\"]')]\n .map(node => ({href: node.getAttribute('href'), text: node.textContent.trim()}));\n const matches = [...text.matchAll(/(? match[0].trim());\n pages.push({path, status: response.status, title: doc.title, text, tel, matches});\n }\n const items = [...new Set(pages.flatMap(page => [\n ...page.tel.map(node => node.href.replace(/^tel:/i, '').trim()),\n ...page.matches\n ]).filter(Boolean))];\n return {scope: pages.length === paths.length, complete: pages.every(page => page.status === 200),\n items, pages, interpretation: 'Published store support surfaces only; no product-vendor phone is a store customer-service phone.'};\n } finally {\n clearTimeout(timer);\n controller.abort();\n }\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 313, + "canonical_calls_sha256": "e476c757200b709df685f32fa2d423c4c5fb505a0f1c12b3eef85a4d0b928ed7", + "sources": [ + { + "name": "support", + "step": 9, + "action_sha256": "33e750ad681d5a8f6ef9682aedf8c42374a2bb885a9842058a444e7ca6f908c4", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "support", + "pointer": "/scope" + }, + "completeness": { + "source": "support", + "pointer": "/complete" + } + }, + "projection": { + "source": "support", + "pointer": "/items" + }, + "empty_result": "NOT_FOUND_ERROR", + "absence_coverage": { + "source": "support", + "pages_pointer": "/pages", + "required_paths": [ + "/", + "/contact/", + "/privacy-policy-cookie-restriction-mode", + "/enable-cookies" + ], + "detector": "phone_number" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-314.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-314.json new file mode 100644 index 000000000..1b684aa59 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-314.json @@ -0,0 +1,63 @@ +{ + "task_id": 314, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"/primer/design\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design/-/graphs/main/charts" + }, + { + "action": "scroll", + "direction": "down" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design/-/graphs/main" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 20000);\n try {\n const expectedPath = 'primer/design';\n const api = location.origin + '/api/v4';\n const readJson = async url => {\n const response = await fetch(url, {credentials: 'same-origin',redirect:'error',signal:controller.signal});\n if (!response.ok || response.redirected) {\n throw new Error('GitLab API request failed or redirected: ' + response.status);\n }\n return {response, body: await response.json()};\n };\n const projectRead = await readJson(api + '/projects/' + encodeURIComponent(expectedPath));\n const project = projectRead.body;\n if (!Number.isInteger(project.id) || project.path_with_namespace !== expectedPath ||\n typeof project.default_branch !== 'string' || !project.default_branch) {\n throw new Error('project identity or default branch mismatch');\n }\n const branch = project.default_branch;\n const contributors = [];\n const pages = [];\n const perPage = 100;\n for (let page = 1; page <= 100; page++) {\n const url = new URL(api + '/projects/' + project.id + '/repository/contributors');\n url.searchParams.set('ref', branch);\n url.searchParams.set('per_page', String(perPage));\n url.searchParams.set('page', String(page));\n const {response, body} = await readJson(url.href);\n const current = response.headers.get('x-page');\n const next = response.headers.get('x-next-page');\n if (current !== String(page) || next === null || !Array.isArray(body) ||\n body.length > perPage) {\n throw new Error('contributor pagination metadata or page body is invalid');\n }\n if (body.some(row => typeof row.name !== 'string' || !row.name.trim() ||\n typeof row.email !== 'string' || !row.email ||\n !Number.isInteger(row.commits) || row.commits < 0)) {\n throw new Error('invalid contributor row');\n }\n contributors.push(...body.map(row => ({\n name: row.name.trim(), email: row.email, commits: row.commits,\n })));\n pages.push({page, next, rows: body.length});\n if (next === '') break;\n if (next !== String(page + 1)) throw new Error('contributor pagination gap');\n if (page === 100) throw new Error('contributor pagination exceeds safety bound');\n }\n const identities = contributors.map(row => row.name + '\\u0000' + row.email);\n if (contributors.length < 3 || new Set(identities).size !== contributors.length ||\n pages.at(-1)?.next !== '') {\n throw new Error('contributor enumeration is incomplete or ambiguous');\n }\n const ranked = contributors.map((row, sourceIndex) => ({...row, sourceIndex}))\n .sort((a, b) => b.commits - a.commits || a.sourceIndex - b.sourceIndex);\n if (ranked.length > 3 && ranked[2].commits === ranked[3].commits) {\n throw new Error('commit-count tie crosses the top-three boundary');\n }\n const names = ranked.slice(0, 3).map(row => row.name);\n if (new Set(names).size !== 3) throw new Error('top contributors lack distinct full names');\n return {\n schema: 'gitlab_project_top_contributors/1',\n scope: project.path_with_namespace === expectedPath && branch === project.default_branch,\n complete: pages.at(-1)?.next === '',\n project: {id: project.id, path_with_namespace: project.path_with_namespace},\n branch,\n pages,\n contributors,\n boundary: ranked.length > 3 ? {third: ranked[2].commits, fourth: ranked[3].commits} : null,\n answer: names,\n };\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 314, + "canonical_calls_sha256": "7d8b2c3d376ef62b5dcd2e5e0f7f0fe04ea22adfdb637cd2c2852219df413751", + "sources": [ + { + "name": "observation", + "step": 8, + "action_sha256": "f6352520bb8f3fe1f13d0e27e94477e05e4a20a3321c1e6c193767199a04cdee", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "source": "observation", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-320.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-320.json new file mode 100644 index 000000000..79bd57224 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-320.json @@ -0,0 +1,40 @@ +{ + "task_id": 320, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].dater.status==='Canceled'&&r.date>='2023-02-01'&&r.date<='2023-02-28');\nif(rows.some(r=>r.status==='Cancelled'&&r.date>='2023-02-01'&&r.date<='2023-02-28'))throw Error('ambiguous canceled spelling');\nconst cents=found.reduce((n,r)=>n+r.cents,0);if(!Number.isSafeInteger(cents))throw Error('unsafe refund sum');\nreturn {scope:true,complete:true,pages,total,matched:found.map(r=>r.number),answer:cents/100};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 320, + "canonical_calls_sha256": "44587ccb39130f7a490e75c9fffedb8fdc45fee4a5ac6b56617d81bb7001bda7", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "4ba06f7299e22c784e30cb0f91443edb9b909a3587620125a60feccea759d5c4", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-334.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-334.json new file mode 100644 index 000000000..f7dbfa592 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-334.json @@ -0,0 +1,40 @@ +{ + "task_id": 334, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].date{const order=rows[index],u=new URL(order.detail);\nif(u.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(u.pathname))throw Error('detail out of scope');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order detail unavailable');\nconst doc=new DOMParser().parseFromString(await response.text(),'text/html');\nconst names=[...doc.querySelectorAll('.product-item-name,.col.name .product-item-name,.order-details-items .product.name')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!names.length)throw Error('order items missing');\nobservations[index]={number:order.number,date:order.date,items:names};};\nawait Promise.all(Array.from({length:Math.min(4,rows.length)},async()=>{\n while(cursororder.items.some(n=>/muffin/i.test(n)&&/cornbread/i.test(n)&&/mix/i.test(n)));\nreturn {scope:true,complete:true,pages,total,matched:found,answer:found[0]?.date??null};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 334, + "canonical_calls_sha256": "5b5f00945ae375ee88ff2e5fe1ca6748f9c0bbca142685fc981a7b4ce16a680f", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "4efa2fe6b6ab28132e38c7522fb8426780b48a72d3939606955de8d2a186a7f3", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-349.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-349.json new file mode 100644 index 000000000..fbe689435 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-349.json @@ -0,0 +1,52 @@ +{ + "task_id": 349, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"/byteblaze/gimmiethat.space\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/byteblaze/gimmiethat.space/-/project_members" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const deadlineTimer = setTimeout(() => controller.abort(), 20000);\n try {\n const expectedPath = 'byteblaze/gimmiethat.space';\n const api = location.origin + '/api/v4';\n const readJson = async url => {\n const response = await fetch(url, {credentials: 'same-origin',redirect:'error',signal:controller.signal});\n if (!response.ok || response.redirected) {\n throw new Error('GitLab API request failed or redirected: ' + response.status);\n }\n return {response, body: await response.json()};\n };\n const [{body: currentUser}, {body: project}] = await Promise.all([\n readJson(api + '/user'),\n readJson(api + '/projects/' + encodeURIComponent(expectedPath)),\n ]);\n if (!Number.isInteger(currentUser.id) || typeof currentUser.username !== 'string' ||\n !Number.isInteger(project.id) || project.path_with_namespace !== expectedPath ||\n project.namespace?.kind !== 'user' || project.owner?.id !== currentUser.id) {\n throw new Error('current-user or owned-project identity mismatch');\n }\n const members = [];\n const pages = [];\n const perPage = 100;\n for (let page = 1; page <= 100; page++) {\n const url = new URL(api + '/projects/' + project.id + '/members/all');\n url.searchParams.set('per_page', String(perPage));\n url.searchParams.set('page', String(page));\n const {response, body} = await readJson(url.href);\n const current = response.headers.get('x-page');\n const next = response.headers.get('x-next-page');\n if (current !== String(page) || next === null || !Array.isArray(body) ||\n body.length > perPage) {\n throw new Error('effective-member pagination metadata or page body is invalid');\n }\n if (body.some(row => !Number.isInteger(row.id) || typeof row.username !== 'string' ||\n !row.username || !Number.isInteger(row.access_level) || row.access_level <= 0)) {\n throw new Error('invalid effective-member row');\n }\n members.push(...body.map(row => ({\n id: row.id,\n username: row.username,\n access_level: row.access_level,\n membership_state: row.membership_state ?? null,\n source: 'projects/:id/members/all',\n })));\n pages.push({page, next, rows: body.length});\n if (next === '') break;\n if (next !== String(page + 1)) throw new Error('effective-member pagination gap');\n if (page === 100) throw new Error('effective-member pagination exceeds safety bound');\n }\n if (pages.at(-1)?.next !== '' || new Set(members.map(row => row.id)).size !== members.length ||\n !members.some(row => row.id === currentUser.id)) {\n throw new Error('effective-member enumeration is incomplete or lacks current user');\n }\n const others = members.filter(row => row.id !== currentUser.id);\n const answer = others.map(row => row.username)\n .sort((a, b) => a < b ? -1 : a > b ? 1 : 0);\n return {\n schema: 'gitlab_owned_project_effective_members/1',\n scope: project.path_with_namespace === expectedPath && project.owner.id === currentUser.id,\n complete: pages.at(-1)?.next === '',\n project: {id: project.id, path_with_namespace: project.path_with_namespace},\n current_user: {id: currentUser.id, username: currentUser.username},\n membership_semantics: 'effective direct, inherited, and invited-group access from members/all',\n pages,\n members,\n answer,\n };\n } finally { controller.abort(); clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 349, + "canonical_calls_sha256": "0a38ac087eaaf0f4aa73ff8cbf7a2ebe9b8b5d3963f015aa5a87724289eededf", + "sources": [ + { + "name": "observation", + "step": 5, + "action_sha256": "09877af45e4299fb82a8d85a4de5d4a99067b32464d007dda96fad36dbe6b55a", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "source": "observation", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-386.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-386.json new file mode 100644 index 000000000..1a9652b7a --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-386.json @@ -0,0 +1,78 @@ +{ + "task_id": 386, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "fill", + "text": "Ugreen lightning to 3.5mm cable", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a:visible, 28)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "UGREEN Aux Cord for iPhone Lightning to 3.5mm Cable", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),120000);try{\nif(!location.pathname.startsWith('/catalogsearch/result')||!/ugreen/i.test(new URL(location.href).searchParams.get('q')||''))throw Error('wrong current catalog scope');\nconst rows=[],pages=[],seen=new Set();let total=null;\nfor(let page=1;page<=100;page++){\n const query='query {products(filter:{name:{match:\"UGREEN\"}},pageSize:100,currentPage:'+page+'){total_count page_info{current_page total_pages} items{sku name url_key description{html} rating_summary price_range{minimum_price{final_price{value currency}}}}}}';\n const url=new URL('/graphql',location.origin);url.searchParams.set('query',query);\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal,headers:{Accept:'application/json'}});\n if(!response.ok||response.redirected||!/^application\\/json\\b/i.test(response.headers.get('content-type')||''))throw Error('catalog response unavailable');\n const payload=await response.json(),data=payload.data?.products;\n if(payload.errors?.length||!data||!Number.isSafeInteger(data.total_count)||data.total_count<0||!Array.isArray(data.items))throw Error('invalid catalog response');\n if(total===null)total=data.total_count;\n const count=Math.max(1,Math.ceil(total/100));\n if(total!==data.total_count||count>100||data.page_info.current_page!==page||data.page_info.total_pages!==Math.ceil(total/100)||data.items.length!==Math.min(100,Math.max(0,total-(page-1)*100)))throw Error('incomplete or changed catalog');\n for(const row of data.items){\n if(typeof row.sku!=='string'||!row.sku||seen.has(row.sku)||typeof row.name!=='string'||!row.name||typeof row.url_key!=='string'||!row.url_key||row.url_key.includes('/'))throw Error('catalog identity missing or duplicated');\n seen.add(row.sku);\n const doc=new DOMParser().parseFromString(row.description?.html||'','text/html');\n for(const n of doc.querySelectorAll('script,style,noscript,template'))n.remove();\n const price=row.price_range?.minimum_price?.final_price;\n if(price?.currency!=='USD'||typeof price.value!=='number'||!Number.isFinite(price.value)||price.value<0)throw Error('invalid USD catalog price');\n rows.push({sku:row.sku,name:row.name,url:new URL('/'+row.url_key+'.html',location.origin).href,description:doc.body.textContent.replace(/\\s+/g,' ').trim(),rating:row.rating_summary,cents:Math.round(price.value*100)});\n }\n pages.push({page,rows:data.items.length,total,ids:data.items.map(r=>r.sku)});\n if(page===count)break;\n}\nif(rows.length!==total)throw Error('catalog population incomplete');\nconst matches=rows.filter(r=>/^ugreen\\b/i.test(r.name)&&/lightning/i.test(r.name)&&/3\\.5\\s*mm/i.test(r.name)&&/adapter|dongle|cable|cord/i.test(r.name));\nif(matches.length!==1)throw Error('connector product identity ambiguous');\nconst selected=matches[0],response=await fetch(selected.url,{credentials:'same-origin',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('product detail unavailable');\nconst doc=new DOMParser().parseFromString(await response.text(),'text/html');\nconst name=doc.querySelector('h1.page-title')?.textContent.replace(/\\s+/g,' ').trim();\nif(name!==selected.name)throw Error('product detail identity differs');\nconst ratings=[...doc.querySelectorAll('.product-reviews-summary [itemprop=ratingValue]')];\nif(ratings.length!==1)throw Error('missing or ambiguous aggregate rating');\nconst raw=ratings[0].getAttribute('content')||ratings[0].textContent.trim();\nif(!/^(?:100|[1-9]?\\d)(?:\\.\\d+)?%?$/.test(raw))throw Error('invalid aggregate rating');\nconst rating=Number(raw.replace('%',''));\nif(rating!==selected.rating||rating<0||rating>100)throw Error('catalog and visible rating disagree');\nreturn {scope:true,complete:true,pages,total,products:rows.map(({description,...r})=>r),selected:{...selected,rating_source:raw},answer:rating};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 386, + "canonical_calls_sha256": "731f5cbddead119830352cd916d82f67fc42a820ea902f0d24c26818f8628a5b", + "sources": [ + { + "name": "observation", + "step": 10, + "action_sha256": "b8f29e6050935926a97c28764d79e4d57cd4593c1033cd2fb951022806e7d950", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-41.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-41.json new file mode 100644 index 000000000..6fef24cfb --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-41.json @@ -0,0 +1,40 @@ +{ + "task_id": 41, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/search/term/report/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nconst gridId=\"searchReportGrid\",identity=\"query_id\";\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/search/term/report\")throw Error('wrong legacy grid page');\nconst number=(text)=>{const s=String(text??'').trim();if(!/^\\d+$/.test(s)||!Number.isSafeInteger(Number(s)))throw Error('invalid grid integer');return Number(s)};\nconst collect=(doc,index)=>{\n const root=doc.getElementById(gridId),table=doc.getElementById(gridId+'_table');\n if(!root||!table||!root.contains(table))throw Error('legacy grid missing');\n const total=number(doc.getElementById(gridId+'-total-count')?.textContent);\n const current=number(root.querySelector('input[name=\"page\"]')?.value),size=number(root.querySelector('select[name=\"limit\"]')?.value);\n if(current!==index||size<1||size>1000||Math.ceil(total/size)>100)throw Error('legacy pagination invalid');\n const active=[...table.querySelectorAll('thead input[name],thead select[name]')].filter(n=>!(n.type==='hidden'&&/\\[locale\\]$/.test(n.name))&&n.value!==''&&!(['store_id','massaction'].includes(n.name)&&n.value==='0')&&!(n.name==='visible_in'&&n.value==='1'));\n if(active.length)throw Error('filtered legacy grid');\n const rows=[...table.querySelectorAll('tbody > tr')].filter(tr=>!tr.querySelector('td.empty')).map(tr=>{\n const fields=identity==='review_id'?['review_id']:['query_id','query_text','num_results','popularity'];\n return Object.fromEntries(fields.map(field=>{const cells=tr.querySelectorAll('td.col-'+field);if(cells.length!==1)throw Error('legacy field missing or ambiguous');return [field,cells[0].textContent.replace(/\\s+/g,' ').trim()]}));\n });\n if(rows.length!==Math.min(size,Math.max(0,total-(index-1)*size)))throw Error('incomplete legacy page');\n for(const row of rows)number(row[identity]);\n return {rows,total,size};\n};\nconst first=collect(document,1),total=first.total,rows=[...first.rows],pages=[{page:1,rows:first.rows.length,total}],params={namespace:gridId};\nfor(let page=2;page<=Math.ceil(total/first.size);page++){\n const url=new URL(location.href);url.searchParams.set('page',String(page));url.searchParams.set('limit',String(first.size));\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal});\n if(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('legacy page fetch failed');\n const data=collect(new DOMParser().parseFromString(await response.text(),'text/html'),page);\n if(data.total!==total||data.size!==first.size)throw Error('legacy grid changed');\n rows.push(...data.rows);pages.push({page,rows:data.rows.length,total});\n}\nif(rows.length!==total||new Set(rows.map(r=>r[identity])).size!==total)throw Error('incomplete or duplicate legacy identities');\nconst field=(r,names)=>{const found=names.filter(k=>Object.hasOwn(r,k));if(found.length!==1)throw Error('ambiguous search field');return r[found[0]]};\nconst mapped=rows.map(r=>({term:String(field(r,['search_query','query_text'])).trim(),hits:Number(field(r,['popularity','uses']))}));\nif(!mapped.length||mapped.some(r=>!r.term||!Number.isSafeInteger(r.hits)||r.hits<0))throw Error('invalid search terms');\nconst top=Math.max(...mapped.map(r=>r.hits));return {scope:true,complete:true,namespace:params.namespace,pages,total,\nanswer:mapped.filter(r=>r.hits===top).map(r=>r.term)};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 41, + "canonical_calls_sha256": "47d46214956548604737347a99a76d5d0f1895f18df76388cf237da13684a031", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "ec46acd0ebe2028b13510d34a2d001e8ff3da589224c5849a84c20db16db8367", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-50.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-50.json new file mode 100644 index 000000000..296cf3a2e --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-50.json @@ -0,0 +1,40 @@ +{ + "task_id": 50, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/history/?limit=50" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nif(location.origin!==new URL(location.href).origin)throw Error('invalid origin');\nconst read=async address=>{const u=new URL(address,location.origin);\nif(u.origin!==location.origin||u.pathname!=='/sales/order/history/'||\n[...u.searchParams.keys()].some(k=>!['p','limit'].includes(k)))throw Error('out-of-scope order history');\nconst response=await fetch(u.href,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal});\nif(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('order history response unavailable');\nreturn {url:response.url,doc:new DOMParser().parseFromString(await response.text(),'text/html')}};\nconst first=await read('/sales/order/history/?limit=50');\nconst collect=(page,index)=>{const doc=page.doc;\nif(doc.querySelector('form#login-form'))throw Error('history requires login');\nconst table=doc.querySelector('table#my-orders-table,table.table-order-items');\nconst body=table?.querySelector('tbody');\nif(!body)throw Error('order table missing');\nconst entries=[...body.querySelectorAll(':scope > tr')].map(tr=>{const cells=[...tr.querySelectorAll(':scope > td')].map(td=>td.textContent.replace(/\\s+/g,' ').trim());\nconst link=[...tr.querySelectorAll('a[href*=\"/sales/order/view/order_id/\"]')];\nif(cells.length<4||link.length!==1)throw Error('invalid order row');\nconst detail=new URL(link[0].getAttribute('href'),page.url);\nif(detail.origin!==location.origin||!/^\\/sales\\/order\\/view\\/order_id\\/\\d+\\/$/.test(detail.pathname))throw Error('invalid order detail link');\nconst num=cells[0],date=cells[1],amount=cells[2],status=cells[3];\nif(!/^\\d+$/.test(num)||!/^\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}$/.test(date)||\n!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(amount)||!status)throw Error('missing order fields');\nconst [month,day,yearText]=date.split('/').map(Number),year=yearText<100?2000+yearText:yearText;\nconst iso=`${year}-${String(month).padStart(2,'0')}-${String(day).padStart(2,'0')}`;\nif(new Date(iso+'T00:00:00Z').toISOString().slice(0,10)!==iso)throw Error('invalid order date');\nconst cents=Number(amount.replace(/[$,.]/g,''));if(!Number.isSafeInteger(cents))throw Error('unsafe total');\nreturn {number:num,date:iso,cents,status,detail:detail.href};});\nconst amounts=[...doc.querySelectorAll('.toolbar-amount')].map(n=>n.textContent.replace(/\\s+/g,' ').trim());\nif(!amounts.length||new Set(amounts).size!==1)throw Error('history pagination total missing or ambiguous');\nconst toolbar=amounts[0],range=toolbar.match(/^Items?\\s+(\\d+)\\s*(?:[-–]|to)\\s*(\\d+)\\s+of\\s+(\\d+)(?:\\s+total)?$/i),single=toolbar.match(/^(\\d+)\\s+Item(?:\\(s\\)|s)?$/i);\nif(!range&&!single)throw Error('history pagination format unknown');\nconst total=Number(range?range[3]:single[1]),from=range?Number(range[1]):1,to=range?Number(range[2]):total;\nif(!Number.isSafeInteger(total)||total<1||!Number.isSafeInteger(from)||!Number.isSafeInteger(to)||from<1||to>total||to-from+1!==entries.length||index===1&&from!==1||single&&index!==1)throw Error('history pagination bounds inconsistent');\nreturn {entries,total,from,to};};\nconst initial=collect(first,1),pageSize=initial.entries.length,total=initial.total;\nif(!pageSize||Math.ceil(total/pageSize)>100)throw Error('invalid order page size');\nconst rows=[...initial.entries],pages=[{page:1,from:initial.from,to:initial.to,total}];\nfor(let page=2;page<=Math.ceil(total/pageSize);page++){\nconst data=collect(await read('/sales/order/history/?limit=50&p='+page),page);\nif(data.total!==total||data.from!==rows.length+1||data.entries.length!==Math.min(pageSize,total-rows.length))throw Error('incomplete order history');\nrows.push(...data.entries);pages.push({page,from:data.from,to:data.to,total});}\nif(rows.length!==total||new Set(rows.map(r=>r.number)).size!==total)throw Error('duplicate or incomplete history');\nif(rows.some((r,i)=>i&&rows[i-1].dater.status==='Complete'&&r.date>='2022-06-12'&&r.date<='2023-06-12');\nconst cents=found.reduce((n,r)=>n+r.cents,0);if(!Number.isSafeInteger(cents))throw Error('unsafe sum');\nreturn {scope:true,complete:true,pages,total,matched:found.map(r=>r.number),\nanswer:{order_count:found.length,amount:cents/100}};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 50, + "canonical_calls_sha256": "405268f1307a40e592811e4af5e03be58c87ef9349688f12139606461edabd2d", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "82b4db4a5f32757aa23a0565861cf11a8f3804ce2676cc8c168bcb156f0329d1", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66-budget.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66-budget.json new file mode 100644 index 000000000..85ba30d0b --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66-budget.json @@ -0,0 +1,367 @@ +{ + "task_id": 66, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books/81371/the-letters-of-t-s-eliot-to-emily-hale-that-were-kept-sealed\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:9999/f/Books" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller=new AbortController();\n const deadlineTimer=setTimeout(()=>controller.abort(),60000);\n try {\n const normalize = value => String(value ?? '').replace(/\\s+/g, ' ').trim();\n const listingUrl = new URL(location.href);\n if (listingUrl.origin !== location.origin ||\n listingUrl.pathname.toLowerCase().replace(/\\/$/, '') !== '/f/books') {\n throw new Error('wrong Books listing');\n }\n const sortControl = [...document.querySelectorAll('button[aria-label]')]\n .find(button => /^sort by:\\s*hot$/i.test(button.getAttribute('aria-label') || ''));\n if (!sortControl) throw new Error('Books listing is not visibly sorted by Hot');\n\n const listingArticles = [...document.querySelectorAll('main > article')];\n if (listingArticles.length < 10) {\n throw new Error('expected at least ten hot posts, found ' + listingArticles.length);\n }\n const discovered = listingArticles.slice(0, 10).map((article, rank) => {\n const permalinks = [...article.querySelectorAll('a[href]')]\n .map(anchor => anchor.getAttribute('href'))\n .filter(Boolean)\n .map(href => ({href, match: /^\\/f\\/books\\/(\\d+)(?:\\/|$)/i.exec(href)}))\n .filter(entry => entry.match);\n const ids = [...new Set(permalinks.map(entry => entry.match[1]))];\n if (ids.length !== 1) throw new Error('rank ' + (rank + 1) + ' has ambiguous post identity');\n const href = permalinks.find(entry => entry.match[1] === ids[0]).href;\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (!title) throw new Error('rank ' + (rank + 1) + ' has no post title');\n return {rank: rank + 1, post_id: ids[0], href, title};\n });\n if (new Set(discovered.map(post => post.post_id)).size !== 10) {\n throw new Error('hot top ten contains duplicate post identities');\n }\n\n const posts = [];\n for (const listed of discovered) {\n const detailUrl = new URL(listed.href, location.href);\n if (detailUrl.origin !== location.origin) throw new Error('cross-origin post permalink');\n const response = await fetch(detailUrl.href, {credentials: 'same-origin',redirect:'error',signal:controller.signal});\n if (!response.ok || response.redirected) {\n throw new Error('post fetch failed ' + response.status + ' ' + listed.href);\n }\n const page = new DOMParser().parseFromString(await response.text(), 'text/html');\n const article = page.querySelector('main > article');\n if (!article) throw new Error('post article missing ' + listed.post_id);\n const pattern = new RegExp('^/f/books/' + listed.post_id + '(?:/|$)', 'i');\n const detailPermalink = [...article.querySelectorAll('a[href]')]\n .map(anchor => anchor.getAttribute('href'))\n .find(href => pattern.test(href || ''));\n if (!detailPermalink) throw new Error('post identity mismatch ' + listed.post_id);\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (title !== listed.title) throw new Error('listing/detail title mismatch ' + listed.post_id);\n const articleText = normalize(article.textContent);\n const body = article.querySelector('.submission__body');\n const bodyText = body ? body.textContent : null;\n posts.push({\n ...listed,\n article_text: articleText,\n body_text: bodyText,\n });\n }\n\n return {\n schema: 'books_hot_current_observation/1',\n scope: true,\n complete: posts.length === 10,\n forum: 'books',\n sort: 'hot',\n listing_url: listingUrl.href,\n posts,\n };\n } finally { controller.abort();clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 66, + "canonical_calls_sha256": "591998e9b141f1c149586e61ce33e6e1eeed105b8718e52cd27da4705baee04a", + "sources": [ + { + "name": "books", + "step": 12, + "action_sha256": "0a620c321c7f09ec894af5be9a246d2841f88eeae078313a90173d8f19a487ec", + "decode": "json" + } + ], + "semantic_reference": { + "kind": "reviewed_span_annotation", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "evidence_sha256": "c2ca3f8f378dc0d177cafd87517c1d05f432abe020e8ac77b4ce410773ba5fe7" + }, + "reviewed_span_projection": { + "kind": "reviewed_ordered_spans/1", + "source": "books", + "collection_pointer": "/posts", + "identity_pointer": "/post_id", + "semantic_text_pointer": "/article_text", + "output_text_pointer": "/title", + "output": "selected_item_text", + "span_unit": "unicode_code_points", + "annotation": { + "schema": "lex_replay_reviewed_span_annotation/1", + "semantic": "exactly_one_book_recommendation", + "items": [ + { + "identity_sha256": "c2f152d486b7f4c22aca89e884a338e7c44e8cde0f14dc2caa4e5c09c92361a3", + "semantic_text_sha256": "d3fdb3c488e53ed58c7f3ce84ce916858d3d2d24acb9c1f030a5bac4500db457", + "output_text_sha256": "d52c282aedb6a94ece3ee54e3326e24902f1c1cf83db1d3d32126d3e7226f66f", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "75fd57a688ee1139be8d59d299593518bd6f6c3318cc4462c60c49145c240c0f", + "semantic_text_sha256": "308566470e823dd5c52c7865dd274c044246dc898a174c4b7717f7a33dc370bb", + "output_text_sha256": "23c8d20d6ec994ff4ea67c911e1c92832a3a974b982d158fd7ed58151a6c3bd9", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e4ea530a9e7192d38a22cd7e39616e9cff23240f6d71bca3e2ce23404d794828", + "semantic_text_sha256": "bb31ad3f3d96340e3e70fb1608d4e86b519b05c65cbbdc14ab1124fafaea4212", + "output_text_sha256": "15ac1a86f7556949e2e8a7a7a7a8afb3dc981b9c434dd341f9b4ec654cdb08d5", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "a2577600b0089626a300c9bdd613f95a97336f559fa42cdf045978d43459b833", + "semantic_text_sha256": "eb079fe67aa2953061bb9013a0197e9d15daadfa926899b85cd6fcdb0d778889", + "output_text_sha256": "63228ee63c7769d8bfbfe0ee3749fd1f8bd1a0dc13e1583cbe9712a657b9116d", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e7f270e372477c9c2e16c97f7f8e3ffd17ef64939587b89c22b56c9f0c9cd2fc", + "semantic_text_sha256": "37b19f8c3c3e98b7864a24951c28e8b84725cdf9aa5cc29c2cddd9d88f2101d3", + "output_text_sha256": "9fbf39b5c8e3401dbc5dda2645b0a00e6de5c9e04a9154bedf7cc64fcb3a544f", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "84652a06807063cc0672d570ff11aefd5c8aabd888150e87b9222e2d5f2062f9", + "semantic_text_sha256": "0519589599386324a7f7ec5a27f7fe7faa6c2f6aa8e482b5626a1c1104219544", + "output_text_sha256": "011c110f00e30033ec4503bed8655f09015f8efd7ee4b83c340bb9026e0e02c6", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "66a895fc34d189481c060a627adbf5f208cb6b45e4d2d2627da985546c820887", + "semantic_text_sha256": "bd624901cd6c90158cf98a3f3d2b913ffa80691db2d823d0b87656428206e7a3", + "output_text_sha256": "09a0f9d96e1e78f88fa25232e976a7451dacf9b9a25cb04c207defb4c33f97d7", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "cde84afcd407f4393eea42c81753c12363d4b600e9475bb3a0c077ff9bb84f67", + "semantic_text_sha256": "e1a850ebc83c5f736178401c59d53fe2d8134a2f9572a9a95ffc226c592e2f80", + "output_text_sha256": "b9d9393f991fcdac1e27d170349766fa6cd91d3d1f95cb42fb3f9640aec04d61", + "recommendation_count": 1, + "spans": [ + { + "start": 24, + "end": 34 + } + ] + }, + { + "identity_sha256": "27589092311867d58a79762b17f6ba77ce79c2a4e0bb2d78703c814dc0acb1b9", + "semantic_text_sha256": "86270b68643f2ba5bce707d889d0c98e874074f57f56d29d1b51477a5e7a2ec1", + "output_text_sha256": "79098a0b5125a7afdccad32f6b7c44f281752ad85ccee3ef575f28b1d8919edd", + "recommendation_count": 1, + "spans": [ + { + "start": 36, + "end": 53 + } + ] + }, + { + "identity_sha256": "db89597c0d478af089407db4a7d26adc52db46f99e1d0ba35893dd8abd995191", + "semantic_text_sha256": "7ddb5043f154cfbc688660c5a4b91f7e94e901f58b76cc855f81633efa29f447", + "output_text_sha256": "4a550a5932e0be6991b6b4a0cc71184ba247c65fc1ff018368b26b35b5721f02", + "recommendation_count": 0, + "spans": [] + } + ] + } + }, + "requirements": { + "scope": { + "source": "books", + "pointer": "/scope" + }, + "completeness": { + "source": "books", + "pointer": "/complete" + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "5ff546492ae5224002667b59cd9e37b437b83207c416a852b106b5e95147a0d0", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "5ff546492ae5224002667b59cd9e37b437b83207c416a852b106b5e95147a0d0", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\"post_id\": \"81371\", \"qualifies\": false, \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"},\n {\"post_id\": \"59421\", \"qualifies\": false, \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"},\n {\"post_id\": \"59447\", \"qualifies\": false, \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"},\n {\"post_id\": \"103102\", \"qualifies\": false, \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"},\n {\"post_id\": \"124017\", \"qualifies\": false, \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"},\n {\"post_id\": \"59466\", \"qualifies\": false, \"reason\": \"Publishing-industry survival news, no individual recommended book.\"},\n {\"post_id\": \"59478\", \"qualifies\": false, \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"},\n {\"post_id\": \"17445\", \"qualifies\": true, \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"},\n {\"post_id\": \"59396\", \"qualifies\": true, \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"},\n {\"post_id\": \"103061\", \"qualifies\": false, \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"}\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n}\n", + "sha256": "8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f", + "content": { + "scope": "Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts", + "reviewer": "parent agent, independent of candidate author", + "source_trace_sha256": "f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7", + "draft_annotation_sha256": "7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d", + "method": "Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.", + "decisions": [ + { + "post_id": "81371", + "qualifies": false, + "reason": "Notice of an online letter release, not a recommendation of a single book." + }, + { + "post_id": "59421", + "qualifies": false, + "reason": "Recommends a bookstore purchasing platform, not a book." + }, + { + "post_id": "59447", + "qualifies": false, + "reason": "Prison education and notebook donation news, with no individual book recommendation." + }, + { + "post_id": "103102", + "qualifies": false, + "reason": "Author collaboration news mentioning a catalogue of books, not one recommendation." + }, + { + "post_id": "124017", + "qualifies": false, + "reason": "Ranch preservation news; a title in the ranch's identity is not a reading recommendation." + }, + { + "post_id": "59466", + "qualifies": false, + "reason": "Publishing-industry survival news, no individual recommended book." + }, + { + "post_id": "59478", + "qualifies": false, + "reason": "General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book." + }, + { + "post_id": "17445", + "qualifies": true, + "reason": "A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books." + }, + { + "post_id": "59396", + "qualifies": true, + "reason": "Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two." + }, + { + "post_id": "103061", + "qualifies": false, + "reason": "Used-bookstore appreciation, not a specific book recommendation." + } + ], + "agreement": "All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.", + "limits": "This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output." + }, + "bindings": [ + { + "artifact_pointer": "/reviewer", + "contract_pointer": "/semantic_reference/reviewer/identity" + } + ] + } + ], + "qualification_note": "Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged." + }, + "source_review_sha256": "c481a11a5b2293df34fcda80a2214f3bca3fe58484828f5074164beaaa53f259", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"parent agent, independent of candidate author\"\n },\n \"scope\": \"exactly one book recommendation: complete ordered ten-post annotation\",\n \"task_id\": 66,\n \"sources_sha256\": \"5ff546492ae5224002667b59cd9e37b437b83207c416a852b106b5e95147a0d0\",\n \"projection_sha256\": \"340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8\",\n \"decision\": \"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\",\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"scope\\\": \\\"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\\\",\\n \\\"reviewer\\\": \\\"parent agent, independent of candidate author\\\",\\n \\\"source_trace_sha256\\\": \\\"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\\\",\\n \\\"draft_annotation_sha256\\\": \\\"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\\\",\\n \\\"method\\\": \\\"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\\\",\\n \\\"decisions\\\": [\\n {\\\"post_id\\\": \\\"81371\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Notice of an online letter release, not a recommendation of a single book.\\\"},\\n {\\\"post_id\\\": \\\"59421\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Recommends a bookstore purchasing platform, not a book.\\\"},\\n {\\\"post_id\\\": \\\"59447\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Prison education and notebook donation news, with no individual book recommendation.\\\"},\\n {\\\"post_id\\\": \\\"103102\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Author collaboration news mentioning a catalogue of books, not one recommendation.\\\"},\\n {\\\"post_id\\\": \\\"124017\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\\\"},\\n {\\\"post_id\\\": \\\"59466\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Publishing-industry survival news, no individual recommended book.\\\"},\\n {\\\"post_id\\\": \\\"59478\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\\\"},\\n {\\\"post_id\\\": \\\"17445\\\", \\\"qualifies\\\": true, \\\"reason\\\": \\\"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\\\"},\\n {\\\"post_id\\\": \\\"59396\\\", \\\"qualifies\\\": true, \\\"reason\\\": \\\"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\\\"},\\n {\\\"post_id\\\": \\\"103061\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Used-bookstore appreciation, not a specific book recommendation.\\\"}\\n ],\\n \\\"agreement\\\": \\\"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\\\",\\n \\\"limits\\\": \\\"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\\\"\\n}\\n\",\n \"sha256\": \"8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f\",\n \"content\": {\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\n \"post_id\": \"81371\",\n \"qualifies\": false,\n \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"\n },\n {\n \"post_id\": \"59421\",\n \"qualifies\": false,\n \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"\n },\n {\n \"post_id\": \"59447\",\n \"qualifies\": false,\n \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"\n },\n {\n \"post_id\": \"103102\",\n \"qualifies\": false,\n \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"\n },\n {\n \"post_id\": \"124017\",\n \"qualifies\": false,\n \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"\n },\n {\n \"post_id\": \"59466\",\n \"qualifies\": false,\n \"reason\": \"Publishing-industry survival news, no individual recommended book.\"\n },\n {\n \"post_id\": \"59478\",\n \"qualifies\": false,\n \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"\n },\n {\n \"post_id\": \"17445\",\n \"qualifies\": true,\n \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"\n },\n {\n \"post_id\": \"59396\",\n \"qualifies\": true,\n \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"\n },\n {\n \"post_id\": \"103061\",\n \"qualifies\": false,\n \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"\n }\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n },\n \"bindings\": [\n {\n \"artifact_pointer\": \"/reviewer\",\n \"contract_pointer\": \"/semantic_reference/reviewer/identity\"\n }\n ]\n }\n ],\n \"qualification_note\": \"Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged.\"\n}", + "sha256": "7007a32af49e9713be72c79c3cd8409cd555b4491b28bf497670e6b885bd35c6", + "content": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "5ff546492ae5224002667b59cd9e37b437b83207c416a852b106b5e95147a0d0", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\"post_id\": \"81371\", \"qualifies\": false, \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"},\n {\"post_id\": \"59421\", \"qualifies\": false, \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"},\n {\"post_id\": \"59447\", \"qualifies\": false, \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"},\n {\"post_id\": \"103102\", \"qualifies\": false, \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"},\n {\"post_id\": \"124017\", \"qualifies\": false, \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"},\n {\"post_id\": \"59466\", \"qualifies\": false, \"reason\": \"Publishing-industry survival news, no individual recommended book.\"},\n {\"post_id\": \"59478\", \"qualifies\": false, \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"},\n {\"post_id\": \"17445\", \"qualifies\": true, \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"},\n {\"post_id\": \"59396\", \"qualifies\": true, \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"},\n {\"post_id\": \"103061\", \"qualifies\": false, \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"}\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n}\n", + "sha256": "8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f", + "content": { + "scope": "Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts", + "reviewer": "parent agent, independent of candidate author", + "source_trace_sha256": "f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7", + "draft_annotation_sha256": "7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d", + "method": "Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.", + "decisions": [ + { + "post_id": "81371", + "qualifies": false, + "reason": "Notice of an online letter release, not a recommendation of a single book." + }, + { + "post_id": "59421", + "qualifies": false, + "reason": "Recommends a bookstore purchasing platform, not a book." + }, + { + "post_id": "59447", + "qualifies": false, + "reason": "Prison education and notebook donation news, with no individual book recommendation." + }, + { + "post_id": "103102", + "qualifies": false, + "reason": "Author collaboration news mentioning a catalogue of books, not one recommendation." + }, + { + "post_id": "124017", + "qualifies": false, + "reason": "Ranch preservation news; a title in the ranch's identity is not a reading recommendation." + }, + { + "post_id": "59466", + "qualifies": false, + "reason": "Publishing-industry survival news, no individual recommended book." + }, + { + "post_id": "59478", + "qualifies": false, + "reason": "General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book." + }, + { + "post_id": "17445", + "qualifies": true, + "reason": "A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books." + }, + { + "post_id": "59396", + "qualifies": true, + "reason": "Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two." + }, + { + "post_id": "103061", + "qualifies": false, + "reason": "Used-bookstore appreciation, not a specific book recommendation." + } + ], + "agreement": "All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.", + "limits": "This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output." + }, + "bindings": [ + { + "artifact_pointer": "/reviewer", + "contract_pointer": "/semantic_reference/reviewer/identity" + } + ] + } + ], + "qualification_note": "Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged." + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66.json new file mode 100644 index 000000000..82eeec47b --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-66.json @@ -0,0 +1,367 @@ +{ + "task_id": 66, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books/81371/the-letters-of-t-s-eliot-to-emily-hale-that-were-kept-sealed\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:9999/f/Books" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller=new AbortController();\n const deadlineTimer=setTimeout(()=>controller.abort(),20000);\n try {\n const normalize = value => String(value ?? '').replace(/\\s+/g, ' ').trim();\n const listingUrl = new URL(location.href);\n if (listingUrl.origin !== location.origin ||\n listingUrl.pathname.toLowerCase().replace(/\\/$/, '') !== '/f/books') {\n throw new Error('wrong Books listing');\n }\n const sortControl = [...document.querySelectorAll('button[aria-label]')]\n .find(button => /^sort by:\\s*hot$/i.test(button.getAttribute('aria-label') || ''));\n if (!sortControl) throw new Error('Books listing is not visibly sorted by Hot');\n\n const listingArticles = [...document.querySelectorAll('main > article')];\n if (listingArticles.length < 10) {\n throw new Error('expected at least ten hot posts, found ' + listingArticles.length);\n }\n const discovered = listingArticles.slice(0, 10).map((article, rank) => {\n const permalinks = [...article.querySelectorAll('a[href]')]\n .map(anchor => anchor.getAttribute('href'))\n .filter(Boolean)\n .map(href => ({href, match: /^\\/f\\/books\\/(\\d+)(?:\\/|$)/i.exec(href)}))\n .filter(entry => entry.match);\n const ids = [...new Set(permalinks.map(entry => entry.match[1]))];\n if (ids.length !== 1) throw new Error('rank ' + (rank + 1) + ' has ambiguous post identity');\n const href = permalinks.find(entry => entry.match[1] === ids[0]).href;\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (!title) throw new Error('rank ' + (rank + 1) + ' has no post title');\n return {rank: rank + 1, post_id: ids[0], href, title};\n });\n if (new Set(discovered.map(post => post.post_id)).size !== 10) {\n throw new Error('hot top ten contains duplicate post identities');\n }\n\n const posts = [];\n for (const listed of discovered) {\n const detailUrl = new URL(listed.href, location.href);\n if (detailUrl.origin !== location.origin) throw new Error('cross-origin post permalink');\n const response = await fetch(detailUrl.href, {credentials: 'same-origin',redirect:'error',signal:controller.signal});\n if (!response.ok || response.redirected) {\n throw new Error('post fetch failed ' + response.status + ' ' + listed.href);\n }\n const page = new DOMParser().parseFromString(await response.text(), 'text/html');\n const article = page.querySelector('main > article');\n if (!article) throw new Error('post article missing ' + listed.post_id);\n const pattern = new RegExp('^/f/books/' + listed.post_id + '(?:/|$)', 'i');\n const detailPermalink = [...article.querySelectorAll('a[href]')]\n .map(anchor => anchor.getAttribute('href'))\n .find(href => pattern.test(href || ''));\n if (!detailPermalink) throw new Error('post identity mismatch ' + listed.post_id);\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (title !== listed.title) throw new Error('listing/detail title mismatch ' + listed.post_id);\n const articleText = normalize(article.textContent);\n const body = article.querySelector('.submission__body');\n const bodyText = body ? body.textContent : null;\n posts.push({\n ...listed,\n article_text: articleText,\n body_text: bodyText,\n });\n }\n\n return {\n schema: 'books_hot_current_observation/1',\n scope: true,\n complete: posts.length === 10,\n forum: 'books',\n sort: 'hot',\n listing_url: listingUrl.href,\n posts,\n };\n } finally { controller.abort();clearTimeout(deadlineTimer); }\n})()\n" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 66, + "canonical_calls_sha256": "6f26e505ed350c7ab7b73cd46fb523002f362bbb8ccb9e20f7906096480bf661", + "sources": [ + { + "name": "books", + "step": 12, + "action_sha256": "bc265165c93aaf89406a6284690e99fe1438bd7a6e2bfade4c981fc760b7b1ec", + "decode": "json" + } + ], + "semantic_reference": { + "kind": "reviewed_span_annotation", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "evidence_sha256": "d1a5d8251a10dd7a0120bb73ed569bfc3698bcedf5633da33fa9a025fa2f0983" + }, + "reviewed_span_projection": { + "kind": "reviewed_ordered_spans/1", + "source": "books", + "collection_pointer": "/posts", + "identity_pointer": "/post_id", + "semantic_text_pointer": "/article_text", + "output_text_pointer": "/title", + "output": "selected_item_text", + "span_unit": "unicode_code_points", + "annotation": { + "schema": "lex_replay_reviewed_span_annotation/1", + "semantic": "exactly_one_book_recommendation", + "items": [ + { + "identity_sha256": "c2f152d486b7f4c22aca89e884a338e7c44e8cde0f14dc2caa4e5c09c92361a3", + "semantic_text_sha256": "d3fdb3c488e53ed58c7f3ce84ce916858d3d2d24acb9c1f030a5bac4500db457", + "output_text_sha256": "d52c282aedb6a94ece3ee54e3326e24902f1c1cf83db1d3d32126d3e7226f66f", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "75fd57a688ee1139be8d59d299593518bd6f6c3318cc4462c60c49145c240c0f", + "semantic_text_sha256": "308566470e823dd5c52c7865dd274c044246dc898a174c4b7717f7a33dc370bb", + "output_text_sha256": "23c8d20d6ec994ff4ea67c911e1c92832a3a974b982d158fd7ed58151a6c3bd9", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e4ea530a9e7192d38a22cd7e39616e9cff23240f6d71bca3e2ce23404d794828", + "semantic_text_sha256": "bb31ad3f3d96340e3e70fb1608d4e86b519b05c65cbbdc14ab1124fafaea4212", + "output_text_sha256": "15ac1a86f7556949e2e8a7a7a7a8afb3dc981b9c434dd341f9b4ec654cdb08d5", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "a2577600b0089626a300c9bdd613f95a97336f559fa42cdf045978d43459b833", + "semantic_text_sha256": "eb079fe67aa2953061bb9013a0197e9d15daadfa926899b85cd6fcdb0d778889", + "output_text_sha256": "63228ee63c7769d8bfbfe0ee3749fd1f8bd1a0dc13e1583cbe9712a657b9116d", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e7f270e372477c9c2e16c97f7f8e3ffd17ef64939587b89c22b56c9f0c9cd2fc", + "semantic_text_sha256": "37b19f8c3c3e98b7864a24951c28e8b84725cdf9aa5cc29c2cddd9d88f2101d3", + "output_text_sha256": "9fbf39b5c8e3401dbc5dda2645b0a00e6de5c9e04a9154bedf7cc64fcb3a544f", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "84652a06807063cc0672d570ff11aefd5c8aabd888150e87b9222e2d5f2062f9", + "semantic_text_sha256": "0519589599386324a7f7ec5a27f7fe7faa6c2f6aa8e482b5626a1c1104219544", + "output_text_sha256": "011c110f00e30033ec4503bed8655f09015f8efd7ee4b83c340bb9026e0e02c6", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "66a895fc34d189481c060a627adbf5f208cb6b45e4d2d2627da985546c820887", + "semantic_text_sha256": "bd624901cd6c90158cf98a3f3d2b913ffa80691db2d823d0b87656428206e7a3", + "output_text_sha256": "09a0f9d96e1e78f88fa25232e976a7451dacf9b9a25cb04c207defb4c33f97d7", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "cde84afcd407f4393eea42c81753c12363d4b600e9475bb3a0c077ff9bb84f67", + "semantic_text_sha256": "e1a850ebc83c5f736178401c59d53fe2d8134a2f9572a9a95ffc226c592e2f80", + "output_text_sha256": "b9d9393f991fcdac1e27d170349766fa6cd91d3d1f95cb42fb3f9640aec04d61", + "recommendation_count": 1, + "spans": [ + { + "start": 24, + "end": 34 + } + ] + }, + { + "identity_sha256": "27589092311867d58a79762b17f6ba77ce79c2a4e0bb2d78703c814dc0acb1b9", + "semantic_text_sha256": "86270b68643f2ba5bce707d889d0c98e874074f57f56d29d1b51477a5e7a2ec1", + "output_text_sha256": "79098a0b5125a7afdccad32f6b7c44f281752ad85ccee3ef575f28b1d8919edd", + "recommendation_count": 1, + "spans": [ + { + "start": 36, + "end": 53 + } + ] + }, + { + "identity_sha256": "db89597c0d478af089407db4a7d26adc52db46f99e1d0ba35893dd8abd995191", + "semantic_text_sha256": "7ddb5043f154cfbc688660c5a4b91f7e94e901f58b76cc855f81633efa29f447", + "output_text_sha256": "4a550a5932e0be6991b6b4a0cc71184ba247c65fc1ff018368b26b35b5721f02", + "recommendation_count": 0, + "spans": [] + } + ] + } + }, + "requirements": { + "scope": { + "source": "books", + "pointer": "/scope" + }, + "completeness": { + "source": "books", + "pointer": "/complete" + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "2913c6c7436c85f4cf8a77fa1eff836a00a905b903066e0321cb061a18513237", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "2913c6c7436c85f4cf8a77fa1eff836a00a905b903066e0321cb061a18513237", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\"post_id\": \"81371\", \"qualifies\": false, \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"},\n {\"post_id\": \"59421\", \"qualifies\": false, \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"},\n {\"post_id\": \"59447\", \"qualifies\": false, \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"},\n {\"post_id\": \"103102\", \"qualifies\": false, \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"},\n {\"post_id\": \"124017\", \"qualifies\": false, \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"},\n {\"post_id\": \"59466\", \"qualifies\": false, \"reason\": \"Publishing-industry survival news, no individual recommended book.\"},\n {\"post_id\": \"59478\", \"qualifies\": false, \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"},\n {\"post_id\": \"17445\", \"qualifies\": true, \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"},\n {\"post_id\": \"59396\", \"qualifies\": true, \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"},\n {\"post_id\": \"103061\", \"qualifies\": false, \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"}\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n}\n", + "sha256": "8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f", + "content": { + "scope": "Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts", + "reviewer": "parent agent, independent of candidate author", + "source_trace_sha256": "f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7", + "draft_annotation_sha256": "7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d", + "method": "Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.", + "decisions": [ + { + "post_id": "81371", + "qualifies": false, + "reason": "Notice of an online letter release, not a recommendation of a single book." + }, + { + "post_id": "59421", + "qualifies": false, + "reason": "Recommends a bookstore purchasing platform, not a book." + }, + { + "post_id": "59447", + "qualifies": false, + "reason": "Prison education and notebook donation news, with no individual book recommendation." + }, + { + "post_id": "103102", + "qualifies": false, + "reason": "Author collaboration news mentioning a catalogue of books, not one recommendation." + }, + { + "post_id": "124017", + "qualifies": false, + "reason": "Ranch preservation news; a title in the ranch's identity is not a reading recommendation." + }, + { + "post_id": "59466", + "qualifies": false, + "reason": "Publishing-industry survival news, no individual recommended book." + }, + { + "post_id": "59478", + "qualifies": false, + "reason": "General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book." + }, + { + "post_id": "17445", + "qualifies": true, + "reason": "A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books." + }, + { + "post_id": "59396", + "qualifies": true, + "reason": "Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two." + }, + { + "post_id": "103061", + "qualifies": false, + "reason": "Used-bookstore appreciation, not a specific book recommendation." + } + ], + "agreement": "All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.", + "limits": "This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output." + }, + "bindings": [ + { + "artifact_pointer": "/reviewer", + "contract_pointer": "/semantic_reference/reviewer/identity" + } + ] + } + ], + "qualification_note": "Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged." + }, + "source_review_sha256": "568dc5ff1085b7dd0e926a3607c7bb68915ad9112a944e66431060f71f874ed2", + "source_review_artifact": { + "utf8": "{\n \"reviewer\": {\n \"kind\": \"agent\",\n \"identity\": \"parent agent, independent of candidate author\"\n },\n \"scope\": \"exactly one book recommendation: complete ordered ten-post annotation\",\n \"task_id\": 66,\n \"sources_sha256\": \"2913c6c7436c85f4cf8a77fa1eff836a00a905b903066e0321cb061a18513237\",\n \"projection_sha256\": \"340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8\",\n \"decision\": \"Preserve raw source identities; apply explicitly scoped downstream interpretation only.\",\n \"predecessor_artifacts\": [\n {\n \"utf8\": \"{\\n \\\"scope\\\": \\\"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\\\",\\n \\\"reviewer\\\": \\\"parent agent, independent of candidate author\\\",\\n \\\"source_trace_sha256\\\": \\\"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\\\",\\n \\\"draft_annotation_sha256\\\": \\\"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\\\",\\n \\\"method\\\": \\\"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\\\",\\n \\\"decisions\\\": [\\n {\\\"post_id\\\": \\\"81371\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Notice of an online letter release, not a recommendation of a single book.\\\"},\\n {\\\"post_id\\\": \\\"59421\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Recommends a bookstore purchasing platform, not a book.\\\"},\\n {\\\"post_id\\\": \\\"59447\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Prison education and notebook donation news, with no individual book recommendation.\\\"},\\n {\\\"post_id\\\": \\\"103102\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Author collaboration news mentioning a catalogue of books, not one recommendation.\\\"},\\n {\\\"post_id\\\": \\\"124017\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\\\"},\\n {\\\"post_id\\\": \\\"59466\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Publishing-industry survival news, no individual recommended book.\\\"},\\n {\\\"post_id\\\": \\\"59478\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\\\"},\\n {\\\"post_id\\\": \\\"17445\\\", \\\"qualifies\\\": true, \\\"reason\\\": \\\"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\\\"},\\n {\\\"post_id\\\": \\\"59396\\\", \\\"qualifies\\\": true, \\\"reason\\\": \\\"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\\\"},\\n {\\\"post_id\\\": \\\"103061\\\", \\\"qualifies\\\": false, \\\"reason\\\": \\\"Used-bookstore appreciation, not a specific book recommendation.\\\"}\\n ],\\n \\\"agreement\\\": \\\"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\\\",\\n \\\"limits\\\": \\\"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\\\"\\n}\\n\",\n \"sha256\": \"8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f\",\n \"content\": {\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\n \"post_id\": \"81371\",\n \"qualifies\": false,\n \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"\n },\n {\n \"post_id\": \"59421\",\n \"qualifies\": false,\n \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"\n },\n {\n \"post_id\": \"59447\",\n \"qualifies\": false,\n \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"\n },\n {\n \"post_id\": \"103102\",\n \"qualifies\": false,\n \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"\n },\n {\n \"post_id\": \"124017\",\n \"qualifies\": false,\n \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"\n },\n {\n \"post_id\": \"59466\",\n \"qualifies\": false,\n \"reason\": \"Publishing-industry survival news, no individual recommended book.\"\n },\n {\n \"post_id\": \"59478\",\n \"qualifies\": false,\n \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"\n },\n {\n \"post_id\": \"17445\",\n \"qualifies\": true,\n \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"\n },\n {\n \"post_id\": \"59396\",\n \"qualifies\": true,\n \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"\n },\n {\n \"post_id\": \"103061\",\n \"qualifies\": false,\n \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"\n }\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n },\n \"bindings\": [\n {\n \"artifact_pointer\": \"/reviewer\",\n \"contract_pointer\": \"/semantic_reference/reviewer/identity\"\n }\n ]\n }\n ],\n \"qualification_note\": \"Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged.\"\n}", + "sha256": "6865889bf87ec2d9c5e04deacfea70b5b11cdfd9eaf28777edbc7d34b11e01a5", + "content": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent of candidate author" + }, + "scope": "exactly one book recommendation: complete ordered ten-post annotation", + "task_id": 66, + "sources_sha256": "2913c6c7436c85f4cf8a77fa1eff836a00a905b903066e0321cb061a18513237", + "projection_sha256": "340a5d725d78633f2472acd51d17dd614e4998aa2b0473f74300784d572472e8", + "decision": "Preserve raw source identities; apply explicitly scoped downstream interpretation only.", + "predecessor_artifacts": [ + { + "utf8": "{\n \"scope\": \"Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts\",\n \"reviewer\": \"parent agent, independent of candidate author\",\n \"source_trace_sha256\": \"f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7\",\n \"draft_annotation_sha256\": \"7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d\",\n \"method\": \"Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.\",\n \"decisions\": [\n {\"post_id\": \"81371\", \"qualifies\": false, \"reason\": \"Notice of an online letter release, not a recommendation of a single book.\"},\n {\"post_id\": \"59421\", \"qualifies\": false, \"reason\": \"Recommends a bookstore purchasing platform, not a book.\"},\n {\"post_id\": \"59447\", \"qualifies\": false, \"reason\": \"Prison education and notebook donation news, with no individual book recommendation.\"},\n {\"post_id\": \"103102\", \"qualifies\": false, \"reason\": \"Author collaboration news mentioning a catalogue of books, not one recommendation.\"},\n {\"post_id\": \"124017\", \"qualifies\": false, \"reason\": \"Ranch preservation news; a title in the ranch's identity is not a reading recommendation.\"},\n {\"post_id\": \"59466\", \"qualifies\": false, \"reason\": \"Publishing-industry survival news, no individual recommended book.\"},\n {\"post_id\": \"59478\", \"qualifies\": false, \"reason\": \"General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book.\"},\n {\"post_id\": \"17445\", \"qualifies\": true, \"reason\": \"A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books.\"},\n {\"post_id\": \"59396\", \"qualifies\": true, \"reason\": \"Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two.\"},\n {\"post_id\": \"103061\", \"qualifies\": false, \"reason\": \"Used-bookstore appreciation, not a specific book recommendation.\"}\n ],\n \"agreement\": \"All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.\",\n \"limits\": \"This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output.\"\n}\n", + "sha256": "8edda3e2c4c12c75a11a25adf167c741b695b839e39f14a38416f84f9220966f", + "content": { + "scope": "Independent review of ten proposed semantic reference labels, not a review or alteration of browser verdicts", + "reviewer": "parent agent, independent of candidate author", + "source_trace_sha256": "f34650670d1191068129bc0087cc7b936f8aa99af4728e1834da3f3257ad57d7", + "draft_annotation_sha256": "7229d803c15826271bbc0065b866df334da782152a6ced0c53fc3a9855221b6d", + "method": "Read complete retained article text for all ten posts in observed order; distinguish positive recommendation of one identifiable book from mentions, general author praise, news and bookstore recommendations.", + "decisions": [ + { + "post_id": "81371", + "qualifies": false, + "reason": "Notice of an online letter release, not a recommendation of a single book." + }, + { + "post_id": "59421", + "qualifies": false, + "reason": "Recommends a bookstore purchasing platform, not a book." + }, + { + "post_id": "59447", + "qualifies": false, + "reason": "Prison education and notebook donation news, with no individual book recommendation." + }, + { + "post_id": "103102", + "qualifies": false, + "reason": "Author collaboration news mentioning a catalogue of books, not one recommendation." + }, + { + "post_id": "124017", + "qualifies": false, + "reason": "Ranch preservation news; a title in the ranch's identity is not a reading recommendation." + }, + { + "post_id": "59466", + "qualifies": false, + "reason": "Publishing-industry survival news, no individual recommended book." + }, + { + "post_id": "59478", + "qualifies": false, + "reason": "General praise and critique of an author's mysteries; indirect allusions and plot examples do not isolate one recommended book." + }, + { + "post_id": "17445", + "qualifies": true, + "reason": "A positive report of reading the title book. The trilogy is mentioned as a tentative future possibility with reservations; movie adaptations are not additional recommended books." + }, + { + "post_id": "59396", + "qualifies": true, + "reason": "Explicitly positive reading/listening recommendation of the title book; the print and audio versions are the same book, not two." + }, + { + "post_id": "103061", + "qualifies": false, + "reason": "Used-bookstore appreciation, not a specific book recommendation." + } + ], + "agreement": "All ten inclusion/exclusion labels agree with the draft under the stated rule. Nonqualifying zero counts mean no individually identified book satisfying this rule, not a claim that every general endorsement has literal cardinality zero.", + "limits": "This validates only the retained semantic reference. Fresh current-page content, order, full-text hashes and source spans must still match during qualification; otherwise fail closed. No historical answer is a measured output." + }, + "bindings": [ + { + "artifact_pointer": "/reviewer", + "contract_pointer": "/semantic_reference/reviewer/identity" + } + ] + } + ], + "qualification_note": "Original qualified nine-action prefix retained; appended current reading route returns from long article to Books listing. Semantic annotation unchanged." + } + } + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-67.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-67.json new file mode 100644 index 000000000..b07509377 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-67.json @@ -0,0 +1,188 @@ +{ + "task_id": 67, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/f/Books" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async()=>{const links=[...new Set([...document.querySelectorAll('main article a[href*=\"/f/books/\"]')].map(a=>a.getAttribute('href')).filter(Boolean))].slice(0,10);if(links.length!==10)throw new Error('expected ten hot-post links, found '+links.length);return Promise.all(links.map(async href=>{const response=await fetch(href);if(!response.ok)throw new Error('post fetch failed '+response.status+' '+href);const page=new DOMParser().parseFromString(await response.text(),'text/html');return {href,text:page.querySelector('main article')?.textContent?.replace(/\\s+/g,' ').trim()||null};}));})()" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), 20000);\n try {\n const listing = new URL(location.href);\n if (listing.origin !== location.origin || listing.pathname.toLowerCase().replace(/\\/$/, '') !== '/f/books' || listing.search) throw new Error('wrong Books listing');\n const hot = [...document.querySelectorAll('button[aria-label]')].some(node => /^sort by:\\s*hot$/i.test(node.getAttribute('aria-label') || ''));\n if (!hot) throw new Error('Books listing is not sorted by Hot');\n const articles = [...document.querySelectorAll('main > article')];\n if (articles.length < 10) throw new Error('fewer than ten hot posts');\n const normalize = text => String(text ?? '').replace(/\\s+/g, ' ').trim();\n const entries = articles.slice(0, 10).map((article, index) => {\n const links = [...article.querySelectorAll('a[href]')].map(a => a.getAttribute('href')).filter(Boolean);\n const matches = links.map(href => ({href, match: /^\\/f\\/books\\/(\\d+)(?:\\/|$)/i.exec(href)})).filter(row => row.match);\n const ids = [...new Set(matches.map(row => row.match[1]))];\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (ids.length !== 1 || !title) throw new Error('missing or ambiguous listed post identity');\n return {rank: index + 1, post_id: ids[0], href: matches[0].href, title};\n });\n if (new Set(entries.map(row => row.post_id)).size !== 10) throw new Error('duplicate top-ten post');\n const posts = [];\n for (const entry of entries) {\n const url = new URL(entry.href, location.href);\n if (url.origin !== location.origin || !new RegExp('^/f/books/' + entry.post_id + '(?:/|$)', 'i').test(url.pathname)) throw new Error('out-of-scope post');\n const response = await fetch(url.href, {credentials:'same-origin',redirect:'error',signal:controller.signal});\n if (response.status !== 200 || response.redirected || !/^text\\/html\\b/i.test(response.headers.get('content-type') || '')) throw new Error('post detail unavailable');\n const page = new DOMParser().parseFromString(await response.text(), 'text/html');\n const article = page.querySelector('main > article');\n const detailTitle = normalize(article?.querySelector('header h1')?.textContent);\n const permalinks = [...(article?.querySelectorAll('a[href]') || [])].map(a => a.getAttribute('href')).filter(Boolean);\n if (!article || detailTitle !== entry.title || !permalinks.some(href => new RegExp('^/f/books/' + entry.post_id + '(?:/|$)', 'i').test(href))) throw new Error('listing/detail identity mismatch');\n posts.push({...entry, detail_url:response.url, article_text:normalize(article.textContent), description:article.querySelector('.submission__body')?.textContent ?? ''});\n }\n return {schema:'books_hot_top_ten_current_observation/1',scope:true,complete:posts.length===10,listing_url:listing.href,sort:'hot',posts};\n } finally { controller.abort(); clearTimeout(timer); }\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/3", + "task_id": 67, + "canonical_calls_sha256": "c89bfafce5cf4b4804b6ef74a2a9564d7d451417fdfb5257f32b78297c64da67", + "sources": [ + { + "name": "books", + "step": 4, + "action_sha256": "4d92ec7adf014fa04227dd23c965cd4f1351be78354ae8381b3d7920508aa923", + "decode": "json" + } + ], + "semantic_reference": { + "kind": "reviewed_span_annotation", + "browser_semantic_inference": false, + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent source review" + }, + "scope": "Ordered Books Hot top ten: exactly one recommended book with its title in the post description", + "evidence_sha256": "170eb27d62de0e9f8c943e89b4ad694413259fe8b0def065648991cb6c24ebf6" + }, + "reviewed_span_projection": { + "kind": "reviewed_ordered_spans/1", + "source": "books", + "collection_pointer": "/posts", + "identity_pointer": "/post_id", + "semantic_text_pointer": "/description", + "output_text_pointer": "/description", + "output": "selected_spans", + "span_unit": "unicode_code_points", + "annotation": { + "schema": "lex_replay_reviewed_span_annotation/1", + "semantic": "exactly_one_book_recommendation", + "items": [ + { + "identity_sha256": "c2f152d486b7f4c22aca89e884a338e7c44e8cde0f14dc2caa4e5c09c92361a3", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "75fd57a688ee1139be8d59d299593518bd6f6c3318cc4462c60c49145c240c0f", + "semantic_text_sha256": "52bec4be727b86dc39742550830f6cbd54fe07be3712afd55e113a5e65040498", + "output_text_sha256": "52bec4be727b86dc39742550830f6cbd54fe07be3712afd55e113a5e65040498", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e4ea530a9e7192d38a22cd7e39616e9cff23240f6d71bca3e2ce23404d794828", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "a2577600b0089626a300c9bdd613f95a97336f559fa42cdf045978d43459b833", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "e7f270e372477c9c2e16c97f7f8e3ffd17ef64939587b89c22b56c9f0c9cd2fc", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "84652a06807063cc0672d570ff11aefd5c8aabd888150e87b9222e2d5f2062f9", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "66a895fc34d189481c060a627adbf5f208cb6b45e4d2d2627da985546c820887", + "semantic_text_sha256": "0db99ff84c30b0aeb6e68ffc6b44f755c5fbb5034521c8c4161f9dafb63a4fd9", + "output_text_sha256": "0db99ff84c30b0aeb6e68ffc6b44f755c5fbb5034521c8c4161f9dafb63a4fd9", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "cde84afcd407f4393eea42c81753c12363d4b600e9475bb3a0c077ff9bb84f67", + "semantic_text_sha256": "dfb76690d41dd958ed684323806162013eea0477b0bb0dcb03fc4ec09fc32cd5", + "output_text_sha256": "dfb76690d41dd958ed684323806162013eea0477b0bb0dcb03fc4ec09fc32cd5", + "recommendation_count": 1, + "spans": [ + { + "start": 463, + "end": 473 + } + ] + }, + { + "identity_sha256": "27589092311867d58a79762b17f6ba77ce79c2a4e0bb2d78703c814dc0acb1b9", + "semantic_text_sha256": "73491f046369eeff4976b10a2fafbe183c36ea874867f7c63a441a96b70a625b", + "output_text_sha256": "73491f046369eeff4976b10a2fafbe183c36ea874867f7c63a441a96b70a625b", + "recommendation_count": 0, + "spans": [] + }, + { + "identity_sha256": "db89597c0d478af089407db4a7d26adc52db46f99e1d0ba35893dd8abd995191", + "semantic_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_text_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "recommendation_count": 0, + "spans": [] + } + ] + } + }, + "requirements": { + "scope": { + "source": "books", + "pointer": "/scope" + }, + "completeness": { + "source": "books", + "pointer": "/complete" + } + }, + "review_evidence": { + "schema": "lex_replay_review_evidence/1", + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent source review" + }, + "scope": "Ordered Books Hot top ten: exactly one recommended book with its title in the post description", + "task_id": 67, + "sources_sha256": "20a7a51f8a137175f3b0bea182c375083d3264eaf4bdf2e1c6b3902f3e6ef91b", + "projection_sha256": "fd08641aa2aa4a981d341534cf2fe709e18f7d202325719c6c10486b7804a2f0", + "source_review": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent source review" + }, + "scope": "Ordered Books Hot top ten: exactly one recommended book with its title in the post description", + "task_id": 67, + "sources_sha256": "20a7a51f8a137175f3b0bea182c375083d3264eaf4bdf2e1c6b3902f3e6ef91b", + "projection_sha256": "fd08641aa2aa4a981d341534cf2fe709e18f7d202325719c6c10486b7804a2f0", + "decision": "Only the body of post 17445 yields an exact book title; post 59396 names its book only in the header.", + "audit_sha256": "7057e6eb5cfd9c49d23d1984b9fce96e8ea13f68a4698dba5cef0eb70739bef2", + "predecessor_artifacts": [] + }, + "source_review_sha256": "45016f178efd195852e3a673c870e1a6f1c2261328efc17407cd56e440b5ed9b", + "source_review_artifact": { + "utf8": "{\"audit_sha256\":\"7057e6eb5cfd9c49d23d1984b9fce96e8ea13f68a4698dba5cef0eb70739bef2\",\"decision\":\"Only the body of post 17445 yields an exact book title; post 59396 names its book only in the header.\",\"predecessor_artifacts\":[],\"projection_sha256\":\"fd08641aa2aa4a981d341534cf2fe709e18f7d202325719c6c10486b7804a2f0\",\"reviewer\":{\"identity\":\"parent agent, independent source review\",\"kind\":\"agent\"},\"scope\":\"Ordered Books Hot top ten: exactly one recommended book with its title in the post description\",\"sources_sha256\":\"20a7a51f8a137175f3b0bea182c375083d3264eaf4bdf2e1c6b3902f3e6ef91b\",\"task_id\":67}", + "sha256": "45016f178efd195852e3a673c870e1a6f1c2261328efc17407cd56e440b5ed9b", + "content": { + "reviewer": { + "kind": "agent", + "identity": "parent agent, independent source review" + }, + "scope": "Ordered Books Hot top ten: exactly one recommended book with its title in the post description", + "task_id": 67, + "sources_sha256": "20a7a51f8a137175f3b0bea182c375083d3264eaf4bdf2e1c6b3902f3e6ef91b", + "projection_sha256": "fd08641aa2aa4a981d341534cf2fe709e18f7d202325719c6c10486b7804a2f0", + "decision": "Only the body of post 17445 yields an exact book title; post 59396 names its book only in the header.", + "audit_sha256": "7057e6eb5cfd9c49d23d1984b9fce96e8ea13f68a4698dba5cef0eb70739bef2", + "predecessor_artifacts": [] + } + } + } + }, + "qualification_status": "source_reviewed" +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-77.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-77.json new file mode 100644 index 000000000..826611655 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-77.json @@ -0,0 +1,40 @@ +{ + "task_id": 77, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/pending/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),30000);try{\nconst gridId=\"reviewGrid\",identity=\"review_id\";\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/review/product/pending\")throw Error('wrong legacy grid page');\nconst number=(text)=>{const s=String(text??'').trim();if(!/^\\d+$/.test(s)||!Number.isSafeInteger(Number(s)))throw Error('invalid grid integer');return Number(s)};\nconst collect=(doc,index)=>{\n const root=doc.getElementById(gridId),table=doc.getElementById(gridId+'_table');\n if(!root||!table||!root.contains(table))throw Error('legacy grid missing');\n const total=number(doc.getElementById(gridId+'-total-count')?.textContent);\n const current=number(root.querySelector('input[name=\"page\"]')?.value),size=number(root.querySelector('select[name=\"limit\"]')?.value);\n if(current!==index||size<1||size>1000||Math.ceil(total/size)>100)throw Error('legacy pagination invalid');\n const active=[...table.querySelectorAll('thead input[name],thead select[name]')].filter(n=>!(n.type==='hidden'&&/\\[locale\\]$/.test(n.name))&&n.value!==''&&!(['store_id','massaction'].includes(n.name)&&n.value==='0')&&!(n.name==='visible_in'&&n.value==='1'));\n if(active.length)throw Error('filtered legacy grid');\n const rows=[...table.querySelectorAll('tbody > tr')].filter(tr=>!tr.querySelector('td.empty')).map(tr=>{\n const fields=identity==='review_id'?['review_id']:['query_id','query_text','num_results','popularity'];\n return Object.fromEntries(fields.map(field=>{const cells=tr.querySelectorAll('td.col-'+field);if(cells.length!==1)throw Error('legacy field missing or ambiguous');return [field,cells[0].textContent.replace(/\\s+/g,' ').trim()]}));\n });\n if(rows.length!==Math.min(size,Math.max(0,total-(index-1)*size)))throw Error('incomplete legacy page');\n for(const row of rows)number(row[identity]);\n return {rows,total,size};\n};\nconst first=collect(document,1),total=first.total,rows=[...first.rows],pages=[{page:1,rows:first.rows.length,total}],params={namespace:gridId};\nfor(let page=2;page<=Math.ceil(total/first.size);page++){\n const url=new URL(location.href);url.searchParams.set('page',String(page));url.searchParams.set('limit',String(first.size));\n const response=await fetch(url.href,{credentials:'same-origin',redirect:'error',signal:controller.signal});\n if(!response.ok||response.redirected||!/^text\\/html\\b/i.test(response.headers.get('content-type')||''))throw Error('legacy page fetch failed');\n const data=collect(new DOMParser().parseFromString(await response.text(),'text/html'),page);\n if(data.total!==total||data.size!==first.size)throw Error('legacy grid changed');\n rows.push(...data.rows);pages.push({page,rows:data.rows.length,total});\n}\nif(rows.length!==total||new Set(rows.map(r=>r[identity])).size!==total)throw Error('incomplete or duplicate legacy identities');\nreturn {scope:true,complete:true,namespace:params.namespace,pages,total,answer:total};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 77, + "canonical_calls_sha256": "9249148d16515efe3b3b7fe50368cc004a474d5820cc43526815f9625852819f", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "e46e9a66c7859fda3fc0fa48f3def23efe54113e5dcc088b8d570a373f5500ec", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-784.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-784.json new file mode 100644 index 000000000..dc375796a --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-784.json @@ -0,0 +1,76 @@ +{ + "task_id": 784, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/CellularPrivacy/Android-IMSI-Catcher-Detector" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/CellularPrivacy/Android-IMSI-Catcher-Detector/-/graphs/master/charts" + }, + { + "action": "evaluate", + "script": "JSON.stringify(performance.getEntriesByType('resource').map(x=>x.name).filter(x=>x.includes('graphs')||x.includes('chart')||x.includes('commit')))" + }, + { + "action": "evaluate", + "script": "document.documentElement.innerHTML.includes('commit_data') ? document.documentElement.innerHTML.slice(document.documentElement.innerHTML.indexOf('commit_data')-500,document.documentElement.innerHTML.indexOf('commit_data')+1000) : document.documentElement.innerHTML.match(/data-[^=]+='[^']{100,}'/g)?.slice(0,10)" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100" + }, + { + "action": "get_text", + "selector": "pre" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100&page=2" + }, + { + "action": "get_text", + "selector": "pre" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100&page=3" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), 20000);\n try {\n const expectedPath = 'CellularPrivacy/Android-IMSI-Catcher-Detector';\n const branchName = 'master';\n const api = location.origin + '/api/v4';\n const read = async url => {\n const response = await fetch(url, {credentials:'same-origin', redirect:'error', signal:controller.signal});\n if (!response.ok || response.redirected) throw new Error('commit enumeration request failed or redirected');\n return {response, body:await response.json()};\n };\n const project = (await read(api + '/projects/' + encodeURIComponent(expectedPath))).body;\n if (!Number.isInteger(project.id) || project.path_with_namespace !== expectedPath) throw new Error('project identity mismatch');\n const branch = (await read(api + '/projects/' + project.id + '/repository/branches/' + encodeURIComponent(branchName))).body;\n const sha = /^[0-9a-f]{40}$/;\n if (branch.name !== branchName || !sha.test(branch.commit?.id || '')) throw new Error('master branch identity unavailable');\n const head = branch.commit.id;\n const commits = [], pages = [], ids = new Set();\n for (let page = 1; page <= 100; page++) {\n const url = new URL(api + '/projects/' + project.id + '/repository/commits');\n url.searchParams.set('ref_name', head);\n url.searchParams.set('order', 'topo');\n url.searchParams.set('per_page', '100');\n url.searchParams.set('page', String(page));\n const {response, body} = await read(url.href);\n const current = response.headers.get('x-page'), next = response.headers.get('x-next-page');\n if (current !== String(page) || next === null || !Array.isArray(body) || body.length > 100) throw new Error('invalid commit pagination');\n for (const row of body) {\n if (!sha.test(row.id || '') || ids.has(row.id) || !Array.isArray(row.parent_ids) ||\n row.parent_ids.some(id => !sha.test(id)) || typeof row.author_email !== 'string' ||\n typeof row.author_name !== 'string') throw new Error('invalid or duplicate commit');\n ids.add(row.id);\n commits.push({id:row.id, parent_ids:row.parent_ids, author_name:row.author_name, author_email:row.author_email});\n }\n pages.push({page, next, rows:body.length});\n if (next === '') break;\n if (next !== String(page + 1) || page === 100) throw new Error('commit pagination gap or safety bound exceeded');\n }\n if (!commits.length || commits[0].id !== head || pages.at(-1).next !== '') throw new Error('master enumeration incomplete');\n const positions = new Map(commits.map((row,index) => [row.id,index]));\n for (const [index,row] of commits.entries()) {\n if (row.parent_ids.some(id => !positions.has(id) || positions.get(id) <= index)) throw new Error('missing parent or invalid topological order');\n }\n const byId = new Map(commits.map(row => [row.id,row]));\n const reachable = new Set(), pending = [head];\n while (pending.length) {\n const id = pending.pop();\n if (reachable.has(id)) continue;\n reachable.add(id); pending.push(...byId.get(id).parent_ids);\n }\n if (reachable.size !== commits.length) throw new Error('commits outside pinned master history');\n // Match deployed GitLab contribution statistics: non-merge commits grouped\n // by author_email, including anonymous email. Never filter before ranking.\n const totals = new Map();\n for (const row of commits) {\n if (row.parent_ids.length > 1) continue;\n totals.set(row.author_email, (totals.get(row.author_email) || 0) + 1);\n }\n const contributors = [...totals].map(([email,count]) => ({email,commits:count}));\n const maximum = Math.max(...contributors.map(row => row.commits));\n const leaders = contributors.filter(row => row.commits === maximum);\n if (leaders.length !== 1) throw new Error('no unique maximum contributor');\n if (!leaders[0].email.trim()) throw new Error('winning contributor has no email address');\n return {schema:'gitlab_pinned_branch_full_contributors/1', scope:true, complete:true,\n project:{id:project.id,path_with_namespace:project.path_with_namespace}, branch:branchName, head,\n count_semantics:'all reachable non-merge commits grouped by exact author_email',\n pages, commits, contributors, answer:leaders[0].email};\n } finally {controller.abort(); clearTimeout(timer);}\n})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 784, + "canonical_calls_sha256": "c19a76b510700ab55ec51f4e29b64007bc45e9006a5efc270d754f5841dd1739", + "sources": [ + { + "name": "observation", + "step": 10, + "action_sha256": "c1a96e53c2a73e752516c2f69f43f2ab26ac7cfb614037cf359bfc1ee3b5a062", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "observation", + "pointer": "/scope" + }, + "completeness": { + "source": "observation", + "pointer": "/complete" + } + }, + "projection": { + "array": [ + { + "source": "observation", + "pointer": "/answer" + } + ] + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-94.json b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-94.json new file mode 100644 index 000000000..269969b38 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-94.json @@ -0,0 +1,40 @@ +{ + "task_id": 94, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/invoice/" + }, + { + "action": "evaluate", + "script": "(async()=>{const controller=new AbortController(), timer=setTimeout(()=>controller.abort(),60000);try{\nif(location.pathname.replace(/\\/$/,'')!==\"/admin/sales/invoice\")throw Error('wrong current page');\nconst readyDeadline=Date.now()+30000;\nlet p=null;\nwhile(Date.now(){if(item.component==='Magento_Ui/js/grid/provider'&&item.params?.namespace==='sales_order_invoice_grid'&&item.update_url)providers.push(item)});\n if(providers.length>1)throw Error('ambiguous current grid provider');\n const candidate=providers[0];\n if(candidate){\n if(candidate.lastError===true)throw Error('grid load failed');\n const completed=registry.get(candidate.storageConfig?.name)?.getRequest(candidate.params);\n if(completed?.errorMessage)throw Error('grid request failed: '+completed.errorMessage);\n if(candidate.firstLoad===false&&candidate.lastError===false&&!candidate.triggerDataReload&&completed&&\n Array.isArray(completed.ids)&&/^\\d+$/.test(String(completed.totalRecords))&&\n /^\\d+$/.test(String(candidate.data?.totalRecords))&&Number.isSafeInteger(Number(completed.totalRecords))&&\n Number(completed.totalRecords)===Number(candidate.data?.totalRecords)&&\n completed.ids.length===candidate.data?.items?.length){p=candidate;break;}\n }\n }\n await new Promise(resolve=>setTimeout(resolve,100));\n}\nif(!p)throw Error('grid not completely loaded within 30000ms');\nconst registry=require('uiRegistry');\nconst params={...p.params,sorting:{field:'entity_id',direction:'asc'},search:'',filters:{placeholder:true},paging:{current:1,pageSize:200}},size=params.paging.pageSize;\nif(!Number.isInteger(size)||size<1||size>1000)throw Error('invalid page size');\nconst endpoint=new URL(p.update_url,location.href);\nif(endpoint.origin!==location.origin||endpoint.pathname!=='/admin/mui/index/render/')throw Error('wrong grid endpoint');\nconst flatten=(v,prefix='')=>Object.entries(v).flatMap(([k,x])=>{const key=prefix?`${prefix}[${k}]`:k;\nreturn x!==null&&typeof x==='object'?flatten(x,key):[[key,String(x)]]});\nfor(const [k,v] of flatten(params))endpoint.searchParams.set(k,v);\nlet total=null;const rows=[],pages=[];\nfor(let page=1;page<=100;page++){endpoint.searchParams.set('paging[current]',String(page));\nconst response=await fetch(endpoint.href,{credentials:'same-origin',cache:'no-store',redirect:'error',\nsignal:controller.signal,headers:{Accept:'application/json'}});\nif(!response.ok||response.redirected||!/^application\\/json(?:\\s*;|\\s*$)/i.test(response.headers.get('content-type')||''))throw Error('grid fetch failed');\nconst data=await response.json();\nif(typeof data.totalRecords==='string'&&/^\\d+$/.test(data.totalRecords))data.totalRecords=Number(data.totalRecords);\nif(!Number.isInteger(data.totalRecords)||data.totalRecords<0||!Array.isArray(data.items))throw Error('invalid grid JSON');\nif(total===null)total=data.totalRecords;\nif(total!==data.totalRecords||Math.ceil(total/size)>100||\ndata.items.length!==Math.min(size,Math.max(0,total-(page-1)*size)))throw Error('incomplete or changed page');\nrows.push(...data.items);pages.push({page,rows:data.items.length,total});\nif(page>=Math.max(1,Math.ceil(total/size)))break;\n}\nif(rows.length!==total||new Set(rows.map(r=>String(r.entity_id))).size!==total||\nrows.some(r=>r.entity_id===null||r.entity_id===undefined))throw Error('incomplete or duplicate grid identities');\nif(!/invoice/i.test(params.namespace))throw Error('wrong invoice grid');\nconst matches=rows.filter(r=>String(r.increment_id)==='000000001');\nif(matches.length!==1)throw Error('invoice identity missing or duplicated');\nconst raw=matches[0].grand_total;\nif(typeof raw!=='string'||!/^\\$(?:\\d+|\\d{1,3}(?:,\\d{3})+)\\.\\d{2}$/.test(raw))throw Error('invalid invoice USD total');\nconst cents=Number(raw.replace(/[$,.]/g,''));\nif(!Number.isSafeInteger(cents))throw Error('unsafe invoice cents');\nreturn {scope:true,complete:true,namespace:params.namespace,pages,total,invoice:String(matches[0].increment_id),\nanswer:cents/100};\n}finally{controller.abort();clearTimeout(timer)}})()" + } + ], + "retrieval_contract": { + "schema": "lex_replay_current_run_retrieval/1", + "task_id": 94, + "canonical_calls_sha256": "bf11ed4e5aaabd7c28869f197d2db5144b8d00c8550225e1208d0afc02eaa16d", + "sources": [ + { + "name": "current", + "step": 2, + "action_sha256": "0d80acb7fc540a4d95e48f68ec91ffaa2133293739fcd40e390a4eb021380996", + "decode": "json" + } + ], + "requirements": { + "scope": { + "source": "current", + "pointer": "/scope" + }, + "completeness": { + "source": "current", + "pointer": "/complete" + } + }, + "projection": { + "source": "current", + "pointer": "/answer" + } + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json new file mode 100644 index 000000000..b98132b31 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260918.json @@ -0,0 +1,270 @@ +{ + "schema": "lex_webarena_replay_task_corrections/1", + "scope": "Browser Trajectory Replay standard-trajectory validation only", + "historical_browser_use_scores_immutable": true, + "upstream": { + "benchmark": "WebArena-Verified", + "package_version": "1.2.3", + "dataset_sha256": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30" + }, + "corrections": [ + { + "task_id": 225, + "official_revision": 2, + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + }, + { + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." + }, + { + "op": "replace", + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + }, + { + "op": "replace", + "path": "/instantiation_dict", + "value": { + "brand_text": "Sephora", + "product_text": "Brush" + } + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-complete-bluetooth-catalog-20260918", + "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + } + ], + "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "reference_receipt": { + "path": "research/trajectory-replay-task279-revised-reference.json", + "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + }, + "regression_ids": [ + "tr-279-complete-thirteen-products", + "tr-279-reject-omitted-product", + "tr-279-reject-wired-product" + ] + }, + { + "task_id": 448, + "official_revision": 2, + "correction_id": "wv-448-profile-save-success-variants-20260918", + "rationale": "GitLab can persist this profile form through either a 302 redirect or a 200 JSON success response. Match the submitted website URL before event recency and require an explicit success response for either form.", + "root_cause": "A single expected HTTP 302 falsely rejects a successful 200 JSON profile update when the redirect request is canceled; an unqualified 200 would also admit validation errors.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__GITLAB__/-/profile" + }, + { + "path": "/eval/1/expected/post_data/user[website_url]", + "equals": "https://egg.tart.com" + }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, + { + "path": "/eval/1/last_event_only", + "equals": true + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/last_event_only", + "value": false + } + ], + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] + } + ] + }, + { + "task_id": 510, + "official_revision": 2, + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "rationale": "The prose combines rating and price without an ordering rule, and the cart-totals response body is not available through every browser evidence interface. Replay freezes the intended product and verifies the one-item cart in the rendered page while retaining add-to-cart, payment and success network checks.", + "preconditions": [ + { + "path": "/intent", + "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + }, + { + "path": "/eval/1/expected/post_data/product", + "equals": "75640" + }, + { + "path": "/eval/2/expected/response_content/items_qty", + "equals": 1 + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." + }, + { + "op": "add", + "path": "/instantiation_dict/selection_policy", + "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" + }, + { + "op": "add", + "path": "/instantiation_dict/target_product_id", + "value": "75640" + }, + { + "op": "remove", + "path": "/eval/2" + } + ] + }, + { + "task_id": 528, + "official_revision": 2, + "correction_id": "wv-528-dom-state-without-submit-20260911", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 547, + "official_revision": 2, + "correction_id": "wv-547-existing-brown-option-20260912", + "rationale": "Brown already exists as global color option 51 in the seeded fixture. The parent product save request already proves creation of the S-Brown matrix row, so replay removes only the redundant attribute-save requirement.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93" + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ] + }, + { + "task_id": 699, + "official_revision": 3, + "correction_id": "wv-699-all-customer-groups-20260918", + "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", + "preconditions": [ + { + "path": "/eval/1/expected/post_data/customer_group_ids[0]", + "equals": "1" + }, + { + "path": "/eval/1/post_data_schema", + "equals": null + } + ], + "operations": [ + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[1]", + "value": "2" + }, + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[2]", + "value": "3" + } + ] + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-pre-510.json b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-pre-510.json new file mode 100644 index 000000000..78c09cc25 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/revisions/task-corrections-20260920-pre-510.json @@ -0,0 +1,272 @@ +{ + "schema": "lex_webarena_replay_task_corrections/1", + "scope": "Browser Trajectory Replay standard-trajectory validation only", + "historical_browser_use_scores_immutable": true, + "upstream": { + "benchmark": "WebArena-Verified", + "package_version": "1.2.3", + "dataset_sha256": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30" + }, + "corrections": [ + { + "task_id": 225, + "official_revision": 2, + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" + ], + "preconditions": [ + { + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" + }, + { + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." + }, + { + "op": "replace", + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." + }, + { + "op": "replace", + "path": "/instantiation_dict", + "value": { + "brand_text": "Sephora", + "product_text": "Brush" + } + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-complete-bluetooth-catalog-20260918", + "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "preconditions": [ + { + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" + } + ], + "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "reference_receipt": { + "path": "research/trajectory-replay-task279-revised-reference.json", + "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + }, + "regression_ids": [ + "tr-279-complete-thirteen-products", + "tr-279-reject-omitted-product", + "tr-279-reject-wired-product" + ] + }, + { + "task_id": 448, + "official_revision": 2, + "correction_id": "wv-448-final-profile-save-success-variants-20260920", + "rationale": "Evaluate the last POST to the profile endpoint, then require the intended website and either a 302 redirect or an explicit 200 JSON success response. An earlier correct save cannot excuse a later incorrect save.", + "root_cause": "Disabling last_event_only made any earlier matching save sufficient, even when a later successful save replaced the website with a wrong value. Select the final endpoint/method event before checking its value; preserve the two valid response representations.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request", + "tr-448-reject-correct-then-wrong", + "tr-448-accept-wrong-then-correct" + ], + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__GITLAB__/-/profile" + }, + { + "path": "/eval/1/expected/post_data/user[website_url]", + "equals": "https://egg.tart.com" + }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, + { + "path": "/eval/1/last_event_only", + "equals": true + } + ], + "operations": [ + { + "op": "replace", + "path": "/eval/1/last_event_only", + "value": true + } + ], + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] + } + ] + }, + { + "task_id": 510, + "official_revision": 2, + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "rationale": "The prose combines rating and price without an ordering rule, and the cart-totals response body is not available through every browser evidence interface. Replay freezes the intended product and verifies the one-item cart in the rendered page while retaining add-to-cart, payment and success network checks.", + "preconditions": [ + { + "path": "/intent", + "equals": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant." + }, + { + "path": "/eval/1/expected/post_data/product", + "equals": "75640" + }, + { + "path": "/eval/2/expected/response_content/items_qty", + "equals": 1 + } + ], + "operations": [ + { + "op": "replace", + "path": "/intent", + "value": "Buy the least expensive product in the Home Audio Speaker category among products with at least 5 reviews. If prices tie, choose the product with the higher rating. Buy Atlantic Technology FS-7.0-GLB 7-channel Surround Bar (Gloss Black) (Discontinued by Manufacturer), product 75640, and choose any available variant." + }, + { + "op": "add", + "path": "/instantiation_dict/selection_policy", + "value": "minimum price among products with at least 5 reviews; rating descending only as a tie-break" + }, + { + "op": "add", + "path": "/instantiation_dict/target_product_id", + "value": "75640" + }, + { + "op": "remove", + "path": "/eval/2" + } + ] + }, + { + "task_id": 528, + "official_revision": 2, + "correction_id": "wv-528-dom-state-without-submit-20260911", + "rationale": "The task requires a filled but unsubmitted form, so a synthetic POST cannot be positive evidence; final DOM values plus absence of the real submit request express the contract.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "^http://.*/dummy_bin$" + }, + { + "path": "/eval/2/should_not_exist", + "equals": true + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ], + "replay_assertions": [ + { + "kind": "final_dom_form_state", + "page_url": "__SHOPPING__/contact", + "form": "#contact-form", + "fields": { + "input[name='name']": "Emma Lopez", + "input[name='email']": "emma.lopez@gmail.com", + "textarea[name='comment']": "I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use." + }, + "forbid_network_event": { + "url": "__SHOPPING__/contact/index/post", + "http_method": "POST" + } + } + ] + }, + { + "task_id": 547, + "official_revision": 2, + "correction_id": "wv-547-existing-brown-option-20260912", + "rationale": "Brown already exists as global color option 51 in the seeded fixture. The parent product save request already proves creation of the S-Brown matrix row, so replay removes only the redundant attribute-save requirement.", + "preconditions": [ + { + "path": "/eval/1/expected/url", + "equals": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93" + } + ], + "operations": [ + { + "op": "remove", + "path": "/eval/1" + } + ] + }, + { + "task_id": 699, + "official_revision": 3, + "correction_id": "wv-699-all-customer-groups-20260918", + "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", + "preconditions": [ + { + "path": "/eval/1/expected/post_data/customer_group_ids[0]", + "equals": "1" + }, + { + "path": "/eval/1/post_data_schema", + "equals": null + } + ], + "operations": [ + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[1]", + "value": "2" + }, + { + "op": "add", + "path": "/eval/1/expected/post_data/customer_group_ids[2]", + "value": "3" + } + ] + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/selection.json b/evaluation/trajectory-replay/webarena-verified/selection.json new file mode 100644 index 000000000..46766a7c3 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/selection.json @@ -0,0 +1,661 @@ +{ + "schema": "lex_webarena_verified_pilot/1", + "metadata": { + "benchmark": "WebArena-Verified", + "benchmark_version": "1.2.3", + "difficulty_definition": "estimated minimum visible UI operations after authentication: easy 1-4, medium 5-7, hard 8+", + "selection_policy": "The pre-registered 90-task pilot selection, with task 243 using the independently verifiable nickname contract from the pinned WebArena-Verified fork; identical task IDs for all three methods.", + "tier_counts": { + "easy": 15, + "medium": 30, + "hard": 45 + }, + "site_counts": { + "shopping": 23, + "shopping_admin": 23, + "reddit": 22, + "gitlab": 22 + }, + "task_type_counts": { + "RETRIEVE": 37, + "NAVIGATE": 13, + "MUTATE": 40 + }, + "excluded_sites": { + "wikipedia": "requires a large external ZIM data mount and has not completed the local execution qualification gate", + "map": "requires multiple external map datasets and persistent volumes and has not completed the local execution qualification gate" + } + }, + "tasks": [ + { + "task_id": 44, + "tier": "easy", + "estimated_min_steps": 2, + "official_hard": true, + "rationale": "打开一个需要登录的 GitLab 目标页" + }, + { + "task_id": 41, + "tier": "easy", + "estimated_min_steps": 3, + "official_hard": false, + "rationale": "打开一份报表并读取第一个条目" + }, + { + "task_id": 77, + "tier": "easy", + "estimated_min_steps": 3, + "official_hard": false, + "rationale": "打开评论列表并读取一种状态的数量" + }, + { + "task_id": 94, + "tier": "easy", + "estimated_min_steps": 4, + "official_hard": false, + "rationale": "搜索一张发票并读取总金额" + }, + { + "task_id": 67, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "检查十个热门帖子并精确提取单书推荐标题" + }, + { + "task_id": 404, + "tier": "easy", + "estimated_min_steps": 4, + "official_hard": false, + "rationale": "进入一个论坛、选择最新帖子并点赞" + }, + { + "task_id": 313, + "tier": "easy", + "estimated_min_steps": 2, + "official_hard": false, + "rationale": "从商城页面直接读取客服号码" + }, + { + "task_id": 274, + "tier": "easy", + "estimated_min_steps": 3, + "official_hard": false, + "rationale": "提交一个商品关键词并停留在搜索结果页" + }, + { + "task_id": 260, + "tier": "easy", + "estimated_min_steps": 2, + "official_hard": false, + "rationale": "从商城导航进入指定品类页" + }, + { + "task_id": 386, + "tier": "easy", + "estimated_min_steps": 4, + "official_hard": false, + "rationale": "搜索一个明确商品并读取评分" + }, + { + "task_id": 374, + "tier": "easy", + "estimated_min_steps": 3, + "official_hard": false, + "rationale": "在后台导航到指定主题设置页" + }, + { + "task_id": 157, + "tier": "easy", + "estimated_min_steps": 3, + "official_hard": true, + "rationale": "在后台打开全部客户详情视图" + }, + { + "task_id": 595, + "tier": "easy", + "estimated_min_steps": 4, + "official_hard": false, + "rationale": "从指定论坛热门帖子页完成订阅" + }, + { + "task_id": 645, + "tier": "easy", + "estimated_min_steps": 4, + "official_hard": true, + "rationale": "在已知论坛发布只有标题的新帖子" + }, + { + "task_id": 156, + "tier": "easy", + "estimated_min_steps": 2, + "official_hard": true, + "rationale": "打开分配给当前用户的合并请求页" + }, + { + "task_id": 258, + "tier": "easy", + "estimated_min_steps": 2, + "official_hard": false, + "rationale": "打开 GitLab 公共项目列表" + }, + { + "task_id": 45, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "进入项目议题列表并应用排序和状态筛选" + }, + { + "task_id": 118, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "把需求转成商品查询并打开匹配详情页" + }, + { + "task_id": 117, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "进入订单历史，经翻页或排序后读取最早日期" + }, + { + "task_id": 132, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "找到项目贡献者活动页并限定一个日期" + }, + { + "task_id": 168, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "打开个人项目并检查星标与项目标识" + }, + { + "task_id": 173, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "按标题和更新时间筛选个人议题，再读取状态" + }, + { + "task_id": 183, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "按库存量筛选后台表格并收集 SKU" + }, + { + "task_id": 188, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "按状态筛选订单，打开最新一单并读取总额" + }, + { + "task_id": 198, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "筛选并排序订单，再打开最新的匹配客户" + }, + { + "task_id": 399, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": true, + "rationale": "打开个人资料设置，修改简介并保存" + }, + { + "task_id": 225, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "定位商品评论并提取低评分评论标题" + }, + { + "task_id": 231, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "筛选订单状态并读取最近订单号" + }, + { + "task_id": 298, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "筛选最近已完成订单并打开详情页" + }, + { + "task_id": 465, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "搜索明确商品并加入愿望清单" + }, + { + "task_id": 279, + "tier": "medium", + "estimated_min_steps": 7, + "official_hard": false, + "rationale": "筛选品牌商品并汇总名称与价格区间" + }, + { + "task_id": 284, + "tier": "medium", + "estimated_min_steps": 7, + "official_hard": true, + "rationale": "按容量和价格比较商品并打开目标详情页" + }, + { + "task_id": 288, + "tier": "medium", + "estimated_min_steps": 7, + "official_hard": false, + "rationale": "从取消订单历史中定位客户并读取邮箱" + }, + { + "task_id": 243, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "定位商品唯一最低分评论并读取评论昵称" + }, + { + "task_id": 676, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "按欺诈状态筛选后台订单列表" + }, + { + "task_id": 470, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "搜索指定订单并执行取消" + }, + { + "task_id": 605, + "tier": "medium", + "estimated_min_steps": 7, + "official_hard": false, + "rationale": "选择相关论坛并发布一个问题" + }, + { + "task_id": 620, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "在关系论坛填写标题和正文并发帖" + }, + { + "task_id": 625, + "tier": "medium", + "estimated_min_steps": 7, + "official_hard": true, + "rationale": "为讨论主题选择相关论坛并发帖" + }, + { + "task_id": 630, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "在已知论坛按给定标题和正文发帖" + }, + { + "task_id": 714, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "进入论坛历史榜并踩排名第一的帖子" + }, + { + "task_id": 475, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "创建一个命名明确的空仓库" + }, + { + "task_id": 339, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "在当前项目打开并筛选带缺陷标签的议题" + }, + { + "task_id": 418, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": true, + "rationale": "打开个人设置并更新 GitLab 状态" + }, + { + "task_id": 389, + "tier": "medium", + "estimated_min_steps": 6, + "official_hard": false, + "rationale": "定位指定合并请求并发表评论" + }, + { + "task_id": 448, + "tier": "medium", + "estimated_min_steps": 5, + "official_hard": false, + "rationale": "打开个人资料并修改主页网址" + }, + { + "task_id": 28, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "找到最新论坛帖子，并按作者和票数条件检查多条评论" + }, + { + "task_id": 50, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "检查一整年的分页订单并汇总数量和支出" + }, + { + "task_id": 66, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "检查十个排序帖子，并根据描述归类推荐内容" + }, + { + "task_id": 108, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "应用日期和状态条件，并汇总五个月度数量" + }, + { + "task_id": 110, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "应用日期和状态条件，并汇总十一个月度数量" + }, + { + "task_id": 124, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "搜索商品、检查结果页并计算价格区间" + }, + { + "task_id": 127, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "结合搜索词排名与独立的商品可用性检查" + }, + { + "task_id": 184, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "筛选库存、检查变体并返回成对商品字段" + }, + { + "task_id": 193, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": true, + "rationale": "排序已完成订单、检查两份详情并合计付款" + }, + { + "task_id": 196, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "检查两组各四笔订单，并计算付款差额" + }, + { + "task_id": 204, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "找到最新完成订单，并规范化、排序多个行项目" + }, + { + "task_id": 506, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": false, + "rationale": "清空购物车，按预算和评分筛选品类，选择商品并结账" + }, + { + "task_id": 441, + "tier": "hard", + "estimated_min_steps": 11, + "official_hard": true, + "rationale": "定位项目文件、修改标题、提交到默认分支并验证" + }, + { + "task_id": 163, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": true, + "rationale": "遍历商品评论并收集全部低评分标题" + }, + { + "task_id": 320, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "筛选指定月份的取消订单并汇总含运费退款" + }, + { + "task_id": 334, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": false, + "rationale": "在历史订单中跨条目定位描述匹配商品的最近日期" + }, + { + "task_id": 528, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "从订单历史提取订单号和金额，再填写但不提交退款表单" + }, + { + "task_id": 571, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": true, + "rationale": "进入账户地址簿并完整更新邮寄地址" + }, + { + "task_id": 585, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "从购买历史定位商品并提交指定评分和评论" + }, + { + "task_id": 510, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": false, + "rationale": "按品类、评论数、评分和价格比较商品并完成结账" + }, + { + "task_id": 464, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": false, + "rationale": "统计商品高评分评论后更新商品描述" + }, + { + "task_id": 547, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": false, + "rationale": "编辑可配置商品并为指定尺码增加颜色选项" + }, + { + "task_id": 694, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "按属性集、库存、尺码、颜色和价格创建商品" + }, + { + "task_id": 699, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": false, + "rationale": "创建面向注册客户的全站折扣价格规则" + }, + { + "task_id": 771, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "筛选全部高评分待审评论并逐项批准" + }, + { + "task_id": 580, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "创建论坛并配置名称、标题、描述和侧栏" + }, + { + "task_id": 610, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "发布书评后继续定位新帖子并追加评论" + }, + { + "task_id": 615, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "从来源帖子提取图片地址并转载到另一个论坛" + }, + { + "task_id": 640, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": true, + "rationale": "在指定论坛创建包含日期信息的活动通知" + }, + { + "task_id": 719, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": false, + "rationale": "筛选指定用户在论坛的全部投稿并逐项点赞" + }, + { + "task_id": 725, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": false, + "rationale": "筛选指定用户在论坛的全部投稿并逐项点踩" + }, + { + "task_id": 731, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": false, + "rationale": "定位自己的指定帖子并在正文末尾追加一行" + }, + { + "task_id": 29, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "定位最新帖子并按作者和票数条件统计评论" + }, + { + "task_id": 635, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": false, + "rationale": "判断最合适论坛并发布包含预算的推荐请求" + }, + { + "task_id": 600, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": false, + "rationale": "自主选择最可能获得回答的论坛并发布问题" + }, + { + "task_id": 349, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": true, + "rationale": "进入仓库成员设置并列出其他可访问用户" + }, + { + "task_id": 784, + "tier": "hard", + "estimated_min_steps": 8, + "official_hard": false, + "rationale": "定位仓库分支贡献者并读取提交最多者邮箱" + }, + { + "task_id": 303, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "按年度范围检查提交历史并汇总指定用户次数" + }, + { + "task_id": 314, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": false, + "rationale": "对仓库贡献者排序并提取前三名完整姓名" + }, + { + "task_id": 658, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": true, + "rationale": "创建议题并配置受理人和远期截止日期" + }, + { + "task_id": 666, + "tier": "hard", + "estimated_min_steps": 12, + "official_hard": false, + "rationale": "选择源分支和目标分支创建合并请求并指定审核人" + }, + { + "task_id": 590, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "在项目中创建并配置起止日期里程碑" + }, + { + "task_id": 576, + "tier": "hard", + "estimated_min_steps": 10, + "official_hard": true, + "rationale": "定位仓库并以指定角色添加两名成员" + }, + { + "task_id": 415, + "tier": "hard", + "estimated_min_steps": 9, + "official_hard": true, + "rationale": "定位合并请求、检查最后评论作者并条件化回复" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/task-corrections.json b/evaluation/trajectory-replay/webarena-verified/task-corrections.json index 54ff761b9..42c659824 100644 --- a/evaluation/trajectory-replay/webarena-verified/task-corrections.json +++ b/evaluation/trajectory-replay/webarena-verified/task-corrections.json @@ -9,77 +9,95 @@ }, "corrections": [ { - "task_id": 279, + "task_id": 225, "official_revision": 2, - "correction_id": "wv-279-reference-title-drift-20260911", - "rationale": "The current seeded catalog title contains two closing parentheses; the frozen reference contains one.", - "preconditions": [ - { - "path": "/eval/0/expected/retrieved_data/0/names/9", - "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" - } + "correction_id": "wv-225-explicit-title-scope-20260918", + "rationale": "Magento has no populated manufacturer attribute for this catalog. The source-complete storefront query identifies the relevant two products by titles containing Sephora and Brush, and both have no reviews.", + "root_cause": "The upstream manufacturer wording has no structured source field, while the historical trajectory inspected only two products without proving membership. A broad text search also has unstable pagination. The corrected intent states the observable title rule and the reference is rederived from complete GraphQL and database evidence.", + "reference_receipt": { + "path": "research/trajectory-replay-task225-revised-reference.json", + "sha256": "0e6f2a1a0e9d85a6f8c6f9a03832253d97be4934159be502b8786e8ae96a567a" + }, + "regression_ids": [ + "tr-225-complete-two-products-no-low-reviews", + "tr-225-reject-incomplete-product-scope", + "tr-225-reject-new-low-review" ], - "operations": [ - { - "op": "replace", - "path": "/eval/0/expected/retrieved_data/0/names/9", - "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" - } - ] - }, - { - "task_id": 339, - "official_revision": 2, - "correction_id": "wv-339-issues-route-slash-equivalence-20260911", - "rationale": "GitLab's canonical issues collection has no trailing slash, while the upstream regex requires one; navigate-task GET selection also checks the last document before matching the expected route.", "preconditions": [ { - "path": "/eval/1/expected/url", - "equals": "^__GITLAB__/a11yproject/a11yproject.com/-/issues/.*$" + "path": "/intent", + "equals": "Return the titles for reviews with 3 stars or below for brush from sephora" }, { - "path": "/eval/2/expected/headers/referer", - "equals": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?state=opened&label_name%5B%5D=bug" + "path": "/eval/0/expected/status", + "equals": "NOT_FOUND_ERROR" } ], "operations": [ { "op": "replace", - "path": "/eval/1/expected/url", - "value": "__GITLAB__/a11yproject/a11yproject.com/-/issues" + "path": "/intent", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found." }, { "op": "replace", - "path": "/eval/2/expected/headers", - "value": null + "path": "/intent_template", + "value": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found." }, { "op": "replace", - "path": "/eval/2/expected/post_data", + "path": "/instantiation_dict", "value": { - "$.variables.state": "opened", - "$.variables.labelName": "bug" + "brand_text": "Sephora", + "product_text": "Brush" } - }, + } + ] + }, + { + "task_id": 279, + "official_revision": 2, + "correction_id": "wv-279-complete-bluetooth-catalog-20260918", + "rationale": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "preconditions": [ { - "op": "replace", - "path": "/eval/2/last_event_only", - "value": false + "path": "/eval/0/expected/retrieved_data/0/names/9", + "equals": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)" } ], - "replay_assertions": [ + "operations": [ { - "kind": "network_event_selection", - "policy": "match_expected_url_method_and_query_before_recency", - "target": "canonical issues document and filtered issues GraphQL request" + "op": "replace", + "path": "/eval/0/expected/retrieved_data/0/names/9", + "value": "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))" } + ], + "root_cause": "The reference omitted the seeded Sony WI-1000XM2 because its storefront description lacks the word Bluetooth, although the exact model has Bluetooth capability. This is separate from the current observer duplicate-product exception.", + "reference_receipt": { + "path": "research/trajectory-replay-task279-revised-reference.json", + "sha256": "c6064850ebb16044bd3ba945333686d378575822c4563e08564343e52fe54b85" + }, + "regression_ids": [ + "tr-279-complete-thirteen-products", + "tr-279-reject-omitted-product", + "tr-279-reject-wired-product" ] }, { "task_id": 448, "official_revision": 2, - "correction_id": "wv-448-match-profile-save-before-recency-20260915", - "rationale": "GitLab emits a successful profile-save POST followed by another profile event. The acceptance contract must match the POST carrying user[website_url] before applying event recency.", + "correction_id": "wv-448-final-profile-save-success-variants-20260920", + "rationale": "Evaluate the last POST to the profile endpoint, then require the intended website and either a 302 redirect or an explicit 200 JSON success response. An earlier correct save cannot excuse a later incorrect save.", + "root_cause": "Disabling last_event_only made any earlier matching save sufficient, even when a later successful save replaced the website with a wrong value. Select the final endpoint/method event before checking its value; preserve the two valid response representations.", + "regression_ids": [ + "tr-448-302-success", + "tr-448-200-json-success", + "tr-448-200-json-failure", + "tr-448-wrong-website", + "tr-448-canceled-request", + "tr-448-reject-correct-then-wrong", + "tr-448-accept-wrong-then-correct" + ], "preconditions": [ { "path": "/eval/1/expected/url", @@ -89,6 +107,10 @@ "path": "/eval/1/expected/post_data/user[website_url]", "equals": "https://egg.tart.com" }, + { + "path": "/eval/1/expected/response_status", + "equals": 302 + }, { "path": "/eval/1/last_event_only", "equals": true @@ -98,40 +120,36 @@ { "op": "replace", "path": "/eval/1/last_event_only", - "value": false - } - ] - }, - { - "task_id": 475, - "official_revision": 2, - "correction_id": "wv-475-unchecked-checkbox-is-false-20260911", - "rationale": "An unchecked HTML checkbox is omitted from a form submission; omission is the browser-standard representation of false.", - "preconditions": [ - { - "path": "/eval/1/expected/post_data/project[initialize_with_readme]", - "equals": "0" + "value": true } ], - "operations": [ - { - "op": "remove", - "path": "/eval/1/expected/post_data/project[initialize_with_readme]" - } - ], - "replay_assertions": [ - { - "kind": "gitlab_project_state", - "project_path": "byteblaze/chatgpt_plugin", - "empty_repository": true + "network_event_alternatives": [ + { + "evaluator_index": 1, + "operations": [ + { + "op": "replace", + "path": "/eval/1/expected/response_status", + "value": 200 + }, + { + "op": "replace", + "path": "/eval/1/expected/response_content", + "value": { + "message": "Profile was successfully updated" + } + } + ] } ] }, { "task_id": 510, "official_revision": 2, - "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", - "rationale": "The prose combines rating and price without an ordering rule, and the cart-totals response body is not available through every browser evidence interface. Replay freezes the intended product and verifies the one-item cart in the rendered page while retaining add-to-cart, payment and success network checks.", + "correction_id": "wv-510-restore-cart-quantity-contract-20260920", + "rationale": "Retain the declared product-selection revision but restore the upstream cart-totals check for quantity and name. A single DOM product row does not prove a single unit; all three frozen task-510 HARs contain the totals response required for zero-call rescoring.", + "root_cause": "Removing the totals evaluator dropped items_qty=1. The replacement trajectory checked only the number of product rows, so one row containing two units was accepted.", + "regression_ids": ["tr-510-quantity-one", "tr-510-reject-quantity-two", "tr-510-reject-missing-totals", "tr-510-reject-wrong-product"], "preconditions": [ { "path": "/intent", @@ -161,10 +179,6 @@ "op": "add", "path": "/instantiation_dict/target_product_id", "value": "75640" - }, - { - "op": "remove", - "path": "/eval/2" } ] }, @@ -224,56 +238,15 @@ } ] }, - { - "task_id": 676, - "official_revision": 2, - "correction_id": "wv-676-match-filter-xhr-20260911", - "rationale": "The acceptance event is the MUI grid XHR carrying filters[status]=fraud, not the surrounding sales/order document navigation.", - "preconditions": [ - { - "path": "/eval/1/expected/url", - "equals": "^__SHOPPING_ADMIN__/mui/index/render/.*$" - }, - { - "path": "/eval/1/last_event_only", - "equals": true - }, - { - "path": "/eval/1/expected/headers/referer", - "equals": "__SHOPPING_ADMIN__/sales/order/" - } - ], - "operations": [ - { - "op": "replace", - "path": "/eval/1/last_event_only", - "value": false - }, - { - "op": "replace", - "path": "/eval/1/expected/headers", - "value": null - } - ], - "replay_assertions": [ - { - "kind": "network_event_selection", - "policy": "match_expected_url_method_and_query_before_recency", - "target": "mui/index/render XHR with filters[status]=fraud" - } - ] - }, { "task_id": 699, - "official_revision": 2, - "correction_id": "wv-699-browser-form-array-fields-20260912", - "rationale": "Magento submits multi-select values as indexed HTML form fields. The acceptance contract must check website_ids[0] and customer_group_ids[0..2] directly; the upstream unindexed keys cannot be produced by this form.", + "official_revision": 3, + "correction_id": "wv-699-all-customer-groups-20260918", + "rationale": "The visible Magento price rule selects all registered customer groups. The current upstream form contract verifies only the first indexed group, so require the other two indexed fields as well.", "preconditions": [ { - "path": "/eval/1/expected/post_data/customer_group_ids", - "equals": [ - 1 - ] + "path": "/eval/1/expected/post_data/customer_group_ids[0]", + "equals": "1" }, { "path": "/eval/1/post_data_schema", @@ -281,24 +254,6 @@ } ], "operations": [ - { - "op": "remove", - "path": "/eval/1/expected/post_data/website_ids" - }, - { - "op": "remove", - "path": "/eval/1/expected/post_data/customer_group_ids" - }, - { - "op": "add", - "path": "/eval/1/expected/post_data/website_ids[0]", - "value": "1" - }, - { - "op": "add", - "path": "/eval/1/expected/post_data/customer_group_ids[0]", - "value": "1" - }, { "op": "add", "path": "/eval/1/expected/post_data/customer_group_ids[1]", diff --git a/evaluation/trajectory-replay/webarena-verified/trajectories-corrected.json b/evaluation/trajectory-replay/webarena-verified/trajectories-corrected.json new file mode 100644 index 000000000..f0ab1d344 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectories-corrected.json @@ -0,0 +1,7762 @@ +{ + "schema": "lex_webarena_verified_trajectory_dataset/1", + "benchmark": "WebArena-Verified", + "benchmark_version": "1.2.3", + "official_data_checksum": "d65275660814663375028e9017e1f929e3c38321041b125795e2713b52243d30", + "selection_sha256": "06d8d999164c7c2de6cb21d15ec12f5260bfc2a58ca81b098a8c7779d4d2d91a", + "semantics": { + "trajectory_is_one_verified_feasible_path": true, + "trajectory_is_the_only_correct_path": false, + "browser_use_scoring_authority": "official task outcome contract", + "browser_use_agent_visibility": "hidden", + "trajectory_replay_role": "fixed BrowserTool action input", + "evaluation_input_role": "official evaluator input only; never an Agent prompt" + }, + "promotion_policy": { + "qualification_browser": "chromium", + "measured_browsers_are_not_promotion_gates": true, + "process_status": "pass", + "evaluator_status": "pass", + "model_calls": 0 + }, + "provenance": { + "seed_manifest_sha256": "bb5e95bba930d8ec6e45cc20531b4f3aff860e6970c982b99171c6a23a022567", + "profile_sha256": "3ca5d91cfec570490df202c9cb332957f7e850b70fc5af80b684709e97ef0893", + "replay_receipts": [ + { + "sha256": "bca4dc72e547fbe9b100ffbf24d90d5008507104bc2b2970da30b7d28c5a9e1a", + "source_seed_manifest_sha256": "fc1cc571c48feac7514845a2dc81f73877fbaac4602812e7aad8e9fe43655234", + "profile_sha256": "5322bd5f100d630686639893de08472be717e955e8cfa75df6a9cd146c141b04" + }, + { + "sha256": "d2b2817940fe3c0a849278d707fcd32ec9992f5fc126c258ed7cf637006a1dee", + "source_seed_manifest_sha256": "d5b3edffceec850892225d5eb6e96ddc82f0a46af742eb917037e001e3da63b6", + "profile_sha256": "f5ef0dd6f2d3f1775093149d53f425b562c0bb1d13a579aab19a4a271112d0d9" + }, + { + "sha256": "0671d3a390a97cd8d8fcf58642c8e86b9dbc8fb41f4a5e8bee745a1be9ac46d1", + "source_seed_manifest_sha256": "76ea9bce6eccd877cc2939eaeeed2a8489b4078f3b02f892e3d741b15e13b2ac", + "profile_sha256": "50488ded31860e13377fcbc424d5cf361968e7557cda6c65760801c2679ada92" + }, + { + "sha256": "f758a9bdcb1cb81347046badecfb1a41f6794c45ef9753938d1783a6dc36ea6b", + "source_seed_manifest_sha256": "231d1b5cbadc5fc83431bc0a5484a69c15b9f12ccbd58cf99d1d49d5d4a9f290", + "profile_sha256": "4b9cca016055bf626d77aab1996efec4cc29e9ba3bb7379b3da54593b7c43be3" + }, + { + "sha256": "37779197d4051833d83f7489d9f513448db3ce0f8fd09f1c2ee33975f49713c6", + "source_seed_manifest_sha256": "56499afea9aeb96641e815da824d3cc60b8c2313cbbc891c5886cac5f9fe28a4", + "profile_sha256": "2b26d3df29f8662576b9b52715df91abc8cd4a55718c84af80d3e7215e96bce7" + }, + { + "sha256": "f3910e7715137714db6dbeba54fde9f012dc01680fc85d0af8737443f06d5494", + "source_seed_manifest_sha256": "c84876cedb21d7c7e32eee5cb77689ce8d1cbc26806d2353fa64b3fbb9f54a98", + "profile_sha256": "d252c299914d61fef61c3c248b710c533d2c20f05c104081a478bfa5d6815570" + }, + { + "sha256": "f59d6568105587acb21768d7fb28d8b592c81f9c5f0b07046c70ed8a27ad677b", + "source_seed_manifest_sha256": "bb5e95bba930d8ec6e45cc20531b4f3aff860e6970c982b99171c6a23a022567", + "profile_sha256": "3ca5d91cfec570490df202c9cb332957f7e850b70fc5af80b684709e97ef0893" + } + ] + }, + "counts": { + "verified_tasks": 90, + "verified_trajectories": 90 + }, + "trajectories": [ + { + "id": "webarena-verified-44-trajectory-1", + "task_id": 44, + "task": { + "intent": "Open my todos page", + "intent_template_id": 303, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/dashboard/todos\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-44", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "eab05ae2e7ca04df24aed3d437412a9e2dec06f83bdc7a6fff679a8e4ac5203d", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "4971d2ec4b2a99e4f3a978921400bf2b93c31f2bc698722192388d16a43cec07" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-41-trajectory-1", + "task_id": 41, + "task": { + "intent": "Get the top 1 search term(s) in my store", + "intent_template_id": 285, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "hollister" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-41", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "4d6f174990189a64add34b969c43ff7e87f671756642b54d30b2d3aeedb24e42", + "response_sha256": "419e1a97f7e0959373082754252f6c26971c91f50997211c9f3f4da7364baa47", + "evaluation_sha256": "49d1071240a2a66ada503cac2c7d9e5cdc209339f9761602bfefbc84d0a4802d" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-77-trajectory-1", + "task_id": 77, + "task": { + "intent": "Get the total number of Pending reviews amongst all the reviews", + "intent_template_id": 277, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/review/product/pending/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 5 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-77", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "2b80fe4ffa5ff80dea38b2a5e13182f0d32da73ff2e5aaa91327a954136d9352", + "response_sha256": "f0b58420f5943b2cd6c7b91f61e3943ca00e024dfcaac6d85f6ae66ed88dba18", + "evaluation_sha256": "d053f429af7a1f29fab7ff5fb7d8de8598610c121e21f2167cf355cb02e70846" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-94-trajectory-1", + "task_id": 94, + "task": { + "intent": "Get the grand total of invoice 000000001. Return the value as a number (e.g., 10.99) only, without any additional details.", + "intent_template_id": 274, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/invoice/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 3000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 36.39 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-94", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "2817e109f2b63ef9807859906afd738e2c9df482387f52746feea8647c548c85", + "response_sha256": "da576cd73e6c506859f3ce8dbd49f80f7622995e7f62ddde42af26af44bd7a0f", + "evaluation_sha256": "e92146f255ebfd6077e756a97df90d8cd51b07459729f5efc5d9d031043116c4" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-67-trajectory-1", + "task_id": 67, + "task": { + "intent": "Among the top 10 hottest posts in the \"Books\" forum, get the book titles, exactly as they appear in the post description, from those that recommend exactly one book.", + "intent_template_id": 17, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ], + "intent_template": "Among the top {{number}} hottest posts in the \"{{forum}}\" forum, get {{description}} from those that {{filter_criterion}}.{{retrieved_data_format_spec}}", + "instantiation_dict": { + "number": 10, + "description": "the book titles, exactly as they appear in the post description,", + "forum": "Books", + "filter_criterion": "recommend exactly one book", + "retrieved_data_format_spec": "" + } + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/f/Books" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async()=>{const links=[...new Set([...document.querySelectorAll('main article a[href*=\"/f/books/\"]')].map(a=>a.getAttribute('href')).filter(Boolean))].slice(0,10);if(links.length!==10)throw new Error('expected ten hot-post links, found '+links.length);return Promise.all(links.map(async href=>{const response=await fetch(href);if(!response.ok)throw new Error('post fetch failed '+response.status+' '+href);const page=new DOMParser().parseFromString(await response.text(),'text/html');return {href,text:page.querySelector('main article')?.textContent?.replace(/\\s+/g,' ').trim()||null};}));})()" + }, + { + "action": "evaluate", + "script": "(async () => {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), 20000);\n try {\n const listing = new URL(location.href);\n if (listing.origin !== location.origin || listing.pathname.toLowerCase().replace(/\\/$/, '') !== '/f/books' || listing.search) throw new Error('wrong Books listing');\n const hot = [...document.querySelectorAll('button[aria-label]')].some(node => /^sort by:\\s*hot$/i.test(node.getAttribute('aria-label') || ''));\n if (!hot) throw new Error('Books listing is not sorted by Hot');\n const articles = [...document.querySelectorAll('main > article')];\n if (articles.length < 10) throw new Error('fewer than ten hot posts');\n const normalize = text => String(text ?? '').replace(/\\s+/g, ' ').trim();\n const entries = articles.slice(0, 10).map((article, index) => {\n const links = [...article.querySelectorAll('a[href]')].map(a => a.getAttribute('href')).filter(Boolean);\n const matches = links.map(href => ({href, match: /^\\/f\\/books\\/(\\d+)(?:\\/|$)/i.exec(href)})).filter(row => row.match);\n const ids = [...new Set(matches.map(row => row.match[1]))];\n const title = normalize(article.querySelector('header h1')?.textContent);\n if (ids.length !== 1 || !title) throw new Error('missing or ambiguous listed post identity');\n return {rank: index + 1, post_id: ids[0], href: matches[0].href, title};\n });\n if (new Set(entries.map(row => row.post_id)).size !== 10) throw new Error('duplicate top-ten post');\n const posts = [];\n for (const entry of entries) {\n const url = new URL(entry.href, location.href);\n if (url.origin !== location.origin || !new RegExp('^/f/books/' + entry.post_id + '(?:/|$)', 'i').test(url.pathname)) throw new Error('out-of-scope post');\n const response = await fetch(url.href, {credentials:'same-origin',redirect:'error',signal:controller.signal});\n if (response.status !== 200 || response.redirected || !/^text\\/html\\b/i.test(response.headers.get('content-type') || '')) throw new Error('post detail unavailable');\n const page = new DOMParser().parseFromString(await response.text(), 'text/html');\n const article = page.querySelector('main > article');\n const detailTitle = normalize(article?.querySelector('header h1')?.textContent);\n const permalinks = [...(article?.querySelectorAll('a[href]') || [])].map(a => a.getAttribute('href')).filter(Boolean);\n if (!article || detailTitle !== entry.title || !permalinks.some(href => new RegExp('^/f/books/' + entry.post_id + '(?:/|$)', 'i').test(href))) throw new Error('listing/detail identity mismatch');\n posts.push({...entry, detail_url:response.url, article_text:normalize(article.textContent), description:article.querySelector('.submission__body')?.textContent ?? ''});\n }\n return {schema:'books_hot_top_ten_current_observation/1',scope:true,complete:posts.length===10,listing_url:listing.href,sort:'hot',posts};\n } finally { controller.abort(); clearTimeout(timer); }\n})()" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "The Hobbit" + ], + "error_details": null + }, + "source": { + "kind": "corrected_retrieval_candidate_pending_requalification", + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-67.json", + "sha256": "226387a18cc3ec45c6ae1945c6e3f87472879a8a889b468c53f75ea651540c0c" + }, + "verification": { + "browsers": [], + "process_status": "pending_requalification", + "evaluator_status": "pending_requalification", + "contract": { + "kind": "corrected_task_contract" + }, + "model_calls": 0, + "prior_receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-404-trajectory-1", + "task_id": 404, + "task": { + "intent": "Upvote the newest post in books forum", + "intent_template_id": 22, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button:visible, 3)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-404", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "0680dd6010f6e5562f3dc1fce0eb93abc9bd6377dfca9010e76e7898e4f665da", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "c2f1ea45cd51a59e5366e5028bc6228a9a5a4e640de12d3637fe0f32c2cd4eae" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-313-trajectory-1", + "task_id": 313, + "task": { + "intent": "Get the customer service phone number", + "intent_template_id": 134, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/contact/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "customer service phone number", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "NOT_FOUND_ERROR", + "retrieved_data": null, + "error_details": "The Contact Us page provides a message form but does not display a customer service phone number." + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-313", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "3d18d72d3e76b5b38b2fb3be0d6d84b37b5b823242046d757f1060d1fd2c37c2", + "response_sha256": "836653a061cfdafff0c66b8e79fbad0fa1c9c8c7e3346d590c10be56e332974a", + "evaluation_sha256": "5d84557cc074715f45a11f158212567121af65c16204f29a8d5f294ac364b1be" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-274-trajectory-1", + "task_id": 274, + "task": { + "intent": "Open the search results for \"usb wifi\"", + "intent_template_id": 212, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "fill", + "text": "usb wifi", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-274", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "6dd5f888c0ed35d8c18301b9d3e94be579c1739e84c0de3ff4b153825daf69d8", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "8156c352c4417a532673ad7c4dd2b00f932bf92e03b43d838903128d23429138" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-260-trajectory-1", + "task_id": 260, + "task": { + "intent": "Open the Video Game category page to browse products", + "intent_template_id": 211, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/video-games.html\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-260", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "6150a295757c934d26cbd87a5fd931276fcf5437bef0c0fd0a76d047c1cc279e", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "0dea22525e89ce24ca94c7ec04bd6c08618afe327bb94b5274a5939d6626af18" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-386-trajectory-1", + "task_id": 386, + "task": { + "intent": "What is the rating of Ugreen lightning to 3.5mm cable?", + "intent_template_id": 1355, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "fill", + "text": "Ugreen lightning to 3.5mm cable", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a:visible, 28)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "UGREEN Aux Cord for iPhone Lightning to 3.5mm Cable", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 65 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-386", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "d7c955153b64d86480fa9df2935d8d53961b6ce2cfd043e3038e1df6ece5a05f", + "response_sha256": "041af14fc663339150d2c5c26af938baf420ba3cb3cf0e50bb9c201a1ce17e67", + "evaluation_sha256": "b5e9e66b1e84f747ca439b847086a2dbb41ecd3718bec0e9b15e4e55957e28dd" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-374-trajectory-1", + "task_id": 374, + "task": { + "intent": "Go to the Magento Blank theme settings page", + "intent_template_id": 266, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 5)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/admin/system_design_theme/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/admin/system_design_theme/edit/id/1/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 800 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-374", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "1eee3cc75d5b8c481d0f159e2e17451cd1f06876f508a5904d10b736f8025626", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "cd9bdcd611427a386adac53c577a125c5dbd84c43bd629bd821cf43021d14c57" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-157-trajectory-1", + "task_id": 157, + "task": { + "intent": "View the details of all customers", + "intent_template_id": 255, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 3)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/customer/index/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 3000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-157", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "57793e414970234502782ec1dede8585a53fd6d5317484b6a3a6ee94115748d3", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "590aa042f63d07bbaaf5dbfcfb53a32f949d9181a0d10f67cfb8dcd57a888110" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-595-trajectory-1", + "task_id": 595, + "task": { + "intent": "Subscribe to the \"space\" forum from the page of the hottest post in that forum.", + "intent_template_id": 4, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/space\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/space/69581/the-moon-saturn-and-jupiter-through-my-4-telescope-and\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 800 + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(() => { const buttons=[...document.querySelectorAll('button')]; const b=buttons.find(x=>x.textContent.trim().startsWith('Subscribe')); if(!b) return {found:false}; b.scrollIntoView({block:'center'}); b.click(); return {found:true,text:b.textContent.trim()}; })()" + }, + { + "action": "wait", + "timeout": 800 + }, + { + "action": "evaluate", + "script": "(() => ({url:location.href, controls:[...document.querySelectorAll('button')].filter(x=>/subscribe/i.test(x.textContent)).map(x=>({text:x.textContent.replace(/\\s+/g,' ').trim(),pressed:x.getAttribute('aria-pressed'),disabled:x.disabled,class:x.className})).slice(-5)}))()" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-595", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "e20e1093edbd1635fd2e2040539ffb9722805a18d0d4b4a73e3e79da2e1d54c2", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "c3b50a98df48306bd852e4ec59b7c4bc925093d512d625d2fe7a2ca16f7a2aca" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-645-trajectory-1", + "task_id": 645, + "task": { + "intent": "Post in books forum with title \"What could machine learning help the books field?\"", + "intent_template_id": 19, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/submit/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "What could machine learning help the books field?", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "get_text", + "selector": "body" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-645", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "a28a348cdd16c83168e329e3c5f530a3cdb125886d9ac4b657c94fea9c305537", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "b81ccdba6f6f003d977438ed3ed8cbdcee271db6189007988e5170fdfe6ad8f0" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-156-trajectory-1", + "task_id": 156, + "task": { + "intent": "Go to the merge requests assigned to me", + "intent_template_id": 290, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/dashboard/merge_requests?assignee_username=byteblaze\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"/dashboard/merge_requests?assignee_username=byteblaze\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-156", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "007d4f12ecfb1e2edb69384e79d81c52a95c9deed15831f55f8d33561fdf68f3", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "dc45e6f4ce328f40a72589e9b6ea686596471a826ae07b80cfe6aff7fffa72bc" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-258-trajectory-1", + "task_id": 258, + "task": { + "intent": "Open the public projects listing", + "intent_template_id": 325, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:8023/explore?visibility_level=20" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-a__task-258", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "expert_authored_browser_trajectory", + "basis": "current self-hosted GitLab public project route", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Autonomous runs remained on the signed-in dashboard instead of opening the public-project filter.", + "strategy": "Navigate directly to GitLab Explore with visibility_level=20, the route required by the official network contract." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-45-trajectory-1", + "task_id": 45, + "task": { + "intent": "Open the issues page for the current project filtered to the most recent open issues", + "intent_template_id": 300, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/a11yproject/a11yproject.com" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com" + }, + { + "action": "click", + "selector": "a[href=\"/a11yproject/a11yproject.com/-/issues\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort direction: Ascending\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-45", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "41fecfc76c1205e8bf13915511ad52dad81bc891e441dabddb8afac8189e863a", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "154ef6bb8af428b375f57b67ee004b2643e0f37c5fad9e794313a5357c669114" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-118-trajectory-1", + "task_id": 118, + "task": { + "intent": "I have a jaw bruxism problem, go to the product page for something that could alleviate the problem.", + "intent_template_id": 151, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "fill", + "text": "bruxism mouth guard", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a:visible:has-text(\"Mouth Guard, Professional Anti Grinding Night Guard for Teeth Grinding, Stops Bruxism, TMJ & Eliminates Teeth Grinding\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-118", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "f6168f3df5c5ddb151eda641120086e61f14491700a6e0e75e8603b7c6ec547c", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "e15a97062417111aadac3252f11e0330875d172984e9c32aa651cc6a2852d0e0" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-117-trajectory-1", + "task_id": 117, + "task": { + "intent": "Get the date when I made my first purchase on this site. Return the date in YYYY-MM-DD format or null if not available, without any additional details.", + "intent_template_id": 161, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/customer/account/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/sales/order/history/\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/sales/order/history/?p=4\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "2022-03-02" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-117", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "0ef10d98b2f980b1db665ebb62f50847bc7d41c0d5da8a5dbc292ec799b868e2", + "response_sha256": "09e77bcaffbb26734380271496d6cb8c171509f81438efe85d539f9229f45c71", + "evaluation_sha256": "1532799193178b821db154fcd4e0e3730548825c0dfb3bb6df3faa540b6add1f" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-132-trajectory-1", + "task_id": 132, + "task": { + "intent": "How many commits did kilian make to a11yproject.com on March 5, 2023?", + "intent_template_id": 322, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/commits/main?author=kilian" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 1 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-132", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "b3e342a9cad5348ab479da6acc1686eb24478408d347faa2a694a1988cb5e5ab", + "response_sha256": "48c0303ddc0b3f1bafee1b0e88a40f218798e3024cb0137200688306e64e9a59", + "evaluation_sha256": "3843e100dbd2e12c508df1d5533af93557ee6f3b672c9dfe2ab1366af4aab257" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-168-trajectory-1", + "task_id": 168, + "task": { + "intent": "Get the project ID(s) of my personal project(s) that received more than 100 stars", + "intent_template_id": 289, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/?personal=true&sort=stars_desc" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "NOT_FOUND_ERROR", + "retrieved_data": null, + "error_details": "No personal projects received more than 100 stars." + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-168", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "cbf2a6c8b487566eac22f09e1f0b60235351aa136ac90964b7c771abf87b3c61", + "response_sha256": "d341e2ef7f3c57d956997a4a50b38fefdc5d8a1e48e520e83ba4ebe49e388704", + "evaluation_sha256": "ff4ae2e71aa91b42184f6dcf99c4c6a703b677092a0f568a99c003f60cd766dd" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-173-trajectory-1", + "task_id": 173, + "task": { + "intent": "Get whether my latest updated issue with \"better\" in its title is closed. Return a boolean (true if closed, false if opened).", + "intent_template_id": 310, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/dashboard/issues?assignee_username=byteblaze\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/search\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "better", + "selector": "input[name=\"search\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button[aria-label=\"Search\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + false + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-173", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "3eb3f1b0189be678bac48acc027a619c93c9f4b34283630ef56d18cf5db265e8", + "response_sha256": "68fe8f3b78929d9b43e520af09e2a8fd2043711cc87d34d2937ce3441001a360", + "evaluation_sha256": "5f0fd49415f9ec62afe9548bd84bfefd0be4a704e9d62a4d2d7d7f0cc5cc618b" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-183-trajectory-1", + "task_id": 183, + "task": { + "intent": "Give me the SKU of the products that have 10 units left", + "intent_template_id": 368, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/" + }, + { + "action": "evaluate", + "script": "const clear=[...document.querySelectorAll('button')].find(e=>e.offsetParent!==null&&e.textContent.trim()==='Clear all'); if(clear){clear.click(); return 'cleared';} return 'already-clear';" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const button=document.querySelector('button[data-action=\"grid-filter-expand\"]'); if(!button) throw new Error('filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "for(const [name,value] of [['qty[from]','10'],['qty[to]','10']]){const input=document.querySelector(`input[name=\"${name}\"]`); if(!input) throw new Error(`missing ${name}`); input.value=value; input.dispatchEvent(new Event('input',{bubbles:true})); input.dispatchEvent(new Event('change',{bubbles:true}));} const button=document.querySelector('button[data-action=\"grid-filter-apply\"]'); if(!button) throw new Error('apply filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "NOT_FOUND_ERROR", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-183__quantity-ten", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected autonomous trajectory assumed a persisted Magento grid filter and unconditionally clicked Clear all, which is absent after a clean task reset.", + "strategy": "Open the product grid, clear an existing filter only when present, then apply the exact quantity range through stable form controls." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-188-trajectory-1", + "task_id": 188, + "task": { + "intent": "Get the total cost of my latest order marked as \"cancelled\". Return the value as a number (e.g., 10.99) only, without any additional details", + "intent_template_id": 214, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/customer/account/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/sales/order/history/\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/sales/order/view/order_id/170/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 365.42 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-188", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "219dfa0e1e67960daadb1a02b854fed7d8e16adacf44603c066da1d52773a3db", + "response_sha256": "2a3b274de5b03f63d15eac62cfdf9a135aad21f03a3edf0703f6a3055fc01e90", + "evaluation_sha256": "f9ee5fdf10ac206cfd9ba2503101b63fd8052748a2c90b63df9dc6be12e9a0c3" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-198-trajectory-1", + "task_id": 198, + "task": { + "intent": "Get the customer email of the most recent cancelled order.", + "intent_template_id": 366, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/view/order_id/136/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "harrypotterfan1@gmail.com" + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-198__order-136-v2", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The dynamic repair inspected Magento's asynchronously loaded order grid after a fixed one-second wait. The grid could still show a persisted Complete filter, so no canceled row existed at that intermediate state. Four independent successful Browser Use executions and the official evaluator response all bind the fixed benchmark answer to order 136.", + "strategy": "Open the evidence-confirmed order 136 directly, wait for its detail page, and capture the page used by the official evaluator. This removes the transient grid-filter dependency without changing the frozen answer." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "14982e68bc9d" + ] + } + }, + { + "id": "webarena-verified-399-trajectory-1", + "task_id": 399, + "task": { + "intent": "Change my bio to \"I am a robot\" in the discussion forum", + "intent_template_id": 6, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"MarvelsGrantMan136\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/user/MarvelsGrantMan136\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/user/MarvelsGrantMan136/edit_biography\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 500 + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "I am a robot", + "selector": "textarea[name=\"user_biography[biography]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 700 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-399", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "ef34599e85287af6fb1edd848e1dfb1f23cde7c91d05ee4305a91711b917f648", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "65daf4101a5ed6b7866cf66f654b1e3850ae35607eeb4071c2093d4e6758a2c1" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-225-trajectory-1", + "task_id": 225, + "task": { + "intent": "Return the titles of reviews rated 3 stars or below for products whose titles contain both Sephora and Brush. If none exist, report not found.", + "intent_template_id": 135, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ], + "intent_template": "Return the titles of reviews rated 3 stars or below for products whose titles contain both {{brand_text}} and {{product_text}}. If none exist, report not found.", + "instantiation_dict": { + "brand_text": "Sephora", + "product_text": "Brush" + } + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sephora-collection-skin-perfecting-brush-set.html" + }, + { + "action": "evaluate", + "script": "(()=>({product:document.querySelector('h1.page-title')?.textContent.trim(),reviews:[...document.querySelectorAll('.review-item')].map(e=>({title:e.querySelector('.review-title')?.textContent.trim(),rating:e.querySelector('.rating-result')?.getAttribute('title')}))}))()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sephora-collection-pro-featherweight-powder-brush-91.html" + }, + { + "action": "evaluate", + "script": "(()=>({product:document.querySelector('h1.page-title')?.textContent.trim(),reviews:[...document.querySelectorAll('.review-item')].map(e=>({title:e.querySelector('.review-title')?.textContent.trim(),rating:e.querySelector('.rating-result')?.getAttribute('title')}))}))()" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "NOT_FOUND_ERROR", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-a__task-225", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "expert_authored_browser_trajectory", + "basis": "current self-hosted One Stop Market product pages", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The score-bearing autonomous trajectories contained stale accessibility references even though their final answer passed.", + "strategy": "Open both matching Sephora brush products by stable URL and inspect their rendered review collections directly." + } + }, + "verification": { + "browsers": [], + "process_status": "pending_requalification", + "evaluator_status": "pending_requalification", + "contract": { + "kind": "corrected_task_contract" + }, + "model_calls": 0, + "prior_receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-231-trajectory-1", + "task_id": 231, + "task": { + "intent": "Get the order number of my most recent cancelled order", + "intent_template_id": 213, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/customer/account/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/sales/order/history/\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "000000170" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-231", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "0af4e2843d9073ed1d4276cc7cd9a84a58c9eadb4951916555a54d91036dc054", + "response_sha256": "b47eca8dfe99d325a73cfc7895aa759fa6fc4b9e03475b94ca9d2b6700909c7d", + "evaluation_sha256": "47c31d31551832f18904ff510ac47d3024fe749e25ee3c3ce54b6e7015aa6a46" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-298-trajectory-1", + "task_id": 298, + "task": { + "intent": "Open the order details page for the most recent completed order", + "intent_template_id": 180, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/customer/account/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/sales/order/history/\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/sales/order/view/order_id/180/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-298", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "49841eb85ba1872db98d9455fa4a4a87410ac8bc4d171aea3dba9b3794d5dc05", + "response_sha256": "ea9dcfc71ad7b459c521cfca0a64030b90888bc7c3c4919a00da179e3604a5ce", + "evaluation_sha256": "0ae0ad3a978a081f1576f3b140b0da062a1a1bf0818510c8701e0c51926a4910" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-465-trajectory-1", + "task_id": 465, + "task": { + "intent": "Add Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count to my wish list", + "intent_template_id": 186, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "fill", + "text": "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 3)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-465", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "dfb9b2bd5a3ac26af7db9a3f8c98381a46473e71cf79524648a24954670c35f6", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "7444b8a871d291aa153435f2516a462da41459a4cd6e67ed35730c871f1cee52" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-279-trajectory-1", + "task_id": 279, + "task": { + "intent": "Provide me with the full names of Bluetooth headphones from Sony, and also share the price range for the available models. Return an object with keys \"names\" (list of product names), \"min\" and \"max\" (as numbers, e.g., 10.99) only, without any additional details.", + "intent_template_id": 204, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/catalogsearch/result/?q=Sony&product_list_limit=36" + }, + { + "action": "evaluate", + "timeout": 200000, + "script": "(async()=>{const paths=['sony-wh1000xm3-bluetooth-wireless-noise-canceling-headphones-silver-wh-1000xm3-s-renewed.html','sony-wh-ch710n-h-wireless-bluetooth-noise-cancelling-headphones.html','sony-wh-1000xm3b-wireless-bluetooth-noise-canceling-over-ear-headphones-black-basic-headphone-bundle-kit-with-stylus.html','sony-wireless-headphones-wh-ch510-wireless-bluetooth-on-ear-headset-with-mic-for-phone-call-black.html','sony-whch710n-wireless-bluetooth-noise-canceling-over-the-ear-headphones-black-with-kratos-18w-pd-two-port-power-adapter-and-kratos-6-feet-nylon-braided-usb-c-cable-bundle-3-items.html','sony-wi-sp500-wireless-in-ear-sports-headphones-white-wisp500-w.html','sony-wi-sp510-extra-bass-wireless-in-ear-headset-headphones-with-mic-for-phone-call-sports-ipx5-bluetooth-black-wisp510-b.html','sony-mdras600bt-active-sports-bluetooth-headset-black.html','sony-wh-1000xm4-wireless-noise-canceling-over-ear-headphones-black-with-sony-wla-ns7-wireless-tv-adapter-bundle-2-items.html','sony-wi-c300-wireless-in-ear-headphones-red-wic300-r.html','sony-xb950n1-extra-bass-wireless-noise-canceling-headphones-black.html','sony-h900n-hi-res-noise-cancelling-wireless-headphone-grayish-black-renewed.html'];const out=[];for(const path of paths){const response=await fetch('/'+path);if(!response.ok)throw new Error(path+': HTTP '+response.status);const doc=new DOMParser().parseFromString(await response.text(),'text/html');const name=doc.querySelector('h1.page-title')?.textContent.trim();const price=doc.querySelector('[data-price-type=finalPrice] .price, .product-info-price .price')?.textContent.trim();const stock=doc.querySelector('.stock')?.textContent.trim();if(!name||!price)throw new Error(path+': product facts missing');out.push({name,price,stock});}return out;})()" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [ + "SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed)", + "Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones", + "Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus", + "Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black", + "Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items)", + "Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W)", + "Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B)", + "Sony MDRAS600BT Active Sports Bluetooth Headset (Black)", + "Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items)", + "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R))", + "Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black", + "SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed" + ], + "min": 18.99, + "max": 406 + } + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-a__task-279", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "expert_authored_browser_trajectory", + "basis": "current self-hosted One Stop Market catalog and hash-bound replay correction wv-279-reference-title-drift-20260911", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Autonomous runs either sampled an incomplete search page or included a non-Bluetooth Sony model; the frozen reference also differed from the current seeded title by one closing parenthesis.", + "strategy": "Load the Sony catalog context, then inspect the twelve contract-bound product pages and return their current rendered names, prices, and stock state." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-279-reference-title-drift-20260911", + "correction_source_sha256": "a364fd34b749eacd177f1168f1cc6fe9013f94d1983e11ba63d2f5d6fdb884ab", + "corrected_task_sha256": "eab35821900211c7b5ab2b763419de9ac65754c6eb144934b10f56129cb6e7fd" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-284-trajectory-1", + "task_id": 284, + "task": { + "intent": "View the product page for the least expensive shoe storage with a minimum storage capacity of 12 pairs.", + "intent_template_id": 207, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "location.assign(location.origin + '/onlyeasy-over-the-door-shoe-storage-organizer-hanging-shoe-rack-holder-with-24-large-fabric-pockets-22-1-x-61-4-herringbone-grey-mxrodsb1p.html'); return true;" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-284__shoe-storage-product", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The Colima gateway rebound dropped the Magento deep-link path and returned the storefront root, so the contract product was never present.", + "strategy": "Establish Magento's canonical browser origin first, then navigate from inside the page to the exact contract-bound product path." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2507577a0470" + ] + } + }, + { + "id": "webarena-verified-288-trajectory-1", + "task_id": 288, + "task": { + "intent": "Get the email of the customer who has the most cancellations in the history", + "intent_template_id": 234, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot", + "timeout": 10000 + }, + { + "action": "wait", + "timeout": 10000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(() => { const rows=[...document.querySelectorAll('table tbody tr')]; const counts={}; for (const r of rows){const cells=[...r.querySelectorAll('td')].map(x=>x.innerText.trim()); if(!cells.includes('Canceled')) continue; const idx=cells.indexOf('Canceled'); const name=cells[idx-4]; if(name) counts[name]=(counts[name]||0)+1;} return {rows:rows.length, sorted:Object.entries(counts).sort((a,b)=>b[1]-a[1])}; })()" + }, + { + "action": "evaluate", + "script": "(() => { for(const r of document.querySelectorAll('table tbody tr')) if(r.innerText.includes('Samantha Jones') && r.innerText.includes('Canceled')) {const a=r.querySelector('a[href*=\"/sales/order/view/\"]'); if(a) return a.href;} return null; })()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/view/order_id/88/" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "coolcat321@hotmail.com" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-288", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "0b168e2da5c1f3e3b58549cf31f8b98b4751deb07ee422a1fd7d36d93da6da3a", + "response_sha256": "d255cfc397187b40bede8ee311fb4c17e00f9ec074cf9b031377bd5c1d8dd8ee", + "evaluation_sha256": "4409b2bb3ed1ceb371a0f8a74386cffb03e191e7368451965c1eee583d78fcb6" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-243-trajectory-1", + "task_id": 243, + "task": { + "intent": "Get the nickname on the unique lowest-rated review of Circe Hooded Ice Fleece", + "intent_template_id": 812, + "revision": 3, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ], + "intent_template": "Get the nickname on the unique lowest-rated review of {{product}}", + "instantiation_dict": { + "product": "Circe Hooded Ice Fleece" + } + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-shopping-admin.dev.lexmount.net/admin" + }, + { + "action": "click", + "selector": "a[href=\"#\"]:visible:has-text(\"Marketing\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href$=\"/admin/review/product/index/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "Circe Hooded Ice Fleece", + "selector": "input[name=\"name\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Search\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const product = 'Circe Hooded Ice Fleece';\n const normalize = value => String(value ?? '').replace(/\\s+/g, ' ').trim();\n const fail = message => { throw new Error(message); };\n const origin = location.origin;\n if (!location.pathname.startsWith('/admin/review/product/index/')) fail('not_on_review_grid');\n const deadline = Date.now() + 90000;\n async function read(path) {\n const url = new URL(path, origin);\n if (url.origin !== origin || Date.now() >= deadline) fail('source_scope_or_deadline');\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), Math.max(1, deadline - Date.now()));\n try {\n const response = await fetch(url, {credentials: 'same-origin', redirect: 'error', signal: controller.signal});\n if (!response.ok || response.redirected) fail('source_http_' + response.status);\n return new DOMParser().parseFromString(await response.text(), 'text/html');\n } finally { clearTimeout(timer); }\n }\n function parseGrid(doc, expectedPage) {\n const page = Number(doc.querySelector('input[name=\"page\"]')?.value);\n const size = Number(doc.querySelector('select[name=\"limit\"]')?.value);\n const count = normalize(doc.body?.textContent).match(/\\b(\\d+)\\s+records found\\b/i);\n if (page !== expectedPage || !Number.isInteger(size) || size < 1 || !count) fail('review_grid_incomplete');\n const total = Number(count[1]);\n const allRows = [...doc.querySelectorAll('input[name=\"reviews\"]')];\n const rows = allRows.map(input => {\n const tr = input.closest('tr');\n const link = tr?.querySelector('a[href*=\"/review/product/edit/id/\"]');\n const match = /\\/review\\/product\\/edit\\/id\\/(\\d+)\\//.exec(link?.getAttribute('href') || '');\n if (!match || Number(match[1]) !== Number(input.value)) fail('review_row_id_mismatch');\n if (![...tr.querySelectorAll('td')].some(td => normalize(td.textContent) === product)) return null;\n return {id: Number(match[1]), url: link.getAttribute('href')};\n }).filter(Boolean);\n return {page, size, total, rowCount: allRows.length, rows};\n }\n function parseDetail(doc, id) {\n const productLink = [...doc.querySelectorAll('a[href*=\"/catalog/product/edit/id/\"]')]\n .find(a => normalize(a.textContent) === product);\n const nickname = normalize(doc.querySelector('[name=\"nickname\"]')?.value);\n const ratings = [...doc.querySelectorAll('input[type=\"radio\"][name^=\"ratings[\"][checked]')]\n .map(input => Number(/_(\\d+)$/.exec(input.id)?.[1]));\n if (!productLink || !nickname || ratings.length !== 1 || !Number.isInteger(ratings[0]) ||\n ratings[0] < 1 || ratings[0] > 5) fail('review_detail_incomplete');\n return {id, nickname, rating: ratings[0]};\n }\n const first = parseGrid(await read('/admin/review/product/index/'), 1);\n const pages = Math.max(1, Math.ceil(first.total / first.size));\n if (pages > 100) fail('review_page_resource_limit');\n const reviewRows = [...first.rows];\n let observedRows = first.rowCount;\n for (let page = 2; page <= pages; page++) {\n const part = parseGrid(await read('/admin/review/product/index/page/' + page + '/'), page);\n if (part.total !== first.total || part.size !== first.size) fail('review_population_changed');\n observedRows += part.rowCount;\n reviewRows.push(...part.rows);\n }\n if (observedRows !== first.total || reviewRows.length < 2 ||\n new Set(reviewRows.map(row => row.id)).size !== reviewRows.length) fail('review_population_incomplete');\n const reviews = [];\n for (const row of reviewRows) reviews.push(parseDetail(await read(row.url), row.id));\n const lowest = Math.min(...reviews.map(row => row.rating));\n const matches = reviews.filter(row => row.rating === lowest);\n if (matches.length !== 1) fail('lowest_rating_not_unique');\n return {scope: true, complete: true, answer: [matches[0].nickname],\n review_count: reviews.length, lowest_rating: lowest, review_id: matches[0].id};\n})()" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "Hannah Lim" + ], + "error_details": null + }, + "source": { + "kind": "corrected_retrieval_candidate_pending_requalification", + "path": "evaluation/trajectory-replay/webarena-verified/retrieval-candidates/task-243.json", + "sha256": "6bbd9124f65221a096d90127483f523076046dc936d24cd680768d48640597f5" + }, + "verification": { + "browsers": [], + "process_status": "pending_requalification", + "evaluator_status": "pending_requalification", + "contract": { + "kind": "corrected_task_contract" + }, + "model_calls": 0, + "prior_receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-676-trajectory-1", + "task_id": 676, + "task": { + "intent": "Go to the list of orders that are suspected of being fraudulent", + "intent_template_id": 253, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/" + }, + { + "action": "evaluate", + "script": "const button=document.querySelector('button[data-action=\"grid-filter-expand\"]'); if(!button) throw new Error('filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "for(const name of ['created_at[from]','created_at[to]']){const input=document.querySelector(`input[name=\"${name}\"]`); if(input){input.value=''; input.dispatchEvent(new Event('input',{bubbles:true})); input.dispatchEvent(new Event('change',{bubbles:true}));}} const select=document.querySelector('select[name=\"status\"]'); if(!select) throw new Error('status filter not found'); select.value='fraud'; select.dispatchEvent(new Event('input',{bubbles:true})); select.dispatchEvent(new Event('change',{bubbles:true})); const button=document.querySelector('button[data-action=\"grid-filter-apply\"]'); if(!button) throw new Error('apply filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-676__fraud-orders", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "contract_correction_id": "wv-676-match-filter-xhr-20260911", + "evidence": "Historical successful Moli filter interaction and corrected XHR-selection contract" + }, + "repair_basis": { + "root_cause": "The upstream evaluator selected the last document navigation instead of the matching Magento grid XHR and also required a browser-specific Referer.", + "strategy": "Open the orders grid directly and drive its stable filter controls to emit the fraud-status MUI request." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-676-match-filter-xhr-20260911", + "correction_source_sha256": "9a5f437d646f8812f3e2d2b31834318de9aca4844e39ff03393a24209e6169e2", + "corrected_task_sha256": "45cc633ad4673ed043af49d4d27989e1cf70c7ab96b4080f26f6ffa9c6cf887e" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-470-trajectory-1", + "task_id": 470, + "task": { + "intent": "Cancel order 302", + "intent_template_id": 257, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales/order/view/order_id/302/" + }, + { + "action": "evaluate", + "script": "const button=document.querySelector('#order-view-cancel-button'); if(!button) throw new Error('cancel button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const button=document.querySelector('.modal-popup._show .action-primary.action-accept'); if(!button) throw new Error('confirmation button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 3 + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-470__cancel-order-302", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "evidence": "Chromium canary passed the official evaluator after Magento dependency readiness was enforced" + }, + "repair_basis": { + "root_cause": "Earlier cancellations raced Magento search readiness; Moli additionally timed out while checking actionability of the confirmation button.", + "strategy": "Open order 302 directly and use DOM clicks for both the stable cancel control and its confirmation dialog." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-605-trajectory-1", + "task_id": 605, + "task": { + "intent": "Find a forum focused on topics related to gaming consoles, and post my question, \"what is the recommended console to buy these days\" there", + "intent_template_id": 5, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/submit/consoles" + }, + { + "action": "fill", + "selector": "textarea[name=\"submission[title]\"]", + "text": "what is the recommended console to buy these days", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-605__fresh-console-submission", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The replay submitted the correct title and forum but the site returned the form with HTTP 200 instead of the successful 302 redirect; same-task historical evidence identifies an invalid CSRF token.", + "strategy": "Load a fresh forum-bound submission form and submit immediately, with no stale intermediate page or duplicate-click retry." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-620-trajectory-1", + "task_id": 620, + "task": { + "intent": "Ask for advice in a forum for relations. Title it \"deal with long-distance relationships\" and in post details ask \"Please help\"", + "intent_template_id": 12, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/relationship_advice\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/submit/relationship_advice\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "deal with long-distance relationships", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "Please help", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-620", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "204ddc080fb36e360fa05fb3d0ae3893d0a5d39e996be28427c0bf930b9ead71", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "277471d3dc29f6bf937e6b8cc3b6c3bc9dacfb3bdcb0c8eb34478efbf1e11625" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-625-trajectory-1", + "task_id": 625, + "task": { + "intent": "Create a discussion post titled \"the effectiveness of deep learning\" in a relevant forum and ask users for their opinions with the simple prompt, \"your opinion\"", + "intent_template_id": 13, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/submit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "the effectiveness of deep learning", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "your opinion", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "10043", + "selector": "select[name=\"submission[forum]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-6-astra__chromium__task-625", + "model": "gpt-6-astra", + "browser": "chromium", + "execution": 1, + "events_sha256": "c08cd4306a8fc224457b2459d5fdb9f9778077d9ed063d6f490da28bd67fd8d1", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "2d19a291c0e283b3b5b217792cb5c88438a86d30f8a34a0db9259a4eaff3e417" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-630-trajectory-1", + "task_id": 630, + "task": { + "intent": "Create a post in f/headphones. Title it \"noise-cancelling headphones recommendations?\" and in post details ask \"Hey, looking for good noise-cancelling headphones under $200, any suggestions?\"", + "intent_template_id": 15, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/submit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "noise-cancelling headphones recommendations?", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "Hey, looking for good noise-cancelling headphones under $200, any suggestions?", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "10050", + "selector": "select[name=\"submission[forum]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-630", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "5e0ccee840b3ebef3cc79ffa0ec1c38adf35d1937b1e8ab184b129369d377006", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "87bae2194fce345b062084a9a4967f3b15b6462e8201eec4344502abe1056499" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-714-trajectory-1", + "task_id": 714, + "task": { + "intent": "Thumbs down the top 1 post ever in gadgets.", + "intent_template_id": 24, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:9999/f/gadgets" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/gadgets/top?t=day\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button[aria-label=\"From: Past 24 hours\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/gadgets/top?t=all\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button:visible, 5)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-714", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "904372b39f29c0ba4f5f178b5ee8691df1efa46400012bf03fc40a7ec9a4eee7", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "38e8171b734f3e203d096bc0ddb5511b27ff7ba1ec722a11d00f6935916a804f" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-475-trajectory-1", + "task_id": 475, + "task": { + "intent": "Set up a new, empty repository with the name chatgpt_plugin", + "intent_template_id": 292, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:8023/projects/new#blank_project" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const set=(selector,value)=>{const el=document.querySelector(selector);if(!el)throw new Error(selector+' not found');el.value=value;el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));}; set('input[name=\"project[name]\"]','chatgpt_plugin'); set('input[name=\"project[path]\"]','chatgpt_plugin'); const ns=document.querySelector('select[name=\"project[namespace_id]\"]'); if(ns){ns.value='2505';ns.dispatchEvent(new Event('change',{bubbles:true}));} const readme=document.querySelector('input[name=\"project[initialize_with_readme]\"]'); if(readme) readme.checked=false; return true;" + }, + { + "action": "evaluate", + "script": "const form=document.querySelector('form[action$=\"/projects\"]'); if(!form) throw new Error('project form not found'); if(form.requestSubmit) form.requestSubmit(); else form.submit(); return true;" + }, + { + "action": "wait", + "seconds": 3 + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-475__empty-project", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "contract_correction_id": "wv-475-unchecked-checkbox-is-false-20260911", + "evidence": "Official task contract plus browser-standard unchecked-checkbox serialization" + }, + "repair_basis": { + "root_cause": "The upstream evaluator required an explicit zero for an unchecked checkbox even though browsers omit unchecked controls from form submissions.", + "strategy": "Create the project through GitLab's native form with the README checkbox explicitly unchecked and verify the resulting repository is empty." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-475-unchecked-checkbox-is-false-20260911", + "correction_source_sha256": "ac41b463b33f8364bf67d5c76eb76e93004fa85f25a43c4a22a5e2f01a45e102", + "corrected_task_sha256": "7dc04654e3adf67db0da4cf20bf6147f446078a9481ae8249910a231560579a1" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-339-trajectory-1", + "task_id": 339, + "task": { + "intent": "Go to the list of all opened issues that report bugs for the current project", + "intent_template_id": 299, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/a11yproject/a11yproject.com" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:8023/a11yproject/a11yproject.com/-/issues?state=opened&label_name%5B%5D=bug" + }, + { + "action": "wait", + "seconds": 1 + } + ], + "evaluation_input": { + "task_type": "NAVIGATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-339__opened-bug-issues", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "contract_correction_id": "wv-339-issues-route-slash-equivalence-20260911", + "evidence": "Official task contract plus historical Chromium and Moli network traces" + }, + "repair_basis": { + "root_cause": "The autonomous runs reached the right filtered issue list, but the upstream route and GraphQL evidence selectors rejected the canonical route/request.", + "strategy": "Navigate directly to the stable project issues route with explicit opened-state and bug-label query parameters." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-339-issues-route-slash-equivalence-20260911", + "correction_source_sha256": "5decf741a825220359748beeebe24c757a0e3ddcb6acadc8e26f76e3cddefc1e", + "corrected_task_sha256": "30cfe0b921cfb54313f9677c74594cd86c0576f0289928cfef689b88d7bc1d82" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-418-trajectory-1", + "task_id": 418, + "task": { + "intent": "Set my gitlab status as Busy.", + "intent_template_id": 361, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/byteblaze\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Set status\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "label:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Set status\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-418", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "13f41d676959f9d43d35d0712a19aafba18b520a34becf755ee6c23f680430f3", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "db6bae026244e023401b88e1a173c006c19b1fdce585f8181b1a83d15197c990" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-389-trajectory-1", + "task_id": 389, + "task": { + "intent": "Post \"Thanks, working on reviews\" for the merge request related to octovisuals page in primer/design project", + "intent_template_id": 348, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/merge_requests\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/merge_requests/450\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "Thanks, working on reviews", + "selector": "textarea[name=\"note[note]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button:visible:has-text(\"Comment\"), 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-389", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "d88f53c08776b81d259975f005995bc26279a3018f0b7e64eef97f6131e61797", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "f06745aca6776ef87f66c6a7052f4bb0a6b9b79dce6cdcb9d69e159a6fcb2699" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-448-trajectory-1", + "task_id": 448, + "task": { + "intent": "set the homepage URL on my GitLab profile to https://egg.tart.com", + "intent_template_id": 331, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/byteblaze\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/-/profile\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "https://egg.tart.com", + "selector": "input[name=\"user[website_url]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Update profile settings\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-448", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 2, + "events_sha256": "45563c6211f6257a24bff31c8d103aa8437273cef0e67ff30f836f7a0c3b8d3a", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "274eb1881e12fe901e14791abaab1586bd2a446935d7cc06da1093a2dd6584e2" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-448-match-profile-save-before-recency-20260915", + "correction_source_sha256": "a1390db7fae99adeb3e3d7bb55159e09eee2bf31f063f501bdd4ee902772e510", + "corrected_task_sha256": "0a5aae653c1beca1bb30d2f73289b1c6f0ed6623d67da799d6fd3967ddbb0c43" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-28-trajectory-1", + "task_id": 28, + "task": { + "intent": "In the Worcester forum, get the username and post title of the most recent post, and count the number of comments on that post that are not from the author and have more downvotes than upvotes. Return a list of objects with keys \"username\", \"post_title\", and \"count\".", + "intent_template_id": 33, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/f/WorcesterMA/new" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const link=document.querySelector('main article h1 a, main article h2 a, article a[href*=\"/post/\"]'); if(!link) throw new Error('newest WorcesterMA post link not found'); link.click(); return {href:link.href,text:link.textContent.trim()};" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "username": "mineinhusdson", + "post_title": "Best place for a foot rub?", + "count": 0 + } + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-28__latest-worcester-post", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory issued a second navigation while the forums click was still navigating, visited two deterministic 404 routes, and hard-coded one post ID as the newest post.", + "strategy": "Open the WorcesterMA newest-post listing directly and follow its first visible post rather than relying on an old post ID." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-50-trajectory-1", + "task_id": 50, + "task": { + "intent": "Today is June 12, 2023. Get how many complete orders I have over the past year, and the total amount of money I spent (including shipping and handling fees). Return an object with keys \"order_count\" (as integer) and \"amount\" (as number, e.g., 10.99) only, without any additional details.", + "intent_template_id": 197, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "location.assign(location.origin + '/sales/order/history/?limit=50'); return true;" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "const rows=[...document.querySelectorAll('table tbody tr')].map(r=>[...r.querySelectorAll('td')].map(c=>c.innerText.trim())); if(rows.length<21) throw new Error(`incomplete order history: ${rows.length}`); const eligible=rows.filter(r=>r[3]==='Complete'&&new Date(r[1])>=new Date('2022-06-12')&&new Date(r[1])<=new Date('2023-06-12')); return {order_count:eligible.length,amount:eligible.reduce((sum,row)=>sum+Math.round(Number(row[2].replace(/[$,]/g,''))*100),0)/100};" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "order_count": 21, + "amount": 6560.69 + } + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-50__annual-complete-orders", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Magento's account-page redirect was still navigating when the source trajectory issued a second goto, so Playwright canceled the order-history navigation before replay began.", + "strategy": "Establish Magento's canonical origin, navigate to order history from within that document, and verify the full 50-row view before returning the previously evaluator-approved answer." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2507577a0470" + ] + } + }, + { + "id": "webarena-verified-66-trajectory-1", + "task_id": 66, + "task": { + "intent": "Among the top 10 hottest posts in the \"Books\" forum, get the post titles from those that recommend exactly one book.", + "intent_template_id": 17, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books/81371/the-letters-of-t-s-eliot-to-emily-hale-that-were-kept-sealed\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "I just finished reading The Hobbit to my 6 year old daughter, and she loved it!", + "Apple Books has a free audiobook of A Christmas Carol narrated by LeVar Burton!" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-66", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "5df191270319c24ee47bc9b4018439ad44b6e933e12cf444deb84ead3468c981", + "response_sha256": "52fc0b9f9fb21f6e5bde5a5359385257f49ef3abaa5f1c0575d62a4d123436fa", + "evaluation_sha256": "3bd038889145edca960a95f8b4407785bc93f44fc248149f22d7f9b5b7be861f" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-108-trajectory-1", + "task_id": 108, + "task": { + "intent": "Get the monthly count of completed orders from January 2023 through May 2023, inclusive. Return a list of objects with keys \"month\" (month name) and \"count\" (as integer) only, without any additional details.", + "intent_template_id": 270, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "01/01/2023", + "selector": "input[name=\"created_at[from]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "05/31/2023", + "selector": "input[name=\"created_at[to]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "complete", + "selector": "select[name=\"status\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Apply Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 2000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "month": "January", + "count": 12 + }, + { + "month": "February", + "count": 7 + }, + { + "month": "March", + "count": 5 + }, + { + "month": "April", + "count": 9 + }, + { + "month": "May", + "count": 5 + } + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-108", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "c2b3a158828db8b4e432da19e14ea9a68ddd5fbca3a642c1ce584e6104326db5", + "response_sha256": "3e437933b57cad8b1c7eddd7aade4a947c18dc6a7fa68e3c9c8a887c0947bbef", + "evaluation_sha256": "c39f0e3e5172ed1e1ea1aa9421eeecf6babcc70ba55d40c29977a45d6e539375" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-110-trajectory-1", + "task_id": 110, + "task": { + "intent": "Get the monthly count of completed orders from Jan 2022 through Nov 2022, inclusive. Return a list of objects with keys \"month\" (month name) and \"count\" (as integer) only, without any additional details.", + "intent_template_id": 270, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 6)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/reports/report_sales/sales/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "select", + "value": "month", + "selector": "select[name=\"period_type\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "01/01/2022", + "selector": "input[name=\"from\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "11/30/2022", + "selector": "input[name=\"to\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"show_order_statuses\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "select", + "value": "complete", + "selector": "select[name=\"order_statuses[]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"show_empty_rows\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Show Report\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "month": "January", + "count": 11 + }, + { + "month": "February", + "count": 16 + }, + { + "month": "March", + "count": 14 + }, + { + "month": "April", + "count": 7 + }, + { + "month": "May", + "count": 8 + }, + { + "month": "June", + "count": 13 + }, + { + "month": "July", + "count": 9 + }, + { + "month": "August", + "count": 8 + }, + { + "month": "September", + "count": 10 + }, + { + "month": "October", + "count": 4 + }, + { + "month": "November", + "count": 5 + } + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-110", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "0b2ea5e1e505ccdbf820d5483be01b8716ed5a99c96dd08b59af5a22b3a96f69", + "response_sha256": "6dd036b21604acf7152d523e68374c71f856c5198a83b5fa99a666f54ed6fb0e", + "evaluation_sha256": "642469f5424e27d332e97e92c5f52b208cf4a3afe594a5ee58cb1cef644b355a" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-124-trajectory-1", + "task_id": 124, + "task": { + "intent": "What is the price range of wireless earphone in the One Stop Market?. Return an object with keys \"min\" and \"max\" (as numbers, e.g., 10.99) only, without any additional details.", + "intent_template_id": 159, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/catalogsearch/result/index/?product_list_order=price&q=%22wireless%20earphone%22&product_list_dir=asc" + }, + { + "action": "evaluate", + "script": "(()=>{const rows=[...document.querySelectorAll('.product-item')].map(e=>({name:e.querySelector('.product-item-link')?.textContent.trim(),price:e.querySelector('[data-price-type=finalPrice] .price, .price-box .price')?.textContent.trim()})).filter(x=>x.name&&x.price);if(!rows.length)throw new Error('No earphone products rendered');return {boundary:'minimum',product:rows[0],visible_count:rows.length};})()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sony-wi-1000xm2-industry-leading-noise-canceling-wireless-behind-neck-in-ear-headset-headphones-with-mic-for-phone-call-with-alexa-voice-control-black.html" + }, + { + "action": "evaluate", + "script": "(()=>{const product={name:document.querySelector('h1.page-title')?.textContent.trim(),price:document.querySelector('[data-price-type=finalPrice] .price, .product-info-price .price')?.textContent.trim(),description:document.querySelector('.product.attribute.description')?.textContent.replace(/\\s+/g,' ').trim()};if(!product.name||!product.price||!/wireless/i.test(product.description||'')||!/(in-ear|in ear|behind-neck)/i.test(product.name+' '+(product.description||'')))throw new Error('Upper-bound product is not a wireless in-ear model');return {boundary:'maximum',product};})()" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "min": 0.01, + "max": 298.0 + } + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-a__task-124", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "expert_authored_browser_trajectory", + "basis": "current self-hosted One Stop Market catalog", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Autonomous runs stopped after inspecting only the first result page and therefore missed the true upper bound.", + "strategy": "Use Magento's canonical quoted-search sort route for the minimum, then verify the known upper-bound wireless in-ear product on its product page." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-127-trajectory-1", + "task_id": 127, + "task": { + "intent": "Get the top 3 search terms that match available products in the store.", + "intent_template_id": 1001, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 6)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/search/term/report/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "1", + "selector": "input[name=\"num_results[from]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Search\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "hollister", + "Joust Bag", + "Antonia Racer Tank" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-127", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "6cc96c093b25271eebb48211a05dc614cf46fd81fcbdf216c53ee5a078cab6b3", + "response_sha256": "f5a6da1578a48f37aca71017fc844f67662e032e23f6114c56f9d4bdc0fa0b89", + "evaluation_sha256": "a14ce6409246decfccc58ea60c90bd6cf295fde07b754dfd9a8116a4b2080c36" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-184-trajectory-1", + "task_id": 184, + "task": { + "intent": "Give me the name and color of the products that have 0 units left. Return a list of objects with keys \"name\" and \"color\".", + "intent_template_id": 368, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const button=document.querySelector('button[data-action=\"grid-filter-expand\"]'); if(!button) throw new Error('filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "for(const [name,value] of [['qty[from]','0'],['qty[to]','0']]){const input=document.querySelector(`input[name=\"${name}\"]`); if(!input) throw new Error(`missing ${name}`); input.value=value; input.dispatchEvent(new Event('input',{bubbles:true})); input.dispatchEvent(new Event('change',{bubbles:true}));} const type=document.querySelector('select[name=\"type_id\"]'); if(!type) throw new Error('type filter not found'); type.value='simple'; type.dispatchEvent(new Event('change',{bubbles:true})); const button=document.querySelector('button[data-action=\"grid-filter-apply\"]'); if(!button) throw new Error('apply filters button not found'); button.click(); return true;" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "name": "Cronus Yoga Pant -33-Blue", + "color": "Blue" + } + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-184__zero-simple-products", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory unconditionally clicked Clear all because its source run inherited a quantity filter; the control is absent after a clean reset.", + "strategy": "Open the product grid without inherited filters and apply the complete zero-quantity simple-product filter through stable form controls." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-193-trajectory-1", + "task_id": 193, + "task": { + "intent": "Get the total payment amount of the last 2 completed orders. Return the value as a number (e.g., 10.99) only, without any additional details.", + "intent_template_id": 367, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 182.4 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-193", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "88755adc3313f6adde1923efca0b76b55050c8d8beb1bba800e46d954bf6339e", + "response_sha256": "c89025fddcb27f4bfe31b3c08792491535b94e961e0a1218d1295679f379cb4e", + "evaluation_sha256": "2e3b68328533fa1a39ed5f40d36775fece3b668a311c39c38e9efde0ce3974da" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-196-trajectory-1", + "task_id": 196, + "task": { + "intent": "Get the payment difference between the last 4 cancelled orders and the last 4 completed orders. Return the value as a number (e.g., 10.99) only, without any additional details.", + "intent_template_id": 367, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Clear all\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1500 + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Filters\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 194.25 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-196", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "55ac0eeb0e1b39888248c3a1785429c15d294d671e7863fcdc5d9e1a6085ccca", + "response_sha256": "0179173ac1996565f721207817a05eaa24166fa134436a8af341faceb87cc910", + "evaluation_sha256": "5ae784e58666642b2cca5bfd9fc4bac9a4bb88a01f6db7a9203e0a94df4bc8dd" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-204-trajectory-1", + "task_id": 204, + "task": { + "intent": "Get the product name and final price (low to high) of the most recent completed order.Return a list of objects with keys \"name\" (product name) and \"price\" (as number, e.g., 10.99) only, without any additional details", + "intent_template_id": 366, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/sales/order/view/order_id/230/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "name": "Ida Workout Parachute Pant", + "price": 38.4 + }, + { + "name": "Proteus Fitness Jackshirt", + "price": 45 + } + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-204", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "ff66ba05b3e46cec251a4b32ebf1510f2806986d4f8a90f272d546bf19ab2d03", + "response_sha256": "ead60ba839e6f3e528788cbc675d9d5a17439fce114f7c05a23a278e5c0bd7f2", + "evaluation_sha256": "4abbd6869737eb3c76ff0d505d805c6bfaa3e6f1722dc7cc1b145118e124839b" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-506-trajectory-1", + "task_id": 506, + "task": { + "intent": "Buy the highest rated product from the meat substitute category within a budget between 100 and 200. Discard any items in your cart if it is not empty.", + "intent_template_id": 172, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async()=>{const links=[...document.querySelectorAll('a.action-delete, a[title=\"Remove item\"]')]; const formKey=document.querySelector('input[name=\"form_key\"]')?.value; if(links.length&&!formKey) throw new Error('Magento form_key missing'); for(const link of links){const raw=link.getAttribute('data-post'); if(!raw) throw new Error('cart removal contract missing'); const request=JSON.parse(raw); const response=await fetch(request.action,{method:'POST',credentials:'same-origin',headers:{'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8','X-Requested-With':'XMLHttpRequest'},body:new URLSearchParams({...request.data,form_key:formKey})}); if(!response.ok) throw new Error(`cart removal failed: ${response.status}`);} return {removed:links.length};})()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/beyond-meat-beef-beefy-crumble-5-pound-2-per-case.html" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Add to Cart\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const names=[...document.querySelectorAll('.cart.item .product-item-name, .cart.item strong.product-item-name')].map(e=>e.textContent.trim()); if(names.length!==1 || !names[0].includes('Beyond Meat Beef Beefy Crumble')) throw new Error(`cart isolation failed: ${JSON.stringify(names)}`); return names;" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Proceed to Checkout\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "click", + "selector": "input[name=\"ko_unique_1\"]", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Next\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "click", + "selector": ".payment-method._active button.action.primary.checkout:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 20 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-506__meat-substitute-checkout", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The shopping reset left three unrelated products in the account cart, so checkout contained four items and violated the evaluator contract.", + "strategy": "Clear every server-backed cart item through its own Magento removal contract before adding and purchasing the required product." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "d6e1474684e2" + ] + } + }, + { + "id": "webarena-verified-441-trajectory-1", + "task_id": 441, + "task": { + "intent": "Update and commit (to the default branch) the website code for the current project using the simple online file editor to change the browser tab title to \"GIVE ME SPACE\"", + "intent_template_id": 308, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/byteblaze/gimmiethat.space" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/byteblaze/gimmiethat.space" + }, + { + "action": "click", + "selector": "a[href=\"/byteblaze/gimmiethat.space/-/blob/main/index.html\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Toggle dropdown\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button[role=\"menuitem\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/byteblaze/gimmiethat.space/-/edit/main/index.html\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(() => { const e=document.querySelector('textarea[aria-label^=\"Editor content\"]'); return e ? e.value : null; })()" + }, + { + "action": "fill", + "text": "\n\n\n\n\n\n\nGIVE ME SPACE\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n

gimmiethat.space

\n\n", + "selector": "textarea[aria-label=\"Editor content;Press Alt+F1 for Accessibility Options.\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "Change browser tab title to GIVE ME SPACE", + "selector": "textarea[name=\"commit_message\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "get_text", + "selector": "textarea[aria-label^=\"Editor content\"]" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Commit changes\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__moli__task-441", + "model": "gpt-5.6-sol", + "browser": "moli", + "execution": 1, + "events_sha256": "c03872ea11c034863ab87fdefd40bf201e3e8d0af785c40f1b2bf10f77bde025", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "bf3b67b9f93f3fc74d3b374aaeccfbd3672653e766673d3641562e66da0bb2f7" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-163-trajectory-1", + "task_id": 163, + "task": { + "intent": "Get all review titles with 2 stars or below for the product on the current page.", + "intent_template_id": 136, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html" + }, + { + "action": "click", + "selector": "a[href=\"#reviews\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/review/product/listAjax/id/38805/?p=2\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "Memory Card Came Defective" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-163", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "c5cd7554294768e9670fdd45a363bd2b373cfc79352c2e3b11816d34fbafcf59", + "response_sha256": "e03c3465597c675088ccda122b463fda5e3174dcf5dbd0260d70cc4d2285c6df", + "evaluation_sha256": "bf530a12b1da141044d0139f8e15c4a2933720c23d9126fa4de98bc30d3c8524" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-320-trajectory-1", + "task_id": 320, + "task": { + "intent": "How much refund should I expect from my orders canceled, if any, in Feb 2023, including shipping fee.. Return the value as a number (e.g., 10.99) only, without any additional details.", + "intent_template_id": 160, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/view/order_id/156/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/view/order_id/158/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 406.53 + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-320__february-refunds", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory started another navigation before the order-detail click had finished navigating.", + "strategy": "Visit the two contract-bearing canceled orders by stable URLs with an explicit settling interval between documents." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-334-trajectory-1", + "task_id": 334, + "task": { + "intent": "Return the date I last ordered my muffin cornbread mix. Return the date in YYYY-MM-DD format or null if not available, without any additional details.", + "intent_template_id": 169, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/sales/order/view/order_id/180/" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "2023-03-11" + ], + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-334__last-cornbread-order", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory queried a pagination control before the order-history navigation had completed.", + "strategy": "Open the contract-bearing order directly and wait for the document before reading it." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-528-trajectory-1", + "task_id": 528, + "task": { + "intent": "Fill out the contact us form with this refund message for the phone screen protector bought March 2023: \"I need a refund. Order: . Amount: . Reason: It broke after three days of use.\" Leave the filled form ready for review and do not submit it.", + "intent_template_id": 154, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/contact" + }, + { + "action": "evaluate", + "script": "const values={\"input[name='name']\":'Emma Lopez',\"input[name='email']\":'emma.lopez@gmail.com',\"textarea[name='comment']\":'I need a refund. Order: 000000180. Amount: $12.99. Reason: It broke after three days of use.'}; for(const [selector,value] of Object.entries(values)){const el=document.querySelector(selector);if(!el)throw new Error(selector+' not found');el.value=value;el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));} return Object.fromEntries(Object.keys(values).map(selector=>[selector,document.querySelector(selector).value]));" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-528__refund-form-dom", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "contract_correction_id": "wv-528-dom-state-without-submit-20260911", + "evidence": "Official task contract, seeded March 2023 order, and corrected final-DOM acceptance contract" + }, + "repair_basis": { + "root_cause": "The upstream contract simultaneously required an artificial POST and prohibited the real contact-form POST, contradicting the instruction to leave the form unsubmitted.", + "strategy": "Open Contact Us directly, set the three required DOM values, and leave the native form unsubmitted for review." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-528-dom-state-without-submit-20260911", + "correction_source_sha256": "c9fc0e69b795fa4636926c2de68e349187ee45a0c6d7e5747c4bba46037de6f2", + "corrected_task_sha256": "6d3f1296cfc7a0288a66e3278e5ec285a3eb4ca614b37f2dc0a98124de8dd7e7" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-571-trajectory-1", + "task_id": 571, + "task": { + "intent": "I recently moved, my address is 231 Willow Way, Suite 100, Chicago, IL, 60601, update my information on OneStopShopping accordingly", + "intent_template_id": 165, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7770" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7770/customer/account/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"http://host.docker.internal:7770/customer/address/edit/id/26/\"]:visible, 1)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "231 Willow Way", + "selector": "input[name=\"street[0]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "Suite 100", + "selector": "input[name=\"street[1]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "23", + "selector": "select[name=\"region_id\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "Chicago", + "selector": "input[name=\"city\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "60601", + "selector": "input[name=\"postcode\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save Address\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-571", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "5d5af474be266a7f6f5fa383f91f25caae22e13e9ffb7f7392cec2d83d8d40ab", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "41e9331f4e3b80a990fb11832f215ee5834e7cbd7f336083d186a1cfa9f3b9b0" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-585-trajectory-1", + "task_id": 585, + "task": { + "intent": "Rate my recently purchased floor lamp with 5 stars using my nickname Emma Lopez, with the summary \"Good purchase\" and review \"I like it\"", + "intent_template_id": 194, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/quoizel-tf9404m-grove-park-tiffany-multi-color-floor-lamp.html" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "click", + "selector": "a[href=\"#reviews\"]", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "click", + "selector": "input[name=\"ratings[4]\"][value=\"20\"]", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "selector": "input[name=\"nickname\"]", + "text": "Emma Lopez", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "selector": "input[name=\"title\"]", + "text": "Good purchase", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "selector": "textarea[name=\"detail\"]", + "text": "I like it", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Submit Review\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-585__five-star-review", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory addressed the fifth visible rating input before the Reviews panel had rendered, making both timing and position part of the selector.", + "strategy": "Open the contract product, wait for the Reviews panel, and select the five-star value by its form value." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-510-trajectory-1", + "task_id": 510, + "task": { + "intent": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive. Choose any available variant.", + "intent_template_id": 216, + "revision": 2, + "sites": [ + "shopping" + ], + "start_urls": [ + "__SHOPPING__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const quantities=[...document.querySelectorAll('input[name^=\"cart[\"][name$=\"[qty]\"]')]; if(!quantities.length) return 'cart already empty'; for(const field of quantities) field.value='0'; const form=quantities[0].form; if(!form || !/checkout\\/cart\\/updatePost\\/?$/.test(form.action)) throw new Error('cart update form missing'); form.submit(); return `submitted removal of ${quantities.length} cart item(s)`; })()" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/atlantic-technology-fs-7-0-glb-7-channel-surround-bar-gloss-black-discontinued-by-manufacturer.html" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "click", + "selector": "#product-addtocart-button" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const rows=[...document.querySelectorAll('#shopping-cart-table tbody.cart.item')]; if(rows.length!==1) throw new Error(`expected one cart item, found ${rows.length}`); const name=(rows[0].querySelector('.product-item-name')?.textContent||rows[0].textContent||'').replace(/\\s+/g,' ').trim(); if(!name.includes('Atlantic Technology FS-7.0-GLB 7-channel Surround Bar')) throw new Error(`wrong cart item: ${name}`); return name; })()" + }, + { + "action": "click", + "selector": "button[data-role='proceed-to-checkout']" + }, + { + "action": "wait", + "seconds": 6 + }, + { + "action": "evaluate", + "script": "(() => { const shipping=document.querySelector('input[type=radio][name=ko_unique_1]') || document.querySelector('input[type=radio][value=flatrate_flatrate]'); if(shipping && !shipping.checked) shipping.click(); const next=document.querySelector('button[data-role=opc-continue]'); if(!next) throw new Error('shipping continue button missing'); next.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 6 + }, + { + "action": "evaluate", + "script": "(() => { const radio=document.querySelector('input[type=radio][value=checkmo]'); if(!radio) throw new Error('checkmo payment method missing'); if(!radio.checked) radio.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const place=[...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Place Order' && !e.disabled); if(!place) throw new Error('place order button missing'); place.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": ".checkout-success" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-c-510", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The autonomous attempts checked out a different product, and the authenticated fixture starts with unrelated items in its persistent cart; the evaluator requires a one-item cart containing product 75640.", + "strategy": "Clear the authenticated cart through the storefront form, add exactly one unit of product 75640, then complete checkout through stable DOM selectors." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-510-freeze-target-and-use-cart-dom-20260912", + "correction_source_sha256": "5a141413e71ad077c19515596b7b4a904fac427bdaa43de656a8f978aab7839e", + "corrected_task_sha256": "5fb4e1b3f5d734001981c6382a37e3a4ddfeda523f8da198e76bdc764108e7ad" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-464-trajectory-1", + "task_id": 464, + "task": { + "intent": "Update the product description of Antonia Racer Tank to \"{count} customer(s) love it!\" where count is the number of reviews with 4 stars or above, or \"don't miss out on this amazing product\" when there are no such reviews.", + "intent_template_id": 251, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/edit/id/1796/" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const title=document.querySelector('[data-index=\"content\"] > .fieldset-wrapper-title'); if(!title) throw new Error('Content accordion title not found'); const ko=require('ko'); const context=ko.contextFor(title); const viewModel=context&&context.$data; if(!viewModel||typeof viewModel.toggleOpened!=='function') throw new Error('Content accordion binding not found'); viewModel.toggleOpened(); return typeof viewModel.opened==='function'?viewModel.opened():true;" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "const fields=[...document.querySelectorAll('textarea')].filter(el=>el.name==='short_description'||el.name==='product[short_description]'); if(fields.length!==1) throw new Error('expected one live short_description field, found '+fields.length); const field=fields[0]; field.value='

1 customer(s) love it!

'; field.dispatchEvent(new Event('input',{bubbles:true})); field.dispatchEvent(new Event('change',{bubbles:true})); return {name:field.name,value:field.value};" + }, + { + "action": "evaluate", + "script": "const form=require('uiRegistry').get('product_form.product_form'); const source=form?.source; if(!source?.client||typeof source.client._save!=='function') throw new Error('Magento product data-source save component not found'); source.set('data.product.short_description','

1 customer(s) love it!

'); setTimeout(()=>source.client._save(source.get('data'),{redirect:false,ajaxSave:false,attributes:{id:form.namespace}}),500); return {component:source.name,action:'save'};" + }, + { + "action": "wait", + "seconds": 3 + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-464__short-description", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "task_revision": 2, + "evidence": "Official task contract and seeded product-review records for product 1796" + }, + "repair_basis": { + "root_cause": "The autonomous runs edited the long Page Builder description instead of the product short_description field required by the evaluator.", + "strategy": "Open product 1796 directly, set the short-description form control to the verified one-review result, and submit the native product form." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-547-trajectory-1", + "task_id": 547, + "task": { + "intent": "Add a new color option brown to the size S of Phoebe Zipper Sweatshirt", + "intent_template_id": 252, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/edit/id/1130/" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "evaluate", + "script": "(() => { const button=[...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Edit Configurations'); if(!button) throw new Error('Edit Configurations button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const dialog=[...document.querySelectorAll('aside[role=dialog]')].find(e=>e.offsetParent!==null); const next=[...(dialog||document).querySelectorAll('button')].find(e=>e.textContent.trim()==='Next' && !e.disabled); if(!next) throw new Error('configuration Next button missing'); next.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const boxes=[...document.querySelectorAll('input[type=checkbox]')]; const sizeValues=new Set(['55 cm','XS','65 cm','S','75 cm','M','6 foot','L','8 foot','XL','10 foot','28','29','30','31','32','33','34','36','38']); const colorValues=new Set(['Black','Blue','Brown','Gray','Green','Lavender','Multi','Orange','Purple','Red','White','Yellow']); const seen=[]; for(const box of boxes){const label=(box.labels?.[0]?.textContent||'').trim(); if(!sizeValues.has(label)&&!colorValues.has(label)) continue; const should=label==='S'||label==='Brown'; if(box.checked!==should) box.click(); if(should) seen.push(label);} if(!seen.includes('S')||!seen.includes('Brown')) throw new Error(`missing Size/Color options: ${seen.join(',')}`); const next=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Next'&&!e.disabled).at(-1); if(!next) throw new Error('Attribute Values Next button missing'); next.click(); return seen; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const price=document.querySelector('#apply-single-price-radio'); const quantity=document.querySelector('#apply-single-inventory-radio'); if(!price||!quantity) throw new Error('bulk price or inventory controls missing'); price.click(); quantity.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "(() => { const set=(name,value)=>{const field=document.querySelector(`input[name=${name}]`); if(!field) throw new Error(`missing ${name}`); field.value=value; field.dispatchEvent(new Event('input',{bubbles:true})); field.dispatchEvent(new Event('change',{bubbles:true}));}; set('ko_unique_7','59'); set('ko_unique_11','100'); const next=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Next'&&!e.disabled).at(-1); if(!next) throw new Error('Bulk Images and Price Next button missing'); next.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const button=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Generate Products'&&!e.disabled).at(-1); if(!button) throw new Error('Generate Products button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 5 + }, + { + "action": "evaluate", + "script": "(() => { const button=document.querySelector('#save-button')||[...document.querySelectorAll('button')].find(e=>e.offsetParent!==null&&e.textContent.trim()==='Save'); if(!button) throw new Error('parent Save button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-c-547", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Brown already exists as global color option 51; prior attempts created or edited a child but did not save the parent configurable-product matrix.", + "strategy": "Use the product configuration wizard to create and persist the missing S/Brown variation on parent product 1130." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-547-existing-brown-option-20260912", + "correction_source_sha256": "4506c41226bbe787123484e251f89116cbaa9770dd77b2949aedcaf5cf00433a", + "corrected_task_sha256": "492d978fb60cce180734859e2547b79ad385300ec02bfd30be57c89274856fe0" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-694-trajectory-1", + "task_id": 694, + "task": { + "intent": "Add a simple product named \"Energy-Bulk Women Shirt\" with 50 in stock, available in size S and color blue, priced at $60 using the appropriate attribute set.", + "intent_template_id": 256, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/new/set/9/type/simple/" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "evaluate", + "script": "(() => { const values={'product[name]':'Energy-Bulk Women Shirt','product[sku]':'energy-bulk-women-shirt','product[price]':'60','product[quantity_and_stock_status][qty]':'50','product[quantity_and_stock_status][is_in_stock]':'1','product[weight]':'1','product[size]':'167','product[color]':'50'}; for(const [name,value] of Object.entries(values)){const el=document.querySelector(`[name=\"${name}\"]`); if(!el) throw new Error(`missing field ${name}`); el.value=value; el.dispatchEvent(new Event('input',{bubbles:true})); el.dispatchEvent(new Event('change',{bubbles:true}));} const status=document.querySelector('input[name=\"product[status]\"]'); if(status && !status.checked) status.click(); return Object.fromEntries(Object.keys(values).map(name=>[name,document.querySelector(`[name=\"${name}\"]`).value])); })()" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "(() => { const button=document.querySelector('#save-button'); if(!button) throw new Error('save button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-c-694", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Earlier saves raced Magento dependency readiness; Moli also timed out while waiting for actionability on fields that were already present.", + "strategy": "Open the Top attribute-set simple-product form directly, set every evaluator-bearing field through DOM events, and invoke the save button through evaluate." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-699-trajectory-1", + "task_id": 699, + "task": { + "intent": "Create a new marketing price rule called \"spring sale\" for all registered customers that offers a 20 percent discount site-wide", + "intent_template_id": 258, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/sales_rule/promo_quote/new/" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "evaluate", + "script": "(() => { const setValue=(selector,value)=>{const el=document.querySelector(selector);if(!el)throw new Error(`missing ${selector}`);el.value=value;el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return el;}; setValue('input[name=name]','spring sale'); const website=document.querySelector('select[name=\"website_ids\"]'); if(!website) throw new Error('missing website_ids'); [...website.options].forEach(o=>o.selected=o.value==='1'); website.dispatchEvent(new Event('change',{bubbles:true})); const groups=document.querySelector('select[name=\"customer_group_ids\"]'); if(!groups) throw new Error('missing customer_group_ids'); [...groups.options].forEach(o=>o.selected=['1','2','3'].includes(o.value)); groups.dispatchEvent(new Event('change',{bubbles:true})); const active=document.querySelector('input[name=is_active]'); if(active && !active.checked) active.click(); return {website:[...website.selectedOptions].map(o=>o.value),groups:[...groups.selectedOptions].map(o=>o.value)}; })()" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "(() => { const panel=document.querySelector('[data-index=\"actions\"]'); const title=panel?.querySelector('.admin__collapsible-title') || [...document.querySelectorAll('strong')].find(e=>e.textContent.trim()==='Actions'); if(!title) throw new Error('Actions accordion missing'); title.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const action=document.querySelector('select[name=simple_action]'); const discount=document.querySelector('input[name=discount_amount]'); if(!action || !discount) throw new Error(`Actions fields missing: simple_action=${!!action}, discount_amount=${!!discount}`); action.value='by_percent'; action.dispatchEvent(new Event('change',{bubbles:true})); discount.value='20'; discount.dispatchEvent(new Event('input',{bubbles:true})); discount.dispatchEvent(new Event('change',{bubbles:true})); return {action:action.value,discount:discount.value}; })()" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "(() => { const button=document.querySelector('#save_and_apply') || [...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Save'); if(!button) throw new Error('save button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert-repair-c-699", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "Prior attempts created a Catalog Price Rule; the task and evaluator require a Cart Price Rule, and all registered groups are IDs 1, 2, and 3.", + "strategy": "Open the Cart Price Rule form directly, select website 1 and registered groups 1/2/3, choose percent discount, and save." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "corrected_replay_contract", + "correction_id": "wv-699-browser-form-array-fields-20260912", + "correction_source_sha256": "4af179b0fbd508d3aae239bf4315021bdcf8989460cae7b764df3cf3192fa0f6", + "corrected_task_sha256": "5f4c022c31dc69bcd038b3b38c6d4bcc92b1562049c5ef8a6ec22b4acac1894e" + }, + "model_calls": 0, + "receipt_run_ids": [ + "2904cf117286" + ] + } + }, + { + "id": "webarena-verified-771-trajectory-1", + "task_id": 771, + "task": { + "intent": "Approve reviews with four stars or higher to display in our store.", + "intent_template_id": 243, + "revision": 2, + "sites": [ + "shopping_admin" + ], + "start_urls": [ + "__SHOPPING_ADMIN__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/review/product/pending/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/review/product/edit/id/353/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(() => { const f=document.querySelector('form'); return {checked:[...document.querySelectorAll('input[type=radio]')].map(x=>({name:x.name,value:x.value,checked:x.checked,outer:x.outerHTML})), ratings:[...document.querySelectorAll('[class*=rating]')].map(x=>({tag:x.tagName,cls:x.className,text:x.innerText,html:x.innerHTML.slice(0,500)}))}; })()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/352/" + }, + { + "action": "evaluate", + "script": "(()=>({rating:[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id,status:document.querySelector('[name=status_id]')?.value}))()" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"status_id\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save Review\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/351/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/349/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/347/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"status_id\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save Review\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/pending/" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__moli__task-771", + "model": "gpt-5.6-sol", + "browser": "moli", + "execution": 1, + "events_sha256": "37c14caa1a034341e11a34a7bd7fccc51c509ca55a82732394f0185580719f86", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "f7d73fb103ab4d8bb64e63f13c76c053cca33fd78aa84f087e8baaf669b78818" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-580-trajectory-1", + "task_id": 580, + "task": { + "intent": "Create a new forum with name and title \"sci_fi\", description \"A wild place for sci-fi enthusiasts\", and sidebar \"**New**, Classic, *Movies*, Post my novel, Random\"", + "intent_template_id": 7, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/create_forum\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "sci_fi", + "selector": "input[name=\"forum[name]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "sci_fi", + "selector": "input[name=\"forum[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "A wild place for sci-fi enthusiasts", + "selector": "textarea[name=\"forum[description]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "**New**, Classic, *Movies*, Post my novel, Random", + "selector": "textarea[name=\"forum[sidebar]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create forum\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-580", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "222e9f7f4f52d31156694dee79d0f0e252ee5d86f07345ce7dc5a34144625145", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "7a80c8e435ab4e173e967882a7c6c892815a33d09cf8fab229b7147c0a0c7c0f" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-610-trajectory-1", + "task_id": 610, + "task": { + "intent": "Post a review of my recent reading, titled \"To Kill a Mockingbird by Harper Lee\" in f/books and then comment \"good book!\" on the created post", + "intent_template_id": 9, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/submit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "To Kill a Mockingbird by Harper Lee", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "10037", + "selector": "select[name=\"submission[forum]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1500 + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "good book!", + "selector": "textarea[name=\"reply_to_submission_2[comment]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Post\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1200 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__moli__task-610", + "model": "gpt-5.6-sol", + "browser": "moli", + "execution": 1, + "events_sha256": "50e3008fd411dc269b2ecbc3e51cc86745607e3b90b22e34585316d6060ad750", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "5344844f668e97c98d7a37f0bb4d4bba1f9d556fe63f5dad3c57053785239c94" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-615-trajectory-1", + "task_id": 615, + "task": { + "intent": "Re-post the image of Bald Eagle from this forum to connecticut forum using the image URL and title \"from /f/pics\"", + "intent_template_id": 11, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__/f/pics" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999/f/pics" + }, + { + "action": "click", + "selector": "a[href=\"/f/pics/110715/amazing-shot-of-a-blue-jay-pestering-a-bald-eagle\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "() => document.querySelector('main article header h1 a').href" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:9999/submit/connecticut" + }, + { + "action": "fill", + "text": "http://192.168.5.2:9999/submission_images/b02113033af32feae9ff147dbbe3764039368d67d193885bd04e65c2e6beea9c.jpg", + "selector": "input[name=\"submission[url]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "from /f/pics", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-615", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "e2f0d470e6df98f3b360d52e9563929ecc5250a328c07389c01602941752c1ca", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "0070d0cdcd78359932c360ac08d67e1712b110bac231537a15f589c2775fe430" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-640-trajectory-1", + "task_id": 640, + "task": { + "intent": "Post a notice in f/books titled \"book reading Meet up!\". Set post details to \"virtual meetup for book reading on March 15th\"", + "intent_template_id": 16, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/submit/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "book reading Meet up!", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "virtual meetup for book reading on March 15th", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-640", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "1f257cfc9f15b416694d7a34273efbc0c35e86ee64c44a5af1593a13dfe5cde7", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "8117560e00cdf1162583df6b99216f498a97adf7a72362833c4f8ec4ebf91cd3" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-719-trajectory-1", + "task_id": 719, + "task": { + "intent": "Like all submissions created by ThetaGang_wsb in forum wallstreetbets", + "intent_template_id": 25, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "fill", + "text": "ThetaGang_wsb", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:9999/user/ThetaGang_wsb" + }, + { + "action": "click", + "selector": "a[href=\"/user/ThetaGang_wsb/submissions\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": ":nth-match(button:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-719", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "4b1c2c8a46353ef6bf8c7249a45376ad6576f248c558c93f22f08b4db9ede35c", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "039901b5db2c20877a8e3a08ca69515d3046e962afbc160c52d97656de8d36c2" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-725-trajectory-1", + "task_id": 725, + "task": { + "intent": "DisLike all submissions created by RickyDontLoseThat in forum massachusetts", + "intent_template_id": 1510, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/user/RickyDontLoseThat/submissions" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const article=[...document.querySelectorAll('main article')].find(e=>e.querySelector('a[href=\"/f/massachusetts\"]')); const button=article?.querySelector('form button:has(img[alt=\"down\"])'); if(!button) throw new Error('massachusetts submission downvote control not found'); button.click(); return {post:article.querySelector('h1,h2')?.textContent.trim()};" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-725__massachusetts-submissions-downvote", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory started a forum navigation while the preceding forums click was still navigating and later addressed the downvote button by page position.", + "strategy": "Open the author's submissions directly, bind the target article to its massachusetts forum link, and click that article's downvote control." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-731-trajectory-1", + "task_id": 731, + "task": { + "intent": "Edit my post on Nvidia RTX 4090 by adding a line to the body that says \"EDIT: This news aged well\"", + "intent_template_id": 27, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "fill", + "text": "Nvidia RTX 4090", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/MachineLearning/1/-/edit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "Crazy device for ML!\n\nEDIT: This news aged well", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Edit submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-731", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "aaa017895be2ca14837144f58e9ff75cc508e49553687fa7e6ec50ca45ef3f74", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "23d569a765d8246d28287610ae6ae6b30e562ed02c744a32157a9709deaffc17" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-29-trajectory-1", + "task_id": 29, + "task": { + "intent": "In the DIY forum, get the username and post title of the most recent post, and count the number of comments on that post that are not from the author and have more downvotes than upvotes. Return a list of objects with keys \"username\", \"post_title\", and \"count\".", + "intent_template_id": 33, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:9999/f/DIY" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/DIY/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "username": "ziostraccette", + "post_title": "How can I bring an HDMI cable from my pc downstairs to my TV upstairs?", + "count": 0 + } + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-29", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "d45ad8af17b459778a306b4248985922d203f0a39ed1a3daddaea56391c3e558", + "response_sha256": "a0bb7efc94340a501b9ca76425c1a118d86a128c0fdf2c3fd150dadb211df07f", + "evaluation_sha256": "78211058a546985022e4a35b4363bcd51ca61785717134e0458a56322cfafb5c" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-635-trajectory-1", + "task_id": 635, + "task": { + "intent": "Create a post in the most appropriate forum. Title it \"noise-cancelling headphones recommendations\" and in post details ask \"I need recommendations for noise-cancelling headphones within a budget of $200 please\"", + "intent_template_id": 6100, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "headphones", + "selector": "input[name=\"q\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "press", + "key": "Enter" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/submit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "noise-cancelling headphones recommendations", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "I need recommendations for noise-cancelling headphones within a budget of $200 please", + "selector": "textarea[name=\"submission[body]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "10050", + "selector": "select[name=\"submission[forum]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-635", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "48637fa9879333955635b0fb06c772f4651dcb2cdbb0478f4cc3d3767872c802", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "5d8aa27f8653fcc6967b375ad325281a67db4aaeea5db831d9a996b3fbcb881d" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-600-trajectory-1", + "task_id": 600, + "task": { + "intent": "Post my question with the title \"what is the recommended console to buy these days?\", in a forum where I'm likely to get an answer", + "intent_template_id": 3765, + "revision": 2, + "sites": [ + "reddit" + ], + "start_urls": [ + "__REDDIT__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:9999/submit/consoles" + }, + { + "action": "fill", + "selector": "textarea[name=\"submission[title]\"]", + "text": "what is the recommended console to buy these days?", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-600__fresh-console-question", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory navigated away while a gaming submission-form click was still completing, before it reached the intended consoles form.", + "strategy": "Load the consoles submission form directly and submit the exact title immediately." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + }, + { + "id": "webarena-verified-349-trajectory-1", + "task_id": 349, + "task": { + "intent": "Get the usernames of other users who have access to my repo gimmiethat.space", + "intent_template_id": 298, + "revision": 3, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"/byteblaze/gimmiethat.space\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/byteblaze/gimmiethat.space/-/project_members" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "yjlou" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-349", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "e4d03b360807c4e00fdf28fea8c873c1e3d52fcc56398f093aaa044128189f3f", + "response_sha256": "f97261dea0353bfbf196f251d69704cb1306c77001e01414b8fa27a23f8bf384", + "evaluation_sha256": "26bdfc4b29c6d675206e0ebb5e7688543b306ec5a164459397363a8cc51d6cac" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-784-trajectory-1", + "task_id": 784, + "task": { + "intent": "Get the email address of the contributor who has the most commits to branch master in the current repository", + "intent_template_id": 316, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/CellularPrivacy/Android-IMSI-Catcher-Detector" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/CellularPrivacy/Android-IMSI-Catcher-Detector" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/CellularPrivacy/Android-IMSI-Catcher-Detector/-/graphs/master/charts" + }, + { + "action": "evaluate", + "script": "JSON.stringify(performance.getEntriesByType('resource').map(x=>x.name).filter(x=>x.includes('graphs')||x.includes('chart')||x.includes('commit')))" + }, + { + "action": "evaluate", + "script": "document.documentElement.innerHTML.includes('commit_data') ? document.documentElement.innerHTML.slice(document.documentElement.innerHTML.indexOf('commit_data')-500,document.documentElement.innerHTML.indexOf('commit_data')+1000) : document.documentElement.innerHTML.match(/data-[^=]+='[^']{100,}'/g)?.slice(0,10)" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100" + }, + { + "action": "get_text", + "selector": "pre" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100&page=2" + }, + { + "action": "get_text", + "selector": "pre" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/api/v4/projects/61/repository/contributors?ref=master&per_page=100&page=3" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "SecUpwN@users.noreply.github.com" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-784", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "a8f613a4972fa09661766a1d142ce5465629d7eb70fc76fd763cefd62dae9c31", + "response_sha256": "ead8795236d5df5d6b2e57260b6a982f6811245d9522d04850865b42f9cc2ff7", + "evaluation_sha256": "753c1b34dc2b4c0578b9ef17e243ef3d5dc17b849418b5ae463ebae86cf2e7d7" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-303-trajectory-1", + "task_id": 303, + "task": { + "intent": "How many commits did Kilian make during 2023 in the current repository?", + "intent_template_id": 321, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/a11yproject/a11yproject.com" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/graphs/main/charts?from=2023-01-01&to=2023-12-31" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/a11yproject/a11yproject.com/-/commits/main?author=Kilian&since=2023-01-01T00%3A00%3A00Z&until=2023-12-31T23%3A59%3A59Z" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + 1 + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-303", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "6c012e5222af2ee572100e734478ebd231c775bfe3360ab0109d90c056cfe6b1", + "response_sha256": "48c0303ddc0b3f1bafee1b0e88a40f218798e3024cb0137200688306e64e9a59", + "evaluation_sha256": "ab21ae38549b911a7f435f56d934476069a27c33fb69f41b691d148596ba8c0a" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-314-trajectory-1", + "task_id": 314, + "task": { + "intent": "Get the full names of the top 3 contributors (by commit count) to primer/design repo", + "intent_template_id": 324, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"/primer/design\"]:visible, 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design/-/graphs/main/charts" + }, + { + "action": "scroll", + "direction": "down" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design/-/graphs/main" + } + ], + "evaluation_input": { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + "Shawn Allen", + "Inayaili León", + "Aurora Pleguezuelo" + ], + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-314", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "4bc55b8f0ace842ae720765cae5c84bdd227a399e6e6ce30742436ea5909b3d1", + "response_sha256": "b41e6cbb9c76364be698488998edf5291a7ff93f6f6fb15c9e3079f7797c805c", + "evaluation_sha256": "e17a6f649b7762962ba7bafa0e0d988a17122795e38b651b1e65fd12a104b7f8" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-658-trajectory-1", + "task_id": 658, + "task": { + "intent": "Create an issue in a11yproject repo with title \"401 bad gateway\". Assign the issue to Roshanjossey (you can invite users to the project if needed). Set due date to be the end of 2030", + "intent_template_id": 327, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023" + }, + { + "action": "click", + "selector": "a[href=\"/a11yproject/a11yproject.com/-/issues\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/a11yproject/a11yproject.com/-/issues/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "401 bad gateway", + "selector": "input[name=\"issue[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "2030-12-31", + "selector": "input[name=\"issue[due_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Unassigned\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create issue\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-658", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "fff5eb90cdaf06e9dd9dededcb99a11747d1e02a50d7c9c79f116b90fa74396e", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "da32ff219fba62d6b783eaae0aa173fe6fe027233aa883f24bfc42603ab0963e" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-666-trajectory-1", + "task_id": 666, + "task": { + "intent": "Submit a merge request for dialog-component in the current repository to be merged into dialog branch, assign Primer as the reviewer", + "intent_template_id": 335, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/primer/design" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/merge_requests\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/merge_requests/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Select source branch\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 11)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"main\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 10)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Compare branches and continue\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(button:visible:has-text(\"Unassigned\"), 2)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 5)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create merge request\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "get_text", + "selector": "body" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-666", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "b6ffaebf0c4c1eddb26c26f11e11ab9a6622c73d67ce052dee7949b0e76e124e", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "4f2b0895d07fa0d02ba721cf98d7cac5fd783ffa1921fb3bf2267074d1ef3620" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-590-trajectory-1", + "task_id": 590, + "task": { + "intent": "Create a milestone in the current repo with title \"product launch\" for the upcoming event of product launch starting on January 16, 2023 and ending on January 30, 2023", + "intent_template_id": 339, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__/primer/design" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/issues\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://192.168.5.2:8023/primer/design/-/milestones" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/milestones/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "product launch", + "selector": "input[name=\"milestone[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "2023-01-16", + "selector": "input[name=\"milestone[start_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "fill", + "text": "2023-01-30", + "selector": "input[name=\"milestone[due_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create milestone\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "authoritative_successful_browser_use_execution", + "candidate_id": "gpt-5.6-sol__chromium__task-590", + "model": "gpt-5.6-sol", + "browser": "chromium", + "execution": 1, + "events_sha256": "1ee105deb614907c0fc4ec4ce74c4a94a9cef43fa49537fec1460cea5aca4bae", + "response_sha256": "db765f7e78b6c552e25494cde987b92c12e4d11d6990604b328f604c66bbf9c7", + "evaluation_sha256": "2dd47c26e79713a107c31acd1a8336e1a159d69042938c210a323df9f905a5d6" + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "ab8cbb4dd809" + ] + } + }, + { + "id": "webarena-verified-576-trajectory-1", + "task_id": 576, + "task": { + "intent": "Add the following users to repo a11y-webring.club as developer: \"abisubramanya27\" and \"lahwaacz\"", + "intent_template_id": 351, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:8023/byteblaze/a11y-webring.club/-/project_members" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Invite members\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const heading=[...document.querySelectorAll('h4')].find(e=>e.textContent.trim()==='Invite members'); const root=heading?.closest('[role=dialog],.modal,.gl-modal')||heading?.parentElement?.parentElement; const input=[...(root?.querySelectorAll('input[type=text]')||[])].find(e=>e.placeholder!=='YYYY-MM-DD'); if(!input) throw new Error('member lookup input not found'); const setter=Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value').set; setter.call(input,'abisubramanya27'); input.dispatchEvent(new Event('input',{bubbles:true})); return true;" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const button=[...document.querySelectorAll('button[role=menuitem],button')].find(e=>e.offsetParent!==null&&e.textContent.includes('abisubramanya27')); if(!button) throw new Error('abisubramanya27 suggestion not found'); button.click(); return true;" + }, + { + "action": "evaluate", + "script": "const heading=[...document.querySelectorAll('h4')].find(e=>e.textContent.trim()==='Invite members'); const root=heading?.closest('[role=dialog],.modal,.gl-modal')||heading?.parentElement?.parentElement; const input=[...(root?.querySelectorAll('input[type=text]')||[])].find(e=>e.placeholder!=='YYYY-MM-DD'); if(!input) throw new Error('member lookup input not found'); const setter=Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value').set; setter.call(input,'lahwaacz'); input.dispatchEvent(new Event('input',{bubbles:true})); return true;" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "const button=[...document.querySelectorAll('button[role=menuitem],button')].find(e=>e.offsetParent!==null&&e.textContent.includes('lahwaacz')); if(!button) throw new Error('lahwaacz suggestion not found'); button.click(); return true;" + }, + { + "action": "select", + "selector": ".modal select:visible", + "value": "30", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": ":nth-match(button:visible:has-text(\"Invite\"), 3)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-576__invite-two-developers", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The selected trajectory addressed the third visible text input, which was the expiration-date field after reset, then waited for a user menu that could never appear.", + "strategy": "Scope member lookup to the Invite members dialog, select each named user by text, set Developer, and submit once." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "6bd92e821d69" + ] + } + }, + { + "id": "webarena-verified-415-trajectory-1", + "task_id": 415, + "task": { + "intent": "Reply on the merge request assigned to me for verification functions: if the last comment is from the author, reply \"Thank you\"; otherwise tag the author (e.g., @user) as a reminder.", + "intent_template_id": 360, + "revision": 2, + "sites": [ + "gitlab" + ], + "start_urls": [ + "__GITLAB__" + ] + }, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:8023/byteblaze/a11y-webring.club/-/merge_requests/40" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "fill", + "selector": "textarea[name=\"note[note]\"]:visible", + "text": "@davepgreene", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Comment\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "evaluation_input": { + "task_type": "MUTATE", + "status": "SUCCESS", + "retrieved_data": null, + "error_details": null + }, + "source": { + "kind": "expert_repair_spec", + "candidate_id": "expert__task-415__verification-reminder", + "repair_spec_sha256": "ea34996acd3f137da912dcc386858c2ab51b86514cacc23502d1448224bbb830", + "details": { + "kind": "reviewed_task_contract_and_existing_browser_evidence", + "task_revision": 2 + }, + "repair_basis": { + "root_cause": "The comment became visible in the page, but the trajectory finalized evidence immediately after the click, before the required notes POST was captured.", + "strategy": "Open the contract merge request directly, submit the required author reminder, and allow the request to settle before evidence finalization." + } + }, + "verification": { + "browsers": [ + "chromium" + ], + "process_status": "pass", + "evaluator_status": "pass", + "contract": { + "kind": "official_task_contract" + }, + "model_calls": 0, + "receipt_run_ids": [ + "13f17cee19ab" + ] + } + } + ], + "revision": { + "schema": "webarena_replay_dataset_revision/1", + "base_dataset_sha256": "61bb38bce552b5c7eea0e1f5fcca1bc898f55e2e39ee6e72498cff346ca1e4e3", + "correction_catalog_sha256": "25619b9e0345c49a79e579a88d7f81280870119e4ab4e8d72bd0a415b9de60b0", + "changed_tasks": [ + { + "task_id": 67, + "correction_id": "fork-67-books-description-only", + "correction_record_sha256": "7057e6eb5cfd9c49d23d1984b9fce96e8ea13f68a4698dba5cef0eb70739bef2", + "original_task_sha256": "03dbaee9f03fac5aa766b36b1004236ff9ebea9d5544816c7e3108817db2a98e", + "corrected_task_sha256": "2d92caf6476a5fb9370aef62da974343e3b1bd10993cf08dbcdbffa1a114aacf", + "task_contract_sha256": "32d581ee7901a5647b53304aa147a5e74f98b1f7a7d1d7259444fb38c4a31408" + }, + { + "task_id": 225, + "correction_id": "wv-225-explicit-title-scope-20260918", + "correction_record_sha256": "eec22076f2d0ab06f8221f097b3cba1e50e935d7ccd1e61878429f0e0abc31ca", + "original_task_sha256": "463f9647d0d033794c2a4e41700d6c4d6b9ea1654943641d7fbaca0254046c90", + "corrected_task_sha256": "6fc98d838713546fa75b019a0ec77e256402d51c49594b73c344986148282f4c", + "task_contract_sha256": "a8b6a85f2ebe49767f353bf4b6380aa680cb91106af1f90079a45064bb45a484" + }, + { + "task_id": 243, + "correction_id": "fork-243-review-nickname", + "correction_record_sha256": "2779ea13532610de196ed6977a6518cd56545eca51b6e9a18289f52f0bf60cc9", + "original_task_sha256": "219b7055268f0e21c1900a0f5c03afa5b0a85aa966fff9dd9e4f1302cc5057e6", + "corrected_task_sha256": "538fa655a6a69a78ac66fa2524d7117b2b2ee0583b2fab3bebe7acf4f583a47d", + "task_contract_sha256": "6a340931057fa929aeafb91739d39e114486d734f5bf13c3beaf8eb63649574d" + } + ], + "qualification_status": "pending_full_scope_reference_requalification" + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-404.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-404.json new file mode 100644 index 000000000..c33665b5c --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-404.json @@ -0,0 +1,63 @@ +{ + "schema": "webarena_qualified_trajectory_revision/1", + "task_id": 404, + "canonical_calls_sha256": "477ba17cd281b03681fb6070402ef31d4a88eceb292a8871cb2645095fe8be14", + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-reddit.dev.lexmount.net" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/books/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "selector": "button[aria-label=\"Sort by: New\"]:visible", + "timeout": 30000 + }, + { + "action": "evaluate", + "script": "(async () => { if (document.readyState !== 'complete') await new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('vote_page_load_timeout')), 15000); window.addEventListener('load', () => { clearTimeout(timer); resolve(); }, {once: true}); }); await new Promise(resolve => setTimeout(resolve, 500)); const form = document.querySelector('article:has(a[href^=\"/f/books/124260/\"]) form[data-controller~=\"vote\"]'); if (!form || !form.querySelector('button[data-action=\"vote#up\"]')) throw new Error('target_vote_controller_missing'); return {ready_state: document.readyState, target_post_id: 124260, vote_controller_declared: true}; })()" + }, + { + "action": "click", + "selector": "article:has(a[href^=\"/f/books/124260/\"]) form button:has(img[alt=\"up\"]):visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 2 + } + ], + "repair": { + "cause": "The observer entered the old execution context while the New-list navigation was pending.", + "change": "Wait for the destination-specific visible sort control before observing the vote controller." + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-464.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-464.json new file mode 100644 index 000000000..01ee46781 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-464.json @@ -0,0 +1,37 @@ +{ + "schema": "webarena_qualified_trajectory_revision/1", + "task_id": 464, + "canonical_calls_sha256": "582dec5936d0e10a0752a2c4e2af0f164074fab5780cfce6151bc12c26d7ee78", + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-shopping-admin.dev.lexmount.net/admin/catalog/product/edit/id/1796/" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "const title=document.querySelector('[data-index=\"content\"] > .fieldset-wrapper-title'); if(!title) throw new Error('Content accordion title not found'); const ko=require('ko'); const context=ko.contextFor(title); const viewModel=context&&context.$data; if(!viewModel||typeof viewModel.toggleOpened!=='function') throw new Error('Content accordion binding not found'); viewModel.toggleOpened(); return typeof viewModel.opened==='function'?viewModel.opened():true;" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "const fields=[...document.querySelectorAll('textarea')].filter(el=>el.name==='short_description'||el.name==='product[short_description]'); if(fields.length!==1) throw new Error('expected one live short_description field, found '+fields.length); const field=fields[0]; const value='

1 customer(s) love it!

'; field.value=value; field.dispatchEvent(new Event('input',{bubbles:true})); field.dispatchEvent(new Event('change',{bubbles:true})); const form=require('uiRegistry').get('product_form.product_form'); const source=form?.source; if(!source||typeof source.set!=='function') throw new Error('Magento product data source not found'); source.set('data.product.short_description',value); if(source.get('data.product.short_description')!==value) throw new Error('product data source did not accept value'); return {name:field.name,value,component:source.name};" + }, + { + "action": "click", + "selector": "button:has-text(\"Save\"):not(:has-text(\"Save options\")):visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "selector": "div.message-success:visible", + "timeout": 30000 + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-510.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-510.json new file mode 100644 index 000000000..cdc27d644 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-510.json @@ -0,0 +1,92 @@ +{ + "task_id": 510, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const quantities=[...document.querySelectorAll('input[name^=\"cart[\"][name$=\"[qty]\"]')]; if(!quantities.length) return 'cart already empty'; for(const field of quantities) field.value='0'; const form=quantities[0].form; if(!form || !/checkout\\/cart\\/updatePost\\/?$/.test(form.action)) throw new Error('cart update form missing'); form.submit(); return `submitted removal of ${quantities.length} cart item(s)`; })()" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/atlantic-technology-fs-7-0-glb-7-channel-surround-bar-gloss-black-discontinued-by-manufacturer.html" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "click", + "selector": "#product-addtocart-button" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7770/checkout/cart/" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const rows=[...document.querySelectorAll('#shopping-cart-table tbody.cart.item')]; if(rows.length!==1) throw new Error(`expected one cart item, found ${rows.length}`); const name=(rows[0].querySelector('.product-item-name')?.textContent||rows[0].textContent||'').replace(/\\s+/g,' ').trim(); if(!name.includes('Atlantic Technology FS-7.0-GLB 7-channel Surround Bar')) throw new Error(`wrong cart item: ${name}`); const fields=[...rows[0].querySelectorAll('input[name^=\"cart[\"][name$=\"[qty]\"]')]; if(fields.length!==1) throw new Error('expected one cart quantity field, found '+fields.length); const raw=fields[0].value; const quantity=Number(raw); if(!raw.trim()||quantity!==1) throw new Error('expected cart quantity 1, found '+raw); return {name,quantity}; })()" + }, + { + "action": "click", + "selector": "button[data-role='proceed-to-checkout']" + }, + { + "action": "wait", + "seconds": 6 + }, + { + "action": "evaluate", + "script": "(() => { const shipping=document.querySelector('input[type=radio][name=ko_unique_1]') || document.querySelector('input[type=radio][value=flatrate_flatrate]'); if(shipping && !shipping.checked) shipping.click(); const next=document.querySelector('button[data-role=opc-continue]'); if(!next) throw new Error('shipping continue button missing'); next.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 6 + }, + { + "action": "evaluate", + "script": "(() => { const radio=document.querySelector('input[type=radio][value=checkmo]'); if(!radio) throw new Error('checkmo payment method missing'); if(!radio.checked) radio.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 3 + }, + { + "action": "evaluate", + "script": "(() => { const place=[...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Place Order' && !e.disabled); if(!place) throw new Error('place order button missing'); place.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": ".checkout-success" + } + ], + "repair": { + "cause": "A single cart row can contain multiple units. The frozen DOM assertion checked row count and product name but not quantity after the totals evaluator was removed.", + "change": "Preserve the original task-510 trajectory and add a fail-closed quantity=1 check to its pre-checkout cart assertion. This candidate is not a new measured score.", + "predecessor_dataset_sha256": "624a0ea83c6e4c3533714e961656d7e4159189a9e4941017b06a5299e4dc31d5", + "changed_step": 11, + "qualification": "Local DOM counterexamples required; full site qualification pending. The frozen cohort is unchanged." + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-547.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-547.json new file mode 100644 index 000000000..e6225b747 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-547.json @@ -0,0 +1,77 @@ +{ + "task_id": 547, + "tool_calls": [ + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/catalog/product/edit/id/1130/" + }, + { + "action": "wait", + "seconds": 4 + }, + { + "action": "evaluate", + "script": "(() => { const button=[...document.querySelectorAll('button')].find(e=>e.textContent.trim()==='Edit Configurations'); if(!button) throw new Error('Edit Configurations button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(async()=>{\nconst deadline=Date.now()+30000;\nconst visible=e=>{const s=getComputedStyle(e);return s.visibility!=='hidden'&&s.display!=='none'&&e.getClientRects().length>0};\nlet step,dialog,nextButton;\nwhile(Date.now(){if(typeof p.doSelectSavedAttributes==='function')components.push(p)});\n if(dialogs.length===1&&components.length===1){\n const p=components[0],selected=p.multiselect?.()?.selected?.();\n const labels=Array.isArray(selected)?selected.map(id=>p.attributesLabels?.[id]).sort():[];\n if(p.stepInitialized===true&&labels.join(',')==='Color,Size'){\n const buttons=[...dialogs[0].querySelectorAll('button')].filter(e=>visible(e)&&e.textContent.trim()==='Next'&&!e.disabled);\n if(buttons.length===1&&typeof jQuery==='function'&&requirejs.defined('ko')){\n const button=buttons[0],owner=require('ko').contextFor(button)?.$data;\n const handlers=jQuery._data(button,'events')?.click;\n if(owner?.wizard===p.wizard&&owner?.wizard?.getStep()===p&&\n owner.selectedStep?.()===p.name&&Array.isArray(handlers)&&handlers.length>0){\n step=p;dialog=dialogs[0];nextButton=button;break;\n }\n }\n }\n }\n await new Promise(r=>setTimeout(r,100));\n}\nif(!step||!dialog||!nextButton)throw Error('configuration attributes or Next event binding not initialized');\nnextButton.click();\nif(step.wizard.index!==1)throw Error('configuration Next did not advance: '+JSON.stringify({index:step.wizard.index,notification:step.notificationMessage}));\nwhile(Date.now()(e.labels?.[0]?.textContent||'').trim());\n if(labels.includes('S')&&labels.includes('Brown'))return {attributes:['Color','Size'],values_ready:true};\n await new Promise(r=>setTimeout(r,100));\n}\nthrow Error('configuration Attribute Values did not become ready');\n})()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const boxes=[...document.querySelectorAll('input[type=checkbox]')]; const sizeValues=new Set(['55 cm','XS','65 cm','S','75 cm','M','6 foot','L','8 foot','XL','10 foot','28','29','30','31','32','33','34','36','38']); const colorValues=new Set(['Black','Blue','Brown','Gray','Green','Lavender','Multi','Orange','Purple','Red','White','Yellow']); const seen=[]; for(const box of boxes){const label=(box.labels?.[0]?.textContent||'').trim(); if(!sizeValues.has(label)&&!colorValues.has(label)) continue; const should=label==='S'||label==='Brown'; if(box.checked!==should) box.click(); if(should) seen.push(label);} if(!seen.includes('S')||!seen.includes('Brown')) throw new Error(`missing Size/Color options: ${seen.join(',')}`); const next=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Next'&&!e.disabled).at(-1); if(!next) throw new Error('Attribute Values Next button missing'); next.click(); return seen; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const price=document.querySelector('#apply-single-price-radio'); const quantity=document.querySelector('#apply-single-inventory-radio'); if(!price||!quantity) throw new Error('bulk price or inventory controls missing'); price.click(); quantity.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 1 + }, + { + "action": "evaluate", + "script": "(() => { const set=(name,value)=>{const field=document.querySelector(`input[name=${name}]`); if(!field) throw new Error(`missing ${name}`); field.value=value; field.dispatchEvent(new Event('input',{bubbles:true})); field.dispatchEvent(new Event('change',{bubbles:true}));}; set('ko_unique_7','59'); set('ko_unique_11','100'); const next=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Next'&&!e.disabled).at(-1); if(!next) throw new Error('Bulk Images and Price Next button missing'); next.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "evaluate", + "script": "(() => { const button=[...document.querySelectorAll('button')].filter(e=>e.textContent.trim()==='Generate Products'&&!e.disabled).at(-1); if(!button) throw new Error('Generate Products button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 5 + }, + { + "action": "evaluate", + "script": "(() => { const button=document.querySelector('#save-button')||[...document.querySelectorAll('button')].find(e=>e.offsetParent!==null&&e.textContent.trim()==='Save'); if(!button) throw new Error('parent Save button missing'); button.click(); return true; })()" + }, + { + "action": "wait", + "seconds": 8 + }, + { + "action": "get_text", + "selector": "main" + } + ], + "repair": { + "cause": "Attribute selection can initialize before the visible Next control has its Knockout click handler and matching wizard owner.", + "change": "Wait for selected Color/Size, matching active wizard and bound Next click handler; click once and verify actual wizard advancement and values." + } +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-590.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-590.json new file mode 100644 index 000000000..d02ee711c --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-590.json @@ -0,0 +1,82 @@ +{ + "task_id": 590, + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-gitlab.dev.lexmount.net/primer/design" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/issues\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-gitlab.dev.lexmount.net/primer/design/-/milestones" + }, + { + "action": "click", + "selector": "a[href=\"/primer/design/-/milestones/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "product launch", + "selector": "input[name=\"milestone[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async name => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (!el || el.disabled) throw new Error('Date input is unavailable');\n if (document.activeElement !== el) { el.focus(); stableSince = null; }\n else {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n }\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input focus did not settle');\n})(\"milestone[start_date]\")" + }, + { + "action": "fill", + "text": "2023-01-16", + "selector": "input[name=\"milestone[start_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async ({name, expected}) => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (el && el.value === expected && document.activeElement === el) {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n } else stableSince = null;\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input did not retain its value and focus');\n})({\"name\": \"milestone[start_date]\", \"expected\": \"2023-01-16\"})" + }, + { + "action": "evaluate", + "script": "(async name => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (!el || el.disabled) throw new Error('Date input is unavailable');\n if (document.activeElement !== el) { el.focus(); stableSince = null; }\n else {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n }\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input focus did not settle');\n})(\"milestone[due_date]\")" + }, + { + "action": "fill", + "text": "2023-01-30", + "selector": "input[name=\"milestone[due_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async ({name, expected}) => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (el && el.value === expected && document.activeElement === el) {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n } else stableSince = null;\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input did not retain its value and focus');\n})({\"name\": \"milestone[due_date]\", \"expected\": \"2023-01-30\"})" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(expected => { for (const [name, value] of Object.entries(expected)) { const el = document.querySelector(`input[name=\"${name}\"]`); if (!el || el.value !== value) throw new Error('Date field differs before submit: ' + name); } return {completion:'date_fields_verified', fields:expected}; })({\"milestone[start_date]\": \"2023-01-16\", \"milestone[due_date]\": \"2023-01-30\"})" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create milestone\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const started = Date.now();\n const sameOrigin = location.origin;\n const read = async (path) => {\n const response = await fetch(path, {method: 'GET', credentials: 'same-origin',\n redirect: 'error', cache: 'no-store'});\n if (response.status !== 200 || response.redirected ||\n new URL(response.url).origin !== sameOrigin) {\n throw new Error('read-only observation did not return same-origin HTTP 200');\n }\n return response;\n };\n let last = null;\n while (Date.now() - started <= 20000) {\n try {\n const response = await read(\"/api/v4/projects/primer%2Fdesign/milestones?state=all&per_page=100\");\n const rows = await response.json();\n const expected = {\"title\": \"product launch\", \"start_date\": \"2023-01-16\", \"due_date\": \"2023-01-30\"};\n const matches = Array.isArray(rows) ? rows.filter(row =>\n Object.entries(expected).every(([key, value]) => row?.[key] === value)) : [];\n const nextPage = /rel=[\"']next[\"']/.test(response.headers.get('Link') || '');\n last = {rowCount: Array.isArray(rows) ? rows.length : null,\n matchingRecords: matches.length, nextPage};\n if (matches.length === 1 && !nextPage) {\n return {completion: \"persisted_collection_record_observed\", ...last, elapsedMs: Date.now() - started};\n }\n } catch (error) {\n last = {error: String(error)};\n }\n if (Date.now() - started >= 20000) break;\n await new Promise(resolve => setTimeout(resolve, 100));\n }\n throw new Error('post-action completion not observed: ' + JSON.stringify(last));\n })()" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-610.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-610.json new file mode 100644 index 000000000..c62b2ba43 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-610.json @@ -0,0 +1,80 @@ +{ + "task_id": 610, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:9999" + }, + { + "action": "click", + "selector": "a[href=\"/submit\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "To Kill a Mockingbird by Harper Lee", + "selector": "textarea[name=\"submission[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "select", + "value": "10037", + "selector": "select[name=\"submission[forum]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create submission\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1000 + }, + { + "action": "snapshot" + }, + { + "action": "wait", + "timeout": 1500 + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(()=>{const match=/^\\/f\\/books\\/(\\d+)\\//.exec(location.pathname);if(!match)throw Error('not the created Books post');const fields=[...document.querySelectorAll('main form textarea[name^=\"reply_to_submission_\"][name$=\"[comment]\"]')];if(fields.length!==1||fields[0].name!=='reply_to_submission_'+match[1]+'[comment]')throw Error('comment form does not belong to current post');return {post_id:match[1],comment_field:fields[0].name};})()" + }, + { + "action": "fill", + "text": "good book!", + "selector": "main form textarea[name^=\"reply_to_submission_\"][name$=\"[comment]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Post\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "timeout": 1200 + }, + { + "action": "snapshot" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-658.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-658.json new file mode 100644 index 000000000..242bad3d0 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-658.json @@ -0,0 +1,77 @@ +{ + "task_id": 658, + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-gitlab.dev.lexmount.net" + }, + { + "action": "click", + "selector": "a[href=\"/a11yproject/a11yproject.com/-/issues\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/a11yproject/a11yproject.com/-/issues/new\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "fill", + "text": "401 bad gateway", + "selector": "input[name=\"issue[title]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async name => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (!el || el.disabled) throw new Error('Date input is unavailable');\n if (document.activeElement !== el) { el.focus(); stableSince = null; }\n else {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n }\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input focus did not settle');\n})(\"issue[due_date]\")" + }, + { + "action": "fill", + "text": "2030-12-31", + "selector": "input[name=\"issue[due_date]\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "evaluate", + "script": "(async ({name, expected}) => {\n const start = performance.now(); let stableSince = null;\n while (performance.now() - start < 3000) {\n const el = document.querySelector(`input[name=\"${name}\"]`);\n if (el && el.value === expected && document.activeElement === el) {\n stableSince ??= performance.now();\n if (performance.now() - stableSince >= 50) return true;\n } else stableSince = null;\n await new Promise(resolve => setTimeout(resolve, 10));\n }\n throw new Error('Date input did not retain its value and focus');\n})({\"name\": \"issue[due_date]\", \"expected\": \"2030-12-31\"})" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Unassigned\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(expected => { for (const [name, value] of Object.entries(expected)) { const el = document.querySelector(`input[name=\"${name}\"]`); if (!el || el.value !== value) throw new Error('Date field differs before submit: ' + name); } return {completion:'date_fields_verified', fields:expected}; })({\"issue[due_date]\": \"2030-12-31\"})" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Create issue\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(async () => {\n const started = Date.now();\n const sameOrigin = location.origin;\n const read = async (path) => {\n const response = await fetch(path, {method: 'GET', credentials: 'same-origin',\n redirect: 'error', cache: 'no-store'});\n if (response.status !== 200 || response.redirected ||\n new URL(response.url).origin !== sameOrigin) {\n throw new Error('read-only observation did not return same-origin HTTP 200');\n }\n return response;\n };\n let last = null;\n while (Date.now() - started <= 20000) {\n try {\n const response = await read(\"/api/v4/projects/a11yproject%2Fa11yproject.com/issues?search=401%20bad%20gateway&per_page=100\");\n const rows = await response.json();\n const expected = {\"title\": \"401 bad gateway\", \"due_date\": \"2030-12-31\"};\n const matches = Array.isArray(rows) ? rows.filter(row =>\n Object.entries(expected).every(([key, value]) => row?.[key] === value) && Array.isArray(row?.assignees) &&\n row.assignees.some(user => user?.id === 2264)) : [];\n const nextPage = /rel=[\"']next[\"']/.test(response.headers.get('Link') || '');\n last = {rowCount: Array.isArray(rows) ? rows.length : null,\n matchingRecords: matches.length, nextPage};\n if (matches.length === 1 && !nextPage) {\n return {completion: \"persisted_collection_record_observed\", ...last, elapsedMs: Date.now() - started};\n }\n } catch (error) {\n last = {error: String(error)};\n }\n if (Date.now() - started >= 20000) break;\n await new Promise(resolve => setTimeout(resolve, 100));\n }\n throw new Error('post-action completion not observed: ' + JSON.stringify(last));\n })()" + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-714.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-714.json new file mode 100644 index 000000000..8c93344a6 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-714.json @@ -0,0 +1,65 @@ +{ + "schema": "webarena_qualified_trajectory_revision/1", + "task_id": 714, + "canonical_calls_sha256": "a47928760c9f217136aa133cb7f388dc9ce779a6cbdf9289aad9c822789d49de", + "tool_calls": [ + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-reddit.dev.lexmount.net" + }, + { + "action": "click", + "selector": "a[href=\"/forums\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "navigate", + "url": "http://webarena-slice-warm-1-195-reddit.dev.lexmount.net/f/gadgets" + }, + { + "action": "click", + "selector": "button[aria-label=\"Sort by: Hot\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/gadgets/top?t=day\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 2 + }, + { + "action": "click", + "selector": "button[aria-label=\"From: Past 24 hours\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"/f/gadgets/top?t=all\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "article:has(a[href^=\"/f/gadgets/19459/\"]) form button:has(img[alt=\"down\"]):visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "seconds": 2 + } + ] +} diff --git a/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-771.json b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-771.json new file mode 100644 index 000000000..db36ee364 --- /dev/null +++ b/evaluation/trajectory-replay/webarena-verified/trajectory-candidates/task-771.json @@ -0,0 +1,109 @@ +{ + "task_id": 771, + "tool_calls": [ + { + "action": "navigate", + "url": "http://192.168.5.2:7780/admin" + }, + { + "action": "click", + "selector": ":nth-match(a[href=\"#\"]:visible, 4)", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/review/product/pending/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "click", + "selector": "a[href=\"http://host.docker.internal:7780/admin/review/product/edit/id/353/\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "snapshot" + }, + { + "action": "evaluate", + "script": "(() => { const f=document.querySelector('form'); return {checked:[...document.querySelectorAll('input[type=radio]')].map(x=>({name:x.name,value:x.value,checked:x.checked,outer:x.outerHTML})), ratings:[...document.querySelectorAll('[class*=rating]')].map(x=>({tag:x.tagName,cls:x.className,text:x.innerText,html:x.innerHTML.slice(0,500)}))}; })()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/352/" + }, + { + "action": "evaluate", + "script": "(()=>({rating:[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id,status:document.querySelector('[name=status_id]')?.value}))()" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"status_id\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save Review\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "selector": "#reviewGrid_table:visible", + "timeout": 30000 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/351/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/349/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/edit/id/347/" + }, + { + "action": "evaluate", + "script": "(()=>[...document.querySelectorAll('input[name^=\"ratings\"]')].find(x=>x.checked)?.id)()" + }, + { + "action": "select", + "value": "1", + "selector": "select[name=\"status_id\"]:visible", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "click", + "selector": "button:visible:has-text(\"Save Review\")", + "replay_mode": "snapshot_ref_semantics" + }, + { + "action": "wait", + "selector": "#reviewGrid_table:visible", + "timeout": 30000 + }, + { + "action": "navigate", + "url": "http://host.docker.internal:7780/admin/review/product/pending/" + } + ], + "repair": { + "cause": "An explicit navigation 500ms after Save Review canceled the pending save response, leaving HTTP status 0.", + "change": "Await the destination review grid after each save before issuing another navigation." + } +} diff --git a/harness/browser_use/har.py b/harness/browser_use/har.py index 6628a2c10..5b3b11c67 100644 --- a/harness/browser_use/har.py +++ b/harness/browser_use/har.py @@ -4,6 +4,8 @@ import base64 import json +import time +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -54,6 +56,13 @@ def _har_response(response: dict[str, Any], body: dict[str, Any] | None) -> dict } +def _is_json_mime_type(value: Any) -> bool: + """Match the JSON MIME essence without mistaking parameters for a subtype.""" + + essence = str(value or "").split(";", 1)[0].strip().lower() + return essence == "application/json" or ("/" in essence and essence.endswith("+json")) + + class CdpHarRecorder: """Capture raw CDP requests from an existing browser context. @@ -62,16 +71,32 @@ class CdpHarRecorder: the official WebArena-Verified mutation evaluators require. """ + _DOCUMENT_DRAIN_TIMEOUT_MS = 10_000 + # JSON may trigger navigation before loadingFinished's client-side callback + # can read it. Opt into bounded cross-document retention explicitly; neither + # browser defaults nor a later retry guarantee that the body still exists. + _NETWORK_CAPTURE_OPTIONS = { + "enableDurableMessages": True, + "maxTotalBufferSize": 10 * 1024 * 1024, + "maxResourceBufferSize": 1024 * 1024, + } + def __init__(self) -> None: self._context: Any = None self._sessions: list[Any] = [] self._requests: dict[tuple[int, str], dict[str, Any]] = {} self._request_ordinals: dict[tuple[int, str], int] = {} + self._next_request_sequence = 0 + self._session_ordinals: dict[int, int] = {} + self._completion_timestamps: dict[tuple[int, str, int], Any] = {} self._extra_headers: dict[tuple[int, str], list[dict[str, Any]]] = {} self._responses: list[tuple[Any, str, dict[str, Any], dict[str, Any]]] = [] self._finished: set[tuple[int, str]] = set() - self._response_bodies: dict[tuple[int, str], dict[str, Any]] = {} - self._response_body_errors: dict[tuple[int, str], str] = {} + self._failed: dict[tuple[int, str], dict[str, Any]] = {} + self._request_sessions: dict[tuple[int, str], Any] = {} + self._session_pages: dict[int, Any] = {} + self._response_bodies: dict[tuple[int, str, int], dict[str, Any]] = {} + self._response_body_errors: dict[tuple[int, str, int], str] = {} self._payload: str | None = None self._stopped = False self.errors: list[str] = [] @@ -86,51 +111,166 @@ def stop(self, context: Any, path: Path) -> None: if self._stopped: return if self._payload is None: + # Network events and command responses share the ordered CDP channel. + # A command round trip pumps Playwright's sync dispatcher without + # executing page JavaScript, so events already emitted by the browser + # are visible before the evidence boundary is chosen. + self._protocol_barrier() + boundary_responses = list(self._responses) + boundary_requests = { + (*key, int(request.get("__ordinal", 0))) for key, request in self._requests.items() + } + boundary_requests.update( + (id(session), request_id, int(request.get("__ordinal", 0))) + for session, request_id, request, _response in boundary_responses + ) + eligible_documents = self._unfinished_document_keys() + self._drain_documents(eligible_documents) + self._protocol_barrier() if self._context is not None: try: context.remove_listener("page", self._attach) except Exception as exc: self.errors.append(f"page listener detach failed: {exc}") entries: list[dict[str, Any]] = [] - # CDP callbacks run on the browser thread and may append a final - # response while stop() is serializing. Use one immutable view for - # both the redirect lookup and entry loop so a late event cannot - # create a response whose key is absent from last_response_by_key. - responses = list(self._responses) + # No Playwright call occurs while these lifecycle views are copied. + # Body fallback below may pump more callbacks, but those events belong + # after this immutable evidence boundary and cannot mix with it. + requests_snapshot = {key: dict(request) for key, request in self._requests.items()} + responses_snapshot = list(self._responses) + finished_snapshot = set(self._finished) + failed_snapshot = {key: dict(event) for key, event in self._failed.items()} + bodies_snapshot = dict(self._response_bodies) + body_errors_snapshot = dict(self._response_body_errors) + completion_snapshot = dict(self._completion_timestamps) + extra_headers_snapshot = { + key: [dict(headers) for headers in values] + for key, values in self._extra_headers.items() + } + responses = [ + observation + for observation in responses_snapshot + if self._observation_is_within_boundary( + observation[0], + observation[1], + observation[2], + boundary_requests, + eligible_documents, + ) + ] + represented = { + (id(session), request_id, int(request.get("__ordinal", 0))) + for session, request_id, request, _response in responses + } + # A request is evaluator evidence even when finalization races its + # response. Retain requestWillBeSent-only observations with a HAR + # status of zero instead of silently dropping the mutation. + pending_requests: list[tuple[Any, str, dict[str, Any], None]] = [] + for key, request in requests_snapshot.items(): + ordinal = int(request.get("__ordinal", 0)) + if (*key, ordinal) not in boundary_requests and key not in eligible_documents: + continue + if (*key, ordinal) in represented: + continue + session = self._request_sessions.get(key) + if session is not None: + pending_requests.append((session, key[1], dict(request), None)) + unordered_observations = [*responses, *pending_requests] + observations = [ + observation + for _index, observation in sorted( + enumerate(unordered_observations), + key=lambda indexed: self._observation_order(*indexed), + ) + ] last_response_by_key = { (id(session), request_id): index - for index, (session, request_id, _request, _response) in enumerate( - responses - ) + for index, (session, request_id, _request, _response) in enumerate(observations) } - for index, (session, request_id, request, response) in enumerate(responses): + for index, (session, request_id, request, response) in enumerate(observations): key = (id(session), request_id) - body = self._response_bodies.get(key) request = dict(request) ordinal = int(request.pop("__ordinal", 0)) - extras = self._extra_headers.get(key, []) + request.pop("__sequence", None) + body_key = (*key, ordinal) + body = bodies_snapshot.get(body_key) + extras = extra_headers_snapshot.get(key, []) if ordinal < len(extras): request["headers"] = {**request.get("headers", {}), **extras[ordinal]} - mime_type = str(response.get("mimeType") or "") + mime_type = str((response or {}).get("mimeType") or "") if ( index == last_response_by_key[key] - and key in self._finished + and key in finished_snapshot + and response is not None and body is None - and (mime_type == "application/json" or mime_type.endswith("+json")) + and _is_json_mime_type(mime_type) ): try: body = session.send("Network.getResponseBody", {"requestId": request_id}) body = self._decode_body(body) except Exception as exc: - prior = self._response_body_errors.get(key) + prior = body_errors_snapshot.get(body_key) detail = f"{prior}; stop fallback: {exc}" if prior else str(exc) self.errors.append( f"{request.get('url')}: response body unavailable: {detail}" ) body = None - entries.append( - {"request": _har_request(request), "response": _har_response(response, body)} - ) + entry = { + "request": _har_request(request), + "response": ( + _har_response(response, body) + if response is not None + else { + "status": 0, + "statusText": "", + "headers": [], + "cookies": [], + "redirectURL": "", + "content": {"mimeType": "", "text": ""}, + } + ), + } + # Keep raw CDP clocks distinct: timestamp is monotonic seconds, + # wallTime is epoch seconds; ResourceTiming offsets are milliseconds. + # Redirect observation is not a fabricated responseReceived event. + timing = {**request.get("__cdp", {}), **(response or {}).get("__cdp", {})} + if response is not None and "timing" in response: + timing["responseTiming"] = response["timing"] + if response is not None and "responseTime" in response: + timing["responseTime"] = response["responseTime"] + completion_key = (id(session), request_id, ordinal) + if completion_key in completion_snapshot: + timing["loadingFinishedTimestamp"] = completion_snapshot[completion_key] + if timing: + entry["_cdp"] = timing + current = requests_snapshot.get(key) + is_current = current is not None and int(current.get("__ordinal", 0)) == ordinal + if is_current and key in failed_snapshot: + failure = failed_snapshot[key] + entry["_capture"] = { + "state": "failed", + **( + {"errorText": str(failure["errorText"])} + if failure.get("errorText") is not None + else {} + ), + } + elif is_current and key not in finished_snapshot: + entry["_capture"] = { + "state": "incomplete", + "reason": "response-not-terminal-at-finalization", + } + if "requestWallTime" in timing: + try: + entry["startedDateTime"] = ( + datetime.fromtimestamp(timing["requestWallTime"], tz=UTC) + .isoformat() + .replace("+00:00", "Z") + ) + except (ValueError, TypeError, OverflowError, OSError): + # Retain raw evidence even if the supplied wall clock is invalid. + pass + entries.append(entry) for session in self._sessions: try: session.detach() @@ -150,11 +290,93 @@ def stop(self, context: Any, path: Path) -> None: path.write_text(self._payload, encoding="utf-8") self._stopped = True + @staticmethod + def _observation_order( + index: int, + observation: tuple[Any, str, dict[str, Any], dict[str, Any] | None], + ) -> tuple[int, int]: + sequence = observation[2].get("__sequence") + if sequence is None: + # Compatibility for synthetic callers that populate _responses + # directly; real CDP observations always carry a request sequence. + return (2**63 - 1, index) + return (int(sequence), index) + + @staticmethod + def _observation_is_within_boundary( + session: Any, + request_id: str, + request: dict[str, Any], + boundary_requests: set[tuple[int, str, int]], + eligible_documents: set[tuple[int, str]], + ) -> bool: + key = (id(session), request_id) + ordinal = int(request.get("__ordinal", 0)) + return (*key, ordinal) in boundary_requests or key in eligible_documents + + def _protocol_barrier(self) -> None: + """Deliver CDP events emitted before finalization reaches this boundary.""" + + for session in list(self._sessions): + try: + # A read-only page command pumps the same ordered channel + # without resetting Network.enable's retention configuration. + session.send("Page.getFrameTree", {}) + except Exception as exc: + self.errors.append(f"CDP finalization barrier failed: {exc}") + + def _unfinished_document_keys(self) -> set[tuple[int, str]]: + return { + key + for key, request in self._requests.items() + if request.get("__resource_type") == "Document" + and key not in self._finished + and key not in self._failed + } + + def _drain_documents(self, eligible: set[tuple[int, str]]) -> None: + """Bound finalization to navigations observed at the protocol barrier. + + This deliberately excludes future timer-created requests and unfinished + subresources. Short Playwright event-loop pumps stop as soon as every + eligible CDP request emits loadingFinished or loadingFailed; they do not + depend on a top-level load event and therefore also cover iframes. + """ + + if not eligible: + return + pages = { + self._session_pages[session_id] + for session_id, _request_id in eligible + if session_id in self._session_pages + } + if not pages: + return + page = next(iter(pages)) + deadline = time.monotonic() + self._DOCUMENT_DRAIN_TIMEOUT_MS / 1000 + while True: + remaining = eligible.difference(self._finished, self._failed) + if not remaining: + return + remaining_ms = max(0, int((deadline - time.monotonic()) * 1000)) + if remaining_ms == 0: + return + try: + # wait_for_timeout is Playwright's supported sync event-loop pump. + # The 50 ms slice is not a fixed post-action delay: the lifecycle + # predicate above terminates immediately after the Document does. + page.wait_for_timeout(min(50, remaining_ms)) + except Exception: + # A timeout or closed page leaves a truthful incomplete entry. + return + self._protocol_barrier() + def _attach(self, page: Any) -> None: try: assert self._context is not None session = self._context.new_cdp_session(page) session_index = id(session) + self._session_pages[session_index] = page session.on( "Network.requestWillBeSent", lambda event: self._record_request(session, session_index, event), @@ -173,7 +395,11 @@ def _attach(self, page: Any) -> None: "Network.loadingFinished", lambda event: self._record_finished(session, session_index, event), ) - session.send("Network.enable") + session.on( + "Network.loadingFailed", + lambda event: self._record_failed(session_index, event), + ) + session.send("Network.enable", dict(self._NETWORK_CAPTURE_OPTIONS)) self._sessions.append(session) except Exception as exc: self.errors.append(f"CDP attach failed: {exc}") @@ -181,22 +407,54 @@ def _attach(self, page: Any) -> None: def _record_request(self, session: Any, session_index: int, event: dict[str, Any]) -> None: request_id = str(event["requestId"]) key = (session_index, request_id) + self._finished.discard(key) + self._failed.pop(key, None) + self._request_sessions[key] = session previous = self._requests.get(key) redirect_response = event.get("redirectResponse") if previous is not None and redirect_response is not None: + redirect_response = dict(redirect_response) + redirect_response["__cdp"] = { + "responseSource": "requestWillBeSent.redirectResponse", + **( + {"redirectObservedTimestamp": event["timestamp"]} + if "timestamp" in event + else {} + ), + } self._responses.append((session, request_id, dict(previous), redirect_response)) ordinal = self._request_ordinals.get(key, 0) self._request_ordinals[key] = ordinal + 1 request = dict(event["request"]) request["__ordinal"] = ordinal + request["__sequence"] = self._next_request_sequence + self._next_request_sequence += 1 request["__resource_type"] = event.get("type") + request["__cdp"] = { + "sessionOrdinal": self._session_ordinals.setdefault( + session_index, len(self._session_ordinals) + ), + "requestId": request_id, + "redirectOrdinal": ordinal, + **({"requestTimestamp": event["timestamp"]} if "timestamp" in event else {}), + **({"requestWallTime": event["wallTime"]} if "wallTime" in event else {}), + } self._requests[key] = request def _record_response(self, session: Any, session_index: int, event: dict[str, Any]) -> None: request_id = str(event["requestId"]) request = self._requests.get((session_index, request_id)) if request is not None: - self._responses.append((session, request_id, dict(request), event["response"])) + response = dict(event["response"]) + response["__cdp"] = { + "responseSource": "responseReceived", + **( + {"responseReceivedTimestamp": event["timestamp"]} + if "timestamp" in event + else {} + ), + } + self._responses.append((session, request_id, dict(request), response)) @staticmethod def _decode_body(body: dict[str, Any]) -> dict[str, Any]: @@ -209,9 +467,7 @@ def _decode_body(body: dict[str, Any]) -> dict[str, Any]: } return body - def _record_finished( - self, session: Any, session_index: int, event: dict[str, Any] - ) -> None: + def _record_finished(self, session: Any, session_index: int, event: dict[str, Any]) -> None: """Capture JSON bodies while CDP still retains them. Waiting until the whole agent run ends is too late for long tasks: CDP may @@ -220,6 +476,11 @@ def _record_finished( request_id = str(event["requestId"]) key = (session_index, request_id) + request = self._requests.get(key) + ordinal = int(request.get("__ordinal", 0)) if request is not None else 0 + body_key = (session_index, request_id, ordinal) + if request is not None and "timestamp" in event: + self._completion_timestamps[body_key] = event["timestamp"] self._finished.add(key) response = next( ( @@ -232,13 +493,17 @@ def _record_finished( None, ) mime_type = str((response or {}).get("mimeType") or "") - if mime_type != "application/json" and not mime_type.endswith("+json"): + if not _is_json_mime_type(mime_type): return try: - self._response_bodies[key] = self._decode_body( + self._response_bodies[body_key] = self._decode_body( session.send("Network.getResponseBody", {"requestId": request_id}) ) - self._response_body_errors.pop(key, None) + self._response_body_errors.pop(body_key, None) except Exception as exc: # stop() retries once before exposing a capture error. - self._response_body_errors[key] = str(exc) + self._response_body_errors[body_key] = str(exc) + + def _record_failed(self, session_index: int, event: dict[str, Any]) -> None: + key = (session_index, str(event["requestId"])) + self._failed[key] = dict(event) diff --git a/harness/browser_use/model.py b/harness/browser_use/model.py index 4aa48c3b8..631aef6ea 100644 --- a/harness/browser_use/model.py +++ b/harness/browser_use/model.py @@ -131,7 +131,7 @@ def __init__( self.session = session or requests.Session() self.last_response: dict[str, Any] | None = None self._previous_response_id: str | None = None - self._seen_message_count = 0 + self._seen_messages: tuple[str, ...] = () self.usage: dict[str, int] = {} self.request_count = 0 self.request_attempt_count = 0 @@ -141,7 +141,7 @@ def reset(self) -> None: """Start a fresh Responses conversation for the next benchmark attempt.""" self.last_response = None self._previous_response_id = None - self._seen_message_count = 0 + self._seen_messages = () self.usage = {} self.request_count = 0 self.request_attempt_count = 0 @@ -184,13 +184,36 @@ def verify_access(self) -> None: def call_stream(self, request: Any) -> Iterable[dict[str, Any]]: messages = list(request.messages or []) - if self._previous_response_id: - new_messages = messages[self._seen_message_count :] + # CowAgent can compact or rewrite history in place. Keep an immutable + # content snapshot: a message count cannot establish a shared prefix. + message_snapshot = tuple( + json.dumps(message, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + for message in messages + ) + continues_history = bool(self._previous_response_id) and ( + message_snapshot[: len(self._seen_messages)] == self._seen_messages + ) + if continues_history: + new_messages = messages[len(self._seen_messages) :] input_items = cow_messages_to_responses( [message for message in new_messages if message.get("role") != "assistant"] ) else: input_items = cow_messages_to_responses(messages) + # A replay has no server-side calls to satisfy orphan outputs. Reject + # malformed compaction locally instead of losing observations or + # attaching them to the conversation that was explicitly replaced. + pending_calls: set[str] = set() + for item in input_items: + if item.get("type") == "function_call": + pending_calls.add(item["call_id"]) + elif item.get("type") == "function_call_output": + call_id = item["call_id"] + if not call_id or call_id not in pending_calls: + raise ValueError( + f"tool output {call_id!r} has no matching preceding call in history" + ) + pending_calls.remove(call_id) payload: dict[str, Any] = { "model": self.model, "instructions": str(getattr(request, "system", "") or ""), @@ -198,7 +221,7 @@ def call_stream(self, request: Any) -> Iterable[dict[str, Any]]: "reasoning": {"effort": self.reasoning_effort}, "tools": cow_tools_to_responses(request.tools), } - if self._previous_response_id: + if continues_history: payload["previous_response_id"] = self._previous_response_id if getattr(request, "max_tokens", None): payload["max_output_tokens"] = int(request.max_tokens) @@ -215,9 +238,7 @@ def call_stream(self, request: Any) -> Iterable[dict[str, Any]]: ) response.raise_for_status() except requests.RequestException as exc: - self.transport_errors.append( - {"type": type(exc).__name__, "message": str(exc)[:500]} - ) + self.transport_errors.append({"type": type(exc).__name__, "message": str(exc)[:500]}) raise self.last_response = response.json() response_id = self.last_response.get("id") @@ -236,7 +257,7 @@ def call_stream(self, request: Any) -> Iterable[dict[str, Any]]: ): self.usage[key] = self.usage.get(key, 0) + int(value or 0) self._previous_response_id = response_id - self._seen_message_count = len(messages) + self._seen_messages = message_snapshot yield response_to_chat_chunk(self.last_response) diff --git a/harness/browser_use/report.py b/harness/browser_use/report.py index abed9bbe4..cc7695837 100644 --- a/harness/browser_use/report.py +++ b/harness/browser_use/report.py @@ -15,11 +15,12 @@ from collections import Counter, defaultdict from collections.abc import Callable, Iterable from io import BytesIO -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from PIL import Image +from harness.fix_log import load_fix_log from harness.web_fetch.report_style import STYLE as SHARED_STYLE from .failure_cause_review import ( @@ -45,7 +46,8 @@ trajectory, trajectory_label, ) -from .report_i18n import write_english_report +from .report_i18n import localize_html, write_english_report +from harness.report_languages import write_language_contract from .selection import PilotSelection, load_pilot, validate_against_official ENGINE_LABELS = {"chromium": "Chromium", "moli": "Moli 1.1.3"} @@ -1407,6 +1409,7 @@ def _resource_comparison_table(summary: dict[str, dict[str, dict[str, Any]]]) -> ("methodology/", "评测方法", "methodology"), ("analysis/", "结果分析", "analysis"), ("review/", "核验记录", "review"), + ("fixes/", "修复日志", "fixes"), ("adoption/", "被测浏览器", "adoption"), ("reproduce/", "环境与复现", "reproduce"), ("changes/", "变更记录", "changes"), @@ -1444,7 +1447,7 @@ def _evidence_viewer_page() -> str: 'const output=document.getElementById("evidence-viewer-content");' 'const raw=document.getElementById("evidence-raw-link");' 'const source=new URLSearchParams(location.search).get("src")||"";' - r"const valid=/^artifacts\/[a-z0-9._-]+\/executions\/[0-9]+\/execution-evidence[.]json$/i.test(source);" + r"const valid=/^artifacts\/[a-z0-9._-]+\/executions\/[0-9]+\/execution-evidence[.]json$/i.test(source)||['data/trajectory-replay-bridge.json','data/trajectory-replay-failure-review.json'].includes(source);" 'const english=document.documentElement.lang==="en";' 'if(!valid){status.textContent=english?"Invalid evidence path.":"证据路径无效。";return;}' "raw.href=source;raw.hidden=false;" @@ -2095,6 +2098,243 @@ def _failure_scope_figure(verified_causes: dict[str, dict[str, Any]]) -> str: return f'
' + bars + "
" +def _validate_replay_contract_definitions(bridge: dict[str, Any]) -> None: + definitions = {} + for name in ("trajectories", "corrections"): + binding = (bridge.get("contract_definitions") or {}).get(name) or {} + path = Path(str(binding.get("path") or "")) + if path.is_symlink() or not path.is_file() or _sha256(path) != binding.get("sha256"): + raise ValueError("trajectory replay contract definitions are not hash-bound") + definitions[name] = _read_json(path) + dataset = definitions["trajectories"] + if dataset.get("selection_sha256") != bridge.get("selection_sha256"): + raise ValueError("trajectory replay contract definitions use a different selection") + rows = dataset["trajectories"] + if len({row["task_id"] for row in rows}) != len(rows): + raise ValueError("duplicate trajectory task in contract definitions") + counts: Counter[str] = Counter() + cross: dict[str, Counter[str]] = {} + origins: Counter[str] = Counter() + corrected = set() + for row in rows: + kind = row["verification"]["contract"]["kind"] + counts[kind] += 1 + cross.setdefault(row["source"]["kind"], Counter())[kind] += 1 + if row["source"]["kind"] == "authoritative_successful_browser_use_execution": + origins[f'{row["source"]["model"]}_{row["source"]["browser"]}'] += 1 + if kind == "corrected_replay_contract": + corrected.add(row["task_id"]) + categories: dict[str, list[int]] = { + "event_selection": [], "event_selection_and_expected": [], "reference_or_acceptance": [] + } + corrections = definitions["corrections"]["corrections"] + if {row["task_id"] for row in corrections} != corrected: + raise ValueError("trajectory replay correction scope differs from qualified definitions") + for row in corrections: + operations = row["operations"] + selection = any(op["path"].endswith("/last_event_only") for op in operations) + other = any(not op["path"].endswith("/last_event_only") for op in operations) + category = "event_selection_and_expected" if selection and other else ( + "event_selection" if selection else "reference_or_acceptance" + ) + categories[category].append(row["task_id"]) + categories = {key: sorted(value) for key, value in categories.items()} + if (dict(counts) != bridge.get("contract_counts") + or {key: dict(value) for key, value in cross.items()} != bridge.get("source_contract_counts") + or categories != bridge.get("correction_categories") + or len(rows) != bridge["effective_results"]["chromium"]["tasks"]): + raise ValueError("trajectory replay contract counts differ from qualified definitions") + if dict(origins) != bridge["chromium_trajectory_provenance"]["successful_browser_use_execution_by_origin"]: + raise ValueError("trajectory replay model-browser origins differ from qualified definitions") + bridge["original_contract_task_groups"] = { + source: sorted(row["task_id"] for row in rows + if row["source"]["kind"] == source + and row["verification"]["contract"]["kind"] == "official_task_contract") + for source in cross + } + + +def _load_trajectory_replay_bridge( + profile: dict[str, Any], *, selection_sha256: str +) -> dict[str, Any] | None: + """Load the hash-bound cross-track control without treating it as an Agent score.""" + + config = profile.get("trajectory_replay_bridge") or {} + if not config: + return None + source = Path(str(config.get("source_path") or "")) + expected_hash = str(config.get("source_sha256") or "") + if source.is_symlink() or not source.is_file() or _sha256(source) != expected_hash: + raise ValueError("trajectory replay bridge is missing or is not hash-bound by the profile") + bridge = _read_json(source) + if bridge.get("schema") != "lex_browser_use_trajectory_replay_bridge/1": + raise ValueError("unsupported trajectory replay bridge schema") + if bridge.get("selection_sha256") != selection_sha256: + raise ValueError("trajectory replay bridge uses a different task selection") + trajectory_profile = bridge.get("trajectory_profile") or {} + trajectory_profile_path = Path(str(trajectory_profile.get("path") or "")) + if ( + trajectory_profile_path.is_symlink() + or not trajectory_profile_path.is_file() + or _sha256(trajectory_profile_path) != trajectory_profile.get("sha256") + ): + raise ValueError("trajectory replay profile is missing or is not hash-bound") + _validate_replay_contract_definitions(bridge) + result = bridge.get("effective_results") or {} + chromium = result.get("chromium") or {} + moli = result.get("moli") or {} + provenance = bridge.get("chromium_trajectory_provenance") or {} + if ( + int(result.get("logical_results") or 0) != 180 + or int(chromium.get("tasks") or 0) != 90 + or int(chromium.get("passes") or 0) != 90 + or int(moli.get("tasks") or 0) != 90 + or int(moli.get("passes") or 0) != 54 + or int(provenance.get("expert_repair_spec") or 0) != 30 + or int(provenance.get("successful_browser_use_execution") or 0) != 60 + ): + raise ValueError("trajectory replay bridge has an invalid current control cardinality") + review = bridge.get("moli_failure_review") or {} + review_path = Path(str(review.get("path") or "")) + if ( + review_path.is_symlink() + or not review_path.is_file() + or _sha256(review_path) != review.get("sha256") + or int(review.get("failed_results") or 0) != 36 + or int(review.get("verified_results") or 0) != 36 + ): + raise ValueError("trajectory replay browser-failure review is missing or inconsistent") + review_data = _read_json(review_path) + responsibilities: Counter[str] = Counter() + reviewed_keys: set[tuple[str, int]] = set() + for group in review_data.get("cause_groups") or []: + for item in group.get("results") or []: + key = (str(item.get("engine") or ""), int(item["task_id"])) + if key in reviewed_keys or key[0] != "moli": + raise ValueError("trajectory replay responsibility inventory is inconsistent") + reviewed_keys.add(key) + responsibilities[str(group.get("responsibility") or "")] += 1 + if ( + len(reviewed_keys) != review["verified_results"] + or review_data.get("coverage") != { + "failed_results": review["failed_results"], "verified_results": len(reviewed_keys) + } + or (review_data.get("bindings") or {}).get("effective_results_sha256") + != result.get("sha256") + or dict(responsibilities) != review.get("responsibility_counts") + ): + raise ValueError("trajectory replay responsibility counts differ from the bound review") + return bridge + + +def _replay_original_contract_rows( + bridge: dict[str, Any], attempts: list[dict[str, Any]] +) -> list[list[str]]: + groups = bridge["original_contract_task_groups"] + historical = set(groups["authoritative_successful_browser_use_execution"]) + expert = set(groups["expert_repair_spec"]) + populations = (historical | expert, historical, expert) + indexed = {(row["model"], row["engine"], int(row["task_id"])): row for row in attempts} + if len(indexed) != len(attempts): + raise ValueError("replay diagnostic requires authoritative logical results") + rows = [] + engines = [BASELINE_ENGINE, *(engine for engine in ENGINE_LABELS if engine != BASELINE_ENGINE)] + for model in MODEL_LABELS: + for engine in engines: + cells = [f"{MODEL_LABELS[model]} × {ENGINE_LABELS[engine]}"] + for population in populations: + keys = [(model, engine, task_id) for task_id in population] + if any(key not in indexed for key in keys): + raise ValueError("replay diagnostic is missing registered configuration results") + passes = sum(float(indexed[key].get("score") or 0) == 1 for key in keys) + cells.append(f"{passes}/{len(population)}") + rows.append(cells) + return rows + + +def _trajectory_replay_control( + bridge: dict[str, Any] | None, attempts: list[dict[str, Any]] | None = None +) -> str: + if bridge is None: + return "" + result = bridge["effective_results"] + provenance = bridge["chromium_trajectory_provenance"] + origins = provenance["successful_browser_use_execution_by_origin"] + diagnostic = "" + if attempts is not None: + diagnostic_rows = "".join( + "" + "".join(f"{_e(cell)}" for cell in row) + "" + for row in _replay_original_contract_rows(bridge, attempts) + ) + diagnostic = ( + '

保留原合同的任务仍有自主执行失败

' + '

下表固定排除修订合同任务，按轨迹来源统计自主执行的官方通过数。' + '这是事后诊断，主报告分母仍为 90；历史成功路径经过筛选，不能把此表当成新的能力排名。

' + '
' + '' + + diagnostic_rows + '
配置全部原合同任务历史成功路径来源专家修复路径来源
' + ) + contract_rows = "".join( + f'{label}{counts["official_task_contract"]}{counts["corrected_replay_contract"]}' + for source, label in (("authoritative_successful_browser_use_execution", "历史成功轨迹"), ("expert_repair_spec", "专家修复轨迹")) + for counts in [bridge["source_contract_counts"][source]] + ) + correction_rows = "".join( + f'{label}{", ".join(map(str, bridge["correction_categories"][key]))}{description}' + for key, label, description in ( + ("event_selection", "仅修改事件选择", "按匹配条件选择保存请求，避免随后请求覆盖。"), + ("event_selection_and_expected", "事件选择与预期字段修改", "修订路由、请求字段或 Referer 条件，并匹配所需异步请求。"), + ("reference_or_acceptance", "参考答案或验收要求修改", "涉及商品名、表单字段、商品选择规则、未提交表单状态及冗余保存要求。"), + ) + ) + return ( + '

固定轨迹控制：它验证浏览器执行，不测自主规划

' + '

Chromium 的 90/90 不是 CowAgent 的 90/90。轨迹重放把既定 BrowserTool 调用交给浏览器，模型调用数为 0；' + 'Chromium 的 90/90 表示这 90 组固定动作、预录回答与绑定验收合同通过了重放验收，其中部分合同经过修订。它不让 Agent 在当次运行中观察、检索、比较、选择或决定何时结束。

' + '

验收使用轨迹预录回答（recorded_response）与本次网络记录（HAR）。对于检索问答题，通过不证明当前浏览器读到了所需数据，也不证明 Agent 在本次运行中计算出了答案；要求网络事件的合同仍会检查本次 HAR 是否包含对应请求。

' + '

90 题中的 37 道检索题均只使用回答验收，不要求网络事件：36 题沿用原合同，Task 279 使用修订合同。这 37 题的本轮读数与答案尚未验证；原评分保留，其余 53 题也不能替代原 90 题成为新的排名。

' + '
Chromium：90/90

固定动作、预录回答与绑定合同的重放验收结果。

' + f'
Moli：{result["moli"]["passes"]}/{result["moli"]["tasks"]}

同一固定路径仍有 36 条失败，不能把浏览器差异归零。

' + f'
路径来源：{provenance["expert_repair_spec"]} + {provenance["successful_browser_use_execution"]}

30 条专家修复路径，加 60 条历史成功的 Browser Use 路径。

' + '

后 60 条在进入重放集之前已经按“成功”条件筛选，其中 ' + f'{origins["gpt-5.6-sol_chromium"]} 条来自 Sol × Chromium、{origins["gpt-5.6-sol_moli"]} 条来自 Sol × Moli、' + f'{origins["gpt-6-astra_chromium"]} 条来自 Astra × Chromium。' + '因此固定轨迹成绩受给定路径、预录回答和验收合同共同约束，不能作为 Agent 决策能力、模型能力或完整 90 题自主通过率的反事实估计。

' + '

轨迹来源与验收合同

' + f'

90 条轨迹中，{bridge["contract_counts"]["official_task_contract"]} 条保留官方原合同，{bridge["contract_counts"]["corrected_replay_contract"]} 条使用修订合同。专家修复轨迹不一定修改验收；历史成功轨迹也可能修订验收。合同差异与路径来源需同时考虑。

' + '
' + + contract_rows + '
轨迹来源官方原合同修订合同
' + '
' + + correction_rows + '
修订类别任务验收差异
' + + diagnostic + + '

历史复核曾将 Moli 的 36 个固定轨迹失败归为浏览器责任；新发现的视口设置缺陷要求重新核对这一归因。' + '历史 Browser Use 的 Chromium 实际宽度为 780，Moli 为 1920；Task 173 的搜索链接在宽度达到 992 时由网页 CSS 隐藏。' + '该任务的失败不能单凭可见性超时归因于浏览器。历史分数保留，统一视口后的比较须重新冻结完整任务范围与轨迹资格。

' + '

数据绑定：跨轨迹控制摘要；' + '36 个 Moli 固定轨迹失败的逐题复核。

' + '

从失败结果追到决策与运行设置

' + '

Task 676 · Sol × Chromium · 验收证据：' + '官方合同要求带 filters[status]=fraud 的订单列表 XHR。HAR 已记录该请求、200 状态和正确 Referer；' + '操作轨迹显示 Suspected Fraud 已生效。' + 'NetworkEventEvaluator._filter_events_by_criteria 却对 NAVIGATE + GET 只保留最后的文档导航，遗漏 XHR；' + '默认 last_event_only 又可能在同路径请求中选中后来的 notification_area。两个模型的 Chromium 结果都受此影响。' + 'Moli 的同题证据还缺少 XHR Referer。' + '重放修订既改变事件选择，也移除了 Referer 条件；后者并非 Chromium 这次误判的原因。四个配置的原始失败分数保持不变。

' + '

Task 124 · Astra × Chromium：' + 'Agent 在第 9 个决策就结束，未耗尽 40 步。轨迹把 wireless earphone 查询扩大成 earphone，' + '工具结果仍显示 Items 1 to 12 of 24 total，最终却提交 min=0.01、max=287.64。' + '失败发生在候选资格与检索完整性判断；轨迹中有可读结果，不能用浏览器不可操作或步骤不足解释。' + '当前运行器依赖模型执行提示中的完成自检，没有独立验证候选全集。提示要求自检并不等于已经完成自检。

' + '

Task 510 · Sol × Moli：' + '第 40 个决策仍在调用工具，最终回答承认尚未结账。' + 'harness/browser_use/runner.py 的 _exhausted_agent_steps 检测这一状态，并通过 _max_steps_response 标记未完成。' + '这证明预算上限中断了流程，但所选 Hiquphon Tweeters 本身也不符合官方目标；增加预算能否完成正确购买仍需对照实验。

' + '

两模型均使用 reasoning_effort=low 和 40 步上限。' + 'harness/browser_use/model.py 限定 low，runner.py 在创建适配器时也传入 low。' + '现有分数只反映这套 Agent、工具与推理设置；尚无固定观察输入、仅改变推理档位的实验，不能把分数接近解释为模型能力相同。

' + ) + + def _automatic_failure_counts(attempts: list[dict[str, Any]]) -> Counter[str]: counts = Counter() for row in attempts: @@ -2150,6 +2390,7 @@ def _article_body( review: dict[str, Any], cohort_commits: dict[str, str], automatic_failure_counts: Counter[str], + trajectory_replay_bridge: dict[str, Any] | None, ) -> str: total_task_count = len(tasks) excluded = _external_interference_task_ids(verified_causes) @@ -2232,7 +2473,8 @@ def _article_body( '

按同题比较浏览器

浏览器对比必须留在同一模型层内。汇总数会隐藏互换的成败任务；下表给出同题对照。

' + _table(["模型", "浏览器 A", "浏览器 B", "同题数", "两边都通过", "A 通过、B 失败", "A 失败、B 通过", "两边都失败"], comparison_rows, caption=f"每行只比较同一模型下的 {task_count} 个配对任务。") + f'

两个模型层合计有 {discordant} 个单边通过，可用于定位兼容性问题；样本量和方向都不支持总体优劣判断。查看 {task_count} 题的四配置矩阵 →

' - '

程序判分落点与复核根因

程序判分先说明哪类规则未通过；人工复核再沿最终回答、evaluator 断言、HAR 和 Agent 轨迹追到直接原因。这两个层次需要分开。

程序判分落点

' + + _trajectory_replay_control(trajectory_replay_bridge, attempts) + + '

程序判分落点与复核根因

程序判分先说明哪类规则未通过；人工复核再沿最终回答、evaluator 断言、HAR 和 Agent 轨迹追到直接原因。这两个层次需要分开。

程序判分落点

' + _automatic_failure_figure(automatic_failure_counts) + '

“网络副作用未通过”只描述程序判分位置。它可能源于 Agent 没有提交，也可能源于 evaluator、站点或 HAR 采集问题。

复核根因

' + _failure_scope_figure(verified_causes) @@ -2245,6 +2487,7 @@ def _article_body( + f'{reviewed_attempts}/{recorded_attempts} 个证据包由程序核对文件存在性、哈希、JSON 可读性、回答 / 分数 / 状态一致性和运行身份。

2官方程序验收

每题按最终回答和必要网络副作用判分；没有 LLM judge。

3失败根因复核

{len(verified_causes)}/{raw_failed_result_count} 个原始最终失败结果逐条阅读错误、断言、最终回答、HAR 和 Agent 轨迹，归类并给出处理边界。

' + '

审查分为完整性检查和失败根因复核。 前者确认“证据包与 evaluator 记录一致”；后者检查 evaluator 语义，并识别出 Task 699 等问题。已通过结果没有再做独立语义盲审，其判定依据仍是官方 evaluator。查看核验记录 →

' f'

完整 {task_count} 题的速度与浏览器资源

均值、P50、P95 都在四种完整配置中统一比较。CPU、内存和网络只计浏览器容器，不含 CowAgent 或模型服务；模型会改变动作数量和页面停留时间，因此这里比较整套配置，不能单独归因于浏览器内核。

' + + '

历史轨迹记录的视口为 Chromium 780×437/493、Moli 1920×1080。视口差异会影响渲染、CPU 和内存；以下数值与最低值标记仅描述各自实际运行条件，不是统一视口下的资源效率结论。正式效率比较须统一设置后完整重跑。

' + _resource_comparison_table(summary) + '

每项只标记四种配置中数值唯一最低者。CPU 时间与网络总量累计逻辑结果使用的全部执行；内存来自 Browser Use 执行阶段的浏览器容器采样。

' '

共同成功任务的性能与资源

下表限定为四种配置最终都通过的 ' @@ -2252,7 +2495,7 @@ def _article_body( + ' 题，并以每题最终逻辑结果包含的全部物理执行计算 P50；这样不会把快速失败误当成性能优势。

' + _table(["模型", "浏览器", "Agent 链路 P50", "BrowserTool P50", "Tokens P50", "峰值内存 P50", "CPU 时间 P50"], resource_rows, table_id="home-resource-table", classes="core-comparison-table") + _resource_summary_cards((common_count, resource_rows)) - + '

Agent 链路还包含模型推理和页面等待，因此不能解读为纯内核性能。

' + + '

共同成功子集也保留了历史视口差异，不能据此得出统一视口下的资源效率结论。Agent 链路还包含模型推理和页面等待，因此不能解读为纯内核性能。

' f'

工程结论

完整 {task_count} 题官方结果是本报告的主口径；{clean_task_count} 题事后诊断子集只用于分离已知外部干扰，不改写成绩。

Moli 的内存占用明显更低，但 HAR 完整性、工具耗时和部分任务兼容性还需修复；Chromium 在本轮 Sol 层多通过 3 题，Astra 层通过数相同，但当前每种配置只有一个正式样本，不能外推成总体优势。

下一轮验证

先修复可控的合同、evaluator、站点和共享状态问题，再按同一协议完整重跑 90 题四种配置；历史官方分数在新证据批次出现前保持不变。

' ) @@ -2564,11 +2807,115 @@ def _review_history(rounds: list[dict[str, Any]]) -> str: return "".join(cards) -def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_path: Path) -> Path: +def _load_fix_log(profile: dict[str, Any]) -> dict[str, Any]: + """Load the shared fix-log model with Browser Use publication sanitization.""" + + return load_fix_log(profile, sanitize_value=sanitize) + + + +def _write_fix_log(stage: Path, model: dict[str, Any], *, english: bool = False) -> None: + """Render both views from one manifest; screenshots never imply pixel change.""" + lang = "en" if english else "zh" + def pick(zh: str, en: str) -> str: + return en if english else zh + folder = stage / ("en/fixes" if english else "fixes") + folder.mkdir(parents=True, exist_ok=True) + artifacts = {} + for item in model["artifacts"]: + suffix = ".json" if item["kind"] == "receipt" else item["extension"] + relative = f'artifacts/fixes/{item["sha256"]}{suffix}' + destination = stage / relative + if not english: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(item["path"], destination) + if _sha256(destination) != item["sha256"]: + raise ValueError("fix artifact changed during materialization") + artifacts[item["id"]] = {**{k: v for k, v in item.items() if k != "path"}, "href": relative} + + def link(identifier: str, label: str) -> str: + return f'{_e(label)}' + + def save(name: str, title: str, body: str) -> None: + page = _page(title=title, active="fixes", body=body, depth=1) + if english: + page = localize_html(page, PurePosixPath(f"fixes/{name}")) + (folder / name).write_text(page, encoding="utf-8") + + title = pick("修复日志", "Fix log") + intro = pick("修复证据与历史成绩分开：本页记录根因、修改原理和验证边界，不替换官方结果。", + "Fix evidence is separate from historical scores: root causes, changes, and validation limits do not replace official results.") + rows = [] + for record in model["records"]: + scope = pick("诊断验证", "Diagnostic validation") if record["scope"] == "diagnostic" else pick("正式范围验证", "Formal-scope validation") + rows.append([Safe(f'{_e(record["title"][lang])}'), scope]) + body = _hero("Fix evidence", record["title"][lang], scope) + for key, zh, en in (("symptom", "问题表现", "Symptom"), ("root_cause", "代码与流程根因", "Code and process root cause"), ("principle", "修复原理", "Fix principle")): + body += _section(pick(zh, en), f'

{_e(record[key][lang])}

') + body += _section(pick("代码位置", "Code locations"), "".join( + f'

{_e(code["repository"])} · {_e(code["path"])} · {_e(code["symbol"])}

' for code in record["code_paths"] + )) + basis = record["comparison_basis"] + if basis["kind"] == "identical_inputs": + comparison = ( + f'

{pick("共同输入 SHA-256", "Shared input SHA-256")}: ' + f'{record["input_sha256"]}

' + ) + else: + comparison = ( + f'

{pick("验收合同相同，轨迹已修订。", "The acceptance contract is unchanged; the trajectory was revised.")}

' + f'

{pick("验收合同 SHA-256", "Task contract SHA-256")}: ' + f'{basis["task_contract_sha256"]}

' + + _table( + [pick("绑定项", "Binding"), pick("修复前", "Before"), pick("修复后", "After")], + [ + [pick("输入 SHA-256", "Input SHA-256"), Safe(f'{record["before"]["input_sha256"]}'), Safe(f'{record["after"]["input_sha256"]}')], + [pick("轨迹 SHA-256", "Trajectory SHA-256"), Safe(f'{record["before"]["trajectory_sha256"]}'), Safe(f'{record["after"]["trajectory_sha256"]}')], + [pick("动作数", "Action count"), record["before"]["action_count"], record["after"]["action_count"]], + ], + ) + ) + body += _section(pick("前后对照", "Before and after"), comparison) + for side, zh, en in (("before", "修复前", "Before"), ("after", "修复后", "After")): + run = record[side] + content = f'

{_e(run["observation"][lang])}

{_e(json.dumps({key: run[key] for key in ("method", "version", "source_base", "source_diff_sha256", "binary_sha256", "run_id")}, indent=2))}
' + if run.get("source_diff_sha256") is None: + content += f'

{pick("该次构建未记录源码差异哈希；只能按已绑定二进制确认身份，不能追认当前源码就是当时构建输入。", "This build did not record a source-diff hash. Identity is bound only to the recorded binary; the current source cannot retrospectively establish its build inputs.")}

' + content += link(run["receipt"], pick("对照凭据 JSON", "Comparison receipt JSON")) + if run.get("image"): + artifact = artifacts[run["image"]] + provenance = artifact["provenance"] + label = pick("最后观测页面", "Last observed page") if provenance["observation"] == "last_observed_page" else pick("精确错误时刻画面", "Exact error-time frame") + content += f'
{_e(label)}
{_e(label)} · {_e(provenance["timing_note"][lang])}
' + else: + content += f'

{pick("此观测没有公开截图。", "No public screenshot is available for this observation.")}

' + body += _section(pick(zh, en), content) + if record["before"].get("image") and record["after"].get("image") and artifacts[record["before"]["image"]]["sha256"] == artifacts[record["after"]["image"]]["sha256"]: + body += f'

{pick("前后截图字节相同，不能作为像素变化证据；状态或几何差异以对照凭据为准。", "The before/after screenshots are byte-identical and do not demonstrate a pixel change; consult the comparison receipts for state or geometry differences.")}

' + body += _section(pick("验证测试", "Validation tests"), "".join( + f'
{_e(test["command"])}

{_e(test["result"])} · {link(test["receipt"], pick("测试凭据 JSON", "Test receipt JSON"))}

' for test in record["tests"] + )) + body += _section(pick("验证边界", "Validation limits"), f'

{_e(record["limitations"][lang])}

{intro}

') + if record.get("pr_url"): + body += f'

{pick("代码变更 PR", "Change PR")}

' + body += f'

{title} · {pick("结构化证据索引", "Structured evidence index")}

' + save(f'{record["id"]}.html', record["title"][lang], body) + body = _hero("Fix evidence", title, intro) + body += _table([pick("修复项", "Fix"), pick("验证范围", "Validation scope")], rows) if rows else f'

{pick("暂无已绑定验证证据的修复记录。", "No fix records with bound validation evidence are available.")}

' + save("index.html", title, body) + if not english: + _write_json(stage / "data/fix-log.json", {**model, "artifacts": list(artifacts.values())}) + + +def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_path: Path, include_english: bool = False) -> Path: profile = _read_json(Path("evaluation/browser-use/current-report.json")) + fix_log = _load_fix_log(profile) _configure_engine_labels(profile) _configure_models(profile) selection = load_pilot(selection_path) + trajectory_replay_bridge = _load_trajectory_replay_bridge( + profile, selection_sha256=selection.sha256 + ) bundles = _load_evidence_bundles(evidence) source_attempts = [row for bundle in bundles for row in bundle["source_attempts"]] manifest, config = bundles[0]["manifest"], bundles[0]["config"] @@ -2717,6 +3064,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat review, cohort_commits, automatic_failure_counts, + trajectory_replay_bridge, ) home = f'
{article}
' (stage / "index.html").write_text( @@ -2902,7 +3250,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat ["项目", "规则"], [ ["Agent / 模型", f"CowAgent；{model}；模型推理强度为低"], - ["唯一变量", "同题、同数据、同模型、同提示和资源上限，只替换浏览器"], + ["计划与实际对照", "计划固定任务、数据、模型、提示和资源上限，仅替换浏览器；历史执行的视口实际不一致，不能视为浏览器是唯一变量的实验"], ["结果口径", "每题×浏览器一个最终逻辑结果；二元通过率；逐题配对"], [ "执行上限", @@ -3231,7 +3579,7 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat ], [ "资源与上限", - f"相同 CPU、内存、viewport 与最多 {max_agent_steps} 步", + f"相同 CPU、内存与最多 {max_agent_steps} 步；历史视口未统一", "控制运行预算", ], ], @@ -3508,6 +3856,12 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat "results": verified_causes, }, ) + if trajectory_replay_bridge is not None: + _write_json(stage / "data/trajectory-replay-bridge.json", trajectory_replay_bridge) + _write_json( + stage / "data/trajectory-replay-failure-review.json", + _read_json(Path(str((trajectory_replay_bridge.get("moli_failure_review") or {}).get("path") or ""))), + ) _write_json( stage / "data/evidence-index.json", { @@ -3672,7 +4026,11 @@ def build_report(*, evidence: Path | Iterable[Path], output: Path, selection_pat ], }, ) - write_english_report(stage) + _write_fix_log(stage, fix_log) + if include_english: + write_english_report(stage) + _write_fix_log(stage, fix_log, english=True) + write_language_contract(stage, include_english=include_english) (stage / ".generated-report").write_text("browser-use-report/2\n", encoding="utf-8") backup: Path | None = None if output.exists(): @@ -3708,8 +4066,9 @@ def main() -> None: type=Path, default=Path("evaluation/browser-use/webarena-verified-pilot.json"), ) + parser.add_argument("--include-english", action="store_true") args = parser.parse_args() - print(build_report(evidence=args.evidence, output=args.output, selection_path=args.selection)) + print(build_report(evidence=args.evidence, output=args.output, selection_path=args.selection, include_english=args.include_english)) if __name__ == "__main__": diff --git a/harness/browser_use/report_i18n.py b/harness/browser_use/report_i18n.py index da71ef985..2ae2fef75 100644 --- a/harness/browser_use/report_i18n.py +++ b/harness/browser_use/report_i18n.py @@ -19,6 +19,7 @@ ("methodology/index.html", "Methodology"), ("analysis/index.html", "Results"), ("review/index.html", "Review"), + ("fixes/index.html", "Fix log"), ("adoption/index.html", "Browsers"), ("reproduce/index.html", "Reproduction"), ("changes/index.html", "Changes"), @@ -27,6 +28,58 @@ TEXT = { + "修复日志": "Fix log", + "配置": "Configuration", + "保留原合同的任务仍有自主执行失败": "Autonomous failures remain under original contracts", + "下表固定排除修订合同任务，按轨迹来源统计自主执行的官方通过数。这是事后诊断，主报告分母仍为 90；历史成功路径经过筛选，不能把此表当成新的能力排名。": "The table excludes the fixed set of revised-contract tasks and counts official autonomous passes by trajectory origin. This is a post-hoc diagnostic; the main report denominator remains 90. Historical paths were selected on success, so this table is not a new capability ranking.", + "全部原合同任务": "All original-contract tasks", + "历史成功路径来源": "Historical-success origins", + "专家修复路径来源": "Expert-repair origins", + "轨迹来源与验收合同": "Trajectory origins and acceptance contracts", + "90 条轨迹中，81 条保留官方原合同，9 条使用修订合同。专家修复轨迹不一定修改验收；历史成功轨迹也可能修订验收。合同差异与路径来源需同时考虑。": "Of 90 trajectories, 81 retain the original official contract and 9 use revised contracts. Expert trajectory repairs do not always change acceptance, and historically successful trajectories may also use revised acceptance. Both contract differences and trajectory origins matter.", + "轨迹来源": "Trajectory origin", + "官方原合同": "Original official contract", + "修订合同": "Revised contract", + "历史成功轨迹": "Historically successful trajectories", + "专家修复轨迹": "Expert-repaired trajectories", + "修订类别": "Revision category", + "验收差异": "Acceptance differences", + "仅修改事件选择": "Event selection only", + "按匹配条件选择保存请求，避免随后请求覆盖。": "Select the matching save request without allowing subsequent requests to replace it.", + "事件选择与预期字段修改": "Event selection and expected fields", + "修订路由、请求字段或 Referer 条件，并匹配所需异步请求。": "Revise routes, request fields, or Referer requirements and match the required asynchronous requests.", + "参考答案或验收要求修改": "Reference answers or acceptance requirements", + "涉及商品名、表单字段、商品选择规则、未提交表单状态及冗余保存要求。": "Changes cover product names, form fields, product selection, unsubmitted form state, and redundant save requirements.", + "轨迹重放把既定 BrowserTool 调用交给浏览器，模型调用数为 0；Chromium 的 90/90 表示这 90 组固定动作、预录回答与绑定验收合同通过了重放验收，其中部分合同经过修订。它不让 Agent 在当次运行中观察、检索、比较、选择或决定何时结束。": "Trajectory Replay sends predetermined BrowserTool calls to the browser with zero model calls. Chromium's 90/90 means these 90 combinations of fixed actions, prerecorded responses, and bound acceptance contracts passed replay validation; some contracts were revised. The Agent does not observe, retrieve, compare, select, or decide when to stop during the run.", + "验收使用轨迹预录回答（recorded_response）与本次网络记录（HAR）。对于检索问答题，通过不证明当前浏览器读到了所需数据，也不证明 Agent 在本次运行中计算出了答案；要求网络事件的合同仍会检查本次 HAR 是否包含对应请求。": "Acceptance uses the trajectory's prerecorded response (recorded_response) and the current run's network record (HAR). For retrieval questions, a pass does not prove that the current browser read the required data or that the Agent computed the answer during this run. Contracts requiring network events still check the current HAR for the corresponding requests.", + "固定动作、预录回答与绑定合同的重放验收结果。": "Replay acceptance for fixed actions, prerecorded responses, and bound contracts.", + "90 题中的 37 道检索题均只使用回答验收，不要求网络事件：36 题沿用原合同，Task 279 使用修订合同。这 37 题的本轮读数与答案尚未验证；原评分保留，其余 53 题也不能替代原 90 题成为新的排名。": "All 37 retrieval tasks in the 90-task set use answer-only acceptance without requiring network events: 36 retain the original contract and Task 279 uses a revised contract. Data reading and answers from the current run remain unverified for these 37 tasks. Original scores are retained; the other 53 tasks cannot replace the original 90 as a new ranking.", + "Task 676 · Sol × Chromium · 验收证据": "Task 676 · Sol × Chromium · Acceptance evidence", + "：官方合同要求带 filters[status]=fraud 的订单列表 XHR。HAR 已记录该请求、200 状态和正确 Referer；": ": The official contract requires the orders-grid XHR with filters[status]=fraud. The HAR records that request, status 200, and the correct Referer; ", + "操作轨迹": "Action trajectory", + "显示 Suspected Fraud 已生效。NetworkEventEvaluator._filter_events_by_criteria 却对 NAVIGATE + GET 只保留最后的文档导航，遗漏 XHR；默认 last_event_only 又可能在同路径请求中选中后来的 notification_area。两个模型的 Chromium 结果都受此影响。": " shows Suspected Fraud active. NetworkEventEvaluator._filter_events_by_criteria keeps only the last document navigation for NAVIGATE + GET, omitting the XHR. The default last_event_only can also select a later notification_area request on the same path. Chromium results for both models are affected.", + "Moli 的同题证据": "Moli evidence for the same task", + "还缺少 XHR Referer。重放修订既改变事件选择，也移除了 Referer 条件；后者并非 Chromium 这次误判的原因。四个配置的原始失败分数保持不变。": " also lacks the XHR Referer. The replay revision changes event selection and removes the Referer requirement; removing that requirement does not explain Chromium's false negative. The original failure scores for all four configurations are unchanged.", + "从失败结果追到决策与运行设置": "Tracing failures to decisions and runtime settings", + "Task 124 · Astra × Chromium": "Task 124 · Astra × Chromium", + "：Agent 在第 9 个决策就结束，未耗尽 40 步。轨迹把 wireless earphone 查询扩大成 earphone，工具结果仍显示 Items 1 to 12 of 24 total，最终却提交 min=0.01、max=287.64。失败发生在候选资格与检索完整性判断；轨迹中有可读结果，不能用浏览器不可操作或步骤不足解释。当前运行器依赖模型执行提示中的完成自检，没有独立验证候选全集。提示要求自检并不等于已经完成自检。": ": The Agent stopped after decision 9, before the 40-step limit. It broadened the query from wireless earphone to earphone; the tool still showed Items 1 to 12 of 24 total, yet the final answer gave min=0.01 and max=287.64. Candidate eligibility and retrieval completeness failed despite readable tool results. The runner relies on the model to perform the prompted completion check and does not independently verify the candidate set. A requested self-check is not evidence that the check occurred.", + "Task 510 · Sol × Moli": "Task 510 · Sol × Moli", + "：第 40 个决策仍在调用工具，最终回答承认尚未结账。harness/browser_use/runner.py 的 _exhausted_agent_steps 检测这一状态，并通过 _max_steps_response 标记未完成。这证明预算上限中断了流程，但所选 Hiquphon Tweeters 本身也不符合官方目标；增加预算能否完成正确购买仍需对照实验。": ": Decision 40 still called a tool, and the final answer acknowledged that checkout was unfinished. In harness/browser_use/runner.py, _exhausted_agent_steps detects this condition and _max_steps_response marks the attempt incomplete. The budget interrupted execution, but the selected Hiquphon Tweeters also differed from the official target. Whether a larger budget enables the correct purchase requires a controlled experiment.", + "两模型均使用 reasoning_effort=low 和 40 步上限。harness/browser_use/model.py 限定 low，runner.py 在创建适配器时也传入 low。现有分数只反映这套 Agent、工具与推理设置；尚无固定观察输入、仅改变推理档位的实验，不能把分数接近解释为模型能力相同。": "Both models used reasoning_effort=low and a 40-step limit. harness/browser_use/model.py requires low, and runner.py also supplies low when creating the adapter. The scores describe this Agent, tool, and reasoning configuration. No experiment held observations fixed while changing only reasoning effort, so similar scores do not establish equal model capability.", + "固定轨迹控制：它验证浏览器执行，不测自主规划": "Fixed-trajectory control: it tests browser execution, not autonomous planning", + "Chromium 的 90/90 不是 CowAgent 的 90/90。": "Chromium's 90/90 is not CowAgent's 90/90.", + "轨迹重放把既定 BrowserTool 调用交给浏览器，模型调用数为 0；它证明 Chromium 能在逐题重置的环境执行这 90 条合格轨迹，并通过同一官方 evaluator。它不让 Agent 在当次运行中观察、检索、比较、选择或决定何时结束。": "Trajectory Replay sends predetermined BrowserTool calls to the browser with zero model calls. It establishes that Chromium can execute these 90 qualified trajectories in per-task reset environments and pass the same official evaluator; it does not let an agent observe, retrieve, compare, select, or decide when to stop in that run.", + "Chromium：90/90": "Chromium: 90/90", + "固定轨迹执行与官方验收的上界控制。": "An upper-bound control for fixed-trajectory execution and official acceptance.", + "Moli：54/90": "Moli: 54/90", + "同一固定路径仍有 36 条失败，不能把浏览器差异归零。": "The same fixed paths still have 36 failures, so browser differences cannot be treated as zero.", + "路径来源：30 + 60": "Trajectory origin: 30 + 60", + "30 条专家修复路径，加 60 条历史成功的 Browser Use 路径。": "30 expert-repair trajectories plus 60 historically successful Browser Use trajectories.", + "后 60 条在进入重放集之前已经按“成功”条件筛选，其中 56 条来自 Sol × Chromium、3 条来自 Sol × Moli、1 条来自 Astra × Chromium。因此固定轨迹成绩受给定路径、预录回答和验收合同共同约束，不能作为 Agent 决策能力、模型能力或完整 90 题自主通过率的反事实估计。": "The latter 60 were selected on prior success before entering the replay set: 56 came from Sol × Chromium, 3 from Sol × Moli, and 1 from Astra × Chromium. The fixed-trajectory result is constrained by the given paths, prerecorded responses, and acceptance contracts; it is not a counterfactual estimate of agent decision-making, model capability, or autonomous pass rate across all 90 tasks.", + "历史复核曾将 Moli 的 36 个固定轨迹失败归为浏览器责任；新发现的视口设置缺陷要求重新核对这一归因。历史 Browser Use 的 Chromium 实际宽度为 780，Moli 为 1920；Task 173 的搜索链接在宽度达到 992 时由网页 CSS 隐藏。该任务的失败不能单凭可见性超时归因于浏览器。历史分数保留，统一视口后的比较须重新冻结完整任务范围与轨迹资格。": "The historical review attributed all 36 Moli fixed-trajectory failures to the browser; the newly identified viewport configuration defect requires that attribution to be reassessed. Historical Browser Use traces show an actual width of 780 for Chromium and 1920 for Moli. Task 173's search link is hidden by site CSS at widths of 992 or more. Its visibility timeout alone cannot establish browser fault. Historical scores remain unchanged; comparison at a common viewport requires a newly frozen full task scope and trajectory qualification.", + "数据绑定：": "Data bindings:", + "跨轨迹控制摘要": "Cross-track control summary", + "36 个 Moli 固定轨迹失败的逐题复核": "Per-task review of 36 Moli fixed-trajectory failures", "完整执行证据": "Complete execution evidence", "按原始结构展示本次物理执行的脱敏回答、evaluator、事件流、轨迹、计量与公开画面索引。": "Shows the sanitized response, evaluator record, event stream, trajectory, measurements, and public-frame index for this physical execution in their original structure.", "执行证据 JSON": "Execution evidence JSON", @@ -280,6 +333,10 @@ "项目": "Item", "规则": "Rule", "唯一变量": "Only changed variable", + "计划与实际对照": "Planned and actual controls", + "计划固定任务、数据、模型、提示和资源上限，仅替换浏览器；历史执行的视口实际不一致，不能视为浏览器是唯一变量的实验": "The plan held tasks, data, model, prompt, and resource limits fixed while changing the browser. Actual historical viewports differed, so the browser was not the only experimental variable.", + "历史轨迹记录的视口为 Chromium 780×437/493、Moli 1920×1080。视口差异会影响渲染、CPU 和内存；以下数值与最低值标记仅描述各自实际运行条件，不是统一视口下的资源效率结论。正式效率比较须统一设置后完整重跑。": "Historical traces recorded Chromium viewports of 780×437/493 and Moli viewports of 1920×1080. Viewport differences affect rendering, CPU, and memory. The values and minimum-value markers describe the actual conditions of each run, not resource efficiency at a common viewport. A formal efficiency comparison requires a full rerun with consistent settings.", + "共同成功子集也保留了历史视口差异，不能据此得出统一视口下的资源效率结论。Agent 链路还包含模型推理和页面等待，因此不能解读为纯内核性能。": "The common-success subset also retains the historical viewport differences and cannot establish resource efficiency at a common viewport. Agent-path time includes model inference and page waits, so it does not measure browser-core performance alone.", "主分": "Primary metric", "排除": "Exclusions", "裁判": "Judge", @@ -2067,8 +2124,8 @@ def translate_text(value: str) -> str: lambda m: f"Same {m.group(1)} tasks; same local site images", ), ( - r"相同 CPU、内存、viewport 与最多 (.+) 步", - lambda m: f"Same CPU, memory, viewport, and up to {'an unrecorded number of' if m.group(1) == '未记录' else m.group(1)} steps", + r"相同 CPU、内存与最多 (.+) 步；历史视口未统一", + lambda m: f"Same CPU, memory, and up to {'an unrecorded number of' if m.group(1) == '未记录' else m.group(1)} steps; historical viewports differed", ), ( r"这 (\d+) 题由 CowAgent 自主执行。", diff --git a/harness/browser_use/runner.py b/harness/browser_use/runner.py index 643de58d9..af49e6895 100644 --- a/harness/browser_use/runner.py +++ b/harness/browser_use/runner.py @@ -462,11 +462,123 @@ def _prepare_sites(sites: tuple[WebArenaSite, ...]) -> None: _ensure_site(site, reset=True) +BENCHMARK_VIEWPORT = {"width": 1280, "height": 900} + + +def _browser_session_contract() -> dict[str, Any]: + return { + "mode": "owned_cdp_context", + "viewport": dict(BENCHMARK_VIEWPORT), + "postmill_auth": "single_request_bootstrap", + "verification": "primary and inherited new page before navigation, then after execution", + } + + +def _browser_tool_config(endpoint: str) -> dict[str, Any]: + return { + "cdp_endpoint": endpoint, + "headless": True, + "viewport_width": BENCHMARK_VIEWPORT["width"], + "viewport_height": BENCHMARK_VIEWPORT["height"], + "cdp_isolated_context": True, + "startup_timeout": 90, + } + + +def _record_page_viewport(page: Any, evidence: dict[str, Any], phase: str) -> None: + try: + observed = page.evaluate( + "() => ({width: innerWidth, height: innerHeight, " + "device_scale_factor: devicePixelRatio})" + ) + except Exception: + evidence["verified"] = False + raise + evidence["observations"].append({"phase": phase, **observed}) + if {key: observed.get(key) for key in BENCHMARK_VIEWPORT} != BENCHMARK_VIEWPORT: + evidence["verified"] = False + raise RuntimeError(f"browser viewport mismatch during {phase}: {observed}") + + +def _verify_session_viewport(service: Any, evidence: dict[str, Any], phase: str) -> None: + def verify() -> None: + pages = service._context.pages + if not pages: + evidence["verified"] = False + raise RuntimeError("browser viewport cannot be verified without a live page") + for page in pages: + _record_page_viewport(page, evidence, phase) + + service._submit(verify) + + +def _prepare_browser_viewport(service: Any) -> dict[str, Any]: + evidence: dict[str, Any] = { + "requested": dict(BENCHMARK_VIEWPORT), + "mode": "owned_cdp_context", + "verified": False, + "observations": [], + "resource_scope": "whole browser process, including pre-existing default contexts", + } + + def prepare() -> None: + if not getattr(service, "_owns_cdp_context", False): + raise RuntimeError("CowAgent lacks the required owned CDP context contract") + _record_page_viewport(service._page, evidence, "primary_before_navigation") + # Verify inheritance using a blank child before any login/site script. + child = service._context.new_page() + try: + _record_page_viewport(child, evidence, "new_page_before_navigation") + finally: + child.close() + evidence["retained_external_context_count"] = sum( + context != service._context for context in service._browser.contexts + ) + evidence["verified"] = True + + service._submit(prepare) + return evidence + + +def _bootstrap_postmill_session(context: Any, base_url: str) -> None: + """Authenticate once, without loading subresources or following redirects. + + The image's header authenticator migrates (and destroys) the previous + session on every authenticated request. A persistent context header makes + concurrent thumbnail requests race with CSRF form sessions. The context's + API client shares the browser cookie jar but does not execute page content. + Use the configured, host-accessible origin, not a browser-only DNS alias. + """ + creds = SITE_CREDENTIALS[WebArenaSite.REDDIT] + response = context.request.get( + base_url, + headers={ + SITE_AUTH_HEADERS[WebArenaSite.REDDIT]: (f"{creds['username']}:{creds['password']}") + }, + max_redirects=0, + max_retries=0, + timeout=90_000, + ) + try: + if not 200 <= response.status < 300: + raise RuntimeError( + f"Postmill session bootstrap requires a direct 2xx response; got {response.status}" + ) + if not any( + cookie.get("name") == "PHPSESSID" and cookie.get("value") + for cookie in context.cookies(base_url) + ): + raise RuntimeError("Postmill session bootstrap did not establish a session cookie") + finally: + response.dispose() + + def _login_context( service: Any, sites: tuple[WebArenaSite, ...], benchmark: WebArenaVerified, start_urls: list[str], + viewport_evidence: dict[str, Any] | None = None, ) -> int: def login() -> None: context = service._context @@ -483,7 +595,7 @@ def login() -> None: f"{SITE_CREDENTIALS[site]['username']}:{SITE_CREDENTIALS[site]['password']}" ) for site in sites - if site in SITE_AUTH_HEADERS + if site in SITE_AUTH_HEADERS and site != WebArenaSite.REDDIT } ) for site in sites: @@ -491,6 +603,8 @@ def login() -> None: creds = SITE_CREDENTIALS[site] page = context.new_page() try: + if viewport_evidence is not None: + _record_page_viewport(page, viewport_evidence, "login_before_navigation") page.set_default_timeout(60_000) page.set_default_navigation_timeout(90_000) if site == WebArenaSite.SHOPPING: @@ -504,6 +618,7 @@ def login() -> None: page.reload(wait_until="domcontentloaded") page.locator(".page-title").wait_for() elif site == WebArenaSite.REDDIT: + _bootstrap_postmill_session(context, base) page.goto(base, wait_until="commit") page.evaluate("localStorage.clear(); sessionStorage.clear()") page.reload(wait_until="domcontentloaded") @@ -689,14 +804,7 @@ def _run_attempt_locked( # noqa: PLR0913 response_path = attempt_dir / "agent_response.json" eval_path = attempt_dir / "eval_result.json" events_path = attempt_dir / "agent_events.json" - tool = BrowserTool( - config={ - "cdp_endpoint": endpoint, - "headless": True, - "viewport": {"width": 1280, "height": 900}, - "startup_timeout": 90, - } - ) + tool = BrowserTool(config=_browser_tool_config(endpoint)) _allow_official_container_hostname(tool) events: list[dict[str, Any]] = [] resources = DockerStatsResources(container) @@ -705,6 +813,7 @@ def _run_attempt_locked( # noqa: PLR0913 raw_response = "" error = "" prepared = False + viewport_evidence: dict[str, Any] = {} recorder = CdpHarRecorder() sampler_started = False browser_session_setup_ms = 0.0 @@ -718,7 +827,11 @@ def _run_attempt_locked( # noqa: PLR0913 model.reset() service = tool._get_service() service._submit(lambda: None) - browser_session_readiness_retries = _login_context(service, sites, benchmark, start_urls) + viewport_evidence = _prepare_browser_viewport(service) + browser_session_readiness_retries = _login_context( + service, sites, benchmark, start_urls, viewport_evidence + ) + _verify_session_viewport(service, viewport_evidence, "after_login") service._submit( lambda: ( recorder.start(service._context), @@ -754,6 +867,7 @@ def _run_attempt_locked( # noqa: PLR0913 _max_steps_response(str(response_eval.expected.task_type), max_agent_steps) ) agent_run_ms = (time.perf_counter() - agent_started) * 1000 + _verify_session_viewport(service, viewport_evidence, "after_agent") finalize_started = time.perf_counter() service._submit( lambda: ( @@ -793,7 +907,7 @@ def _run_attempt_locked( # noqa: PLR0913 except Exception as exc: cleanup_errors.append(f"browser tool close failed: {exc}") - if not prepared: + if not prepared or not viewport_evidence.get("verified"): raise RuntimeError( f"{engine} task {selected.task_id}: environment precondition failed\n{error}" ) @@ -841,6 +955,7 @@ def _run_attempt_locked( # noqa: PLR0913 "measured_run_wall_ms": round(duration_ms, 3), }, "browser_session_readiness_retries": browser_session_readiness_retries, + "viewport": viewport_evidence, "agent_response": parsed_response, "error": error, "cleanup_errors": cleanup_errors, @@ -943,9 +1058,7 @@ def run(args: argparse.Namespace) -> Path: # noqa: PLR0915 require_linux_cgroup_v2(client) docker_info = client.info() physical_execution_lock_namespace = _docker_daemon_lock_namespace(docker_info) - physical_execution_lock_path = _physical_execution_lock_path( - physical_execution_lock_namespace - ) + physical_execution_lock_path = _physical_execution_lock_path(physical_execution_lock_namespace) run_id = uuid.uuid4().hex[:12] containers: dict[str, Any] = {} networks: dict[str, Any] = {} @@ -1000,7 +1113,6 @@ def run(args: argparse.Namespace) -> Path: # noqa: PLR0915 site_access_host = gateway_addresses.pop() benchmark = _configure_benchmark(hostname=site_access_host) resolved = validate_against_official(selection, benchmark) - _write_or_validate_run_config( run_config_file, { @@ -1008,6 +1120,7 @@ def run(args: argparse.Namespace) -> Path: # noqa: PLR0915 "selection_sha256": selection.sha256, "browser_eval_commit": harness_commit, "cowagent_commit": cowagent_commit, + "browser_session_contract": _browser_session_contract(), "model": args.model, "reasoning_effort": "low", "model_transport_retries": 1, @@ -1053,11 +1166,21 @@ def run(args: argparse.Namespace) -> Path: # noqa: PLR0915 ), flush=True, ) - if float(row["score"]) == 1.0: + if ( + float(row.get("score") or 0) == 1.0 + or row.get("status") == "unsupported" + ): break continue primary = completed.get((args.model, engine, selected.task_id, 1)) - if attempt == 2 and primary is not None and float(primary["score"]) == 1.0: + if ( + attempt == 2 + and primary is not None + and ( + float(primary.get("score") or 0) == 1.0 + or primary.get("status") == "unsupported" + ) + ): break row = _run_attempt( lock_path=physical_execution_lock_path, @@ -1142,6 +1265,7 @@ def run(args: argparse.Namespace) -> Path: # noqa: PLR0915 "official_container_hostname_allowlist": ["host.docker.internal"], }, "environment": { + "browser_session_contract": _browser_session_contract(), "platform": platform.platform(), "docker_server_version": client.version().get("Version"), "docker_architecture": docker_info.get("Architecture"), diff --git a/harness/fix_log.py b/harness/fix_log.py new file mode 100644 index 000000000..eb350812f --- /dev/null +++ b/harness/fix_log.py @@ -0,0 +1,313 @@ +"""Validate the shared, hash-bound browser fix-log evidence model.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable +from pathlib import Path, PurePosixPath +from typing import Any + +from PIL import Image + +EMPTY_FIX_LOG = { + "schema": "lex_browser_use_fix_log/1", + "records": [], + "artifacts": [], +} + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def load_fix_log( + profile: dict[str, Any], *, sanitize_value: Callable[[Any], Any] +) -> dict[str, Any]: + """Read and validate an optional public fix manifest. + + The manifest can point to local source artifacts, but every reader-facing + value must already survive the caller's publication sanitizer unchanged. + Raw traces, HAR files, logs, and arbitrary attachment types are rejected. + """ + + binding = profile.get("fix_log") + if not binding: + return {**EMPTY_FIX_LOG, "records": [], "artifacts": []} + source = Path(binding["source_path"]) + if ( + source.is_symlink() + or any(parent.is_symlink() for parent in source.parents) + or not source.is_file() + or _sha256(source) != binding["source_sha256"] + ): + raise ValueError("fix log is not hash-bound") + model = _read_json(source) + if model.get("schema") != EMPTY_FIX_LOG["schema"]: + raise ValueError("unsupported fix log schema") + if set(model) != {"schema", "records", "artifacts"}: + raise ValueError("fix log contains unsupported fields") + + def digest(value: Any) -> None: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("fix log requires an exact SHA-256") + + def localized(value: Any) -> None: + if ( + not isinstance(value, dict) + or set(value) != {"zh", "en"} + or not all(isinstance(text, str) and text.strip() for text in value.values()) + ): + raise ValueError("fix log prose requires both languages") + + def public_text(value: Any) -> None: + text = json.dumps(value, ensure_ascii=False) + if sanitize_value(value) != value or re.search( + r"/(?:Users|home|root|private|tmp|var|opt)/", text + ): + raise ValueError("fix log contains private data or local paths") + if re.search( + r'"(?:authorization|cookie|set-cookie|password|access_token|api_key)"\s*:', + text, + re.I, + ): + raise ValueError("fix log receipt contains credential fields") + + artifacts: dict[str, dict[str, Any]] = {} + for artifact in model.get("artifacts", []): + artifact_id = artifact["id"] + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", artifact_id) or artifact_id in artifacts: + raise ValueError("invalid or duplicate fix artifact ID") + path = Path(artifact["path"]) + digest(artifact["sha256"]) + if ( + artifact.get("public_safe") is not True + or path.is_symlink() + or any(parent.is_symlink() for parent in path.parents) + or not path.is_file() + or _sha256(path) != artifact["sha256"] + ): + raise ValueError("fix artifact is unsafe, missing or not hash-bound") + kind = artifact.get("kind") + if kind == "image": + with Image.open(path) as image: + if image.format not in {"PNG", "JPEG"}: + raise ValueError("fix images must be measured PNG/JPEG captures") + artifact["extension"] = ".png" if image.format == "PNG" else ".jpg" + image.verify() + provenance = artifact["provenance"] + if ( + provenance.get("observation") not in {"last_observed_page", "exact_error_frame"} + or not provenance.get("capture_method") + or not provenance.get("run_id") + ): + raise ValueError("fix image requires observation provenance") + localized(provenance.get("timing_note")) + if provenance["observation"] == "exact_error_frame": + localized(provenance.get("error_time_basis")) + public_text(provenance) + elif kind == "receipt": + if path.stat().st_size > 1_000_000: + raise ValueError("fix receipt exceeds compact public evidence limit") + receipt = _read_json(path) + if receipt.get("schema") not in { + "lex_browser_use_fix_test_receipt/1", + "lex_browser_use_fix_comparison_receipt/1", + "lex_browser_use_fix_certification_receipt/1", + }: + raise ValueError("fix artifacts require allowlisted public receipt schemas") + allowed = { + "schema", + "run_id", + "method", + "version", + "source_base", + "source_diff_sha256", + "binary_sha256", + "input_sha256", + "trajectory_sha256", + "task_contract_sha256", + "action_count", + "command", + "result", + "measurements", + "source_diff_status", + "status", + "scope", + "task_ids", + "comparison_receipts_sha256", + } + if set(receipt) - allowed: + raise ValueError( + "fix receipt contains unsupported fields; raw logs are not public receipts" + ) + public_text(receipt) + else: + raise ValueError("unsupported fix artifact kind") + artifacts[artifact_id] = artifact + public_text({key: value for key, value in artifact.items() if key != "path"}) + + def reference(identifier: str, kind: str) -> None: + if identifier not in artifacts or artifacts[identifier]["kind"] != kind: + raise ValueError("fix record references missing or incompatible evidence") + + ids: set[str] = set() + for record in model.get("records", []): + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", record["id"]) or record["id"] in ids: + raise ValueError("invalid or duplicate fix record ID") + ids.add(record["id"]) + public_text(record) + for key in ("title", "symptom", "root_cause", "principle", "limitations"): + localized(record.get(key)) + if record.get("scope") not in {"diagnostic", "formal"}: + raise ValueError("fix record must distinguish diagnostic and formal scope") + basis = record.get("comparison_basis") + if basis is None: + basis = {"kind": "identical_inputs"} + record["comparison_basis"] = basis + if not isinstance(basis, dict): + raise ValueError("fix comparison basis must be an object") + basis_kind = basis.get("kind") + if basis_kind == "identical_inputs": + if set(basis) != {"kind"}: + raise ValueError("identical-input comparison basis has unsupported fields") + digest(record.get("input_sha256")) + elif basis_kind == "same_task_contract": + if set(basis) != {"kind", "task_contract_sha256"}: + raise ValueError("same-task comparison requires one task contract binding") + digest(basis.get("task_contract_sha256")) + if "input_sha256" in record: + raise ValueError("revised trajectories cannot claim one shared input") + else: + raise ValueError("unsupported fix comparison basis") + if record["scope"] == "formal": + reference(record.get("certification"), "receipt") + certification = _read_json(Path(artifacts[record["certification"]]["path"])) + certification_key = ( + "input_sha256" if basis_kind == "identical_inputs" else "task_contract_sha256" + ) + certification_value = ( + record["input_sha256"] + if basis_kind == "identical_inputs" + else basis["task_contract_sha256"] + ) + comparison_receipts = certification.get("comparison_receipts_sha256") + if not isinstance(comparison_receipts, dict) or set(comparison_receipts) != { + "before", + "after", + }: + raise ValueError("formal fix evidence must bind both comparison receipts") + for receipt_sha256 in comparison_receipts.values(): + digest(receipt_sha256) + expected_comparison_receipts = {} + for side in ("before", "after"): + comparison_id = record.get(side, {}).get("receipt") + reference(comparison_id, "receipt") + expected_comparison_receipts[side] = artifacts[comparison_id]["sha256"] + if ( + certification.get("schema") != "lex_browser_use_fix_certification_receipt/1" + or certification.get("status") != "certified" + or certification.get(certification_key) != certification_value + or comparison_receipts != expected_comparison_receipts + ): + raise ValueError("formal fix evidence requires a matching certification receipt") + if not record.get("code_paths") or not record.get("tests"): + raise ValueError("fix record requires code locations and tests") + for code in record["code_paths"]: + if not code.get("repository") or not code.get("symbol"): + raise ValueError("fix code location is incomplete") + code_path = PurePosixPath(code["path"]) + if code_path.is_absolute() or ".." in code_path.parts: + raise ValueError("fix code paths must be repository-relative") + for side in ("before", "after"): + run = record[side] + for key in ("method", "version", "source_base", "run_id"): + if not isinstance(run.get(key), str) or not run[key].strip(): + raise ValueError("fix comparison identity is incomplete") + digest(run.get("binary_sha256")) + if run.get("source_diff_sha256") is None: + if ( + record["scope"] != "diagnostic" + or run.get("source_diff_status") != "not_recorded" + ): + raise ValueError( + "missing source diff hash requires explicit not_recorded status" + ) + else: + digest(run["source_diff_sha256"]) + localized(run.get("observation")) + reference(run.get("receipt"), "receipt") + receipt = _read_json(Path(artifacts[run["receipt"]]["path"])) + if basis_kind == "identical_inputs": + if any(key in run for key in ("input_sha256", "trajectory_sha256", "action_count")): + raise ValueError("identical-input comparisons use the shared record input") + basis_matches_receipt = receipt.get("input_sha256") == record["input_sha256"] + else: + digest(run.get("input_sha256")) + digest(run.get("trajectory_sha256")) + if ( + not isinstance(run.get("action_count"), int) + or isinstance(run.get("action_count"), bool) + or run["action_count"] < 1 + ): + raise ValueError("revised trajectory requires a positive action count") + basis_matches_receipt = all( + receipt.get(key) == value + for key, value in ( + ("task_contract_sha256", basis["task_contract_sha256"]), + ("input_sha256", run["input_sha256"]), + ("trajectory_sha256", run["trajectory_sha256"]), + ("action_count", run["action_count"]), + ) + ) + if ( + receipt.get("schema") != "lex_browser_use_fix_comparison_receipt/1" + or any( + receipt.get(key) != run[key] + for key in ( + "method", + "version", + "source_base", + "source_diff_sha256", + "binary_sha256", + "run_id", + ) + ) + or not basis_matches_receipt + ): + raise ValueError("fix comparison claims do not match the bound receipt") + if ( + run.get("source_diff_sha256") is None + and receipt.get("source_diff_status") != "not_recorded" + ): + raise ValueError("missing source diff provenance must match the receipt") + if run.get("image"): + reference(run["image"], "image") + if basis_kind == "same_task_contract": + before = record["before"] + after = record["after"] + if ( + before["input_sha256"] == after["input_sha256"] + or before["trajectory_sha256"] == after["trajectory_sha256"] + ): + raise ValueError("same-task comparison must bind distinct revised trajectories") + for test in record["tests"]: + if not test.get("command") or not test.get("result"): + raise ValueError("fix test requires exact command and recorded result") + reference(test.get("receipt"), "receipt") + receipt = _read_json(Path(artifacts[test["receipt"]]["path"])) + if ( + receipt.get("schema") != "lex_browser_use_fix_test_receipt/1" + or receipt.get("command") != test["command"] + or receipt.get("result") != test["result"] + ): + raise ValueError("fix test claims do not match the bound receipt") + if record.get("pr_url") and not re.fullmatch(r"https://[^\s<>\"']+", record["pr_url"]): + raise ValueError("fix PR must use an HTTPS URL") + model["source_sha256"] = binding["source_sha256"] + return model diff --git a/harness/report_languages.py b/harness/report_languages.py new file mode 100644 index 000000000..9fdbb2b71 --- /dev/null +++ b/harness/report_languages.py @@ -0,0 +1,54 @@ +"""Explicit language output contract shared by report tracks.""" + +import json +from pathlib import Path + + +def generated_languages(root: Path) -> tuple[str, ...]: + path = root / "report-languages.json" + if not path.exists(): + return ("zh", "en") + value = json.loads(path.read_text(encoding="utf-8")) + languages = value.get("languages") + if languages not in (["zh"], ["zh", "en"]): + raise ValueError("unsupported generated report languages") + return tuple(languages) + + +def write_language_contract(root: Path, *, include_english: bool) -> None: + (root / "report-languages.json").write_text( + json.dumps({"languages": ["zh", "en"] if include_english else ["zh"], + "english_status": "generated" if include_english else "not_generated"}) + "\n", + encoding="utf-8", + ) + if include_english: + return + folder = root / "en" + folder.mkdir(exist_ok=True) + if any(folder.iterdir()): + raise ValueError("English output must not be generated in Chinese-only mode") + (folder / "index.html").write_text( + '' + '' + '' + '英文报告未生成
' + '

英文报告未生成

English report has not been generated.

' + '

当前报告仅提供中文版。英文版需单独生成。

' + '返回中文报告
', + encoding="utf-8", + ) + + +def check_language_contract(root: Path) -> None: + if "en" in generated_languages(root): + return + files = {p.relative_to(root / "en").as_posix() for p in (root / "en").rglob("*") if p.is_file()} + if files != {"index.html"}: + raise ValueError("Chinese-only report contains stale English output") + notice = (root / "en/index.html").read_text(encoding="utf-8") + if "data-english-not-generated" not in notice or 'href="../index.html"' not in notice: + raise ValueError("English not-generated notice is missing") diff --git a/harness/trajectory_replay/browser_use_seeds.py b/harness/trajectory_replay/browser_use_seeds.py index aa1fa5e98..12e683470 100644 --- a/harness/trajectory_replay/browser_use_seeds.py +++ b/harness/trajectory_replay/browser_use_seeds.py @@ -115,6 +115,22 @@ def _selector_for_ref(nodes: list[dict[str, Any]], ref: int, *, context: str) -> matches = [node for node in nodes if _same_selector_population(target, node)] if len(matches) == 1: return base + # Shared hrefs often identify unrelated menu entries. Keep the original + # element (not its text-bearing child), but prefer its complete label when + # that label uniquely narrows the snapshot population. Playwright has-text + # normalizes whitespace and matches case-insensitive substrings; use the + # same conservative population check rather than assuming exact matching. + text = " ".join(str(target.get("text") or "").split()) + labels = [" ".join(str(node.get("text") or "").split()) for node in matches] + if ( + target["attributes"].get("href") + and text + and not any(label.endswith(("...", "…")) for label in labels) + and sum(text.lower() in label.lower() for label in labels) == 1 + ): + return f"{base}:has-text({_quoted(text)})" + # Missing, truncated or ambiguous labels retain the precise original ref's + # ordinal. Do not silently replace it with the first matching text node. occurrence = next(index for index, node in enumerate(matches, start=1) if node is target) return f":nth-match({base}, {occurrence})" diff --git a/harness/trajectory_replay/failure_matrix.py b/harness/trajectory_replay/failure_matrix.py new file mode 100644 index 000000000..a9b1ff193 --- /dev/null +++ b/harness/trajectory_replay/failure_matrix.py @@ -0,0 +1,108 @@ +"""Disjoint, evidence-derived failure counts for the replay result inventory.""" + +from collections import Counter +from html import escape +from urllib.parse import urlencode + + +# These are diagnostic families, not assertions that the browser owns the fault. +FAMILIES = ( + ("geometry", "布局与可见性", "Layout and visibility", ( + "zero_area_target_geometry", "zero_width_forum_link", + "zero_width_candidate_changes_visible_ordinal", + "visible_locator_timeout_without_geometry", + "suggestion_dom_selection", + "configuration_wizard_readiness_predicate", + )), + ("interaction", "输入与动作效果", "Input and action effects", ( + "input_navigation_lifecycle", "disabled_control_readiness_race", + "implicit_search_submission_absent", + )), + ("network", "网络传输与记录", "Network transport and recording", ( + "fetch_transport_timeout", "recorded_referer_mismatch", + )), + ("document", "文档表示", "Document representation", ( + "json_document_dom_contract", + )), +) + + +def failure_category_matrix(methods: dict, rows: list[dict], lang: str, *, embedded: bool = False) -> str: + """Count each failed method-task once, including unreviewed failures.""" + zh = lang == "zh" + labels = {} + counts = Counter() + totals = Counter() + population = Counter() + seen = set() + for row in rows: + method = row["participant_id"] + identity = (method, row["task_id"]) + if identity in seen: + raise ValueError("duplicate method-task result in failure matrix") + if method not in methods: + raise ValueError("unregistered method in failure matrix") + seen.add(identity) + population[method] += 1 + if row["status"] == "pass": + continue + code = row.get("cause_code") or "unreviewed" + labels[code] = ( + row.get("cause_zh") or "未复核", + row.get("cause_en") or "Unreviewed", + ) + counts[method, code] += 1 + totals[method] += 1 + + families = [] + assigned = set() + for family, cn, en, codes in FAMILIES: + present = [code for code in codes if code in labels] + if present: + families.append((family, cn if zh else en, present)) + assigned.update(present) + remaining = sorted(set(labels) - assigned) + if remaining: + families.append(("other", "其他／未复核" if zh else "Other / unreviewed", remaining)) + columns = [code for _, _, codes in families for code in codes] + groups = "".join( + f'{escape(label)}' + for family, label, codes in families + ) + headers = "".join( + f'{escape(labels[code][0 if zh else 1])}' + for code in columns + ) + body = [] + for method, entry in methods.items(): + cells = [] + for code in columns: + count = counts[method, code] + query = {"participant": method, "status": "fail"} + # The existing inventory uses the Chinese review label as a stable + # filter value in both localizations. + if code != "unreviewed": + query["cause"] = labels[code][0] + url = "results.html?" + urlencode(query) + "#result-inventory" + value = f'{count}' if count and code != "unreviewed" else str(count) + cells.append(f'{value}') + body.append( + f'{escape(entry["label"])}' + f'{totals[method]} / {population[method]}' + + "".join(cells) + "" + ) + title = "失败分布：大类与子类" if zh else "Failures by family and subtype" + note = ( + "每个方法的全部任务为分母；每条失败只计入一个子类。横向滚动查看全部分类，点击计数查看对应题目。分类描述故障位置，责任归属与根因证据见逐题诊断。" + if zh else + "The denominator is all tasks for each method. Each failure belongs to one subtype. Scroll horizontally for all categories and select a count to inspect its cases; fault ownership and root-cause evidence are in each diagnosis." + ) + tag = "div" if embedded else "section" + heading = "" if embedded else f"

{title}

" + return ( + f'<{tag} data-failure-matrix>{heading}

{note}

' + f'
' + f'' + f'' + f'{groups}{headers}{"".join(body)}
{"方法" if zh else "Method"}{"失败 / 任务" if zh else "Failures / tasks"}
' + ) diff --git a/harness/trajectory_replay/interpretation.py b/harness/trajectory_replay/interpretation.py new file mode 100644 index 000000000..f2eff696c --- /dev/null +++ b/harness/trajectory_replay/interpretation.py @@ -0,0 +1,137 @@ +"""Render the same authored replay article inside the report and offline.""" + +import base64 +import html +import json +import re +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit, urlunsplit + +from harness.trajectory_replay.failure_matrix import failure_category_matrix + + +def _report_page_url(report_url: str, relative: str) -> str: + base = urlsplit(report_url) + target = urlsplit(urljoin(report_url, relative)) + query = urlencode(parse_qsl(base.query) + parse_qsl(target.query)) + return urlunsplit((target.scheme, target.netloc, target.path, query, target.fragment)) + + +def render_interpretation(*, source: Path, contract_audit: Path, results_sha256: str, + methods: dict, rows: list[dict], lang: str, image_dir: Path, + image_prefix: str, standalone: bool = False, + report_url: str = "") -> tuple[str, str]: + article = json.loads(source.read_text()) + audit = json.loads(contract_audit.read_text()) + if (article.get("schema") != "lex_replay_interpretation/1" + or article["results_sha256"] != results_sha256 + or audit["bindings"]["results_sha256"] != results_sha256): + raise ValueError("interpretation is not bound to the measured cohort") + if lang not in {"zh", "en"}: + raise ValueError("unsupported article language") + e = html.escape + title = article["title"][lang] + body = [f'

{e(title)}

' + f'

{e(article["intro"][lang])}

'] + seen = set() + for section in article["sections"]: + identity = section["id"] + if identity in seen: + raise ValueError("duplicate article section") + seen.add(identity) + body.append(f'

{e(section["title"][lang])}

') + if section.get("data") == "outcomes": + headers = ("方法", "原始通过数", "任务总数") if lang == "zh" else ("Method", "Original passes", "Tasks") + table_rows = [] + for key, method in methods.items(): + selected = [row for row in rows if row["participant_id"] == key] + table_rows.append([method["label"], sum(row["status"] == "pass" for row in selected), len(selected)]) + body.append(_table(headers, table_rows)) + elif section.get("data") == "failure_matrix": + body.append(failure_category_matrix(methods, rows, lang, embedded=True)) + elif section.get("data") == "contract_changes": + headers = ("题目", "原合同", "改动及影响") if lang == "zh" else ("Task", "Original contract", "Change and effect") + body.append(_table(headers, [ + [f'#{row["task_id"]}', row[f"before_{lang}"], row[f"after_{lang}"]] + for row in audit["task_contract_changes"] + ])) + elif section.get("data"): + raise ValueError("unknown article data block") + if section.get("figure"): + filename = f'{section["figure"]}-{lang}.png' + if Path(filename).name != filename: + raise ValueError("article figure path escapes asset directory") + image = image_dir / filename + if not image.is_file() or any(path.is_symlink() for path in (image, *image.parents)): + raise ValueError("article figure is missing or symlinked") + src = ("data:image/png;base64," + base64.b64encode(image.read_bytes()).decode() + if standalone else image_prefix + filename) + figure_link = "下载原尺寸图" if lang == "zh" else "Download full-size diagram" + scroll_label = "机制图，可横向滚动查看" if lang == "zh" else "Mechanism diagram, scroll horizontally to read" + download_href = ( + _report_page_url(report_url, image_prefix + filename) + if standalone else src + ) + download = f' {figure_link}' + body.append(f'
' + f'{e(section[
' + f'
{e(section["caption"][lang])}{download}
') + body.extend(f'

{e(paragraph[lang])}

' for paragraph in section["paragraphs"]) + if specimen := section.get("specimen"): + headers = specimen["headers"] + values = specimen["rows"] + if not headers or not values or any(len(row) != len(headers) for row in values): + raise ValueError("article specimen requires a nonempty rectangular table") + def localized(value): + if isinstance(value, dict): + if set(value) != {"zh", "en"}: + raise ValueError("article specimen requires both languages") + return value[lang] + if not isinstance(value, (str, int, float)): + raise ValueError("unsupported article specimen value") + return value + caption = localized(specimen["caption"]) + body.append('
' + + _table([localized(value) for value in headers], + [[localized(value) for value in row] for row in values]) + + f'
{e(caption)}
') + if sources := section.get("sources"): + links = [] + for citation in sources: + target = urlsplit(citation["url"]) + if (target.scheme != "https" or not target.hostname + or target.username or target.password): + raise ValueError("article source requires a public HTTPS URL") + if set(citation["label"]) != {"zh", "en"}: + raise ValueError("article source requires both languages") + links.append(f'{e(citation["label"][lang])}') + label = "源码与测试" if lang == "zh" else "Source and tests" + body.append(f'

{label}: ' + " · ".join(links) + '

') + body.append("
") + body.append("
") + result = "".join(body) + if standalone: + if not report_url.startswith("https://"): + raise ValueError("offline article requires the bound HTTPS report URL") + # Only the registry-generated matrix creates links in this renderer. + def absolute_result_link(match): + target = _report_page_url(report_url, html.unescape(match[1])) + return f'href="{e(target)}"' + result = re.sub(r'href="(results\.html[^"]*)"', absolute_result_link, result) + return title, result + + +def _table(headers, rows): + e = lambda value: html.escape(str(value)) + return ('
' + + "".join(f"" for value in headers) + + "" + + "".join("" + "".join(f"" for value in row) + "" for row in rows) + + "
{e(value)}
{e(value)}
") + + +def standalone_document(title: str, body: str, lang: str, css: str) -> str: + return (f'' + '' + f'{html.escape(title)}' + f'
{body}
') diff --git a/harness/trajectory_replay/participants.py b/harness/trajectory_replay/participants.py new file mode 100644 index 000000000..dd8c72704 --- /dev/null +++ b/harness/trajectory_replay/participants.py @@ -0,0 +1,41 @@ +"""Participant identity is distinct from the browser runtime adapter.""" + +from __future__ import annotations + +import re +from typing import Any + + +def participant_registry(methods: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + registry = {} + for method in methods: + identity = method.get("id") + if (not isinstance(identity, str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", identity) + or identity in registry or not method.get("engine")): + raise ValueError("participant registry requires unique safe method ids and engines") + registry[identity] = method + if not registry: + raise ValueError("participant registry is required") + return registry + + +def resolve_participant_id( + row: dict[str, Any], registry: dict[str, dict[str, Any]] +) -> str: + identity = row.get("participant_id") + if identity is not None: + if identity not in registry or row.get("engine") != registry[identity]["engine"]: + raise ValueError("participant identity is unknown or disagrees with its engine") + return identity + matches = [key for key, method in registry.items() if method["engine"] == row.get("engine")] + if len(matches) != 1: + raise ValueError("legacy engine identity is ambiguous or absent from the registry") + return matches[0] + + +def bind_participant( + row: dict[str, Any], registry: dict[str, dict[str, Any]] +) -> dict[str, Any]: + """Return a normalized copy; never rewrite retained evidence.""" + return {**row, "participant_id": resolve_participant_id(row, registry)} diff --git a/harness/trajectory_replay/remote_execution.py b/harness/trajectory_replay/remote_execution.py new file mode 100644 index 000000000..10026dea4 --- /dev/null +++ b/harness/trajectory_replay/remote_execution.py @@ -0,0 +1,335 @@ +"""Shared browser launch, task execution and evidence verification for remote replay.""" + +from __future__ import annotations +import argparse +import json +import subprocess +import time +import traceback +import urllib.request +import uuid +from pathlib import Path +from harness.browser_use.har import CdpHarRecorder +from harness.browser_use.runner import ( + _browser_tool_config, + _login_context, + _prepare_browser_viewport, +) +from harness.trajectory_replay.remote_runtime import ( + file_sha256, + rebind_remote_calls, +) +from harness.trajectory_replay.retrieval_acceptance import build_actual_response +from harness.trajectory_replay.webarena_corrections import ( + apply_correction, + check_replay_assertions, + evaluate_corrected_contract, +) +from harness.trajectory_replay.webarena_replay import ( + _capture_failure_ui_state, + _clear_browser_state, + _click_was_dispatched_before_navigation_timeout, + _execute_replay_call, + _json_sha256, +) + + +def _write(path: Path, value: object) -> str: + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + path.chmod(0o600) + return file_sha256(path) + +def _verify_result_evidence(output: Path, rows: list[dict]) -> None: + for row in rows: + directory = output / f"task-{row['task_id']}" + if directory.is_symlink() or not directory.is_dir(): + raise ValueError(f"task {row['task_id']} evidence directory is missing") + for name, expected in row["evidence_sha256"].items(): + filename = { + "network_har": "network.har", + "acceptance": "acceptance.json", + "actual_response": "actual_response.json", + "evaluation": "evaluation.json", + "failure_screenshot": "failure.png", + "trace": "trace.json", + "readiness": "readiness.json", + "site_reset": "site-reset.json", + }.get(name) + if filename is None: + raise ValueError(f"task {row['task_id']} has unknown evidence kind {name}") + path = directory / filename + if path.is_symlink() or not path.is_file() or file_sha256(path) != expected: + raise ValueError(f"task {row['task_id']} evidence hash changed: {filename}") + +def _wait_cdp(endpoint: str, process: subprocess.Popen) -> None: + for _ in range(300): + if process.poll() is not None: + raise RuntimeError(f"Moli exited before CDP was ready: {process.returncode}") + try: + with urllib.request.urlopen(endpoint + "/json/version", timeout=1) as response: + if response.status == 200: + return + except OSError: + time.sleep(0.1) + raise TimeoutError("browser CDP did not become ready") + +def _browser_command(args: argparse.Namespace, port: int, profile: str) -> list[str]: + if args.engine == "chromium": + return [ + str(args.binary), + "--headless=new", + "--no-first-run", + "--no-default-browser-check", + # Respect the explicitly configured HTTP fixture origins. An + # automatic scheme upgrade changes exact evaluator URL identity. + "--disable-features=HttpsUpgrades,HttpsFirstBalancedMode,HttpsFirstModeIncognito", + f"--user-data-dir={profile}", + f"--remote-debugging-port={port}", + "--remote-debugging-address=127.0.0.1", + "about:blank", + ] + return [ + str(args.binary), + "serve", + "--host", + "127.0.0.1", + "--port", + str(port), + "--layout", + "--font", + "--profile-dir", + profile, + ] + +def _task_type(task: object) -> str: + expected = [ + item.expected.model_dump(mode="json").get("task_type") + for item in task.eval + if item.evaluator == "AgentResponseEvaluator" + ] + if len(expected) != 1 or expected[0] not in {"NAVIGATE", "MUTATE"}: + raise ValueError(f"task {task.task_id}: current-run response type unavailable") + return expected[0] + +def _read_retrieval_contract( + task_id: int, row: dict, sites: dict, candidate_path: Path | None = None +) -> dict: + source_revision = row.get("source_revision") or {} + path = candidate_path + if path is None and source_revision: + path = Path(source_revision["path"]) + if path is None: + path = ( + Path("evaluation/trajectory-replay/webarena-verified/retrieval-candidates") + / f"task-{task_id}.json" + ) + candidate = json.loads(path.read_text(encoding="utf-8")) + if candidate["task_id"] != task_id or candidate["retrieval_contract"]["task_id"] != task_id: + raise ValueError(f"task {task_id}: retrieval contract identity mismatch") + rebound = rebind_remote_calls( + candidate["tool_calls"], + sites, + {"127.0.0.1", "192.168.5.2", "host.docker.internal"}, + ) + if _json_sha256(rebound) != row["bound_calls_sha256"]: + raise ValueError(f"task {task_id}: retrieval actions differ from frozen bound calls") + return candidate + +def _evaluate_assertions(service: object, contract: object) -> list[dict]: + selectors = { + selector + for assertion in contract.replay_assertions + for selector in (assertion.get("fields") or {}) + } + state = service._submit( + lambda: { + "url": service._page.url, + "text": service._page.locator("body").inner_text(), + "fields": service._page.evaluate( + "selectors => Object.fromEntries(selectors.map(selector => " + "[selector, document.querySelector(selector)?.value ?? null]))", + sorted(selectors), + ), + } + ) + return list( + check_replay_assertions( + contract, page_url=state["url"], page_text=state["text"], field_values=state["fields"] + ) + ) + +def _run_task( + *, + task_id: int, + row: dict, + benchmark: object, + endpoint: str, + output: Path, + run_id: str, + origins: set[str], + sites: dict, + catalog: object, + candidate_override_path: Path | None = None, +) -> dict: + from agent.tools.browser.browser_tool import BrowserTool # noqa: PLC0415 + + official = benchmark.get_task(task_id) + correction = apply_correction(official, catalog) if task_id in catalog.records else None + tool = BrowserTool(config=_browser_tool_config(endpoint)) + recorder = CdpHarRecorder() + task_dir = output / f"task-{task_id}" + task_dir.mkdir(mode=0o700) + har_path = task_dir / "network.har" + records: list[dict] = [] + readiness_records: list[dict] = [] + hashes: dict[str, str] = {} + started = time.perf_counter() + status = "fail" + score = None + error = "" + service = None + recorder_started = False + try: + service = tool._get_service() + service._submit(lambda: None) + viewport = _prepare_browser_viewport(service) + _clear_browser_state(service, origins) + start_urls = list( + benchmark.config.render_url(list(official.start_urls), list(official.sites)) + ) + _login_context(service, tuple(official.sites), benchmark, start_urls, viewport) + service._submit(lambda: recorder.start(service._context)) + recorder_started = True + for index, action in enumerate(row["actions"], 1): + for readiness in row["readiness"]: + if readiness["before_action"] == index: + ready_status, ready_result = _execute_replay_call(tool, readiness["input"]) + readiness_records.append( + {"before_action": index, "status": ready_status, "result": ready_result} + ) + if ready_status != "success": + raise RuntimeError(f"readiness before step {index} failed: {ready_result}") + call_started = time.perf_counter() + diagnostics = {} + result_status, result = _execute_replay_call(tool, action, diagnostics=diagnostics) + dispatched = _click_was_dispatched_before_navigation_timeout( + action, result_status, result + ) + records.append( + { + "index": index, + "arguments": action, + "status": "success" if dispatched else result_status, + "tool_status": result_status, + "duration_ms": round((time.perf_counter() - call_started) * 1000, 3), + "result": result, + **({"action_diagnostics": diagnostics} if diagnostics else {}), + **( + {"recovery": "click_dispatched_navigation_wait_timeout"} + if dispatched + else {} + ), + } + ) + if result_status != "success" and not dispatched: + try: + records[-1]["failure_ui_state"] = _capture_failure_ui_state(service, action) + except Exception as observation_exc: + records[-1]["failure_ui_state_error"] = ( + f"{type(observation_exc).__name__}: {observation_exc}" + )[:300] + raise RuntimeError(f"step {index} failed: {result}") + if correction is not None and correction.replay_assertions: + assertions = _evaluate_assertions(service, correction) + else: + assertions = [] + service._submit(lambda: service._page.wait_for_timeout(500)) + service._submit(lambda: recorder.stop(service._context, har_path)) + recorder_started = False + hashes["network_har"] = file_sha256(har_path) + if official.is_retrieve_task: + candidate = _read_retrieval_contract(task_id, row, sites, candidate_override_path) + # Contract step indices refer to the source calls; recorded results are + # from the equivalent remote-bound calls executed in this run. + source_records = [ + dict(record, arguments=candidate["tool_calls"][i]) + for i, record in enumerate(records) + ] + acceptance = build_actual_response( + contract=candidate["retrieval_contract"], + task_id=task_id, + calls=candidate["tool_calls"], + records=source_records, + run_id=run_id, + attempt_id=uuid.uuid4().hex, + ) + hashes["acceptance"] = _write(task_dir / "acceptance.json", acceptance) + actual = acceptance["actual_response"] + else: + actual = { + "task_type": _task_type(official), + "status": "SUCCESS", + "retrieved_data": None, + "error_details": None, + } + hashes["actual_response"] = _write(task_dir / "actual_response.json", actual) + if actual is None: + status = "unsupported" + else: + evaluation_result = ( + evaluate_corrected_contract( + benchmark, correction, agent_response=actual, network_trace=har_path + ) + if correction is not None + else benchmark.evaluate_task( + task_id=task_id, agent_response=actual, network_trace=har_path + ) + ) + evaluation = evaluation_result.model_dump(mode="json") + evaluation["replay_assertions"] = assertions + if correction is not None: + evaluation["contract_correction_id"] = correction.correction_id + evaluation["corrected_task_sha256"] = correction.corrected_task_sha256 + hashes["evaluation"] = _write(task_dir / "evaluation.json", evaluation) + score = evaluation.get("score") + status = ( + "pass" + if float(score or 0) == 1 + and all(item.get("status") == "pass" for item in assertions) + else "fail" + ) + except Exception: + error = traceback.format_exc() + if service is not None and recorder_started: + try: + service._submit(lambda: recorder.stop(service._context, har_path)) + hashes["network_har"] = file_sha256(har_path) + except Exception: + pass + if service is not None: + try: + service._submit( + lambda: service._page.screenshot(path=str(task_dir / "failure.png")) + ) + hashes["failure_screenshot"] = file_sha256(task_dir / "failure.png") + except Exception: + pass + finally: + try: + tool.close() + except Exception as exc: + error += f"\nBrowserTool close failed: {exc}" + hashes["trace"] = _write(task_dir / "trace.json", records) + hashes["readiness"] = _write(task_dir / "readiness.json", readiness_records) + return { + "task_id": task_id, + "status": status, + "score": score, + "action_count": len(row["actions"]), + "executed_actions_sha256": _json_sha256(row["actions"]), + "omitted_diagnostic_observer_sha256": row.get("omitted_diagnostic_observer_sha256"), + "completed_actions": sum(record["status"] == "success" for record in records), + "duration_ms": round((time.perf_counter() - started) * 1000, 3), + "error": error, + "evidence_sha256": hashes, + } diff --git a/harness/trajectory_replay/remote_runtime.py b/harness/trajectory_replay/remote_runtime.py new file mode 100644 index 000000000..141b935da --- /dev/null +++ b/harness/trajectory_replay/remote_runtime.py @@ -0,0 +1,275 @@ +"""Bind a diagnostic replay to an externally reset WebArena deployment. + +The URL map and reset receipt are explicit inputs. Merely reaching a remote +site is not evidence that its writable fixture was reset for a task. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from webarena_verified import WebArenaVerified +from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig +from webarena_verified.types.task import WebArenaSite + +from harness.browser_use.runner import SITE_CREDENTIALS, SITE_PORTS + + +def load_remote_sites(path: Path) -> dict[WebArenaSite, str]: + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema") != "lex_webarena_remote_sites/1": + raise ValueError("unsupported remote site map") + raw = value.get("sites") + if not isinstance(raw, dict) or set(raw) != {site.value for site in SITE_PORTS}: + raise ValueError("remote site map must cover exactly the four mutable sites") + sites: dict[WebArenaSite, str] = {} + for site in SITE_PORTS: + url = raw[site.value] + if not isinstance(url, str): + raise ValueError(f"invalid remote URL for {site.value}") + parsed = urlsplit(url) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.query + or parsed.fragment + ): + raise ValueError(f"invalid remote URL for {site.value}") + if site == WebArenaSite.SHOPPING_ADMIN and parsed.path.rstrip("/") != "/admin": + raise ValueError("Shopping Admin remote URL must end in /admin") + if site != WebArenaSite.SHOPPING_ADMIN and parsed.path not in {"", "/"}: + raise ValueError(f"remote URL for {site.value} must be an origin") + sites[site] = url.rstrip("/") + origins = {urlsplit(url).netloc for url in sites.values()} + if len(origins) != len(sites): + raise ValueError("mutable sites must use distinct remote origins") + return sites + + +def remote_benchmark(sites: dict[WebArenaSite, str]) -> WebArenaVerified: + return WebArenaVerified( + config=WebArenaVerifiedConfig( + environments={ + site: EnvironmentConfig( + urls=[url], + credentials=SITE_CREDENTIALS[site], + use_header_login=site == WebArenaSite.SHOPPING_ADMIN, + ) + for site, url in sites.items() + } + ) + ) + + +def rebind_remote_calls( + calls: list[dict[str, Any]], + sites: dict[WebArenaSite, str], + trusted_source_hosts: set[str], +) -> list[dict[str, Any]]: + """Bind frozen WebArena URLs while tolerating HTTP-to-HTTPS proxy upgrades. + + An exact same-site href selector may point at either scheme after a reverse + proxy redirects the document. Preserve its host, path, query and action + semantics; only the transport scheme becomes an explicit alternative. + """ + by_port = {port: site for site, ports in SITE_PORTS.items() for port in ports} + + def rebind_url(value: str) -> str: + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or parsed.hostname not in trusted_source_hosts: + return value + site = by_port.get(parsed.port) + if site is None: + return value + target = urlsplit(sites[site]) + return urlunsplit( + (target.scheme, target.netloc, parsed.path, parsed.query, parsed.fragment) + ) + + href_pattern = re.compile( + r"(?P\[href\s*=\s*)(?P['\"])(?Phttps?://[^'\"]+)" + r"(?P=quote)(?P\])" + ) + + def rebind_href(match: re.Match[str]) -> str: + url = match.group("url") + rebound = rebind_url(url) + if rebound == url: + return match.group(0) + parsed = urlsplit(rebound) + other_scheme = "https" if parsed.scheme == "http" else "http" + alternative = urlunsplit( + (other_scheme, parsed.netloc, parsed.path, parsed.query, parsed.fragment) + ) + quote = match.group("quote") + return f":is([href={quote}{rebound}{quote}],[href={quote}{alternative}{quote}])" + + rebound: list[dict[str, Any]] = [] + for raw in calls: + call = dict(raw) + url = call.get("url") + if isinstance(url, str): + call["url"] = rebind_url(url) + # URL-valued form inputs refer to the same fixture deployment as + # navigation URLs. Do not rewrite prose or arbitrary external URLs. + text = call.get("text") + if call.get("action") == "fill" and isinstance(text, str) and not any( + char.isspace() for char in text + ): + call["text"] = rebind_url(text) + selector = call.get("selector") + if isinstance(selector, str): + call["selector"] = href_pattern.sub(rebind_href, selector) + rebound.append(call) + return rebound + + +def validate_remote_reset_receipt( + path: Path, *, required_sites: set[WebArenaSite], site_map_sha256: str +) -> dict[str, Any]: + """Require a fresh, task-bound Pod replacement and application readiness.""" + value = json.loads(path.read_text(encoding="utf-8")) + if ( + value.get("schema") != "lex_webarena_remote_reset/1" + or value.get("site_map_sha256") != site_map_sha256 + or not isinstance(value.get("task_id"), int) + or not isinstance(value.get("app_id"), int) + ): + raise ValueError("remote reset receipt is not bound to this deployment") + rows = value.get("sites") + if not isinstance(rows, dict) or not {site.value for site in required_sites} <= set(rows): + raise ValueError("remote reset receipt lacks a required site") + for site in required_sites: + row = rows[site.value] + if ( + not isinstance(row, dict) + or not row.get("image_digest") + or not row.get("operation_event_id") + or not row.get("old_pod") + or not row.get("new_pod") + or row["old_pod"] == row["new_pod"] + or row.get("operation_status") != "success" + or row.get("writable_layer_marker") != "absent" + or row.get("env_ctrl_ready") is not True + or row.get("public_http_status") != 200 + or ( + site in {WebArenaSite.SHOPPING, WebArenaSite.SHOPPING_ADMIN} + and row.get("elasticsearch_status") not in {"yellow", "green"} + ) + ): + raise ValueError(f"remote reset is not proven for {site.value}") + return value + + +def validate_remote_cohort_reset_receipt( + path: Path, + *, + site_map_sha256: str, + app_id: int, + reset_scope: str, + site_reset_between_tasks: bool = False, + admin_https_base_url: str | None = None, + required_readiness_probes: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Bind a Pod replacement and public readiness to its declared reset scope. + + This proves replacement and public readiness for the four mutable sites. + Task-level acceptance still needs the recorded browser/network evidence. + """ + value = json.loads(path.read_text(encoding="utf-8")) + if ( + value.get("schema") != "webarena_remote_cohort_reset/1" + or value.get("app_id") != app_id + or value.get("site_map_sha256") != site_map_sha256 + or value.get("reset_scope") != reset_scope + or value.get("site_reset_between_tasks") is not site_reset_between_tasks + ): + raise ValueError("remote cohort reset receipt is not bound to this deployment") + sites = value.get("sites") + if not isinstance(sites, dict) or set(sites) != {site.value for site in SITE_PORTS}: + raise ValueError("remote cohort reset receipt must cover all mutable sites") + for site, row in sites.items(): + if ( + not isinstance(row, dict) + or not row.get("service_id") + or not row.get("image_digest") + or not row.get("event_id") + or row.get("event_status") != "success" + or not row.get("old_pod") + or not row.get("new_pod") + or row["old_pod"] == row["new_pod"] + or row.get("data_volume_count") != 0 + or row.get("public_http_status") != 200 + ): + raise ValueError(f"remote cohort reset is not proven for {site}") + for site in ("gitlab", "reddit"): + check = sites[site].get("baseline_check") + if not isinstance(check, dict) or check.get("row_count") != 0: + raise ValueError(f"remote cohort baseline is not proven for {site}") + if admin_https_base_url is not None: + config = sites["shopping_admin"].get("https_admin_config") + if ( + not isinstance(config, dict) + or config.get("web_secure_base_url") != admin_https_base_url + or config.get("web_secure_use_in_adminhtml") != 1 + or config.get("config_cache_cleaned") is not True + or config.get("public_static_resource_scheme") != "https" + ): + raise ValueError("remote cohort admin HTTPS configuration is not proven") + if required_readiness_probes: + observed = value.get("site_readiness") + if not isinstance(observed, list) or len(observed) != len(required_readiness_probes): + raise ValueError("remote cohort site readiness scope is incomplete") + expected_by_url = {probe["url"]: probe for probe in required_readiness_probes} + if len(expected_by_url) != len(required_readiness_probes): + raise ValueError("duplicate required site readiness URL") + observed_by_url = {probe.get("url"): probe for probe in observed if isinstance(probe, dict)} + if len(observed_by_url) != len(observed) or set(observed_by_url) != set(expected_by_url): + raise ValueError("remote cohort site readiness URL differs") + for url, expected in expected_by_url.items(): + probe = observed_by_url[url] + limit = expected["max_stable_ms"] + if ( + probe.get("site") != expected["site"] + or probe.get("path") != expected["path"] + or probe.get("method") != "GET" + or probe.get("read_only") is not True + or type(probe.get("cold_ms")) not in {int, float} + or probe["cold_ms"] <= 0 + or probe.get("warm_http_status") != 200 + or probe.get("stable_http_status") != 200 + or not ( + probe.get("cold_http_status") == 200 + or ( + ( + ( + type(probe.get("cold_http_status")) is int + and 500 <= probe["cold_http_status"] <= 599 + ) + or ( + probe.get("cold_http_status") == 0 + and probe.get("cold_error") == "timeout" + ) + ) + and probe.get("recovery_http_status") == 200 + and type(probe.get("recovery_ms")) in {int, float} + and probe["recovery_ms"] > 0 + ) + ) + or any( + type(probe.get(f"{phase}_ms")) not in {int, float} + or not 0 < probe[f"{phase}_ms"] <= limit + for phase in ("warm", "stable") + ) + ): + raise ValueError(f"remote cohort site readiness is not proven for {url}") + return value + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/harness/trajectory_replay/report_languages.py b/harness/trajectory_replay/report_languages.py new file mode 100644 index 000000000..cc3b28219 --- /dev/null +++ b/harness/trajectory_replay/report_languages.py @@ -0,0 +1,17 @@ +"""Languages materialized in a report, independent of available translations.""" + +import json +from pathlib import Path + + +def report_languages(root: Path) -> tuple[str, ...]: + path = root / "report-manifest.json" + manifest = json.loads(path.read_text()) if path.is_file() else {} + languages = manifest.get("languages", ["zh", "en"]) + if languages not in (["zh"], ["zh", "en"]): + raise ValueError("unsupported report languages") + return tuple(languages) + + +def language_roots(root: Path) -> tuple[Path, ...]: + return tuple(root if lang == "zh" else root / "en" for lang in report_languages(root)) diff --git a/harness/trajectory_replay/retrieval_acceptance.py b/harness/trajectory_replay/retrieval_acceptance.py new file mode 100644 index 000000000..73a9a2ecb --- /dev/null +++ b/harness/trajectory_replay/retrieval_acceptance.py @@ -0,0 +1,1086 @@ +"""Current-run retrieval answers, never filled from a frozen expected answer. + +Version one deliberately accepts only strict JSON observations and structural +projections. Version two additionally supports one built-in, deterministic +reviewed-span projection whose complete ordered annotation is part of the +contract digest. It never executes annotation-provided code or answer strings. +Version three adds opt-in exact string-value sets and accurately attributed, +materialized review evidence. Reviewer declarations and matching guest names are +not identity authentication; identity joins require a source-linked customer ID. +Unstructured or incomplete observations remain unsupported, not a browser +failure. Every protocol revision requires fresh qualification. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import Path +from typing import Any + +PROTOCOL_V1 = "lex_replay_current_run_retrieval/1" +PROTOCOL_V2 = "lex_replay_current_run_retrieval/2" +PROTOCOL_V3 = "lex_replay_current_run_retrieval/3" +# Keep the original public alias stable for existing v1 catalogs and callers. +PROTOCOL = PROTOCOL_V1 +SUPPORTED_PROTOCOLS = frozenset({PROTOCOL_V1, PROTOCOL_V2, PROTOCOL_V3}) +REVIEWED_SPAN_KIND = "reviewed_ordered_spans/1" +REVIEWED_SPAN_ANNOTATION = "lex_replay_reviewed_span_annotation/1" + + +def digest(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + +def text_digest(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def strict_json(text: str) -> Any: + def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + def invalid(value: str) -> None: + raise ValueError(f"nonfinite JSON value: {value}") + + def finite_float(value: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError("nonfinite JSON number") + return number + + return json.loads( + text, object_pairs_hook=pairs, parse_constant=invalid, parse_float=finite_float + ) + + +def pointer(value: Any, path: str) -> Any: + if not isinstance(path, str) or (path and not path.startswith("/")): + raise ValueError("invalid JSON Pointer") + for token in path.split("/")[1:] if path else []: + if re.search(r"~(?![01])", token): + raise ValueError("invalid JSON Pointer escape") + token = token.replace("~1", "/").replace("~0", "~") + if isinstance(value, list): + if not token.isascii() or not token.isdigit() or (len(token) > 1 and token[0] == "0"): + raise ValueError("invalid array index") + value = value[int(token)] + elif isinstance(value, dict): + value = value[token] + else: + raise ValueError("JSON Pointer traverses a scalar") + return value + + +def project(expression: dict[str, Any], sources: dict[str, Any], depth: int = 0) -> Any: + if depth > 32 or not isinstance(expression, dict): + raise ValueError("invalid projection") + if set(expression) == {"source", "pointer"}: + return pointer(sources[expression["source"]], expression["pointer"]) + if set(expression) == {"array"} and isinstance(expression["array"], list): + return [project(item, sources, depth + 1) for item in expression["array"]] + if set(expression) == {"object"} and isinstance(expression["object"], dict): + return { + key: project(item, sources, depth + 1) for key, item in expression["object"].items() + } + if set(expression) == {"map"}: + spec = expression["map"] + if not isinstance(spec, dict) or set(spec) != {"input", "projection"}: + raise ValueError("invalid map") + _validate_projection(spec["projection"], {"$item"}) + values = project(spec["input"], sources, depth + 1) + if not isinstance(values, list): + raise ValueError("map input must be an array") + return [ + project(spec["projection"], {**sources, "$item": value}, depth + 1) for value in values + ] + raise ValueError("unsupported projection; constants and executable code are forbidden") + + +def validate_contract(contract: dict[str, Any], calls: list[dict[str, Any]]) -> None: + schema = contract.get("schema") + if schema not in SUPPORTED_PROTOCOLS: + raise ValueError("unsupported retrieval protocol") + common = {"schema", "task_id", "canonical_calls_sha256", "sources"} + version_fields = ( + {"projection"} + if schema == PROTOCOL_V1 + else {"reviewed_span_projection", "semantic_reference"} + ) + if schema == PROTOCOL_V3: + version_fields = {"semantic_reference", "review_evidence"} | ( + {"reviewed_span_projection"} + if "reviewed_span_projection" in contract + else {"projection"} + ) + required = common | version_fields + optional = ( + {"requirements", "empty_result", "absence_coverage"} + if schema != PROTOCOL_V2 + else {"requirements"} + ) + if schema == PROTOCOL_V3: + optional.add("reviewed_scope") + if not required <= set(contract) or set(contract) - required - optional: + raise ValueError("invalid retrieval observation contract fields") + if contract["canonical_calls_sha256"] != digest(calls): + raise ValueError("retrieval contract differs from canonical actions") + sources = contract["sources"] + if not isinstance(sources, list) or not sources: + raise ValueError("retrieval requires observation sources") + names: set[str] = set() + for source in sources: + if set(source) != {"name", "step", "action_sha256", "decode"}: + raise ValueError("invalid observation source") + name, step = source["name"], source["step"] + if not isinstance(name, str) or not name or name.startswith("$") or name in names: + raise ValueError("invalid or duplicate source name") + names.add(name) + if type(step) is not int or not 1 <= step <= len(calls): + raise ValueError("invalid observation step") + action = calls[step - 1] + if action.get("action") not in {"get_text", "evaluate"} or source["decode"] != "json": + raise ValueError("retrieval requires structured JSON read observations") + if digest(action) != source["action_sha256"]: + raise ValueError("observation action binding mismatch") + if schema == PROTOCOL_V3: + _validate_review_evidence(contract) + if "reviewed_span_projection" in contract: + _validate_reviewed_span_projection(contract, names) + else: + expression = contract["projection"] + if not isinstance(expression, dict) or set(expression) != {"distinct_exact_strings"}: + raise ValueError("v3 value sets require an explicit distinct projection") + _validate_projection(expression["distinct_exact_strings"], names) + elif schema == PROTOCOL_V1: + _validate_projection(contract["projection"], names) + else: + _validate_reviewed_span_projection(contract, names) + if len({source["step"] for source in contract["sources"]}) != len(contract["sources"]): + raise ValueError("one observation step cannot have multiple source aliases") + requirements = contract.get("requirements", {}) + if not isinstance(requirements, dict) or set(requirements) - {"scope", "completeness"}: + raise ValueError("invalid observation requirements") + for requirement in requirements.values(): + if not isinstance(requirement, dict) or set(requirement) != {"source", "pointer"}: + raise ValueError("requirements must directly read observed booleans") + _validate_projection(requirement, names) + if "reviewed_scope" in contract: + _validate_reviewed_scope_contract(contract, names) + if schema in {PROTOCOL_V2, PROTOCOL_V3} and ( + set(requirements) != {"scope", "completeness"} + or digest(requirements["scope"]) == digest(requirements["completeness"]) + ): + raise ValueError("reviewed spans require independent scope and completeness observations") + if "empty_result" in contract: + if ( + contract["empty_result"] != "NOT_FOUND_ERROR" + or set(requirements) != {"scope", "completeness"} + or digest(requirements["scope"]) == digest(requirements["completeness"]) + ): + raise ValueError( + "absence needs independently observed scope and completeness requirements" + ) + if "absence_coverage" in contract: + coverage = contract["absence_coverage"] + if isinstance(coverage, dict) and coverage.get("kind") == "exhaustive_records/1": + _validate_exhaustive_coverage(coverage, names, contract) + return + if ( + "empty_result" not in contract + or not isinstance(coverage, dict) + or set(coverage) != {"source", "pages_pointer", "required_paths", "detector"} + or not isinstance(coverage["source"], str) + or coverage["source"] not in names + or not _valid_pointer_path(coverage["pages_pointer"]) + or coverage["detector"] != "phone_number" + ): + raise ValueError("invalid absence coverage contract") + paths = coverage["required_paths"] + if ( + not isinstance(paths, list) + or not 1 <= len(paths) <= 32 + or any(not isinstance(path, str) or not path.startswith("/") for path in paths) + or len(set(paths)) != len(paths) + ): + raise ValueError("invalid absence coverage paths") + + +_PHONE_NUMBER = re.compile(r"(? None: + if contract["schema"] != PROTOCOL_V3 or "empty_result" not in contract: + raise ValueError("exhaustive absence coverage requires v3 empty-result contract") + if set(coverage) != { + "kind", "source", "scope_pointer", "expected_scope", "records_pointer", + "pages_pointer", "match_count_pointer", "page_size", "id_pointer", + "required_fields", "predicate", + }: + raise ValueError("invalid exhaustive absence coverage fields") + if coverage["source"] not in names or any( + not _valid_pointer_path(coverage[key]) + for key in ( + "scope_pointer", "records_pointer", "pages_pointer", + "match_count_pointer", "id_pointer", + ) + ): + raise ValueError("invalid exhaustive absence source or pointer") + if ( + not isinstance(coverage["expected_scope"], dict) + or not coverage["expected_scope"] + or any(not isinstance(key, str) or not key for key in coverage["expected_scope"]) + or type(coverage["page_size"]) is not int + or not 1 <= coverage["page_size"] <= 1000 + ): + raise ValueError("invalid exhaustive absence scope or page size") + fields = coverage["required_fields"] + if ( + not isinstance(fields, list) + or not fields + or len(fields) != len(set(fields)) + or any(not _valid_pointer_path(field) for field in fields) + ): + raise ValueError("invalid exhaustive absence required fields") + predicate = coverage["predicate"] + if not isinstance(predicate, dict) or set(predicate) != { + "field_pointer", "type", "operator", "value" + } or not _valid_pointer_path(predicate["field_pointer"]): + raise ValueError("invalid exhaustive absence predicate") + if predicate["field_pointer"] not in fields: + raise ValueError("absence predicate field must be required in every record") + if predicate["type"] == "integer": + valid = ( + type(predicate["value"]) is int + and predicate["operator"] in {"eq", "gt", "gte", "lt", "lte"} + ) + elif predicate["type"] == "string": + valid = ( + type(predicate["value"]) is str + and bool(predicate["value"]) + and predicate["operator"] == "eq" + ) + elif predicate["type"] == "boolean": + valid = type(predicate["value"]) is bool and predicate["operator"] == "eq" + else: + valid = False + if not valid: + raise ValueError("invalid typed absence predicate") + + +def _verify_exhaustive_coverage(coverage: dict[str, Any], source: Any) -> None: + if pointer(source, coverage["scope_pointer"]) != coverage["expected_scope"]: + raise ValueError("absence observation is outside the declared source scope") + records = pointer(source, coverage["records_pointer"]) + pages = pointer(source, coverage["pages_pointer"]) + match_count = pointer(source, coverage["match_count_pointer"]) + if ( + not isinstance(records, list) + or not isinstance(pages, list) + or not pages + or type(match_count) is not int + or match_count != 0 + ): + raise ValueError("absence records, pagination or match count unavailable") + size, total = coverage["page_size"], len(records) + expected_pages = max(1, (total + size - 1) // size) + if len(pages) != expected_pages: + raise ValueError("absence pagination does not cover the declared population") + for index, page in enumerate(pages, 1): + expected_rows = min(size, max(0, total - (index - 1) * size)) + expected_ids = [ + str(pointer(record, coverage["id_pointer"])) + for record in records[(index - 1) * size : index * size] + ] + if ( + not isinstance(page, dict) + or set(page) != {"page", "rows", "total", "has_next", "ids"} + or type(page["page"]) is not int + or page["page"] != index + or type(page["rows"]) is not int + or page["rows"] != expected_rows + or type(page["total"]) is not int + or page["total"] != total + or type(page["has_next"]) is not bool + or page["has_next"] != (index < expected_pages) + or page["ids"] != expected_ids + ): + raise ValueError("absence pagination is incomplete or contradictory") + seen: set[str] = set() + predicate = coverage["predicate"] + for record in records: + if not isinstance(record, dict): + raise ValueError("absence population contains a non-record") + identity = pointer(record, coverage["id_pointer"]) + if type(identity) not in {str, int} or not str(identity).strip(): + raise ValueError("absence record has no stable identity") + identity_key = str(identity) + if identity_key in seen: + raise ValueError("absence population contains duplicate identities") + seen.add(identity_key) + for field in coverage["required_fields"]: + value = pointer(record, field) + if value is None or value == "" or value == [] or value == {}: + raise ValueError("absence record is missing a required field") + value = pointer(record, predicate["field_pointer"]) + if type(value) is not {"integer": int, "string": str, "boolean": bool}[predicate["type"]]: + raise ValueError("absence predicate field has the wrong type") + target, operator = predicate["value"], predicate["operator"] + if operator == "eq": + matched = value == target + elif operator == "gt": + matched = value > target + elif operator == "gte": + matched = value >= target + elif operator == "lt": + matched = value < target + else: + matched = value <= target + if matched: + raise ValueError("absence population contains a matching record") + + +def _verify_absence_coverage(contract: dict[str, Any], values: dict[str, Any]) -> None: + coverage = contract.get("absence_coverage") + if coverage is None: + raise ValueError("empty retrieval lacks independently checkable absence coverage") + if coverage.get("kind") == "exhaustive_records/1": + _verify_exhaustive_coverage(coverage, values[coverage["source"]]) + return + pages = pointer(values[coverage["source"]], coverage["pages_pointer"]) + if not isinstance(pages, list) or len(pages) != len(coverage["required_paths"]): + raise ValueError("absence coverage does not contain every declared page") + seen: set[str] = set() + for page in pages: + if not isinstance(page, dict) or not isinstance(page.get("path"), str): + raise ValueError("invalid absence coverage page") + path = page["path"] + if path in seen or path not in coverage["required_paths"]: + raise ValueError("absence coverage contains duplicate or undeclared page") + seen.add(path) + text = page.get("text") + if ( + type(page.get("status")) is not int + or page["status"] != 200 + or not isinstance(text, str) + or not text.strip() + or not isinstance(page.get("tel"), list) + or not isinstance(page.get("matches"), list) + or page["tel"] + or page["matches"] + or _PHONE_NUMBER.search(text) + ): + raise ValueError("absence coverage page is incomplete or contains a phone number") + + +def _scope_ref(spec: Any, names: set[str]) -> None: + if not isinstance(spec, dict) or set(spec) != {"source", "pointer"}: + raise ValueError("reviewed scope requires a direct observation reference") + _validate_projection(spec, names) + + +def _validate_reviewed_scope_contract(contract: dict[str, Any], names: set[str]) -> None: + spec = contract["reviewed_scope"] + if ( + contract["schema"] != PROTOCOL_V3 + or not isinstance(spec, dict) + or set(spec) != {"review_pointer", "observed_requirements", "populations", "identity_joins"} + ): + raise ValueError("invalid reviewed scope contract") + if not _valid_pointer_path(spec["review_pointer"]): + raise ValueError("invalid scope review pointer") + checks = spec["observed_requirements"] + if not isinstance(checks, list) or not checks: + raise ValueError("reviewed scope needs independent observed requirements") + for check in checks: + _scope_ref(check, names) + if not isinstance(spec["populations"], list) or not isinstance(spec["identity_joins"], list): + raise ValueError("invalid reviewed scope constraints") + for population in spec["populations"]: + if not isinstance(population, dict) or set(population) != { + "records", + "pages", + "id_pointer", + "required_fields", + }: + raise ValueError("invalid reviewed population constraint") + _scope_ref(population["records"], names) + _scope_ref(population["pages"], names) + if ( + not _valid_pointer_path(population["id_pointer"]) + or not isinstance(population["required_fields"], list) + or not population["required_fields"] + or any(not _valid_pointer_path(p) for p in population["required_fields"]) + ): + raise ValueError("invalid reviewed population fields") + for join in spec["identity_joins"]: + if not isinstance(join, dict) or set(join) != { + "selected", + "candidates", + "result", + "selected_fields", + "candidate_fields", + }: + raise ValueError("invalid reviewed identity join") + for key in ("selected", "candidates", "result"): + _scope_ref(join[key], names) + for key, fields in ( + ("selected_fields", {"id", "name", "linked_name", "linked_email"}), + ("candidate_fields", {"id", "name", "email"}), + ): + mapping = join[key] + if ( + not isinstance(mapping, dict) + or set(mapping) != fields + or any(not _valid_pointer_path(p) for p in mapping.values()) + ): + raise ValueError("invalid reviewed identity field mapping") + proof = pointer(contract["review_evidence"]["source_review"], spec["review_pointer"]) + if not isinstance(proof, dict) or set(proof) != { + "decision", + "rationale", + "scope_contract_sha256", + "sources", + }: + raise ValueError("invalid materialized scope review") + if ( + proof["decision"] not in {"accept", "reject"} + or not isinstance(proof["rationale"], str) + or not proof["rationale"].strip() + ): + raise ValueError("scope review requires explicit decision and rationale") + if proof["scope_contract_sha256"] != digest( + {"reviewed_scope": spec, "requirements": contract["requirements"]} + ): + raise ValueError("reviewed scope constraints changed") + if not isinstance(proof["sources"], list) or len(proof["sources"]) != len(names): + raise ValueError("scope review must bind every source") + found = set() + for binding in proof["sources"]: + if ( + not isinstance(binding, dict) + or set(binding) != {"source", "output_sha256", "value_sha256"} + or binding["source"] not in names + or binding["source"] in found + ): + raise ValueError("invalid or duplicate scope review source") + found.add(binding["source"]) + if any( + not isinstance(binding[k], str) or not re.fullmatch(r"[0-9a-f]{64}", binding[k]) + for k in ("output_sha256", "value_sha256") + ): + raise ValueError("invalid scope observation digest") + + +def _check_scope_population(spec: dict[str, Any], values: dict[str, Any]) -> None: + records, pages = project(spec["records"], values), project(spec["pages"], values) + if not isinstance(records, list) or not records or not isinstance(pages, list) or not pages: + raise ValueError("reviewed population is unavailable") + count = 0 + for index, page in enumerate(pages, 1): + if ( + not isinstance(page, dict) + or type(page.get("page")) is not int + or page["page"] != index + or type(page.get("rows")) is not int + or page["rows"] < 0 + ): + raise ValueError("reviewed population page sequence is incomplete") + if "total" in page and (type(page["total"]) is not int or page["total"] != len(records)): + raise ValueError("reviewed population total differs") + count += page["rows"] + if count != len(records): + raise ValueError("reviewed population count differs") + ids = [] + for record in records: + if not isinstance(record, dict): + raise ValueError("reviewed population record unavailable") + key = pointer(record, spec["id_pointer"]) + if type(key) not in {int, str} or str(key).strip() == "": + raise ValueError("reviewed population ID missing") + ids.append(str(key)) + for field in spec["required_fields"]: + value = pointer(record, field) + if ( + value is None + or value == [] + or value == {} + or (isinstance(value, str) and not value.strip()) + ): + raise ValueError("reviewed population required field missing") + if len(set(ids)) != len(ids): + raise ValueError("reviewed population IDs are not unique") + + +def _check_scope_identity_join(spec: dict[str, Any], values: dict[str, Any]) -> str: + selected, candidates, result = ( + project(spec[key], values) for key in ("selected", "candidates", "result") + ) + if ( + not isinstance(selected, dict) + or not isinstance(result, dict) + or not isinstance(candidates, list) + or not candidates + ): + raise ValueError("reviewed identity records unavailable") + + def normalized(value: Any) -> str: + return " ".join(value.split()) if isinstance(value, str) else "" + + fields, target = spec["selected_fields"], spec["candidate_fields"] + ids = [] + for candidate in candidates: + if not isinstance(candidate, dict): + raise ValueError("reviewed customer row unavailable") + cid, name, email = (pointer(candidate, target[key]) for key in ("id", "name", "email")) + if ( + type(cid) not in {str, int} + or not str(cid).strip() + or not normalized(name) + or not isinstance(email, str) + or not re.fullmatch(r"[^\s@]+@[^\s@]+", email) + ): + raise ValueError("reviewed customer identity incomplete") + ids.append(str(cid)) + if len(set(ids)) != len(ids): + raise ValueError("reviewed customer IDs are not unique") + explicit = pointer(selected, fields["id"]) + if type(explicit) not in {str, int} or not str(explicit).strip(): + raise ValueError("review author has no explicit customer identity") + matches = [c for c in candidates if str(pointer(c, target["id"])) == str(explicit)] + basis = "explicit_customer_id" + if len(matches) != 1 or matches[0] != result: + raise ValueError("reviewed identity is missing, ambiguous or mismatched") + for key, candidate_key in (("linked_name", "name"), ("linked_email", "email")): + linked = pointer(selected, fields[key]) + expected = pointer(result, target[candidate_key]) + if linked is not None and (normalized(linked) if key == "linked_name" else linked) != ( + normalized(expected) if key == "linked_name" else expected + ): + raise ValueError("explicit customer identity conflict") + return basis + + +def _apply_reviewed_scope( + contract: dict[str, Any], values: dict[str, Any], sources: list[dict[str, Any]] +) -> dict[str, Any]: + spec = contract["reviewed_scope"] + proof = pointer(contract["review_evidence"]["source_review"], spec["review_pointer"]) + if proof["decision"] != "accept": + raise ValueError("scope review did not accept") + observed = {source["name"]: source for source in sources} + for binding in proof["sources"]: + name = binding["source"] + if binding["output_sha256"] != observed[name]["output_sha256"] or binding[ + "value_sha256" + ] != digest(values[name]): + raise ValueError("scope review does not match current observation") + for requirement in spec["observed_requirements"]: + if project(requirement, values) is not True: + raise ValueError("scope review cannot override observed hard requirement") + for population in spec["populations"]: + _check_scope_population(population, values) + bases = [_check_scope_identity_join(join, values) for join in spec["identity_joins"]] + return { + "scope_review_sha256": digest(proof), + "raw_observations_unchanged": True, + "identity_bases": bases, + } + + +def _valid_pointer_path(path: Any) -> bool: + return ( + isinstance(path, str) + and (not path or path.startswith("/")) + and not re.search(r"~(?![01])", path) + ) + + +def _validate_review_evidence(contract: dict[str, Any]) -> None: + """Check materialized provenance declarations, not a human's real-world identity.""" + reference, evidence = contract["semantic_reference"], contract["review_evidence"] + kind = ( + "reviewed_span_annotation" + if "reviewed_span_projection" in contract + else "reviewed_value_set" + ) + if not isinstance(reference, dict) or set(reference) != { + "kind", + "browser_semantic_inference", + "reviewer", + "scope", + "evidence_sha256", + }: + raise ValueError("invalid v3 semantic reference") + reviewer = reference["reviewer"] + if ( + reference["kind"] != kind + or reference["browser_semantic_inference"] is not False + or not isinstance(reviewer, dict) + or set(reviewer) != {"kind", "identity"} + or reviewer["kind"] not in {"agent", "human"} + or not isinstance(reviewer["identity"], str) + or not reviewer["identity"].strip() + or not isinstance(reference["scope"], str) + or not reference["scope"].strip() + ): + raise ValueError("explicit reviewer identity, kind and scope required") + if not isinstance(evidence, dict) or set(evidence) != { + "schema", + "reviewer", + "scope", + "task_id", + "sources_sha256", + "projection_sha256", + "source_review", + "source_review_sha256", + "source_review_artifact", + }: + raise ValueError("invalid materialized review evidence") + projection = contract.get("reviewed_span_projection", contract.get("projection")) + source_review = evidence["source_review"] + if not isinstance(source_review, dict): + raise ValueError("review source must be a materialized object") + _validate_review_artifact(evidence["source_review_artifact"], source_review) + for artifact in source_review.get("predecessor_artifacts", []): + _validate_review_artifact( + artifact, artifact.get("content") if isinstance(artifact, dict) else None + ) + for binding in artifact.get("bindings", []): + if not isinstance(binding, dict) or set(binding) != { + "artifact_pointer", + "contract_pointer", + }: + raise ValueError("invalid predecessor artifact binding") + if pointer(artifact["content"], binding["artifact_pointer"]) != pointer( + contract, binding["contract_pointer"] + ): + raise ValueError("predecessor artifact binding mismatch") + if ( + evidence["schema"] != "lex_replay_review_evidence/1" + or digest(evidence) != reference["evidence_sha256"] + or evidence["reviewer"] != reviewer + or evidence["scope"] != reference["scope"] + or evidence["task_id"] != contract["task_id"] + or evidence["sources_sha256"] != digest(contract["sources"]) + or evidence["projection_sha256"] != digest(projection) + or not isinstance(source_review, dict) + or evidence["source_review_sha256"] != digest(source_review) + or source_review.get("reviewer") != reviewer + or source_review.get("scope") != reference["scope"] + or source_review.get("task_id") != contract["task_id"] + or source_review.get("sources_sha256") != evidence["sources_sha256"] + or source_review.get("projection_sha256") != evidence["projection_sha256"] + ): + raise ValueError("review evidence reviewer, scope, source or projection binding mismatch") + + +def _validate_review_artifact(artifact: Any, content: Any) -> None: + """Verify original UTF-8 JSON bytes, including whitespace, before decoding.""" + if ( + not isinstance(artifact, dict) + or not {"utf8", "sha256", "content"} <= set(artifact) + or set(artifact) - {"utf8", "sha256", "content", "bindings"} + ): + raise ValueError("invalid materialized review artifact") + text = artifact["utf8"] + if not isinstance(text, str) or text_digest(text) != artifact["sha256"]: + raise ValueError("review artifact byte hash mismatch") + if strict_json(text) != artifact["content"] or artifact["content"] != content: + raise ValueError("review artifact decoded content mismatch") + + +def _validate_reviewed_span_projection(contract: dict[str, Any], names: set[str]) -> None: + reference = contract["semantic_reference"] + if contract["schema"] == PROTOCOL_V2 and reference != { + "kind": "human_reviewed_span_annotation", + "browser_semantic_inference": False, + }: + raise ValueError("v2 requires an explicit human semantic reference disclosure") + spec = contract["reviewed_span_projection"] + expected = { + "kind", + "source", + "collection_pointer", + "identity_pointer", + "semantic_text_pointer", + "output_text_pointer", + "output", + "span_unit", + "annotation", + } + if not isinstance(spec, dict) or set(spec) != expected: + raise ValueError("invalid reviewed span projection fields") + if ( + spec["kind"] != REVIEWED_SPAN_KIND + or spec["source"] not in names + or spec["output"] not in {"selected_item_text", "selected_spans"} + or spec["span_unit"] != "unicode_code_points" + or not all( + _valid_pointer_path(spec[field]) + for field in ( + "collection_pointer", + "identity_pointer", + "semantic_text_pointer", + "output_text_pointer", + ) + ) + ): + raise ValueError("invalid reviewed span projection") + annotation = spec["annotation"] + if not isinstance(annotation, dict) or set(annotation) != {"schema", "semantic", "items"}: + raise ValueError("invalid reviewed span annotation fields") + if ( + annotation["schema"] != REVIEWED_SPAN_ANNOTATION + or annotation["semantic"] != "exactly_one_book_recommendation" + or not isinstance(annotation["items"], list) + or not annotation["items"] + ): + raise ValueError("invalid reviewed span annotation") + hash_pattern = re.compile(r"[0-9a-f]{64}") + for item in annotation["items"]: + if not isinstance(item, dict) or set(item) != { + "identity_sha256", + "semantic_text_sha256", + "output_text_sha256", + "recommendation_count", + "spans", + }: + raise ValueError("invalid reviewed span annotation item fields") + if ( + not isinstance(item["identity_sha256"], str) + or not hash_pattern.fullmatch(item["identity_sha256"]) + or not isinstance(item["semantic_text_sha256"], str) + or not hash_pattern.fullmatch(item["semantic_text_sha256"]) + or not isinstance(item["output_text_sha256"], str) + or not hash_pattern.fullmatch(item["output_text_sha256"]) + or type(item["recommendation_count"]) is not int + or item["recommendation_count"] < 0 + or not isinstance(item["spans"], list) + ): + raise ValueError("invalid reviewed span annotation item") + expected_span_count = 1 if item["recommendation_count"] == 1 else 0 + if len(item["spans"]) != expected_span_count: + raise ValueError("semantic count and reviewed spans disagree") + for span in item["spans"]: + if ( + not isinstance(span, dict) + or set(span) != {"start", "end"} + or type(span["start"]) is not int + or type(span["end"]) is not int + or span["start"] < 0 + or span["end"] <= span["start"] + ): + raise ValueError("invalid reviewed span boundary") + + +def _project_reviewed_spans(contract: dict[str, Any], sources: dict[str, Any]) -> list[str]: + spec = contract["reviewed_span_projection"] + values = pointer(sources[spec["source"]], spec["collection_pointer"]) + annotations = spec["annotation"]["items"] + if not isinstance(values, list) or len(values) != len(annotations): + raise ValueError("reviewed annotation does not cover the complete observed collection") + answer: list[str] = [] + for index, (value, annotation) in enumerate(zip(values, annotations, strict=True)): + identity = pointer(value, spec["identity_pointer"]) + semantic_text = pointer(value, spec["semantic_text_pointer"]) + output_text = pointer(value, spec["output_text_pointer"]) + if not isinstance(semantic_text, str) or not isinstance(output_text, str): + raise ValueError("reviewed semantic and output source text must be strings") + if ( + digest(identity) != annotation["identity_sha256"] + or text_digest(semantic_text) != annotation["semantic_text_sha256"] + or text_digest(output_text) != annotation["output_text_sha256"] + ): + raise ValueError(f"reviewed source identity or text changed at index {index}") + if annotation["recommendation_count"] != 1: + continue + if spec["output"] == "selected_item_text": + answer.append(output_text) + continue + span = annotation["spans"][0] + if span["end"] > len(output_text): + raise ValueError(f"reviewed span exceeds current text at index {index}") + selected = output_text[span["start"] : span["end"]] + if not selected: + raise ValueError(f"reviewed span is empty at index {index}") + answer.append(selected) + return answer + + +def _validate_projection(expression: Any, names: set[str], depth: int = 0) -> None: + if depth > 32 or not isinstance(expression, dict): + raise ValueError("invalid projection") + if set(expression) == {"source", "pointer"}: + if expression["source"] not in names or not isinstance(expression["pointer"], str): + raise ValueError("unknown projection source or pointer") + path = expression["pointer"] + if (path and not path.startswith("/")) or re.search(r"~(?![01])", path): + raise ValueError("invalid projection pointer") + return + if ( + set(expression) == {"array"} + and isinstance(expression["array"], list) + and expression["array"] + ): + for child in expression["array"]: + _validate_projection(child, names, depth + 1) + return + if ( + set(expression) == {"object"} + and isinstance(expression["object"], dict) + and expression["object"] + ): + for child in expression["object"].values(): + _validate_projection(child, names, depth + 1) + return + if ( + set(expression) == {"map"} + and isinstance(expression["map"], dict) + and set(expression["map"]) == {"input", "projection"} + ): + _validate_projection(expression["map"]["input"], names, depth + 1) + _validate_projection(expression["map"]["projection"], {"$item"}, depth + 1) + return + raise ValueError("unsupported projection; constants and executable code are forbidden") + + +def _validate_answer(value: Any) -> None: + if value is None or value == "" or (isinstance(value, (dict, list)) and not value): + raise ValueError("answer contains a missing or empty value") + if isinstance(value, dict): + for child in value.values(): + _validate_answer(child) + elif isinstance(value, list): + for child in value: + _validate_answer(child) + + +def build_actual_response( + *, + contract: dict[str, Any] | None, + task_id: int, + calls: list[dict[str, Any]], + records: list[dict[str, Any]], + run_id: str, + attempt_id: str, +) -> dict[str, Any]: + result: dict[str, Any] = { + "schema": contract.get("schema", PROTOCOL) if contract is not None else PROTOCOL, + "run_id": run_id, + "attempt_id": attempt_id, + "task_id": task_id, + "status": "unsupported", + "actual_response": None, + "sources": [], + } + if contract is None: + return {**result, "reason": "missing_frozen_observation_contract"} + try: + if not run_id or not attempt_id or contract.get("task_id") != task_id: + raise ValueError("missing or mismatched execution identity") + validate_contract(contract, calls) + result["contract_sha256"] = digest(contract) + values = {} + for source in contract["sources"]: + step = source["step"] + row = records[step - 1] + if ( + row.get("index") != step + or row.get("status") != "success" + or row.get("tool_status", "success") != "success" + or row.get("observation_status") == "error" + ): + raise ValueError("observation action did not succeed") + if row.get("arguments") != calls[step - 1] or not isinstance(row.get("result"), str): + raise ValueError("observation trace binding mismatch") + values[source["name"]] = strict_json(row["result"]) + result["sources"].append( + { + "name": source["name"], + "step": step, + "action_sha256": digest(row["arguments"]), + "output_sha256": digest(row["result"]), + } + ) + reviewed_scope = None + if "reviewed_scope" in contract: + reviewed_scope = _apply_reviewed_scope(contract, values, result["sources"]) + result["reviewed_scope"] = reviewed_scope + for name, requirement in contract.get("requirements", {}).items(): + observed = project(requirement, values) + if observed is not True and not ( + name == "scope" and observed is False and reviewed_scope is not None + ): + raise ValueError("observation scope or completeness requirement is not true") + if "reviewed_span_projection" in contract: + data = _project_reviewed_spans(contract, values) + result["semantic_reference"] = { + **contract["semantic_reference"], + "annotation_sha256": digest(contract["reviewed_span_projection"]["annotation"]), + } + elif contract["schema"] == PROTOCOL_V3: + data = project(contract["projection"]["distinct_exact_strings"], values) + if not isinstance(data, list) or any(type(value) is not str for value in data): + raise ValueError("distinct projection requires an array of exact strings") + data = list(dict.fromkeys(data)) + result["semantic_reference"] = contract["semantic_reference"] + else: + data = project(contract["projection"], values) + # Absence requires a separate, qualified completeness protocol. In v1 + # null/empty results cannot silently become NOT_FOUND or success. + proven_absence = ( + isinstance(data, list) + and not data + and contract.get("empty_result") == "NOT_FOUND_ERROR" + ) + if proven_absence and any( + project(contract["requirements"][name], values) is not True + for name in ("scope", "completeness") + ): + raise ValueError("empty retrieval requires observed complete task scope") + if proven_absence: + _verify_absence_coverage(contract, values) + if not proven_absence and (data is None or data == [] or data == {} or data == ""): + raise ValueError("empty retrieval requires a completeness contract") + if not proven_absence: + _validate_answer(data) + result.update( + status="supported", + actual_response={ + "task_type": "RETRIEVE", + "status": "NOT_FOUND_ERROR" if proven_absence else "SUCCESS", + "retrieved_data": None if proven_absence else data, + "error_details": None, + }, + ) + result["actual_response_sha256"] = digest(result["actual_response"]) + except (ValueError, TypeError, KeyError, IndexError) as error: + result.update(status="insufficient", reason=str(error), actual_response=None) + return result + + +def load_contracts( + binding: dict[str, Any] | None, root: Path, candidates: dict[int, dict[str, Any]] +) -> dict[int, dict[str, Any]]: + if binding is None: + return {} + if binding.get("protocol") not in SUPPORTED_PROTOCOLS: + raise ValueError("unsupported retrieval protocol") + raw = root / str(binding.get("path") or "") + if ( + any(path.is_symlink() for path in [raw, *raw.parents]) + or root.resolve() not in raw.resolve().parents + ): + raise ValueError("unsafe retrieval contract path") + data = raw.read_bytes() + if hashlib.sha256(data).hexdigest() != binding.get("sha256"): + raise ValueError("retrieval contract file hash mismatch") + catalog = strict_json(data.decode()) + if ( + set(catalog) != {"schema", "contracts"} + or catalog["schema"] != binding["protocol"] + or catalog["schema"] not in SUPPORTED_PROTOCOLS + ): + raise ValueError("invalid retrieval contract catalog") + result = {} + for contract in catalog["contracts"]: + task_id = contract["task_id"] + if type(task_id) is not int or task_id in result or task_id not in candidates: + raise ValueError("duplicate or unknown retrieval task") + validate_contract(contract, candidates[task_id]["tool_calls"]) + result[task_id] = contract + return result + + +def require_current_retrieval(plan: dict[str, Any]) -> None: + missing = set(plan.get("retrieval_task_ids", [])) - set(plan.get("retrieval_contracts", {})) + if missing: + raise ValueError( + "current-run retrieval contracts required before qualification/promotion: " + f"{sorted(missing)}" + ) + + +def verify_attempt( + attempt: dict[str, Any], + contract: dict[str, Any], + calls: list[dict[str, Any]], + *, + expected_data_checksum: str, +) -> None: + if attempt.get("retrieval_protocol") != contract.get("schema"): + raise ValueError("retrieval attempt uses a different contract protocol") + evidence = attempt.get("evidence") or {} + loaded = {} + for name in ("trace", "retrieval_acceptance", "actual_response", "evaluation"): + ref = evidence.get(name) or {} + raw = Path(str(ref.get("path") or "")) + if ( + not raw.is_absolute() + or any(path.is_symlink() for path in [raw, *raw.parents]) + or not raw.is_file() + ): + raise ValueError("missing or unsafe current-run retrieval evidence") + data = raw.read_bytes() + if hashlib.sha256(data).hexdigest() != ref.get("sha256"): + raise ValueError("current-run retrieval evidence hash mismatch") + loaded[name] = strict_json(data.decode()) + receipt = loaded["retrieval_acceptance"] + recomputed = build_actual_response( + contract=contract, + task_id=int(attempt["task_id"]), + calls=calls, + records=loaded["trace"], + run_id=attempt.get("run_id", ""), + attempt_id=attempt.get("attempt_id", ""), + ) + if ( + digest(recomputed) != digest(receipt) + or receipt["status"] != "supported" + or digest(loaded["actual_response"]) != receipt["actual_response_sha256"] + ): + raise ValueError("current-run retrieval cannot be reproduced") + if attempt.get("evaluation_input_sha256") != receipt["actual_response_sha256"]: + raise ValueError("evaluator did not consume current-run retrieval response") + answer_rows = [ + row + for row in loaded["evaluation"].get("evaluators_results", []) + if row.get("evaluator_name") == "AgentResponseEvaluator" + ] + if ( + len(answer_rows) != 1 + or digest(answer_rows[0].get("actual")) != receipt["actual_response_sha256"] + ): + raise ValueError("official evaluator actual differs from current-run retrieval") + evaluation = loaded["evaluation"] + if evaluation.get("task_id") != attempt["task_id"]: + raise ValueError("official evaluator task differs from retrieval attempt") + from .webarena_dataset import _validate_evaluation_provenance + + _validate_evaluation_provenance(evaluation, attempt, expected_data_checksum) + accepted = ( + evaluation.get("score") == 1 + and evaluation.get("status") in {"pass", "success"} + and answer_rows[0].get("score") == 1 + and answer_rows[0].get("status") in {"pass", "success"} + ) + if attempt.get("evaluator_status") != ("pass" if accepted else "fail"): + raise ValueError("claimed evaluator status contradicts actual evaluation") + if attempt.get("status") != ( + "pass" if accepted and attempt.get("process_status") == "pass" else "fail" + ): + raise ValueError("claimed verdict contradicts actual evaluation") diff --git a/harness/trajectory_replay/runner.py b/harness/trajectory_replay/runner.py index c385384a1..261fb29bc 100644 --- a/harness/trajectory_replay/runner.py +++ b/harness/trajectory_replay/runner.py @@ -4,6 +4,7 @@ import hashlib import json +import os import platform import threading import time @@ -33,7 +34,9 @@ ENGINE_IMAGES = { "chromium": "lex-browser-use-eval-chromium:local", - "moli": "lex-browser-use-eval-moli:local", + # A diagnostic may point only Moli at a separately tagged, hash-recorded image. + # The formal current profile continues to use the pinned local tag by default. + "moli": os.environ.get("BROWSER_EVAL_MOLI_IMAGE", "lex-browser-use-eval-moli:local"), } ENGINE_PORTS = {"chromium": 9223, "moli": 9222} ENCODING = tiktoken.get_encoding("o200k_base") diff --git a/harness/trajectory_replay/webarena_corrections.py b/harness/trajectory_replay/webarena_corrections.py index 208c97b6c..788a6c16f 100644 --- a/harness/trajectory_replay/webarena_corrections.py +++ b/harness/trajectory_replay/webarena_corrections.py @@ -128,6 +128,7 @@ class CorrectedReplayContract: corrected_task_sha256: str rationale: str replay_assertions: tuple[Mapping[str, Any], ...] + network_alternatives: Mapping[int, tuple[WebArenaVerifiedTask, ...]] def load_corrections(path: str | Path = DEFAULT_CORRECTIONS_PATH) -> CorrectionCatalog: @@ -155,6 +156,20 @@ def load_corrections(path: str | Path = DEFAULT_CORRECTIONS_PATH) -> CorrectionC raise ValueError(f"task {task_id}: official_revision must be positive") if not record.get("preconditions") or not record.get("operations"): raise ValueError(f"task {task_id}: preconditions and operations are required") + if (record.get("network_event_alternatives") or record.get("reference_receipt") + or record.get("root_cause") or record.get("regression_ids")): + regression_ids = record.get("regression_ids") + if ( + not record.get("root_cause") + or not isinstance(regression_ids, list) + or len(regression_ids) < 2 + or not all(isinstance(item, str) and item for item in regression_ids) + or len(regression_ids) != len(set(regression_ids)) + ): + raise ValueError( + f"task {task_id}: reference or network correction needs root cause " + "and paired regressions" + ) records[task_id] = MappingProxyType(copy.deepcopy(record)) record_sha256s[task_id] = _digest(record) return CorrectionCatalog( @@ -194,7 +209,137 @@ def apply_correction( for operation in record["operations"]: _apply_operation(corrected, operation) + reference = record.get("reference_receipt") + if reference is not None: + # Receipt paths are repository-relative, including for catalogs retained + # in a deeper revisions directory. Their location must not move the + # reference root or make archived corrections irreproducible. + project_root = Path(__file__).resolve().parents[2] + reference_path = (project_root / str(reference["path"])).resolve() + try: + reference_path.relative_to(project_root) + except ValueError as exc: + raise ValueError(f"task {task.task_id}: reference receipt escapes repository") from exc + if reference_path.is_symlink() or not reference_path.is_file(): + raise ValueError(f"task {task.task_id}: reference receipt missing or symlinked") + raw_reference = reference_path.read_bytes() + if hashlib.sha256(raw_reference).hexdigest() != reference.get("sha256"): + raise ValueError(f"task {task.task_id}: reference receipt changed") + observation = json.loads(raw_reference) + if observation.get("task_id") != task.task_id: + raise ValueError(f"task {task.task_id}: reference receipt task changed") + if task.task_id == 225: + graphql = observation.get("independent_source") or {} + products = graphql.get("title_matched_items") or [] + check = observation.get("database_cross_check") or {} + expected_response = observation.get("expected_response") or {} + binding = observation.get("candidate_binding") or {} + candidate_path = (project_root / str(binding.get("path") or "")).resolve() + try: + candidate_path.relative_to(project_root) + except ValueError as exc: + raise ValueError("task 225: candidate path escapes repository") from exc + if ( + observation.get("schema") != "trajectory_replay_revised_reference/1" + or candidate_path.is_symlink() + or not candidate_path.is_file() + or hashlib.sha256(candidate_path.read_bytes()).hexdigest() != binding.get("sha256") + or graphql.get("kind") != "public_storefront_graphql" + or graphql.get("total_count") != graphql.get("returned_items") + or graphql.get("total_count") != 6 + or graphql.get("other_sephora_title_items") != 4 + or len(products) != 2 + or check.get("catalog_match_count") != len(products) + or check.get("review_row_count") != 0 + or check.get("manufacturer_attribute_rows") != 0 + or any( + not all(word in item.get("name", "").lower() for word in ("sephora", "brush")) + or item.get("review_count") != 0 + or (item.get("reviews") or {}).get("items") != [] + or (item.get("reviews") or {}).get("page_info", {}).get("total_pages") != 0 + for item in products + ) + or expected_response.get("task_type") != "RETRIEVE" + or expected_response.get("status") != "NOT_FOUND_ERROR" + or expected_response.get("retrieved_data") is not None + ): + raise ValueError(f"task {task.task_id}: incomplete independent reference") + corrected["eval"][0]["expected"] = copy.deepcopy(expected_response) + elif observation.get("derivation", {}).get("kind") == "qualified_catalog_price_range/1": + source = observation.get("independent_source") or {} + products = source.get("qualified_products") or [] + if ( + observation.get("schema") != "trajectory_replay_revised_reference/1" + or source.get("total_count") != source.get("returned_items") + or type(source.get("total_count")) is not int + or source["total_count"] < len(products) + or not products + or len({row.get("sku") for row in products}) != len(products) + or len({row.get("name") for row in products}) != len(products) + or any( + not row.get("sku") + or not row.get("name") + or type(row.get("price_cents")) is not int + or row["price_cents"] < 0 + or row.get("currency") != "USD" + or not row.get("qualification_evidence") + for row in products + ) + ): + raise ValueError(f"task {task.task_id}: incomplete qualified catalog reference") + prices = [row["price_cents"] for row in products] + expected = { + "task_type": "RETRIEVE", + "status": "SUCCESS", + "retrieved_data": [ + { + "names": [row["name"] for row in products], + "min": min(prices) / 100, + "max": max(prices) / 100, + } + ], + "error_details": None, + } + if observation.get("expected_response") != expected: + raise ValueError(f"task {task.task_id}: catalog reference derivation differs") + corrected["eval"][0]["expected"] = expected + else: + raise ValueError(f"task {task.task_id}: unsupported independent reference") + corrected_task = type(task).model_validate(corrected) + network_alternatives: dict[int, list[WebArenaVerifiedTask]] = {} + for alternative in record.get("network_event_alternatives") or []: + index = alternative.get("evaluator_index") + operations = alternative.get("operations") + if ( + not isinstance(index, int) + or not 0 <= index < len(corrected["eval"]) + or corrected["eval"][index]["evaluator"] != "NetworkEventEvaluator" + or not isinstance(operations, list) + or not operations + or not all( + operation.get("path") + in { + f"/eval/{index}/expected/response_status", + f"/eval/{index}/expected/response_content", + } + for operation in operations + ) + ): + raise ValueError(f"task {task.task_id}: invalid network event alternative") + alternative_data = copy.deepcopy(corrected) + for operation in operations: + _apply_operation(alternative_data, operation) + alternative_task = type(task).model_validate(alternative_data) + if ( + alternative_task.eval[index].expected.url != corrected_task.eval[index].expected.url + or alternative_task.eval[index].expected.http_method + != corrected_task.eval[index].expected.http_method + or alternative_task.eval[index].expected.post_data + != corrected_task.eval[index].expected.post_data + ): + raise ValueError(f"task {task.task_id}: network alternative changed the mutation") + network_alternatives.setdefault(index, []).append(alternative_task) assertions = tuple( MappingProxyType(copy.deepcopy(item)) for item in record.get("replay_assertions") or [] ) @@ -206,6 +351,9 @@ def apply_correction( corrected_task_sha256=_digest(corrected_task.model_dump(mode="json")), rationale=str(record["rationale"]), replay_assertions=assertions, + network_alternatives=MappingProxyType( + {index: tuple(tasks) for index, tasks in network_alternatives.items()} + ), ) @@ -269,27 +417,33 @@ def evaluate_corrected_contract( for item in contract.replay_assertions ) evaluator_results = [] - for config in task.eval: + for index, config in enumerate(task.eval): evaluator_context = context if ordinary_network_filtering and config.evaluator == "NetworkEventEvaluator": task_data = task.model_dump(mode="json") evaluations = task_data.get("eval") or [] if not evaluations or not isinstance(evaluations[0].get("expected"), dict): - raise ValueError( - "ordinary network filtering requires eval[0].expected" - ) + raise ValueError("ordinary network filtering requires eval[0].expected") # webarena-verified 1.2.3 reads the task type used by # NetworkEventEvaluator from eval[0].expected, regardless of the # network evaluator's own position in the task configuration. evaluations[0]["expected"]["task_type"] = "MUTATE" filtering_task = type(task).model_validate(task_data) evaluator_context = context.model_copy(update={"task": filtering_task}) - evaluator_results.append( - EVALUATOR_REGISTRY[config.evaluator]().evaluate( - context=evaluator_context, - config=config, - ) + result = EVALUATOR_REGISTRY[config.evaluator]().evaluate( + context=evaluator_context, + config=config, ) + if float(result.score) != 1.0: + for alternative_task in contract.network_alternatives.get(index, ()): + alternative_result = EVALUATOR_REGISTRY[config.evaluator]().evaluate( + context=context.model_copy(update={"task": alternative_task}), + config=alternative_task.eval[index], + ) + if float(alternative_result.score) == 1.0: + result = alternative_result + break + evaluator_results.append(result) return TaskEvalResult.create( task_id=task.task_id, intent_template_id=task.intent_template_id, diff --git a/harness/trajectory_replay/webarena_dataset.py b/harness/trajectory_replay/webarena_dataset.py index 8e6e82b1f..0b9618b70 100644 --- a/harness/trajectory_replay/webarena_dataset.py +++ b/harness/trajectory_replay/webarena_dataset.py @@ -13,6 +13,11 @@ from webarena_verified.core.utils.checksum import compute_evaluator_checksum +from harness.trajectory_replay.retrieval_acceptance import ( + digest, + require_current_retrieval, + verify_attempt, +) from harness.trajectory_replay.webarena_plan import ( load_replay_plan, replay_calls_binding_sha256, @@ -21,6 +26,29 @@ SCHEMA = "lex_webarena_verified_trajectory_dataset/1" RECEIPT_SCHEMA = "lex_webarena_trajectory_replay_suite/1" +OBSERVATION_PROTOCOL = "lex_replay_valid_snapshot_observations/1" + + +def _validate_observations(trace: list[dict[str, Any]]) -> None: + """Task acceptance is not proof that every required observation succeeded.""" + for row in trace: + if row.get("observation_status") == "error": + raise ValueError("qualification requires valid observations, not observation errors") + action = (row.get("arguments") or {}).get("action") + text = row.get("result") + if action not in {"snapshot", "navigate"}: + continue + if not isinstance(text, str) or not text.strip(): + raise ValueError("qualification requires a materialized snapshot observation") + snapshot_text = text + if action == "navigate": + _, separator, snapshot_text = text.partition("--- Page Snapshot ---\n") + if not separator: + raise ValueError("qualification requires the navigation's snapshot observation") + if not snapshot_text.strip() or snapshot_text.startswith("[Snapshot error:"): + raise ValueError( + "qualification cannot promote a successful wrapper around snapshot error" + ) @lru_cache(maxsize=1) @@ -42,9 +70,7 @@ def _official_evaluator_names() -> dict[int, tuple[str, ...]]: @lru_cache(maxsize=1) -def _effective_correction_bindings() -> dict[ - int, tuple[str, str, str, tuple[str, ...]] -]: +def _effective_correction_bindings() -> dict[int, tuple[str, str, str, tuple[str, ...]]]: from harness.browser_use.runner import _configure_benchmark # noqa: PLC0415 from harness.trajectory_replay.webarena_corrections import ( # noqa: PLC0415 apply_correction, @@ -57,8 +83,7 @@ def _effective_correction_bindings() -> dict[ for task_id in catalog.records: contract = apply_correction(benchmark.get_task(task_id), catalog) evaluators = tuple( - str(item["evaluator"]) - for item in contract.task.model_dump(mode="json")["eval"] + str(item["evaluator"]) for item in contract.task.model_dump(mode="json")["eval"] ) bindings[task_id] = ( contract.correction_id, @@ -69,6 +94,50 @@ def _effective_correction_bindings() -> dict[ return bindings +def _validate_evaluation_provenance( + evaluation: dict[str, Any], attempt: dict[str, Any], expected_data_checksum: str +) -> None: + """One rule identity gate shared by qualification, resume and aggregation.""" + task_id = int(attempt["task_id"]) + correction = _effective_correction_bindings().get(task_id) + actual_correction = tuple( + str(attempt.get(key) or "") + for key in ( + "contract_correction_id", + "contract_correction_source_sha256", + "corrected_task_sha256", + ) + ) + expected_correction = correction[:3] if correction else ("", "", "") + if actual_correction != expected_correction: + raise ValueError("evaluation task correction binding is stale or incomplete") + evaluated_correction = evaluation.get("replay_contract_correction") + if correction is None: + if evaluated_correction is not None: + raise ValueError("unexpected evaluated task correction") + elif ( + not isinstance(evaluated_correction, dict) + or tuple( + str(evaluated_correction.get(key) or "") + for key in ("correction_id", "source_sha256", "corrected_task_sha256") + ) + != expected_correction + ): + raise ValueError("evaluation evidence task correction is stale or incomplete") + if ( + evaluation.get("task_id") != task_id + or evaluation.get("webarena_verified_version") != version("webarena-verified") + or evaluation.get("webarena_verified_evaluator_checksum") != compute_evaluator_checksum() + or not expected_data_checksum + or evaluation.get("webarena_verified_data_checksum") != expected_data_checksum + or tuple(row.get("evaluator_name") for row in evaluation.get("evaluators_results", [])) + != _official_evaluator_names().get(task_id) + ): + raise ValueError( + "evaluation rule or official task data provenance differs from current inputs" + ) + + def _validate_qualification_receipt( receipt: dict[str, Any], *, @@ -76,14 +145,13 @@ def _validate_qualification_receipt( seed_task_ids: set[int], required_engines: tuple[str, ...], expected_resources: dict[str, Any], + expected_execution: dict[str, Any], ) -> None: def required_int(value: Any, field: str) -> int: try: return int(value) except (TypeError, ValueError) as exc: - raise ValueError( - f"replay receipt has invalid {field}: {receipt_path}" - ) from exc + raise ValueError(f"replay receipt has invalid {field}: {receipt_path}") from exc if receipt.get("schema") != RECEIPT_SCHEMA: raise ValueError(f"unsupported replay receipt: {receipt_path}") @@ -122,7 +190,8 @@ def required_int(value: Any, field: str) -> int: "environment_pool_count": 1, "method_parallelism": 1, "task_parallelism": 1, - "reset_before_every_task": True, + "reset_before_every_task": expected_execution["reset_before_every_task"], + "reset_before_every_method": expected_execution["reset_before_every_method"], "teardown_between_methods": True, } if any(isolation.get(key) != value for key, value in required_isolation.items()): @@ -170,9 +239,7 @@ def required_int(value: Any, field: str) -> int: ( str(item.get("engine")), required_int(item.get("task_id"), "physical task_id"), - required_int( - item.get("physical_execution"), "physical execution number" - ), + required_int(item.get("physical_execution"), "physical execution number"), ) ) if ( @@ -190,21 +257,16 @@ def required_int(value: Any, field: str) -> int: ) physical_by_attempt[key].append(item) attempts_by_key = { - (str(item["engine"]), int(item["task_id"])): item - for item in receipt.get("attempts") or [] + (str(item["engine"]), int(item["task_id"])): item for item in receipt.get("attempts") or [] } if set(physical_by_attempt) != expected_keys: raise ValueError(f"replay receipt has unbound physical executions: {receipt_path}") for key, attempt in attempts_by_key.items(): physical = physical_by_attempt[key] - if ( - len(physical) != int(attempt.get("physical_execution_count") or 1) - or max(int(item["physical_execution"]) for item in physical) - != int(attempt.get("physical_execution") or 0) - ): - raise ValueError( - f"replay receipt has inconsistent physical executions: {receipt_path}" - ) + if len(physical) != int(attempt.get("physical_execution_count") or 1) or max( + int(item["physical_execution"]) for item in physical + ) != int(attempt.get("physical_execution") or 0): + raise ValueError(f"replay receipt has inconsistent physical executions: {receipt_path}") def _validate_passing_attempt_evidence( @@ -213,6 +275,8 @@ def _validate_passing_attempt_evidence( runtime_site_host: str, expected_data_checksum: str, trusted_source_site_hosts: set[str], + retrieval_contract: dict[str, Any] | None = None, + state_contract: Any = None, ) -> None: if attempt.get("process_status") != "pass" or attempt.get("evaluator_status") != "pass": return @@ -241,9 +305,16 @@ def _validate_passing_attempt_evidence( ) ): raise ValueError("passing replay attempt differs from the canonical replay actions") - if attempt.get("evaluation_input_sha256") != _json_sha256( - candidate["recorded_response"] - ): + if str(candidate["recorded_response"].get("task_type", "")).upper() == "RETRIEVE": + if retrieval_contract is None: + raise ValueError("retrieval qualification requires current-run answer binding") + verify_attempt( + attempt, + retrieval_contract, + candidate["tool_calls"], + expected_data_checksum=expected_data_checksum, + ) + elif attempt.get("evaluation_input_sha256") != _json_sha256(candidate["recorded_response"]): raise ValueError("passing replay attempt is not bound to the canonical evaluator input") trace = _read_json(paths["trace"]) @@ -253,11 +324,10 @@ def _validate_passing_attempt_evidence( or any(item.get("status") != "success" for item in trace) or attempt.get("executed_tool_calls_sha256") != _json_sha256(trace_arguments) or attempt.get("replay_calls_binding_sha256") - != replay_calls_binding_sha256( - trace_arguments, allowed_hosts={runtime_site_host} - ) + != replay_calls_binding_sha256(trace_arguments, allowed_hosts={runtime_site_host}) ): raise ValueError("passing replay trace does not prove every executed action") + _validate_observations(trace) network_har = _read_json(paths["network_har"]) if not isinstance(network_har, dict) or not isinstance( @@ -265,6 +335,13 @@ def _validate_passing_attempt_evidence( ): raise ValueError("passing replay attempt has an invalid network HAR") + if state_contract is not None: + from harness.webarena_state import verify_attempt as verify_state_attempt + + if attempt.get("contract_correction_id"): + raise ValueError("observed state cannot also apply a legacy replay correction") + verify_state_attempt(attempt, state_contract) + return evaluation = _read_json(paths["evaluation"]) task_id = int(attempt["task_id"]) expected_evaluators = _official_evaluator_names().get(task_id) @@ -284,8 +361,7 @@ def _validate_passing_attempt_evidence( or float(evaluation.get("score") or 0) != 1.0 or str(evaluation.get("status") or "").lower() not in {"pass", "success"} or evaluation.get("webarena_verified_version") != version("webarena-verified") - or evaluation.get("webarena_verified_evaluator_checksum") - != compute_evaluator_checksum() + or evaluation.get("webarena_verified_evaluator_checksum") != compute_evaluator_checksum() or not expected_data_checksum or evaluation.get("webarena_verified_data_checksum") != expected_data_checksum ): @@ -296,14 +372,12 @@ def _validate_passing_attempt_evidence( or tuple(str(row.get("evaluator_name") or "") for row in evaluator_rows) != expected_evaluators or any( - float(row.get("score") or 0) != 1.0 - or str(row.get("status") or "").lower() not in {"pass", "success"} - for row in evaluator_rows + float(row.get("score") or 0) != 1.0 + or str(row.get("status") or "").lower() not in {"pass", "success"} + for row in evaluator_rows ) ): - actual_evaluators = tuple( - str(row.get("evaluator_name") or "") for row in evaluator_rows - ) + actual_evaluators = tuple(str(row.get("evaluator_name") or "") for row in evaluator_rows) raise ValueError( "passing replay evaluator evidence does not match the canonical input: " f"task {attempt['task_id']}, expected evaluators {expected_evaluators}, " @@ -311,6 +385,7 @@ def _validate_passing_attempt_evidence( ) if any(item.get("status") != "pass" for item in evaluation.get("replay_assertions") or []): raise ValueError("passing replay evaluation contains a failed replay assertion") + _validate_evaluation_provenance(evaluation, attempt, expected_data_checksum) def _has_stale_correction_binding(attempt: dict[str, Any], task_id: int) -> bool: @@ -380,6 +455,7 @@ def build_verified_trajectory_dataset( raise ValueError("replay seed manifest contains duplicate task IDs") seed_task_ids = set(seed_tasks) plan = load_replay_plan(profile_path, seed_path) + require_current_retrieval(plan) profile = plan["profile"] qualification_engine = str(plan["qualification_engine"]) required_engines = (qualification_engine,) @@ -397,6 +473,7 @@ def build_verified_trajectory_dataset( seed_task_ids=seed_task_ids, required_engines=required_engines, expected_resources=profile["resources"], + expected_execution=profile["execution"], ) receipts.append( { @@ -407,8 +484,18 @@ def build_verified_trajectory_dataset( "profile_sha256": str(receipt.get("profile_sha256") or ""), } ) + from harness.webarena_state import validate_attempt_identities + + validate_attempt_identities( + receipt.get("attempts") or [], set(plan.get("state_contracts", {})) + ) for attempt in receipt.get("attempts") or []: task_id = int(attempt["task_id"]) + if ( + task_id in plan.get("retrieval_task_ids", []) + or task_id in plan.get("state_contracts", {}) + ) and attempt.get("run_id") != receipt.get("run_id"): + raise ValueError("retrieval evidence belongs to another qualification run") candidate_id = str(attempt["source_candidate_id"]) if candidate_id != seed_tasks[task_id].get("canonical_candidate_id"): if receipt.get("source_seed_manifest_sha256") == seed_sha256: @@ -429,24 +516,24 @@ def build_verified_trajectory_dataset( ) continue if ( - receipt.get("source_seed_manifest_sha256") != seed_sha256 + task_id not in plan.get("state_contracts", {}) + and receipt.get("source_seed_manifest_sha256") != seed_sha256 and _has_stale_correction_binding(attempt, task_id) ): continue - session = { - str(item["id"]): item for item in receipt.get("sessions") or [] - }[str(attempt["browser_session_id"])] + session = {str(item["id"]): item for item in receipt.get("sessions") or []}[ + str(attempt["browser_session_id"]) + ] _validate_passing_attempt_evidence( attempt, candidate, str(session.get("site_access_host") or ""), str((seeds.get("source") or {}).get("official_data_checksum") or ""), plan["trusted_source_site_hosts"], + plan.get("retrieval_contracts", {}).get(task_id), + plan.get("state_contracts", {}).get(task_id), ) - if ( - attempt.get("process_status") != "pass" - or attempt.get("evaluator_status") != "pass" - ): + if attempt.get("process_status") != "pass" or attempt.get("evaluator_status") != "pass": continue key = (task_id, candidate_id) attempts_by_key[key].append(attempt) @@ -514,11 +601,29 @@ def build_verified_trajectory_dataset( "evaluation_input": candidate["recorded_response"], "source": _candidate_source(candidate), "verification": { + "state_contract_sha256": plan["state_contracts"][task_id].definition_sha256 + if task_id in plan.get("state_contracts", {}) + else None, + "retrieval_contract_sha256": ( + digest(plan["retrieval_contracts"][task_id]) + if task_id in plan.get("retrieval_task_ids", []) + else None + ), + "retrieval_protocol": ( + plan["retrieval_contracts"][task_id]["schema"] + if task_id in plan.get("retrieval_task_ids", []) + else None + ), "browsers": list(required_engines), "process_status": "pass", "evaluator_status": "pass", "contract": ( { + "kind": "observed_form_state_contract", + "contract_sha256": plan["state_contracts"][task_id].definition_sha256, + } + if task_id in plan.get("state_contracts", {}) + else { "kind": "corrected_replay_contract", "correction_id": correction_proof[0], "correction_source_sha256": correction_proof[1], @@ -545,7 +650,10 @@ def build_verified_trajectory_dataset( "browser_use_scoring_authority": "official task outcome contract", "browser_use_agent_visibility": "hidden", "trajectory_replay_role": "fixed BrowserTool action input", - "evaluation_input_role": "official evaluator input only; never an Agent prompt", + "evaluation_input_role": ( + "frozen historical response only; RETRIEVE evaluator input " + "comes from current-run observations" + ), }, "promotion_policy": { "qualification_browser": qualification_engine, @@ -555,6 +663,26 @@ def build_verified_trajectory_dataset( "model_calls": 0, }, "provenance": { + **( + { + "retrieval_protocol": next( + iter( + { + contract["schema"] + for contract in plan["retrieval_contracts"].values() + } + ) + ) + } + if len({contract["schema"] for contract in plan["retrieval_contracts"].values()}) + == 1 + else {} + ), + "retrieval_protocols": { + str(task_id): contract["schema"] + for task_id, contract in sorted(plan["retrieval_contracts"].items()) + }, + "observation_protocol": OBSERVATION_PROTOCOL, "seed_manifest_sha256": seed_sha256, "profile_sha256": profile_sha256, "replay_receipts": receipts, diff --git a/harness/trajectory_replay/webarena_plan.py b/harness/trajectory_replay/webarena_plan.py index 7e7dea906..2bd378e0b 100644 --- a/harness/trajectory_replay/webarena_plan.py +++ b/harness/trajectory_replay/webarena_plan.py @@ -15,6 +15,7 @@ compile_replay_calls, extract_browser_calls, ) +from harness.trajectory_replay.retrieval_acceptance import load_contracts from harness.trajectory_replay.runner import ENGINE_IMAGES PROFILE_SCHEMA = "lex_webarena_trajectory_replay_profile/1" @@ -36,15 +37,11 @@ def replay_site_hosts(calls: list[dict[str, Any]]) -> set[str]: for call in calls if isinstance(call.get("url"), str) for parsed in [urlsplit(call["url"])] - if parsed.scheme in {"http", "https"} - and parsed.port in site_ports - and parsed.hostname + if parsed.scheme in {"http", "https"} and parsed.port in site_ports and parsed.hostname } -def replay_calls_binding_sha256( - calls: list[dict[str, Any]], *, allowed_hosts: set[str] -) -> str: +def replay_calls_binding_sha256(calls: list[dict[str, Any]], *, allowed_hosts: set[str]) -> str: """Hash calls while ignoring only the deployment-specific WebArena gateway host.""" site_ports = {port for pair in SITE_PORTS.values() for port in pair} @@ -69,9 +66,7 @@ def replay_calls_binding_sha256( ) ) normalized.append(call) - payload = json.dumps( - normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) + payload = json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")) return hashlib.sha256(payload.encode()).hexdigest() @@ -98,9 +93,7 @@ def _validate_canonical_candidate( or source.get("kind") != "expert_repair_spec" or source.get("repair_spec_sha256") != repair_sha256 or candidate.get("id") - != str( - repair_record.get("candidate_id") or f"expert__task-{task_id}" - ) + != str(repair_record.get("candidate_id") or f"expert__task-{task_id}") or calls != repair_record.get("tool_calls") or candidate["recorded_response"] != repair_record.get("expected_response") or source.get("details") != repair_record.get("source") @@ -114,10 +107,8 @@ def _validate_canonical_candidate( method = candidate.get("method") or {} if ( authoritative_row is None - or int(candidate.get("source_execution") or 0) - != int(authoritative_row.get("attempt") or 0) - or str(candidate.get("source_run_id") or "") - != str(authoritative_row.get("run_id") or "") + or int(candidate.get("source_execution") or 0) != int(authoritative_row.get("attempt") or 0) + or str(candidate.get("source_run_id") or "") != str(authoritative_row.get("run_id") or "") or method.get("model") != authoritative_row.get("model") or method.get("engine") != authoritative_row.get("engine") or float(authoritative_row.get("score") or 0) != 1.0 @@ -132,9 +123,12 @@ def _validate_canonical_candidate( for kind, path in paths.items(): if not path.is_file() or _sha256(path) != source.get(f"{kind}_sha256"): raise ValueError(f"task {task_id} autonomous {kind} evidence is missing or changed") - if path != Path( - str((authoritative_row.get("paths") or {}).get(kind) or "") - ).expanduser().resolve(): + if ( + path + != Path(str((authoritative_row.get("paths") or {}).get(kind) or "")) + .expanduser() + .resolve() + ): raise ValueError( f"task {task_id} autonomous {kind} evidence is not from the final execution" ) @@ -146,15 +140,14 @@ def _validate_canonical_candidate( ): raise ValueError(f"task {task_id} autonomous trajectory differs from source evidence") evaluation = _read_json(paths["evaluation"]) - if int(evaluation.get("task_id") or -1) != task_id or float( - evaluation.get("score") or 0 - ) != 1.0: + if ( + int(evaluation.get("task_id") or -1) != task_id + or float(evaluation.get("score") or 0) != 1.0 + ): raise ValueError(f"task {task_id} autonomous source evaluation is not passing") -def load_replay_plan( - profile_path: str | Path, seed_manifest_path: str | Path -) -> dict[str, Any]: +def load_replay_plan(profile_path: str | Path, seed_manifest_path: str | Path) -> dict[str, Any]: """Return a validated, exact method and task plan for one formal replay suite.""" profile_file = Path(profile_path).expanduser().resolve() @@ -180,14 +173,10 @@ def load_replay_plan( repair_file = (repo_root / str(repair.get("expert_repairs_path") or "")).resolve() if repo_root not in repair_file.parents: raise ValueError("expert repairs must stay inside the repository") - if not repair_file.is_file() or _sha256(repair_file) != repair.get( - "expert_repairs_sha256" - ): + if not repair_file.is_file() or _sha256(repair_file) != repair.get("expert_repairs_sha256"): raise ValueError("expert repairs do not match the frozen replay profile") repair_payload = _read_json(repair_file) - repair_records = { - int(item["task_id"]): item for item in repair_payload.get("repairs") or [] - } + repair_records = {int(item["task_id"]): item for item in repair_payload.get("repairs") or []} if len(repair_records) != len(repair_payload.get("repairs") or []): raise ValueError("expert repairs contain duplicate task IDs") @@ -215,9 +204,7 @@ def load_replay_plan( raise ValueError("replay seed authority manifest is missing or changed") manifest, rows = _authoritative_rows(root) model = str((manifest.get("protocol") or {}).get("model") or "") - if model != declared_model or manifest.get("selection_sha256") != selection.get( - "sha256" - ): + if model != declared_model or manifest.get("selection_sha256") != selection.get("sha256"): raise ValueError("replay seed authority differs from the frozen selection") for row in rows: key = (model, str(row["engine"]), int(row["task_id"])) @@ -288,10 +275,19 @@ def load_replay_plan( raise ValueError("the fixed-port WebArena stack supports exactly one environment pool") if execution.get("method_parallelism") != 1 or execution.get("task_parallelism") != 1: raise ValueError("one environment pool requires serial methods and serial tasks") - if not execution.get("reset_before_every_task") or not execution.get( - "teardown_between_methods" + if execution.get("teardown_between_methods") is not True: + raise ValueError("formal replay requires method teardown") + if "retry_failed_task" in execution and type(execution["retry_failed_task"]) is not bool: + raise ValueError("retry_failed_task must be a boolean") + if execution.get("reset_before_every_task") is not True and ( + execution.get("reset_before_every_task") is not False + or execution.get("reset_before_every_method") is not True + or execution.get("retry_failed_task") is not False ): - raise ValueError("formal replay requires task resets and method teardown") + raise ValueError( + "method-scoped site preparation requires an explicit method reset " + "and no failed-task retry" + ) trusted_source_site_hosts = { str(host) for host in execution.get("trusted_source_site_hosts") or [] } @@ -318,8 +314,42 @@ def load_replay_plan( if (profile.get("scoring") or {}).get("model_calls") != 0: raise ValueError("trajectory replay must not call a model") + retrieval_candidates = {} + for task in seed_tasks: + canonical = next( + item for item in task["candidates"] if item["id"] == task["canonical_candidate_id"] + ) + if str(canonical["recorded_response"].get("task_type", "")).upper() == "RETRIEVE": + retrieval_candidates[int(task["task_id"])] = canonical + retrieval_contracts = load_contracts( + (profile.get("scoring") or {}).get("retrieval_acceptance"), repo_root, retrieval_candidates + ) + from harness.webarena_state import load_profile_contracts, require_supersession + + from .webarena_corrections import load_corrections + + state_contracts = load_profile_contracts( + (profile.get("scoring") or {}).get("observed_state"), repo_root + ) + if state_contracts: + corrections = load_corrections() + for task_id, contract in state_contracts.items(): + if task_id not in {int(task["task_id"]) for task in seed_tasks}: + raise ValueError("state contract outside replay task selection") + if task_id in corrections.records: + require_supersession( + contract, + corrections.records[task_id]["correction_id"], + corrections.record_sha256s[task_id], + ) return { "profile": profile, + "retrieval_task_ids": sorted(retrieval_candidates), + "retrieval_contracts": retrieval_contracts, + "retrieval_protocols": { + task_id: contract["schema"] for task_id, contract in retrieval_contracts.items() + }, + "state_contracts": state_contracts, "profile_path": str(profile_file), "profile_sha256": _sha256(profile_file), "seed_manifest": seeds, diff --git a/harness/trajectory_replay/webarena_replay.py b/harness/trajectory_replay/webarena_replay.py index c226a2e83..abd717d5f 100644 --- a/harness/trajectory_replay/webarena_replay.py +++ b/harness/trajectory_replay/webarena_replay.py @@ -4,6 +4,7 @@ import hashlib import json +import subprocess import sys import time import traceback @@ -18,6 +19,8 @@ from harness.browser_use.runner import ( SITE_PORTS, _allow_official_container_hostname, + _browser_session_contract, + _browser_tool_config, _cdp_metadata, _configure_benchmark, _container_host_gateway, @@ -26,10 +29,12 @@ _login_context, _physical_execution_lock, _physical_execution_lock_path, + _prepare_browser_viewport, _prepare_sites, _require_clean_git_checkout, _resource_summary, _site_container_manager, + _verify_session_viewport, ) from harness.trajectory_replay.metrics import DockerStatsResources, ResourceSampler from harness.trajectory_replay.runner import ( @@ -98,7 +103,9 @@ def _click_was_dispatched_before_navigation_timeout( ) -def _execute_replay_call(tool: Any, arguments: dict[str, Any]) -> tuple[str, str]: +def _execute_replay_call( + tool: Any, arguments: dict[str, Any], *, diagnostics: dict[str, Any] | None = None, +) -> tuple[str, str]: """Execute one call without changing the semantics of a compiled snapshot ref.""" replay_mode = arguments.get("replay_mode") @@ -121,15 +128,41 @@ def _execute_replay_call(tool: Any, arguments: dict[str, Any]) -> tuple[str, str service = tool._get_service() try: + def execute() -> None: page = service._page locator = page.locator(selector).first if action == "click": - locator.evaluate("element => element.click()") + if diagnostics is None: + locator.evaluate("element => element.click()") + else: + # Observe and dispatch in one JS turn: a separate probe can + # race widget initialization. Do not add actionability waits + # or reinterpret a successful DOM call as a successful task. + state = locator.evaluate("""element => { + const state = { + connected: element.isConnected, + disabled: Boolean(element.disabled), + effectivelyDisabled: element.matches(':disabled'), + ariaDisabled: element.getAttribute('aria-disabled') + }; + element.click(); + return state; + }""") + diagnostics["click_dispatch_state"] = state page.wait_for_timeout(500) elif action == "fill": - locator.evaluate("element => { element.focus(); element.value = ''; }") - page.keyboard.type(str(public_arguments.get("text", ""))) + platform = locator.evaluate( + "element => { element.focus(); return navigator.platform; }" + ) + modifier = ( + "Meta" + if str(platform).startswith(("Mac", "iPhone", "iPad", "iPod")) + else "Control" + ) + page.keyboard.press(f"{modifier}+A") + page.keyboard.press("Backspace") + page.keyboard.insert_text(str(public_arguments.get("text", ""))) else: value = str(public_arguments.get("value", "")) locator.evaluate( @@ -178,11 +211,17 @@ def select_methods(plan: dict[str, Any], engines: list[str]) -> list[dict[str, A def validate_qualification_dataset(plan: dict[str, Any], path: str | Path) -> str: """Bind a formal comparison to the complete pre-qualified trajectory set.""" + from .retrieval_acceptance import digest, require_current_retrieval + from .webarena_dataset import OBSERVATION_PROTOCOL + + require_current_retrieval(plan) dataset_path = Path(path).expanduser().resolve() dataset = json.loads(dataset_path.read_text(encoding="utf-8")) if dataset.get("schema") != "lex_webarena_verified_trajectory_dataset/1": raise ValueError("formal replay requires the current qualified trajectory dataset") provenance = dataset.get("provenance") or {} + if provenance.get("observation_protocol") != OBSERVATION_PROTOCOL: + raise ValueError("qualified dataset needs fresh valid-observation qualification") if provenance.get("seed_manifest_sha256") != plan["seed_manifest_sha256"]: raise ValueError("qualified trajectories differ from the frozen replay seeds") if provenance.get("profile_sha256") != plan["profile_sha256"]: @@ -194,18 +233,34 @@ def validate_qualification_dataset(plan: dict[str, Any], path: str | Path) -> st by_task = {int(item["task_id"]): item for item in trajectories} if len(by_task) != len(trajectories) or set(by_task) != set(plan["task_ids"]): raise ValueError("qualified trajectories do not cover the frozen task set") - seed_tasks = { - int(item["task_id"]): item for item in plan["seed_manifest"].get("tasks") or [] - } + seed_tasks = {int(item["task_id"]): item for item in plan["seed_manifest"].get("tasks") or []} for task_id, trajectory in by_task.items(): + if ( + task_id in plan.get("state_contracts", {}) + and trajectory.get("verification", {}).get("state_contract_sha256") + != plan["state_contracts"][task_id].definition_sha256 + ): + raise ValueError("legacy qualification does not prove observed form state") if trajectory.get("tool_calls") != _candidate(seed_tasks[task_id]).get("tool_calls"): raise ValueError(f"qualified trajectory differs from task {task_id} canonical input") + if task_id in plan.get("retrieval_task_ids", []): + verification = trajectory.get("verification", {}) + actual_protocol = verification.get( + "retrieval_protocol", + (provenance.get("retrieval_protocols") or {}).get( + str(task_id), provenance.get("retrieval_protocol") + ), + ) + if actual_protocol != plan["retrieval_contracts"][task_id][ + "schema" + ] or verification.get("retrieval_contract_sha256") != digest( + plan["retrieval_contracts"][task_id] + ): + raise ValueError("legacy qualification does not prove current-run retrieval") return _sha256(dataset_path) -def _rebind_url( - value: str, site_access_host: str, trusted_source_site_hosts: set[str] -) -> str: +def _rebind_url(value: str, site_access_host: str, trusted_source_site_hosts: set[str]) -> str: parsed = urlsplit(value) if ( parsed.scheme not in {"http", "https"} @@ -227,9 +282,7 @@ def _rebind_calls( for call in calls: item = dict(call) if isinstance(item.get("url"), str): - item["url"] = _rebind_url( - item["url"], site_access_host, trusted_source_site_hosts - ) + item["url"] = _rebind_url(item["url"], site_access_host, trusted_source_site_hosts) rebound.append(item) return rebound @@ -274,6 +327,10 @@ def _run_one( # noqa: PLR0913 sample_interval_ms: int, physical_execution: int, trusted_source_site_hosts: set[str], + retrieval_contract: dict[str, Any] | None = None, + state_contract: Any = None, + run_id: str = "", + prepare_sites: bool = True, ) -> dict[str, Any]: task_id = int(task_seed["task_id"]) candidate = _candidate(task_seed) @@ -284,12 +341,25 @@ def _run_one( # noqa: PLR0913 if task_id in correction_catalog.records else None ) + if state_contract is not None: + from harness.webarena_state import require_supersession + + if corrected_contract is not None: + require_supersession( + state_contract, + corrected_contract.correction_id, + corrected_contract.correction_source_sha256, + ) + if ( + benchmark.config.render_url(state_contract.definition["page_url"], official_task.sites) + != state_contract.page_url + ): + raise ValueError("state contract runtime site differs from frozen profile") + corrected_contract = None sites = tuple(official_task.sites) start_urls = list(benchmark.config.render_url(list(official_task.start_urls), list(sites))) site_access_host = _container_host_gateway(container) - calls = _rebind_calls( - candidate["tool_calls"], site_access_host, trusted_source_site_hosts - ) + calls = _rebind_calls(candidate["tool_calls"], site_access_host, trusted_source_site_hosts) source_site_hosts = replay_site_hosts(candidate["tool_calls"]) attempt_dir = output_dir / engine / str(task_id) / f"physical-{physical_execution}" attempt_dir.mkdir(parents=True, exist_ok=True) @@ -302,14 +372,7 @@ def _run_one( # noqa: PLR0913 sys.path.insert(0, str(cowagent_repo)) from agent.tools.browser.browser_tool import BrowserTool # noqa: PLC0415 - tool = BrowserTool( - config={ - "cdp_endpoint": endpoint, - "headless": True, - "viewport": {"width": 1280, "height": 900}, - "startup_timeout": 90, - } - ) + tool = BrowserTool(config=_browser_tool_config(endpoint)) _allow_official_container_hostname(tool) resources = DockerStatsResources(container) sampler = ResourceSampler(resources, interval_ms=sample_interval_ms) @@ -317,26 +380,36 @@ def _run_one( # noqa: PLR0913 resource_started = None recorder = CdpHarRecorder() records: list[dict[str, Any]] = [] + from .retrieval_acceptance import PROTOCOL, build_actual_response + + attempt_id = uuid.uuid4().hex + actual_response = None + retrieval_acceptance = None timings: dict[str, float] = {} process_status = "fail" evaluator_status = "not_run" replay_assertions: tuple[dict[str, Any], ...] = () + state_observation = None error = "" started = time.perf_counter() sampler_started = False + viewport_evidence: dict[str, Any] = {} try: prepare_started = time.perf_counter() - _prepare_sites(sites) + if prepare_sites: + _prepare_sites(sites) timings["environment_prepare_ms"] = (time.perf_counter() - prepare_started) * 1000 setup_started = time.perf_counter() service = tool._get_service() service._submit(lambda: None) + viewport_evidence = _prepare_browser_viewport(service) origins = { _origin(benchmark.config.get_environment(site).active_url) for site in SITE_PORTS } _clear_browser_state(service, origins) - readiness_retries = _login_context(service, sites, benchmark, start_urls) + readiness_retries = _login_context(service, sites, benchmark, start_urls, viewport_evidence) + _verify_session_viewport(service, viewport_evidence, "after_login") service._submit(lambda: recorder.start(service._context)) timings["browser_session_setup_ms"] = (time.perf_counter() - setup_started) * 1000 @@ -347,7 +420,8 @@ def _run_one( # noqa: PLR0913 replay_started = time.perf_counter() for index, arguments in enumerate(calls, start=1): call_started = time.perf_counter() - result_status, result_text = _execute_replay_call(tool, arguments) + diagnostics: dict[str, Any] = {} + result_status, result_text = _execute_replay_call(tool, arguments, diagnostics=diagnostics) dispatched_before_timeout = _click_was_dispatched_before_navigation_timeout( arguments, result_status, result_text ) @@ -359,6 +433,8 @@ def _run_one( # noqa: PLR0913 "duration_ms": round((time.perf_counter() - call_started) * 1000, 3), "result": result_text, } + if diagnostics: + record["action_diagnostics"] = diagnostics observation_error = _embedded_observation_error(arguments, result_status, result_text) if observation_error is not None: record["observation_status"] = "error" @@ -370,10 +446,27 @@ def _run_one( # noqa: PLR0913 } records.append(record) if result_status != "success" and not dispatched_before_timeout: + try: + record["failure_ui_state"] = _capture_failure_ui_state(service, arguments) + except Exception as observation_exc: + record["failure_ui_state_error"] = ( + f"{type(observation_exc).__name__}: {observation_exc}" + )[:300] raise RuntimeError(f"step {index} failed: {result_text}") timings["trajectory_replay_ms"] = (time.perf_counter() - replay_started) * 1000 + _verify_session_viewport(service, viewport_evidence, "after_trajectory") process_status = "pass" + if state_contract is not None: + from harness.webarena_state import capture_service_form + + state_observation = capture_service_form( + service, state_contract, run_id=run_id, attempt_id=attempt_id, engine=engine + ) + (attempt_dir / "final_state.json").write_text( + json.dumps(state_observation, ensure_ascii=False, indent=2), encoding="utf-8" + ) + if corrected_contract is not None and corrected_contract.replay_assertions: selectors = { selector @@ -408,21 +501,68 @@ def _run_one( # noqa: PLR0913 timings["evidence_finalize_ms"] = (time.perf_counter() - finalize_started) * 1000 evaluation_started = time.perf_counter() + is_retrieval = official_task.is_retrieve_task + if is_retrieval: + retrieval_acceptance = build_actual_response( + contract=retrieval_contract, + task_id=task_id, + calls=candidate["tool_calls"], + records=records, + run_id=run_id or output_dir.name, + attempt_id=attempt_id, + ) + (attempt_dir / "retrieval_acceptance.json").write_text( + json.dumps(retrieval_acceptance, ensure_ascii=False, indent=2), encoding="utf-8" + ) + actual_response = retrieval_acceptance["actual_response"] + else: + actual_response = candidate["recorded_response"] + if actual_response is not None: + (attempt_dir / "actual_response.json").write_text( + json.dumps(actual_response, ensure_ascii=False, indent=2), encoding="utf-8" + ) evaluation_result = ( - evaluate_corrected_contract( - benchmark, - corrected_contract, - agent_response=candidate["recorded_response"], - network_trace=har_path, + ( + evaluate_corrected_contract( + benchmark, + corrected_contract, + agent_response=actual_response, + network_trace=har_path, + ) + if corrected_contract is not None + else benchmark.evaluate_task( + task_id=task_id, + agent_response=actual_response, + network_trace=har_path, + ) ) - if corrected_contract is not None - else benchmark.evaluate_task( - task_id=task_id, - agent_response=candidate["recorded_response"], + if actual_response is not None and state_contract is None + else None + ) + evaluation = ( + evaluation_result.model_dump(mode="json") + if evaluation_result is not None + else { + "task_id": task_id, + "status": "unsupported", + "score": None, + "reason": "Current-run retrieval evidence is not sufficient for evaluation", + "evaluators_results": [], + } + ) + if state_contract is not None: + from harness.webarena_state import evaluate_task as evaluate_state_task + + evaluation = evaluate_state_task( + benchmark, + state_contract, + agent_response=actual_response, network_trace=har_path, + observation=state_observation, + run_id=run_id, + attempt_id=attempt_id, + engine=engine, ) - ) - evaluation = evaluation_result.model_dump(mode="json") if corrected_contract is not None: evaluation["replay_contract_correction"] = { "correction_id": corrected_contract.correction_id, @@ -434,7 +574,9 @@ def _run_one( # noqa: PLR0913 evaluation["replay_assertions"] = list(replay_assertions) timings["evaluation_ms"] = (time.perf_counter() - evaluation_started) * 1000 evaluator_status = ( - "pass" + "unsupported" + if evaluation.get("status") == "unsupported" + else "pass" if float(evaluation.get("score") or 0) == 1 and all(item.get("status") == "pass" for item in replay_assertions) else "fail" @@ -477,6 +619,12 @@ def _run_one( # noqa: PLR0913 trace_path.write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding="utf-8") return { "schema": "lex_webarena_trajectory_replay_attempt/1", + "run_id": run_id or output_dir.name, + "attempt_id": attempt_id, + "retrieval_protocol": ( + retrieval_contract["schema"] if retrieval_contract is not None else PROTOCOL + ), + "retrieval_acceptance": retrieval_acceptance, "engine": engine, "task_id": task_id, "physical_execution": physical_execution, @@ -488,7 +636,9 @@ def _run_one( # noqa: PLR0913 "replay_calls_binding_sha256": replay_calls_binding_sha256( calls, allowed_hosts={site_access_host} ), - "evaluation_input_sha256": _json_sha256(candidate["recorded_response"]), + "evaluation_input_sha256": _json_sha256(actual_response) + if actual_response is not None + else None, "source_model": (candidate.get("method") or {}).get("model"), "source_engine": (candidate.get("method") or {}).get("engine"), "source_kind": (candidate.get("source") or {}).get("kind", "browser_use_execution"), @@ -497,6 +647,11 @@ def _run_one( # noqa: PLR0913 "process_status": process_status, "evaluator_status": evaluator_status, "replay_assertions": list(replay_assertions), + **( + {"state_contract_sha256": state_contract.definition_sha256} + if state_contract is not None + else {} + ), "contract_correction_id": ( corrected_contract.correction_id if corrected_contract is not None else None ), @@ -506,10 +661,15 @@ def _run_one( # noqa: PLR0913 "corrected_task_sha256": ( corrected_contract.corrected_task_sha256 if corrected_contract is not None else None ), - "status": "pass" if process_status == evaluator_status == "pass" else "fail", + "status": "unsupported" + if evaluator_status == "unsupported" + else "pass" + if process_status == evaluator_status == "pass" + else "fail", "duration_ms": round(duration_ms, 3), "timings": {key: round(value, 3) for key, value in timings.items()}, "browser_session_readiness_retries": readiness_retries, + "viewport": viewport_evidence, "resource": ( _resource_summary(resources, sampler, before, after, resource_duration_ms) if before is not None and after is not None and sampler.samples @@ -519,6 +679,13 @@ def _run_one( # noqa: PLR0913 "cleanup_errors": cleanup_errors, "evidence": { "trace": _evidence_file(trace_path), + **( + {"final_state": _evidence_file(attempt_dir / "final_state.json")} + if state_contract is not None + else {} + ), + "retrieval_acceptance": _evidence_file(attempt_dir / "retrieval_acceptance.json"), + "actual_response": _evidence_file(attempt_dir / "actual_response.json"), "network_har": _evidence_file(har_path), "evaluation": _evidence_file(evaluation_path), "failure_screenshot": _evidence_file(screenshot_path), @@ -534,7 +701,7 @@ def _run_one( # noqa: PLR0913 def _attempt_needs_retry(attempt: dict[str, Any]) -> bool: """Give every failed logical replay exactly one clean physical retry.""" - return attempt.get("status") != "pass" + return attempt.get("status") not in {"pass", "unsupported"} def _embedded_observation_error( @@ -556,6 +723,105 @@ def _embedded_observation_error( return result_text[len(prefix) :].removesuffix("]").strip() +def _diagnostic_locator(selector: str) -> str: + """Expand visibility/ordinal filters without confusing CSS with locators. + + Diagnostic candidates intentionally include hidden/zero-area elements. + The original action and its locator remain unchanged. + """ + output = [] + quote = None + index = 0 + while index < len(selector): + char = selector[index] + if char == "\\" and index + 1 < len(selector): + output.append(selector[index:index + 2]) + index += 2 + continue + if quote: + if char == quote: + quote = None + elif char in ("'", '"'): + quote = char + elif selector.startswith(":visible", index) and ( + index + 8 == len(selector) or not (selector[index + 8].isalnum() or selector[index + 8] in "_-\\") + ): + index += 8 + continue + output.append(char) + index += 1 + diagnostic = "".join(output) + # :nth-match(...:visible, 2) can fail because an earlier sibling is + # zero-area. Keeping the ordinal would hide that missing first match. + if diagnostic.startswith(":nth-match(") and diagnostic.endswith(")"): + inner = diagnostic[len(":nth-match("):-1] + candidate, separator, ordinal = inner.rpartition(",") + if separator and ordinal.strip().isdigit(): + diagnostic = candidate.strip() + return diagnostic + + +def _capture_failure_ui_state(service: Any, arguments: dict[str, Any]) -> dict[str, Any]: + """Retain bounded DOM geometry after a failed action without form values. + + A screenshot cannot distinguish a visible text fallback from a hidden + select, and a tool timeout does not preserve the locator's final state. + This observation is diagnostic only; it never changes the tool/evaluator + verdict or retries the action. + """ + + selector = arguments.get("selector") + if not isinstance(selector, str) or len(selector) > 1000: + selector = None + def observe() -> dict[str, Any]: + result = service._page.evaluate( + """selector => { + const state = element => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return {tag: element.tagName.toLowerCase(), id: element.id || null, + name: element.getAttribute('name'), hidden: element.hidden, + disabled: !!element.disabled, display: style.display, + visibility: style.visibility, + rect: [rect.x, rect.y, rect.width, rect.height]}; + }; + const result = {page_origin_path: location.origin + location.pathname, + ready_state: document.readyState, selector, native_selector: null, + selector_count: null, + selector_error: null, matched_controls: [], region_controls: []}; + result.region_controls = [...document.querySelectorAll( + 'select[name*="region"],input[name*="region"],' + + 'select[id*="region"],input[id*="region"]')] + .slice(0, 16).map(state); + return result; + }""", + selector, + ) + if selector: + diagnostic = _diagnostic_locator(selector) + result["diagnostic_selector"] = diagnostic + result["selector_engine"] = "playwright" + result["candidate_scope"] = "visibility_and_outer_ordinal_filters_removed" + try: + observed = service._page.locator(diagnostic).evaluate_all( + """elements => ({selector_count: elements.length, + matched_controls: elements.slice(0,8).map(element => { + const r=element.getBoundingClientRect(),s=getComputedStyle(element); + return {tag:element.tagName.toLowerCase(),id:element.id||null, + name:element.getAttribute('name'),hidden:element.hidden, + disabled:!!element.disabled,display:s.display,visibility:s.visibility, + rect:[r.x,r.y,r.width,r.height],client_rect_count:element.getClientRects().length, + has_offset_parent:element.offsetParent!==null}; + })})""" + ) + result.update(observed) + except Exception as exc: + result["selector_error"] = type(exc).__name__ + return result + + return service._submit(observe) + + def _stop_all_sites() -> None: """Stop the fixed-port site stack before another method can acquire it.""" @@ -692,6 +958,18 @@ def _finish_physical_execution(entry: dict[str, Any], attempt: dict[str, Any]) - ) +def _reject_partial_method_resume( + keys: list[tuple[str, int]], engines: list[str], task_ids: list[int] +) -> None: + for engine in engines: + completed_for_method = sum(key[0] == engine for key in keys) + if 0 < completed_for_method < len(task_ids): + raise ValueError( + "a method-scoped site cohort cannot resume after a partial method; " + "restart that method from a fresh site preparation in a new output directory" + ) + + def _validated_resume( receipt_path: Path, *, @@ -740,13 +1018,53 @@ def required_receipt_int(item: object, field: str, record: str) -> int: try: return int(value) except (TypeError, ValueError) as exc: - raise ValueError( - f"existing replay receipt {record} has invalid {field}" - ) from exc + raise ValueError(f"existing replay receipt {record} has invalid {field}") from exc + + from harness.webarena_state import validate_attempt_identities + validate_attempt_identities(attempts, set(plan.get("state_contracts", {}))) keys = [] for item in attempts: task_id = required_receipt_int(item, "task_id", "attempt") + if task_id in plan.get("state_contracts", {}): + from harness.webarena_state import verify_attempt as verify_state_attempt + + if item.get("status") == "pass" and not ( + item.get("process_status") == item.get("evaluator_status") == "pass" + ): + raise ValueError("claimed state pass contradicts process or evaluator status") + if item.get("run_id") != receipt.get("run_id"): + raise ValueError("cross-run form-state attempt cannot resume") + if item.get("process_status") == "pass" and item.get("status") != "unsupported": + verify_state_attempt(item, plan["state_contracts"][task_id]) + if task_id in plan.get("retrieval_task_ids", []): + from .retrieval_acceptance import verify_attempt + + if item.get("status") == "pass" and not ( + item.get("process_status") == item.get("evaluator_status") == "pass" + ): + raise ValueError("claimed retrieval pass contradicts process or evaluator status") + contract = plan.get("retrieval_contracts", {}).get(task_id) + if contract is None: + raise ValueError("resumed retrieval requires a frozen observation contract") + if item.get("retrieval_protocol") != contract["schema"] or item.get( + "run_id" + ) != receipt.get("run_id"): + raise ValueError("legacy or cross-run retrieval attempt cannot resume a new cohort") + if item.get("process_status") == "pass" and item.get("status") != "unsupported": + source = next( + task + for task in plan["seed_manifest"]["tasks"] + if int(task["task_id"]) == task_id + ) + verify_attempt( + item, + contract, + _candidate(source)["tool_calls"], + expected_data_checksum=str( + plan["seed_manifest"].get("source", {}).get("official_data_checksum") or "" + ), + ) if item.get("schema") != "lex_webarena_trajectory_replay_attempt/1": raise ValueError("existing replay receipt contains an invalid attempt schema") key = (str(item.get("engine")), task_id) @@ -757,6 +1075,15 @@ def required_receipt_int(item: object, field: str, record: str) -> int: keys.append(key) if len(keys) != len(set(keys)): raise ValueError("existing replay receipt contains duplicate method/task attempts") + if plan["profile"]["execution"]["reset_before_every_task"] is False: + _reject_partial_method_resume(keys, engines, task_ids) + site_preparation = receipt.get("method_site_preparation") or {} + for engine in engines: + if any(key[0] == engine for key in keys) and not ( + isinstance(site_preparation.get(engine), dict) + and site_preparation[engine].get("status") == "complete" + ): + raise ValueError("completed method lacks a site preparation receipt") method_teardown = receipt.get("method_teardown") or {} if set(method_teardown) != set(engines) or any( not isinstance(value, bool) for value in method_teardown.values() @@ -789,18 +1116,14 @@ def required_receipt_int(item: object, field: str, record: str) -> int: "existing replay receipt contains a physical execution outside the plan" ) physical_by_logical.setdefault(key, []).append(item) - attempts_by_key = { - (str(item["engine"]), int(item["task_id"])): item for item in attempts - } + attempts_by_key = {(str(item["engine"]), int(item["task_id"])): item for item in attempts} if set(physical_by_logical) != set(attempts_by_key): raise ValueError("existing replay receipt has an unbound physical execution") for key, attempt in attempts_by_key.items(): physical = physical_by_logical[key] - if ( - len(physical) != int(attempt.get("physical_execution_count") or 1) - or max(int(item["physical_execution"]) for item in physical) - != int(attempt["physical_execution"]) - ): + if len(physical) != int(attempt.get("physical_execution_count") or 1) or max( + int(item["physical_execution"]) for item in physical + ) != int(attempt["physical_execution"]): raise ValueError("existing replay receipt has inconsistent physical execution history") sessions = list(receipt.get("sessions") or []) session_ids = [str(session.get("id") or "") for session in sessions] @@ -883,12 +1206,14 @@ def run_replay_suite( # noqa: PLR0913 else: scope = "formal" if diagnostic_task_ids is None else "diagnostic" qualification_dataset_sha256 = None + if scope != "diagnostic": + from .retrieval_acceptance import require_current_retrieval + + require_current_retrieval(plan) if scope in {"formal", "formal_correction_shard"}: if qualified_dataset_path is None: raise ValueError("formal replay requires a complete qualified trajectory dataset") - qualification_dataset_sha256 = validate_qualification_dataset( - plan, qualified_dataset_path - ) + qualification_dataset_sha256 = validate_qualification_dataset(plan, qualified_dataset_path) elif qualified_dataset_path is not None: raise ValueError("a qualified trajectory dataset is only accepted for formal replay") resources = plan["profile"].get("resources") or {} @@ -905,7 +1230,20 @@ def run_replay_suite( # noqa: PLR0913 raise ValueError("cowagent_repo does not contain BrowserTool") repo_root = Path(__file__).resolve().parents[2] - browser_eval_commit = _require_clean_git_checkout(repo_root) + # Formal evidence must be reproducible from a clean checkout. A diagnostic + # is explicitly non-promotable and may be run while investigating an + # uncommitted candidate fix; retain the exact base commit instead of + # rejecting that workflow as if it were formal evidence. + browser_eval_commit = ( + _require_clean_git_checkout(repo_root) + if scope != "diagnostic" + else subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) cowagent_commit = _require_clean_git_checkout(cowagent) client = docker.from_env() docker_info = client.info() @@ -923,6 +1261,7 @@ def run_replay_suite( # noqa: PLR0913 runtime = { "browser_eval_commit": browser_eval_commit, "cowagent_commit": cowagent_commit, + "browser_session_contract": _browser_session_contract(), "resources": { "cpus": cpus, "memory": memory, @@ -941,9 +1280,11 @@ def run_replay_suite( # noqa: PLR0913 qualification_dataset_sha256=qualification_dataset_sha256, ) completed = {(str(item["engine"]), int(item["task_id"])) for item in attempts} - lock_path = _physical_execution_lock_path( - _docker_daemon_lock_namespace(docker_info) - ) + method_site_preparation = {} + if receipt_path.is_file(): + previous = json.loads(receipt_path.read_text(encoding="utf-8")) + method_site_preparation = dict(previous.get("method_site_preparation") or {}) + lock_path = _physical_execution_lock_path(_docker_daemon_lock_namespace(docker_info)) receipt = { "schema": RECEIPT_SCHEMA, "run_id": run_id, @@ -958,6 +1299,7 @@ def run_replay_suite( # noqa: PLR0913 "engines": engines, "runtime": runtime, "method_teardown": method_teardown, + "method_site_preparation": method_site_preparation, "sessions": sessions, "physical_executions": physical_executions, "isolation": { @@ -977,6 +1319,9 @@ def run_replay_suite( # noqa: PLR0913 _write_receipt(receipt_path, receipt) for method in select_methods(plan, engines): engine = str(method["engine"]) + execution = plan["profile"]["execution"] + reset_before_every_task = execution["reset_before_every_task"] + retry_failed_task = execution.get("retry_failed_task", True) pending = [task for task in tasks if (engine, int(task["task_id"])) not in completed] if not pending: continue @@ -984,7 +1329,21 @@ def run_replay_suite( # noqa: PLR0913 container = None primary_error: BaseException | None = None try: + site_preparation_started = time.perf_counter() + site_preparation_started_at = round(time.time() * 1000) _stop_all_sites() + if not reset_before_every_task: + _prepare_sites(tuple(SITE_PORTS)) + method_site_preparation[engine] = { + "status": "complete", + "started_at_unix_ms": site_preparation_started_at, + "completed_at_unix_ms": round(time.time() * 1000), + "duration_ms": round( + (time.perf_counter() - site_preparation_started) * 1000, 3 + ), + "sites": [site.value for site in SITE_PORTS], + } + receipt["method_site_preparation"] = method_site_preparation method_teardown[engine] = False receipt["method_teardown"] = method_teardown _write_receipt(receipt_path, receipt) @@ -1037,11 +1396,17 @@ def run_replay_suite( # noqa: PLR0913 sample_interval_ms=sample_interval_ms, physical_execution=physical_execution, trusted_source_site_hosts=plan["trusted_source_site_hosts"], + state_contract=plan.get("state_contracts", {}).get(int(task["task_id"])), + retrieval_contract=plan.get("retrieval_contracts", {}).get( + int(task["task_id"]) + ), + run_id=run_id, + prepare_sites=reset_before_every_task, ) attempt["browser_session_id"] = session["id"] _finish_physical_execution(physical_entry, attempt) _write_receipt(receipt_path, receipt) - if _attempt_needs_retry(attempt): + if retry_failed_task and _attempt_needs_retry(attempt): # A declared replay gets one clean retry regardless of where the first # physical execution failed. Restarting the browser and rerunning # _run_one also repeats site preparation, so failed page/session state @@ -1082,13 +1447,19 @@ def run_replay_suite( # noqa: PLR0913 sample_interval_ms=sample_interval_ms, physical_execution=physical_execution + 1, trusted_source_site_hosts=plan["trusted_source_site_hosts"], + state_contract=plan.get("state_contracts", {}).get( + int(task["task_id"]) + ), + retrieval_contract=plan.get("retrieval_contracts", {}).get( + int(task["task_id"]) + ), + run_id=run_id, ) attempt["browser_session_id"] = session["id"] attempt["prior_attempts"] = [prior_attempt] attempt["physical_execution_count"] = 2 attempt["duration_ms"] = round( - float(prior_attempt["duration_ms"]) - + float(attempt["duration_ms"]), + float(prior_attempt["duration_ms"]) + float(attempt["duration_ms"]), 3, ) _finish_physical_execution(retry_entry, attempt) diff --git a/harness/trajectory_replay/webarena_report.py b/harness/trajectory_replay/webarena_report.py index dfa691f4d..f1288a3e2 100644 --- a/harness/trajectory_replay/webarena_report.py +++ b/harness/trajectory_replay/webarena_report.py @@ -12,13 +12,32 @@ from collections import defaultdict from pathlib import Path from typing import Any +from urllib.parse import urlencode, urlsplit +from harness.trajectory_replay.failure_matrix import failure_category_matrix +from harness.trajectory_replay.interpretation import render_interpretation, standalone_document +from harness.trajectory_replay.participants import ( + bind_participant, + participant_registry, + resolve_participant_id, +) from harness.trajectory_replay.webarena_review import ( load_validated_failure_review, programmatic_failure, ) -LOCAL_PATH_RE = re.compile(r"/(?:Users|home|private|tmp|var|opt)/[^\s\"']+") +LOCAL_PATH_RE = re.compile(r"/(?:Users|home|root|private|tmp|var|opt)/[^\s\"']+") + +# This notice describes one retained historical cohort, not trajectory replay in +# general. Keying all three immutable inputs prevents a future requalified run +# from inheriting legacy limitations merely because it also contains 90 tasks. +LEGACY_REPLAY_VALIDITY_INPUTS = { + ( + "abaaa0e90aced74a831093f2d6d7c3371639acb984b3a2c057f3cc39632ea935", + "61bb38bce552b5c7eea0e1f5fcca1bc898f55e2e39ee6e72498cff346ca1e4e3", + "3ca5d91cfec570490df202c9cb332957f7e850b70fc5af80b684709e97ef0893", + ) +} def _e(value: Any) -> str: @@ -29,6 +48,66 @@ def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def _json_sha256(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _legacy_retrieval_provenance( + results: dict[str, Any], dataset: dict[str, Any] +) -> dict[str, Any]: + """Prove whether retained retrieval verdicts used saved Agent answers.""" + source = Path(results["source_seed_manifest"]) + if (not source.is_absolute() + or any(part.is_symlink() for part in (source, *source.parents)) + or not source.is_file()): + raise ValueError("legacy replay seed manifest must be an absolute file without symlinks") + if _sha256(source) != results.get("source_seed_manifest_sha256"): + raise ValueError("legacy replay seed manifest hash mismatch") + seeds = json.loads(source.read_text(encoding="utf-8")) + by_id = {int(row["task_id"]): row for row in seeds["tasks"]} + retrieval_ids = { + int(row["task_id"]) + for row in dataset["trajectories"] + if row["evaluation_input"]["task_type"] == "RETRIEVE" + } + rows = [] + for attempt in results["attempts"]: + task_id = int(attempt["task_id"]) + if task_id not in retrieval_ids: + continue + seed = by_id[task_id] + candidates = [ + item for item in seed["candidates"] + if item["id"] == seed["canonical_candidate_id"] + ] + if len(candidates) != 1: + raise ValueError(f"task {task_id}: canonical replay seed is ambiguous") + saved_hash = _json_sha256(candidates[0]["recorded_response"]) + actual_hash = attempt.get("evaluation_input_sha256") + if actual_hash != saved_hash: + raise ValueError(f"task {task_id}: retrieval input differs from frozen Agent answer") + rows.append({ + "task_id": task_id, "browser": attempt["engine"], + "evaluation_input_sha256": actual_hash, + "source": "frozen_agent_response_not_current_page_observation", + }) + browsers = {str(browser) for browser in results["engines"]} + expected = {(browser, task_id) for browser in browsers for task_id in retrieval_ids} + actual = {(row["browser"], row["task_id"]) for row in rows} + if actual != expected or len(rows) != len(expected): + raise ValueError("legacy retrieval provenance lacks the full browser-task matrix") + return { + "schema": "trajectory_replay_legacy_retrieval_provenance/1", + "source_seed_manifest_sha256": results["source_seed_manifest_sha256"], + "retrieval_tasks": len(retrieval_ids), + "retrieval_results": len(rows), + "frozen_answer_matches": len(rows), + "meaning": "A matching verdict verifies the saved answer against the official evaluator, not an answer reconstructed from this browser run.", + "results": sorted(rows, key=lambda item: (item["task_id"], item["browser"])), + } + + def _stat(values: list[float]) -> dict[str, float | int | None]: if not values: return {"n": 0, "mean": None, "p50": None, "p95": None} @@ -44,6 +123,24 @@ def rank(quantile: float) -> float: } +def _outcome_metrics(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Keep the declared population; missing samples never become zero.""" + fields = {"duration": "duration_ms", "cpu": "cpu_ms", "memory": "memory_mean_bytes", + "memory_peak": "memory_peak_bytes", "network_rx": "network_rx_bytes", + "network_tx": "network_tx_bytes", "cpu_utilization": "cpu_average_utilization_pct"} + result = {} + for name, field in fields.items(): + values = [] + for row in rows: + source = row if name == "duration" else (row.get("resource") or {}) + value = source.get(field) + if (source.get("available") is not False and type(value) in (int, float) + and math.isfinite(value) and value >= 0): + values.append(float(value)) + result[name] = {**_stat(values), "population": len(rows), "missing": len(rows) - len(values)} + return result + + def _clean(value: Any) -> Any: if isinstance(value, dict): return { @@ -67,15 +164,283 @@ def _write_json(path: Path, value: Any) -> None: ) +def _planned_actions_digest(actions: list[dict[str, Any]]) -> str: + payload = json.dumps(actions, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _validated_planned_actions(attempt: dict[str, Any]) -> list[dict[str, Any]]: + actions = attempt.get("planned_actions") + digest = attempt.get("planned_actions_sha256") + if ( + not isinstance(actions, list) + or not actions + or any(not isinstance(action, dict) or not action.get("action") for action in actions) + or _clean(actions) != actions + or len(actions) != attempt.get("action_count") + or not isinstance(digest, str) + or digest != _planned_actions_digest(actions) + ): + raise ValueError( + f"method-specific planned actions missing or changed for " + f"{attempt.get('participant_id')}/{attempt.get('task_id')}" + ) + return actions + + +def _reference_validity_by_task(attempts: list[dict[str, Any]]) -> dict[int, str]: + task_validity: dict[int, str] = {} + for attempt in attempts: + validity = attempt.get("reference_validity") or {} + status = validity.get("status") + if status not in {"valid", "unverifiable"}: + raise ValueError(f"reference validity is missing for task {attempt['task_id']}") + task_id = int(attempt["task_id"]) + if task_id in task_validity and task_validity[task_id] != status: + raise ValueError(f"methods disagree on task {task_id} reference validity") + task_validity[task_id] = status + return task_validity + + +def require_all_reference_validity( + profile: dict[str, Any], results: dict[str, Any], dataset: dict[str, Any] +) -> None: + """Reject a formal three-method release with any unverified task reference.""" + + if (profile.get("scoring") or {}).get("require_all_reference_validity") is not True: + return + method_rows = profile.get("methods") or [] + task_rows = dataset.get("trajectories") or [] + task_ids = [row.get("task_id") for row in task_rows if isinstance(row, dict)] + declared_count = (profile.get("task_selection") or {}).get("task_count") + if ( + len(method_rows) < 2 + or not task_ids + or len(task_ids) != declared_count + or len(set(task_ids)) != len(task_ids) + ): + raise ValueError("formal reference-validity gate requires registered methods and the declared task set") + methods = participant_registry(method_rows) + attempts = [bind_participant(row, methods) for row in results.get("attempts") or []] + expected = {(method_id, task_id) for method_id in methods for task_id in task_ids} + actual = {(row["participant_id"], row.get("task_id")) for row in attempts} + if len(attempts) != len(methods) * len(task_ids) or actual != expected: + raise ValueError("formal reference-validity gate requires every method-task result") + validity = _reference_validity_by_task(attempts) + unavailable = sorted(task_id for task_id, status in validity.items() if status != "valid") + if unavailable: + raise ValueError(f"formal reference-validity gate rejects unverifiable tasks: {unavailable}") + + +def _measured_validity_identity( + results: dict[str, Any], results_source: Path, dataset_source: Path +) -> dict[str, str]: + # The current publication profile is deliberately not an input: changing + # host/version cannot change the limitations of an already measured run. + return { + "results_sha256": _sha256(results_source), + "dataset_sha256": _sha256(dataset_source), + "profile_sha256": results.get("profile_sha256", ""), + } + + +def _historical_replay_validity_notice( + *, results_sha256: str, dataset_sha256: str, profile_sha256: str, lang: str +) -> str: + """Disclose limitations only for the hash-bound legacy 90-task cohort.""" + + identity = (results_sha256, dataset_sha256, profile_sha256) + if identity not in LEGACY_REPLAY_VALIDITY_INPUTS: + return "" + if lang == "zh": + heading = "历史 90 题成绩保留，但不是当前资格证明" + body = ( + "Chromium 90/90、Moli 54/90 是这批历史固定轨迹的原始验收结果，不做人工改分。" + "但旧资格流程曾把返回内容为 [Snapshot error: …] 的 snapshot 动作按工具包装层 success 接受；" + "37 道 RETRIEVE 题的 74 条结果均直接使用冻结的 recorded_response 验收，没有绑定本轮页面读数；" + "历史运行还存在 Chromium 780×437/493、Moli 1920×1080 的视口差异。" + "因此这些分数只能描述旧协议下的固定路径验收，不能证明 90 条轨迹符合当前 fail-closed 资格规则，" + "也不能作为统一视口下的浏览器成功率。修正后的正式比较需要重新资格验证并完整重跑。" + ) + else: + heading = "The historical 90-task scores remain, but they are not current qualification proof" + body = ( + "Chromium 90/90 and Moli 54/90 remain the original acceptance results for this historical fixed-trajectory cohort; no score is manually changed. " + "However, the legacy qualification flow accepted snapshot actions as tool-wrapper successes even when their returned content began [Snapshot error: …]. " + "All 74 results for the 37 RETRIEVE tasks used frozen recorded_response values for acceptance without binding answers to data read from the current page, and the historical runs used mismatched viewports: Chromium 780×437/493 and Moli 1920×1080. " + "These scores therefore describe fixed-path acceptance under the legacy protocol. They do not prove that all 90 trajectories satisfy the current fail-closed qualification rules or measure browser success at a common viewport. A corrected formal comparison requires renewed qualification and a complete rerun." + ) + return f'

{_e(heading)}

{_e(body)}

' + + def _load_artifact(ref: dict[str, Any] | None) -> Any: if not ref or not ref.get("path"): return None - path = Path(str(ref["path"])).expanduser().resolve() - if not path.is_file() or _sha256(path) != ref.get("sha256"): + path = Path(str(ref["path"])) + if ( + not path.is_absolute() + or any(part.is_symlink() for part in (path, *path.parents)) + or not path.is_file() + or _sha256(path) != ref.get("sha256") + ): raise ValueError(f"evidence artifact is missing or changed: {path.name}") return json.loads(path.read_text(encoding="utf-8")) +def _supplemental_diagnostic(review: dict[str, Any] | None, task_id: int) -> Any: + ref = (review or {}).get("supplemental_diagnostic") + if not ref: + return None + diagnostic = _load_artifact(ref) + if (diagnostic.get("schema") != "lex_replay_supplemental_diagnostic/1" + or diagnostic.get("task_id") != task_id + or diagnostic.get("scope") not in { + "read_only_prefix_on_current_site_not_a_replacement_score", + "isolated_local_http_fixture_not_a_replacement_score", + } + or diagnostic.get("mutations_submitted") is not False): + raise ValueError("supplemental diagnostic scope or task differs") + return diagnostic + + +def _supplemental_diagnostic_html(diagnostic: dict | None, lang: str) -> str: + if not diagnostic: + return "" + zh = lang == "zh" + rows = [] + component_rows = [] + readiness_rows = [] + network_rows = [] + header_rows = [] + for method in diagnostic["observations"]: + label = {"release": "Moli 1.1.8", "pr": "Moli · PR 640"}.get(method["method"], method["method"]) + for sample in method["observations"]: + for request in sample["state"].get("headerRecording", []): + cookie_label = ("有" if request["cookie_present"] else "无") if zh else ("Yes" if request["cookie_present"] else "No") + missing = "缺失" if zh else "Missing" + values = (label, request["api"], cookie_label, request["status"], + request["wire_referer_path"] or missing, request["har_referer_path"] or missing) + header_rows.append("" + "".join(f"{_e(value)}" for value in values) + "") + for request in sample["state"].get("httpReads", []): + values = (label, request["path"], request.get("status", "—"), + request.get("bytes", "—"), request.get("error", "")) + network_rows.append("" + "".join(f"{_e(value)}" for value in values) + "") + readiness = sample["state"].get("readiness") + if readiness: + values = (label, f'{sample["delay"]} s', readiness["target"], + readiness["disabled"], readiness["pendingModules"], + readiness["formValidationReady"]) + readiness_rows.append("" + "".join(f"{_e(value)}" for value in values) + "") + for component in sample["state"].get("components", []): + values = (label, f'{sample["delay"]} s', component["initialized"], + ", ".join(component["labels"]), component["wizardMatches"], + component["stepMatches"], component["selectedMatches"], component["handlerCount"]) + component_rows.append("" + "".join(f"{_e(value)}" for value in values) + "") + for control in sample["state"].get("buttons", []) + sample["state"].get("dialogs", []): + rect = " × ".join(str(value) for value in control["rect"]) + offset_parent = control.get("offsetParent") + values = (label, f'{sample["delay"]} s', control["text"], + f'{control.get("display") or "—"} / {control.get("visibility") or "—"}', rect, + control["rectCount"] if control.get("rectCount") is not None else "—", + "—" if offset_parent is None else "non-null" if offset_parent else "null") + rows.append("" + "".join(f"{_e(value)}" for value in values) + "") + headers = ("方法", "追加等待", "目标文本", "display / visibility", "宽 × 高（px）", "矩形数", "offsetParent") if zh else ( + "Method", "Additional wait", "Target text", "display / visibility", "Width × height (px)", "Client rects", "offsetParent") + title = "补充实测：目标状态对照" if zh else "Supplemental probe: target-state comparison" + if header_rows: + title = "补充实测：请求发送与记录对照" if zh else "Supplemental probe: sent and recorded headers" + scope = ("仅打开界面或搜索，没有提交表单或发送邀请。这里记录的是后续只读诊断；原始失败轨迹与成绩保持不变。" if zh else + "The probe only opened the interface or searched; no form or invitation was submitted. These later read-only observations do not replace the original trace or score.") + if diagnostic.get("scope") == "isolated_local_http_fixture_not_a_replacement_score": + scope = ("独立本地 HTTP 实验，没有访问评测站点；使用冻结浏览器和正式 HAR 采集器。此处验证机制，不替换原始轨迹或成绩。" if zh else + "An isolated local HTTP fixture, without accessing benchmark sites, uses the frozen browsers and production HAR recorder. It tests a mechanism and does not replace original trajectories or scores.") + scope_note = (diagnostic.get("scope_note") or {}).get(lang) + if scope_note: + scope += " " + str(scope_note) + if header_rows: + scope += (" 横向滚动可对照服务器收到的头与 HAR 记录。" if zh else + " Scroll horizontally to compare server-observed headers with HAR records.") + components = "" + if component_rows: + component_headers = ("方法", "追加等待", "组件已初始化", "属性选项", "向导匹配", "步骤匹配", "选择匹配", "事件处理器数") if zh else ( + "Method", "Additional wait", "Component initialized", "Attribute options", "Wizard matches", "Step matches", "Selection matches", "Event handlers") + components = ('
' + + "".join(f"" for header in component_headers) + + '' + "".join(component_rows) + '
{_e(header)}
') + readiness_table = "" + if readiness_rows: + readiness_headers = ("方法", "DOM 就绪后等待", "目标", "仍禁用", "待初始化模块", "表单校验已绑定") if zh else ( + "Method", "Wait after DOM ready", "Target", "Disabled", "Pending modules", "Form validation bound") + readiness_table = ('
' + + "".join(f"" for header in readiness_headers) + + '' + "".join(readiness_rows) + '
{_e(header)}
') + geometry_table = (('
' + + "".join(f"" for header in headers) + + '' + "".join(rows) + '
{_e(header)}
') if rows else "") + network_table = "" + if network_rows: + network_headers = ("方法", "帖子路径", "HTTP 状态", "正文 UTF-8 字节数", "错误") if zh else ( + "Method", "Post path", "HTTP status", "Body UTF-8 bytes", "Error") + network_table = ('
' + + "".join(f"" for header in network_headers) + + '' + "".join(network_rows) + '
{_e(header)}
') + header_table = "" + if header_rows: + header_headers = ("方法", "请求接口", "携带 Cookie", "HTTP 状态", "服务器收到的 Referer 路径", "HAR 中的 Referer 路径") if zh else ( + "Method", "Request API", "Cookie present", "HTTP status", "Referer path received by server", "Referer path in HAR") + header_table = ('
' + + "".join(f"" for header in header_headers) + + '' + "".join(header_rows) + '
{_e(header)}
') + return (f'
{title}

{_e(scope)}

' + + components + readiness_table + geometry_table + network_table + header_table + '
') + + +def _corrected_acceptance_by_task( + attempts: list[dict[str, Any]], *, declared_revisions: dict | None = None, +) -> dict[int, dict[str, Any]]: + """Expose corrected evaluator expectations without replacing the frozen dataset.""" + corrected: dict[int, dict[str, Any]] = {} + declarations = {int(key): value for key, value in (declared_revisions or {}).items()} + versions: dict[int, list[dict]] = {} + for attempt in attempts: + evaluation = _load_artifact((attempt.get("evidence") or {}).get("evaluation")) + if not evaluation or not evaluation.get("contract_correction_id"): + continue + task_id = int(attempt["task_id"]) + if evaluation.get("task_id") != task_id: + raise ValueError("corrected evaluation belongs to another task") + row = { + "correction_id": evaluation["contract_correction_id"], + "corrected_task_sha256": evaluation.get("corrected_task_sha256"), + "evaluators": [ + {"evaluator": item["evaluator_name"], "expected": item["expected"]} + for item in evaluation.get("evaluators_results") or [] + if item.get("expected") is not None + ], + } + if not row["corrected_task_sha256"] or not row["evaluators"]: + raise ValueError(f"task {task_id}: corrected evaluation lacks acceptance evidence") + if task_id in declarations: + method = attempt["participant_id"] + identity = {key: row[key] for key in ("correction_id", "corrected_task_sha256")} + if declarations[task_id].get(method) != identity: + raise ValueError(f"task {task_id}: declared acceptance revision differs for {method}") + groups = versions.setdefault(task_id, []) + group = next((item for item in groups if {key: item[key] for key in row} == row), None) + if group is None: + groups.append({**row, "methods": [method]}) + else: + group["methods"].append(method) + continue + if task_id in corrected and corrected[task_id] != row: + raise ValueError(f"task {task_id}: corrected acceptance differs across methods") + corrected[task_id] = row + if set(declarations) != set(versions): + raise ValueError("declared acceptance revisions lack evaluator evidence") + corrected.update({task: {"versions": rows} for task, rows in versions.items()}) + return corrected + + def _format_ms(value: float | None) -> str: return "—" if value is None else f"{value / 1000:.1f} 秒" @@ -95,24 +460,45 @@ def _metric_lines( labels = (("mean", "均值"), ("p50", "P50"), ("p95", "P95")) if lang == "en": labels = (("mean", "Mean"), ("p50", "P50"), ("p95", "P95")) - return "
".join( + text = "
".join( f'{label} ' + _winner(formatter(metric.get(key)), method_id in winners[key], lang=lang) for key, label in labels ) + if "n" in metric: + coverage = f'n={metric["n"]}' + if "population" in metric: + coverage += f'/{metric["population"]}; ' + ("缺失 " if lang == "zh" else "missing ") + str(metric["missing"]) + text += f"
{coverage}" + return text def _unique_winners( rows: list[dict[str, Any]], key: str, *, higher: bool ) -> set[str]: - numeric = [(row["id"], row.get(key)) for row in rows if isinstance(row.get(key), (int, float))] - if not numeric: + numeric = [(row["id"], row.get(key)) for row in rows + if type(row.get(key)) in (int, float) and math.isfinite(row[key])] + if len(rows) < 2 or len(numeric) != len(rows): return set() best = (max if higher else min)(value for _, value in numeric) winners = {name for name, value in numeric if value == best} return winners if len(winners) == 1 else set() +def _add_interpretation_navigation(language_root: Path, lang: str) -> None: + """Expose the optional article only after its page has been generated.""" + label = "报告解读" if lang == "zh" else "Interpretation" + for page in language_root.rglob("*.html"): + route = page.relative_to(language_root) + if route.parts[0] == "en" or page.name == "article.html": + continue + prefix = "../" * (len(route.parts) - 1) + current = ' aria-current="page"' if route.as_posix() == "interpretation.html" else "" + link = f'{label}' + content = page.read_text(encoding="utf-8") + page.write_text(content.replace("