From dcb999b596c6dd22545f40d155da7ec6cb27057b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:46:29 +0800 Subject: [PATCH 1/3] Own Web Fetch task definitions and standalone evaluation --- .github/workflows/ci.yml | 13 +- AGENTS.md | 4 +- CITATION.cff | 2 +- CONTRIBUTING.md | 4 +- MANIFEST.in | 2 + Makefile | 2 +- README.md | 26 +- docs/README.zh-CN.md | 26 +- docs/data.md | 6 +- docs/protocol.md | 2 +- examples/answer.txt | 1 + examples/missing.txt | 1 + examples/task.json | 1 + provenance.json | 51 +- pyproject.toml | 10 +- src/lexbench_webfetch/benchmark.py | 62 +- src/lexbench_webfetch/body_coverage.py | 501 + src/lexbench_webfetch/cli.py | 31 +- src/lexbench_webfetch/composite_tasks.py | 1079 + .../evaluation}/dataset-catalog.json | 4 +- .../dataset-catalog.json | 14 +- .../source-inputs.json | 17 + .../tasks.json | 51 +- .../native-retrieval-contracts.json | 73 + .../popular-sites/dataset-catalog.json | 50 +- .../popular-sites/source-inputs.json | 58 +- .../evaluation}/popular-sites/tasks.json | 1890 +- .../risk-control/body-contracts.json | 2248 ++ .../risk-control/current/dataset-catalog.json | 1011 + .../risk-control/current/source-inputs.json | 13 + .../risk-control/current}/tasks.json | 244 +- .../risk-control}/dataset-catalog.json | 20 +- .../risk-control/first-30-rule-review.md | 372 + .../history/v5/dataset-catalog.json | 1011 + .../risk-control/list-contracts.json | 780 + .../qualification/cohort-policy.json | 33 + .../qualification/cohort/cohort-manifest.json | 1331 + .../qualification/cohort/cohort-receipt.json | 86 + .../qualification/cohort/input-manifest.json | 34 + .../risk-control/remaining-70-rule-review.md | 692 + .../risk-control/rule-authoring-guide.md | 227 + .../risk-control/source-inputs.json | 13 + .../evaluation/scoring-regressions.json | 24240 ++++++++++++++++ .../evaluation}/source-inputs.json | 11 +- .../evaluation}/tasks.json | 119 +- .../definitions/registry.json | 49 + .../dynamic_entry_matching.py | 166 + src/lexbench_webfetch/dynamic_lists.py | 869 + src/lexbench_webfetch/dynamic_reference.py | 58 + src/lexbench_webfetch/evaluation.py | 19 + src/lexbench_webfetch/file_acceptance.py | 21 + src/lexbench_webfetch/inputs.py | 130 + src/lexbench_webfetch/list_verification.py | 66 + src/lexbench_webfetch/metrics.py | 915 + src/lexbench_webfetch/models.py | 72 + src/lexbench_webfetch/ordered_tree.py | 98 + src/lexbench_webfetch/paths.py | 41 + src/lexbench_webfetch/reference_quality.py | 350 + src/lexbench_webfetch/response_input.py | 55 + src/lexbench_webfetch/risk_control_dataset.py | 507 + src/lexbench_webfetch/scoring.py | 304 + src/lexbench_webfetch/scoring_regressions.py | 162 + src/lexbench_webfetch/stable_tasks.py | 1100 + src/lexbench_webfetch/structured_tables.py | 495 + .../access-restrictions/source-inputs.json | 13 - .../suites/dynamic-content/source-inputs.json | 13 - src/lexbench_webfetch/suites/registry.json | 45 - tests/test_annotation_evidence.py | 31 + tests/test_benchmark.py | 31 +- tests/test_body_text_acceptance_roles.py | 141 + tests/test_composite_tasks.py | 933 + tests/test_currency_matching.py | 22 + tests/test_dynamic_lists.py | 259 + tests/test_link_layout_normalization.py | 30 + tests/test_list_verification.py | 34 + tests/test_metrics.py | 408 + tests/test_ordered_tree.py | 54 + tests/test_percentile.py | 18 + tests/test_risk_control_compiler.py | 238 + tests/test_source_identity.py | 27 + tests/test_stable_tasks.py | 777 + tests/test_standalone_evaluation.py | 84 + tests/test_table_header_equivalence.py | 60 + tests/test_webfetch_format_normalization.py | 54 + uv.lock | 215 +- 85 files changed, 43624 insertions(+), 1806 deletions(-) create mode 100644 examples/answer.txt create mode 100644 examples/missing.txt create mode 100644 examples/task.json create mode 100644 src/lexbench_webfetch/body_coverage.py create mode 100644 src/lexbench_webfetch/composite_tasks.py rename src/lexbench_webfetch/{suites/webfetch => definitions/evaluation}/dataset-catalog.json (99%) rename src/lexbench_webfetch/{suites/dynamic-content => definitions/evaluation/dynamic-content-100-moli-webfetch}/dataset-catalog.json (98%) create mode 100644 src/lexbench_webfetch/definitions/evaluation/dynamic-content-100-moli-webfetch/source-inputs.json rename src/lexbench_webfetch/{suites/dynamic-content => definitions/evaluation/dynamic-content-100-moli-webfetch}/tasks.json (99%) create mode 100644 src/lexbench_webfetch/definitions/evaluation/native-retrieval-contracts.json rename src/lexbench_webfetch/{suites => definitions/evaluation}/popular-sites/dataset-catalog.json (90%) rename src/lexbench_webfetch/{suites => definitions/evaluation}/popular-sites/source-inputs.json (84%) rename src/lexbench_webfetch/{suites => definitions/evaluation}/popular-sites/tasks.json (92%) create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/body-contracts.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/current/dataset-catalog.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/current/source-inputs.json rename src/lexbench_webfetch/{suites/access-restrictions => definitions/evaluation/risk-control/current}/tasks.json (99%) rename src/lexbench_webfetch/{suites/access-restrictions => definitions/evaluation/risk-control}/dataset-catalog.json (98%) create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/first-30-rule-review.md create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/history/v5/dataset-catalog.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/list-contracts.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/qualification/cohort-policy.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/qualification/cohort/cohort-manifest.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/qualification/cohort/cohort-receipt.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/qualification/cohort/input-manifest.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/remaining-70-rule-review.md create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/rule-authoring-guide.md create mode 100644 src/lexbench_webfetch/definitions/evaluation/risk-control/source-inputs.json create mode 100644 src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json rename src/lexbench_webfetch/{suites/webfetch => definitions/evaluation}/source-inputs.json (99%) rename src/lexbench_webfetch/{suites/webfetch => definitions/evaluation}/tasks.json (99%) create mode 100644 src/lexbench_webfetch/definitions/registry.json create mode 100644 src/lexbench_webfetch/dynamic_entry_matching.py create mode 100644 src/lexbench_webfetch/dynamic_lists.py create mode 100644 src/lexbench_webfetch/dynamic_reference.py create mode 100644 src/lexbench_webfetch/evaluation.py create mode 100644 src/lexbench_webfetch/file_acceptance.py create mode 100644 src/lexbench_webfetch/inputs.py create mode 100644 src/lexbench_webfetch/list_verification.py create mode 100644 src/lexbench_webfetch/metrics.py create mode 100644 src/lexbench_webfetch/models.py create mode 100644 src/lexbench_webfetch/ordered_tree.py create mode 100644 src/lexbench_webfetch/paths.py create mode 100644 src/lexbench_webfetch/reference_quality.py create mode 100644 src/lexbench_webfetch/response_input.py create mode 100644 src/lexbench_webfetch/risk_control_dataset.py create mode 100644 src/lexbench_webfetch/scoring.py create mode 100644 src/lexbench_webfetch/scoring_regressions.py create mode 100644 src/lexbench_webfetch/stable_tasks.py create mode 100644 src/lexbench_webfetch/structured_tables.py delete mode 100644 src/lexbench_webfetch/suites/access-restrictions/source-inputs.json delete mode 100644 src/lexbench_webfetch/suites/dynamic-content/source-inputs.json delete mode 100644 src/lexbench_webfetch/suites/registry.json create mode 100644 tests/test_annotation_evidence.py create mode 100644 tests/test_body_text_acceptance_roles.py create mode 100644 tests/test_composite_tasks.py create mode 100644 tests/test_currency_matching.py create mode 100644 tests/test_dynamic_lists.py create mode 100644 tests/test_link_layout_normalization.py create mode 100644 tests/test_list_verification.py create mode 100644 tests/test_metrics.py create mode 100644 tests/test_ordered_tree.py create mode 100644 tests/test_percentile.py create mode 100644 tests/test_risk_control_compiler.py create mode 100644 tests/test_source_identity.py create mode 100644 tests/test_stable_tasks.py create mode 100644 tests/test_standalone_evaluation.py create mode 100644 tests/test_table_header_equivalence.py create mode 100644 tests/test_webfetch_format_normalization.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 344fc3d..59930f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,13 +13,18 @@ jobs: - uses: astral-sh/setup-uv@v6 with: version: "0.7.8" - - run: uv sync --frozen --python ${{ matrix.python }} - - run: uv run --frozen python -m unittest discover -s tests -v + - run: uv sync --frozen --extra test --python ${{ matrix.python }} + - run: uv run --frozen --extra test python -m pytest tests -q - run: uv run --frozen lexbench-webfetch validate - - run: uv build --sdist --wheel + - run: uv build --sdist + - run: uv build dist/*.tar.gz --wheel - name: Validate an isolated wheel installation run: | uv venv /tmp/lexbench-wheel --python ${{ matrix.python }} - uv pip install --python /tmp/lexbench-wheel/bin/python dist/*.whl + for lexbench_wheel in dist/*.whl; do + uv pip install --python /tmp/lexbench-wheel/bin/python "${lexbench_wheel}[test]" + done + cp -R tests /tmp/lexbench-wheel-tests cd /tmp /tmp/lexbench-wheel/bin/lexbench-webfetch validate + uv run --no-project --no-sync --python /tmp/lexbench-wheel/bin/python python -m pytest /tmp/lexbench-wheel-tests -q diff --git a/AGENTS.md b/AGENTS.md index 1520437..a1bac33 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,9 @@ # Repository guidance -This repository owns Web Fetch task definitions, acceptance contracts, frozen input bindings and independent input tools. Keep browser/provider adapters, generic scoring implementations, execution manifests, measured outputs and report renderers in the execution harness or external evidence storage. +This repository owns Web Fetch task definitions, acceptance contracts, frozen input bindings and independent scoring/input tools. Keep browser/provider adapters, execution manifests, measured outputs and report renderers in the execution harness or external evidence storage. Use English for code documentation and contribution guides. Keep README.md and docs/README.zh-CN.md aligned. Preserve original-language tasks, quotes and hash-bound artifacts. Keep prose concise and reader-facing. Run Python through `uv run --frozen python` or Make. Run `make check` after changes. Tests must cover accepted and rejected input scenarios. Preserve frozen input versions and add provenance for revisions. Never add credentials, third-party captures or run-local evidence to Git. + +All code changes go through a feature branch and pull request. Never push directly to main or merge a pull request. diff --git a/CITATION.cff b/CITATION.cff index 1072500..e077256 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,4 +6,4 @@ authors: - name: LexBench Web Fetch contributors repository-code: "https://github.com/lexmount/lexbench-webfetch" license: MIT -version: 0.1.0 +version: 0.2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 834220f..9b1284a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,8 @@ # Contributing -Task definitions and acceptance requirements belong here. Browser adapters, execution scheduling, generic scoring code and report rendering belong in the selected execution harness. Include a task ID and a reproducible accepted/rejected example when reporting a rule problem. +Task definitions and acceptance requirements belong here. Task models, acceptance scoring, file checks and compilation also belong here. Browser adapters, scheduling, reference-capture orchestration and report rendering belong in the selected execution harness. Include a task ID and a reproducible accepted/rejected example when reporting a rule problem. -Use Python 3.11 or newer and [uv](https://docs.astral.sh/uv/). Run `uv sync --frozen`, then `make check`. Tests use synthetic inputs and need no browser, network access or historical source bundle. Keep the English and Chinese READMEs aligned; other guides use English. +Use Python 3.11 or newer and [uv](https://docs.astral.sh/uv/). Run `uv sync --frozen`, then `make check`. Run `uv sync --frozen --extra test` to install the test tools. Tests use synthetic inputs and need no browser, network access or historical source bundle. Keep the English and Chinese READMEs aligned; other guides use English. A task revision must explain the intended requirement, preserve the prior input version, and include scenarios that should pass and fail. Update the original dataset, catalog, public task definitions, input manifest and registry together after independent source verification. Do not regenerate old hashes to disguise an input correction. Frozen quotes retain their original language. diff --git a/MANIFEST.in b/MANIFEST.in index ccdfa9c..ae55a71 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include AGENTS.md CITATION.cff CONTRIBUTING.md SECURITY.md THIRD_PARTY.md provenance.json uv.lock Makefile recursive-include docs *.md *.svg recursive-include tests *.py +recursive-include src/lexbench_webfetch/definitions *.json *.md +recursive-include examples *.json *.txt diff --git a/Makefile b/Makefile index b9531b2..99bc9dd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ .PHONY: check check: uv run --frozen lexbench-webfetch validate - uv run --frozen python -m unittest discover -s tests -v + uv run --frozen --extra test python -m pytest tests -q diff --git a/README.md b/README.md index d744c2e..cc18e81 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ uv run --frozen lexbench-webfetch validate This checks the public definitions without a browser, API key or Browser Eval checkout. It does not fetch websites or compute benchmark scores. -To reproduce a frozen suite, obtain an approved copy of its inputs. The required files and hashes are in `src/lexbench_webfetch/suites//source-inputs.json`. Third-party snapshots are not redistributed here; no public download bundle is currently provided. +To reproduce a frozen suite, obtain an approved copy of its inputs. The required files and hashes are in `src/lexbench_webfetch/definitions/registry.json`. Third-party snapshots are not redistributed here; no public download bundle is currently provided. ```sh uv run --frozen lexbench-webfetch verify-inputs \ @@ -72,6 +72,24 @@ uv run --frozen lexbench-webfetch export \ The export includes only manifest-listed files and prints the executable dataset path. Missing files, changed bytes and symlinks fail validation. Existing output directories are never reused. +## Score a saved answer + +The benchmark includes its own scorer. It needs no Browser Eval installation: + +```sh +uv run --frozen lexbench-webfetch score --task examples/task.json --response examples/answer.txt +uv run --frozen lexbench-webfetch score --task examples/task.json --response examples/missing.txt +``` + +The first answer passes; the second fails. Supply the complete task JSON from a verified dataset for a real result. Public task projections bind reference bodies by hash and cannot replace those complete rules. File tasks use repeated `--file /absolute/path/file` arguments; acceptance is derived from the actual bytes, signature, length and declared hash, not a supplied pass flag. Optional ordered-tree tasks require `uv sync --frozen --extra tree`. + +Compile the audited access-restriction rules with independently captured references: + +```sh +uv run --frozen lexbench-webfetch compile-access-restrictions \ + --evidence-home /absolute/path/reference-data-home --output /absolute/path/new-dataset.json +``` + ## Run an evaluation A runner receives each task's URL and prompt, saves its response or files, and evaluates the declared requirements against the verified reference inputs. Record the benchmark commit, suite ID, input hashes, method versions, execution policy and scorer revision. Follow the [evaluation protocol](docs/protocol.md) when reporting scores or timing. @@ -92,12 +110,14 @@ Run these two commands from Browser Eval. Ordinary runs are diagnostic; a formal | Path | Purpose | | --- | --- | -| `src/lexbench_webfetch/suites/` | Task definitions, catalogs and frozen input manifests | +| `src/lexbench_webfetch/definitions/` | Task definitions, catalogs and frozen input manifests | | `src/lexbench_webfetch/benchmark.py` | Independent definition checks and verified input export | +| `src/lexbench_webfetch/scoring.py` | Authoritative task acceptance and content diagnostics | +| `src/lexbench_webfetch/risk_control_dataset.py` | Compile audited access-restriction task requirements | | `docs/protocol.md` | Acceptance, evidence, timing and result-reporting rules | | `tests/` | Accepted/rejected input scenarios and export behavior | -Browser adapters, credentials, measured runs, report renderers and hosting configuration belong to the execution harness or its evidence storage. This repository publishes no measured leaderboard. Changes to task requirements belong here; generic execution or scorer bugs belong in the harness that implements them. +Browser adapters, credentials, measured runs, report renderers and hosting configuration belong to the execution harness or its evidence storage. This repository owns task models, deterministic acceptance scoring, file-byte checks and task compilation. Browser Eval imports these implementations as a pinned dependency. This repository publishes no measured leaderboard. ## Contribute and cite diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 59230af..762334d 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -60,7 +60,7 @@ uv run --frozen lexbench-webfetch validate 这一步检查公开任务定义，无需浏览器、API Key 或 Browser Eval。它不访问网站，也不计算评测分数。 -复现冻结任务集需要取得获准使用的原始输入。所需文件及哈希列在 `src/lexbench_webfetch/suites//source-inputs.json` 中。本仓库不分发第三方页面快照，目前也没有公开的完整输入下载包。 +复现冻结任务集需要取得获准使用的原始输入。所需文件及哈希列在 `src/lexbench_webfetch/definitions/registry.json` 中。本仓库不分发第三方页面快照，目前也没有公开的完整输入下载包。 ```sh uv run --frozen lexbench-webfetch verify-inputs \ @@ -72,6 +72,24 @@ uv run --frozen lexbench-webfetch export \ 导出工具只复制清单内的文件，并打印可执行数据集路径。缺文件、内容变化或符号链接都会使校验失败，已有输出目录不会被复用。 +## 对保存的响应评分 + +本仓库自带评分器，无需安装 Browser Eval： + +```sh +uv run --frozen lexbench-webfetch score --task examples/task.json --response examples/answer.txt +uv run --frozen lexbench-webfetch score --task examples/task.json --response examples/missing.txt +``` + +第一份响应通过，第二份失败。实际评分时，请使用已校验数据集中的完整任务 JSON。公开任务文件中的参考正文只有哈希绑定，不能替代完整验收输入。文件任务通过可重复的 `--file /absolute/path/file` 提供真实文件；评分器检查字节签名、长度和声明的哈希，不接受调用者自报的通过标记。可选的树结构任务需要先运行 `uv sync --frozen --extra tree`。 + +使用独立采集的参考输入编译已审定的访问受限任务规则： + +```sh +uv run --frozen lexbench-webfetch compile-access-restrictions \ + --evidence-home /absolute/path/reference-data-home --output /absolute/path/new-dataset.json +``` + ## 执行评测 执行工具读取任务网址与提示词，保存响应或文件，再按任务规则和已校验的参考输入验收。记录 benchmark 提交版本、任务集 ID、输入哈希、待测方法版本、执行策略和评分器版本。分数与耗时的报告方式见[评测协议](protocol.md)。 @@ -92,12 +110,14 @@ uv run --frozen browser-harness-eval run \ | 路径 | 用途 | | --- | --- | -| `src/lexbench_webfetch/suites/` | 任务定义、目录和冻结输入清单 | +| `src/lexbench_webfetch/definitions/` | 任务定义、目录和冻结输入清单 | | `src/lexbench_webfetch/benchmark.py` | 独立的定义检查与输入导出 | +| `src/lexbench_webfetch/scoring.py` | 统一的任务验收与内容诊断 | +| `src/lexbench_webfetch/risk_control_dataset.py` | 将已审定的访问受限任务规则编译为数据集 | | `docs/protocol.md` | 验收、证据、耗时和结果报告规则 | | `tests/` | 输入接受、拒绝及导出行为测试 | -浏览器适配器、凭据、实测结果、报告生成器和托管配置由执行脚手架或外部证据存储管理。本仓库没有实测排行榜。任务要求的修改在这里维护；通用执行或评分实现的问题在对应脚手架修复。 +浏览器适配器、凭据、实测结果、报告生成器和托管配置由执行脚手架或外部证据存储管理。任务模型、确定性验收评分、文件字节校验和任务编译由本仓库维护，Browser Eval 通过固定版本的依赖直接调用。本仓库没有实测排行榜。 ## 参与贡献与引用 diff --git a/docs/data.md b/docs/data.md index 1d9893b..379a546 100644 --- a/docs/data.md +++ b/docs/data.md @@ -1,6 +1,6 @@ # Data layout and rights -The suite registry is `src/lexbench_webfetch/suites/registry.json`. It binds every shipped suite file by SHA-256 and records each population's ID and size. +The suite registry is `src/lexbench_webfetch/definitions/registry.json`. Its suite directories contain the current inputs. It binds every shipped suite file by SHA-256 and records each population's ID and size. | File within a suite | Role | | --- | --- | @@ -8,9 +8,9 @@ The suite registry is `src/lexbench_webfetch/suites/registry.json`. It binds eve | `dataset-catalog.json` | Original ordered task inventory, source URLs and complete task-definition hashes | | `source-inputs.json` | Exact executable dataset and external reference-file hashes | -The four initial versions were imported from Browser Eval commit `894b38f8`. The full source commit and per-file hashes are in `provenance.json`. Original catalog and manifest bytes are retained, including original dataset IDs and task languages. Their names may contain historical method labels; those labels identify frozen input versions and confer no method-specific acceptance exception. +The initial versions were imported from Browser Eval commit `894b38f8`. The current access-restriction suite selects the later frozen version declared in the registry. Source commits and per-file hashes are in `provenance.json`. Original catalog and manifest bytes are retained, including original dataset IDs and task languages. Their names may contain historical method labels; those labels identify frozen input versions and confer no method-specific acceptance exception. -Public task requirements are maintained in this repository. Browser Eval retains hash-bound historical records needed by existing experiments; those records preserve old experiments and are not a second source for new task revisions. +Public task requirements are maintained in this repository. Frozen historical catalogs and manifests also live under `definitions/evaluation/` with their original relative paths and bytes. Browser Eval resolves its historical logical paths through this installed package. It retains experiment profiles and compact hash bindings, rather than duplicate definition files. ## External inputs diff --git a/docs/protocol.md b/docs/protocol.md index 546e3f1..9856caa 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -8,7 +8,7 @@ Use the original dataset verified by the suite's `source-inputs.json`. The publi A method receives the task URL and prompt. Follow the declared task kind: return body text, structured records with the required identities and links, discover the specified resources, or deliver the requested file. A successful HTTP request or nonempty response alone is insufficient. Method-specific alternate task requirements are not allowed. -Task requirements are data: expected markers, fact groups, forbidden output, minimum length, body coverage, structure and resource/file contracts are selected by the task. Missing reference coverage is unavailable, not zero or a perfect score. Binary task acceptance and text-quality diagnostics are separate. Browser Eval implements generic scoring through `score_attempt` and the shared `task_passed` predicate; other harnesses must disclose their evaluator implementation and demonstrate equivalent task-rule behavior before comparing scores. +Task requirements are data: expected markers, fact groups, forbidden output, minimum length, body coverage, structure and resource/file contracts are selected by the task. Missing reference coverage is unavailable, not zero or a perfect score. Binary task acceptance and text-quality diagnostics are separate. LexBench owns `score_attempt`, the shared `task_passed` predicate and file-byte acceptance. Browser Eval calls the same installed implementations. Other harnesses can use `lexbench_webfetch.evaluation.evaluate_response` or the `score` command without Browser Eval; disclose the benchmark/scorer revision when comparing scores. ## Live sources and frozen evidence diff --git a/examples/answer.txt b/examples/answer.txt new file mode 100644 index 0000000..ceeecac --- /dev/null +++ b/examples/answer.txt @@ -0,0 +1 @@ +Alpha article diff --git a/examples/missing.txt b/examples/missing.txt new file mode 100644 index 0000000..7c9da2c --- /dev/null +++ b/examples/missing.txt @@ -0,0 +1 @@ +Please sign in to continue diff --git a/examples/task.json b/examples/task.json new file mode 100644 index 0000000..957521a --- /dev/null +++ b/examples/task.json @@ -0,0 +1 @@ +{"id":"article","name":"Local article","url":"https://example.test/article","category":"web","expected_any":["Alpha article"],"min_content_chars":1} diff --git a/provenance.json b/provenance.json index 1548a9b..9924071 100644 --- a/provenance.json +++ b/provenance.json @@ -12,5 +12,54 @@ "evaluation/risk-control/dataset-catalog.json": "43e79ab63c12f56ed025d78fa6e417cfc91b812faa6a69b9016b99979a498a98", "evaluation/risk-control/source-inputs.json": "230c3a88585c7ebb838bc2d7d956fbbfbb4a2c23d2ff1f0a84eb103909c6ee14" }, - "task_projection": "Task instructions and acceptance structure; body reference text/source_text are replaced by SHA-256 and character-count bindings. Full references, answers and capture/review payloads remain in external inputs. Original full task hashes remain in each catalog." + "task_projection": "Task instructions and acceptance structure; body reference text/source_text are replaced by SHA-256 and character-count bindings. Full references, answers and capture/review payloads remain in external inputs. Original full task hashes remain in each catalog.", + "implementation_import": { + "source_repository": "https://github.com/lexmount/browser-eval", + "source_commit": "b722587f9e8e9e8b438cd17f3c625096fd2af315", + "source_files": { + "harness/web_fetch/scoring.py": "6488d212d51cd4ac2e902f0c90e4ff56ffca7f9e63798a0de588300e05f0c91e", + "harness/web_fetch/metrics.py": "d3c37702ff79a952f0ab1d75e3030ec45ca388663c226f31a45a5dab46400029", + "harness/web_fetch/stable_tasks.py": "8767205609ddf47b59afab6110043d4076247eda6319a84d42557587586e20c3", + "harness/web_fetch/body_coverage.py": "71f1306f73e13c726376220eb542e0dbd1332ae692586d0ca7422c6b5c3822e3", + "harness/web_fetch/composite_tasks.py": "245ba0c069d6951ac339aa69fba6bd6c0724bf69c347a4141831b90a014078ea", + "harness/web_fetch/dynamic_lists.py": "5b2a6e9afc197806601703d1e1d58956970920100a664b573342d875b0e42fdd", + "harness/web_fetch/reference_quality.py": "e9c961b9ec90ea17f6dc84f5cbb702290b81f86204889287a8a2a3690ea7d162", + "harness/web_fetch/dynamic_entry_matching.py": "206f483bac2f11280f2ecc04f8d6aeb61ea68a2a1b47ac2abee3ded219f3e4d3", + "harness/web_fetch/dynamic_reference.py": "b0ea992fdb83f7af92c396919b944cb5947fe916b72a791e4e34bc178027996e", + "harness/web_fetch/response_input.py": "7ab4008509d382bc33455bb1415da9d6254f0ac23fda56dee029e45b646adb7e", + "harness/web_fetch/structured_tables.py": "2cd2d8e45ffd1d7b789eebe071708b476cdd774ff4397e406df00d579fafcd8f", + "harness/web_fetch/ordered_tree.py": "78c418fbe07188f3b7e8acbbccc64c16074208631517e77ca6edf7b7ceae166c", + "harness/web_fetch/list_verification.py": "c311bb3ef9d46f21319b7eef35e34120580ec6cad720958b6336fe59e5a24bf7", + "harness/web_fetch/scoring_regressions.py": "133853bbb7a9d2d20c100807a13a83c65eed83e5d6b2a9534d0c83fe5a49d10a", + "harness/web_fetch/risk_control_dataset.py": "58d0703d4138329a9b8f90a15eac23a0c5d0ad1b4f845b42fb69834c4c545ab9", + "harness/web_fetch/models.py": "9edd2aaf363f6e684425579f28ba73b088bd604db932f2dd37eb9997febd4e60", + "harness/web_fetch/safety.py": "a2c8974631e9e003fc493b7e2b3b32a74cf6ce5e87311e79038609f685511066", + "harness/web_fetch/reference_inputs.py": "3a7e5415f6a2ca305fefff2e15673965465ddbc801792ac017d346bd94460ab1", + "harness/web_fetch/call_boundary.py": "0e367fc148f3271b9600fb543db1a291a4e2e4bbe0f1ec0f223ae1c77a364a55" + } + }, + "definition_import": { + "evaluation/dataset-catalog.json": "994fa53f7fc7c3170692a7bb493ebc5af9ad8ffb5b576c022fa123e6e6fb3838", + "evaluation/dynamic-content-100-moli-webfetch/dataset-catalog.json": "ca1572aa80fc2324b2f00800a79b053cfd584d1831ffb16e6906ae0981522c6c", + "evaluation/dynamic-content-100-moli-webfetch/source-inputs.json": "710f9c91705b29bb9e4aa15af97dcc856e64e406baa4b9966446408d7e3d26a8", + "evaluation/popular-sites/dataset-catalog.json": "f2b9dfb550230cdf8dbbba544d58158ac957620c1d064173790ead70754c178d", + "evaluation/popular-sites/source-inputs.json": "32c31cfd5458a1dfe06d370fc2411a1eea5d1a05da8ef88e0dcd4fd3b0f3c810", + "evaluation/risk-control/body-contracts.json": "de5a1ff1b4333e980dd824848e79f2e7717175b53ffade512491afeb4f6c13f9", + "evaluation/risk-control/current/dataset-catalog.json": "52ec968b2ada26e5e061ccb25cbe4a73747539e556fa88f31bece6d7fb79b3cb", + "evaluation/risk-control/current/source-inputs.json": "c61d6825abbff2a92098592f83055347cd0b0101374794023046c0fd0323dd5f", + "evaluation/risk-control/dataset-catalog.json": "02e30a5c408de3b5113e28f27359cf5e454814de612f860e6b7d7536c892efb0", + "evaluation/risk-control/first-30-rule-review.md": "6945a6aa7b0d8d5990d8abf5d24fc41b5a42a22c70940a7c996d8698c6e9bf81", + "evaluation/risk-control/history/v5/dataset-catalog.json": "73b1a3cd945c535d5043f0ba01316b98a26af25ed5156d900cdff79f2cbad4ef", + "evaluation/risk-control/list-contracts.json": "a566fb13328bd1a88b189b5a49fb7b3565a8d037c22aaa19c2b86087cbbe0881", + "evaluation/risk-control/qualification/cohort/cohort-manifest.json": "aeec7ec26d24650896263574135a27fc4848f4d362bd38f0a5b5c167f82abc26", + "evaluation/risk-control/qualification/cohort/cohort-receipt.json": "a5929b62ffb8b64da1ac902c5c5be61a75fa01ed171a497f115de14442a9a872", + "evaluation/risk-control/qualification/cohort/input-manifest.json": "8a61be8ffdf09a4f5ba84ebd2a621243a6b59d726825b1884f9c71295ecffd27", + "evaluation/risk-control/qualification/cohort-policy.json": "4835abe67852adf1609cb3033b19cbef839650d14879c0e24dd76116af2acd2e", + "evaluation/risk-control/remaining-70-rule-review.md": "301574dddcfddf6d30ed4d882cdd40b1ae859fdc95a0a02cf9868f55e3fd1079", + "evaluation/risk-control/rule-authoring-guide.md": "74363ab502978179e5438a0b992bac7563090e41c91784e492636832dc1cd088", + "evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258", + "evaluation/scoring-regressions.json": "d519f86e327d9e2fd1c884670642eb31714435864e834592e2c7b65ef21ee655", + "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708", + "evaluation/native-retrieval-contracts.json": "210281ea49c61aa9c1ae4976be0f8b97adf1ef0eed6e08a74d77f54bab80b71b" + } } diff --git a/pyproject.toml b/pyproject.toml index 56494f3..d3d39a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,15 @@ [project] name = "lexbench-webfetch" -version = "0.1.0" +version = "0.2.0" description = "Versioned Web Fetch tasks, acceptance contracts and frozen reference bindings" readme = "README.md" requires-python = ">=3.11" license = "MIT" -dependencies = [] +dependencies = ["beautifulsoup4==4.14.2", "soupsieve==2.9.2"] + +[project.optional-dependencies] +test = ["pytest>=8,<10", "apted==1.0.3", "rapidfuzz==3.14.1"] +tree = ["apted==1.0.3", "rapidfuzz==3.14.1"] [project.scripts] lexbench-webfetch = "lexbench_webfetch.cli:main" @@ -22,4 +26,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -lexbench_webfetch = ["suites/*.json", "suites/*/*.json"] +lexbench_webfetch = ["definitions/*.json", "definitions/evaluation/*.json", "definitions/evaluation/*/*.json", "definitions/evaluation/*/*.md", "definitions/evaluation/*/*/*.json", "definitions/evaluation/*/*/*/*.json"] diff --git a/src/lexbench_webfetch/benchmark.py b/src/lexbench_webfetch/benchmark.py index efd67bd..f8b7be4 100644 --- a/src/lexbench_webfetch/benchmark.py +++ b/src/lexbench_webfetch/benchmark.py @@ -7,9 +7,11 @@ import re import shutil import tempfile -from pathlib import Path, PurePosixPath +from pathlib import Path +from .paths import contained_path +from .inputs import digest, load_manifest, verify_catalog -SUITES = Path(__file__).parent / "suites" +SUITES = Path(__file__).parent / "definitions" # Full reference bodies and source-capture/review payloads stay in frozen inputs. PUBLIC_TASK_FIELDS = ( @@ -54,27 +56,18 @@ def public_task(case: dict) -> dict: routing["risk_control"] = {"navigation_contract": navigation} if routing: task["source_benchmark"] = routing + verification = {key: case.get("source_verification", {})[key] + for key in ("mime", "file_sha256") if key in case.get("source_verification", {})} + if verification: + task["source_verification"] = verification review = case.get("rule_review", {}) if review: task["rule_review"] = {key: review[key] for key in ("status", "reason") if key in review} return task -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() -def bound_path(root: Path, relative: str) -> Path: - path = PurePosixPath(relative) - if not relative or path.is_absolute() or ".." in path.parts or "\\" in relative: - raise ValueError(f"Unsafe input path: {relative}") - target = root / relative - for part in (root, *target.parents, target): - if part.is_symlink(): - raise ValueError(f"Symlinked input: {part}") - if not target.resolve().is_relative_to(root.resolve()): - raise ValueError(f"Escaping input: {relative}") - return target def read(path: Path) -> dict: @@ -96,11 +89,11 @@ def suite(identifier: str) -> tuple[dict, Path]: if len(matches) != 1: raise ValueError(f"Unknown suite: {identifier}") entry = matches[0] - root = bound_path(SUITES, identifier) + root = contained_path(SUITES, entry.get("directory", identifier), reject_symlinks=True) for relative, expected in entry["files"].items(): if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ValueError(f"Invalid SHA-256: {relative}") - if digest(bound_path(root, relative)) != expected: + if digest(contained_path(root, relative, reject_symlinks=True)) != expected: raise ValueError(f"Changed suite file: {identifier}/{relative}") return entry, root @@ -109,13 +102,9 @@ def validate(identifier: str) -> dict: entry, root = suite(identifier) catalog = read(root / "dataset-catalog.json") tasks = read(root / "tasks.json") - manifest = read(root / "source-inputs.json") - if (manifest.get("schema_version") != 2 - or manifest.get("kind") != "frozen_reference_inputs" - or manifest["catalog_sha256"] != digest(root / "dataset-catalog.json") - or manifest["dataset_sha256"] != catalog["dataset_sha256"] - or manifest["files"].get(manifest["dataset"]) != manifest["dataset_sha256"]): - raise ValueError("Inconsistent frozen input bindings") + manifest = load_manifest(root / "source-inputs.json") + if manifest['catalog'] != 'dataset-catalog.json': + raise ValueError('Manifest must select the registry-bound task catalog') if not (entry["dataset_id"] == catalog["id"] == tasks["id"]): raise ValueError("Inconsistent dataset identity") declared = catalog["cases"] @@ -129,7 +118,7 @@ def validate(identifier: str) -> dict: if key in row and row[key] != task.get(key): raise ValueError(f"Task catalog mismatch: {task['id']}/{key}") for relative, expected in input_files(manifest).items(): - bound_path(root, relative) + contained_path(root, relative, reject_symlinks=True) if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ValueError(f"Invalid input SHA-256: {relative}") return {"suite": identifier, "case_count": len(ids), "dataset_sha256": catalog["dataset_sha256"]} @@ -151,20 +140,15 @@ def verify_inputs(identifier: str, source: Path) -> dict: _, root = suite(identifier) manifest = read(root / "source-inputs.json") for relative, expected in input_files(manifest).items(): - if digest(bound_path(source, relative)) != expected: + if digest(contained_path(source, relative, reject_symlinks=True)) != expected: raise ValueError(f"Changed frozen input: {relative}") - dataset = read(bound_path(source, manifest["dataset"])) - catalog = read(root / "dataset-catalog.json") - if (len(dataset["cases"]) != catalog["case_count"] - or any(dataset.get(k) != catalog.get(k) for k in ("id", "name", "coverage_plan"))): - raise ValueError("Frozen dataset inventory differs from catalog") - tasks = read(root / "tasks.json")["cases"] - for case, row, task in zip(dataset["cases"], catalog["cases"], tasks): - encoded = json.dumps(case, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() - if hashlib.sha256(encoded).hexdigest() != row["definition_sha256"]: - raise ValueError(f"Changed task definition: {row['id']}") + raw = contained_path(source, manifest['dataset'], reject_symlinks=True).read_bytes() + verify_catalog(root/'source-inputs.json', manifest, raw) + dataset = json.loads(raw) + tasks = read(root/'tasks.json')['cases'] + for case, task in zip(dataset['cases'], tasks): if public_task(case) != task: - raise ValueError(f"Public task differs from frozen input: {row['id']}") + raise ValueError(f"Public task differs from frozen input: {case['id']}") return result @@ -180,9 +164,9 @@ def export(identifier: str, source: Path, output: Path) -> Path: stage = Path(tempfile.mkdtemp(prefix=".lexbench-", dir=output.parent)) try: for relative in input_files(manifest): - target = bound_path(stage, relative) + target = contained_path(stage, relative, reject_symlinks=True) target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(bound_path(source, relative), target) + shutil.copyfile(contained_path(source, relative, reject_symlinks=True), target) verify_inputs(identifier, stage) # A materialized dataset is intentionally independent of the historical # Browser Eval external-input resolver; do not copy source-inputs.json. diff --git a/src/lexbench_webfetch/body_coverage.py b/src/lexbench_webfetch/body_coverage.py new file mode 100644 index 0000000..8127824 --- /dev/null +++ b/src/lexbench_webfetch/body_coverage.py @@ -0,0 +1,501 @@ +"""Declared, distributed body coverage; independent of the full-text F1 metric.""" +import re +import hashlib +from collections import Counter +from itertools import combinations +from math import ceil + + +def tokens(text): + from .metrics import normalize_for_match + return re.findall(r'[\u3400-\u9fff]|[^\W_\u3400-\u9fff]+', normalize_for_match(text)) + + +def _group_contiguous(units, count): + """Group natural units contiguously without splitting an individual unit.""" + count = min(count, len(units)) + if count <= 1: + return ['\n\n'.join(units)] + weights = [max(1, len(tokens(unit))) for unit in units] + total = sum(weights) + groups = [] + start = 0 + consumed = 0 + for index in range(1, count): + target = total * index / count + end = start + while end + 1 <= len(units) - (count - index): + next_total = consumed + sum(weights[start:end + 1]) + after_total = consumed + sum(weights[start:end + 2]) + if end > start and abs(next_total - target) <= abs(after_total - target): + break + end += 1 + end = max(start + 1, end) + groups.append('\n\n'.join(units[start:end])) + consumed += sum(weights[start:end]) + start = end + groups.append('\n\n'.join(units[start:])) + return [group for group in groups if group.strip()] + + +def _natural_regions(reference, desired): + text = str(reference or '').replace('\r\n', '\n').replace('\r', '\n').strip() + paragraphs = [part.strip() for part in re.split(r'\n\s*\n+', text) if part.strip()] + heading = re.compile(r'^(?:#{1,6}\s+|[A-Z][^.!?。！？]{0,70}:?$)') + has_explicit_headings = any( + re.match(r'^#{1,6}\s+', paragraph.splitlines()[0].strip()) + for paragraph in paragraphs + ) + if len(paragraphs) >= 2: + sections = [] + current = [] + for paragraph in paragraphs: + first = paragraph.splitlines()[0].strip() + is_heading = (bool(re.match(r'^#{1,6}\s+', first)) + or (not has_explicit_headings + and len(paragraph.splitlines()) == 1 + and len(tokens(first)) <= 12 + and bool(heading.match(first)))) + if is_heading and current: + sections.append('\n\n'.join(current)) + current = [paragraph] + else: + current.append(paragraph) + if current: + sections.append('\n\n'.join(current)) + if len(sections) >= 2: + return _group_contiguous(sections, desired), 'sections' + if has_explicit_headings: + units = [] + index = 0 + while index < len(paragraphs): + paragraph = paragraphs[index] + if (re.match(r'^#{1,6}\s+', paragraph.splitlines()[0].strip()) + and index + 1 < len(paragraphs)): + units.append(paragraph + '\n\n' + paragraphs[index + 1]) + index += 2 + else: + units.append(paragraph) + index += 1 + return _group_contiguous(units, desired), 'paragraphs' + return _group_contiguous(paragraphs, desired), 'paragraphs' + lines = [line.strip() for line in text.splitlines() if line.strip()] + if len(lines) >= 4: + sections = [] + current = [] + for index, line in enumerate(lines): + next_line = lines[index + 1] if index + 1 < len(lines) else '' + is_heading = ( + bool(re.match(r'^#{1,6}\s+', line)) + or (len(tokens(line)) <= 12 + and not re.search(r'[.!?。！？;；]$', line) + and bool(next_line) + and (len(tokens(next_line)) > len(tokens(line)) + or re.search(r'[.!?。！？]$', next_line))) + ) + if is_heading and current: + sections.append('\n'.join(current)) + current = [line] + else: + current.append(line) + if current: + sections.append('\n'.join(current)) + if len(sections) >= 2: + return _group_contiguous(sections, desired), 'sections' + return _group_contiguous(lines, desired), 'line_blocks' + sentences = [part.strip() for part in re.split(r'(?<=[.!?。！？])\s+', text) + if part.strip()] + if len(sentences) >= 2: + return _group_contiguous(sentences, desired), 'sentences' + words = tokens(text) + return ([' '.join(words[len(words) * index // desired: + len(words) * (index + 1) // desired]) + for index in range(desired)], 'balanced_tokens_fallback') + + +def body_requirements(reference, *, structure_hint=None): + words = tokens(reference) + desired = min(5, max(1, len(words) // 20)) + if structure_hint == 'whole_declared_component': + natural, method = [reference], 'whole_declared_component' + else: + natural, method = _natural_regions(reference, desired) + regions = [] + for index, text in enumerate(natural): + normalized = ' '.join(tokens(text)) + if normalized: + regions.append({'id': f'body-region-{index + 1}', 'text': normalized, + 'source_text': text, 'segmentation_method': method}) + count = len(regions) + return {'version': 3, 'unit': 'region_compact_hexagram_recall', + 'minimum_coverage': 0.8, 'minimum_floor': 0.6, + 'minimum_region_passes': max(1, count - 1), + 'segmentation_method': method, 'regions': regions} + + +def compact_trigrams(text): + return compact_ngrams(text, 3) + + +def compact_ngrams(text, size): + from .metrics import normalize_for_match + compact = ''.join(character for character in normalize_for_match(text) + if character.isalnum()) + return [compact[index:index+size] for index in range(max(0, len(compact)-size+1))] + + +def _compact_with_positions(text): + from .metrics import normalize_for_match + normalized = normalize_for_match(text) + positions = [index for index, character in enumerate(normalized) if character.isalnum()] + return normalized, ''.join(normalized[index] for index in positions), positions + + +def _merge_compact_spans(spans, positions): + if not spans: + return [] + merged = [] + for start, end in sorted(spans): + if merged and start <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], end) + else: + merged.append([start, end]) + return [{ + 'compact_start': start, + 'compact_end': end, + 'normalized_start': positions[start], + 'normalized_end': positions[end - 1] + 1, + } for start, end in merged] + + +def _physical_match_spans(content, expected_by_id, assigned_by_id): + """Bind allocated fragment counts to deterministic physical occurrences.""" + normalized, compact, positions = _compact_with_positions(content) + all_units = sorted(set().union(*(set(row) for row in assigned_by_id.values()))) + wanted = set(all_units) + occurrences = {unit: [] for unit in all_units} + for start in range(max(0, len(compact) - 5)): + unit = compact[start:start + 6] + if unit in wanted: + occurrences[unit].append(start) + cursor = Counter() + spans = {identifier: [] for identifier in assigned_by_id} + for unit in all_units: + for identifier in sorted(assigned_by_id): + for _ in range(assigned_by_id[identifier][unit]): + index = cursor[unit] + if index >= len(occurrences[unit]): + break + start = occurrences[unit][index] + spans[identifier].append((start, start + len(unit))) + cursor[unit] += 1 + return normalized, { + identifier: _merge_compact_spans(rows, positions) + for identifier, rows in spans.items() + } + + +def _reference_match_spans(reference, matched): + normalized, compact, positions = _compact_with_positions(reference) + starts_by_unit = {unit: [] for unit in matched} + for start in range(max(0, len(compact) - 5)): + unit = compact[start:start + 6] + if unit in starts_by_unit: + starts_by_unit[unit].append(start) + spans = [] + for unit in sorted(matched): + starts = starts_by_unit[unit] + spans.extend((start, start + len(unit)) for start in starts[:matched[unit]]) + return normalized, _merge_compact_spans(spans, positions) + + +def _span_passages(text, spans): + return [{**span, 'text': text[span['normalized_start']:span['normalized_end']]} + for span in spans] + + +def _unmatched_passages(text, spans): + passages = [] + cursor = 0 + for span in spans: + start, end = span['normalized_start'], span['normalized_end'] + if text[cursor:start].strip(): + passages.append({'normalized_start': cursor, 'normalized_end': start, + 'text': text[cursor:start]}) + cursor = max(cursor, end) + if text[cursor:].strip(): + passages.append({'normalized_start': cursor, 'normalized_end': len(text), + 'text': text[cursor:]}) + return passages + + +def _maximum_allocation(capacities, demands, limits): + """Allocate fragment instances globally, with deterministic max-flow ties.""" + region_ids = sorted(demands) + allocation = {identifier: Counter() for identifier in region_ids} + + # Units owned by only one region cannot affect another region's quota. Assign + # them directly so the flow graph only contains genuinely shared fragments. + shared = {} + for unit in sorted(set().union(*(set(row) for row in demands.values()))): + claimants = [identifier for identifier in region_ids if demands[identifier][unit]] + capacity = min(capacities[unit], sum(demands[row][unit] for row in claimants)) + if capacity <= 0: + continue + if len(claimants) == 1: + identifier = claimants[0] + allocation[identifier][unit] = min(capacity, demands[identifier][unit]) + else: + shared[unit] = capacity + + residual_limits = { + identifier: max(0, limits[identifier] - sum(allocation[identifier].values())) + for identifier in region_ids + } + required_flow = sum(residual_limits.values()) + if not required_flow: + return allocation, True + + source = ('source',) + sink = ('sink',) + graph = {} + + def add_edge(start, end, capacity): + graph.setdefault(start, {}) + graph.setdefault(end, {}) + graph[start][end] = graph[start].get(end, 0) + capacity + graph[end].setdefault(start, 0) + + for unit, capacity in shared.items(): + unit_node = ('unit', unit) + add_edge(source, unit_node, capacity) + for identifier in region_ids: + capacity = demands[identifier][unit] + if capacity: + add_edge(unit_node, ('region', identifier), capacity) + for identifier in region_ids: + add_edge(('region', identifier), sink, residual_limits[identifier]) + + total = 0 + ordered = {node: sorted(neighbors, key=repr) for node, neighbors in graph.items()} + while total < required_flow: + level = {source: 0} + queue = [source] + for node in queue: + for neighbor in ordered.get(node, ()): + if graph[node][neighbor] > 0 and neighbor not in level: + level[neighbor] = level[node] + 1 + queue.append(neighbor) + if sink not in level: + break + cursor = {node: 0 for node in graph} + + def send(node, amount): + if node == sink: + return amount + sent = 0 + neighbors = ordered.get(node, ()) + while cursor[node] < len(neighbors) and sent < amount: + neighbor = neighbors[cursor[node]] + capacity = graph[node][neighbor] + if capacity > 0 and level.get(neighbor) == level[node] + 1: + pushed = send(neighbor, min(amount - sent, capacity)) + if pushed: + graph[node][neighbor] -= pushed + graph[neighbor][node] += pushed + sent += pushed + continue + cursor[node] += 1 + return sent + + while total < required_flow: + pushed = send(source, required_flow - total) + if not pushed: + break + total += pushed + + for unit, capacity in shared.items(): + unit_node = ('unit', unit) + for identifier in region_ids: + original = demands[identifier][unit] + remaining = graph[unit_node].get(('region', identifier), 0) + if original > remaining: + allocation[identifier][unit] += original - remaining + return allocation, total == required_flow + + +def _global_region_allocation(predicted, expected, floor_counts, target_counts, + required_regions): + """Find one simultaneous allocation satisfying the declared region quotas.""" + region_ids = sorted(expected) + total_expected = Counter() + for row in expected.values(): + total_expected.update(row) + if all(predicted[unit] >= count for unit, count in total_expected.items()): + return {identifier: expected[identifier].copy() for identifier in region_ids}, True + available_total = sum(min(predicted[unit], count) + for unit, count in total_expected.items()) + minimum_total = (sum(floor_counts.values()) + + sum(sorted(target_counts[identifier] - floor_counts[identifier] + for identifier in region_ids)[:required_regions])) + if available_total < minimum_total: + # The total-capacity lower bound already proves infeasibility. Produce a + # deterministic maximum-size diagnostic allocation without building flow + # graphs for every possible target-region subset. + allocation = {identifier: Counter() for identifier in region_ids} + for unit in sorted(total_expected): + remaining = min(predicted[unit], total_expected[unit]) + for identifier in region_ids: + matched = min(remaining, expected[identifier][unit]) + if matched: + allocation[identifier][unit] = matched + remaining -= matched + if not remaining: + break + return allocation, False + for targets in combinations(region_ids, required_regions): + target_set = set(targets) + limits = {identifier: (target_counts[identifier] if identifier in target_set + else floor_counts[identifier]) + for identifier in region_ids} + allocation, feasible = _maximum_allocation(predicted, expected, limits) + if feasible: + break + else: + limits = floor_counts + allocation, feasible = _maximum_allocation(predicted, expected, limits) + if not feasible: + limits = {identifier: sum(row.values()) for identifier, row in expected.items()} + allocation, _ = _maximum_allocation(predicted, expected, limits) + return allocation, False + + # Quotas establish the verdict; spend remaining fragment capacity globally so + # the per-region diagnostic counts describe as much of the answer as possible. + used = Counter() + for row in allocation.values(): + used.update(row) + remaining_capacity = predicted - used + remaining_demand = { + identifier: expected[identifier] - allocation[identifier] + for identifier in region_ids + } + extra_limits = {identifier: sum(row.values()) for identifier, row in remaining_demand.items()} + extras, _ = _maximum_allocation(remaining_capacity, remaining_demand, extra_limits) + for identifier in region_ids: + allocation[identifier].update(extras[identifier]) + return allocation, True + + +def _evaluate_body_surface(content, contract): + if not contract: + return {'body_coverage_pass': True, 'body_region_results': []} + version = contract.get('version') + unit = contract.get('unit') + valid_units = {(1, 'region_token_recall'), + (2, 'region_compact_trigram_recall'), + (3, 'region_compact_hexagram_recall')} + if ((version, unit) not in valid_units + or not 0 < contract.get('minimum_coverage', 0) <= 1 or not contract.get('regions')): + raise ValueError('Invalid distributed body requirements') + if version == 3: + floor = contract.get('minimum_floor') + required_regions = contract.get('minimum_region_passes') + if (not isinstance(required_regions, int) + or not 1 <= required_regions <= len(contract['regions']) + or not isinstance(floor, (int, float)) + or not 0 < floor <= contract['minimum_coverage']): + raise ValueError('Invalid distributed body requirements') + else: + floor = contract['minimum_coverage'] + required_regions = len(contract['regions']) + units = tokens(content) if version == 1 else compact_ngrams(content, 6 if version == 3 else 3) + predicted = Counter(units) + expected_by_id = {} + regions_by_id = {} + if version == 3: + for region in contract['regions']: + identifier = region.get('id') + if not isinstance(identifier, str) or not identifier or identifier in regions_by_id: + raise ValueError('Invalid distributed body requirements') + expected_units = compact_ngrams(region.get('text', ''), 6) + if not expected_units: + raise ValueError('Empty body region') + regions_by_id[identifier] = region + expected_by_id[identifier] = Counter(expected_units) + floor_counts = {identifier: ceil(floor * sum(row.values()) - 1e-12) + for identifier, row in expected_by_id.items()} + target_counts = {identifier: ceil(contract['minimum_coverage'] * sum(row.values()) - 1e-12) + for identifier, row in expected_by_id.items()} + assigned_by_id, coverage_pass = _global_region_allocation( + predicted, expected_by_id, floor_counts, target_counts, required_regions) + matching_text, output_spans_by_id = _physical_match_spans( + content, expected_by_id, assigned_by_id) + checks = [] + for region in contract['regions']: + expected_units = (region['text'].split() if version == 1 + else compact_ngrams(region['text'], 6 if version == 3 else 3)) + expected = Counter(expected_units) + if not expected: + raise ValueError('Empty body region') + matched = assigned_by_id[region['id']] if version == 3 else predicted & expected + missing = expected - matched + coverage = sum(matched.values()) / sum(expected.values()) + label = 'token' if version == 1 else 'fragment' + target_pass = coverage >= contract['minimum_coverage'] + floor_pass = coverage >= floor + checks.append({'id': region['id'], 'coverage': round(coverage, 4), + 'expected_units': sum(expected.values()), + 'matched_units': sum(matched.values()), + 'missing_units': sum(missing.values()), + 'missing_sample': [{label: value, 'count': count} + for value, count in missing.most_common(40)], + 'minimum': contract['minimum_coverage'], 'floor': floor, + 'target_pass': target_pass, 'floor_pass': floor_pass, + 'pass': target_pass}) + if version == 3: + reference_matching_text, reference_spans = _reference_match_spans( + region.get('text', ''), matched) + output_spans = output_spans_by_id[region['id']] + checks[-1].update({ + 'matching_text_sha256': hashlib.sha256(matching_text.encode()).hexdigest(), + 'matched_output_spans': output_spans, + 'matched_output_passages': _span_passages(matching_text, output_spans), + 'reference_matching_text_sha256': hashlib.sha256( + reference_matching_text.encode()).hexdigest(), + 'matched_reference_spans': reference_spans, + 'matched_reference_passages': _span_passages( + reference_matching_text, reference_spans), + 'unmatched_reference_passages': _unmatched_passages( + reference_matching_text, reference_spans), + }) + if version == 1: + checks[-1].update(expected_tokens=checks[-1]['expected_units'], + matched_tokens=checks[-1]['matched_units'], + missing_tokens=checks[-1]['missing_units']) + if version != 3: + coverage_pass = (all(c['floor_pass'] for c in checks) + and sum(c['target_pass'] for c in checks) >= required_regions) + return {'body_coverage_pass': coverage_pass, + 'body_target_regions': sum(c['target_pass'] for c in checks), + 'body_required_target_regions': required_regions, + **({'body_matching_text_sha256': hashlib.sha256( + matching_text.encode()).hexdigest()} if version == 3 else {}), + 'body_region_results': checks} + + +def evaluate_body_requirements(content, contract): + """Evaluate prose directly, then one structure-preserving table column at a time.""" + result = _evaluate_body_surface(content, contract) + if not contract or result['body_coverage_pass']: + return result + from .structured_tables import table_header_column_surfaces + for surface in table_header_column_surfaces(content): + candidate = _evaluate_body_surface(surface['text'], contract) + if candidate['body_coverage_pass']: + candidate.update({ + 'body_matching_surface': 'table_header_and_column', + 'body_matching_table_index': surface['table_index'], + 'body_matching_column_index': surface['column_index'], + }) + return candidate + return result diff --git a/src/lexbench_webfetch/cli.py b/src/lexbench_webfetch/cli.py index 204e092..8207caf 100644 --- a/src/lexbench_webfetch/cli.py +++ b/src/lexbench_webfetch/cli.py @@ -18,9 +18,34 @@ def main() -> None: command.add_argument("--source", type=Path, required=True) if name == "export": command.add_argument("--output", type=Path, required=True) + score = sub.add_parser("score", help="Evaluate a saved response and delivered files") + score.add_argument("--task", type=Path, required=True, + help="Complete task JSON including its frozen reference requirements") + score.add_argument("--response", type=Path, required=True) + score.add_argument("--file", type=Path, action="append", default=[]) + score.add_argument("--list-reference", type=Path) + score.add_argument("--request-failed", action="store_true") + score.add_argument("--status-code", type=int) + score.add_argument("--final-url") + score.add_argument("--title") + compile_rules = sub.add_parser("compile-access-restrictions") + compile_rules.add_argument("--output", type=Path, required=True) + compile_rules.add_argument("--definition-dir", type=Path) + compile_rules.add_argument("--evidence-home", type=Path, required=True) args = parser.parse_args() try: - if args.command == "list": + if args.command == "score": + from .evaluation import evaluate_response + result = evaluate_response( + json.loads(args.task.read_text(encoding="utf-8")), args.response.read_bytes(), + files=[path.read_bytes() for path in args.file], ok=not args.request_failed, + list_reference=(json.loads(args.list_reference.read_text(encoding="utf-8")) + if args.list_reference else None), status_code=args.status_code, + title=args.title, final_url=args.final_url) + elif args.command == "compile-access-restrictions": + from .risk_control_dataset import build, RISK + result = build(args.output, args.definition_dir or RISK, evidence_home=args.evidence_home) + elif args.command == "list": result = [{k: row[k] for k in ("id", "dataset_id", "case_count")} for row in registry()["suites"]] elif args.command == "validate": result = [validate(row["id"]) for row in registry()["suites"] @@ -34,3 +59,7 @@ def main() -> None: except (ValueError, KeyError, OSError) as error: parser.exit(1, f"{error}\n") print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/lexbench_webfetch/composite_tasks.py b/src/lexbench_webfetch/composite_tasks.py new file mode 100644 index 0000000..11dc516 --- /dev/null +++ b/src/lexbench_webfetch/composite_tasks.py @@ -0,0 +1,1079 @@ +"""Composite acceptance contracts for independently scoped page records. + +The existing stable-task contract deliberately describes one homogeneous +record set. Some pages contain several lists which must each meet their own +threshold, or rows whose extra fields must belong to the same record. This +module adds that composition without teaching the provider or report code +about individual sites. It is pure scoring code and performs no I/O. +""" + +from __future__ import annotations + +import re +import unicodedata +from urllib.parse import parse_qsl, unquote, urljoin, urlsplit + +from .structured_tables import _pipe_cells, table_text_surface +from .dynamic_lists import _canonical_entry_path, _named_candidate, extract_named_entries +from .reference_quality import clean_reference_text, markdown_links, visible_markdown_links + + +def _key(value: str) -> str: + value = unicodedata.normalize("NFKC", unquote(value)).casefold() + return re.sub(r"\s+", " ", value).strip(" *#>`") + + +def _scope(content: str, group: dict) -> str: + """Return one declared section, failing closed when a boundary is absent.""" + start = group.get("start_pattern") + end = group.get("end_pattern") + begin = 0 + + def boundary(pattern: str, value: str): + matches = list(re.finditer(pattern, value, re.I | re.M)) + return next( + ( + match + for match in matches + if re.match(r"\s*#{1,6}\s+", value[value.rfind("\n", 0, match.start()) + 1 :]) + ), + matches[0] if matches else None, + ) + + if start: + match = boundary(start, content) + if not match: + return "" + begin = match.start() + finish = len(content) + if end: + match = boundary(end, content[begin:]) + if not match: + return "" + finish = begin + match.start() + return content[begin:finish] + + +def _validate_group(group: dict) -> None: + allowed = { + "id", + "kind", + "minimum_records", + "hosts", + "entry_path", + "entry_identity_pattern", + "identity_query_keys", + "link_base_url", + "record_name_exclusions", + "record_line_pattern", + "start_pattern", + "end_pattern", + "required_fields", + "required_columns", + "identity_column", + "required_literals", + "fixed_values", + "link_column", + "combined_columns", + } + if ( + not isinstance(group, dict) + or set(group) - allowed + or not isinstance(group.get("id"), str) + or not group["id"] + or group.get("kind") + not in { + "named_link", + "name_description", + "table", + "linked_table", + "literals", + "field_report", + } + ): + raise ValueError("Invalid composite record group") + for key in ("start_pattern", "end_pattern", "record_line_pattern"): + if key in group: + if not isinstance(group[key], str) or not group[key]: + raise ValueError("Invalid composite scope pattern") + re.compile(group[key]) + kind = group["kind"] + if "combined_columns" in group and kind != "table": + raise ValueError("Combined columns require a table group") + minimum = group.get("minimum_records") + if kind == "literals": + literals = group.get("required_literals") + if ( + minimum is not None + or not isinstance(literals, list) + or not literals + or any(not isinstance(value, str) or not value for value in literals) + ): + raise ValueError("Invalid composite literal group") + return + if kind == "field_report": + fixed = group.get("fixed_values", {}) + if ( + minimum is not None + or not fields_are_valid(group.get("required_fields", [])) + or not isinstance(fixed, dict) + or any( + not isinstance(key, str) or not key or not isinstance(value, str) or not value + for key, value in fixed.items() + ) + ): + raise ValueError("Invalid composite field report") + return + if type(minimum) is not int or minimum < 1: + raise ValueError("Composite record groups require a positive minimum") + if kind in {"named_link", "linked_table"}: + hosts = group.get("hosts") + if ( + not isinstance(hosts, list) + or not hosts + or any(not isinstance(host, str) or not host for host in hosts) + or not isinstance(group.get("entry_path"), str) + or not group["entry_path"] + ): + raise ValueError("Composite named links require hosts and entry_path") + re.compile(group["entry_path"]) + if pattern := group.get("entry_identity_pattern"): + if re.compile(pattern).groups != 1: + raise ValueError("Composite link identity must capture once") + if kind in {"table", "linked_table"}: + columns = group.get("required_columns") + if ( + not isinstance(columns, list) + or not columns + or any(not isinstance(row, dict) or "aliases" not in row for row in columns) + or not fields_are_valid(columns) + or group.get("identity_column") not in {row["id"] for row in columns} + ): + raise ValueError("Invalid composite table columns") + if kind == "linked_table" and group.get("link_column") not in { + row["id"] for row in columns + }: + raise ValueError("Invalid composite table link column") + combinations = group.get("combined_columns", []) + if not isinstance(combinations, list) or (combinations and kind != "table"): + raise ValueError("Invalid combined table columns") + for combined in combinations: + if (not isinstance(combined, dict) or set(combined) != {"aliases", "pattern"} + or not isinstance(combined["aliases"], list) or not combined["aliases"] + or any(not isinstance(a, str) or not a.strip() for a in combined["aliases"]) + or not isinstance(combined["pattern"], str) or not combined["pattern"]): + raise ValueError("Invalid combined table columns") + pattern = re.compile(combined["pattern"]) + if (len(pattern.groupindex) < 2 or pattern.groups != len(pattern.groupindex) + or not set(pattern.groupindex) <= {row["id"] for row in columns}): + raise ValueError("Combined columns must capture declared fields by name") + fields = group.get("required_fields", []) + if not fields_are_valid(fields): + raise ValueError("Invalid composite required fields") + + +def fields_are_valid(fields: object) -> bool: + allowed_field_keys = {"id", "kind", "aliases", "minimum_chars", "pattern"} + allowed_field_kinds = {"text", "integer", "date_or_time", "price", "area", "version"} + if not isinstance(fields, list) or any( + not isinstance(field, dict) + or set(field) - allowed_field_keys + or not isinstance(field.get("id"), str) + or not field["id"] + or field.get("kind") not in allowed_field_kinds + or ( + "aliases" in field + and ( + not isinstance(field["aliases"], list) + or not field["aliases"] + or any(not isinstance(value, str) or not value for value in field["aliases"]) + ) + ) + or ( + "minimum_chars" in field + and ( + field["kind"] != "text" + or type(field["minimum_chars"]) is not int + or field["minimum_chars"] < 1 + ) + ) + or ("pattern" in field and (not isinstance(field["pattern"], str) or not field["pattern"])) + for field in fields + ): + return False + if len({field["id"] for field in fields}) != len(fields): + return False + for field in fields: + if "pattern" in field: + re.compile(field["pattern"]) + return True + + +def validate_composite_acceptance(contract: dict) -> None: + if ( + not isinstance(contract, dict) + or contract.get("version") != 1 + or set(contract) != {"version", "groups"} + or not isinstance(contract.get("groups"), list) + or not contract["groups"] + ): + raise ValueError("Unsupported composite acceptance contract") + ids = set() + for group in contract["groups"]: + _validate_group(group) + if group["id"] in ids: + raise ValueError("Duplicate composite group id") + ids.add(group["id"]) + + +def _markdown_contexts(content: str) -> list[tuple[object, str]]: + links = markdown_links(content) + contexts = [] + for link in links: + line_start = content.rfind("\n", 0, link.start) + 1 + line_end = content.find("\n", link.end) + line_end = len(content) if line_end < 0 else line_end + line = content[line_start:line_end] + if "|" in line: + context = line + else: + block_end = content.find("\n\n", link.end) + block_end = len(content) if block_end < 0 else block_end + context = content[line_start:block_end] + # A list item owns its indented continuation lines, not the next item. + before = content[line_start : link.start] + if re.fullmatch(r"\s*(?:[-*+] |\d+[.)] )[^\n]*", before): + following = re.search(r"(?m)^\s*(?:[-*+] |\d+[.)] )", content[line_end:block_end]) + if following: + context = content[line_start : line_end + following.start()] + contexts.append((link, clean_reference_text(context))) + return contexts + + +def _link_identity(url: str, group: dict) -> str | None: + parts = urlsplit(url) + if ( + parts.scheme not in {"http", "https"} + or not parts.hostname + or "*" not in group["hosts"] + and parts.hostname not in group["hosts"] + ): + return None + path = _canonical_entry_path(parts.path).rstrip("/") + if not re.fullmatch(group["entry_path"], path): + return None + identity = f"{parts.hostname}::{path}" + if pattern := group.get("entry_identity_pattern"): + match = re.search(pattern, path) + if not match: + return None + identity = f"{parts.hostname}::{match[1]}" + keys = group.get("identity_query_keys", []) + if keys: + values = {key: [] for key in keys} + for key, value in parse_qsl(parts.query, keep_blank_values=True): + if key in values: + values[key].append(value) + if any(len(values[key]) != 1 or not values[key][0].strip() for key in keys): + return None + identity += "?" + "&".join(f"{key}={values[key][0]}" for key in sorted(keys)) + return identity + + +def _html_field_surface(content: str, group: dict, base_url: str) -> tuple[str, str]: + """Keep real DOM ownership while projecting visible links and field text.""" + from bs4 import Comment, NavigableString, Tag + from .dynamic_lists import _scoped_html, visible_html_text + + soup = _scoped_html(content) + identity_cache = {} + def target_identities(node): + if id(node) not in identity_cache: + identity_cache[id(node)] = { + _link_identity(urljoin(group.get('link_base_url', base_url), anchor['href']), group) + for anchor in node.find_all('a', href=True)} - {None} + return identity_cache[id(node)] + + owners = {id(node): node for node in soup.select( + 'article,li,aside,nav,[role="article"],[role="listitem"]')} + # Generic div cards have no semantic tag. Repeated sibling containers with + # their own content and distinct target links supply the same DOM boundary. + for parent in soup.find_all(True): + siblings = parent.find_all('div', recursive=False) + if len(siblings) < 2: + continue + cards = [] + for node in siblings: + identities = target_identities(node) + if identities: + cards.append((node, identities)) + if len(cards) > 1 and len(set().union(*(ids for _, ids in cards))) > 1: + owners.update((id(node), node) for node, _ in cards) + # Lists may represent fields or merely wrap a record title. A nested list + # item is a separate record only when it identifies a narrower target set + # than its enclosing record; otherwise the enclosing card owns its fields. + nested_layout_items = [] + for key, node in owners.items(): + if node.name != 'li' and node.get('role') != 'listitem': + continue + identities = target_identities(node) + enclosing = [ + parent + for parent in node.parents + if id(parent) in owners + and parent.name not in {'aside', 'nav'} + and target_identities(parent) + ] + if enclosing and (not identities or identities == target_identities(enclosing[0])): + nested_layout_items.append(key) + for key in nested_layout_items: + del owners[key] + # A section/header/footer inside an identified record still belongs to + # that record. Outside records, these delimit unrelated page regions. + for node in soup.select('section,header,footer'): + if not any(id(parent) in owners and parent.name not in {'aside','nav'} + for parent in node.parents): + owners[id(node)] = node + # Remove layout-only wrappers only after effective record owners are known. + # This keeps a one-child sibling card as a boundary while making deeper + # presentation wrappers irrelevant to ownership. + for node in reversed(soup.find_all('div')): + elements = [child for child in node.children if isinstance(child, Tag)] + direct_text = any(isinstance(child, NavigableString) and not isinstance(child, Comment) + and str(child).strip() for child in node.children) + if (id(node) not in owners and len(elements) == 1 and not direct_text + and node.get('role') not in {'article','listitem'}): + node.unwrap() + marker = '\ufdd1record-owner-' + while marker in content: + marker += '_' + for index, node in enumerate(owners.values()): + node.insert_before(NavigableString('\n'+marker+'S'+str(index)+'\n')) + node.insert_after(NavigableString('\n'+marker+'E'+str(index)+'\n')) + for node in soup.find_all(['img','br']): + node.replace_with(NavigableString('\ufdd0')) + return visible_html_text(str(soup), preserve_links=True).replace('\ufdd0', '\n\n'), marker + + +def _record_owner_ranges(content: str, marker: str | None): + if marker is None: + return content, [] + parts, starts, ranges, offset, cursor = [], {}, [], 0, 0 + for match in re.finditer(re.escape(marker)+r'([SE])(\d+)', content): + segment = content[cursor:match.start()] + parts.append(segment); offset += len(segment); cursor = match.end() + if match[1] == 'S': + starts[match[2]] = offset + else: + ranges.append((starts.pop(match[2], 0), offset)) + parts.append(content[cursor:]) + clean = ''.join(parts) + ranges.extend((start, len(clean)) for start in starts.values()) + return clean, ranges + + +def _named_link_group(content: str, group: dict, base_url: str, links: list | None, *, ownership_marker: str | None = None) -> dict: + content, owner_ranges = _record_owner_ranges(content, ownership_marker) + standard = { + "version": 1, + "schema": "named_link", + "minimum_records": group["minimum_records"], + "hosts": group["hosts"], + "entry_path": group["entry_path"], + } + for key in ( + "entry_identity_pattern", + "identity_query_keys", + "link_base_url", + "record_name_exclusions", + ): + if key in group: + standard[key] = group[key] + if "*" in group["hosts"]: + entries = [] + for link, _ in _markdown_contexts(content): + destination = urljoin(group.get("link_base_url", base_url), link.destination) + if _link_identity(destination, group) is not None and not link.is_image: + label = visible_markdown_links(link.label).strip() + if label: + entries.append({"url": destination, "title": label}) + else: + entries = extract_named_entries(content, standard, base_url, links) + exclusions = {_key(value) for value in group.get("record_name_exclusions", [])} + entries = [entry for entry in entries if _key(entry["title"]) not in exclusions] + fields = group.get("required_fields", []) + if fields: + + def is_record_start(link) -> bool: + pattern = group.get("record_line_pattern") + if not pattern: + return True + line_start = content.rfind("\n", 0, link.start) + 1 + line_end = content.find("\n", link.end) + line_end = len(content) if line_end < 0 else line_end + return bool(re.search(pattern, content[line_start:line_end], re.I)) + + # A card commonly places its title link, body paragraphs and date on + # separate Markdown blocks. Own everything from the first qualifying + # link for one identity up to the first qualifying link for another + # identity. Repeated title/date links to the same article stay inside + # that record and cannot inflate the count. + source_links = markdown_links(content) + previous_targets = {} + previous_target = None + for source_link in source_links: + previous_targets[source_link.start] = previous_target + source_destination = urljoin(group.get("link_base_url", base_url), source_link.destination) + source_identity = _link_identity(source_destination, group) + if source_identity is not None and not source_link.is_image: + previous_target = (source_identity, source_link.end) + + def previous_scalar(link): + destination = urljoin(group.get("link_base_url", base_url), link.destination) + if link.label.strip() not in {link.destination, destination}: + return None + start = content.rfind("\n", 0, link.start) + 1 + end = content.find("\n", link.end) + end = len(content) if end < 0 else end + if content[start:link.start].strip() or content[link.end:end].strip() or start == 0: + return None + previous_start = content.rfind("\n", 0, start - 1) + 1 + scalar = content[previous_start:start - 1].strip() + if (not scalar or markdown_links(scalar) or re.search(r"<[^>]*>", scalar) + or re.match(r"(?:[#>*]|[-+]\s|\d+[.)]\s|https?://|//)", scalar)): + return None + prior = previous_targets[link.start] + paragraph = content.rfind("\n\n", 0, previous_start) + 2 + if (prior and prior[1] > paragraph and prior[0] != _link_identity(destination, group) + and not (_PRICE.fullmatch(scalar) or _DATE_OR_TIME.fullmatch(scalar) + or _AREA.fullmatch(scalar))): + return None + return previous_start, scalar + + def visible_label(link): + label = link.label + for image in reversed([item for item in markdown_links(label) if item.is_image]): + label = label[:image.start] + label[image.end:] + label = visible_markdown_links(label).strip() + if previous := previous_scalar(link): + scalar = previous[1] + field_like = (_AREA.fullmatch(scalar) or _VERSION.fullmatch(scalar) + or _line_starts_declared_label(scalar, _declared_aliases(fields)) + or any(pattern.fullmatch(scalar) for spec in fields + if (pattern := _field_token_pattern(spec)) is not None)) + if not field_like: + label = scalar + return label + + def name_only(label): + return not (_PRICE.fullmatch(label) or _DATE_OR_TIME.fullmatch(label) + or re.fullmatch(r"(?i)(?:today|tomorrow|\d[\d,.]*|more|details?)", label)) + + allowed_rows = {_link_identity(entry["url"], group): entry for entry in entries + if name_only(entry["title"])} if "*" not in group["hosts"] else {} + # Field cards prove their own ownership through the target identity and + # typed fields. Adjacent status text need not satisfy ordinary list + # prefix rules, but the title must still be real visible link text. + for link in markdown_links(content): + destination = urljoin(group.get("link_base_url", base_url), link.destination) + identity = _link_identity(destination, group) + if identity is None or link.is_image or not is_record_start(link): + continue + label = visible_label(link) + candidate_contract = (standard if "*" not in group["hosts"] else + {**standard, "hosts": [urlsplit(destination).hostname]}) + candidate = _named_candidate(destination, label, candidate_contract, base_url, []) + if candidate and name_only(candidate[1]["title"]) and _key(candidate[1]["title"]) not in exclusions: + allowed_rows.setdefault(identity, candidate[1]) + def owner_for(position): + containing = [(begin, end) for begin, end in owner_ranges + if begin <= position < end] + return min(containing, key=lambda item: item[1] - item[0]) if containing else None + + preferred_titles = {} + for link in markdown_links(content): + destination = urljoin(group.get("link_base_url", base_url), link.destination) + identity = _link_identity(destination, group) + if identity not in allowed_rows or link.is_image or not is_record_start(link): + continue + label = visible_label(link) + field_like = bool( + _PRICE.search(label) + or _DATE_OR_TIME.search(label) + or re.fullmatch(r"(?i)(?:today|tomorrow|\d[\d,.]*|more|details?)", label) + or re.search(r"[A-Za-z][A-Za-z .'-]+,\s*[A-Z]{2}\b$", label) + ) + if (label and not field_like and label not in {link.destination, destination} + and _key(label) not in exclusions): + preferred_titles.setdefault((identity, owner_for(link.start)), label) + candidates = [] + for link in markdown_links(content): + destination = urljoin(group.get("link_base_url", base_url), link.destination) + identity = _link_identity(destination, group) + if identity is None or link.is_image or not is_record_start(link): + continue + entry = allowed_rows.get(identity) + owner = owner_for(link.start) + title = preferred_titles.get( + (identity, owner), + entry["title"] if entry and owner is None else "", + ) + # Even an unnamed target starts a different card. Its fields must + # never complete the preceding named record. + previous = previous_scalar(link) + start = previous[0] if previous else link.start + candidates.append((identity, destination, title, start, link.start, owner)) + owned = {} + # Consecutive links to the same identity belong to one card (for + # example, a title link followed by a linked date). A later duplicate + # after another identity starts a new card and must not enlarge the + # intervening record's field context. + ordered = [] + for candidate in sorted(candidates, key=lambda row: row[3]): + if (not ordered or ordered[-1][0] != candidate[0] + or ordered[-1][5] != candidate[5]): + ordered.append(candidate) + for index, (identity, destination, label, start, link_start, owner) in enumerate(ordered): + line_start = content.rfind("\n", 0, start) + 1 + finish = ordered[index + 1][3] if index + 1 < len(ordered) else len(content) + next_line = content.rfind("\n", 0, finish) + 1 if finish < len(content) else finish + if owner: + begin, end = owner + # The first link can have fields before it in the same card. + first = not any( + begin <= previous[4] < link_start and previous[5] == owner + for previous in ordered[:index] + ) + line_start = begin if first else max(line_start, begin) + next_line = min(next_line, end) + # Exclude unrelated owners rather than discarding the parent's + # own fields on either side of an embedded aside/section. + excluded = sorted((max(begin, line_start), min(end, next_line)) + for begin, end in owner_ranges + if not begin <= link_start < end + and begin < next_line and end > line_start) + pieces, cursor = [], line_start + for begin, end in excluded: + if begin > cursor: + pieces.append(content[cursor:begin]) + cursor = max(cursor, end) + pieces.append(content[cursor:next_line]) + context = "\n\n".join(pieces) + if label and _required_fields(context, label, field_specs=fields): + owned[identity] = {"url": destination, "title": label, "context": context} + records = [row for identity, row in owned.items() if identity in allowed_rows] + else: + records = entries + return { + "count": len(records), + "records": records, + "minimum": group["minimum_records"], + "pass": len(records) >= group["minimum_records"], + } + + +_AMOUNT = r"(?:\d{1,3}(?:[,.\s]\d{3})+|\d+)(?:[.,]\d+)?" +_DATE_OR_TIME = re.compile( + r"(?:\b(?:19|20)\d{2}[-/.]\d{1,2}(?:[-/.]\d{1,2})?\b|" + r"\b\d{1,2}[-/.]\d{1,2}[-/.](?:\d{2}|\d{4})\b|" + r"\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|" + r"Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|" + r"Dec(?:ember)?)\.?\s+\d{1,2}(?:,\s*(?:19|20)\d{2})?\b|" + r"\b\d{1,2}\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|" + r"Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|" + r"Dec(?:ember)?)\.?(?:\s+(?:19|20)\d{2})?\b|" + r"\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)(?:day)?\b|" + r"\b\d{1,2}:\d{2}(?:\s*[ap]m)?\b|" + r"\b(?:an?|\d+)\s+(?:minutes?|hours?|days?)\s+ago\b)", + re.I, +) +_PRICE = re.compile(rf"(?:[$£€¥]\s*{_AMOUNT}|(?:USD|EUR|GBP|CNY)\s*{_AMOUNT})", re.I) +_AREA = re.compile( + rf"\b{_AMOUNT}\s*(?:sq\.?\s*ft|sqft|ft²|m²|square\s+(?:feet|met(?:er|re)s))\b", re.I +) +_VERSION = re.compile( + r"\b(?:v(?:ersion)?\s*)?\d+(?:\.\d+){1,3}(?:[-+._a-z0-9]*)\b|\bcompatib(?:le|ility)\b", re.I +) + + +def _value_matches_spec(value: str, spec: dict, *, whole_cell: bool = False) -> bool: + """Apply the same declared field type rules to card and table values.""" + visible = re.sub(r"\s+", " ", visible_markdown_links(value)).strip() + if not visible or visible in {"-", "—"}: + return False + if pattern := spec.get("pattern"): + if not re.search(pattern, visible, re.I | re.M): + return False + kind = spec["kind"] + if kind == "integer": + pattern = r"\d[\d,.]*" + return bool( + re.fullmatch(pattern, visible) if whole_cell else re.search(rf"\b{pattern}\b", visible) + ) + if kind == "date_or_time": + return bool(_DATE_OR_TIME.search(visible)) + if kind == "price": + return bool(_PRICE.search(visible)) + if kind == "area": + return bool(_AREA.search(visible)) + if kind == "version": + return bool(_VERSION.search(visible)) + minimum = spec.get("minimum_chars", 2) + return len(visible) >= minimum and any(character.isalpha() for character in visible) + + +def _context_lines(context: str, title: str) -> list[str]: + title_key = _key(title) + lines = [] + for raw in clean_reference_text(context).splitlines(): + if re.match(r"^\s*#{1,6}\s+", raw): + continue + line = visible_markdown_links(raw) + line = re.sub(r"(?*|-_") + if not line or _key(line) == title_key or re.fullmatch(r"https?://\S+", line): + continue + # Some native Markdown serializers concatenate venue and city text + # nodes without a separator (for example ``ArenaSeattle, WA``). The + # lowercase-to-uppercase boundary still proves two visible fields. + split = re.search(r"(?<=[a-z])(?=[A-Z][A-Za-z .'-]+,\s*[A-Z]{2}\b)", line) + if split: + lines.extend((line[: split.start()], line[split.start() :])) + continue + # Browser innerText may join adjacent inline field/status nodes. + # Preserve the visible lexical boundary, not a new record/field value. + lines.append(re.sub(r"(?<=[a-z])(?=\d|[A-Z])", " ", line)) + return lines + + +def _field_token_pattern(spec: dict): + """Typed card fields own a distinct value span, not an entire layout line.""" + if spec.get('aliases'): + return None + if spec['kind'] == 'integer' and spec.get('pattern'): + # A number elsewhere in the card cannot prove a numeric labeled field. + return re.compile(rf"\b{_AMOUNT}\s*(?:{spec['pattern']})|(?:{spec['pattern']})", re.I) + return {'price': _PRICE, 'area': _AREA, 'date_or_time': _DATE_OR_TIME, + 'version': _VERSION}.get(spec['kind']) + + +def _declared_aliases(field_specs: list[dict]) -> list[str]: + return sorted( + {alias for spec in field_specs for alias in spec.get("aliases", [])}, + key=len, + reverse=True, + ) + + +def _trimmed_value_span(line: str, start: int, end: int) -> tuple[str, int, int]: + while start < end and line[start] in "| ": + start += 1 + while end > start and line[end - 1] in "| ": + end -= 1 + return line[start:end], start, end + + +def _owned_labeled_values( + line: str, aliases: list[str] +) -> list[tuple[str, str, int, int]]: + """Split one visible line into values owned by its longest declared labels.""" + if not aliases: + return [] + names = "|".join(re.escape(alias) for alias in aliases) + label = rf"(?P