From 5839112c74af448037833324118c7bcb213ddbfe Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:02:24 +0800 Subject: [PATCH 1/5] feat(reporting): bind reports and calls to benchmark editions --- CONTRIBUTING.md | 5 + README.md | 5 + docs/README.zh-CN.md | 4 + docs/benchmarks.md | 48 +++++ docs/report-hosting.md | 7 + evaluation/lexbench-webfetch.json | 2 +- harness/web_fetch/certification.py | 19 ++ harness/web_fetch/published_snapshot.py | 15 +- harness/web_fetch/report.py | 4 + harness/web_fetch/report_build.py | 12 +- harness/web_fetch/report_check.py | 4 + harness/web_fetch/report_version.py | 128 ++++++++++++++ pyproject.toml | 2 +- tests/test_benchmark_release_binding.py | 225 ++++++++++++++++++++++++ tests/test_local_audit.py | 9 + tests/test_run_certified_benchmark.py | 6 + uv.lock | 4 +- 17 files changed, 491 insertions(+), 8 deletions(-) create mode 100644 harness/web_fetch/report_version.py create mode 100644 tests/test_benchmark_release_binding.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 152938b9c..3a6bac756 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,11 @@ Test both accepted and rejected scenarios. Fix scoring rules rather than editing Write documentation in English. Keep the [Chinese README](docs/README.zh-CN.md) aligned when setup or usage changes; other docs do not need duplicate translations. Preserve original task text, quotations and frozen acceptance inputs. Reports follow their track's language settings. +Benchmark changes follow [benchmark editions](docs/benchmarks.md#benchmark-editions). +Keep human-readable editions, immutable dataset/scorer hashes and the package +commit together. A minor scoring correction requires common-edition rescoring; +changed task populations or requests require a major edition. + ## PR and commit titles Use `(): ` for PR titles and new commit subjects. The scope is optional; use the module or feature affected, such as `scoring` or `webfetch`. diff --git a/README.md b/README.md index 586f61ee4..be06ae515 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,11 @@ Install an extra with `uv sync --frozen --extra NAME`. Browser binaries, Docker, ## Read and share a report +Reports retain the selected benchmark edition and exact dataset/scorer hashes. +Package upgrades leave historical results unchanged. See +[benchmark editions](docs/benchmarks.md#benchmark-editions) for patch, minor and +major comparison rules. + | Output | What you will find | | --- | --- | | `report/en/index.html` | English comparisons, task results and evidence links | diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index c5e2ad2a0..7b2bceeb8 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -106,6 +106,10 @@ uv run --frozen browser-harness-eval run --delay 0 ## 查看与分享报告 +报告固定绑定所选评测版本,以及题目和评分源码的精确哈希。升级软件包不会改变历史结果。 +补丁版可以直接比较;评分修正的小版本要统一重评;题目或任务要求改变的大版本分别展示。 +具体规则见[评测版本](benchmarks.md#benchmark-editions)。 + | 输出 | 内容 | | --- | --- | | `report/en/index.html` | 英文比较、任务结果和证据入口 | diff --git a/docs/benchmarks.md b/docs/benchmarks.md index cab5da6cb..71e8d5d50 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -51,6 +51,54 @@ Change task requirements in LexBench, freeze a new dataset/catalog/manifest vers The retained `harness/lexbench/` module concerns historical headless-browser comparisons. It is unrelated to the `lexbench-webfetch` input package and remains an execution/report adapter. +## Benchmark editions + +Use a benchmark version together with exact task/reference and scoring-source +hashes. Keep the package's full Git commit pinned separately. A package update +does not select a new dataset for an existing report. + +| Change | Edition | Result comparison | +| --- | --- | --- | +| Documentation or presentation only | Patch | Direct when execution controls also match | +| Reviewed scoring/reference correction with the same task inputs | Minor | Re-score both saved cohorts under one edition | +| Added/removed cases, changed requests, output scope or routes | Major | Separate full-population results | + +The task package owns `lexbench-webfetch release` and `compare-releases`. +See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/98dd3423a5d179b4662c27beccae0918b94bc2e8/docs/versions.md). +Freeze `benchmark-release.json` beside the compiled inputs before calls. +The certified Web Fetch runner accepts `--benchmark-release`; it also discovers +the sidecar beside the selected dataset. Release creation records the full scorer +commit alongside the independently computed source hash. The runner checks tasks and the active scorer +before external work and retains the binding in execution metadata. The original +call edition remains in benchmark-input-release.json; a report may select a newer +scoring edition after rescoring, retaining both files. + +An instance profile can select an edition explicitly: + +```json +{ + "benchmark_release": { + "path": "benchmark-release.json", + "sha256": "" + } +} +``` + +Paths are relative to the external profile. The existing build copies this +edition into the report manifest and package and displays a link beside the +home-page scope. Checks reject mismatched populations, acceptance rules, +scoring sources, missing sidecars and incorrect version labels. Historical +hash-bound reports remain usable without inventing a version label. An +explicit historical edition preserves the saved scorer identity during a +presentation rebuild; it does not run the current scorer over old results. + +Reuse unchanged saved outputs for a minor correction, including prior +successes, then recompute scores under one edition. Use the existing revision +receipts; do not overwrite the older report. Major editions may retain +diagnostic task-overlap analyses, but their complete throughputs/pass rates +are separate measurements. Latency still requires matching runtime controls +and the existing same-window comparison contract. + ## Report instances Each report track uses its maintained build, check and package entry points. diff --git a/docs/report-hosting.md b/docs/report-hosting.md index 848f1401e..2a539bae4 100644 --- a/docs/report-hosting.md +++ b/docs/report-hosting.md @@ -11,6 +11,13 @@ Make targets. A new instance reuses the same commands with different inputs. Historical full-cohort formats use an explicit `--authority-root` bundle that retains their original relative paths and hash-bound profile bytes. +An optional `benchmark_release` path/hash binding in the external profile fixes +the selected benchmark edition. Builds and packages retain +`benchmark-release.json`; presentation rebuilds keep its dataset and scoring +identity. See [benchmark editions](benchmarks.md#benchmark-editions) for the +version and comparison rules. A newer task package must not silently update +an older report. + ## Report languages The Web Fetch CLI defaults to English. `run --language zh` generates Chinese only; `report --language en` rebuilds both editions and opens English. A rebuild without `--language` keeps the saved choice, including Chinese-only reports created by older versions. diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index 56f98bcff..8f9d07116 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "6bd263ed98030a883e66ac75a47ac4e7fbae4ad1", + "revision": "98dd3423a5d179b4662c27beccae0918b94bc2e8", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index 9a46ab084..617d68045 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -140,6 +140,8 @@ def main(argv: list[str] | None = None) -> None: return parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset", type=Path, default=ROOT / "evaluation/dataset.json") + parser.add_argument("--benchmark-release", type=Path, + help="Frozen benchmark edition; checked before any external calls") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--work-root", type=Path, required=True) parser.add_argument("--providers", required=True) @@ -215,6 +217,16 @@ def main(argv: list[str] | None = None) -> None: publication_state="diagnostic", unqualified_reference_cases=sorted(unqualified), ) + release_raw = None + release_path = args.benchmark_release or args.dataset.parent / 'benchmark-release.json' + if args.benchmark_release or release_path.exists(): + from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 + from lexbench_webfetch.versions import verify_release + if release_path.is_symlink(): + parser.error('Benchmark release rejects symlinks') + release_raw = release_path.read_bytes() + verify_release(json.loads(release_raw), dataset, exact_dataset=False, + scoring_source_sha256=SCORING_SOURCE_SHA256) # Reject malformed execution contracts before doing any external captures. from harness.web_fetch.reference_lifecycle import ( requires_browser_observation, @@ -320,6 +332,13 @@ def main(argv: list[str] | None = None) -> None: if args.provider_config: benchmark.extend(["--provider-config", str(args.provider_config)]) run(benchmark_runner.main, benchmark) + if release_raw is not None: + (args.output / 'benchmark-input-release.json').write_bytes(release_raw) + execution_path = args.output / 'execution.json' + execution = json.loads(execution_path.read_text()) + execution['aggregate']['benchmark_release'] = { + 'path': 'benchmark-input-release.json', 'sha256': hashlib.sha256(release_raw).hexdigest()} + execution_path.write_text(json.dumps(execution, ensure_ascii=False, indent=2) + '\n') # Re-capture HTTP evidence after every provider call has completed, then # choose browser fallbacks from that post-call evidence. Reusing the diff --git a/harness/web_fetch/published_snapshot.py b/harness/web_fetch/published_snapshot.py index 2ece93fa0..2cef0b001 100644 --- a/harness/web_fetch/published_snapshot.py +++ b/harness/web_fetch/published_snapshot.py @@ -64,6 +64,11 @@ def validate_snapshot_audit(source, profile_path): records = load_jsonl(inputs['results']) receipt = json.loads(inputs['receipt'].read_text()) audit = json.loads(inputs['review'].read_text()) + scoring_source = SCORING_SOURCE_SHA256 + if config.get('benchmark_release'): + from .report_version import load_release + _, release = load_release(source, dataset, records, profile=profile_path) + scoring_source = release['scoring_source_sha256'] old_cases = {row['id']: row for row in baseline['cases']} cases = {row['id']: row for row in dataset['cases']} old_rows = {row['attempt_id']: row for row in previous} @@ -75,7 +80,7 @@ def validate_snapshot_audit(source, profile_path): if (receipt.get('kind') != 'published_snapshot_rescore' or receipt.get('provider_calls_added') != 0 or receipt.get('input_dataset_sha256') != file_sha256(source / 'dataset.json') or receipt.get('input_results_sha256') != file_sha256(source / 'results.jsonl') - or receipt.get('scoring_source_sha256') != SCORING_SOURCE_SHA256 + or receipt.get('scoring_source_sha256') != scoring_source or receipt.get('corrected_dataset_sha256') != canonical_json_sha256(dataset)): raise ValueError('Snapshot audit does not bind the saved calls and current scorer') correction_provenance(**config['correction_provenance']) @@ -96,7 +101,7 @@ def validate_snapshot_audit(source, profile_path): if (view['path'] != old['artifact_path'] or view['published_sha256'] != file_sha256(artifact) or view['original_sha256'] != old['metrics']['content_sha256'] or row['metrics']['content_sha256'] != view['scored_sha256'] - or row['metrics']['scoring_source_sha256'] != SCORING_SOURCE_SHA256 + or row['metrics']['scoring_source_sha256'] != scoring_source or row.get('scoring', {}).get('artifact_view') != view['view']): raise ValueError('Snapshot audit artifact identity changed: ' + identifier) expected = {'original': view['original_sha256'], 'recovered_original': view['original_sha256'], @@ -212,6 +217,8 @@ def build_snapshot_audit(source, output, *, profile): (staging / 'dataset.json').write_text(json.dumps(dataset, ensure_ascii=False, indent=2)+'\n') shutil.copy2(inputs['results'], staging / 'results.jsonl') batches = json.loads((staging / 'execution.json').read_text())['batches'] + from .report_version import bind_release + release_binding = bind_release(source, staging, dataset, records, profile=profile) generate(staging, dataset, records, batches, authored_interpretation=True, frozen_scores=True) write_frozen_result_summary(staging, dataset, records) (staging / '.generated-report').write_text('webfetch\n') @@ -221,6 +228,8 @@ def build_snapshot_audit(source, output, *, profile): 'results_sha256':file_sha256(staging/'results.jsonl'), 'generated_languages':list(generated_languages(staging)), 'score_audit':comparison['score_audit'], 'profile_sha256':file_sha256(profile)} + if release_binding: + manifest['benchmark_release'] = release_binding (staging/'report-manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n') check_snapshot_audit(staging) _install_report_tree(staging, output) @@ -241,6 +250,8 @@ def check_snapshot_audit(root): raise ValueError('Snapshot audit input changed: '+name) dataset = json.loads((root/'dataset.json').read_text()) rows = load_jsonl(root/'results.jsonl') + from .report_version import check_release + check_release(root, dataset, rows) if dataset.get('definition_revision'): revision_path=_bound(root,dataset['definition_revision']) historical=root/'audit/history/dataset.json' diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index f96ae5b93..d9e62a1ef 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -2215,6 +2215,10 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english description=f'{len(cases)} 个案例 · {len(names)} 种方法 · {track_label} 类任务。' if local_pass_rate_ranking(dataset): description=model_text(f'{len(cases)} 个案例 · {len(names)} 种方法 · 按本轮任务通过率排序。',f'{len(cases)} cases · {len(names)} methods · Ordered by task pass rate for this sample.') + release_path=out/'benchmark-release.json' + if release_path.is_file(): + release=json.loads(release_path.read_text()) + description+=' · '+esc(release['benchmark']+' v'+release['version'])+'' home='
'+hero('Web Fetch 评测报告',description) from .report_explainer import render_explainer TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else '本报告的任务范围与验收规则' diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index c0765fbcb..84766f889 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -250,7 +250,7 @@ def _install_report_tree(staging: Path, output: Path): def _build_frozen_presentation(source, output, presentation_binding, *, include_english, language, authored_interpretation, - show_local_status_banners): + show_local_status_banners, profile_path=None): """Render a bound historical report using saved scores and public evidence.""" from harness.report_languages import select_report_language from .safety import evidence_tree_identity, copy_evidence_file @@ -291,6 +291,10 @@ def _build_frozen_presentation(source, output, presentation_binding, *, if relative in before: metadata["public_" + kind + "_path"] = relative metadata["public_" + kind + "_sha256"] = before[relative] + from .report_version import bind_release + release_binding = bind_release(source, staging, dataset, records, profile=profile_path) + if release_binding: + manifest['benchmark_release'] = release_binding generate(staging, dataset, records, batches, include_english=include_english, authored_interpretation=authored_interpretation, show_local_status_banners=show_local_status_banners, frozen_scores=True) @@ -380,7 +384,7 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, return _build_frozen_presentation( source, output, presentation_binding, include_english=include_english, language=language, authored_interpretation=authored_interpretation, - show_local_status_banners=show_local_status_banners) + show_local_status_banners=show_local_status_banners, profile_path=profile_path) dataset_probe = json.loads(contained_path(source, 'dataset.json').read_text()) diagnostic_preview = (diagnostic_preview and dataset_probe.get('comparison', {}).get('kind') == 'web_fetch_diagnostic') @@ -771,6 +775,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, }, ensure_ascii=False, indent=2)) from .report_reading import copy_archived_reading copy_archived_reading(source,staging) + from .report_version import bind_release + release_binding = bind_release(source, staging, dataset, records, profile=profile_path) generate(staging,dataset,records,manifests,include_english=include_english, authored_interpretation=authored_interpretation, show_local_status_banners=show_local_status_banners) @@ -782,6 +788,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, 'dataset_sha256': file_sha256(staging/'dataset.json'), 'results_sha256': file_sha256(staging/'results.jsonl'), 'generated_languages': generated_languages(staging)} + if release_binding: + template_manifest['benchmark_release'] = release_binding if dataset.get('comparison', {}).get('kind') == 'uniform_native_local_measurement': template_manifest.update(evidence_state='local_measurement', certified=False, languages=template_manifest['generated_languages'], provider_calls_added=0) diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index c895e0914..4e007e15b 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -300,6 +300,10 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres profile_path = ( (repository / profile).resolve() if not profile.is_absolute() else profile.resolve() ) + if (root / 'dataset.json').is_file() and (root / 'results.jsonl').is_file(): + from .report_version import check_release + check_release(root, json.loads((root / 'dataset.json').read_text()), + load_jsonl(root / 'results.jsonl'), profile=profile_path) if (root / 'report-manifest.json').is_file() and json.loads((root / 'report-manifest.json').read_text()).get('evidence_state') == 'published_snapshot_score_audit': from .published_snapshot import check_snapshot_audit if profile_path and file_sha256(profile_path) != json.loads((root / 'report-manifest.json').read_text()).get('profile_sha256'): diff --git a/harness/web_fetch/report_version.py b/harness/web_fetch/report_version.py new file mode 100644 index 000000000..42aba46cb --- /dev/null +++ b/harness/web_fetch/report_version.py @@ -0,0 +1,128 @@ +"""Carry an explicitly selected benchmark edition through existing reports.""" + +import hashlib +import json +from pathlib import Path + +from lexbench_webfetch.paths import contained_path +from lexbench_webfetch.versions import verify_release + +FILENAME = "benchmark-release.json" + + +def _input_release(root): + execution = root / "execution.json" + if not execution.is_file(): + return None + binding = json.loads(execution.read_text()).get("aggregate", {}).get("benchmark_release") + if binding is not None: + path = contained_path(root, binding["path"], reject_symlinks=True) + if hashlib.sha256(path.read_bytes()).hexdigest() != binding.get("sha256"): + raise ValueError("Measured benchmark input release changed") + return binding + + +def _profile_binding(profile): + if profile is None: + return None + return json.loads(Path(profile).read_text()).get("benchmark_release") + + +def _verify_scores(release, dataset, records): + verify_release(release, dataset, exact_dataset=False) + if not records or any( + ( + row.get("metrics", {}).get("scoring_source_sha256") != release["scoring_source_sha256"] + or row.get("scoring", {}).get("source_sha256", release["scoring_source_sha256"]) + != release["scoring_source_sha256"] + ) + for row in records + ): + raise ValueError("Benchmark release differs from the saved scoring sources; rescore first") + + +def load_release(source, dataset, records, *, profile=None): + """Validate the profile's edition, otherwise the measured input's edition.""" + binding = _profile_binding(profile) + if binding is not None: + selected = contained_path(Path(profile).parent, binding["path"], reject_symlinks=True) + else: + selected = contained_path(source, FILENAME, reject_symlinks=True) + if not selected.exists(): + measured = _input_release(source) + if measured is None: + return None + selected = contained_path(source, measured["path"], reject_symlinks=True) + raw = selected.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if binding is not None and digest != binding.get("sha256"): + raise ValueError("Profile benchmark release changed") + release = json.loads(raw) + _verify_scores(release, dataset, records) + return raw, release + + +def bind_release(source, output, dataset, records, *, profile=None): + """Carry the validated edition into the generated report.""" + selected = load_release(source, dataset, records, profile=profile) + if selected is None: + return None + raw, release = selected + digest = hashlib.sha256(raw).hexdigest() + target = output / FILENAME + target.write_bytes(raw) + measured = _input_release(source) + if measured is not None: + original = contained_path(source, measured["path"], reject_symlinks=True) + archived = contained_path(output, measured["path"], reject_symlinks=True) + archived.parent.mkdir(parents=True, exist_ok=True) + archived.write_bytes(original.read_bytes()) + return { + "path": FILENAME, + "sha256": digest, + "benchmark": release["benchmark"], + "version": release["version"], + } + + +def check_release(root, dataset, records, *, profile=None): + """Check the edition, rendered label and scoring identity without rescoring.""" + from bs4 import BeautifulSoup + + manifest_path = root / "report-manifest.json" + manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {} + binding = manifest.get("benchmark_release") + expected = _profile_binding(profile) + measured = _input_release(root) + release_path = contained_path(root, FILENAME, reject_symlinks=True) + if binding is None: + if expected is not None or release_path.exists() or measured is not None: + raise ValueError("Report lost its benchmark release binding") + return + if binding.get("path") != FILENAME or not release_path.is_file(): + raise ValueError("Report benchmark release is missing") + raw = release_path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != binding.get("sha256") or ( + expected is not None and digest != expected.get("sha256") + ): + raise ValueError("Report benchmark release changed") + release = json.loads(raw) + _verify_scores(release, dataset, records) + if any(binding.get(key) != release[key] for key in ("benchmark", "version")): + raise ValueError("Report benchmark version label differs from its release") + for route in ("index.html", "en/index.html"): + path = root / route + if not path.is_file(): + continue + page = BeautifulSoup(path.read_text(), "html.parser") + if page.select_one("[data-english-not-generated]"): + continue + label = page.select_one("[data-benchmark-release]") + if ( + label is None + or label.get("href") != ("../" if route.startswith("en/") else "") + FILENAME + or label.get("data-benchmark-release") != release["version"] + or label.get_text(" ", strip=True) != release["benchmark"] + " v" + release["version"] + ): + raise ValueError("Report lost its benchmark version label: " + route) diff --git a/pyproject.toml b/pyproject.toml index 687aa557f..ebe908eee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@6bd263ed98030a883e66ac75a47ac4e7fbae4ad1", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@98dd3423a5d179b4662c27beccae0918b94bc2e8", "beautifulsoup4>=4.12,<5", "fastcdc==1.7.0", ] diff --git a/tests/test_benchmark_release_binding.py b/tests/test_benchmark_release_binding.py new file mode 100644 index 000000000..2dc6d5c8c --- /dev/null +++ b/tests/test_benchmark_release_binding.py @@ -0,0 +1,225 @@ +"""Reports preserve an edition and reject mismatched tasks, scorers or labels.""" + +import json + +import pytest +from lexbench_webfetch.versions import freeze_release + +from harness.record_io import file_sha256 +from harness.web_fetch.published_snapshot import build_snapshot_audit, check_snapshot_audit +from harness.web_fetch.report_version import bind_release, check_release +from tests.test_published_snapshot_report import example + + +def configured(tmp_path): + source, profile = example(tmp_path) + data = json.loads(profile.read_text()) + dataset = profile.parent / data["inputs"]["dataset"]["path"] + release = tmp_path / "edition.json" + freeze_release(dataset, release, benchmark="sample-benchmark", version="1.0.0") + data["benchmark_release"] = {"path": release.name, "sha256": file_sha256(release)} + profile.write_text(json.dumps(data)) + return source, profile, release + + +def test_versioned_snapshot_uses_existing_build_check_and_package(tmp_path): + source, profile, release = configured(tmp_path) + before = {p.relative_to(source): p.read_bytes() for p in source.rglob("*") if p.is_file()} + output = tmp_path / "report" + build_snapshot_audit(source, output, profile=profile) + assert (output / "benchmark-release.json").read_bytes() == release.read_bytes() + assert 'data-benchmark-release="1.0.0"' in (output / "index.html").read_text() + assert "2 saved calls" in check_snapshot_audit(output) + dataset = json.loads((output / "dataset.json").read_text()) + rows = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + check_release(output, dataset, rows, profile=profile) + from harness.web_fetch.publication_check import check + from harness.web_fetch.report_package import build + + archive = tmp_path / "report.zip" + build(output, archive, snapshot_audit=True) + check(archive, output, snapshot_audit=True) + assert before == { + p.relative_to(source): p.read_bytes() for p in source.rglob("*") if p.is_file() + } + + +@pytest.mark.parametrize( + "defect", ["missing-file", "wrong-version", "removed-label", "wrong-scorer", "dropped-binding"] +) +def test_report_gate_rejects_release_drift(tmp_path, defect): + source, profile, _ = configured(tmp_path) + output = tmp_path / "report" + build_snapshot_audit(source, output, profile=profile) + if defect == "missing-file": + (output / "benchmark-release.json").unlink() + elif defect in {"wrong-version", "removed-label"}: + page = output / "index.html" + old = 'data-benchmark-release="1.0.0"' + page.write_text( + page.read_text().replace( + old, + 'data-benchmark-release="2.0.0"' + if defect == "wrong-version" + else 'data-removed="1.0.0"', + ) + ) + elif defect == "wrong-scorer": + rows = output / "results.jsonl" + records = [json.loads(line) for line in rows.read_text().splitlines()] + records[0]["metrics"]["scoring_source_sha256"] = "0" * 64 + rows.write_text("".join(json.dumps(row) + "\n" for row in records)) + manifest = output / "report-manifest.json" + data = json.loads(manifest.read_text()) + data["results_sha256"] = file_sha256(rows) + manifest.write_text(json.dumps(data)) + else: + manifest = output / "report-manifest.json" + data = json.loads(manifest.read_text()) + del data["benchmark_release"] + manifest.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="[Bb]enchmark|scoring sources"): + check_snapshot_audit(output) + + +def test_wrong_task_edition_is_rejected_before_rendering(tmp_path): + source, profile, release = configured(tmp_path) + data = json.loads(profile.read_text()) + dataset = json.loads((profile.parent / data["inputs"]["dataset"]["path"]).read_text()) + dataset["cases"][0]["task_prompt"] = "Return a different scope." + records = [ + json.loads(line) + for line in (profile.parent / data["inputs"]["results"]["path"]).read_text().splitlines() + ] + with pytest.raises(ValueError, match="dataset"): + bind_release(source, tmp_path, dataset, records, profile=profile) + assert release.is_file() + + +def test_legacy_hash_bound_reports_do_not_acquire_an_invented_version(tmp_path): + source, profile = example(tmp_path) + output = tmp_path / "report" + build_snapshot_audit(source, output, profile=profile) + assert not (output / "benchmark-release.json").exists() + assert "data-benchmark-release" not in (output / "index.html").read_text() + assert "2 saved calls" in check_snapshot_audit(output) + + +def test_wrong_edition_stops_before_capture_or_provider_calls(tmp_path, monkeypatch): + from lexbench_webfetch.scoring import SCORING_SOURCE_SHA256 + + from harness.web_fetch.certification import main + + source = tmp_path / "dataset.json" + data = { + "cases": [ + { + "id": "sample", + "url": "https://example.test/", + "task_prompt": "Return the complete article.", + } + ] + } + source.write_text(json.dumps(data)) + edition = tmp_path / "benchmark-release.json" + freeze_release(source, edition, benchmark="sample-benchmark", version="1.0.0") + data["cases"][0]["task_prompt"] = "Return only its title." + source.write_text(json.dumps(data)) + calls = [] + monkeypatch.setattr("harness.web_fetch.certification.run", lambda *args: calls.append(args)) + with pytest.raises(ValueError, match="dataset"): + main( + [ + "--dataset", + str(source), + "--output", + str(tmp_path / "run"), + "--work-root", + str(tmp_path / "work"), + "--providers", + "fixture", + "--case-ids", + "sample", + ] + ) + assert not calls + assert not (tmp_path / "work").exists() + assert SCORING_SOURCE_SHA256 + + +def test_explicit_old_edition_preserves_historical_scores_with_new_renderer(tmp_path): + source, profile, edition = configured(tmp_path) + data = json.loads(profile.read_text()) + old_source = "a" * 64 + result_path = profile.parent / data["inputs"]["results"]["path"] + rows = [json.loads(line) for line in result_path.read_text().splitlines()] + for row in rows: + row["metrics"]["scoring_source_sha256"] = old_source + row.setdefault("scoring", {})["source_sha256"] = old_source + result_path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + data["inputs"]["results"]["sha256"] = file_sha256(result_path) + receipt_path = profile.parent / data["inputs"]["receipt"]["path"] + receipt = json.loads(receipt_path.read_text()) + receipt["scoring_source_sha256"] = old_source + receipt_path.write_text(json.dumps(receipt)) + data["inputs"]["receipt"]["sha256"] = file_sha256(receipt_path) + release = json.loads(edition.read_text()) + release["scoring_source_sha256"] = old_source + edition.write_text(json.dumps(release)) + data["benchmark_release"]["sha256"] = file_sha256(edition) + from harness.record_io import canonical_json_sha256 + from harness.web_fetch.report_explainer import _headless_article_input + + dataset_path = profile.parent / data["inputs"]["dataset"]["path"] + cases = {case["id"]: case for case in json.loads(dataset_path.read_text())["cases"]} + article_path = profile.parent / data["inputs"]["interpretation"]["path"] + article = json.loads(article_path.read_text()) + article["evidence_sha256"] = canonical_json_sha256( + _headless_article_input(cases, rows, ["chromium"]) + ) + article_path.write_text(json.dumps(article)) + data["inputs"]["interpretation"]["sha256"] = file_sha256(article_path) + profile.write_text(json.dumps(data)) + output = tmp_path / "historical-report" + build_snapshot_audit(source, output, profile=profile) + assert (output / "results.jsonl").read_bytes() == result_path.read_bytes() + assert "2 saved calls" in check_snapshot_audit(output) + + +def test_rescoring_edition_keeps_the_original_call_edition(tmp_path): + from lexbench_webfetch.versions import create_release + + source, profile, edition = configured(tmp_path) + data = json.loads(profile.read_text()) + dataset = json.loads((profile.parent / data["inputs"]["dataset"]["path"]).read_text()) + rows = [ + json.loads(line) + for line in (profile.parent / data["inputs"]["results"]["path"]).read_text().splitlines() + ] + original = create_release( + dataset, benchmark="sample-benchmark", version="1.0.0", scoring_source_sha256="f" * 64 + ) + original_path = source / "benchmark-input-release.json" + original_path.write_text(json.dumps(original)) + execution = json.loads((source / "execution.json").read_text()) + execution["aggregate"] = { + "benchmark_release": {"path": original_path.name, "sha256": file_sha256(original_path)} + } + (source / "execution.json").write_text(json.dumps(execution)) + corrected = create_release( + dataset, + benchmark="sample-benchmark", + version="1.1.0", + previous=original, + change_note="Correct the saved-response scorer", + ) + edition.write_text(json.dumps(corrected)) + data["benchmark_release"]["sha256"] = file_sha256(edition) + profile.write_text(json.dumps(data)) + output = tmp_path / "revised" + output.mkdir() + selected = bind_release(source, output, dataset, rows, profile=profile) + assert selected["version"] == "1.1.0" + assert json.loads((output / "benchmark-release.json").read_text()) == corrected + assert (output / original_path.name).read_bytes() == original_path.read_bytes() + assert json.loads(original_path.read_text())["version"] == "1.0.0" diff --git a/tests/test_local_audit.py b/tests/test_local_audit.py index a837e7f83..67346246b 100644 --- a/tests/test_local_audit.py +++ b/tests/test_local_audit.py @@ -285,6 +285,9 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp _write(contract_path, contract) dataset["comparison"]["local_run_contract_sha256"] = _digest(contract_path) _write(evidence / "dataset.json", dataset) + from lexbench_webfetch.versions import freeze_release + freeze_release(evidence / "dataset.json", evidence / "benchmark-release.json", + benchmark="local-fixture", version="1.0.0") monkeypatch.setattr(local_execution_contract, "ROOT", tmp_path / "repo") assert local_audit.require_local_audit(evidence, dataset, records) is None changed_contract = {**contract, 'timeout_seconds': timeout + 1} @@ -293,6 +296,8 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp evidence, dataset, records, changed_contract, {'batches': [batch]}) output = tmp_path / "report" build_report(evidence, output, local_audit=True, include_english=include_english) + from harness.web_fetch.report_version import check_release + check_release(output, json.loads((output / "dataset.json").read_text()), records) home = (output / "index.html").read_text() assert "本地未认证诊断快照" in home assert "查看调用链问题和日志留存范围" not in home @@ -345,6 +350,10 @@ def test_new_local_matrix_is_complete_and_keeps_logs_without_claiming_review(tmp (output / "results.jsonl").write_text("".join(json.dumps(row) + "\n" for row in frozen_rows)) manifest = json.loads((output / "report-manifest.json").read_text()) manifest["results_sha256"] = _digest(output / "results.jsonl") + historical_edition = json.loads((output / "benchmark-release.json").read_text()) + historical_edition["scoring_source_sha256"] = historical_scorer + _write(output / "benchmark-release.json", historical_edition) + manifest["benchmark_release"]["sha256"] = _digest(output / "benchmark-release.json") manifest["template"] = {"renderer": "historical-template"} _write(output / "report-manifest.json", manifest) binding = {"comparison_kind": "uniform_native_local_measurement", diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py index bde4e5353..6c768064b 100644 --- a/tests/test_run_certified_benchmark.py +++ b/tests/test_run_certified_benchmark.py @@ -148,6 +148,9 @@ def test_frozen_contract_execution_needs_no_historical_qualification_files(self) 'evidence_sha256': 'b' * 64}} dataset.write_text(json.dumps({'tool': 'web_fetch', 'comparison': {}, 'cases': [case, fixed, resource]})) original = dataset.read_bytes() + from lexbench_webfetch.versions import freeze_release + edition = root / 'benchmark-release.json' + freeze_release(dataset, edition, benchmark='frozen-example', version='1.0.0') config = root / 'providers.json' config.write_text(json.dumps({'providers': {'fixture': {'type': 'fixture'}}})) (root / 'evaluation/list-references').mkdir(parents=True) @@ -175,6 +178,9 @@ def execute(stage, arguments): measured = json.loads((output / 'dataset.json').read_text()) self.assertEqual(measured['cases'], [case, fixed, resource]) self.assertEqual(dataset.read_bytes(), original) + self.assertEqual((output / 'benchmark-input-release.json').read_bytes(), edition.read_bytes()) + release_binding = json.loads((output / 'execution.json').read_text())['aggregate']['benchmark_release'] + self.assertEqual(release_binding['sha256'], hashlib.sha256(edition.read_bytes()).hexdigest()) self.assertEqual(measured['comparison']['source_reference_policy'], 'frozen_benchmark_inputs_with_explicit_live_exact_checks') batches = json.loads((output / 'execution.json').read_text())['batches'] diff --git a/uv.lock b/uv.lock index 2cc83e2d8..789cc6a0d 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=6bd263ed98030a883e66ac75a47ac4e7fbae4ad1" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=98dd3423a5d179b4662c27beccae0918b94bc2e8" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1933,7 +1933,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" version = "0.3.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=6bd263ed98030a883e66ac75a47ac4e7fbae4ad1#6bd263ed98030a883e66ac75a47ac4e7fbae4ad1" } +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=98dd3423a5d179b4662c27beccae0918b94bc2e8#98dd3423a5d179b4662c27beccae0918b94bc2e8" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" }, From 69829cec602e491c25ff0a1f7952d70e7b1bf270 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:02:20 +0800 Subject: [PATCH 2/5] build(deps): pin benchmark versioning after main integration --- evaluation/lexbench-webfetch.json | 2 +- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index 8f9d07116..005ac5007 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "98dd3423a5d179b4662c27beccae0918b94bc2e8", + "revision": "04c2b99e0f86aa0264976ad4159bc4e45b2b10ab", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" diff --git a/pyproject.toml b/pyproject.toml index ebe908eee..b05922db2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@98dd3423a5d179b4662c27beccae0918b94bc2e8", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@04c2b99e0f86aa0264976ad4159bc4e45b2b10ab", "beautifulsoup4>=4.12,<5", "fastcdc==1.7.0", ] diff --git a/uv.lock b/uv.lock index 789cc6a0d..1d046ca88 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=98dd3423a5d179b4662c27beccae0918b94bc2e8" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=04c2b99e0f86aa0264976ad4159bc4e45b2b10ab" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1933,7 +1933,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" version = "0.3.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=98dd3423a5d179b4662c27beccae0918b94bc2e8#98dd3423a5d179b4662c27beccae0918b94bc2e8" } +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=04c2b99e0f86aa0264976ad4159bc4e45b2b10ab#04c2b99e0f86aa0264976ad4159bc4e45b2b10ab" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" }, From 8086c561ebb087e3fbf7ff100baeb143d7d72f22 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:07:31 +0800 Subject: [PATCH 3/5] build(deps): bind the published benchmark catalog edition --- evaluation/lexbench-webfetch.json | 2 +- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index 005ac5007..af2a852a3 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "04c2b99e0f86aa0264976ad4159bc4e45b2b10ab", + "revision": "56aca0cb9b9b9eef9995f0168e0504a72e309cd1", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" diff --git a/pyproject.toml b/pyproject.toml index b05922db2..0d9114012 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@04c2b99e0f86aa0264976ad4159bc4e45b2b10ab", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@56aca0cb9b9b9eef9995f0168e0504a72e309cd1", "beautifulsoup4>=4.12,<5", "fastcdc==1.7.0", ] diff --git a/uv.lock b/uv.lock index 1d046ca88..30b9e9a1b 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=04c2b99e0f86aa0264976ad4159bc4e45b2b10ab" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=56aca0cb9b9b9eef9995f0168e0504a72e309cd1" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1933,7 +1933,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" version = "0.3.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=04c2b99e0f86aa0264976ad4159bc4e45b2b10ab#04c2b99e0f86aa0264976ad4159bc4e45b2b10ab" } +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=56aca0cb9b9b9eef9995f0168e0504a72e309cd1#56aca0cb9b9b9eef9995f0168e0504a72e309cd1" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" }, From 4221ea42adf369f73b2d69c8d7b8cecf40a103a3 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:11:19 +0800 Subject: [PATCH 4/5] build(deps): hash-bind the published benchmark release manifest --- evaluation/lexbench-webfetch.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index af2a852a3..a50f6c973 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -80,6 +80,7 @@ "evaluation/risk-control/source-inputs.json": "77c5a4b62c07d59ed57154a833e85f9e7a97bc739730f51cd4ca3a698edb9258", "evaluation/scoring-regressions.json": "432ddabe23d045d0ce4c23aef69ca3337ba4d240fe3c2061932cee26b7dc00dc", "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708", - "annotation-revisions/serialization-20261021.json": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a" + "annotation-revisions/serialization-20261021.json": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a", + "benchmark-release.json": "75fde8db11bb8b3b8b1460fea0e78385bd2a0bdddecc84fbae8287df5d25e879" } } From c32adca4710b101415f4368a544e46d54b8615e9 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:46:11 +0800 Subject: [PATCH 5/5] fix(reporting): preserve and enforce benchmark edition bindings --- docs/benchmarks.md | 6 ++- evaluation/lexbench-webfetch.json | 4 +- harness/web_fetch/report.py | 2 +- harness/web_fetch/report_build.py | 2 + harness/web_fetch/report_package.py | 8 ++-- harness/web_fetch/report_version.py | 21 ++++++++-- pyproject.toml | 2 +- tests/test_external_local_report.py | 57 +++++++++++++++++++++++++++ tests/test_webfetch_report_package.py | 51 ++++++++++++++++++++++++ uv.lock | 4 +- 10 files changed, 143 insertions(+), 14 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 71e8d5d50..113e435ff 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -57,6 +57,10 @@ Use a benchmark version together with exact task/reference and scoring-source hashes. Keep the package's full Git commit pinned separately. A package update does not select a new dataset for an existing report. +Use the same `benchmark-vMAJOR.MINOR.PATCH` tag prefix in both repositories. +Reports reference their benchmark edition; report rebuilds use the hosting +platform's version ID rather than a separate semantic-version sequence. + | Change | Edition | Result comparison | | --- | --- | --- | | Documentation or presentation only | Patch | Direct when execution controls also match | @@ -64,7 +68,7 @@ does not select a new dataset for an existing report. | Added/removed cases, changed requests, output scope or routes | Major | Separate full-population results | The task package owns `lexbench-webfetch release` and `compare-releases`. -See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/98dd3423a5d179b4662c27beccae0918b94bc2e8/docs/versions.md). +See its [version contract](https://github.com/lexmount/lexbench-webfetch/blob/benchmark-v2.0.1/docs/versions.md). Freeze `benchmark-release.json` beside the compiled inputs before calls. The certified Web Fetch runner accepts `--benchmark-release`; it also discovers the sidecar beside the selected dataset. Release creation records the full scorer diff --git a/evaluation/lexbench-webfetch.json b/evaluation/lexbench-webfetch.json index a50f6c973..14379fbb0 100644 --- a/evaluation/lexbench-webfetch.json +++ b/evaluation/lexbench-webfetch.json @@ -2,7 +2,7 @@ "schema_version": 1, "kind": "external_benchmark_binding", "repository": "https://github.com/lexmount/lexbench-webfetch", - "revision": "56aca0cb9b9b9eef9995f0168e0504a72e309cd1", + "revision": "2ea8c52df67c39eee2ea5113a63f5da34f102160", "ownership": { "benchmark": "Task populations, task model, acceptance scoring, file checks, compiler and frozen definitions", "harness": "Execution, reference lifecycle, evidence and reports" @@ -81,6 +81,6 @@ "evaluation/scoring-regressions.json": "432ddabe23d045d0ce4c23aef69ca3337ba4d240fe3c2061932cee26b7dc00dc", "evaluation/source-inputs.json": "c202abc4624811b938a837402bbaf0ecbe716e375fbca70567346d2edb3b1708", "annotation-revisions/serialization-20261021.json": "e0fe42179e584abb73782cbc459f521cab9cb09d3442de1ec4a6ff54d1b1235a", - "benchmark-release.json": "75fde8db11bb8b3b8b1460fea0e78385bd2a0bdddecc84fbae8287df5d25e879" + "benchmark-release.json": "96ac7f1044601f4a3f768be814a49a48ff46e3ad9802f7f9e17a524460b57c4b" } } diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index d9e62a1ef..b67ae17c8 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -2218,7 +2218,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english release_path=out/'benchmark-release.json' if release_path.is_file(): release=json.loads(release_path.read_text()) - description+=' · '+esc(release['benchmark']+' v'+release['version'])+'' + description=''+description+' · '+esc(release['benchmark']+' v'+release['version'])+'' home='
'+hero('Web Fetch 评测报告',description) from .report_explainer import render_explainer TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else '本报告的任务范围与验收规则' diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index 84766f889..b1b7fd4ff 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -1509,6 +1509,8 @@ def validate_local_presentation(output: Path, binding_path: Path): require_host_metadata_projection(output, binding['publication_projection']) dataset = json.loads((output / 'dataset.json').read_text()) records = load_jsonl(contained_path(output, 'results.jsonl')) + from .report_version import check_release + check_release(output, dataset, records) if binding.get('publication_evidence_views'): from .publication_privacy import require_public_evidence_view_inventory require_public_evidence_view_inventory(output, binding['publication_evidence_views'], records) diff --git a/harness/web_fetch/report_package.py b/harness/web_fetch/report_package.py index 2fd9b22eb..08af4ab63 100644 --- a/harness/web_fetch/report_package.py +++ b/harness/web_fetch/report_package.py @@ -29,6 +29,11 @@ def _require(condition, detail): def validate_release(source: Path, *, presentation_binding: Path | None = None, snapshot_audit=False) -> bool: + from .report_version import check_release + dataset_path = source / "dataset.json" + dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {} + records = load_jsonl(source / "results.jsonl") + check_release(source, dataset, records) manifest_path=source/'report-manifest.json' audit_state=manifest_path.is_file() and json.loads(manifest_path.read_text()).get('evidence_state')=='published_snapshot_score_audit' if snapshot_audit: @@ -43,9 +48,6 @@ def validate_release(source: Path, *, presentation_binding: Path | None = None, from .report_build import validate_local_presentation validate_local_presentation(source, presentation_binding) return False - dataset_path = source / "dataset.json" - dataset = json.loads(dataset_path.read_text()) if dataset_path.exists() else {} - records = load_jsonl(source / "results.jsonl") from .reference_lifecycle import require_certifiable_evidence manifests = [] for name in ('manifest.json', 'execution.json'): diff --git a/harness/web_fetch/report_version.py b/harness/web_fetch/report_version.py index 42aba46cb..d926f2474 100644 --- a/harness/web_fetch/report_version.py +++ b/harness/web_fetch/report_version.py @@ -12,9 +12,13 @@ def _input_release(root): execution = root / "execution.json" - if not execution.is_file(): - return None - binding = json.loads(execution.read_text()).get("aggregate", {}).get("benchmark_release") + binding = (json.loads(execution.read_text()).get("aggregate", {}).get("benchmark_release") + if execution.is_file() else None) + manifest = root / "report-manifest.json" + if manifest.is_file(): + declared = (json.loads(manifest.read_text()).get("benchmark_release") or {}).get("input_release") + if declared is not None and declared != binding: + raise ValueError("Report lost its original benchmark input release binding") if binding is not None: path = contained_path(root, binding["path"], reject_symlinks=True) if hashlib.sha256(path.read_bytes()).hexdigest() != binding.get("sha256"): @@ -77,12 +81,21 @@ def bind_release(source, output, dataset, records, *, profile=None): archived = contained_path(output, measured["path"], reject_symlinks=True) archived.parent.mkdir(parents=True, exist_ok=True) archived.write_bytes(original.read_bytes()) - return { + execution_path = output / "execution.json" + execution = json.loads(execution_path.read_text()) if execution_path.exists() else {} + aggregate = execution.setdefault("aggregate", {}) + if aggregate.get("benchmark_release") != measured: + aggregate["benchmark_release"] = measured + execution_path.write_text(json.dumps(execution, ensure_ascii=False, indent=2) + "\n") + binding = { "path": FILENAME, "sha256": digest, "benchmark": release["benchmark"], "version": release["version"], } + if measured is not None: + binding["input_release"] = measured + return binding def check_release(root, dataset, records, *, profile=None): diff --git a/pyproject.toml b/pyproject.toml index 0d9114012..92d2ceb82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md"] requires-python = ">=3.11" dependencies = [ - "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@56aca0cb9b9b9eef9995f0168e0504a72e309cd1", + "lexbench-webfetch @ git+https://github.com/lexmount/lexbench-webfetch.git@2ea8c52df67c39eee2ea5113a63f5da34f102160", "beautifulsoup4>=4.12,<5", "fastcdc==1.7.0", ] diff --git a/tests/test_external_local_report.py b/tests/test_external_local_report.py index 14904a5a2..b8cdd32f3 100644 --- a/tests/test_external_local_report.py +++ b/tests/test_external_local_report.py @@ -92,6 +92,16 @@ def versioned_fixture(self, case, timeout, **kwargs): "max_failure_retries": 0}) dataset["comparison"]["local_run_contract_sha256"] = contract_digest write(evidence / "dataset.json", dataset) + if case_count == 5: + from lexbench_webfetch.versions import create_release + original = create_release(dataset, benchmark="synthetic", version="1.0.0", + scoring_source_sha256="f" * 64) + original_sha = write(evidence / "benchmark-input-release.json", original) + current = create_release(dataset, benchmark="synthetic", version="1.1.0", + previous=original, change_note="Correct scoring of saved outputs") + write(evidence / "benchmark-release.json", current) + write(evidence / "execution.json", {"batches": batches, "aggregate": { + "benchmark_release": {"path": "benchmark-input-release.json", "sha256": original_sha}}}) output = tmp_path / "report" for script, args in [ ("build_report.py", ["--local-audit", "--evidence", str(evidence), @@ -154,6 +164,28 @@ def versioned_fixture(self, case, timeout, **kwargs): from harness.web_fetch.report_build import build_report from harness.web_fetch.report_package import build as package_report from harness.web_fetch.publication_check import check as check_package + from harness.web_fetch.report_version import check_release + from harness.web_fetch.report_build import validate_local_presentation + original_binding = json.loads((output / "execution.json").read_text())["aggregate"]["benchmark_release"] + assert original_binding["sha256"] == original_sha + ordinary_rebuild = tmp_path / "ordinary-rebuild" + build_report(output, ordinary_rebuild, local_audit=True, language="en", include_english=True) + assert json.loads((ordinary_rebuild / "execution.json").read_text())["aggregate"]["benchmark_release"] == original_binding + assert json.loads((ordinary_rebuild / "benchmark-release.json").read_text())["version"] == "1.1.0" + original_path = ordinary_rebuild / original_binding["path"] + original_bytes = original_path.read_bytes() + original_path.write_text('{}') + with pytest.raises(ValueError, match="input release changed"): + check_release(ordinary_rebuild, dataset, saved) + original_path.write_bytes(original_bytes) + execution_path = ordinary_rebuild / "execution.json" + execution_bytes = execution_path.read_bytes() + changed_execution = json.loads(execution_bytes) + del changed_execution["aggregate"]["benchmark_release"] + write(execution_path, changed_execution) + with pytest.raises(ValueError, match="input release binding"): + check_release(ordinary_rebuild, dataset, saved) + execution_path.write_bytes(execution_bytes) binding_path = tmp_path / "presentation-binding.json" write(binding_path, { "comparison_kind": "uniform_native_local_measurement", @@ -174,6 +206,31 @@ def versioned_fixture(self, case, timeout, **kwargs): archive = tmp_path / "frozen-report.zip" package_report(rebuilt, archive, presentation_binding=binding_path) check_package(archive, rebuilt, presentation_binding=binding_path) + damaged_paths = [rebuilt / "benchmark-release.json", rebuilt / "report-manifest.json", + rebuilt / "index.html"] + pristine = {path: path.read_bytes() for path in damaged_paths} + for damage in ("missing", "changed", "dropped-binding", "wrong-label"): + if damage == "missing": + damaged_paths[0].unlink() + elif damage == "changed": + changed = json.loads(pristine[damaged_paths[0]]) + changed["version"] = "99.0.0" + write(damaged_paths[0], changed) + elif damage == "dropped-binding": + changed = json.loads(pristine[damaged_paths[1]]) + del changed["benchmark_release"] + write(damaged_paths[1], changed) + else: + damaged_paths[2].write_text(pristine[damaged_paths[2]].decode().replace( + 'data-benchmark-release="1.1.0"', 'data-benchmark-release="99.0.0"')) + with pytest.raises(ValueError, match="[Bb]enchmark"): + validate_local_presentation(rebuilt, binding_path) + with pytest.raises(ValueError, match="[Bb]enchmark"): + package_report(rebuilt, archive, presentation_binding=binding_path) + with pytest.raises(ValueError, match="[Bb]enchmark"): + check_package(archive, rebuilt, presentation_binding=binding_path) + for path, raw in pristine.items(): + path.write_bytes(raw) # Language entry normalization must not hide changes to ordinary links. from bs4 import BeautifulSoup from harness.web_fetch.report_check import check diff --git a/tests/test_webfetch_report_package.py b/tests/test_webfetch_report_package.py index 03b3cab8e..8fd8ddfff 100644 --- a/tests/test_webfetch_report_package.py +++ b/tests/test_webfetch_report_package.py @@ -131,6 +131,57 @@ def test_retained_success_and_failure_rerun_can_be_packaged_together(tmp_path): assert archive.is_file() +@pytest.mark.parametrize("damage", [None, "missing", "changed", "dropped-binding", "wrong-label"]) +def test_ordinary_package_and_publication_require_the_bound_benchmark_edition(tmp_path, damage): + from lexbench_webfetch.scoring import score_attempt + from lexbench_webfetch.versions import create_release + from harness.web_fetch.publication_check import check + + source, archive = report_tree(tmp_path, sampled=True) + dataset_path = source / "dataset.json" + dataset = json.loads(dataset_path.read_text()) + for case in dataset["cases"]: + case["url"] = "https://example.test/" + case["id"] + dataset_path.write_text(json.dumps(dataset)) + cases = {case["id"]: case for case in dataset["cases"]} + rows = [json.loads(line) for line in (source / "results.jsonl").read_text().splitlines()] + for row in rows: + row["ok"] = True + row["metrics"] = score_attempt(cases[row["case_id"]], + (source / row["artifact_path"]).read_bytes(), ok=True) + (source / "results.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + release = create_release(dataset, benchmark="sample", version="1.0.0") + sidecar = source / "benchmark-release.json" + sidecar.write_text(json.dumps(release)) + manifest = {"benchmark_release": {"path": sidecar.name, + "sha256": hashlib.sha256(sidecar.read_bytes()).hexdigest(), + "benchmark": "sample", "version": "1.0.0"}} + (source / "report-manifest.json").write_text(json.dumps(manifest)) + home = source / "index.html" + home.write_text(home.read_text() + 'sample v1.0.0') + report_package.build(source, archive) + check(archive, source) + before = archive.read_bytes() + if damage is None: + return + if damage == "missing": + sidecar.unlink() + elif damage == "changed": + release["version"] = "99.0.0" + sidecar.write_text(json.dumps(release)) + elif damage == "dropped-binding": + (source / "report-manifest.json").write_text('{}') + else: + home.write_text(home.read_text().replace('data-benchmark-release="1.0.0"', + 'data-benchmark-release="99.0.0"')) + with pytest.raises(ValueError, match="[Bb]enchmark"): + report_package.build(source, archive) + assert archive.read_bytes() == before + with pytest.raises(ValueError, match="[Bb]enchmark"): + check(archive, source) + + @pytest.mark.parametrize( "damage", [ diff --git a/uv.lock b/uv.lock index 30b9e9a1b..3c0aac23d 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ requires-dist = [ { name = "jieba", marker = "extra == 'webmainbench-eval'", specifier = "==0.42.1" }, { name = "json-repair", marker = "extra == 'browser-use-eval'", specifier = ">=0.54,<1" }, { name = "langchain-community", marker = "extra == 'oss-eval'", specifier = "==0.3.30" }, - { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=56aca0cb9b9b9eef9995f0168e0504a72e309cd1" }, + { name = "lexbench-webfetch", git = "https://github.com/lexmount/lexbench-webfetch.git?rev=2ea8c52df67c39eee2ea5113a63f5da34f102160" }, { name = "markdown", marker = "extra == 'webmainbench-eval'", specifier = ">=3.7,<4" }, { name = "openai", marker = "extra == 'webmainbench-eval'", specifier = ">=2.8,<4" }, { name = "openpyxl", marker = "extra == 'oss-eval'", specifier = "==3.1.5" }, @@ -1933,7 +1933,7 @@ wheels = [ [[package]] name = "lexbench-webfetch" version = "0.3.0" -source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=56aca0cb9b9b9eef9995f0168e0504a72e309cd1#56aca0cb9b9b9eef9995f0168e0504a72e309cd1" } +source = { git = "https://github.com/lexmount/lexbench-webfetch.git?rev=2ea8c52df67c39eee2ea5113a63f5da34f102160#2ea8c52df67c39eee2ea5113a63f5da34f102160" } dependencies = [ { name = "beautifulsoup4" }, { name = "soupsieve" },