diff --git a/.agents/skills/benchmark-report/references/audit-and-revision.md b/.agents/skills/benchmark-report/references/audit-and-revision.md index 9bb7a956f..f98b1d346 100644 --- a/.agents/skills/benchmark-report/references/audit-and-revision.md +++ b/.agents/skills/benchmark-report/references/audit-and-revision.md @@ -30,6 +30,8 @@ Formatting differences are not failures when the output contract treats them as Agreement between the reviewer and scorer proves only that the saved answer fits the current contract; it does not prove that the contract or evaluation path is sound. After recording the blind verdict, deterministically flag suspicious results and investigate the complete chain: task input, frozen reference, provider request contract and runtime, raw response, adapter transformation, scorer, and final verdict. At minimum flag successful requests with empty answers, nonempty raw responses that become empty or implausibly small, identical short answers across unrelated URLs, tool failures that occur before meaningful network work, results that satisfy the positive structure but fail only on excluded content, provider-by-family zeroes or sharp discontinuities, and anomalies raised by the reviewer. This second stage may inspect execution metadata, but must not rewrite the already-recorded blind verdict. +A method result may reveal a benchmark anomaly, but it is never authority for changing a task, reference, annotation, or acceptance rule. Decide each correction from the task contract and method-independent source/reference evidence, without using which method benefits or how scores move. Bind that authority in the correction manifest and reject the revision before rescoring when it is absent. A source presentation state has no standalone meaning: apply it only through the benchmark's declared content scope. + Every triggered anomaly needs concrete evidence at each applicable layer, a root-cause classification, a deterministic equivalence scope expanded to the affected attempts, and one of: a justified no-change decision, a corrected reference/scorer with paired regressions, or a corrected request/adapter/runtime followed by the affected provider calls and renewed review. Missing raw/request evidence is `insufficient`, not proof that the provider failed. Do not publish with unresolved anomalies or a known evaluation defect. Report ordinary provider/origin limitations separately from evaluation defects, and do not retry a valid physical call merely because a later call might score better. ## Decide what to rerun diff --git a/evaluation/webmainbench-full/case-correction-binding.json b/evaluation/webmainbench-full/case-correction-binding.json index 58a7d6c8b..cbd66fec9 100644 --- a/evaluation/webmainbench-full/case-correction-binding.json +++ b/evaluation/webmainbench-full/case-correction-binding.json @@ -1,5 +1,5 @@ { "schema_version": 1, - "relative_path": "evidence/webmainbench-545-origin-correction-seven-pr919-e944d066-rescored-20260929", - "sha256": "4bd72bf64c56f72566f7085cf6bdbb8107b1e34efe2b0f4845546e6fa0d9fc0e" + "relative_path": "evidence/webmainbench-545-origin-correction-seven-pr919-721bd2e-authority-rescored-20261008", + "sha256": "72be654df28c75eeca879101b0eb416dbcbe3b8e2f79e1b5073e9f8ee9ea52f2" } diff --git a/evaluation/webmainbench-full/content-binding.json b/evaluation/webmainbench-full/content-binding.json index 4f1e50514..3166b6dc4 100644 --- a/evaluation/webmainbench-full/content-binding.json +++ b/evaluation/webmainbench-full/content-binding.json @@ -1,5 +1,5 @@ { "schema_version": 1, "relative_path": "seven-method-moli-repairs/report-content.json", - "sha256": "92307a6be0ea8ec3bdae29045d19750fc4c66945dd901c9746eb8349647c054b" + "sha256": "9c6576a4991ba7ef319a9941c2bed13be1b76c487cdd5b82a1b2445c2394fd57" } diff --git a/evaluation/webmainbench-full/illustrations.json b/evaluation/webmainbench-full/illustrations.json index e71830564..2107ee6dd 100644 --- a/evaluation/webmainbench-full/illustrations.json +++ b/evaluation/webmainbench-full/illustrations.json @@ -4,7 +4,7 @@ { "name": "seven-method-overview.png", "role": "overview", - "sha256": "dde509a23ff5e0694df68c7b91974c388ddae3ff9441df6e161caddc6912f06e", + "sha256": "59b8f5025ae770eecc53f0d81446dd275e16458fae31b7ac122bcc36e9c3ac7e", "generator": "deterministic HTML/CSS render", "scope": "seven methods after certified origin, reference, TEDS and shared-population corrections", "claims": { @@ -15,7 +15,7 @@ "chromium": "27.02", "obscura": "31.03", "moli-pr785-fixed": "36.53", - "moli-content-fixed": "39.41" + "moli-content-fixed": "39.27" }, "machine_issues": 25, "request_failures": 15, @@ -51,7 +51,7 @@ { "name": "moli-repair-mechanisms.png", "role": "repair-mechanisms", - "sha256": "9db091bb41d693aa22862f2fe7e8f316ebfaaba1d2d1938fc515df915a8a329a", + "sha256": "f2a8d8e5cede37bf824b8a8ba585875d0e67b084975364b138df88ff5c37f137", "generator": "deterministic HTML/CSS render", "scope": "observed defect to shared cause, implementation layer and verification" }, diff --git a/evaluation/webmainbench-full/moli-content-fixed-report.json b/evaluation/webmainbench-full/moli-content-fixed-report.json index 86806d171..b00e4841a 100644 --- a/evaluation/webmainbench-full/moli-content-fixed-report.json +++ b/evaluation/webmainbench-full/moli-content-fixed-report.json @@ -5,18 +5,31 @@ "case_count": 545, "case_ids_sha256": "5c0cf5810602b70e8f7c1e16723e24a49c3ea207d54865df5c03c38f3d88ffd5", "source_dataset_sha256": "03bc6370a2b37427eba94f95268cf3c935ac89b15b184dfc47f84b78b97fe18e", - "providers": ["moli"], - "versions": {"moli": "1.1.11"}, - "binary_sha256": "e146b853bb06ea02bb8847407a76e28eb2bc31ba4ce741af3a7fdaabb194e209", - "source_commit": "e944d066bc12493bc05e737f07c5721d5be32cf6", + "providers": [ + "moli" + ], + "versions": { + "moli": "1.1.11" + }, + "binary_sha256": "fac987f54939fab2f5db7bf54af41b25cbb3aed3230528c170b4c22ada0b98ac", + "source_commit": "721bd2e329f96c06573db6e6b257a149cf17a6fe", "timeout_seconds": 45, "runs_per_case": 1, "concurrency": 1, "max_failure_retries": 1, "reference_binding_mode": "complete_webmainbench_545_official_frozen_html_and_groundtruth_content", "official_scorer_commit": "f7cb97df2dfb5ac10ebc97cd0b8f9d73ab40f50b", - "metric_config": {"use_llm": false}, - "official_metrics": ["overall", "text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS"], + "metric_config": { + "use_llm": false + }, + "official_metrics": [ + "overall", + "text_edit", + "code_edit", + "formula_edit", + "table_edit", + "table_TEDS" + ], "input_contract": { "body": "official html field encoded as UTF-8 without mutation", "execution": "inline scripts may run; non-loopback subresources and data access are blocked; local engine receives the same body from an OS-isolated loopback HTTPS fixture with a per-run CA", @@ -24,5 +37,5 @@ }, "publication_state": "pending", "require_independent_result_review": false, - "conversion": "Native Markdown from Moli PR #919 after shared fallback semantics, live form state, proven interactive disclosures, bounded one-shot fade-in final-state recovery, media candidate and fragment-target fixes, based on main cb56b3b33" + "conversion": "Native Markdown from Moli PR #919 after shared fallback semantics, live form state, proven interactive disclosures, resolved cascade visibility, and composed final animation effects, based on main cb56b3b33" } diff --git a/evaluation/webmainbench-full/moli-variants.json b/evaluation/webmainbench-full/moli-variants.json index a13d7c6ed..6f1fe9030 100644 --- a/evaluation/webmainbench-full/moli-variants.json +++ b/evaluation/webmainbench-full/moli-variants.json @@ -14,7 +14,7 @@ "provider": "moli-content-fixed", "profile": "moli-content-fixed-report.json", "evidence_alias": "moli-fixed", - "label": "Moli 1.1.11 · PR919 修复版", + "label": "Moli 1.1.11 · PR919 721bd2e 实测版", "short_label": "PR919 修复版", "role": "content_repair", "pull_request": "https://github.com/lexmount/moli/pull/919" diff --git a/evaluation/webmainbench-full/publication-target.json b/evaluation/webmainbench-full/publication-target.json index e8737ceb2..b710142f0 100644 --- a/evaluation/webmainbench-full/publication-target.json +++ b/evaluation/webmainbench-full/publication-target.json @@ -3,10 +3,10 @@ "profile_sha256": "8cbceaf9694e88e7c1ddbba3263573dcccf69b1ac89d7e9ea5aa098f2a3bab02", "variant_profile_sha256": { "moli-pr785-fixed": "db8e7fa408edf3dc2e19d2b0f35645448d0e1efd3ac86926d0bcc9e1a3fb3848", - "moli-content-fixed": "e3a4b3abd03e29ef3aab13e27e11b614d06f0d822fad86b08c61dbde1501b6bf" + "moli-content-fixed": "ee4c91daa5221a0e44a5f5300b03e379a3875623a31c0e0baf59c723a5e093e3" }, "host": "https://artifact-site.lexmount.cn", "site_slug": "zrqPjO672dgg", "share_url": "https://artifact-site.lexmount.cn/v/Avz3qzo6EEujI0iln1LAzCUF93I47OEC", - "expected_remote_version": "ver_9440fb2764424af387e7a3c41cf6720f" + "expected_remote_version": "ver_cdd1a9316d6c4d49a5dc4fee4646fba1" } diff --git a/evaluation/webmainbench-full/scoring-correction-binding.json b/evaluation/webmainbench-full/scoring-correction-binding.json index 786e43d8d..78fd48727 100644 --- a/evaluation/webmainbench-full/scoring-correction-binding.json +++ b/evaluation/webmainbench-full/scoring-correction-binding.json @@ -1,8 +1,8 @@ { "schema_version": 1, "kind": "webmainbench_shared_population_scoring_correction_binding", - "receipt_sha256": "514c99c3c8b55b29eccc38fd87efecc40ebf8602f2e9e84b6807538df696061e", - "scorer_commit": "f7cb97df2dfb5ac10ebc97cd0b8f9d73ab40f50b", + "receipt_sha256": "25281e33ad59cbbe9d7a52ce5e220ff335d769ec6864860989f71820b11ab4f8", + "scorer_commit": "65b49124b923e59f3cfcd839ee1af4628c36f340", "regression_registry_sha256": "ddd89c55e5791a94f3d9282dc3a56fbefed2cb72aad8b0fc43d5275a3bca7cd3", - "results_sha256": "08f4464fe98c57910bcc21b63d3dad8c87d3bd505c8f181c89f169257fd8f439" + "results_sha256": "08f57ca52b337e3d3dbe42c2a84c291de2636383af36f0d401fce786b83b03e9" } diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index 84e727bda..95fb7f776 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -3244,7 +3244,7 @@ def page(route, title, body): anchor("index.html", "WebMainBench · 545") + '' + anchor("en/index.html", "English") + '
' + body + '
') + anchor("https://github.com/lexmount/WebMainBench/tree/lex-main", "lex-main 数据与评分器") + '') target = out / route target.parent.mkdir(parents=True, exist_ok=True) target.write_text(markup, encoding="utf-8") @@ -3504,13 +3504,13 @@ def bound_artifact(path, expected, shard_path): '

测量与评分方法

' + prose('method_lead') + '

' + '

输入

' + prose('method_input') + '

数据 SHA-256：' + esc(profile["source_dataset_sha256"]) + '。原始评分器提交：' + esc(profile["official_scorer_commit"]) + - '' + (('；Lexmain 修正评分器提交：' + esc(scoring_correction['scorer_commit']) + '') if scoring_correction else '') + + '' + (('；lex-main 修正评分器提交：' + esc(scoring_correction['scorer_commit']) + '') if scoring_correction else '') + '。

失败重试

本轮共 ' + str(len(records) + retry_count) + ' 次物理调用，包含 ' + str(retry_count) + ' 次失败后的重试；其中 ' + str(recovered_count) + ' 条重试后返回成功，' + str(final_request_failures) + ' 条最终请求失败均调用两次。返回成功但正文为空或为错误页的结果不触发请求失败重试。

' + '

评分

' + prose('method_scoring') + '

' + prose('method_correction') + '

' + - ((anchor("teds-correction.json", "历史 TEDS 修正收据") + " · " + anchor("teds-correction-review.json", "历史修正复核")) if (out / "teds-correction.json").is_file() else anchor("scoring-correction.json", "Lexmain 统一评分修正收据")) + "
" + + ((anchor("teds-correction.json", "历史 TEDS 修正收据") + " · " + anchor("teds-correction-review.json", "历史修正复核")) if (out / "teds-correction.json").is_file() else anchor("scoring-correction.json", "lex-main 统一评分修正收据")) + "" + '

调用接口

' + table(['方法', '本轮调用方式'], interface_rows) + '
' + '

' + prose('kitesurf_version_note') + '

' + '

公开采用证据

' + prose('adoption_lead') + ' 截至 ' + esc(adoption['cutoff'][:10]) + diff --git a/research/report-changes.json b/research/report-changes.json index f24c8c56f..7ab1485c9 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -1275,6 +1275,61 @@ } ], "releases": [ + { + "id": "webmainbench-545-pr919-721bd2e-authority-20261008", + "date": "2026-10-08", + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], + "title": "保留人工主内容参考并重算 PR919 实测结果", + "summary": "Moli 1.1.11 PR919 提交 721bd2e32 以固定 release 二进制完成 545 页认证运行，目标正文 545/545。复核确认任务 8798f93a 的 91 组可展开比赛明细属于人工标注主内容；撤销错误参考改动后，七方法零调用离线重算，PR919 实测版综合分为 39.27%。", + "changes": [ + [ + "实测版本边界", + "此前把 721bd2e 的运行称为 PR919 当前版本，分支后续重排会让这一称呼失真。", + "继续绑定提交 721bd2e32 与 release 二进制 fac987f5…，并明确它是一次独立实测版本。", + "目标正文 545/545；后续分支提交不借用本次分数。" + ], + [ + "主内容参考复核", + "仅凭页面启动时折叠就删除任务 8798f93a 的 91 组比赛明细，混淆了初始可见文本与主内容抽取契约。", + "核对任务定义、人工 cc-select 标注、main_html 和 groundtruth_content；四者均把明细纳入主内容。恢复原参考，并要求未来修订绑定任务契约与冻结来源，禁止用方法得分授权改题。", + "PR919 实测版在该页的代码分保留为 12.28%；七方法只离线重算，没有新增 provider 调用。" + ], + [ + "七方法结果", + "错误参考把一个真实的方法差距从正式结果中移除。", + "七种方法统一使用 22 条有效数据修订、固定公共总体和同一原网址补测结果。", + "PR919 实测版综合分 39.27%，全文 40.73%、代码 91.06%、公式 28.20%、表格内容 22.62%、表格结构 13.75%；综合分和代码为唯一最高。" + ] + ], + "i18n": { + "en": { + "title": "Retain the annotated main-content reference and rescore the measured PR919 build", + "summary": "Moli 1.1.11 PR919 commit 721bd2e32 completed a certified 545-page run with a pinned release binary and delivered 545/545 target bodies. Review confirmed that the 91 expandable game-detail groups in case 8798f93a belong to the human-annotated main content. After reverting the invalid reference change, zero-call offline rescoring gives the measured PR919 build a 39.27% overall score.", + "changes": [ + [ + "Measured-build boundary", + "Calling the 721bd2e run the current PR919 build became inaccurate after the branch was rebased.", + "Keep the exact 721bd2e32 source and fac987f5… release binary binding and label it as a measured build.", + "It delivers 545/545 target bodies; later branch commits do not inherit these scores." + ], + [ + "Main-content reference review", + "Removing 91 game-detail groups solely because they are collapsed at startup confused initial visible text with the benchmark main-content contract.", + "Check the task definition, human cc-select annotations, main_html, and groundtruth_content. All four include the details. Restore the reference and require future corrections to bind task-contract and frozen-source authority, never method scores.", + "The measured PR919 build retains a 12.28% code score on this page. All seven methods are rescored offline with zero provider calls." + ], + [ + "Seven-method result", + "The invalid reference change removed a real method gap from the formal result.", + "Use the same 22 valid dataset corrections, reference-defined populations, and original-origin replacement result for all seven methods.", + "The measured PR919 build scores 39.27% overall, 40.73% text, 91.06% code, 28.20% formula, 22.62% table content, and 13.75% table structure; overall and code are the unique maxima." + ] + ] + } + } + }, { "id": "2026-10-07-risk-control-historical-presentation", "comparison_kinds": [ diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py index 7a9af6905..9a5ed4fb4 100644 --- a/scripts/check_webmainbench_full_report.py +++ b/scripts/check_webmainbench_full_report.py @@ -133,10 +133,10 @@ def check(root: Path) -> None: raise ValueError('Shared-population scoring correction differs from repository binding') home_html = (root / 'index.html').read_text(encoding='utf-8') methodology_html = (root / 'methodology/index.html').read_text(encoding='utf-8') - if ('https://github.com/lexmount/WebMainBench/tree/lexmain' not in home_html + if ('https://github.com/lexmount/WebMainBench/tree/lex-main' not in home_html or scoring_receipt['scorer_commit'] not in methodology_html - or 'Lexmain 修正评分器提交' not in methodology_html): - raise ValueError('Public report does not identify the bound Lexmain scorer') + or 'lex-main 修正评分器提交' not in methodology_html): + raise ValueError('Public report does not identify the bound lex-main scorer') releases = json.loads((Path(__file__).resolve().parents[1] / 'research/report-changes.json').read_text(encoding='utf-8'))['releases'] expected_releases = [item for item in releases if 'webmainbench_frozen_html_to_markdown' in item.get('comparison_kinds', [])] history = BeautifulSoup((root / 'history/index.html').read_text(encoding='utf-8'), 'html.parser') diff --git a/scripts/correct_webmainbench_scoring.py b/scripts/correct_webmainbench_scoring.py index 763bef78d..9794ab9c2 100644 --- a/scripts/correct_webmainbench_scoring.py +++ b/scripts/correct_webmainbench_scoring.py @@ -45,6 +45,13 @@ def copy_correction_sources(corrections: dict, manifest_path: Path, ) +def prepare_correction_source(destination: Path) -> None: + """Replace inherited provenance when correcting an already corrected cohort.""" + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir() + + def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -165,8 +172,9 @@ def revised_metrics(old: dict, scorer, predicted: str, reference: str, def validate_population_binding(source: Path, source_dataset: dict, corrected_rows: list[dict], manifest: dict) -> None: - """Bind schema-v2 corrections to the complete ordered 545-row dataset.""" - if manifest.get("schema_version") != 2: + """Bind authority-declared corrections to the complete ordered 545-row dataset.""" + validate_correction_authority(manifest) + if manifest.get("schema_version") != 3: raise ValueError("Invalid dataset correction manifest") cases = {case["id"]: case for case in source_dataset.get("cases", [])} original_rows = [] @@ -193,6 +201,54 @@ def validate_population_binding(source: Path, source_dataset: dict, raise ValueError(f"Dataset correction {label} population binding mismatch") +def validate_correction_authority(manifest: dict) -> None: + """Reject reference changes whose declared authority depends on method outcomes.""" + if manifest.get("schema_version") != 3: + raise ValueError("Dataset corrections require decision-authority schema v3") + contracts = manifest.get("decision_contracts") + if not isinstance(contracts, dict) or not contracts: + raise ValueError("Dataset corrections require decision contracts") + for contract in contracts.values(): + evidence = contract.get("required_source_evidence") if isinstance(contract, dict) else None + if not isinstance(contract, dict) or contract.get("changed_field") not in { + "groundtruth_content", "meta", + }: + raise ValueError("Decision contract has an unsupported changed field") + if (not isinstance(evidence, list) or "task_contract" not in evidence + or not any(isinstance(value, str) and ( + value.startswith("frozen_") or value.startswith("original_") + ) for value in evidence)): + raise ValueError("Decision contract lacks method-independent source authority") + forbidden = {"method_score", "method_result", "provider_score", "provider_result"} + + def contains_forbidden_key(value) -> bool: + if isinstance(value, dict): + return bool(forbidden & set(value)) or any( + contains_forbidden_key(item) for item in value.values() + ) + if isinstance(value, list): + return any(contains_forbidden_key(item) for item in value) + return False + + for correction in manifest.get("corrections", []): + if correction.get("decision_contract") not in contracts: + raise ValueError("Dataset correction lacks a declared decision contract") + if not isinstance(correction.get("rationale"), str) or not correction["rationale"].strip(): + raise ValueError("Dataset correction lacks an evidence-based rationale") + if contains_forbidden_key(correction): + raise ValueError("Method outcomes cannot authorize dataset corrections") + fields = set() + if any(key in correction for key in ( + "original_groundtruth_sha256", "replacement_sha256", "replacement_file", + )): + fields.add("groundtruth_content") + if any(key in correction for key in ("original_meta_sha256", "replacement_meta")): + fields.add("meta") + declared = contracts[correction["decision_contract"]]["changed_field"] + if fields != {declared}: + raise ValueError("Correction fields differ from its decision contract") + + def apply(source: Path, corrected_dataset: Path, manifest_path: Path, scorer_repo: Path, scorer_commit: str, output: Path) -> dict: if output.exists(): @@ -223,6 +279,12 @@ def apply(source: Path, corrected_dataset: Path, manifest_path: Path, item = corrections.get(case_id) if not item or item["url"] != before["url"] or item.get("html_sha256") != before["input_html_sha256"]: raise ValueError("Correction binding mismatch: " + case_id) + declared = manifest["decision_contracts"][item["decision_contract"]]["changed_field"] + actual_fields = ({"groundtruth_content"} if reference_changed else set()) | ( + {"meta"} if metadata_changed else set() + ) + if actual_fields != {declared}: + raise ValueError("Changed dataset fields differ from decision contract: " + case_id) if reference_changed and ( item.get("original_groundtruth_sha256") != before["reference_sha256"] or item.get("replacement_sha256") != text_digest(after["groundtruth_content"]) @@ -313,7 +375,7 @@ def apply(source: Path, corrected_dataset: Path, manifest_path: Path, (output / "score-summaries.json").write_text( json.dumps(summaries, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") correction_source = output / "scoring-correction-source" - correction_source.mkdir() + prepare_correction_source(correction_source) shutil.copy2(manifest_path, correction_source / manifest_path.name) shutil.copy2(REGISTRY, correction_source / REGISTRY.name) copy_correction_sources(corrections, manifest_path, correction_source) diff --git a/scripts/run_certified_benchmark.py b/scripts/run_certified_benchmark.py index 2b810c64f..3723ea40b 100644 --- a/scripts/run_certified_benchmark.py +++ b/scripts/run_certified_benchmark.py @@ -112,6 +112,10 @@ def main(argv: list[str] | None = None) -> None: from scripts.certify_webmainbench_case_correction import main as correction_main correction_main((argv or sys.argv[1:])[1:]) return + if (argv or sys.argv[1:])[:1] == ["webmainbench-full"]: + from scripts.run_certified_webmainbench_full import main as webmainbench_full_main + webmainbench_full_main((argv or sys.argv[1:])[1:]) + return if (argv or sys.argv[1:])[:1] == ["wcxb"]: run_wcxb((argv or sys.argv[1:])[1:]) return diff --git a/scripts/run_certified_webmainbench_full.py b/scripts/run_certified_webmainbench_full.py index 02db77c0e..9274e16bb 100644 --- a/scripts/run_certified_webmainbench_full.py +++ b/scripts/run_certified_webmainbench_full.py @@ -94,7 +94,7 @@ def verify_shard(shard: Path, provider: str, case_ids: set[str], profile: dict) "execution_sha256": digest(shard / "execution.json")} -def main() -> None: +def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--source", type=Path, required=True) @@ -102,7 +102,7 @@ def main() -> None: parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json") parser.add_argument("--binary-source", type=Path, help="Exact clean source checkout used to build a profile-bound candidate binary") - args = parser.parse_args() + args = parser.parse_args(argv) if args.output_root.exists(): parser.error("Output root already exists; cohort evidence is immutable") profile = json.loads(args.profile.read_text(encoding="utf-8")) diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py index 46c2b23cb..5eeded188 100644 --- a/tests/test_run_certified_benchmark.py +++ b/tests/test_run_certified_benchmark.py @@ -10,6 +10,12 @@ class CertifiedBenchmarkTests(unittest.TestCase): + def test_dispatches_complete_webmainbench_certification(self) -> None: + arguments = ["--output-root", "/tmp/run", "--source", "/tmp/source.jsonl"] + with patch("scripts.run_certified_webmainbench_full.main") as invoked: + main(["webmainbench-full", *arguments]) + invoked.assert_called_once_with(arguments) + @staticmethod def write_capture(captures: Path, case_id: str, *, status: int = 200, text: str = "reference") -> None: diff --git a/tests/test_webmainbench_full_report.py b/tests/test_webmainbench_full_report.py index ff20ef404..61d2c6d5e 100644 --- a/tests/test_webmainbench_full_report.py +++ b/tests/test_webmainbench_full_report.py @@ -82,7 +82,7 @@ def test_full_report_renders_seven_way_details_and_failure_evidence(tmp_path: Pa ], "variant_labels": { "moli-pr785-fixed": "Moli 1.1.9 · PR785 修复版", - "moli-content-fixed": "Moli 1.1.11 · PR919 修复版", + "moli-content-fixed": "Moli 1.1.11 · PR919 721bd2e 实测版", }, "source_dataset_sha256": "a" * 64, "official_scorer_commit": "b" * 40} profile["versions"]["moli"] = "1.0'