From b54a6bfa00e04d8ae901798dbfc35298cb491d86 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:56:10 +0800 Subject: [PATCH 1/4] Fix equivalent public table answer scoring --- evaluation/scoring-regressions.json | 12 ++++- harness/web_fetch/composite_tasks.py | 18 ++++--- harness/web_fetch/metrics.py | 1 + harness/web_fetch/structured_tables.py | 67 +++++++++++++++++++++++++- tests/test_scoring_regressions.py | 2 + 5 files changed, 90 insertions(+), 10 deletions(-) diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index 82663f684..955425892 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -21777,6 +21777,16 @@ "expected_metrics": { "stable_unique_record_count": 0 } -} +}, + {"id": "table-header-controls-valid", "category": "structured-extraction", "root_cause": "table_header_visual_line", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Name

Resize Name | Amount

Sort |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", "ok": true, "expected_task_pass": true}, + {"id": "table-header-controls-invalid", "category": "structured-extraction", "root_cause": "table_header_visual_line", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Wrong

Resize Name | Amount

Sort |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", "ok": true, "expected_task_pass": false}, + {"id": "table-split-body-valid", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Name

Resize Name | Amount

Sort |\n| --- | --- |\n\n| | |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", "ok": true, "expected_task_pass": true}, + {"id": "table-split-body-invalid", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Name

Resize Name | Amount

Sort |\n| --- | --- |\n\nOther section\n| | |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", "ok": true, "expected_task_pass": false}, + {"id": "mixed-html-linked-table-valid", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "Intro\n
NameOwner
AlphaAlice
BetaBob
", "ok": true, "expected_task_pass": true}, + {"id": "mixed-html-linked-table-missing-owned-field", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "Intro\n
NameOwner
AlphaAlice
Beta
", "ok": true, "expected_task_pass": false}, + {"id": "combined-split-table-valid", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-negative", "case": {"url": "https://example.test/training", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "problems", "kind": "table", "minimum_records": 3, "start_pattern": "(?m)^Section\\s+1\\.", "end_pattern": "(?m)^Section\\s+6\\.", "required_columns": [{"id": "number", "kind": "text", "pattern": "^[A-Z]?[0-9]+$", "aliases": ["Number"]}, {"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "attempts", "kind": "integer", "aliases": ["Tried"]}, {"id": "accepted", "kind": "integer", "aliases": ["AC"]}, {"id": "difficulty", "kind": "integer", "aliases": ["Difficulty"]}], "identity_column": "number", "combined_columns": [{"aliases": ["Problem"], "pattern": "^(?P[A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}}, "artifact": "Section 1. First\n| Problem | Tried | AC | Difficulty |\n| --- | --- | --- | --- |\n\n| | | | |\n| --- | --- | --- | --- |\n| **P1** Hello World | 1 | 1 | 7 |\n| **P2** Output Practice | 2 | 1 | 5 |\n| **B003** Addition Exercise | 3 | 2 | 4 |\nSection 6. End", "ok": true, "expected_task_pass": true}, + {"id": "combined-split-table-missing-field", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-positive", "case": {"url": "https://example.test/training", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "problems", "kind": "table", "minimum_records": 3, "start_pattern": "(?m)^Section\\s+1\\.", "end_pattern": "(?m)^Section\\s+6\\.", "required_columns": [{"id": "number", "kind": "text", "pattern": "^[A-Z]?[0-9]+$", "aliases": ["Number"]}, {"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "attempts", "kind": "integer", "aliases": ["Tried"]}, {"id": "accepted", "kind": "integer", "aliases": ["AC"]}, {"id": "difficulty", "kind": "integer", "aliases": ["Difficulty"]}], "identity_column": "number", "combined_columns": [{"aliases": ["Problem"], "pattern": "^(?P[A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}}, "artifact": "Section 1. First\n| Problem | Tried | AC | Difficulty |\n| --- | --- | --- | --- |\n\n| | | | |\n| --- | --- | --- | --- |\n| **P1** Hello World | 1 | 1 | 7 |\n| **P2** Output Practice | 2 | 1 | 5 |\n| **B003** Addition Exercise | 3 | | 4 |\nSection 6. End", "ok": true, "expected_task_pass": false}, + {"id": "mixed-html-linked-table-example-is-not-answer", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "Example:\n```html\nIntro\n
NameOwner
AlphaAlice
BetaBob
\n```\nNo records", "ok": true, "expected_task_pass": false}, + {"id": "mixed-html-linked-table-hidden-is-not-answer", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "", "ok": true, "expected_task_pass": false} ] } diff --git a/harness/web_fetch/composite_tasks.py b/harness/web_fetch/composite_tasks.py index 348518d54..549a8a5af 100644 --- a/harness/web_fetch/composite_tasks.py +++ b/harness/web_fetch/composite_tasks.py @@ -13,6 +13,7 @@ import unicodedata from urllib.parse import parse_qsl, unquote, urljoin, urlsplit +from .structured_tables import table_text_surface from .dynamic_lists import _canonical_entry_path, extract_named_entries from .reference_quality import clean_reference_text, markdown_links, visible_markdown_links @@ -541,14 +542,14 @@ def _name_description_group(content: str, group: dict, base_url: str) -> dict: def _table_group(content: str, group: dict) -> dict: - lines = [line.strip() for line in clean_reference_text(content).splitlines() if line.strip()] + lines = [line.strip() for line in clean_reference_text(table_text_surface(content)).splitlines() if line.strip()] required = group["required_columns"] records = [] for index in range(len(lines) - 1): delimiter = "|" if "|" in lines[index] else "\t" if "\t" in lines[index] else None if not delimiter: continue - headers = [_key(value) for value in lines[index].strip("|").split(delimiter)] + headers = [_key(visible_markdown_links(value)) for value in lines[index].strip("|").split(delimiter)] separator = lines[index + 1].strip("|").split(delimiter) if delimiter == "|" and not all( re.fullmatch(r"\s*:?-{3,}:?\s*", value) for value in separator @@ -581,7 +582,7 @@ def _table_group(content: str, group: dict) -> dict: if len(values) == len(headers): row = {} for field, (column, pattern) in mapping.items(): - match = pattern.fullmatch(visible_markdown_links(values[column])) if pattern else None + match = pattern.fullmatch(visible_markdown_links(values[column]).replace("**", "").strip()) if pattern else None row[field] = (match[field] if match else '') if pattern else values[column] if all( _value_matches_spec(row[spec["id"]], spec, whole_cell=True) @@ -601,14 +602,14 @@ def _table_group(content: str, group: dict) -> dict: def _linked_table_group(content: str, group: dict, base_url: str) -> dict: - """Read named links and all required fields from the same Markdown row.""" - lines = [line.strip() for line in clean_reference_text(content).splitlines() if line.strip()] + """Read named links and required fields from the same owned table row.""" + lines = [line.strip() for line in clean_reference_text(table_text_surface(content)).splitlines() if line.strip()] required = group["required_columns"] records = {} for index in range(len(lines) - 1): if "|" not in lines[index]: continue - headers = [_key(value) for value in lines[index].strip("|").split("|")] + headers = [_key(visible_markdown_links(value)) for value in lines[index].strip("|").split("|")] separator = lines[index + 1].strip("|").split("|") if not all(re.fullmatch(r"\s*:?-{3,}:?\s*", value) for value in separator): continue @@ -679,12 +680,13 @@ def _field_report_group(content: str, group: dict) -> dict: def evaluate_composite_acceptance( - content: str, contract: dict, *, base_url: str, links: list | None = None, eligible: bool = True + content: str, contract: dict, *, base_url: str, links: list | None = None, eligible: bool = True, table_content: str | None = None ) -> dict: validate_composite_acceptance(contract) results = [] for group in contract["groups"]: - scoped = _scope(content, group) + source = table_content if table_content is not None and group["kind"] in {"table", "linked_table"} else content + scoped = _scope(source, group) if group["kind"] == "named_link": result = _named_link_group( scoped, group, base_url, links if scoped == content else None diff --git a/harness/web_fetch/metrics.py b/harness/web_fetch/metrics.py index 2b5ddaa30..0992b917c 100644 --- a/harness/web_fetch/metrics.py +++ b/harness/web_fetch/metrics.py @@ -643,6 +643,7 @@ def evaluate_content( from .composite_tasks import evaluate_composite_acceptance composite = evaluate_composite_acceptance( visible_content, composite_acceptance, base_url=base_url, links=named_links, + table_content=content, ) result.update(composite) result["content_pass"] = bool(result["content_pass"] and composite["composite_acceptance_pass"]) diff --git a/harness/web_fetch/structured_tables.py b/harness/web_fetch/structured_tables.py index 0d78c3a07..03900d960 100644 --- a/harness/web_fetch/structured_tables.py +++ b/harness/web_fetch/structured_tables.py @@ -174,6 +174,65 @@ def _without_example_fences(text): return ''.join(lines) +def table_header_label(value): + """The first visual line names a column; later lines are controls/help.""" + return re.split(r'', value, maxsplit=1, flags=re.I)[0].strip() + + +def table_text_surface(content): + """Preserve owned HTML cells/links and adjacent split Markdown bodies.""" + from bs4 import BeautifulSoup + content = _without_example_fences(content) + if re.search(r"]*>[\s\S]*?', convert, content, flags=re.I) + lines = content.splitlines() + output = [] + index = 0 + def separator(line): + return '|' in line and all( + re.fullmatch(r'\s*:?-{3,}:?\s*', c) for c in _pipe_cells(line)) + while index < len(lines): + prior = [v for v in output if v.strip()] + cells = _pipe_cells(lines[index]) + if ('|' in lines[index] and all(not c.strip() for c in cells) + and index + 1 < len(lines) and separator(lines[index + 1]) + and len(_pipe_cells(lines[index + 1])) == len(cells) + and len(prior) >= 2 and separator(prior[-1]) + and len(_pipe_cells(prior[-2])) == len(cells) + and any(c.strip() for c in _pipe_cells(prior[-2]))): + while output and not output[-1].strip(): + output.pop() + index += 2 + continue + output.append(lines[index]) + index += 1 + return '\n'.join(output) + + def parse_table_records(content, visible, contract): columns, scope = contract['table_columns'], contract['table_scope'] names = [c['name'] for c in columns] @@ -227,7 +286,13 @@ def parse_table_records(content, visible, contract): if root is None: # Inline layout breaks do not create new Markdown rows/cells. Preserve # a space so split numeric tokens cannot become a different valid value. - visible = re.sub(r'', ' ', visible, flags=re.I) + lines = table_text_surface(visible).splitlines() + for i in range(len(lines) - 1): + cells = _pipe_cells(lines[i]) + if ('|' in lines[i] and len(cells) == len(_pipe_cells(lines[i + 1])) + and all(re.fullmatch(r'\s*:?-{3,}:?\s*', c) for c in _pipe_cells(lines[i + 1]))): + lines[i] = '| ' + ' | '.join(table_header_label(c) for c in cells) + ' |' + visible = re.sub(r'', ' ', '\n'.join(lines), flags=re.I) # Only source-verified, case-declared auxiliary text is removed. # This also normalizes headers before locating the target table. for pattern in scope.get('plain_text_ignored_patterns', []): diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index e816ae3d7..3931ee708 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -95,6 +95,8 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) paired.add("declared_full_page_stable_text_scope") paired.add("inline_emphasis_in_structured_fields") paired.add("markdown_list_prefix_ownership") + paired.update({"table_header_visual_line", "table_adjacent_split_body", + "composite_html_table_ownership"}) for root_cause in paired: self.assertEqual(root_modes.get(root_cause), {"false-positive", "false-negative"}) From 87d26d0bef3b855ab2c05b2d5e78a490eeadad1e Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:04:25 +0800 Subject: [PATCH 2/4] Keep final URL checks diagnostic in shared scoring --- evaluation/scoring-regressions.json | 12 ++++++++++-- harness/web_fetch/README.md | 7 +++++-- harness/web_fetch/metrics.py | 8 ++++---- harness/web_fetch/scoring.py | 16 ++++++++++------ tests/test_scoring.py | 20 +++++++++++++++----- tests/test_scoring_regressions.py | 1 + 6 files changed, 45 insertions(+), 19 deletions(-) diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index 955425892..c6b504cd7 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -2011,7 +2011,7 @@ { "id": "identity", "alternatives": [ - "Search results" + "Mario search results" ] } ] @@ -21787,6 +21787,14 @@ {"id": "combined-split-table-valid", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-negative", "case": {"url": "https://example.test/training", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "problems", "kind": "table", "minimum_records": 3, "start_pattern": "(?m)^Section\\s+1\\.", "end_pattern": "(?m)^Section\\s+6\\.", "required_columns": [{"id": "number", "kind": "text", "pattern": "^[A-Z]?[0-9]+$", "aliases": ["Number"]}, {"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "attempts", "kind": "integer", "aliases": ["Tried"]}, {"id": "accepted", "kind": "integer", "aliases": ["AC"]}, {"id": "difficulty", "kind": "integer", "aliases": ["Difficulty"]}], "identity_column": "number", "combined_columns": [{"aliases": ["Problem"], "pattern": "^(?P[A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}}, "artifact": "Section 1. First\n| Problem | Tried | AC | Difficulty |\n| --- | --- | --- | --- |\n\n| | | | |\n| --- | --- | --- | --- |\n| **P1** Hello World | 1 | 1 | 7 |\n| **P2** Output Practice | 2 | 1 | 5 |\n| **B003** Addition Exercise | 3 | 2 | 4 |\nSection 6. End", "ok": true, "expected_task_pass": true}, {"id": "combined-split-table-missing-field", "category": "structured-extraction", "root_cause": "table_adjacent_split_body", "failure_mode": "false-positive", "case": {"url": "https://example.test/training", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "problems", "kind": "table", "minimum_records": 3, "start_pattern": "(?m)^Section\\s+1\\.", "end_pattern": "(?m)^Section\\s+6\\.", "required_columns": [{"id": "number", "kind": "text", "pattern": "^[A-Z]?[0-9]+$", "aliases": ["Number"]}, {"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "attempts", "kind": "integer", "aliases": ["Tried"]}, {"id": "accepted", "kind": "integer", "aliases": ["AC"]}, {"id": "difficulty", "kind": "integer", "aliases": ["Difficulty"]}], "identity_column": "number", "combined_columns": [{"aliases": ["Problem"], "pattern": "^(?P[A-Z]?[0-9]+)\\s+(?P.+)$"}]}]}}, "artifact": "Section 1. First\n| Problem | Tried | AC | Difficulty |\n| --- | --- | --- | --- |\n\n| | | | |\n| --- | --- | --- | --- |\n| **P1** Hello World | 1 | 1 | 7 |\n| **P2** Output Practice | 2 | 1 | 5 |\n| **B003** Addition Exercise | 3 | | 4 |\nSection 6. End", "ok": true, "expected_task_pass": false}, {"id": "mixed-html-linked-table-example-is-not-answer", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "Example:\n```html\nIntro\n
NameOwner
AlphaAlice
BetaBob
\n```\nNo records", "ok": true, "expected_task_pass": false}, - {"id": "mixed-html-linked-table-hidden-is-not-answer", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "", "ok": true, "expected_task_pass": false} + {"id": "mixed-html-linked-table-hidden-is-not-answer", "category": "structured-extraction", "root_cause": "composite_html_table_ownership", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "link_column": "name"}]}}, "artifact": "", "ok": true, "expected_task_pass": false}, + {"id": "risk_control-diagnostic-url-missing-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "risk_control", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "source_benchmark": {"risk_control": {"navigation_contract": {"requested_url": "https://example.test/start", "observed_final_url": "https://example.test/catalog"}}}}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}}, + {"id": "risk_control-diagnostic-url-missing-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "risk_control", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "source_benchmark": {"risk_control": {"navigation_contract": {"requested_url": "https://example.test/start", "observed_final_url": "https://example.test/catalog"}}}}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}}, + {"id": "risk_control-diagnostic-url-mismatch-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "risk_control", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "source_benchmark": {"risk_control": {"navigation_contract": {"requested_url": "https://example.test/start", "observed_final_url": "https://example.test/catalog"}}}}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog"}, + {"id": "risk_control-diagnostic-url-mismatch-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "risk_control", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "source_benchmark": {"risk_control": {"navigation_contract": {"requested_url": "https://example.test/start", "observed_final_url": "https://example.test/catalog"}}}}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog"}, + {"id": "dynamic-diagnostic-url-missing-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, + {"id": "dynamic-diagnostic-url-missing-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, + {"id": "dynamic-diagnostic-url-mismatch-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, + {"id": "dynamic-diagnostic-url-mismatch-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}} ] } diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index f75241140..524442e4f 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -546,8 +546,11 @@ rankings and catalogs change frequently. Annotate fields, not whole websites: `stable_text_anchors` with independently justified equivalent alternatives. - Frequently changing names, prices, dates and ranking positions use an `acceptance_contract`: required fields, types, record grouping, minimum count, - uniqueness and optional stable page identity. Values are not compared to a - previous ranking. Duplicate cards cannot satisfy the minimum. + uniqueness and stable page text. Values are not compared to a previous + ranking. Duplicate cards cannot satisfy the minimum. Final-URL agreement is + diagnostic: a missing or mismatched URL does not change acceptance or the + evaluated denominator. Declared document-language requirements remain + separate acceptance checks. - Full-reference word precision, recall and F1 remain separate diagnostics. Historical demonstration snapshots describe their captured sample; they do not certify arbitrary later content. Without a comparable full reference, diff --git a/harness/web_fetch/metrics.py b/harness/web_fetch/metrics.py index 0992b917c..85c38b652 100644 --- a/harness/web_fetch/metrics.py +++ b/harness/web_fetch/metrics.py @@ -325,8 +325,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: # Scored records carry the case kind so dataset-free summaries use the same # predicate as the report. Explicit case context still takes precedence. task_kind = task_kind if task_kind is not None else metrics.get('task_kind', 'body') - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') != 'verified'): + if (metrics.get('document_language_required') + and not metrics.get('language_consistent')): return False if (not task_measured(metrics, task_kind) or metrics.get('deadline_overrun') or metrics.get('blocked_page')): @@ -347,8 +347,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') == 'unavailable' + if (metrics.get('document_language_required') + and metrics.get('observed_document_language') is None and metrics.get('request_ok') is not False): return False kind = task_kind or metrics.get('task_kind', 'body') diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py index 7c0b75577..6eea42615 100644 --- a/harness/web_fetch/scoring.py +++ b/harness/web_fetch/scoring.py @@ -51,6 +51,7 @@ def load_list_reference(case: EvalCase | dict[str, Any], source: Path) -> dict | "structure_pass", "named_entry_count", "minimum_entries", "snapshot_f1", "dynamic_titles", "verification_note", "confirmed_entries", "checked_entries", "page_identity_required", "page_identity_status", "final_url_consistent", + "document_language_required", "language_consistent", "observed_document_language", "acceptable_final_urls", "ordered_tree", "ordered_tree_similarity", @@ -187,7 +188,8 @@ def score_attempt( else "unavailable" ) result.update( - page_identity_required=True, + page_identity_required=False, + document_language_required=True, page_identity_status=identity_status, final_url_consistent=final_consistent, language_consistent=language_consistent, @@ -205,7 +207,8 @@ def score_attempt( and any(_same_page_url(final_url, expected) for expected in acceptable_urls) ) result.update( - page_identity_required=True, + page_identity_required=False, + document_language_required=False, page_identity_status=( "verified" if final_consistent else "mismatch" if final_url else "unavailable" ), @@ -213,7 +216,8 @@ def score_attempt( acceptable_final_urls=acceptable_urls, ) else: - result.update(page_identity_required=False, page_identity_status="not_required") + result.update(page_identity_required=False, page_identity_status="not_required", + document_language_required=False) result["task_kind"] = task_kind result["rule_review_status"] = rule_review.get("status", "resolved") if rule_review.get("reason"): @@ -255,13 +259,13 @@ def score_attempt( body_requirements, sort_keys=True, ensure_ascii=False).encode()).hexdigest() if not result["content_score_applicable"] or not ok: result["content_pass"] = False - if result.get("page_identity_required") and result.get("page_identity_status") != "verified": + if result.get("document_language_required") and not result.get("language_consistent"): result["content_pass"] = False if contract: result["stable_task_pass"] = False result["stable_content_pass"] = False - if "page_identity_not_verified" not in result["stable_task_errors"]: - result["stable_task_errors"].append("page_identity_not_verified") + if "document_language_not_verified" not in result["stable_task_errors"]: + result["stable_task_errors"].append("document_language_not_verified") if not ok: if contract: result["stable_task_pass"] = False diff --git a/tests/test_scoring.py b/tests/test_scoring.py index c8635c140..7d6e97e8c 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -63,7 +63,7 @@ def test_untagged_source_requires_explicitly_observed_empty_language(self): final_url="https://example.test/page") self.assertEqual(observed_tagged["page_identity_status"], "mismatch") - def test_risk_control_requires_declared_page_identity(self): + def test_risk_control_final_url_is_diagnostic(self): navigation = { "risk_control": { "navigation_contract": { @@ -85,14 +85,24 @@ def test_risk_control_requires_declared_page_identity(self): source_benchmark=navigation, ) wrong = score_attempt(c, b"Expected", ok=True, final_url="https://other.test/") - self.assertTrue(wrong["page_identity_required"]) + self.assertFalse(wrong["page_identity_required"]) self.assertEqual(wrong["page_identity_status"], "mismatch") self.assertTrue(task_measured(wrong)) - self.assertFalse(task_passed(True, wrong)) + self.assertTrue(task_passed(True, wrong)) missing = score_attempt(c, b"Expected", ok=True) self.assertEqual(missing["page_identity_status"], "unavailable") - self.assertFalse(task_measured(missing)) - self.assertFalse(task_passed(True, missing)) + self.assertTrue(task_measured(missing)) + self.assertTrue(task_passed(True, missing)) + for diagnostic in (wrong, missing): + stats = summarize([{ + "provider": "fixture", "provider_type": "fixture", + "case_id": c.id, "case_name": c.name, "category": c.category, + "ok": True, "elapsed_ms": 1, + "metrics": diagnostic, + }])["providers"]["fixture"] + self.assertEqual(stats["task_evaluated_attempts"], 1) + self.assertEqual(stats["task_unmeasured_attempts"], 0) + self.assertEqual(stats["task_pass_rate"], 1.0) failed = score_attempt(c, b"", ok=False) self.assertEqual(failed["page_identity_status"], "unavailable") self.assertTrue(task_measured(failed)) diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index 3931ee708..e33ddf4cc 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -95,6 +95,7 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) paired.add("declared_full_page_stable_text_scope") paired.add("inline_emphasis_in_structured_fields") paired.add("markdown_list_prefix_ownership") + paired.add("final_url_diagnostic_not_acceptance") paired.update({"table_header_visual_line", "table_adjacent_split_body", "composite_html_table_ownership"}) for root_cause in paired: From e6b3684ddc6cf0c03af575d7b9bb70104bdc72fc Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:10:04 +0800 Subject: [PATCH 3/4] Keep HTML language metadata diagnostic for content tasks --- evaluation/scoring-regressions.json | 14 ++++++++++---- harness/web_fetch/README.md | 4 ++-- harness/web_fetch/metrics.py | 7 ------- harness/web_fetch/scoring.py | 12 +----------- tests/test_scoring_regressions.py | 1 + 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index c6b504cd7..6d0910763 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -1935,7 +1935,7 @@ ] } }, - "artifact": "Catalog", + "artifact": "Unrelated content", "ok": true, "final_url": "https://example.test/fr/catalog", "metadata": { @@ -2098,7 +2098,7 @@ ] } }, - "artifact": "الشخص", + "artifact": "Unrelated content", "ok": true, "final_url": "https://example.test/rtl", "metadata": { @@ -21451,7 +21451,7 @@ {"id":"html-hidden-script-cannot-inflate-visible-length","category":"format-normalization","root_cause":"html_visible_answer_surface","failure_mode":"false-positive","case":{"url":"https://example.test/","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Hello"]}]},"min_content_chars":20},"artifact":"
Hello
","title":"Home","ok":true,"expected_task_pass":false,"expected_metrics":{"length_pass":false}}, {"id":"html-visible-challenge-heading-blocks-body","category":"format-normalization","root_cause":"html_visible_answer_surface","failure_mode":"false-positive","case":{"url":"https://example.test/","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Welcome"]}]}},"artifact":"

CAPTCHA

Welcome

","title":"Home","ok":true,"expected_task_pass":false,"expected_metrics":{"blocked_page":true,"stable_blocked_page":true}}, {"id":"dynamic-undeclared-source-language-can-verify-identity","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-negative","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Visible content","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":""},"expected_task_pass":true,"expected_metrics":{"page_identity_status":"verified","language_consistent":true}}, - {"id":"dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-positive","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Visible content","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false}}, + {"id":"dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-positive","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Unrelated content","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false}}, {"id":"long-metadata-title-keeps-visible-html-anchor","category":"format-normalization","root_cause":"html_title_prefix_detection","failure_mode":"false-negative","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"

Required body

","title":"Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ","ok":true,"expected_task_pass":true,"expected_metrics":{"stable_anchor_hits":1}}, {"id":"long-metadata-title-cannot-expose-script-anchor","category":"format-normalization","root_cause":"html_title_prefix_detection","failure_mode":"false-positive","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"

Other text

","title":"Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ","ok":true,"expected_task_pass":false,"expected_metrics":{"stable_anchor_hits":0}}, {"id":"html-inline-challenge-explanation-is-prose","category":"format-normalization","root_cause":"html_block_line_boundaries","failure_mode":"false-negative","case":{"url":"https://example.test/article","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"

Required body. If you see Verify you are human, complete the verification before continuing.

","ok":true,"expected_task_pass":true,"expected_metrics":{"blocked_page":false,"stable_blocked_page":false}}, @@ -21795,6 +21795,12 @@ {"id": "dynamic-diagnostic-url-missing-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, {"id": "dynamic-diagnostic-url-missing-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, {"id": "dynamic-diagnostic-url-mismatch-valid", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, - {"id": "dynamic-diagnostic-url-mismatch-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}} + {"id": "dynamic-diagnostic-url-mismatch-missing-content", "category": "acceptance-contract", "root_cause": "final_url_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Other content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "mismatch", "final_url_consistent": false, "page_identity_required": false}, "final_url": "https://other.test/catalog", "metadata": {"route": "dynamic_selected_text", "document_language": "en"}}, + {"id": "dynamic-diagnostic-lang-missing-valid", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text"}}, + {"id": "dynamic-diagnostic-lang-missing-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text"}}, + {"id": "dynamic-diagnostic-lang-fr-valid", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "fr"}}, + {"id": "dynamic-diagnostic-lang-fr-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "fr"}}, + {"id": "dynamic-diagnostic-lang-empty-valid", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": ""}}, + {"id": "dynamic-diagnostic-lang-empty-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": ""}} ] } diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 524442e4f..96af5291b 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -549,8 +549,8 @@ rankings and catalogs change frequently. Annotate fields, not whole websites: uniqueness and stable page text. Values are not compared to a previous ranking. Duplicate cards cannot satisfy the minimum. Final-URL agreement is diagnostic: a missing or mismatched URL does not change acceptance or the - evaluated denominator. Declared document-language requirements remain - separate acceptance checks. + evaluated denominator. HTML document-language tags are also diagnostic; + content acceptance comes from the declared answer requirements. - Full-reference word precision, recall and F1 remain separate diagnostics. Historical demonstration snapshots describe their captured sample; they do not certify arbitrary later content. Without a comparable full reference, diff --git a/harness/web_fetch/metrics.py b/harness/web_fetch/metrics.py index 85c38b652..f04f29996 100644 --- a/harness/web_fetch/metrics.py +++ b/harness/web_fetch/metrics.py @@ -325,9 +325,6 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: # Scored records carry the case kind so dataset-free summaries use the same # predicate as the report. Explicit case context still takes precedence. task_kind = task_kind if task_kind is not None else metrics.get('task_kind', 'body') - if (metrics.get('document_language_required') - and not metrics.get('language_consistent')): - return False if (not task_measured(metrics, task_kind) or metrics.get('deadline_overrun') or metrics.get('blocked_page')): return False @@ -347,10 +344,6 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False - if (metrics.get('document_language_required') - and metrics.get('observed_document_language') is None - and metrics.get('request_ok') is not False): - return False kind = task_kind or metrics.get('task_kind', 'body') return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool) diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py index 6eea42615..ec012d965 100644 --- a/harness/web_fetch/scoring.py +++ b/harness/web_fetch/scoring.py @@ -189,7 +189,6 @@ def score_attempt( ) result.update( page_identity_required=False, - document_language_required=True, page_identity_status=identity_status, final_url_consistent=final_consistent, language_consistent=language_consistent, @@ -208,7 +207,6 @@ def score_attempt( ) result.update( page_identity_required=False, - document_language_required=False, page_identity_status=( "verified" if final_consistent else "mismatch" if final_url else "unavailable" ), @@ -216,8 +214,7 @@ def score_attempt( acceptable_final_urls=acceptable_urls, ) else: - result.update(page_identity_required=False, page_identity_status="not_required", - document_language_required=False) + result.update(page_identity_required=False, page_identity_status="not_required") result["task_kind"] = task_kind result["rule_review_status"] = rule_review.get("status", "resolved") if rule_review.get("reason"): @@ -259,13 +256,6 @@ def score_attempt( body_requirements, sort_keys=True, ensure_ascii=False).encode()).hexdigest() if not result["content_score_applicable"] or not ok: result["content_pass"] = False - if result.get("document_language_required") and not result.get("language_consistent"): - result["content_pass"] = False - if contract: - result["stable_task_pass"] = False - result["stable_content_pass"] = False - if "document_language_not_verified" not in result["stable_task_errors"]: - result["stable_task_errors"].append("document_language_not_verified") if not ok: if contract: result["stable_task_pass"] = False diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index e33ddf4cc..4b7ce31cb 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -96,6 +96,7 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) paired.add("inline_emphasis_in_structured_fields") paired.add("markdown_list_prefix_ownership") paired.add("final_url_diagnostic_not_acceptance") + paired.add("html_language_metadata_diagnostic_not_acceptance") paired.update({"table_header_visual_line", "table_adjacent_split_body", "composite_html_table_ownership"}) for root_cause in paired: From 15e96f7826626bc39ba170a5933ca2e87f1c91d3 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:33:58 +0800 Subject: [PATCH 4/4] Preserve visible table scopes and escaped pipe cells --- evaluation/scoring-regressions.json | 20 +++++++++++++++++- harness/web_fetch/composite_tasks.py | 16 +++++++------- harness/web_fetch/stable_tasks.py | 2 +- harness/web_fetch/structured_tables.py | 29 +++++++++++++++++++++++--- tests/test_scoring_regressions.py | 10 +++++++++ 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index 6d0910763..ab6e6f032 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -21801,6 +21801,24 @@ {"id": "dynamic-diagnostic-lang-fr-valid", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "fr"}}, {"id": "dynamic-diagnostic-lang-fr-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": "fr"}}, {"id": "dynamic-diagnostic-lang-empty-valid", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-negative", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Complete catalog", "ok": true, "expected_task_pass": true, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": ""}}, - {"id": "dynamic-diagnostic-lang-empty-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": ""}} + {"id": "dynamic-diagnostic-lang-empty-missing-content", "category": "acceptance-contract", "root_cause": "html_language_metadata_diagnostic_not_acceptance", "failure_mode": "false-positive", "case": {"url": "https://example.test/start", "track": "dynamic", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [{"id": "answer", "alternatives": ["Complete catalog"]}]}, "evaluation_url": "https://example.test/start", "expected_final_url": "https://example.test/catalog", "target_language": "en", "target_region": "US"}, "artifact": "Unrelated content", "ok": true, "expected_task_pass": false, "expected_metrics": {"page_identity_status": "unavailable", "final_url_consistent": false, "page_identity_required": false, "language_consistent": false}, "metadata": {"route": "dynamic_selected_text", "document_language": ""}}, + {"id": "table-visible_section-valid", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "

section_start

NameOwner
AlphaAlice | Bob
BetaAlice | Bob

section_end

NameOwner
OutsideAlice | Bob
OtherAlice | Bob
", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "table-visible_section-invalid", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "

section_start

NameOwner
AlphaAlice | Bob

section_end

NameOwner
OutsideAlice | Bob
OtherAlice | Bob
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "table-escaped_pipe-valid", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name"}]}}, "artifact": "
NameOwner
AlphaAlice | Bob
BetaAlice | Bob
", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "table-escaped_pipe-invalid", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name"}]}}, "artifact": "
NameOwner
AlphaAliceBob
BetaAliceBob
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "linked_table-visible_section-valid", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "

section_start

NameOwner
AlphaAlice | Bob
BetaAlice | Bob

section_end

NameOwner
OutsideAlice | Bob
OtherAlice | Bob
", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "linked_table-visible_section-invalid", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "

section_start

NameOwner
AlphaAlice | Bob

section_end

NameOwner
OutsideAlice | Bob
OtherAlice | Bob
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "linked_table-escaped_pipe-valid", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name"}]}}, "artifact": "
NameOwner
AlphaAlice | Bob
BetaAlice | Bob
", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "linked_table-escaped_pipe-invalid", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name"}]}}, "artifact": "
NameOwner
AlphaAliceBob
BetaAliceBob
", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "table-visible_section-valid-markdown", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "\nsection_start\n\n| Name | Owner |\n| --- | --- |\n| Alpha | Alice \\| Bob |\n| Beta | Alice \\| Bob |\n\nsection_end\n\n| Name | Owner |\n| --- | --- |\n| Outside | Alice \\| Bob |\n| Other | Alice \\| Bob |\n", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "table-visible_section-invalid-markdown", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "\nsection_start\n\n| Name | Owner |\n| --- | --- |\n| Alpha | Alice \\| Bob |\n\nsection_end\n\n| Name | Owner |\n| --- | --- |\n| Outside | Alice \\| Bob |\n| Other | Alice \\| Bob |\n", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "table-escaped_pipe-valid-markdown", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name"}]}}, "artifact": "\n| Name | Owner |\n| --- | --- |\n| Alpha | Alice \\| Bob |\n| Beta | Alice \\| Bob |\n", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "table-escaped_pipe-invalid-markdown", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name"}]}}, "artifact": "\n| Name | Owner |\n| --- | --- |\n| Alpha | Alice | Bob |\n| Beta | Alice | Bob |\n", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "linked_table-visible_section-valid-markdown", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "\nsection_start\n\n| Name | Owner |\n| --- | --- |\n| [Alpha](/repo/0) | Alice \\| Bob |\n| [Beta](/repo/1) | Alice \\| Bob |\n\nsection_end\n\n| Name | Owner |\n| --- | --- |\n| [Outside](/repo/0) | Alice \\| Bob |\n| [Other](/repo/1) | Alice \\| Bob |\n", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "linked_table-visible_section-invalid-markdown", "category": "structured-extraction", "root_cause": "table_visible_section_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name", "start_pattern": "(?m)^section_start$", "end_pattern": "(?m)^section_end$"}]}}, "artifact": "\nsection_start\n\n| Name | Owner |\n| --- | --- |\n| [Alpha](/repo/0) | Alice \\| Bob |\n\nsection_end\n\n| Name | Owner |\n| --- | --- |\n| [Outside](/repo/0) | Alice \\| Bob |\n| [Other](/repo/1) | Alice \\| Bob |\n", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "linked_table-escaped_pipe-valid-markdown", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name"}]}}, "artifact": "\n| Name | Owner |\n| --- | --- |\n| [Alpha](/repo/0) | Alice \\| Bob |\n| [Beta](/repo/1) | Alice \\| Bob |\n", "ok": true, "expected_task_pass": true, "expected_metrics": {"composite_acceptance_pass": true}}, + {"id": "linked_table-escaped_pipe-invalid-markdown", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "composite_acceptance": {"version": 1, "groups": [{"id": "entries", "kind": "linked_table", "minimum_records": 2, "required_columns": [{"id": "name", "kind": "text", "aliases": ["Name"]}, {"id": "owner", "kind": "text", "aliases": ["Owner"]}], "identity_column": "name", "hosts": ["example.test"], "entry_path": "/repo/[^/]+", "link_column": "name"}]}}, "artifact": "\n| Name | Owner |\n| --- | --- |\n| [Alpha](/repo/0) | Alice | Bob |\n| [Beta](/repo/1) | Alice | Bob |\n", "ok": true, "expected_task_pass": false, "expected_metrics": {"composite_acceptance_pass": false}}, + {"id": "stable-table-escaped-pipe-valid", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-negative", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Name | Amount |\n| --- | --- |\n| Alpha \\| Beta | 7 |\n| Gamma \\| Delta | 8 |", "ok": true, "expected_task_pass": true}, + {"id": "stable-table-escaped-pipe-wrong-columns", "category": "structured-extraction", "root_cause": "table_escaped_pipe_boundaries", "failure_mode": "false-positive", "case": {"url": "https://example.test/table", "task_kind": "body", "acceptance_contract": {"version": 1, "schema": "table_row", "minimum_records": 2, "table_columns": [{"name": "name", "label": "Name", "type": "text"}, {"name": "amount", "label": "Amount", "type": "integer"}], "record_identity_fields": ["name"], "table_scope": {"row_selector": "table tbody tr", "cell_selector": "td", "start_pattern": "^Name\\s*$", "end_pattern": "^Footer$", "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)"}}}, "artifact": "| Name | Amount |\n| --- | --- |\n| Alpha | Beta | 7 |\n| Gamma | Delta | 8 |", "ok": true, "expected_task_pass": false} ] } diff --git a/harness/web_fetch/composite_tasks.py b/harness/web_fetch/composite_tasks.py index 549a8a5af..7e5a19920 100644 --- a/harness/web_fetch/composite_tasks.py +++ b/harness/web_fetch/composite_tasks.py @@ -13,7 +13,7 @@ import unicodedata from urllib.parse import parse_qsl, unquote, urljoin, urlsplit -from .structured_tables import table_text_surface +from .structured_tables import _pipe_cells, table_text_surface from .dynamic_lists import _canonical_entry_path, extract_named_entries from .reference_quality import clean_reference_text, markdown_links, visible_markdown_links @@ -549,8 +549,8 @@ def _table_group(content: str, group: dict) -> dict: delimiter = "|" if "|" in lines[index] else "\t" if "\t" in lines[index] else None if not delimiter: continue - headers = [_key(visible_markdown_links(value)) for value in lines[index].strip("|").split(delimiter)] - separator = lines[index + 1].strip("|").split(delimiter) + headers = [_key(visible_markdown_links(value)) for value in (_pipe_cells(lines[index]) if delimiter == "|" else lines[index].split(delimiter))] + separator = (_pipe_cells(lines[index + 1]) if delimiter == "|" else lines[index + 1].split(delimiter)) if delimiter == "|" and not all( re.fullmatch(r"\s*:?-{3,}:?\s*", value) for value in separator ): @@ -578,7 +578,7 @@ def _table_group(content: str, group: dict) -> dict: continue row_index = index + 2 if delimiter == "|" else index + 1 while row_index < len(lines) and delimiter in lines[row_index]: - values = [value.strip() for value in lines[row_index].strip("|").split(delimiter)] + values = [value.strip() for value in (_pipe_cells(lines[row_index]) if delimiter == "|" else lines[row_index].split(delimiter))] if len(values) == len(headers): row = {} for field, (column, pattern) in mapping.items(): @@ -609,8 +609,8 @@ def _linked_table_group(content: str, group: dict, base_url: str) -> dict: for index in range(len(lines) - 1): if "|" not in lines[index]: continue - headers = [_key(visible_markdown_links(value)) for value in lines[index].strip("|").split("|")] - separator = lines[index + 1].strip("|").split("|") + headers = [_key(visible_markdown_links(value)) for value in _pipe_cells(lines[index])] + separator = _pipe_cells(lines[index + 1]) if not all(re.fullmatch(r"\s*:?-{3,}:?\s*", value) for value in separator): continue mapping = {} @@ -626,7 +626,7 @@ def _linked_table_group(content: str, group: dict, base_url: str) -> dict: else: row_index = index + 2 while row_index < len(lines) and "|" in lines[row_index]: - values = [value.strip() for value in lines[row_index].strip("|").split("|")] + values = [value.strip() for value in _pipe_cells(lines[row_index])] if len(values) == len(headers): row = {field: values[column] for field, column in mapping.items()} cell = row[group["link_column"]] @@ -686,6 +686,8 @@ def evaluate_composite_acceptance( results = [] for group in contract["groups"]: source = table_content if table_content is not None and group["kind"] in {"table", "linked_table"} else content + if group["kind"] in {"table", "linked_table"}: + source = table_text_surface(source) scoped = _scope(source, group) if group["kind"] == "named_link": result = _named_link_group( diff --git a/harness/web_fetch/stable_tasks.py b/harness/web_fetch/stable_tasks.py index be0ea4661..37232dcfb 100644 --- a/harness/web_fetch/stable_tasks.py +++ b/harness/web_fetch/stable_tasks.py @@ -143,7 +143,7 @@ def _visible(text, schema=None): text = _unwrap_links(text, schema) # CommonMark punctuation escapes preserve the same visible characters. # Remove them before date/field parsing (for example ``2023\-05\-18``). - text = re.sub(r'\\([!"#$%&\'()*+,\-./:;<=>?@\[\]\\^_`{|}~])', r'\1', text) + text = re.sub(r'\\([!"#$%&\'()*+,\-./:;<=>?@\[\]\\^_`{|}~])', lambda m: m[0] if schema == 'table_row' and m[1] in {'|', '\\'} else m[1], text) if schema == "name_description": # Image-only card links intentionally contribute no visible title. # Remove their now-empty bullet line so blank-delimited card parsing diff --git a/harness/web_fetch/structured_tables.py b/harness/web_fetch/structured_tables.py index 03900d960..68f0c9ce4 100644 --- a/harness/web_fetch/structured_tables.py +++ b/harness/web_fetch/structured_tables.py @@ -186,6 +186,11 @@ def table_text_surface(content): if re.search(r"]*>[\s\S]*?", + lambda m: "\n" + BeautifulSoup(m[0], "html.parser").get_text(" ", strip=True) + "\n", + content, flags=re.I, + ) def convert(match): soup = BeautifulSoup(match[0], 'html.parser') @@ -201,7 +206,7 @@ def convert(match): break for link in cell.find_all('a', href=True): link.replace_with('[' + link.get_text(' ', strip=True) + '](' + link['href'] + ')') - values.append(cell.get_text(' ', strip=True).replace('|', r'\|')) + values.append(cell.get_text(' ', strip=True).replace('\\', r'\\').replace('|', r'\|')) else: if values: lines.append('| ' + ' | '.join(values) + ' |') @@ -374,5 +379,23 @@ def capture(match): def _pipe_cells(line): - # Remove only the table's outer delimiters; adjacent pipes are empty cells. - return line.strip().removeprefix('|').removesuffix('|').split('|') + """Split real delimiters, preserving escaped pipes and empty owned cells.""" + line = line.strip() + cells = [''] + index = 0 + while index < len(line): + char = line[index] + if char == '\\' and index + 1 < len(line) and line[index + 1] in {'|', '\\'}: + cells[-1] += line[index + 1] + index += 2 + continue + if char == '|': + cells.append('') + else: + cells[-1] += char + index += 1 + if line.startswith('|'): + cells.pop(0) + if cells and cells[-1] == '' and line.endswith('|'): + cells.pop() + return cells diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index 4b7ce31cb..c089f6da6 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -73,6 +73,15 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) self.assertEqual(verdict, row["expected_task_pass"]) for key, expected in row.get("expected_metrics", {}).items(): self.assertEqual(metrics.get(key), expected, key) + if (row["root_cause"] == "table_escaped_pipe_boundaries" + and row["expected_task_pass"]): + if row["case"].get("composite_acceptance"): + record = metrics["composite_group_results"][0]["records"][0] + fields = record.get("fields", record) + self.assertEqual(fields["owner"], "Alice | Bob") + else: + self.assertEqual(metrics["stable_records"][0]["fields"]["name"], + "Alpha | Beta") root_modes.setdefault(row["root_cause"], set()).add(row["failure_mode"]) paired = {"answer_surface_omission", "body_scope_calibration", "declared_scope_exclusions", "declared_truncation", @@ -97,6 +106,7 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) paired.add("markdown_list_prefix_ownership") paired.add("final_url_diagnostic_not_acceptance") paired.add("html_language_metadata_diagnostic_not_acceptance") + paired.update({"table_visible_section_boundaries", "table_escaped_pipe_boundaries"}) paired.update({"table_header_visual_line", "table_adjacent_split_body", "composite_html_table_ownership"}) for root_cause in paired: