Skip to content

Fix forbidden phrase boundaries and guard scoped link ownership #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lanyue-llk wants to merge 1 commit into from
Closed

Fix forbidden phrase boundaries and guard scoped link ownership #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
260 changes: 259 additions & 1 deletion evaluation/scoring-regressions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21777,6 +21777,264 @@
"expected_metrics": {
"stable_unique_record_count": 0
}
}
},
{
"id": "forbidden-term-longer-label-is-not-error",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-negative",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"required_facts": [
{
"alternatives": [
"Latest news"
]
}
],
"forbidden_any": [
"Not Found"
],
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/story/[^/]+"
}
},
"artifact": "Latest news\n[Story](https://example.test/story/one)\n[Page Not FoundExtra](https://example.test/column)",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
"forbidden_hits": []
}
},
{
"id": "forbidden-term-complete-phrase-still-fails",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"required_facts": [
{
"alternatives": [
"Latest news"
]
}
],
"forbidden_any": [
"Not Found"
],
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/story/[^/]+"
}
},
"artifact": "Latest news\n[Story](https://example.test/story/one)\nPage Not Found.",
"ok": true,
"expected_task_pass": false,
"expected_metrics": {
"forbidden_hits": [
"Not Found"
]
}
},
{
"id": "forbidden-term-prefixed-word-is-not-error",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-negative",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"required_facts": [
{
"alternatives": [
"Latest news"
]
}
],
"forbidden_any": [
"Not Found"
],
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/story/[^/]+"
}
},
"artifact": "Latest news\n[Story](https://example.test/story/one)\nLatestNot Found",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
"forbidden_hits": []
}
},
{
"id": "named-caption-owned-by-link",
"category": "structured-extraction",
"root_cause": "scoped_markdown_caption_ownership",
"failure_mode": "false-negative",
"case": {
"url": "https://example.test/shows",
"task_kind": "body",
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/series/[^/]+",
"name_validation": "nonempty",
"entry_scope": {
"html_selector": "main a:has(p)",
"markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$"
}
}
},
"artifact": "[Artwork<br>Series One<br>](https://example.test/series/one)",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
"stable_record_count": 1
}
},
{
"id": "named-caption-nearby-is-not-owned",
"category": "structured-extraction",
"root_cause": "scoped_markdown_caption_ownership",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/shows",
"task_kind": "body",
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/series/[^/]+",
"name_validation": "nonempty",
"entry_scope": {
"html_selector": "main a:has(p)",
"markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$"
}
}
},
"artifact": "### Series One\n\n[Mehr Infos](https://example.test/series/other)\n\nSeries One<br>\n",
"ok": true,
"expected_task_pass": false,
"expected_metrics": {
"stable_record_count": 0
}
},
{
"id": "named-caption-unqualified-title-link",
"category": "structured-extraction",
"root_cause": "scoped_markdown_caption_ownership",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/shows",
"task_kind": "body",
"acceptance_contract": {
"version": 1,
"schema": "named_link",
"minimum_records": 1,
"hosts": [
"example.test"
],
"entry_path": "/series/[^/]+",
"name_validation": "nonempty",
"entry_scope": {
"html_selector": "main a:has(p)",
"markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$"
}
}
},
"artifact": "### [Series One](https://example.test/series/one)",
"ok": true,
"expected_task_pass": false,
"expected_metrics": {
"stable_record_count": 0
}
},
{
"id": "forbidden-cjk-remains-unsegmented",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"forbidden_any": [
"访问受限"
]
},
"artifact": "当前页面访问受限，请稍后重试",
"ok": true,
"expected_task_pass": false,
"expected_metrics": {
"forbidden_hits": [
"访问受限"
]
}
},
{
"id": "forbidden-url-path-is-not-visible",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-negative",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"forbidden_any": [
"Not Found"
]
},
"artifact": "[Latest news](https://example.test/not-found)",
"ok": true,
"expected_task_pass": true,
"expected_metrics": {
"forbidden_hits": []
}
},
{
"id": "forbidden-phrase-case-and-markup",
"category": "format-normalization",
"root_cause": "forbidden_phrase_substring_collision",
"failure_mode": "false-positive",
"case": {
"url": "https://example.test/news",
"task_kind": "body",
"forbidden_any": [
"Not Found"
]
},
"artifact": "**NOT** FOUND: requested page is unavailable.",
"ok": true,
"expected_task_pass": false,
"expected_metrics": {
"forbidden_hits": [
"Not Found"
]
}
}
]
}
13 changes: 11 additions & 2 deletions harness/web_fetch/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,15 @@ def fact_in_text(text: str, fact: str) -> bool:
return False


def forbidden_in_text(text: str, phrase: str) -> bool:
"""Match ASCII word boundaries without segmenting CJK phrases."""
if not phrase:
return False
left = r'(?<![a-z0-9_])' if phrase[0].isascii() and phrase[0].isalnum() else ''
right = r'(?![a-z0-9_])' if phrase[-1].isascii() and phrase[-1].isalnum() else ''
return bool(re.search(left + re.escape(phrase) + right, text))


def text_corruption(value: str) -> tuple[int, float, bool]:
"""Detect substantial UTF-8 replacement-character damage in text output."""
count = value.count('\ufffd')
Expand Down Expand Up @@ -534,12 +543,12 @@ def evaluate_content(
]
block_hits = [hit for hit in block_hits if hit not in block_reference_exemptions]
reference_exemptions = [item for item in (forbidden_any or [])
if reference_folded and (text := normalize_for_match(item)) and text in reference_folded]
if reference_folded and forbidden_in_text(reference_folded, normalize_for_match(item))]
if acceptance_contract:
# A historical snapshot cannot exempt an explicit stable-mode block rule.
reference_exemptions = []
forbidden = [item for item in (forbidden_any or [])
if item not in reference_exemptions and (text := normalize_for_match(item)) and text in folded]
if item not in reference_exemptions and forbidden_in_text(folded, normalize_for_match(item))]
lines = [re.sub(r"\s+", " ", line).strip().casefold() for line in answer_text.splitlines()]
lines = [line for line in lines if line]
unique_line_ratio = round(len(set(lines)) / len(lines), 4) if lines else 1.0
Expand Down
2 changes: 2 additions & 0 deletions tests/test_scoring_regressions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self)
paired.add("declared_full_page_stable_text_scope")
paired.add("inline_emphasis_in_structured_fields")
paired.add("markdown_list_prefix_ownership")
paired.add("forbidden_phrase_substring_collision")
paired.add("scoped_markdown_caption_ownership")
for root_cause in paired:
self.assertEqual(root_modes.get(root_cause), {"false-positive", "false-negative"})

Expand Down
Loading