Repository navigation
feat(ocr): add font sanity heuristics and corrupt text OCR routing #4643
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| # SPDX-FileCopyrightText: The Docling Contributors | ||
| # SPDX-License-Identifier: MIT | ||
|
|
||
| import re | ||
| import unicodedata | ||
|
|
||
| # Core PDF dump artifact regexes | ||
| GLYPH_RE = re.compile(r"GLYPH<[0-9A-Fa-f]+>") | ||
| SLASH_G_RE = re.compile(r"(?:/G\d+){2,}") | ||
| FRAG_RE = re.compile(r"\b[A-Za-z](?:/[a-z]{1,3}\.[a-z]{1,3}){2,}\b") | ||
| SLASH_NUMBER_GARBAGE_RE = re.compile(r"(?:/\w+\s*){2,}") | ||
|
|
||
| # Unprintable control characters (excluding standard whitespace \t, \n, \r) | ||
| CONTROL_CHAR_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]") | ||
|
|
||
| # Genuinely corrupt intra-word mojibake noise (e.g. backslash, caret, tilde, pipe, backtick inside words) | ||
| # Legitimate characters like &, +, =, -, *, /, $, % are explicitly permitted | ||
| MOJIBAKE_SYMBOL_RE = re.compile( | ||
|
Contributor
There was a problem hiding this comment. blocker: Intra-word mojibake regex classifies valid symbols and short tokens as corrupt The regex |
||
| r"[A-Za-z0-9\u00C0-\u024F\u0400-\u04FF\u0600-\u06FF\u0900-\u0D7F\u4E00-\u9FFF][\\^~|`#][A-Za-z0-9\u00C0-\u024F\u0400-\u04FF\u0600-\u06FF\u0900-\u0D7F\u4E00-\u9FFF]" | ||
| ) | ||
|
|
||
|
|
||
| def _is_pua(ch: str) -> bool: | ||
| """Check if a character belongs to Unicode Private Use Area.""" | ||
| return unicodedata.category(ch) == "Co" or ( | ||
| "\ue000" <= ch <= "\uf8ff" | ||
| or "\U000f0000" <= ch <= "\U000ffffd" | ||
| or "\U00100000" <= ch <= "\U0010fffd" | ||
| ) | ||
|
|
||
|
|
||
| def _has_corrupt_pua(text: str) -> bool: | ||
| """Detect unmapped PUA font dumps vs benign presentation bullet/icon fonts.""" | ||
| non_space = [ch for ch in text if not ch.isspace()] | ||
| if not non_space: | ||
| return False | ||
| pua_chars = [ch for ch in non_space if _is_pua(ch)] | ||
| if not pua_chars: | ||
| return False | ||
|
|
||
| # Icons / bullets repeat 1 or 2 unique code points; unmapped font alphabets vary | ||
| if len(set(pua_chars)) >= 3: | ||
| return True | ||
|
|
||
| # If PUA chars dominate the text block (>= 20%), it is an unmapped font dump | ||
| if len(pua_chars) / len(non_space) >= 0.20: | ||
|
Contributor
There was a problem hiding this comment. blocker: Standalone PUA bullet point and icon cells trigger false positive corruption PDF parsers frequently emit bullet points and symbols as isolated single-character text cells (for example |
||
| return True | ||
|
|
||
| return False | ||
|
|
||
|
|
||
| def rate_text_quality(text: str) -> float: | ||
| """Rate extracted text quality and detect font corruption/mojibake. | ||
|
|
||
| Returns a score between 0.0 (severely corrupted) and 1.0 (clean text). | ||
| """ | ||
| if not text: | ||
| return 1.0 | ||
|
|
||
| # 1. Hard parser errors: replacement chars, control characters, GLYPH dumps, slash-garbage | ||
| if ( | ||
| "\ufffd" in text | ||
| or CONTROL_CHAR_RE.search(text) | ||
| or GLYPH_RE.search(text) | ||
| or SLASH_G_RE.search(text) | ||
| or SLASH_NUMBER_GARBAGE_RE.match(text) | ||
| ): | ||
| return 0.0 | ||
|
|
||
| # 2. Corrupt unmapped Private Use Area alphabets (while allowing single/repeated bullet icons) | ||
| if _has_corrupt_pua(text): | ||
| return 0.0 | ||
|
|
||
| # 3. Mojibake intra-word noise (e.g., d\b^c~e) | ||
| tokens = text.split() | ||
| if tokens: | ||
| mojibake_count = sum(1 for t in tokens if MOJIBAKE_SYMBOL_RE.search(t)) | ||
| if (mojibake_count / len(tokens)) >= 0.25: | ||
| return 0.0 | ||
|
|
||
| # 4. Repeated font fragment paths penalty | ||
| penalty = 0.0 | ||
| frag_matches = FRAG_RE.findall(text) | ||
| if len(frag_matches) >= 3: | ||
| penalty += 0.1 * len(frag_matches) | ||
|
|
||
| return max(1.0 - penalty, 0.0) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
blocker: Cell merging applies text quality filtering to code and formula cells without layout label awareness
While
_find_pdf_aware_layout_ocr_rects()skips code and formula clusters,_merge_ocr_and_pdf_cells()filters all visible PDF cells throughrate_text_quality(). Code and formula cells containing backslashes, carets, pipes, or hash symbols are marked as corrupt. When OCR cells exist for the page, OCR text overwrites these native PDF code and formula cells. Preserve programmatic text in code and formula regions or avoid cell-level quality filtering where layout structure marks the region as code or formula.