From cfde734bb53507000764f3e3b6e6371c8e774bbf Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:08:57 +0530 Subject: [PATCH 01/15] feat(ocr): replace broken PDF text layers with OCR Some PDFs carry a text layer that does not match the text drawn on the page, such as an unmapped font encoding or leftover noise from a scanner's OCR. Docling trusts that layer, so the noise ends up in the output, and the page's parse_score still reports 1.0 because the existing per-cell checks only look for glyph-name dumps. Add docling.utils.text_quality.is_broken_text_layer(), which judges a page's whole text layer on patterns only broken text produces: control, private-use and replacement characters, symbols inside words, non-ASCII Latin runs, punctuation soup, image file names left by scanners, and unusual characters on scanned pages. Symbols common in ordinary documents never count. Page preprocessing sets parse_score to 0.0 for a broken page, so the score, its poor grade and the document-level score reflect it. Add OcrOptions.replace_broken_text_layer (default False). When enabled, a page whose parse_score is graded poor (below 0.5) is OCR'd in full and its OCR text replaces its PDF text, as OcrMode.FULL_PAGE does, for that page only. The option lives in the base OCR model, so every engine, including external plugins, supports it. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/datamodel/base_models.py | 3 + docling/datamodel/pipeline_options.py | 14 ++ docling/models/base_ocr_model.py | 52 +++-- .../page_preprocessing_model.py | 13 +- docling/utils/text_quality.py | 185 ++++++++++++++++++ tests/test_replace_broken_text_layer.py | 147 ++++++++++++++ tests/test_text_quality.py | 127 ++++++++++++ 7 files changed, 528 insertions(+), 13 deletions(-) create mode 100644 docling/utils/text_quality.py create mode 100644 tests/test_replace_broken_text_layer.py create mode 100644 tests/test_text_quality.py diff --git a/docling/datamodel/base_models.py b/docling/datamodel/base_models.py index e337a8502e..0db67b0aaf 100644 --- a/docling/datamodel/base_models.py +++ b/docling/datamodel/base_models.py @@ -519,6 +519,9 @@ class Page(BaseModel): # conversion results. _shape_lines: list[BoundingBox] | None = PrivateAttr(default=None) _shape_bounding_boxes: list[BoundingBox] | None = PrivateAttr(default=None) + # Quality of the page's PDF text layer, as in its confidence report; read by + # the OCR stage to replace a broken text layer (OcrOptions.replace_broken_text_layer). + _parse_score: float = PrivateAttr(default=float("nan")) _default_image_scale: float = 1.0 # Default image scale for external usage. _image_cache: dict[ float, Image diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index d3f52a7a6b..e090f002d9 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -257,6 +257,20 @@ class OcrOptions(BaseOptions): ), ] = 3.0 + replace_broken_text_layer: Annotated[ + bool, + Field( + description=( + "Replace a broken PDF text layer with OCR, page by page. A page whose" + " `parse_score` is in the `poor` quality grade (below 0.5) is OCR'd in" + " full and its OCR text is used instead of its PDF text" + " for that page only. Other pages, and pages" + " without a text layer, keep the usual behaviour." + ), + examples=[False, True], + ), + ] = False + model_config = ConfigDict( validate_assignment=True, validate_default=True, diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index a39ba5ce29..800ca2ad1d 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -139,6 +139,24 @@ def _segregate_by_visibility( return visible, invisible +# Upper bound of QualityGrade.POOR (PageConfidenceScores._score_to_grade). +_POOR_PARSE_SCORE = 0.5 + + +def _ocr_full_page(options: OcrOptions, page: Page) -> bool: + """Whether a page is OCR'd in full and its PDF cells dropped. + + True in `OcrMode.FULL_PAGE`, and, with `replace_broken_text_layer`, for a page + whose text layer is graded poor. A page without a text layer has no parse score + (NaN) and is never poor. + """ + if options.mode == OcrMode.FULL_PAGE: + return True + if not getattr(options, "replace_broken_text_layer", False): + return False + return page._parse_score < _POOR_PARSE_SCORE + + class BaseOcrModel(BasePageModel, BaseModelWithOptions): r""" Base class for all OCR models. @@ -231,15 +249,14 @@ def get_ocr_rects(self, page: Page) -> list[BoundingBox]: # Compute the OCR rects according to the mode ocr_rects: list[BoundingBox] - # Both DEFAULT and PDF_AWARE_LAYOUT_REGIONS make OCR input as layout detections eliminated by PDF cells - if ( - self.options.mode == OcrMode.DEFAULT - or self.options.mode == OcrMode.PDF_AWARE_LAYOUT_REGIONS - ): - ocr_rects = self._find_pdf_aware_layout_ocr_rects(page) - elif self.options.mode == OcrMode.LAYOUT_REGIONS: - ocr_rects = self._find_layout_ocr_rects(page) - elif self.options.mode == OcrMode.FULL_PAGE: + if _ocr_full_page(self.options, page): + if self.options.mode != OcrMode.FULL_PAGE: + _log.info( + "Page %s: parse score %.2f is graded poor;" + " replacing its text layer with full-page OCR", + page.page_no, + page._parse_score, + ) # A big bbox covering the entire page ocr_rects = [ BoundingBox( @@ -250,6 +267,14 @@ def get_ocr_rects(self, page: Page) -> list[BoundingBox]: coord_origin=CoordOrigin.TOPLEFT, ) ] + # Both DEFAULT and PDF_AWARE_LAYOUT_REGIONS make OCR input as layout detections eliminated by PDF cells + elif ( + self.options.mode == OcrMode.DEFAULT + or self.options.mode == OcrMode.PDF_AWARE_LAYOUT_REGIONS + ): + ocr_rects = self._find_pdf_aware_layout_ocr_rects(page) + elif self.options.mode == OcrMode.LAYOUT_REGIONS: + ocr_rects = self._find_layout_ocr_rects(page) return ocr_rects def _find_layout_ocr_rects(self, page: Page) -> list[BoundingBox]: @@ -427,7 +452,9 @@ def post_process_cells( r""" Post-process the OCR cells and update the page object according to the algorithm: - - If FULL_PAGE: Any existing PDF cells are ignored and only the OCR cells are used. + - If FULL_PAGE, or with `replace_broken_text_layer` the page's parse score is + graded poor: + Any existing PDF cells are ignored and only the OCR cells are used. - If LAYOUT_REGIONS or PDF_AWARE_LAYOUT_REGIONS and the priority parameter is None, the priority is auto-selected based on the OcrMode: - OCR_FIRST when LAYOUT_REGIONS @@ -438,7 +465,8 @@ def post_process_cells( existing_cells = page.cells # Combine existing and OCR cells with overlap filtering - if self.options.mode == OcrMode.FULL_PAGE: + full_page = _ocr_full_page(self.options, page) + if full_page: final_cells = ocr_cells else: if priority is None: @@ -468,7 +496,7 @@ def post_process_cells( page.parsed_page.has_lines = len(final_cells) > 0 # In OcrMode.FULL_PAGE, PDF-extracted word/char cells are unreliable. Keep only OCR cells - if self.options.mode == OcrMode.FULL_PAGE: + if full_page: page.parsed_page.word_cells = [ c for c in page.parsed_page.word_cells if c.from_ocr ] diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index 26a188fb7e..c2b810e8ef 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -20,6 +20,7 @@ from docling.datamodel.settings import settings from docling.models.base_model import BasePageModel from docling.utils.profiling import TimeRecorder +from docling.utils.text_quality import is_broken_text_layer def resolve_skip_cell_extraction(pipeline_options: PdfPipelineOptions) -> bool: @@ -114,12 +115,22 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: warnings.filterwarnings( "ignore", "Mean of empty slice", RuntimeWarning, "numpy" ) - conv_res.confidence.pages[page.page_no].parse_score = float( + parse_score = float( np.nanquantile( text_scores, q=0.10 ) # To emphasise problems in the parse_score, we take the 10% percentile score of all text cells. ) + # Some broken text layers only show across the whole page (a font encoding + # read as control characters, a legacy font read as Latin letters). + if not np.isnan(parse_score) and is_broken_text_layer( + "\n".join(c.text for c in page.cells), + has_bitmap=any(True for _ in page._backend.get_bitmap_rects()), + ): + parse_score = 0.0 + conv_res.confidence.pages[page.page_no].parse_score = parse_score + page._parse_score = parse_score + def draw_cell_boxes( image: Image, text_cells: Iterable[TextCell], diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py new file mode 100644 index 0000000000..45c55bbdff --- /dev/null +++ b/docling/utils/text_quality.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Detect a PDF text layer that does not match the text drawn on the page. + +The checks look at a page's whole text layer and only at patterns broken text +produces, such as unmapped font encodings or leftover OCR noise. +""" + +import re +import unicodedata +from collections.abc import Sequence + +# A short layer that only names an image file, left behind by a scanner. +_IMAGE_FILENAME_RE = re.compile(r"(?i)\.(?:tif|tiff|jpe?g|png|gif|bmp|pict)\b") +_COMMON_PUNCT = frozenset( + ".,;:'\"!?()-[]/%$&+#@=<>/—\N{EN DASH}\N{RIGHT SINGLE QUOTATION MARK}“”«»" +) +# Symbols ordinary documents are full of; never evidence of a broken layer. +_ORDINARY_SYMBOLS = frozenset( + "*•▪■□◆◇●○►▶✓✔☐☒☑_®©™±°|≤≥≈\N{MULTIPLICATION SIGN}÷\N{MINUS SIGN}\N{EN DASH}—…§¶†‡€£¥¢‰\N{PRIME}″←→↑↓" +) +# Symbols that, inside a word, mark a non-Latin font read as Latin letters. +_IN_WORD_SYMBOLS = frozenset("$`©«»¥÷½{}^~±§@#%\\<>¤¦¨¬®¯°¹²³¼¾\N{MULTIPLICATION SIGN}") +# Characters a punctuation-soup token may hold without counting: leaders, $, dashes. +_SOUP_OK = frozenset(".,-\N{EN DASH}—$()%:;'\"/…_•▪*") +_BROKEN_CATEGORIES = frozenset({"Cc", "Co", "Cn", "Cs"}) + +_MIN_CHARS = 20 # Less text than this is left alone. +_SHORT = 120 # Below this length, ratio checks are unreliable. +_BROKEN_CHAR_SHARE = 0.10 # Control, private-use or U+FFFD characters. +_SYMBOL_WORD_SHARE = 0.20 # Words with symbols between their letters. +_NON_ASCII_LATIN_SHARE = 0.5 # Latin letters outside ASCII. +_SOUP_SHARE = 0.30 # Words made mostly of symbols. +_UNUSUAL_PER_100_LETTERS = 2.0 # Unusual characters on a scan. + + +def _letter_scripts(text: str) -> dict[str, int]: + counts = {"latin": 0, "cyrillic": 0, "cjk": 0, "arabic": 0, "other": 0} + for ch in text: + if not ch.isalpha(): + continue + name = unicodedata.name(ch, "") + if "CYRILLIC" in name: + counts["cyrillic"] += 1 + elif any( + token in name + for token in ("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "IDEOGRAPH") + ): + counts["cjk"] += 1 + elif "ARABIC" in name: + counts["arabic"] += 1 + elif "LATIN" in name: + counts["latin"] += 1 + else: + counts["other"] += 1 + return counts + + +def _is_ordinary(ch: str) -> bool: + """Not a letter, but nothing a broken layer produces either.""" + category = unicodedata.category(ch) + return ( + ch.isalnum() + or ch.isspace() + or ch in _COMMON_PUNCT + or ch in _ORDINARY_SYMBOLS + or category[0] == "M" # combining marks: Thai / Devanagari vowel signs + or category == "Co" # private use: icon-font glyphs + ) + + +def _unusual_per_100_letters(text: str) -> float: + letters = 0 + unusual = 0 + for ch in text: + if ch.isalpha(): + letters += 1 + elif not _is_ordinary(ch): + unusual += 1 + return 100.0 * unusual / letters if letters else 0.0 + + +def _broken_char_share(text: str) -> float: + """Share of non-space characters that are control, private-use or U+FFFD.""" + chars = [ch for ch in text if not ch.isspace()] + broken = sum( + 1 + for ch in chars + if unicodedata.category(ch) in _BROKEN_CATEGORIES or ch == "\ufffd" + ) + return broken / len(chars) if chars else 0.0 + + +def _symbol_word_share(words: Sequence[str]) -> float: + """Share of lettered words with two or more symbols between their letters.""" + lettered = [w for w in words if len(w) >= 3 and any(ch.isalpha() for ch in w)] + if not lettered: + return 0.0 + mixed = 0 + for word in lettered: + letters = [i for i, ch in enumerate(word) if ch.isalpha()] + inner = word[letters[0] : letters[-1]] + if sum(ch in _IN_WORD_SYMBOLS for ch in inner) >= 2: + mixed += 1 + return mixed / len(lettered) + + +def _non_ascii_latin_share(text: str) -> float: + """Share of Latin letters outside ASCII, when Latin carries the page.""" + letters = [ch for ch in text if ch.isalpha()] + latin = [ + ch + for ch in letters + if "LATIN" in unicodedata.name(ch, "") and not "\uff00" <= ch <= "\uffef" + ] + if len(latin) < 50 or len(latin) * 2 < len(letters): + return 0.0 + return sum(ord(ch) > 127 for ch in latin) / len(latin) + + +def _soup_share(words: Sequence[str]) -> float: + """Share of words that are mostly symbols beyond leaders, currency and dashes.""" + if not words: + return 0.0 + soup = sum( + 1 + for word in words + if sum(ch.isalnum() for ch in word) * 2 < len(word) + and any(not ch.isalnum() and ch not in _SOUP_OK for ch in word) + ) + return soup / len(words) + + +def is_broken_text_layer(text: str, *, has_bitmap: bool) -> bool: + """Whether a page's PDF text is not the text drawn on the page. + + Args: + text: All text of one page's PDF text layer, cells joined by newlines. + has_bitmap: Whether the page holds an image, i.e. may be a scan. + + Returns: + ``True`` only for text a broken layer produces. When unsure it returns + ``False``, so pages keep their PDF text. + """ + stripped = (text or "").strip() + if not stripped: + return False + + if len(stripped) < _SHORT and _IMAGE_FILENAME_RE.search(stripped): + return True + + # A broken font encoding, whether or not the page holds an image. + if ( + len(stripped) >= _MIN_CHARS + and _broken_char_share(stripped) >= _BROKEN_CHAR_SHARE + ): + return True + + words = stripped.split() + if len(stripped) >= _SHORT and ( + _symbol_word_share(words) >= _SYMBOL_WORD_SHARE + or _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE + or _soup_share(words) >= _SOUP_SHARE + ): + return True + + # The remaining checks only apply to scans with a hidden OCR layer. + if not has_bitmap or len(stripped) < _MIN_CHARS: + return False + + scripts = _letter_scripts(stripped) + if scripts["cyrillic"] >= 30 or scripts["cjk"] >= 20 or scripts["arabic"] >= 20: + return False + + letters = sum(ch.isalpha() for ch in stripped) + unusual = _unusual_per_100_letters(stripped) + if len(stripped) < _SHORT and ( + letters / len(stripped) < 0.5 or unusual >= _UNUSUAL_PER_100_LETTERS + ): + return True + return ( + scripts["latin"] >= max(1, letters * 0.5) + and unusual >= _UNUSUAL_PER_100_LETTERS + ) diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py new file mode 100644 index 0000000000..9a09d67fd2 --- /dev/null +++ b/tests/test_replace_broken_text_layer.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Replacing a broken PDF text layer with OCR (OcrOptions.replace_broken_text_layer). + +A page whose text layer is broken scores 0 in its parse score; with the option +on, the OCR stage reads every page graded poor (below 0.5) in full and keeps only +its OCR text. Other pages, and pages without a text layer, are left as they were. +""" + +import math +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from docling_core.types.doc.page import BoundingRectangle, TextCell + +from docling.datamodel.base_models import ConfidenceReport, Page, Size +from docling.datamodel.pipeline_options import OcrMode, OcrOptions +from docling.models.base_ocr_model import BaseOcrModel, _empty_segmented_page +from docling.models.stages.page_preprocessing.page_preprocessing_model import ( + PagePreprocessingModel, + PagePreprocessingOptions, +) + +_BROKEN = "&>66*;B \x1b8?.;7*7,. \x182;.,=8;< \x17869.7<*=287#;898<*5<" +_CLEAN = "We have audited the accompanying financial statements of the village." + + +def _cell(text: str, *, from_ocr: bool, y: float) -> TextCell: + return TextCell( + rect=BoundingRectangle( + r_x0=10, + r_y0=y, + r_x1=300, + r_y1=y, + r_x2=300, + r_y2=y + 12, + r_x3=10, + r_y3=y + 12, + ), + text=text, + orig=text, + from_ocr=from_ocr, + confidence=0.9 if from_ocr else 1.0, + ) + + +def _page(*pdf_texts: str, parse_score: float = math.nan) -> Page: + page = Page(page_no=1) + page.size = Size(width=600.0, height=800.0) + page.parsed_page = _empty_segmented_page(page) + page.parsed_page.textline_cells = [ + _cell(text, from_ocr=False, y=700 - 20 * i) for i, text in enumerate(pdf_texts) + ] + page.parsed_page.has_lines = bool(pdf_texts) + page._parse_score = parse_score + return page + + +def _ocr_model(mode: OcrMode, replace: bool, **methods: Mock): + options = SimpleNamespace(mode=mode, replace_broken_text_layer=replace) + return SimpleNamespace(options=options, **methods) + + +@pytest.mark.parametrize( + ("texts", "expected"), [((_BROKEN,) * 4, 0.0), ((_CLEAN,) * 4, 1.0)] +) +def test_parse_score_reflects_a_broken_text_layer( + texts: tuple[str, ...], expected: float +) -> None: + page = _page() + segmented = _page(*texts).parsed_page + page._backend = SimpleNamespace( + get_segmented_page=lambda: segmented, get_bitmap_rects=lambda: iter(()) + ) # type: ignore[assignment] + conv_res = SimpleNamespace(confidence=ConfidenceReport()) + model = PagePreprocessingModel(PagePreprocessingOptions(images_scale=None)) + + model._parse_page_cells(conv_res, page) # type: ignore[arg-type] + + assert conv_res.confidence.pages[1].parse_score == expected + assert page._parse_score == expected + + +def test_page_below_threshold_is_ocrd_in_full() -> None: + page = _page(_BROKEN, parse_score=0.0) + find_rects = Mock(return_value=[]) + model = _ocr_model( + OcrMode.DEFAULT, True, _find_pdf_aware_layout_ocr_rects=find_rects + ) + + (rect,) = BaseOcrModel.get_ocr_rects(model, page) # type: ignore[arg-type] + + assert (rect.l, rect.t, rect.r, rect.b) == (0, 0, 600.0, 800.0) + find_rects.assert_not_called() + + +@pytest.mark.parametrize( + ("replace", "parse_score"), + [ + (True, 1.0), # good text layer + (True, math.nan), # no text layer: left to the usual OCR + (False, 0.0), # option off: behaviour unchanged + ], +) +def test_other_pages_keep_their_ocr_regions(replace: bool, parse_score: float) -> None: + page = _page(_CLEAN, parse_score=parse_score) + find_rects = Mock(return_value=[]) + model = _ocr_model( + OcrMode.DEFAULT, replace, _find_pdf_aware_layout_ocr_rects=find_rects + ) + + assert BaseOcrModel.get_ocr_rects(model, page) == [] # type: ignore[arg-type] + find_rects.assert_called_once_with(page) + + +def test_ocr_text_replaces_the_pdf_text_below_threshold() -> None: + page = _page(_BROKEN, parse_score=0.0) + ocr_cell = _cell("Proxy Statement", from_ocr=True, y=700) + merge = Mock() + model = _ocr_model(OcrMode.DEFAULT, True, _merge_ocr_and_pdf_cells=merge) + conv_res = SimpleNamespace(confidence=ConfidenceReport()) + + BaseOcrModel.post_process_cells(model, [ocr_cell], page, conv_res) # type: ignore[arg-type] + + assert page.cells == [ocr_cell] + merge.assert_not_called() + + +def test_pdf_text_is_kept_above_threshold() -> None: + page = _page(_CLEAN, parse_score=1.0) + pdf_cell = page.cells[0] + ocr_cell = _cell("figure label", from_ocr=True, y=100) + merge = Mock(return_value=[pdf_cell, ocr_cell]) + model = _ocr_model(OcrMode.DEFAULT, True, _merge_ocr_and_pdf_cells=merge) + conv_res = SimpleNamespace(confidence=ConfidenceReport()) + + BaseOcrModel.post_process_cells(model, [ocr_cell], page, conv_res) # type: ignore[arg-type] + + assert page.cells == [pdf_cell, ocr_cell] + merge.assert_called_once() + + +def test_option_is_off_by_default() -> None: + assert OcrOptions(lang=[]).replace_broken_text_layer is False + assert OcrOptions(lang=[], replace_broken_text_layer=True).replace_broken_text_layer diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py new file mode 100644 index 0000000000..c33ec3dee8 --- /dev/null +++ b/tests/test_text_quality.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Detection of PDF text layers that are not the text drawn on the page.""" + +import pytest + +from docling.utils.text_quality import is_broken_text_layer + +_REPORT = ( + "For all new facilities we design and construct in North America, we are " + "targeting LEED certification. This process is in progress for the following " + "facilities:\n• Dallas (DFW4)\n• Raleigh-Durham (DUR2)\n• Chicago (CHI6)\n" + "INTRODUCTION | CORPORATE GOVERNANCE | ENVIRONMENTAL IMPACT\n" +) + +# Ordinary text layers that must keep their text, image or not. +GOOD_LAYERS = { + "bullets_and_pipes": (_REPORT * 2, True), + "form_lines_and_checkboxes": ( + "Indicate by check mark whether the registrant is a shell company. " + "Yes ☐ No ☒\nSecurities registered pursuant to Section 12(g) of the Act: " + "None\n______________________________________\n" + "0.800% Notes Due 2040 KO40B New York Stock Exchange\n" * 3, + True, + ), + "units_and_maths": ( + "General Description Product Summary\nRDS(ON) (at VGS=10V) < 8.7mΩ\n" + "Avalanche energy L=0.05mH ± 5%\nTJ, TSTG -55 to 150 °C\nt ≤ 10s " + "Steady-State\n• Latest advanced trench technology\n" * 3, + True, + ), + "square_markers": ( + "TED KOENIG ■■\nChairman & CEO\nJoined 2004, 41* years of experience in " + "private credit and leveraged finance across lower middle market deals.\n" * 3, + True, + ), + "icon_font_glyphs": ( + " Hermes Teil von Brabant Konzession  Süd-Ost Brabant " + "Konzession - Hermes Teil von Arriva Personenvervoer Nederland.\n" * 3, + True, + ), + "asterisk_heavy_report": ( + "Speeches/Conference Papers (150) -- Historical Materials (060)\n" + "EDRS PRICE MF01/PC01 Plus Postage.\n" + "***********************************************************************\n" + "Reproductions supplied by EDRS are the best that can be made\n" * 2, + True, + ), + "dot_leader_table": ( + "Net revenues . . . . . . . . . . . . . . $ 12,345 $ 11,234\n" + "Operating income . . . . . . . . . . . . . 2,345 — 1,987\n" * 6, + False, + ), + "ukrainian": ( + "Цей документ містить загальну інформацію про страховий продукт. " + "Повна інформація надається перед укладенням договору.\n" * 4, + True, + ), + "japanese": ("この文書には保険商品に関する一般情報が含まれています。" * 8, True), + "thai_vowel_signs": ( + "กิจกรรม ปัจฉิมนิเทศ และ กิจกรรม พัฒนาบุคลิกภาพ ของนักเรียนทุกระดับชั้น\n" * 4, + True, + ), + "empty_scan": (" ", True), +} + +# Broken text layers: replaced by OCR when the threshold is set. +BROKEN_LAYERS = { + "font_encoding_as_control_chars": ( + "&>66*;B \x1b8?.;7*7,. \x182;.,=8;< \x17869.7<*=287#;898<*5< \x01\x13\x12\n" + * 4, + False, + ), + "control_char_digits": ( + "360 \x0e\x1081, 3\x01317814\x12\x015\x11\x01,578\x015\x0e2\x10,87\n" * 4, + False, + ), + "legacy_font_read_as_latin": ( + "4. H¥$VrH$m`©H«$_mMr {Xem 4. H¥$VrH$m`©H«$_mMr {Xem n§MdmfI© " + "Am{U Ë`mMm ¶moOZm H$m`©H«$_\n" * 3, + False, + ), + "legacy_font_read_as_accented_latin": ( + "ĊáäåĊĜĈđĂĜûïćĒçĂĒùĒĎđčăđþđĊëùĒ čĜąēïđĉïĉĘĈĘïðĂíïð ĉĘĈĒďĘĆđćĞ\n" * 4, + False, + ), + "private_use_font": ("t o k " * 12, False), + "shifted_font_with_controls": ( + "&RGHRI(WKLFV $SSURYDOVDQG:DLYHUV :KHQQHFHVVDU\\\x0fDVSHFWVRIWKH\x11\x10\n" * 4, + True, + ), + "punctuation_soup_scan": ( + "..-= ,\": —*- s:. 0 £'S § \"s c.' 3 .»•?•§ ill ^^ ^. C/2 ^ ^ < E- ^\n" * 4, + True, + ), + "poor_hidden_ocr_layer": ( + "tlir Alf SHOUSHA, Pasha, .P..e j.0nal Director, preseh:.ed an apology " + "tor· ·tile·:···, due. tdt-urgsn~ ·national affa::rs, of the Chairman, " + "R.I.Dr.N&~b ~. Pl-1th&, the FQ'ptian Mini.star of Health.\n" * 4, + True, + ), + "scan_file_name": ("scan005.TIF", True), + "image_file_name_with_spaces": ("court document 2.tiff", True), + "docket_stamp_over_scan": ( + "4-\nI-\nE\n0\nE\n0\nC\n0\nz\nU,\nI-\n0\n[.\nI-.\nLx~\n18\n", + True, + ), +} + + +@pytest.mark.parametrize("name", sorted(GOOD_LAYERS)) +def test_ordinary_text_layer_is_kept(name: str) -> None: + text, has_bitmap = GOOD_LAYERS[name] + assert is_broken_text_layer(text, has_bitmap=has_bitmap) is False + + +@pytest.mark.parametrize("name", sorted(BROKEN_LAYERS)) +def test_broken_text_layer_is_detected(name: str) -> None: + text, has_bitmap = BROKEN_LAYERS[name] + assert is_broken_text_layer(text, has_bitmap=has_bitmap) is True + + +def test_unusual_symbols_only_count_on_scans() -> None: + # The same poor-OCR-looking text is left alone on a page without an image. + text, _ = BROKEN_LAYERS["poor_hidden_ocr_layer"] + assert is_broken_text_layer(text, has_bitmap=False) is False From f64bb02f5f60bd61c5fd365f8ed086cc8c03df61 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:48:56 +0530 Subject: [PATCH 02/15] feat(ocr): replace broken PDF text layers with OCR Some PDFs carry a text layer that does not match the text drawn on the page, because a font's glyphs were not mapped to real characters. Docling keeps that text, so garbage reaches the output while parse_score still reports 1.0. Add OcrOptions.replace_broken_text_layer and the CLI flag --replace-broken-text-layer (default off). When enabled, page preprocessing checks each page's whole text layer with docling.utils.text_quality.is_broken_text_layer(), which flags text made mostly of control or private-use characters, or Latin text that is mostly non-ASCII letters. It relies on Unicode properties only, with no character lists. A broken page's parse_score drops to 0.0, and every page graded poor (below POOR_SCORE_BOUND, shared with QualityGrade) is OCR'd in full, with its OCR text replacing its PDF text, as OcrMode.FULL_PAGE does, for that page only. parse_score keeps describing the PDF text layer. With the option off, preprocessing and parse_score are unchanged. The option lives in the base OCR model, so every OCR engine, including plugins, supports it. Tests: unit tests on ordinary text (code, LaTeX, JSON, markdown, units, several scripts) and on unmapped-font patterns, plus end-to-end tests with Tesseract on a generated PDF, with the option on and off. The usage skill docs cover the new option. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- .../.agents/skills/docling/references/cli.md | 2 + .../skills/docling/references/python-sdk.md | 10 + docling/cli/main.py | 15 +- docling/datamodel/base_models.py | 6 +- docling/datamodel/pipeline_options.py | 13 +- docling/models/base_ocr_model.py | 10 +- .../page_preprocessing_model.py | 13 +- docling/pipeline/standard_pdf_pipeline.py | 4 + docling/utils/text_quality.py | 165 +++------------ tests/test_replace_broken_text_layer.py | 194 +++++++----------- tests/test_text_quality.py | 167 ++++++--------- 11 files changed, 209 insertions(+), 390 deletions(-) diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index b76fbe65f6..70ed2e33e5 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -68,6 +68,7 @@ Decision guide: | Document | Use | |---|---| | Born-digital PDF (text selectable) | Standard (fast, no GPU) | +| Text looks right on the page but copies out as garbage | Standard with `--replace-broken-text-layer` | | Scanned / image-only PDF | Standard with OCR, or `--pipeline vlm` for best quality | | Complex/multi-column layout, dense tables | `--pipeline vlm` | | Handwriting or formulas | `--pipeline vlm` (standard OCR won't handle these) | @@ -85,6 +86,7 @@ docling scan.pdf --ocr-engine rapidocr --output /tmp/ # lightweight docling scan.pdf --ocr-engine tesserocr --output /tmp/ # needs system Tesseract docling scan.pdf --ocr-engine ocrmac --output /tmp/ # macOS Vision (mac only) docling scan.pdf --force-ocr --output /tmp/ # re-OCR even extractable text +docling report.pdf --replace-broken-text-layer --output /tmp/ # OCR only pages whose text layer is broken docling report.pdf --no-ocr --output /tmp/ # skip OCR (faster) docling scan.pdf --ocr-lang eng,deu --output /tmp/ # the engine's own codes docling scan.pdf --ocr-lang iso:en,iso:de --output /tmp/ # BCP-47 tags behind `iso:` diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index 54cc35d6b2..5cc92d6973 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -89,6 +89,16 @@ opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS Each engine is an optional dependency — see [slim-packaging.md](slim-packaging.md). +When a PDF looks right but its text copies out as garbage (a font whose glyphs +were not mapped to real characters), let OCR replace the text of just those pages. +Off by default; such pages then report `parse_score` 0 in the confidence report: + +```python +opts = PdfPipelineOptions( + do_ocr=True, ocr_options=RapidOcrOptions(replace_broken_text_layer=True) +) +``` + OCR can also run on a remote KServe v2 / Triton server. Page crops are sent to that server, so `enable_remote_services=True` is required: diff --git a/docling/cli/main.py b/docling/cli/main.py index 25291eb357..83990b0279 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -942,6 +942,16 @@ def convert( # noqa: C901 help="Which document regions are fed to the OCR engine.", ), ] = OcrMode.DEFAULT, + replace_broken_text_layer: Annotated[ + bool, + typer.Option( + ..., + help=( + "Replace a page's PDF text with OCR when its text layer is broken, " + "e.g. from a font whose glyphs were not mapped to real characters." + ), + ), + ] = False, tables: Annotated[ bool, typer.Option( @@ -1456,7 +1466,10 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] resolved_ocr_mode = OcrMode.FULL_PAGE else: resolved_ocr_mode = ocr_mode - ocr_kwargs: dict[str, Any] = {"mode": resolved_ocr_mode} + ocr_kwargs: dict[str, Any] = { + "mode": resolved_ocr_mode, + "replace_broken_text_layer": replace_broken_text_layer, + } ocr_lang_list = _split_list(ocr_lang) # `_split_list` returns None only when the option was not given, so an # explicitly empty value reaches the engine as `lang=[]`: "your default". diff --git a/docling/datamodel/base_models.py b/docling/datamodel/base_models.py index 0db67b0aaf..4368a47998 100644 --- a/docling/datamodel/base_models.py +++ b/docling/datamodel/base_models.py @@ -629,6 +629,10 @@ class OpenAiApiResponse(BaseModel): ScoreValue = float +# Scores below this bound are graded QualityGrade.POOR. +POOR_SCORE_BOUND: float = 0.5 + + class QualityGrade(str, Enum): POOR = "poor" FAIR = "fair" @@ -656,7 +660,7 @@ def _coerce_none_or_nan_str(cls, v): return v def _score_to_grade(self, score: ScoreValue) -> QualityGrade: - if score < 0.5: + if score < POOR_SCORE_BOUND: return QualityGrade.POOR elif score < 0.8: return QualityGrade.FAIR diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index e090f002d9..0b934459d0 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -261,11 +261,14 @@ class OcrOptions(BaseOptions): bool, Field( description=( - "Replace a broken PDF text layer with OCR, page by page. A page whose" - " `parse_score` is in the `poor` quality grade (below 0.5) is OCR'd in" - " full and its OCR text is used instead of its PDF text" - " for that page only. Other pages, and pages" - " without a text layer, keep the usual behaviour." + "Replace a broken PDF text layer with OCR, page by page. When enabled," + " a page's `parse_score` also drops to 0 if its text layer comes from" + " a font whose glyphs were not mapped to real characters. A page whose" + " `parse_score` is in the `poor` quality grade (below 0.5), from this" + " check or from the existing glyph-name checks, is OCR'd in full and" + " its OCR text is used instead of its PDF text for that page only." + " Its `parse_score` keeps describing the PDF text layer. Other pages," + " and pages without a text layer, keep the usual behaviour." ), examples=[False, True], ), diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index 800ca2ad1d..25afbb8906 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -22,7 +22,7 @@ from PIL import Image, ImageDraw from docling.datamodel.accelerator_options import AcceleratorOptions -from docling.datamodel.base_models import Page +from docling.datamodel.base_models import POOR_SCORE_BOUND, Page from docling.datamodel.document import ConversionResult from docling.datamodel.pipeline_options import OcrMode, OcrOptions from docling.datamodel.settings import settings @@ -139,10 +139,6 @@ def _segregate_by_visibility( return visible, invisible -# Upper bound of QualityGrade.POOR (PageConfidenceScores._score_to_grade). -_POOR_PARSE_SCORE = 0.5 - - def _ocr_full_page(options: OcrOptions, page: Page) -> bool: """Whether a page is OCR'd in full and its PDF cells dropped. @@ -152,9 +148,7 @@ def _ocr_full_page(options: OcrOptions, page: Page) -> bool: """ if options.mode == OcrMode.FULL_PAGE: return True - if not getattr(options, "replace_broken_text_layer", False): - return False - return page._parse_score < _POOR_PARSE_SCORE + return options.replace_broken_text_layer and page._parse_score < POOR_SCORE_BOUND class BaseOcrModel(BasePageModel, BaseModelWithOptions): diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index c2b810e8ef..b54788f824 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -43,6 +43,8 @@ class PagePreprocessingOptions(BaseModel): False # Skip text cell extraction for VLM-only processing ) capture_reading_order_separators: bool = False + # Score a broken text layer as poor, for OcrOptions.replace_broken_text_layer. + detect_broken_text_layer: bool = False class PagePreprocessingModel(BasePageModel): @@ -121,11 +123,12 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: ) # To emphasise problems in the parse_score, we take the 10% percentile score of all text cells. ) - # Some broken text layers only show across the whole page (a font encoding - # read as control characters, a legacy font read as Latin letters). - if not np.isnan(parse_score) and is_broken_text_layer( - "\n".join(c.text for c in page.cells), - has_bitmap=any(True for _ in page._backend.get_bitmap_rects()), + # Some broken text layers only show across the whole page, such as an + # unmapped font encoding. Judged only when the OCR may replace the layer. + if ( + self.options.detect_broken_text_layer + and not np.isnan(parse_score) + and is_broken_text_layer("\n".join(c.text for c in page.cells)) ): parse_score = 0.0 conv_res.confidence.pages[page.page_no].parse_score = parse_score diff --git a/docling/pipeline/standard_pdf_pipeline.py b/docling/pipeline/standard_pdf_pipeline.py index cdc56bfc68..18df8c40d0 100644 --- a/docling/pipeline/standard_pdf_pipeline.py +++ b/docling/pipeline/standard_pdf_pipeline.py @@ -614,6 +614,10 @@ def _init_models(self) -> None: capture_reading_order_separators=( self.pipeline_options.use_reading_order_separators ), + detect_broken_text_layer=( + self.pipeline_options.do_ocr + and self.pipeline_options.ocr_options.replace_broken_text_layer + ), ) ) self.ocr_model = self._make_ocr_model(art_path) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 45c55bbdff..eacca858ef 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -4,182 +4,65 @@ """Detect a PDF text layer that does not match the text drawn on the page. The checks look at a page's whole text layer and only at patterns broken text -produces, such as unmapped font encodings or leftover OCR noise. +produces, such as unmapped font encodings. """ -import re import unicodedata -from collections.abc import Sequence - -# A short layer that only names an image file, left behind by a scanner. -_IMAGE_FILENAME_RE = re.compile(r"(?i)\.(?:tif|tiff|jpe?g|png|gif|bmp|pict)\b") -_COMMON_PUNCT = frozenset( - ".,;:'\"!?()-[]/%$&+#@=<>/—\N{EN DASH}\N{RIGHT SINGLE QUOTATION MARK}“”«»" -) -# Symbols ordinary documents are full of; never evidence of a broken layer. -_ORDINARY_SYMBOLS = frozenset( - "*•▪■□◆◇●○►▶✓✔☐☒☑_®©™±°|≤≥≈\N{MULTIPLICATION SIGN}÷\N{MINUS SIGN}\N{EN DASH}—…§¶†‡€£¥¢‰\N{PRIME}″←→↑↓" -) -# Symbols that, inside a word, mark a non-Latin font read as Latin letters. -_IN_WORD_SYMBOLS = frozenset("$`©«»¥÷½{}^~±§@#%\\<>¤¦¨¬®¯°¹²³¼¾\N{MULTIPLICATION SIGN}") -# Characters a punctuation-soup token may hold without counting: leaders, $, dashes. -_SOUP_OK = frozenset(".,-\N{EN DASH}—$()%:;'\"/…_•▪*") -_BROKEN_CATEGORIES = frozenset({"Cc", "Co", "Cn", "Cs"}) _MIN_CHARS = 20 # Less text than this is left alone. -_SHORT = 120 # Below this length, ratio checks are unreliable. -_BROKEN_CHAR_SHARE = 0.10 # Control, private-use or U+FFFD characters. -_SYMBOL_WORD_SHARE = 0.20 # Words with symbols between their letters. +_MIN_LATIN_LETTERS = 50 # Too few Latin letters to judge their mix. +_CONTROL_SHARE = 0.03 # Control or U+FFFD characters. +_PRIVATE_USE_SHARE = 0.5 # Private-use characters; icon fonts stay far below. _NON_ASCII_LATIN_SHARE = 0.5 # Latin letters outside ASCII. -_SOUP_SHARE = 0.30 # Words made mostly of symbols. -_UNUSUAL_PER_100_LETTERS = 2.0 # Unusual characters on a scan. - - -def _letter_scripts(text: str) -> dict[str, int]: - counts = {"latin": 0, "cyrillic": 0, "cjk": 0, "arabic": 0, "other": 0} - for ch in text: - if not ch.isalpha(): - continue - name = unicodedata.name(ch, "") - if "CYRILLIC" in name: - counts["cyrillic"] += 1 - elif any( - token in name - for token in ("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "IDEOGRAPH") - ): - counts["cjk"] += 1 - elif "ARABIC" in name: - counts["arabic"] += 1 - elif "LATIN" in name: - counts["latin"] += 1 - else: - counts["other"] += 1 - return counts - - -def _is_ordinary(ch: str) -> bool: - """Not a letter, but nothing a broken layer produces either.""" - category = unicodedata.category(ch) - return ( - ch.isalnum() - or ch.isspace() - or ch in _COMMON_PUNCT - or ch in _ORDINARY_SYMBOLS - or category[0] == "M" # combining marks: Thai / Devanagari vowel signs - or category == "Co" # private use: icon-font glyphs - ) +# Not "Cn" (unassigned): that depends on the Unicode version of the Python running. +_CONTROL = frozenset({"Cc", "Cs"}) -def _unusual_per_100_letters(text: str) -> float: - letters = 0 - unusual = 0 - for ch in text: - if ch.isalpha(): - letters += 1 - elif not _is_ordinary(ch): - unusual += 1 - return 100.0 * unusual / letters if letters else 0.0 - -def _broken_char_share(text: str) -> float: - """Share of non-space characters that are control, private-use or U+FFFD.""" +def _share(text: str, categories: frozenset[str], *, replacement: bool) -> float: + """Share of non-space characters in the given Unicode categories.""" chars = [ch for ch in text if not ch.isspace()] - broken = sum( - 1 + if not chars: + return 0.0 + hits = sum( + unicodedata.category(ch) in categories or (replacement and ch == "\ufffd") for ch in chars - if unicodedata.category(ch) in _BROKEN_CATEGORIES or ch == "\ufffd" ) - return broken / len(chars) if chars else 0.0 - - -def _symbol_word_share(words: Sequence[str]) -> float: - """Share of lettered words with two or more symbols between their letters.""" - lettered = [w for w in words if len(w) >= 3 and any(ch.isalpha() for ch in w)] - if not lettered: - return 0.0 - mixed = 0 - for word in lettered: - letters = [i for i, ch in enumerate(word) if ch.isalpha()] - inner = word[letters[0] : letters[-1]] - if sum(ch in _IN_WORD_SYMBOLS for ch in inner) >= 2: - mixed += 1 - return mixed / len(lettered) + return hits / len(chars) def _non_ascii_latin_share(text: str) -> float: - """Share of Latin letters outside ASCII, when Latin carries the page.""" + """Share of Latin letters outside ASCII, on text where Latin letters dominate.""" letters = [ch for ch in text if ch.isalpha()] latin = [ ch for ch in letters if "LATIN" in unicodedata.name(ch, "") and not "\uff00" <= ch <= "\uffef" ] - if len(latin) < 50 or len(latin) * 2 < len(letters): + if len(latin) < _MIN_LATIN_LETTERS or len(latin) * 2 < len(letters): return 0.0 return sum(ord(ch) > 127 for ch in latin) / len(latin) -def _soup_share(words: Sequence[str]) -> float: - """Share of words that are mostly symbols beyond leaders, currency and dashes.""" - if not words: - return 0.0 - soup = sum( - 1 - for word in words - if sum(ch.isalnum() for ch in word) * 2 < len(word) - and any(not ch.isalnum() and ch not in _SOUP_OK for ch in word) - ) - return soup / len(words) - - -def is_broken_text_layer(text: str, *, has_bitmap: bool) -> bool: - """Whether a page's PDF text is not the text drawn on the page. +def is_broken_text_layer(text: str) -> bool: + """Whether a page's PDF text does not match the text drawn on the page. Args: text: All text of one page's PDF text layer, cells joined by newlines. - has_bitmap: Whether the page holds an image, i.e. may be a scan. Returns: ``True`` only for text a broken layer produces. When unsure it returns ``False``, so pages keep their PDF text. """ - stripped = (text or "").strip() - if not stripped: + stripped = text.strip() + if len(stripped) < _MIN_CHARS: return False - if len(stripped) < _SHORT and _IMAGE_FILENAME_RE.search(stripped): + # A font whose glyphs were mapped to control or private-use code points. + if _share(stripped, _CONTROL, replacement=True) >= _CONTROL_SHARE: return True - - # A broken font encoding, whether or not the page holds an image. - if ( - len(stripped) >= _MIN_CHARS - and _broken_char_share(stripped) >= _BROKEN_CHAR_SHARE - ): - return True - - words = stripped.split() - if len(stripped) >= _SHORT and ( - _symbol_word_share(words) >= _SYMBOL_WORD_SHARE - or _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE - or _soup_share(words) >= _SOUP_SHARE - ): + if _share(stripped, frozenset({"Co"}), replacement=False) >= _PRIVATE_USE_SHARE: return True - # The remaining checks only apply to scans with a hidden OCR layer. - if not has_bitmap or len(stripped) < _MIN_CHARS: - return False - - scripts = _letter_scripts(stripped) - if scripts["cyrillic"] >= 30 or scripts["cjk"] >= 20 or scripts["arabic"] >= 20: - return False - - letters = sum(ch.isalpha() for ch in stripped) - unusual = _unusual_per_100_letters(stripped) - if len(stripped) < _SHORT and ( - letters / len(stripped) < 0.5 or unusual >= _UNUSUAL_PER_100_LETTERS - ): - return True - return ( - scripts["latin"] >= max(1, letters * 0.5) - and unusual >= _UNUSUAL_PER_100_LETTERS - ) + # A non-Latin font whose glyphs were mapped to accented Latin letters. + return _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py index 9a09d67fd2..3000c219ce 100644 --- a/tests/test_replace_broken_text_layer.py +++ b/tests/test_replace_broken_text_layer.py @@ -3,145 +3,97 @@ """Replacing a broken PDF text layer with OCR (OcrOptions.replace_broken_text_layer). -A page whose text layer is broken scores 0 in its parse score; with the option -on, the OCR stage reads every page graded poor (below 0.5) in full and keeps only -its OCR text. Other pages, and pages without a text layer, are left as they were. +The test page shows "Proxy Statement" as an image, over a text layer that either +matches it or is the output of a font decoded as accented Latin letters. """ -import math -from types import SimpleNamespace -from unittest.mock import Mock +import ctypes +from pathlib import Path +import pypdfium2 as pdfium +import pypdfium2.raw as pdfium_c import pytest -from docling_core.types.doc.page import BoundingRectangle, TextCell - -from docling.datamodel.base_models import ConfidenceReport, Page, Size -from docling.datamodel.pipeline_options import OcrMode, OcrOptions -from docling.models.base_ocr_model import BaseOcrModel, _empty_segmented_page -from docling.models.stages.page_preprocessing.page_preprocessing_model import ( - PagePreprocessingModel, - PagePreprocessingOptions, -) - -_BROKEN = "&>66*;B \x1b8?.;7*7,. \x182;.,=8;< \x17869.7<*=287#;898<*5<" -_CLEAN = "We have audited the accompanying financial statements of the village." - - -def _cell(text: str, *, from_ocr: bool, y: float) -> TextCell: - return TextCell( - rect=BoundingRectangle( - r_x0=10, - r_y0=y, - r_x1=300, - r_y1=y, - r_x2=300, - r_y2=y + 12, - r_x3=10, - r_y3=y + 12, - ), - text=text, - orig=text, - from_ocr=from_ocr, - confidence=0.9 if from_ocr else 1.0, - ) - - -def _page(*pdf_texts: str, parse_score: float = math.nan) -> Page: - page = Page(page_no=1) - page.size = Size(width=600.0, height=800.0) - page.parsed_page = _empty_segmented_page(page) - page.parsed_page.textline_cells = [ - _cell(text, from_ocr=False, y=700 - 20 * i) for i, text in enumerate(pdf_texts) - ] - page.parsed_page.has_lines = bool(pdf_texts) - page._parse_score = parse_score - return page - - -def _ocr_model(mode: OcrMode, replace: bool, **methods: Mock): - options = SimpleNamespace(mode=mode, replace_broken_text_layer=replace) - return SimpleNamespace(options=options, **methods) +from PIL import Image, ImageDraw, ImageFont - -@pytest.mark.parametrize( - ("texts", "expected"), [((_BROKEN,) * 4, 0.0), ((_CLEAN,) * 4, 1.0)] +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import ConversionResult +from docling.datamodel.pipeline_options import ( + PdfPipelineOptions, + TesseractCliOcrOptions, ) -def test_parse_score_reflects_a_broken_text_layer( - texts: tuple[str, ...], expected: float -) -> None: - page = _page() - segmented = _page(*texts).parsed_page - page._backend = SimpleNamespace( - get_segmented_page=lambda: segmented, get_bitmap_rects=lambda: iter(()) - ) # type: ignore[assignment] - conv_res = SimpleNamespace(confidence=ConfidenceReport()) - model = PagePreprocessingModel(PagePreprocessingOptions(images_scale=None)) - - model._parse_page_cells(conv_res, page) # type: ignore[arg-type] - - assert conv_res.confidence.pages[1].parse_score == expected - assert page._parse_score == expected - - -def test_page_below_threshold_is_ocrd_in_full() -> None: - page = _page(_BROKEN, parse_score=0.0) - find_rects = Mock(return_value=[]) - model = _ocr_model( - OcrMode.DEFAULT, True, _find_pdf_aware_layout_ocr_rects=find_rects - ) - - (rect,) = BaseOcrModel.get_ocr_rects(model, page) # type: ignore[arg-type] - - assert (rect.l, rect.t, rect.r, rect.b) == (0, 0, 600.0, 800.0) - find_rects.assert_not_called() +from docling.document_converter import DocumentConverter, PdfFormatOption +pytestmark = pytest.mark.ml_ocr -@pytest.mark.parametrize( - ("replace", "parse_score"), +_SHOWN = "Proxy Statement" +_BROKEN_LAYER = " ".join( [ - (True, 1.0), # good text layer - (True, math.nan), # no text layer: left to the usual OCR - (False, 0.0), # option off: behaviour unchanged - ], + "\N{LATIN SMALL LETTER A WITH ACUTE}\N{LATIN SMALL LETTER A WITH DIAERESIS}" + "\N{LATIN SMALL LETTER C WITH CEDILLA}\N{LATIN SMALL LETTER E WITH GRAVE}" + "\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}" + ] + * 12 ) -def test_other_pages_keep_their_ocr_regions(replace: bool, parse_score: float) -> None: - page = _page(_CLEAN, parse_score=parse_score) - find_rects = Mock(return_value=[]) - model = _ocr_model( - OcrMode.DEFAULT, replace, _find_pdf_aware_layout_ocr_rects=find_rects - ) - assert BaseOcrModel.get_ocr_rects(model, page) == [] # type: ignore[arg-type] - find_rects.assert_called_once_with(page) +def _page_pdf(path: Path, text_layer: str) -> Path: + """One page: `_SHOWN` drawn as an image, under a text layer of `text_layer`.""" + width, height = 400, 200 + image = Image.new("RGB", (width * 3, height * 3), "white") + font = ImageFont.load_default(size=110) + ImageDraw.Draw(image).text((60, 230), _SHOWN, fill="black", font=font) + + pdf = pdfium.PdfDocument.new() + page = pdf.new_page(width, height) + picture = pdfium.PdfImage.new(pdf) + picture.set_bitmap(pdfium.PdfBitmap.from_pil(image)) + picture.set_matrix(pdfium.PdfMatrix().scale(width, height)) + page.insert_obj(picture) + + helvetica = pdfium_c.FPDFText_LoadStandardFont(pdf.raw, b"Helvetica") + text = pdfium_c.FPDFPageObj_CreateTextObj(pdf.raw, helvetica, 9.0) + utf16 = (text_layer + "\0").encode("utf-16-le") + pdfium_c.FPDFText_SetText( + text, ctypes.cast(ctypes.c_char_p(utf16), ctypes.POINTER(pdfium_c.FPDF_WCHAR)) + ) + # A broken layer still draws the right glyphs; white ink keeps this one from + # adding marks of its own to the image the OCR reads. + pdfium_c.FPDFPageObj_SetFillColor(text, 255, 255, 255, 255) + pdfium_c.FPDFPageObj_Transform(text, 1, 0, 0, 1, 20, 112) + pdfium_c.FPDFPage_InsertObject(page.raw, text) + page.gen_content() + pdf.save(path) + return path + + +def _convert(path: Path, *, replace: bool) -> ConversionResult: + ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=replace) + options = PdfPipelineOptions(do_ocr=True, do_table_structure=False, ocr_options=ocr) + converter = DocumentConverter( + format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} + ) + return converter.convert(path) -def test_ocr_text_replaces_the_pdf_text_below_threshold() -> None: - page = _page(_BROKEN, parse_score=0.0) - ocr_cell = _cell("Proxy Statement", from_ocr=True, y=700) - merge = Mock() - model = _ocr_model(OcrMode.DEFAULT, True, _merge_ocr_and_pdf_cells=merge) - conv_res = SimpleNamespace(confidence=ConfidenceReport()) - BaseOcrModel.post_process_cells(model, [ocr_cell], page, conv_res) # type: ignore[arg-type] +def test_broken_text_layer_is_replaced_by_ocr(tmp_path: Path) -> None: + result = _convert(_page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=True) - assert page.cells == [ocr_cell] - merge.assert_not_called() + assert result.confidence.pages[1].parse_score == 0.0 + markdown = result.document.export_to_markdown() + assert _SHOWN in markdown + assert _BROKEN_LAYER[:5] not in markdown -def test_pdf_text_is_kept_above_threshold() -> None: - page = _page(_CLEAN, parse_score=1.0) - pdf_cell = page.cells[0] - ocr_cell = _cell("figure label", from_ocr=True, y=100) - merge = Mock(return_value=[pdf_cell, ocr_cell]) - model = _ocr_model(OcrMode.DEFAULT, True, _merge_ocr_and_pdf_cells=merge) - conv_res = SimpleNamespace(confidence=ConfidenceReport()) +def test_option_off_keeps_the_text_layer_and_its_score(tmp_path: Path) -> None: + result = _convert(_page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=False) - BaseOcrModel.post_process_cells(model, [ocr_cell], page, conv_res) # type: ignore[arg-type] + assert result.confidence.pages[1].parse_score == 1.0 + assert _BROKEN_LAYER[:5] in result.document.export_to_markdown() - assert page.cells == [pdf_cell, ocr_cell] - merge.assert_called_once() +def test_good_text_layer_is_kept(tmp_path: Path) -> None: + layer = "Annual Proxy Statement of the Company" + result = _convert(_page_pdf(tmp_path / "good.pdf", layer), replace=True) -def test_option_is_off_by_default() -> None: - assert OcrOptions(lang=[]).replace_broken_text_layer is False - assert OcrOptions(lang=[], replace_broken_text_layer=True).replace_broken_text_layer + assert result.confidence.pages[1].parse_score == 1.0 + assert layer in result.document.export_to_markdown() diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index c33ec3dee8..69aeeddfb1 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -1,127 +1,78 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT -"""Detection of PDF text layers that are not the text drawn on the page.""" +"""Detection of PDF text layers that do not match the text drawn on the page.""" import pytest from docling.utils.text_quality import is_broken_text_layer -_REPORT = ( - "For all new facilities we design and construct in North America, we are " - "targeting LEED certification. This process is in progress for the following " - "facilities:\n• Dallas (DFW4)\n• Raleigh-Durham (DUR2)\n• Chicago (CHI6)\n" - "INTRODUCTION | CORPORATE GOVERNANCE | ENVIRONMENTAL IMPACT\n" -) -# Ordinary text layers that must keep their text, image or not. -GOOD_LAYERS = { - "bullets_and_pipes": (_REPORT * 2, True), - "form_lines_and_checkboxes": ( - "Indicate by check mark whether the registrant is a shell company. " - "Yes ☐ No ☒\nSecurities registered pursuant to Section 12(g) of the Act: " - "None\n______________________________________\n" - "0.800% Notes Due 2040 KO40B New York Stock Exchange\n" * 3, - True, - ), - "units_and_maths": ( - "General Description Product Summary\nRDS(ON) (at VGS=10V) < 8.7mΩ\n" - "Avalanche energy L=0.05mH ± 5%\nTJ, TSTG -55 to 150 °C\nt ≤ 10s " - "Steady-State\n• Latest advanced trench technology\n" * 3, - True, - ), - "square_markers": ( - "TED KOENIG ■■\nChairman & CEO\nJoined 2004, 41* years of experience in " - "private credit and leveraged finance across lower middle market deals.\n" * 3, - True, - ), - "icon_font_glyphs": ( - " Hermes Teil von Brabant Konzession  Süd-Ost Brabant " - "Konzession - Hermes Teil von Arriva Personenvervoer Nederland.\n" * 3, - True, - ), - "asterisk_heavy_report": ( - "Speeches/Conference Papers (150) -- Historical Materials (060)\n" - "EDRS PRICE MF01/PC01 Plus Postage.\n" - "***********************************************************************\n" - "Reproductions supplied by EDRS are the best that can be made\n" * 2, - True, - ), - "dot_leader_table": ( - "Net revenues . . . . . . . . . . . . . . $ 12,345 $ 11,234\n" - "Operating income . . . . . . . . . . . . . 2,345 — 1,987\n" * 6, - False, - ), - "ukrainian": ( - "Цей документ містить загальну інформацію про страховий продукт. " - "Повна інформація надається перед укладенням договору.\n" * 4, - True, - ), - "japanese": ("この文書には保険商品に関する一般情報が含まれています。" * 8, True), - "thai_vowel_signs": ( - "กิจกรรม ปัจฉิมนิเทศ และ กิจกรรม พัฒนาบุคลิกภาพ ของนักเรียนทุกระดับชั้น\n" * 4, - True, - ), - "empty_scan": (" ", True), +def _words(first: int, last: int, count: int = 40) -> str: + """Words made of the letters in a Unicode range, as a stand-in for a script.""" + letters = [chr(cp) for cp in range(first, last + 1) if chr(cp).isalpha()] + return " ".join( + "".join(letters[(i + j) % len(letters)] for j in range(5)) for i in range(count) + ) + + +# Ordinary text layers: never broken. +ORDINARY = { + "latex": r"The loss $L = \frac{1}{n}\sum_{i=1}^{n} (y_i - \hat{y}_i)^2$ with " + r"\alpha \in [0,1] and \beta^{2} \leq \gamma_{k}. " * 3, + "shell": "export PATH=$HOME/bin:$PATH; cd ${USER}/work && ls ~/docs/*.md " + "| grep -v '#draft' > ~/out/${DATE}.log; echo $? " * 3, + "python": "def f(x):\n return {k: v**2 for k, v in x.items() if v > 0}\n" + "print(f({'a': 1, 'b': -2}))\n" * 3, + "json": '{"id": 42, "tags": ["a", "b"], "meta": {"ok": true}, "path": "/v/a.log"}\n' + * 3, + "markdown_table": "| Metric | Value |\n|---|---|\n| F1 | 0.91 |\n**Note:** see " + "`config.yaml` and [docs](https://x.y/z).\n" * 3, + "dot_leaders": "Net revenues . . . . . . $ 12,345 $ 11,234 (1)\n" + "Operating income . . . . 2,345 (1,987)\n" * 5, + "caption": "Q3 2024: $12,345 (+5.2%) vs 11,734 in the prior year", + "units": "RDS(ON) < 8.7 m\N{OHM SIGN} at VGS=10V; TJ = \N{MINUS SIGN}55 to 150 " + "\N{DEGREE SIGN}C; t \N{LESS-THAN OR EQUAL TO} 10 \N{MICRO SIGN}s. " * 3, + "bullets_and_checkboxes": "\N{BULLET} Dallas (DFW4)\n\N{BLACK SMALL SQUARE} " + "Chicago\n\N{BALLOT BOX} No \N{BALLOT BOX WITH X} Yes\n______________\n" * 4, + "icon_font_glyphs": "".join(chr(0xF002) + " Brabant Konzession, Teil von Arriva. ") + * 4, + "accented_european": "L\N{RIGHT SINGLE QUOTATION MARK}\N{LATIN CAPITAL LETTER E WITH ACUTE}cole " + "doctorale accueille aujourd\N{RIGHT SINGLE QUOTATION MARK}hui des \N{LATIN SMALL LETTER E WITH ACUTE}tudiants " + "de la r\N{LATIN SMALL LETTER E WITH ACUTE}gion. " * 4, + "greek": _words(0x03B1, 0x03C9), + "cyrillic": _words(0x0430, 0x044F), + "hebrew": _words(0x05D0, 0x05EA), + "arabic": _words(0x0627, 0x064A), + "thai": _words(0x0E01, 0x0E2E), + "devanagari": _words(0x0915, 0x0939), + "cjk": _words(0x4E00, 0x4E40), } -# Broken text layers: replaced by OCR when the threshold is set. -BROKEN_LAYERS = { - "font_encoding_as_control_chars": ( - "&>66*;B \x1b8?.;7*7,. \x182;.,=8;< \x17869.7<*=287#;898<*5< \x01\x13\x12\n" - * 4, - False, - ), - "control_char_digits": ( - "360 \x0e\x1081, 3\x01317814\x12\x015\x11\x01,578\x015\x0e2\x10,87\n" * 4, - False, - ), - "legacy_font_read_as_latin": ( - "4. H¥$VrH$m`©H«$_mMr {Xem 4. H¥$VrH$m`©H«$_mMr {Xem n§MdmfI© " - "Am{U Ë`mMm ¶moOZm H$m`©H«$_\n" * 3, - False, - ), - "legacy_font_read_as_accented_latin": ( - "ĊáäåĊĜĈđĂĜûïćĒçĂĒùĒĎđčăđþđĊëùĒ čĜąēïđĉïĉĘĈĘïðĂíïð ĉĘĈĒďĘĆđćĞ\n" * 4, - False, - ), - "private_use_font": ("t o k " * 12, False), - "shifted_font_with_controls": ( - "&RGHRI(WKLFV $SSURYDOVDQG:DLYHUV :KHQQHFHVVDU\\\x0fDVSHFWVRIWKH\x11\x10\n" * 4, - True, - ), - "punctuation_soup_scan": ( - "..-= ,\": —*- s:. 0 £'S § \"s c.' 3 .»•?•§ ill ^^ ^. C/2 ^ ^ < E- ^\n" * 4, - True, - ), - "poor_hidden_ocr_layer": ( - "tlir Alf SHOUSHA, Pasha, .P..e j.0nal Director, preseh:.ed an apology " - "tor· ·tile·:···, due. tdt-urgsn~ ·national affa::rs, of the Chairman, " - "R.I.Dr.N&~b ~. Pl-1th&, the FQ'ptian Mini.star of Health.\n" * 4, - True, - ), - "scan_file_name": ("scan005.TIF", True), - "image_file_name_with_spaces": ("court document 2.tiff", True), - "docket_stamp_over_scan": ( - "4-\nI-\nE\n0\nE\n0\nC\n0\nz\nU,\nI-\n0\n[.\nI-.\nLx~\n18\n", - True, - ), +# Text layers from fonts whose glyphs were not mapped to real characters. +BROKEN = { + "glyphs_as_control_codes": "Net assets 2,635 2,849\n" + + "".join(chr(0x10 + i % 15) for i in range(40)) + + "\nTotal 3,004\n", + "replacement_characters": "Annual report \N{REPLACEMENT CHARACTER}" + "\N{REPLACEMENT CHARACTER} 2024 " * 6, + "glyphs_as_private_use": " ".join( + "".join(chr(0xF021 + (i + j) % 60) for j in range(4)) for i in range(30) + ), + "glyphs_as_accented_latin": _words(0x0100, 0x017F), } -@pytest.mark.parametrize("name", sorted(GOOD_LAYERS)) +@pytest.mark.parametrize("name", sorted(ORDINARY)) def test_ordinary_text_layer_is_kept(name: str) -> None: - text, has_bitmap = GOOD_LAYERS[name] - assert is_broken_text_layer(text, has_bitmap=has_bitmap) is False + assert not is_broken_text_layer(ORDINARY[name]) -@pytest.mark.parametrize("name", sorted(BROKEN_LAYERS)) -def test_broken_text_layer_is_detected(name: str) -> None: - text, has_bitmap = BROKEN_LAYERS[name] - assert is_broken_text_layer(text, has_bitmap=has_bitmap) is True +@pytest.mark.parametrize("name", sorted(BROKEN)) +def test_unmapped_font_is_detected(name: str) -> None: + assert is_broken_text_layer(BROKEN[name]) -def test_unusual_symbols_only_count_on_scans() -> None: - # The same poor-OCR-looking text is left alone on a page without an image. - text, _ = BROKEN_LAYERS["poor_hidden_ocr_layer"] - assert is_broken_text_layer(text, has_bitmap=False) is False +def test_short_or_empty_text_is_left_alone() -> None: + assert not is_broken_text_layer("") + assert not is_broken_text_layer("\x01\x02\x03") # too little text to judge From 611494101805dd7dc52a1f8dd5f08c2a0bc1ca75 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:09:11 +0530 Subject: [PATCH 03/15] feat(ocr): detect more broken text layers is_broken_text_layer() looked only at the whole page, and only for control, private-use and non-ASCII Latin characters. It missed a broken font that covers only part of a page, and a broken font that maps glyphs to the wrong ASCII letters. Keep the existing checks and add: - Per-line checks, so one broken cell on a good page is found: a run of 4 or more different private-use characters (icons repeat a few code points), or a token with 2 or more control characters. - U+FFFD density rules: runs on short pages, many U+FFFD, U+FFFD in several lines, or long runs. One stray U+FFFD does not flag a page. - A shifted-letter check for broken ToUnicode maps. The letter-frequency profile keeps the shape of a language, but the letters are in the wrong places. It counts only tokens with a lowercase letter, so acronyms and part numbers are skipped, and it needs 200 letters and a low vowel share. The non-ASCII Latin check now also needs few ASCII letters on the page, to protect mixed-language and Vietnamese text. Tests: ordinary text long enough for the letter check (English, Czech, Welsh, Vietnamese, C code, part numbers, base64, DNA, icon ratings, a single U+FFFD, a pdfium hyphen marker), and broken text that the old checks missed (shifted letters, one broken cell, U+FFFD below the old share) Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 131 +++++++++++++++++++++++++++++++++- tests/test_text_quality.py | 64 +++++++++++++++++ 2 files changed, 192 insertions(+), 3 deletions(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index eacca858ef..51339fe62d 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -3,17 +3,46 @@ """Detect a PDF text layer that does not match the text drawn on the page. -The checks look at a page's whole text layer and only at patterns broken text -produces, such as unmapped font encodings. +The checks look at a page's whole text layer and at each of its lines, and only +at patterns broken text produces, such as unmapped font encodings. """ +import math import unicodedata +from itertools import groupby _MIN_CHARS = 20 # Less text than this is left alone. _MIN_LATIN_LETTERS = 50 # Too few Latin letters to judge their mix. _CONTROL_SHARE = 0.03 # Control or U+FFFD characters. _PRIVATE_USE_SHARE = 0.5 # Private-use characters; icon fonts stay far below. _NON_ASCII_LATIN_SHARE = 0.5 # Latin letters outside ASCII. +_ASCII_LETTER_SHARE = 1 / 3 # Mojibake leaves few ASCII letters. + +# One line is one text cell. +_PRIVATE_USE_RUN = 4 # Different private-use characters in a row. +_MIN_LINE_CHARS = 5 # Less text than this in a line or token is not judged. +_TOKEN_CONTROLS = 2 # Control characters in one token. +_TOKEN_CONTROL_SHARE = 0.05 + +# U+FFFD on the page. +_SHORT_PAGE_CHARS = 80 # On a page this short, a run of 2 is enough. +_SHORT_PAGE_RUN = 2 +_MANY_REPLACEMENTS = 12 +_MANY_REPLACEMENTS_SHARE = 0.05 +_REPLACEMENT_LINES = 3 +_LONG_REPLACEMENT_RUN = 8 +_SOME_REPLACEMENTS_SHARE = 0.025 + +# Letters shifted by a broken ToUnicode map, as in a substitution cipher. +_MIN_CIPHER_LETTERS = 200 # Too few letters to judge their frequencies. +_MAX_VOWEL_SHARE = 0.30 # Latin-script text stays above this. +_MAX_ENGLISH_COSINE = 0.60 # Letters not where English puts them... +_MIN_SHAPE_COSINE = 0.90 # ...but with the frequency profile of a language. +# English letter frequencies, percent, a to z (Lewand, "Cryptological Mathematics"). +_ENGLISH_LETTER_FREQ = ( + 8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, + 6.7, 7.5, 1.9, 0.1, 6.0, 6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07, +) # fmt: skip # Not "Cn" (unassigned): that depends on the Unicode version of the Python running. _CONTROL = frozenset({"Cc", "Cs"}) @@ -31,6 +60,10 @@ def _share(text: str, categories: frozenset[str], *, replacement: bool) -> float return hits / len(chars) +def _is_private_use(ch: str) -> bool: + return unicodedata.category(ch) == "Co" + + def _non_ascii_latin_share(text: str) -> float: """Share of Latin letters outside ASCII, on text where Latin letters dominate.""" letters = [ch for ch in text if ch.isalpha()] @@ -44,6 +77,85 @@ def _non_ascii_latin_share(text: str) -> float: return sum(ord(ch) > 127 for ch in latin) / len(latin) +def _ascii_letter_share(text: str) -> float: + """Share of non-space characters that are ASCII letters.""" + chars = [ch for ch in text if not ch.isspace()] + if not chars: + return 0.0 + return sum(ch.isascii() and ch.isalpha() for ch in chars) / len(chars) + + +def _is_broken_line(line: str) -> bool: + """Whether one line holds a run or a token that only a broken font produces.""" + # Icons repeat a few private-use code points; the letters of words vary. + for is_private_use, run in groupby(line, key=_is_private_use): + if is_private_use and len(set(run)) >= _PRIVATE_USE_RUN: + return True + for token in line.split(): + if len(token) < _MIN_LINE_CHARS: + continue + controls = sum(unicodedata.category(ch) in _CONTROL for ch in token) + if controls >= _TOKEN_CONTROLS and controls >= _TOKEN_CONTROL_SHARE * len( + token + ): + return True + return False + + +def _has_replacement_evidence(text: str, lines: list[str]) -> bool: + """Whether U+FFFD is dense or clustered enough to mark the page broken.""" + count = text.count("\ufffd") + if not count: + return False + chars = sum(not ch.isspace() for ch in text) + longest = max((len(list(run)) for ch, run in groupby(text) if ch == "\ufffd")) + if chars <= _SHORT_PAGE_CHARS and longest >= _SHORT_PAGE_RUN: + return True + share = count / chars + lines_hit = sum("\ufffd" in line for line in lines) + return ( + (count >= _MANY_REPLACEMENTS and share >= _MANY_REPLACEMENTS_SHARE) + or (lines_hit >= _REPLACEMENT_LINES and share >= _SOME_REPLACEMENTS_SHARE) + or (longest >= _LONG_REPLACEMENT_RUN and share >= _SOME_REPLACEMENTS_SHARE) + ) + + +def _cosine(a: list[float], b: list[float]) -> float: + norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b)) + return sum(x * y for x, y in zip(a, b)) / norm if norm else 1.0 + + +def _is_shifted_latin(text: str) -> bool: + """Whether ASCII letters look like language with its letters swapped. + + A broken ToUnicode map often shifts every letter by a constant. That keeps + the shape of the letter-frequency profile but moves each letter. Only tokens + with a lowercase letter count, so acronyms and part numbers are skipped. + """ + counts = [0.0] * 26 + other_letters = 0 + for token in text.split(): + if not any("a" <= ch <= "z" for ch in token): + continue + for ch in token: + if "a" <= ch.lower() <= "z": + counts[ord(ch.lower()) - ord("a")] += 1 + elif ch.isalpha(): + other_letters += 1 + letters = sum(counts) + if letters < _MIN_CIPHER_LETTERS or other_letters > letters: + return False + vowels = sum(counts[ord(v) - ord("a")] for v in "aeiou") + if vowels / letters > _MAX_VOWEL_SHARE: + return False + english = list(_ENGLISH_LETTER_FREQ) + return ( + _cosine(counts, english) < _MAX_ENGLISH_COSINE + and _cosine(sorted(counts, reverse=True), sorted(english, reverse=True)) + >= _MIN_SHAPE_COSINE + ) + + def is_broken_text_layer(text: str) -> bool: """Whether a page's PDF text does not match the text drawn on the page. @@ -57,12 +169,25 @@ def is_broken_text_layer(text: str) -> bool: stripped = text.strip() if len(stripped) < _MIN_CHARS: return False + lines = stripped.splitlines() # A font whose glyphs were mapped to control or private-use code points. if _share(stripped, _CONTROL, replacement=True) >= _CONTROL_SHARE: return True if _share(stripped, frozenset({"Co"}), replacement=False) >= _PRIVATE_USE_SHARE: return True + if _has_replacement_evidence(stripped, lines): + return True + # The same, in one cell of an otherwise good page. + if any(_is_broken_line(line) for line in lines): + return True # A non-Latin font whose glyphs were mapped to accented Latin letters. - return _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE + if ( + _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE + and _ascii_letter_share(stripped) < _ASCII_LETTER_SHARE + ): + return True + + # A Latin font whose glyphs were mapped to the wrong ASCII letters. + return _is_shifted_latin(stripped) diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 69aeeddfb1..1c79f01e9c 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -16,6 +16,17 @@ def _words(first: int, last: int, count: int = 40) -> str: ) +_PROSE = ( + "The quarterly report shows that revenue increased across all regions, " + "driven by strong demand for cloud services and improved margins. " +) * 3 + + +def _shift(text: str, offset: int) -> str: + """Letters moved by a constant, as a broken ToUnicode map does.""" + return "".join(chr(ord(ch) + offset) if ch.isalpha() else ch for ch in text) + + # Ordinary text layers: never broken. ORDINARY = { "latex": r"The loss $L = \frac{1}{n}\sum_{i=1}^{n} (y_i - \hat{y}_i)^2$ with " @@ -47,6 +58,41 @@ def _words(first: int, last: int, count: int = 40) -> str: "thai": _words(0x0E01, 0x0E2E), "devanagari": _words(0x0915, 0x0939), "cjk": _words(0x4E00, 0x4E40), + # Long enough for the letter-frequency check. + "english_prose": _PROSE, + "czech": "Vl\N{LATIN SMALL LETTER A WITH ACUTE}da schv\N{LATIN SMALL LETTER A WITH ACUTE}lila " + "n\N{LATIN SMALL LETTER A WITH ACUTE}vrh z\N{LATIN SMALL LETTER A WITH ACUTE}kona, kter\N{LATIN SMALL LETTER Y WITH ACUTE} " + "zjednodu\N{LATIN SMALL LETTER S WITH CARON}uje stavebn\N{LATIN SMALL LETTER I WITH ACUTE} " + "\N{LATIN SMALL LETTER R WITH CARON}\N{LATIN SMALL LETTER I WITH ACUTE}zen\N{LATIN SMALL LETTER I WITH ACUTE} " + "a podporuje v\N{LATIN SMALL LETTER Y WITH ACUTE}stavbu nov\N{LATIN SMALL LETTER Y WITH ACUTE}ch byt\N{LATIN SMALL LETTER U WITH RING ABOVE} " + "ve velk\N{LATIN SMALL LETTER Y WITH ACUTE}ch m\N{LATIN SMALL LETTER E WITH CARON}stech. " + * 3, + "welsh": "Mae'r llywodraeth wedi cyhoeddi cynllun newydd i gefnogi busnesau bach " + "yng nghefn gwlad Cymru dros y blynyddoedd nesaf. " * 3, + "vietnamese": "\N{LATIN CAPITAL LETTER D WITH STROKE}\N{LATIN SMALL LETTER U WITH HORN AND GRAVE}\N{LATIN SMALL LETTER O WITH HORN}ng " + "\N{LATIN SMALL LETTER D WITH STROKE}\N{LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE}n t\N{LATIN SMALL LETTER U WITH HORN}\N{LATIN SMALL LETTER O WITH HORN}ng " + "lai c\N{LATIN SMALL LETTER U WITH HOOK ABOVE}a nh\N{LATIN SMALL LETTER U WITH HORN AND TILDE}ng ng\N{LATIN SMALL LETTER U WITH HORN}\N{LATIN SMALL LETTER O WITH HORN AND GRAVE}i " + "d\N{LATIN SMALL LETTER A WITH CIRCUMFLEX}n Vi\N{LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW}t Nam " + "r\N{LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE}t d\N{LATIN SMALL LETTER A WITH GRAVE}i. " + * 4, + "c_code": "for (int i = 0; i < n; i++) { ptr->buf[i] = xmm_mul(cfg.k, src[i]); " + "if (!chk(ptr)) return -EINVAL; } " * 4, + "part_numbers": "STM32F407VGT6 LQFP100 GPIO PB12 SPI2_NSS PB13 SPI2_SCK TIM1_CH1N " + "HCLK PCLK1 PCLK2 RCC_CFGR SYSCLK PLLM PLLN\n" * 4, + "base64": "TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcmsuIFRoZSBxdWljayBicm93biBmb3gganVtcHM=\n" + * 5, + "dna": "ACGTTGCAAGCTTGCATGCCTGCAGGTCGACTCTAGAGGATCCCCGGGTACCGAGCTCGAATTC\n" * 5, + # Icon fonts repeat a few private-use code points, also in a row. + "icon_rating_and_footer": _PROSE + + "\n" + + chr(0xF0AB) * 4 + + chr(0xF0AA) + + "\n" + + " ".join(chr(0xF09A + i) for i in range(5)), + # One U+FFFD from a math font, on a full page. + "single_replacement_character": _PROSE + "\nx \N{REPLACEMENT CHARACTER} y\n", + # pdfium can mark one end-of-line hyphen with a control character. + "hyphen_control": _PROSE.replace("regions", "re\x02\ngions", 1), } # Text layers from fonts whose glyphs were not mapped to real characters. @@ -60,6 +106,24 @@ def _words(first: int, last: int, count: int = 40) -> str: "".join(chr(0xF021 + (i + j) % 60) for j in range(4)) for i in range(30) ), "glyphs_as_accented_latin": _words(0x0100, 0x017F), + "glyphs_as_shifted_letters": _shift(_PROSE, 3), + "glyphs_as_letters_and_digits": _shift(_PROSE, -11), + # One cell in a broken font, on an otherwise good page. + "one_private_use_cell": _PROSE * 2 + + "\n" + + "".join(chr(0xF041 + i) for i in range(6)) + + "\n", + "one_control_code_cell": _PROSE * 2 + + "\nTotal " + + "".join(chr(0x10 + i) for i in range(6)) + + "\n", + # Below the share the control check needs, but in runs or in several cells. + "replacement_run_on_short_page": "Annual report of the regional transport " + "authority for 2024: \N{REPLACEMENT CHARACTER}\N{REPLACEMENT CHARACTER} " + "summary, outlook, notes", + "replacement_characters_in_many_cells": _PROSE + + "\n" + + "\n".join("Item " + "\N{REPLACEMENT CHARACTER}" * n for n in (4, 3, 3)), } From c5b3d92f910a5c16018c627beb148234205f8883 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:24:50 +0530 Subject: [PATCH 04/15] fix(ocr): pass replace_broken_text_layer to the auto-selected OCR engine OcrAutoModel built the options of the selected engine from mode and lang only, so with OcrAutoOptions a broken page was graded poor but not OCR'd. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/models/stages/ocr/auto_ocr_model.py | 15 +++++++++++++++ tests/test_replace_broken_text_layer.py | 20 ++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docling/models/stages/ocr/auto_ocr_model.py b/docling/models/stages/ocr/auto_ocr_model.py index 3ff1151f5c..000a6ce2cd 100644 --- a/docling/models/stages/ocr/auto_ocr_model.py +++ b/docling/models/stages/ocr/auto_ocr_model.py @@ -74,6 +74,9 @@ def __init__( options=OcrMacOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -99,6 +102,9 @@ def __init__( options=NemotronOcrOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -130,6 +136,9 @@ def __init__( backend="onnxruntime", mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -159,6 +168,9 @@ def __init__( options=EasyOcrOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -184,6 +196,9 @@ def __init__( backend="torch", mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py index 3000c219ce..c5b11be6ac 100644 --- a/tests/test_replace_broken_text_layer.py +++ b/tests/test_replace_broken_text_layer.py @@ -18,6 +18,8 @@ from docling.datamodel.base_models import InputFormat from docling.datamodel.document import ConversionResult from docling.datamodel.pipeline_options import ( + OcrAutoOptions, + OcrOptions, PdfPipelineOptions, TesseractCliOcrOptions, ) @@ -66,8 +68,11 @@ def _page_pdf(path: Path, text_layer: str) -> Path: return path -def _convert(path: Path, *, replace: bool) -> ConversionResult: - ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=replace) +def _convert( + path: Path, *, replace: bool, ocr: OcrOptions | None = None +) -> ConversionResult: + if ocr is None: + ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=replace) options = PdfPipelineOptions(do_ocr=True, do_table_structure=False, ocr_options=ocr) converter = DocumentConverter( format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} @@ -97,3 +102,14 @@ def test_good_text_layer_is_kept(tmp_path: Path) -> None: assert result.confidence.pages[1].parse_score == 1.0 assert layer in result.document.export_to_markdown() + + +def test_auto_ocr_engine_replaces_the_broken_layer(tmp_path: Path) -> None: + """The engine OcrAutoOptions selects gets the option too.""" + ocr = OcrAutoOptions(replace_broken_text_layer=True) + result = _convert( + _page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=True, ocr=ocr + ) + + assert result.confidence.pages[1].parse_score == 0.0 + assert _BROKEN_LAYER[:5] not in result.document.export_to_markdown() From b19fa045b795f493c26b85dd6ab849675a448aff Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:54:21 +0530 Subject: [PATCH 05/15] docs(ocr): describe replace_broken_text_layer Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docs/concepts/OCR.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 7cfd981785..a79df5c6b4 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -151,6 +151,38 @@ Engines that run one language at a time (RapidOCR, Nemotron-OCR) take the **firs about the rest. The KServe client also sends only the first entry. +## Replace a broken PDF text layer + +Some PDFs look correct on screen, but their text copies out as garbage, for example `8VceZWZTReV` +instead of `Certificate`. This happens when a font does not map its glyphs to real characters. With +`replace_broken_text_layer`, Docling detects these pages and reads them with OCR instead. Other +pages keep their PDF text. + +```python +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import PdfPipelineOptions, RapidOcrOptions +from docling.document_converter import DocumentConverter, PdfFormatOption + +pipeline_options = PdfPipelineOptions( + do_ocr=True, + ocr_options=RapidOcrOptions(replace_broken_text_layer=True), +) +doc_converter = DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options) + } +) +``` + +The option is off by default and works with every OCR engine. Pick an engine and `lang` that can +read the language of the document. A replaced page keeps its low `parse_score` in the +[confidence scores](confidence_scores.md), because the score describes the PDF text layer. + +```bash +docling report.pdf --replace-broken-text-layer +``` + + ## RapidOCR The engine's own vocabulary, in its own codes, is listed in From 48a1eb8b7a51386b8063a1f7b0558b4bc10e8c7e Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:24:22 +0530 Subject: [PATCH 06/15] fix(ocr): judge visible text only and keep invisible text on a replaced page The broken-text-layer check reads visible text only, so an invisible OCR layer or accessibility text cannot change the grade of a page. A replaced page keeps its invisible PDF cells where no OCR cell overlaps them, as the OCR merge already does. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/models/base_ocr_model.py | 29 +++++-- .../page_preprocessing_model.py | 14 ++-- tests/test_replace_broken_text_layer.py | 76 ++++++++++++++++--- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index 25afbb8906..5e1834d771 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -129,16 +129,21 @@ def _segregate_by_visibility( visible: list[TextCell] = [] invisible: list[TextCell] = [] for cell in cells: - if ( - isinstance(cell, PdfTextCell) - and cell.rendering_mode in _INVISIBLE_RENDERING_MODES - ): + if _is_invisible(cell): invisible.append(cell) else: visible.append(cell) return visible, invisible +def _is_invisible(cell: TextCell) -> bool: + """Whether a PDF cell is drawn with a text rendering mode that paints no ink.""" + return ( + isinstance(cell, PdfTextCell) + and cell.rendering_mode in _INVISIBLE_RENDERING_MODES + ) + + def _ocr_full_page(options: OcrOptions, page: Page) -> bool: """Whether a page is OCR'd in full and its PDF cells dropped. @@ -449,6 +454,7 @@ def post_process_cells( - If FULL_PAGE, or with `replace_broken_text_layer` the page's parse score is graded poor: Any existing PDF cells are ignored and only the OCR cells are used. + A page replaced for `replace_broken_text_layer` keeps its invisible cells. - If LAYOUT_REGIONS or PDF_AWARE_LAYOUT_REGIONS and the priority parameter is None, the priority is auto-selected based on the OcrMode: - OCR_FIRST when LAYOUT_REGIONS @@ -460,8 +466,15 @@ def post_process_cells( # Combine existing and OCR cells with overlap filtering full_page = _ocr_full_page(self.options, page) + # A replaced broken text layer keeps its invisible cells, as in the merge + keep_invisible = full_page and self.options.mode != OcrMode.FULL_PAGE if full_page: final_cells = ocr_cells + if keep_invisible: + _, invisible_cells = _segregate_by_visibility(existing_cells) + final_cells = self._merge_cells_by_priority( + final_cells, invisible_cells + ) else: if priority is None: priority = ( @@ -492,10 +505,14 @@ def post_process_cells( # In OcrMode.FULL_PAGE, PDF-extracted word/char cells are unreliable. Keep only OCR cells if full_page: page.parsed_page.word_cells = [ - c for c in page.parsed_page.word_cells if c.from_ocr + c + for c in page.parsed_page.word_cells + if c.from_ocr or (keep_invisible and _is_invisible(c)) ] page.parsed_page.char_cells = [ - c for c in page.parsed_page.char_cells if c.from_ocr + c + for c in page.parsed_page.char_cells + if c.from_ocr or (keep_invisible and _is_invisible(c)) ] page.parsed_page.has_words = len(page.parsed_page.word_cells) > 0 page.parsed_page.has_chars = len(page.parsed_page.char_cells) > 0 diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index b54788f824..3ddf45414b 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -19,6 +19,7 @@ from docling.datamodel.pipeline_options import OcrMode, PdfPipelineOptions from docling.datamodel.settings import settings from docling.models.base_model import BasePageModel +from docling.models.base_ocr_model import _segregate_by_visibility from docling.utils.profiling import TimeRecorder from docling.utils.text_quality import is_broken_text_layer @@ -124,13 +125,12 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: ) # Some broken text layers only show across the whole page, such as an - # unmapped font encoding. Judged only when the OCR may replace the layer. - if ( - self.options.detect_broken_text_layer - and not np.isnan(parse_score) - and is_broken_text_layer("\n".join(c.text for c in page.cells)) - ): - parse_score = 0.0 + # unmapped font encoding. Judged on visible text, only when the OCR may + # replace the layer. + if self.options.detect_broken_text_layer and not np.isnan(parse_score): + visible_cells, _ = _segregate_by_visibility(page.cells) + if is_broken_text_layer("\n".join(c.text for c in visible_cells)): + parse_score = 0.0 conv_res.confidence.pages[page.page_no].parse_score = parse_score page._parse_score = parse_score diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py index c5b11be6ac..b7139f09eb 100644 --- a/tests/test_replace_broken_text_layer.py +++ b/tests/test_replace_broken_text_layer.py @@ -38,8 +38,17 @@ ) -def _page_pdf(path: Path, text_layer: str) -> Path: - """One page: `_SHOWN` drawn as an image, under a text layer of `text_layer`.""" +def _page_pdf( + path: Path, + text_layer: str, + *, + invisible: bool = False, + hidden_text: str | None = None, +) -> Path: + """One page: `_SHOWN` drawn as an image, under a text layer of `text_layer`. + + `invisible` draws the layer with no ink; `hidden_text` adds an invisible line. + """ width, height = 400, 200 image = Image.new("RGB", (width * 3, height * 3), "white") font = ImageFont.load_default(size=110) @@ -52,20 +61,38 @@ def _page_pdf(path: Path, text_layer: str) -> Path: picture.set_matrix(pdfium.PdfMatrix().scale(width, height)) page.insert_obj(picture) + _add_text(pdf, page, text_layer, y=112, invisible=invisible) + if hidden_text is not None: + _add_text(pdf, page, hidden_text, y=20, invisible=True) + page.gen_content() + pdf.save(path) + return path + + +def _add_text( + pdf: pdfium.PdfDocument, + page: pdfium.PdfPage, + text_layer: str, + *, + y: float, + invisible: bool, +) -> None: helvetica = pdfium_c.FPDFText_LoadStandardFont(pdf.raw, b"Helvetica") text = pdfium_c.FPDFPageObj_CreateTextObj(pdf.raw, helvetica, 9.0) utf16 = (text_layer + "\0").encode("utf-16-le") pdfium_c.FPDFText_SetText( text, ctypes.cast(ctypes.c_char_p(utf16), ctypes.POINTER(pdfium_c.FPDF_WCHAR)) ) - # A broken layer still draws the right glyphs; white ink keeps this one from - # adding marks of its own to the image the OCR reads. - pdfium_c.FPDFPageObj_SetFillColor(text, 255, 255, 255, 255) - pdfium_c.FPDFPageObj_Transform(text, 1, 0, 0, 1, 20, 112) + if invisible: + pdfium_c.FPDFTextObj_SetTextRenderMode( + text, pdfium_c.FPDF_TEXTRENDERMODE_INVISIBLE + ) + else: + # A broken layer still draws the right glyphs; white ink keeps this one + # from adding marks of its own to the image the OCR reads. + pdfium_c.FPDFPageObj_SetFillColor(text, 255, 255, 255, 255) + pdfium_c.FPDFPageObj_Transform(text, 1, 0, 0, 1, 20, y) pdfium_c.FPDFPage_InsertObject(page.raw, text) - page.gen_content() - pdf.save(path) - return path def _convert( @@ -113,3 +140,34 @@ def test_auto_ocr_engine_replaces_the_broken_layer(tmp_path: Path) -> None: assert result.confidence.pages[1].parse_score == 0.0 assert _BROKEN_LAYER[:5] not in result.document.export_to_markdown() + + +def test_invisible_text_does_not_grade_the_layer(tmp_path: Path) -> None: + """Only visible text is judged.""" + pdf = _page_pdf(tmp_path / "hidden.pdf", _BROKEN_LAYER, invisible=True) + result = _convert(pdf, replace=True) + + assert result.confidence.pages[1].parse_score == 1.0 + + +def test_replaced_page_keeps_its_invisible_text(tmp_path: Path) -> None: + hidden = "Accessible summary of the statement" + pdf = _page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER, hidden_text=hidden) + ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=True) + options = PdfPipelineOptions( + do_ocr=True, + do_table_structure=False, + generate_parsed_pages=True, + ocr_options=ocr, + ) + converter = DocumentConverter( + format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} + ) + result = converter.convert(pdf) + + assert result.confidence.pages[1].parse_score == 0.0 + parsed = result.pages[0].parsed_page + assert parsed is not None + texts = [c.text for c in parsed.textline_cells] + assert hidden in texts + assert not any(_BROKEN_LAYER[:5] in t for t in texts) From 9677faa976fab439a951dcc840cb8f4dc56f5936 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:50:18 +0530 Subject: [PATCH 07/15] docs(ocr): state which broken text layers the check cannot detect The check finds characters that do not belong in text and Latin letters that are shifted as a whole. It cannot find text that uses the right alphabet but the wrong letters, or the right letters in the wrong order. Such pages keep their PDF text. The docs and the SDK usage reference now say this and point to OcrMode.FULL_PAGE. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/.agents/skills/docling/references/python-sdk.md | 4 ++++ docs/concepts/OCR.md | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index 5cc92d6973..e0152788ca 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -99,6 +99,10 @@ opts = PdfPipelineOptions( ) ``` +It does not detect text that uses the right alphabet but the wrong letters, or +the right letters in the wrong order (e.g. `Cartifecita` for `Certificate`). +Such pages keep their PDF text; `mode=OcrMode.FULL_PAGE` OCRs every page. + OCR can also run on a remote KServe v2 / Triton server. Page crops are sent to that server, so `enable_remote_services=True` is required: diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index a79df5c6b4..12c49cc6e7 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -182,6 +182,13 @@ read the language of the document. A replaced page keeps its low `parse_score` i docling report.pdf --replace-broken-text-layer ``` +Docling detects a broken page from characters that do not belong in text, such as control +characters, `�`, or private-use characters, and from Latin letters that are shifted as a whole, as +in the example above. It cannot detect a page whose text uses the right alphabet but the wrong +letters, or the right letters in the wrong order, for example `Cartifecita` instead of +`Certificate`. Such pages keep their PDF text. To read every page with OCR, set +`mode=OcrMode.FULL_PAGE` (`--ocr-mode full_page`). + ## RapidOCR From d95d5b26f5ff45fdf42e3d96d8a946ed9a462b3a Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:21:47 +0530 Subject: [PATCH 08/15] fix(ocr): count only ASCII letters in the shifted-letter check Some characters lowercase to more than one character, which made the check raise TypeError. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 2 +- tests/test_text_quality.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 51339fe62d..2376a95121 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -138,7 +138,7 @@ def _is_shifted_latin(text: str) -> bool: if not any("a" <= ch <= "z" for ch in token): continue for ch in token: - if "a" <= ch.lower() <= "z": + if ch.isascii() and ch.isalpha(): counts[ord(ch.lower()) - ord("a")] += 1 elif ch.isalpha(): other_letters += 1 diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 1c79f01e9c..451759ebb1 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -93,6 +93,8 @@ def _shift(text: str, offset: int) -> str: "single_replacement_character": _PROSE + "\nx \N{REPLACEMENT CHARACTER} y\n", # pdfium can mark one end-of-line hyphen with a control character. "hyphen_control": _PROSE.replace("regions", "re\x02\ngions", 1), + "dotted_capital_i": _PROSE + + "\nThe office in \N{LATIN CAPITAL LETTER I WITH DOT ABOVE}stanbul opened.\n", } # Text layers from fonts whose glyphs were not mapped to real characters. From 83a31d20c79b0d9701d57118205845b6cdd1bc8e Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:44:08 +0530 Subject: [PATCH 09/15] fix(ocr): replace a page only when the text layer check flags it A low per-cell score alone, for example from unmapped bullet glyphs, no longer sends a page with good text to OCR. The shifted-letter check ignores GLYPH placeholders and tokens with fewer than 3 letters. Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/datamodel/base_models.py | 12 ++++-------- docling/models/base_ocr_model.py | 12 +++++------- .../page_preprocessing/page_preprocessing_model.py | 5 +++-- docling/utils/text_quality.py | 13 ++++++++++--- tests/test_replace_broken_text_layer.py | 11 +++++++++++ tests/test_text_quality.py | 9 +++++++++ 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/docling/datamodel/base_models.py b/docling/datamodel/base_models.py index 2b0dfbcf55..00a89f2e18 100644 --- a/docling/datamodel/base_models.py +++ b/docling/datamodel/base_models.py @@ -581,9 +581,9 @@ class Page(BaseModel): # conversion results. _shape_lines: list[BoundingBox] | None = PrivateAttr(default=None) _shape_bounding_boxes: list[BoundingBox] | None = PrivateAttr(default=None) - # Quality of the page's PDF text layer, as in its confidence report; read by - # the OCR stage to replace a broken text layer (OcrOptions.replace_broken_text_layer). - _parse_score: float = PrivateAttr(default=float("nan")) + # Whether the page's PDF text layer is broken; read by the OCR stage to replace + # it (OcrOptions.replace_broken_text_layer). + _broken_text_layer: bool = PrivateAttr(default=False) _default_image_scale: float = 1.0 # Default image scale for external usage. _image_cache: dict[ float, Image @@ -691,10 +691,6 @@ class OpenAiApiResponse(BaseModel): ScoreValue = float -# Scores below this bound are graded QualityGrade.POOR. -POOR_SCORE_BOUND: float = 0.5 - - class QualityGrade(str, Enum): POOR = "poor" FAIR = "fair" @@ -722,7 +718,7 @@ def _coerce_none_or_nan_str(cls, v): return v def _score_to_grade(self, score: ScoreValue) -> QualityGrade: - if score < POOR_SCORE_BOUND: + if score < 0.5: return QualityGrade.POOR elif score < 0.8: return QualityGrade.FAIR diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index c5c14f9d8b..5436cd4f5a 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -22,7 +22,7 @@ from PIL import Image, ImageDraw from docling.datamodel.accelerator_options import AcceleratorOptions -from docling.datamodel.base_models import POOR_SCORE_BOUND, Page +from docling.datamodel.base_models import Page from docling.datamodel.document import ConversionResult from docling.datamodel.pipeline_options import OcrMode, OcrOptions from docling.datamodel.settings import settings @@ -148,12 +148,11 @@ def _ocr_full_page(options: OcrOptions, page: Page) -> bool: """Whether a page is OCR'd in full and its PDF cells dropped. True in `OcrMode.FULL_PAGE`, and, with `replace_broken_text_layer`, for a page - whose text layer is graded poor. A page without a text layer has no parse score - (NaN) and is never poor. + whose text layer page preprocessing found broken. """ if options.mode == OcrMode.FULL_PAGE: return True - return options.replace_broken_text_layer and page._parse_score < POOR_SCORE_BOUND + return options.replace_broken_text_layer and page._broken_text_layer class BaseOcrModel(BasePageModel, BaseModelWithOptions): @@ -263,10 +262,9 @@ def get_ocr_rects(self, page: Page) -> list[BoundingBox]: if _ocr_full_page(self.options, page): if self.options.mode != OcrMode.FULL_PAGE: _log.info( - "Page %s: parse score %.2f is graded poor;" - " replacing its text layer with full-page OCR", + "Page %s: the text layer is broken;" + " replacing it with full-page OCR", page.page_no, - page._parse_score, ) # A big bbox covering the entire page ocr_rects = [ diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index 3ddf45414b..a283cb3ca3 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -126,13 +126,14 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: # Some broken text layers only show across the whole page, such as an # unmapped font encoding. Judged on visible text, only when the OCR may - # replace the layer. + # replace the layer. Only this check replaces a page: a low per-cell score + # also comes from a few unmapped glyphs, such as bullets, on a good page. if self.options.detect_broken_text_layer and not np.isnan(parse_score): visible_cells, _ = _segregate_by_visibility(page.cells) if is_broken_text_layer("\n".join(c.text for c in visible_cells)): + page._broken_text_layer = True parse_score = 0.0 conv_res.confidence.pages[page.page_no].parse_score = parse_score - page._parse_score = parse_score def draw_cell_boxes( image: Image, diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 2376a95121..954e234ea5 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -8,6 +8,7 @@ """ import math +import re import unicodedata from itertools import groupby @@ -35,6 +36,8 @@ # Letters shifted by a broken ToUnicode map, as in a substitution cipher. _MIN_CIPHER_LETTERS = 200 # Too few letters to judge their frequencies. +_MIN_WORD_LETTERS = 3 # Shorter tokens, such as math variables, are not counted. +_GLYPH_PLACEHOLDER = re.compile(r"GLYPH<[^>]*>") # A parser's unmapped glyph. _MAX_VOWEL_SHARE = 0.30 # Latin-script text stays above this. _MAX_ENGLISH_COSINE = 0.60 # Letters not where English puts them... _MIN_SHAPE_COSINE = 0.90 # ...but with the frequency profile of a language. @@ -129,14 +132,18 @@ def _is_shifted_latin(text: str) -> bool: """Whether ASCII letters look like language with its letters swapped. A broken ToUnicode map often shifts every letter by a constant. That keeps - the shape of the letter-frequency profile but moves each letter. Only tokens - with a lowercase letter count, so acronyms and part numbers are skipped. + the shape of the letter-frequency profile but moves each letter. Only words + count: tokens with a lowercase letter and at least three ASCII letters, so + acronyms, part numbers and math variables are skipped. Placeholders for + unmapped glyphs, such as ``GLYPH<81>``, are not words either. """ counts = [0.0] * 26 other_letters = 0 - for token in text.split(): + for token in _GLYPH_PLACEHOLDER.sub(" ", text).split(): if not any("a" <= ch <= "z" for ch in token): continue + if sum(ch.isascii() and ch.isalpha() for ch in token) < _MIN_WORD_LETTERS: + continue for ch in token: if ch.isascii() and ch.isalpha(): counts[ord(ch.lower()) - ord("a")] += 1 diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py index b7139f09eb..62042604e2 100644 --- a/tests/test_replace_broken_text_layer.py +++ b/tests/test_replace_broken_text_layer.py @@ -131,6 +131,17 @@ def test_good_text_layer_is_kept(tmp_path: Path) -> None: assert layer in result.document.export_to_markdown() +def test_poor_cell_score_alone_keeps_the_text_layer(tmp_path: Path) -> None: + """An unmapped bullet glyph grades the page poor, but its text is good.""" + layer = "GLYPH<81> Annual Report of the Company" + result = _convert(_page_pdf(tmp_path / "bullet.pdf", layer), replace=True) + + assert result.confidence.pages[1].parse_score == 0.0 + markdown = result.document.export_to_markdown() + assert "Annual Report of the Company" in markdown + assert _SHOWN not in markdown + + def test_auto_ocr_engine_replaces_the_broken_layer(tmp_path: Path) -> None: """The engine OcrAutoOptions selects gets the option too.""" ocr = OcrAutoOptions(replace_broken_text_layer=True) diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 451759ebb1..75a84edde4 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -93,6 +93,15 @@ def _shift(text: str, offset: int) -> str: "single_replacement_character": _PROSE + "\nx \N{REPLACEMENT CHARACTER} y\n", # pdfium can mark one end-of-line hyphen with a control character. "hyphen_control": _PROSE.replace("regions", "re\x02\ngions", 1), + # A figure whose axis labels a parser could not map, on a good page. + "glyph_placeholders": _PROSE + "\nGLYPHGLYPHGLYPH" * 60, + # Formulas: many one-letter variables and few vowels. + "math_variables": ( + "(p, q, k) and (p, k, q)\n" + "PP[(q)] = 5 e2 [(k2 g - k k) - (q2 g - q q)]\n" + "= 2 e sin(p ^ q) B (p, q, k) = i2 g sin(k ^ q) (p, q, k) ln(s) ln(p) mn pq\n" + ) + * 6, "dotted_capital_i": _PROSE + "\nThe office in \N{LATIN CAPITAL LETTER I WITH DOT ABOVE}stanbul opened.\n", } From 031cff70229d352e6faf17cc6bd0114675f8cad2 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:56:08 +0530 Subject: [PATCH 10/15] fix(ocr): detect U+FFFD spread across many lines Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 8 ++++++++ tests/test_text_quality.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 954e234ea5..233c8153a1 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -31,6 +31,9 @@ _MANY_REPLACEMENTS = 12 _MANY_REPLACEMENTS_SHARE = 0.05 _REPLACEMENT_LINES = 3 +_REPLACEMENT_LINE_SHARE = ( + 0.10 # Lines with U+FFFD, as the per-cell score's 10% quantile. +) _LONG_REPLACEMENT_RUN = 8 _SOME_REPLACEMENTS_SHARE = 0.025 @@ -119,6 +122,11 @@ def _has_replacement_evidence(text: str, lines: list[str]) -> bool: return ( (count >= _MANY_REPLACEMENTS and share >= _MANY_REPLACEMENTS_SHARE) or (lines_hit >= _REPLACEMENT_LINES and share >= _SOME_REPLACEMENTS_SHARE) + # One U+FFFD in many lines: few characters, but most of the text is affected. + or ( + lines_hit >= _REPLACEMENT_LINES + and lines_hit >= _REPLACEMENT_LINE_SHARE * len(lines) + ) or (longest >= _LONG_REPLACEMENT_RUN and share >= _SOME_REPLACEMENTS_SHARE) ) diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 75a84edde4..a142ca40bf 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -135,6 +135,12 @@ def _shift(text: str, offset: int) -> str: "replacement_characters_in_many_cells": _PROSE + "\n" + "\n".join("Item " + "\N{REPLACEMENT CHARACTER}" * n for n in (4, 3, 3)), + # One U+FFFD in each of many lines: a small share of the characters. + "one_replacement_in_many_lines": "\n".join( + "BIB@QOLKF@ PB@QOFQFBP QO>AFKD\N{REPLACEMENT CHARACTER} IBKAFKD >KA M>VJBKQ" + " PBOSF@BP CLO @LJM>KFBP >KA FKAFSFA§>IP" + for _ in range(8) + ), } From afb05c774f6a2a267df1e9dafcec7251bfb69512 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:15:17 +0530 Subject: [PATCH 11/15] fix(ocr): do not count IPA symbols as accented Latin Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 11 +++++++++-- tests/test_text_quality.py | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 233c8153a1..1e25b970c5 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -71,12 +71,19 @@ def _is_private_use(ch: str) -> bool: def _non_ascii_latin_share(text: str) -> float: - """Share of Latin letters outside ASCII, on text where Latin letters dominate.""" + """Share of Latin letters outside ASCII, on text where Latin letters dominate. + + Phonetic symbols (IPA) are not counted as Latin: no 8-bit code page decodes to + them, so they are never the accented Latin of a misread font. + """ letters = [ch for ch in text if ch.isalpha()] latin = [ ch for ch in letters - if "LATIN" in unicodedata.name(ch, "") and not "\uff00" <= ch <= "\uffef" + if "LATIN" in unicodedata.name(ch, "") + and not "\uff00" <= ch <= "\uffef" + and not "\u0250" <= ch <= "\u02af" + and not "\u1d00" <= ch <= "\u1dbf" ] if len(latin) < _MIN_LATIN_LETTERS or len(latin) * 2 < len(letters): return 0.0 diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index a142ca40bf..acd0ae00b6 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -104,6 +104,14 @@ def _shift(text: str, offset: int) -> str: * 6, "dotted_capital_i": _PROSE + "\nThe office in \N{LATIN CAPITAL LETTER I WITH DOT ABOVE}stanbul opened.\n", + # The 25 vowels of an IPA vowel chart, in three rows: phonetic symbols, not mojibake. + "ipa_vowel_chart": "\n".join( + [ + "\u026a \u028f \u028a e \u00f8 \u0259 \u0250 \u00e6 a \u0251 \u0252 \u028c " + "\u0254 \u0153 \u025c \u025e \u0264 o \u0258 \u0275 \u026f \u0268 \u0289 y u" + ] + * 3 + ), } # Text layers from fonts whose glyphs were not mapped to real characters. From 364e4c01200652be0e0b66bc4bd7182e8c59a52b Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:43:37 +0530 Subject: [PATCH 12/15] fix(ocr): describe the page replacement rule and make the visibility helper public Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/datamodel/pipeline_options.py | 14 +++++++------- docling/models/base_ocr_model.py | 18 +++++++++--------- .../page_preprocessing_model.py | 4 ++-- docs/concepts/OCR.md | 5 +++-- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index f224fd2bc3..a2e13abf14 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -265,13 +265,13 @@ class OcrOptions(BaseOptions): Field( description=( "Replace a broken PDF text layer with OCR, page by page. When enabled," - " a page's `parse_score` also drops to 0 if its text layer comes from" - " a font whose glyphs were not mapped to real characters. A page whose" - " `parse_score` is in the `poor` quality grade (below 0.5), from this" - " check or from the existing glyph-name checks, is OCR'd in full and" - " its OCR text is used instead of its PDF text for that page only." - " Its `parse_score` keeps describing the PDF text layer. Other pages," - " and pages without a text layer, keep the usual behaviour." + " Docling checks the visible text layer of each page. A page whose" + " text layer is broken, for example from a font whose glyphs were not" + " mapped to real characters, is read with full-page OCR, and its OCR" + " text is used instead of its PDF text for that page only. Its" + " `parse_score` drops to 0 and keeps describing the PDF text layer. A" + " low `parse_score` alone does not replace a page. Other pages, and" + " pages without a text layer, keep the usual behavior." ), examples=[False, True], ), diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index 5436cd4f5a..00307699b3 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -118,7 +118,7 @@ class _MergeCellsPriority(str, Enum): _PDF_CELLS_POST_PROCESSING_SEGREGATION = True -def _segregate_by_visibility( +def segregate_by_visibility( cells: Iterable[TextCell], ) -> tuple[list[TextCell], list[TextCell]]: """Split cells into (visible, invisible), preserving the order within each group. @@ -129,14 +129,14 @@ def _segregate_by_visibility( visible: list[TextCell] = [] invisible: list[TextCell] = [] for cell in cells: - if _is_invisible(cell): + if is_invisible(cell): invisible.append(cell) else: visible.append(cell) return visible, invisible -def _is_invisible(cell: TextCell) -> bool: +def is_invisible(cell: TextCell) -> bool: """Whether a PDF cell is drawn with a text rendering mode that paints no ink.""" return ( isinstance(cell, PdfTextCell) @@ -626,8 +626,8 @@ def post_process_cells( r""" Post-process the OCR cells and update the page object according to the algorithm: - - If FULL_PAGE, or with `replace_broken_text_layer` the page's parse score is - graded poor: + - If FULL_PAGE, or with `replace_broken_text_layer` the page's text layer is + broken (`is_broken_text_layer`): Any existing PDF cells are ignored and only the OCR cells are used. A page replaced for `replace_broken_text_layer` keeps its invisible cells. - If LAYOUT_REGIONS or PDF_AWARE_LAYOUT_REGIONS and the priority parameter is None, @@ -646,7 +646,7 @@ def post_process_cells( if full_page: final_cells = ocr_cells if keep_invisible: - _, invisible_cells = _segregate_by_visibility(existing_cells) + _, invisible_cells = segregate_by_visibility(existing_cells) final_cells = self._merge_cells_by_priority( final_cells, invisible_cells ) @@ -682,12 +682,12 @@ def post_process_cells( page.parsed_page.word_cells = [ c for c in page.parsed_page.word_cells - if c.from_ocr or (keep_invisible and _is_invisible(c)) + if c.from_ocr or (keep_invisible and is_invisible(c)) ] page.parsed_page.char_cells = [ c for c in page.parsed_page.char_cells - if c.from_ocr or (keep_invisible and _is_invisible(c)) + if c.from_ocr or (keep_invisible and is_invisible(c)) ] page.parsed_page.has_words = len(page.parsed_page.word_cells) > 0 page.parsed_page.has_chars = len(page.parsed_page.char_cells) > 0 @@ -715,7 +715,7 @@ def _merge_ocr_and_pdf_cells( visible_cells: list[TextCell] invisible_cells: list[TextCell] if segregate_pdf_cells: - visible_cells, invisible_cells = _segregate_by_visibility(pdf_cells) + visible_cells, invisible_cells = segregate_by_visibility(pdf_cells) else: visible_cells, invisible_cells = list(pdf_cells), [] diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index a283cb3ca3..23cb7154e1 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -19,7 +19,7 @@ from docling.datamodel.pipeline_options import OcrMode, PdfPipelineOptions from docling.datamodel.settings import settings from docling.models.base_model import BasePageModel -from docling.models.base_ocr_model import _segregate_by_visibility +from docling.models.base_ocr_model import segregate_by_visibility from docling.utils.profiling import TimeRecorder from docling.utils.text_quality import is_broken_text_layer @@ -129,7 +129,7 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: # replace the layer. Only this check replaces a page: a low per-cell score # also comes from a few unmapped glyphs, such as bullets, on a good page. if self.options.detect_broken_text_layer and not np.isnan(parse_score): - visible_cells, _ = _segregate_by_visibility(page.cells) + visible_cells, _ = segregate_by_visibility(page.cells) if is_broken_text_layer("\n".join(c.text for c in visible_cells)): page._broken_text_layer = True parse_score = 0.0 diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index e8a0ab7968..91625d6879 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -183,8 +183,9 @@ docling report.pdf --replace-broken-text-layer ``` Docling detects a broken page from characters that do not belong in text, such as control -characters, `�`, or private-use characters, and from Latin letters that are shifted as a whole, as -in the example above. It cannot detect a page whose text uses the right alphabet but the wrong +characters, `�`, or private-use characters. It also detects accented Latin letters that stand for +another script, for example `Îò÷¸ò ïîêàçûâàåò` for Russian text, and Latin letters that are +shifted as a whole, as in the example above. It cannot detect a page whose text uses the right alphabet but the wrong letters, or the right letters in the wrong order, for example `Cartifecita` instead of `Certificate`. Such pages keep their PDF text. To read every page with OCR, set `mode=OcrMode.FULL_PAGE` (`--ocr-mode full_page`). From 24889ceeb0b455a8bb700015add4413bac03ad11 Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:05:19 +0530 Subject: [PATCH 13/15] fix(ocr): do not count lone U+FFFD bullets as damaged lines Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 17 +++++++++++++++-- tests/test_text_quality.py | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index 1e25b970c5..f11e6a97b6 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -115,6 +115,18 @@ def _is_broken_line(line: str) -> bool: return False +def _has_replacement_in_text(line: str) -> bool: + """Whether a line has U+FFFD other than a lone one at its start. + + An unmapped bullet glyph is a lone U+FFFD before the item text; a broken font puts + U+FFFD inside words and numbers. + """ + stripped = line.lstrip() + if stripped[:1] == "�" and (len(stripped) == 1 or stripped[1].isspace()): + stripped = stripped[1:] + return "�" in stripped + + def _has_replacement_evidence(text: str, lines: list[str]) -> bool: """Whether U+FFFD is dense or clustered enough to mark the page broken.""" count = text.count("\ufffd") @@ -126,13 +138,14 @@ def _has_replacement_evidence(text: str, lines: list[str]) -> bool: return True share = count / chars lines_hit = sum("\ufffd" in line for line in lines) + lines_damaged = sum(_has_replacement_in_text(line) for line in lines) return ( (count >= _MANY_REPLACEMENTS and share >= _MANY_REPLACEMENTS_SHARE) or (lines_hit >= _REPLACEMENT_LINES and share >= _SOME_REPLACEMENTS_SHARE) # One U+FFFD in many lines: few characters, but most of the text is affected. or ( - lines_hit >= _REPLACEMENT_LINES - and lines_hit >= _REPLACEMENT_LINE_SHARE * len(lines) + lines_damaged >= _REPLACEMENT_LINES + and lines_damaged >= _REPLACEMENT_LINE_SHARE * len(lines) ) or (longest >= _LONG_REPLACEMENT_RUN and share >= _SOME_REPLACEMENTS_SHARE) ) diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index acd0ae00b6..3b6c4ebee2 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -112,6 +112,11 @@ def _shift(text: str, offset: int) -> str: ] * 3 ), + # Unmapped list bullets: a lone U+FFFD before good item text, in 3 of 25 lines. + "bullet_replacement_markers": "\n".join( + [f"\N{REPLACEMENT CHARACTER} Revenue grew in region {n}" for n in range(3)] + + [f"Line {n}: the regional teams reported stable demand." for n in range(22)] + ), } # Text layers from fonts whose glyphs were not mapped to real characters. From bec1a5e84f557af8d9e469edd2c88f61db07724e Mon Sep 17 00:00:00 2001 From: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:20:09 +0530 Subject: [PATCH 14/15] fix(ocr): skip lone U+FFFD bullets in all replacement-character checks Signed-off-by: Yashika Malhotra <106444881+yashikam19@users.noreply.github.com> --- docling/utils/text_quality.py | 22 ++++++++++++---------- tests/test_text_quality.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index f11e6a97b6..fae99ec0c2 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -115,16 +115,16 @@ def _is_broken_line(line: str) -> bool: return False -def _has_replacement_in_text(line: str) -> bool: - """Whether a line has U+FFFD other than a lone one at its start. +def _without_bullet(line: str) -> str: + """Return the line without a lone U+FFFD at its start. An unmapped bullet glyph is a lone U+FFFD before the item text; a broken font puts U+FFFD inside words and numbers. """ stripped = line.lstrip() - if stripped[:1] == "�" and (len(stripped) == 1 or stripped[1].isspace()): - stripped = stripped[1:] - return "�" in stripped + if stripped[:1] == "\ufffd" and (len(stripped) == 1 or stripped[1].isspace()): + return stripped[1:] + return line def _has_replacement_evidence(text: str, lines: list[str]) -> bool: @@ -138,14 +138,13 @@ def _has_replacement_evidence(text: str, lines: list[str]) -> bool: return True share = count / chars lines_hit = sum("\ufffd" in line for line in lines) - lines_damaged = sum(_has_replacement_in_text(line) for line in lines) return ( (count >= _MANY_REPLACEMENTS and share >= _MANY_REPLACEMENTS_SHARE) or (lines_hit >= _REPLACEMENT_LINES and share >= _SOME_REPLACEMENTS_SHARE) # One U+FFFD in many lines: few characters, but most of the text is affected. or ( - lines_damaged >= _REPLACEMENT_LINES - and lines_damaged >= _REPLACEMENT_LINE_SHARE * len(lines) + lines_hit >= _REPLACEMENT_LINES + and lines_hit >= _REPLACEMENT_LINE_SHARE * len(lines) ) or (longest >= _LONG_REPLACEMENT_RUN and share >= _SOME_REPLACEMENTS_SHARE) ) @@ -205,13 +204,16 @@ def is_broken_text_layer(text: str) -> bool: if len(stripped) < _MIN_CHARS: return False lines = stripped.splitlines() + # The U+FFFD checks skip unmapped bullets, which precede good item text. + item_lines = [_without_bullet(line) for line in lines] + item_text = "\n".join(item_lines) # A font whose glyphs were mapped to control or private-use code points. - if _share(stripped, _CONTROL, replacement=True) >= _CONTROL_SHARE: + if _share(item_text, _CONTROL, replacement=True) >= _CONTROL_SHARE: return True if _share(stripped, frozenset({"Co"}), replacement=False) >= _PRIVATE_USE_SHARE: return True - if _has_replacement_evidence(stripped, lines): + if _has_replacement_evidence(item_text, item_lines): return True # The same, in one cell of an otherwise good page. if any(_is_broken_line(line) for line in lines): diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 3b6c4ebee2..32c6333fc1 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -117,6 +117,18 @@ def _shift(text: str, offset: int) -> str: [f"\N{REPLACEMENT CHARACTER} Revenue grew in region {n}" for n in range(3)] + [f"Line {n}: the regional teams reported stable demand." for n in range(22)] ), + # A short slide: a title and 3 items, each bullet in its own cell. + "bullet_slide": "Quarterly results\n" + + "\n".join( + f"\N{REPLACEMENT CHARACTER}\n{item}" + for item in ("Revenue grew by 5%", "Costs fell", "Margins improved") + ), + # An agenda slide: 5 short items, each a bullet, a space and the item text. + "bullet_agenda": "Agenda\n" + + "\n".join( + f"\N{REPLACEMENT CHARACTER} {item}" + for item in ("Welcome", "Results", "Outlook", "Questions", "Close") + ), } # Text layers from fonts whose glyphs were not mapped to real characters. From ea4a65349793a292f986acd0a9f16e7fcf935d52 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Tue, 27 Oct 2026 13:41:58 +0100 Subject: [PATCH 15/15] refactor(ocr): make the broken text layer check compact and script-neutral Replace the 23 thresholds and the English letter-frequency profile with four checks and eight thresholds: - control, U+FFFD or private-use characters: at least 3% of the page - U+FFFD or control characters in at least 10% of the lines (and 3) - Latin letters in U+00C0-U+024F: at least half of the Latin letters - vowel share of Latin letters in words below 0.25; accented vowels count by their base letter, so no language profile is needed The first two checks apply to every script. The last two apply to Latin letters only. One broken cell on an otherwise good page no longer replaces the page, because the OCR would replace all of its good text. Signed-off-by: Christoph Auer --- docling/utils/text_quality.py | 246 ++++++++++------------------------ docs/concepts/OCR.md | 15 ++- tests/test_text_quality.py | 25 ++-- 3 files changed, 90 insertions(+), 196 deletions(-) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py index fae99ec0c2..3009c992ea 100644 --- a/docling/utils/text_quality.py +++ b/docling/utils/text_quality.py @@ -3,191 +3,94 @@ """Detect a PDF text layer that does not match the text drawn on the page. -The checks look at a page's whole text layer and at each of its lines, and only -at patterns broken text produces, such as unmapped font encodings. +The checks look only at patterns that broken font mappings produce. Two of them +apply to every script: characters that do not belong in text, on the page or +in many of its lines. Two apply to Latin letters only, and use no language +profile: accented letters that stand for another script, and too few vowels. """ -import math import re import unicodedata -from itertools import groupby _MIN_CHARS = 20 # Less text than this is left alone. -_MIN_LATIN_LETTERS = 50 # Too few Latin letters to judge their mix. -_CONTROL_SHARE = 0.03 # Control or U+FFFD characters. -_PRIVATE_USE_SHARE = 0.5 # Private-use characters; icon fonts stay far below. -_NON_ASCII_LATIN_SHARE = 0.5 # Latin letters outside ASCII. -_ASCII_LETTER_SHARE = 1 / 3 # Mojibake leaves few ASCII letters. - -# One line is one text cell. -_PRIVATE_USE_RUN = 4 # Different private-use characters in a row. -_MIN_LINE_CHARS = 5 # Less text than this in a line or token is not judged. -_TOKEN_CONTROLS = 2 # Control characters in one token. -_TOKEN_CONTROL_SHARE = 0.05 - -# U+FFFD on the page. -_SHORT_PAGE_CHARS = 80 # On a page this short, a run of 2 is enough. -_SHORT_PAGE_RUN = 2 -_MANY_REPLACEMENTS = 12 -_MANY_REPLACEMENTS_SHARE = 0.05 -_REPLACEMENT_LINES = 3 -_REPLACEMENT_LINE_SHARE = ( - 0.10 # Lines with U+FFFD, as the per-cell score's 10% quantile. -) -_LONG_REPLACEMENT_RUN = 8 -_SOME_REPLACEMENTS_SHARE = 0.025 - -# Letters shifted by a broken ToUnicode map, as in a substitution cipher. -_MIN_CIPHER_LETTERS = 200 # Too few letters to judge their frequencies. -_MIN_WORD_LETTERS = 3 # Shorter tokens, such as math variables, are not counted. +_JUNK_SHARE = 0.03 # Control, U+FFFD or private-use characters on the page. +_JUNK_LINE_SHARE = 0.10 # Lines with U+FFFD or a control character... +_MIN_JUNK_LINES = 3 # ...and at least this many. + +# Latin letters: only judged when there are enough of them. +_MIN_LATIN_LETTERS = 50 +_ACCENTED_LATIN_SHARE = 0.5 # Real text in Latin script stays far below. +_MIN_WORD_LETTERS = 200 +_MIN_VOWEL_SHARE = 0.25 # Real text in Latin script stays above about 0.3. +# Shorter tokens are math variables; longer ones are base64 or hashes. +_WORD_LENGTH = range(3, 21) +# Base vowels, with æ, œ, ø, dotless i and schwa, which have no base letter. +_VOWELS = frozenset("aeiouy\u00e6\u0153\u00f8\u0131\u0259") _GLYPH_PLACEHOLDER = re.compile(r"GLYPH<[^>]*>") # A parser's unmapped glyph. -_MAX_VOWEL_SHARE = 0.30 # Latin-script text stays above this. -_MAX_ENGLISH_COSINE = 0.60 # Letters not where English puts them... -_MIN_SHAPE_COSINE = 0.90 # ...but with the frequency profile of a language. -# English letter frequencies, percent, a to z (Lewand, "Cryptological Mathematics"). -_ENGLISH_LETTER_FREQ = ( - 8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, - 6.7, 7.5, 1.9, 0.1, 6.0, 6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07, -) # fmt: skip - -# Not "Cn" (unassigned): that depends on the Unicode version of the Python running. -_CONTROL = frozenset({"Cc", "Cs"}) - - -def _share(text: str, categories: frozenset[str], *, replacement: bool) -> float: - """Share of non-space characters in the given Unicode categories.""" - chars = [ch for ch in text if not ch.isspace()] - if not chars: - return 0.0 - hits = sum( - unicodedata.category(ch) in categories or (replacement and ch == "\ufffd") - for ch in chars - ) - return hits / len(chars) -def _is_private_use(ch: str) -> bool: - return unicodedata.category(ch) == "Co" +def _is_junk(ch: str) -> bool: + """Whether a character is a control, surrogate, private-use or U+FFFD.""" + category = unicodedata.category(ch) + return ( + ch == "�" or category in ("Cs", "Co") or (category == "Cc" and not ch.isspace()) + ) -def _non_ascii_latin_share(text: str) -> float: - """Share of Latin letters outside ASCII, on text where Latin letters dominate. +def _has_replacement(line: str) -> bool: + """Whether a line has U+FFFD or a control character after its first character. - Phonetic symbols (IPA) are not counted as Latin: no 8-bit code page decodes to - them, so they are never the accented Latin of a misread font. + Private-use characters are not counted: icon fonts put them in many lines. """ - letters = [ch for ch in text if ch.isalpha()] - latin = [ - ch - for ch in letters - if "LATIN" in unicodedata.name(ch, "") - and not "\uff00" <= ch <= "\uffef" - and not "\u0250" <= ch <= "\u02af" - and not "\u1d00" <= ch <= "\u1dbf" - ] - if len(latin) < _MIN_LATIN_LETTERS or len(latin) * 2 < len(letters): - return 0.0 - return sum(ord(ch) > 127 for ch in latin) / len(latin) - - -def _ascii_letter_share(text: str) -> float: - """Share of non-space characters that are ASCII letters.""" - chars = [ch for ch in text if not ch.isspace()] - if not chars: - return 0.0 - return sum(ch.isascii() and ch.isalpha() for ch in chars) / len(chars) - - -def _is_broken_line(line: str) -> bool: - """Whether one line holds a run or a token that only a broken font produces.""" - # Icons repeat a few private-use code points; the letters of words vary. - for is_private_use, run in groupby(line, key=_is_private_use): - if is_private_use and len(set(run)) >= _PRIVATE_USE_RUN: - return True - for token in line.split(): - if len(token) < _MIN_LINE_CHARS: - continue - controls = sum(unicodedata.category(ch) in _CONTROL for ch in token) - if controls >= _TOKEN_CONTROLS and controls >= _TOKEN_CONTROL_SHARE * len( - token - ): - return True - return False + return any( + ch == "�" or (unicodedata.category(ch) == "Cc" and not ch.isspace()) + for ch in line.lstrip()[1:] + ) def _without_bullet(line: str) -> str: - """Return the line without a lone U+FFFD at its start. + """Return the line without a lone junk character at its start. - An unmapped bullet glyph is a lone U+FFFD before the item text; a broken font puts - U+FFFD inside words and numbers. + An unmapped bullet glyph is a lone junk character before the item text; a + broken font puts junk inside words and numbers. """ stripped = line.lstrip() - if stripped[:1] == "\ufffd" and (len(stripped) == 1 or stripped[1].isspace()): + if stripped[:1] and _is_junk(stripped[0]) and stripped[1:2].isspace(): return stripped[1:] + if len(stripped) == 1 and _is_junk(stripped): + return "" return line -def _has_replacement_evidence(text: str, lines: list[str]) -> bool: - """Whether U+FFFD is dense or clustered enough to mark the page broken.""" - count = text.count("\ufffd") - if not count: - return False - chars = sum(not ch.isspace() for ch in text) - longest = max((len(list(run)) for ch, run in groupby(text) if ch == "\ufffd")) - if chars <= _SHORT_PAGE_CHARS and longest >= _SHORT_PAGE_RUN: - return True - share = count / chars - lines_hit = sum("\ufffd" in line for line in lines) - return ( - (count >= _MANY_REPLACEMENTS and share >= _MANY_REPLACEMENTS_SHARE) - or (lines_hit >= _REPLACEMENT_LINES and share >= _SOME_REPLACEMENTS_SHARE) - # One U+FFFD in many lines: few characters, but most of the text is affected. - or ( - lines_hit >= _REPLACEMENT_LINES - and lines_hit >= _REPLACEMENT_LINE_SHARE * len(lines) - ) - or (longest >= _LONG_REPLACEMENT_RUN and share >= _SOME_REPLACEMENTS_SHARE) - ) - +def _is_latin(ch: str) -> bool: + return ch.isalpha() and "LATIN" in unicodedata.name(ch, "") -def _cosine(a: list[float], b: list[float]) -> float: - norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b)) - return sum(x * y for x, y in zip(a, b)) / norm if norm else 1.0 +def _is_wrong_latin(text: str) -> bool: + """Whether the Latin letters stand for another script or were moved. -def _is_shifted_latin(text: str) -> bool: - """Whether ASCII letters look like language with its letters swapped. - - A broken ToUnicode map often shifts every letter by a constant. That keeps - the shape of the letter-frequency profile but moves each letter. Only words - count: tokens with a lowercase letter and at least three ASCII letters, so - acronyms, part numbers and math variables are skipped. Placeholders for - unmapped glyphs, such as ``GLYPH<81>``, are not words either. + A non-Latin 8-bit font read as Latin-1 gives mostly accented letters + (`Îò÷ò`). A broken ToUnicode map that moves letters, or a legacy font that + puts another script on ASCII letters, leaves few vowels. Accented vowels + count by their base letter, so no Latin-script language is penalized. """ - counts = [0.0] * 26 - other_letters = 0 - for token in _GLYPH_PLACEHOLDER.sub(" ", text).split(): - if not any("a" <= ch <= "z" for ch in token): - continue - if sum(ch.isascii() and ch.isalpha() for ch in token) < _MIN_WORD_LETTERS: - continue - for ch in token: - if ch.isascii() and ch.isalpha(): - counts[ord(ch.lower()) - ord("a")] += 1 - elif ch.isalpha(): - other_letters += 1 - letters = sum(counts) - if letters < _MIN_CIPHER_LETTERS or other_letters > letters: + text = _GLYPH_PLACEHOLDER.sub(" ", text) + latin = [ch for ch in text if _is_latin(ch)] + if len(latin) < _MIN_LATIN_LETTERS: return False - vowels = sum(counts[ord(v) - ord("a")] for v in "aeiou") - if vowels / letters > _MAX_VOWEL_SHARE: + # 8-bit code pages decode to U+00C0-U+024F; IPA and Vietnamese tones lie above. + accented = sum("À" <= ch <= "ɏ" for ch in latin) + if accented >= _ACCENTED_LATIN_SHARE * len(latin): + return True + words = [] + for token in text.split(): + letters = [ch for ch in token if _is_latin(ch)] + if len(letters) in _WORD_LENGTH and any(ch.islower() for ch in letters): + words.extend(letters) + if len(words) < _MIN_WORD_LETTERS: return False - english = list(_ENGLISH_LETTER_FREQ) - return ( - _cosine(counts, english) < _MAX_ENGLISH_COSINE - and _cosine(sorted(counts, reverse=True), sorted(english, reverse=True)) - >= _MIN_SHAPE_COSINE - ) + vowels = sum(unicodedata.normalize("NFD", ch)[0].lower() in _VOWELS for ch in words) + return vowels < _MIN_VOWEL_SHARE * len(words) def is_broken_text_layer(text: str) -> bool: @@ -203,28 +106,17 @@ def is_broken_text_layer(text: str) -> bool: stripped = text.strip() if len(stripped) < _MIN_CHARS: return False - lines = stripped.splitlines() - # The U+FFFD checks skip unmapped bullets, which precede good item text. - item_lines = [_without_bullet(line) for line in lines] - item_text = "\n".join(item_lines) + lines = [_without_bullet(line) for line in stripped.splitlines()] + chars = [ch for line in lines for ch in line if not ch.isspace()] + if not chars: + return False - # A font whose glyphs were mapped to control or private-use code points. - if _share(item_text, _CONTROL, replacement=True) >= _CONTROL_SHARE: - return True - if _share(stripped, frozenset({"Co"}), replacement=False) >= _PRIVATE_USE_SHARE: - return True - if _has_replacement_evidence(item_text, item_lines): + # A font whose glyphs were mapped to control, private-use or no code points. + if sum(map(_is_junk, chars)) >= _JUNK_SHARE * len(chars): return True - # The same, in one cell of an otherwise good page. - if any(_is_broken_line(line) for line in lines): - return True - - # A non-Latin font whose glyphs were mapped to accented Latin letters. - if ( - _non_ascii_latin_share(stripped) >= _NON_ASCII_LATIN_SHARE - and _ascii_letter_share(stripped) < _ASCII_LETTER_SHARE - ): + # The same in many lines, such as unmapped ligatures (`publica�on`). + junk_lines = sum(map(_has_replacement, lines)) + if junk_lines >= max(_MIN_JUNK_LINES, _JUNK_LINE_SHARE * len(lines)): return True - # A Latin font whose glyphs were mapped to the wrong ASCII letters. - return _is_shifted_latin(stripped) + return _is_wrong_latin(stripped) diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 91625d6879..32e2774cad 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -183,12 +183,15 @@ docling report.pdf --replace-broken-text-layer ``` Docling detects a broken page from characters that do not belong in text, such as control -characters, `�`, or private-use characters. It also detects accented Latin letters that stand for -another script, for example `Îò÷¸ò ïîêàçûâàåò` for Russian text, and Latin letters that are -shifted as a whole, as in the example above. It cannot detect a page whose text uses the right alphabet but the wrong -letters, or the right letters in the wrong order, for example `Cartifecita` instead of -`Certificate`. Such pages keep their PDF text. To read every page with OCR, set -`mode=OcrMode.FULL_PAGE` (`--ocr-mode full_page`). +characters, `�`, or private-use characters, on the page or in many of its lines. These checks work +for every script. For Latin letters, it also detects accented letters that stand for another +script, for example `Îò÷¸ò ïîêàçûâàåò` for Russian text, and letters with too few vowels, as in +the example above. The vowel check uses no language profile, so it works the same for every +Latin-script language. One broken cell on an otherwise good page does not replace the page. + +Docling cannot detect a page whose text uses the right alphabet but the wrong letters, or the right +letters in the wrong order, for example `Cartifecita` instead of `Certificate`. Such pages keep +their PDF text. To read every page with OCR, set `mode=OcrMode.FULL_PAGE` (`--ocr-mode full_page`). ## RapidOCR diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py index 32c6333fc1..55720d1038 100644 --- a/tests/test_text_quality.py +++ b/tests/test_text_quality.py @@ -129,6 +129,15 @@ def _shift(text: str, offset: int) -> str: f"\N{REPLACEMENT CHARACTER} {item}" for item in ("Welcome", "Results", "Outlook", "Questions", "Close") ), + # One cell in a broken font: OCR would replace the good text of the whole page. + "one_private_use_cell": _PROSE * 2 + + "\n" + + "".join(chr(0xF041 + i) for i in range(6)) + + "\n", + "one_control_code_cell": _PROSE * 2 + + "\nTotal " + + "".join(chr(0x10 + i) for i in range(6)) + + "\n", } # Text layers from fonts whose glyphs were not mapped to real characters. @@ -144,19 +153,9 @@ def _shift(text: str, offset: int) -> str: "glyphs_as_accented_latin": _words(0x0100, 0x017F), "glyphs_as_shifted_letters": _shift(_PROSE, 3), "glyphs_as_letters_and_digits": _shift(_PROSE, -11), - # One cell in a broken font, on an otherwise good page. - "one_private_use_cell": _PROSE * 2 - + "\n" - + "".join(chr(0xF041 + i) for i in range(6)) - + "\n", - "one_control_code_cell": _PROSE * 2 - + "\nTotal " - + "".join(chr(0x10 + i) for i in range(6)) - + "\n", - # Below the share the control check needs, but in runs or in several cells. - "replacement_run_on_short_page": "Annual report of the regional transport " - "authority for 2024: \N{REPLACEMENT CHARACTER}\N{REPLACEMENT CHARACTER} " - "summary, outlook, notes", + # Not only English: the vowel check uses no language profile. + "welsh_as_shifted_letters": _shift(ORDINARY["welsh"], 3), + # Below the share the control check needs, but in several cells. "replacement_characters_in_many_cells": _PROSE + "\n" + "\n".join("Item " + "\N{REPLACEMENT CHARACTER}" * n for n in (4, 3, 3)),