diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index 7e3ca653f8..4d9776aa51 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -79,6 +79,7 @@ Decision guide: | Document | Use | |---|---| | Born-digital PDF (text selectable) | Standard (fast, no GPU) | +| Text looks right on the page but copies out as garbage | Standard with `--replace-broken-text-layer` | | Scanned / image-only PDF | Standard with OCR, or `--pipeline vlm` for best quality | | Complex/multi-column layout, dense tables | `--pipeline vlm` | | Handwriting or formulas | `--pipeline vlm` (standard OCR won't handle these) | @@ -96,6 +97,7 @@ docling scan.pdf --ocr-engine rapidocr --output /tmp/ # lightweight docling scan.pdf --ocr-engine tesserocr --output /tmp/ # needs system Tesseract docling scan.pdf --ocr-engine ocrmac --output /tmp/ # macOS Vision (mac only) docling scan.pdf --force-ocr --output /tmp/ # re-OCR even extractable text +docling report.pdf --replace-broken-text-layer --output /tmp/ # OCR only pages whose text layer is broken docling report.pdf --no-ocr --output /tmp/ # skip OCR (faster) docling scan.pdf --ocr-lang eng,deu --output /tmp/ # the engine's own codes docling scan.pdf --ocr-lang iso:en,iso:de --output /tmp/ # BCP-47 tags behind `iso:` diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index be17238dad..2c53e34132 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -104,6 +104,20 @@ opts = PdfPipelineOptions(do_ocr=True, ocr_options=OcrMacOptions()) # macOS Each engine is an optional dependency — see [slim-packaging.md](slim-packaging.md). +When a PDF looks right but its text copies out as garbage (a font whose glyphs +were not mapped to real characters), let OCR replace the text of just those pages. +Off by default; such pages then report `parse_score` 0 in the confidence report: + +```python +opts = PdfPipelineOptions( + do_ocr=True, ocr_options=RapidOcrOptions(replace_broken_text_layer=True) +) +``` + +It does not detect text that uses the right alphabet but the wrong letters, or +the right letters in the wrong order (e.g. `Cartifecita` for `Certificate`). +Such pages keep their PDF text; `mode=OcrMode.FULL_PAGE` OCRs every page. + OCR can also run on a remote KServe v2 / Triton server. Page crops are sent to that server, so `enable_remote_services=True` is required: diff --git a/docling/cli/main.py b/docling/cli/main.py index 3cf58f8d4b..ab0f97d9c9 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -965,6 +965,16 @@ def convert( # noqa: C901 help="Which document regions are fed to the OCR engine.", ), ] = OcrMode.DEFAULT, + replace_broken_text_layer: Annotated[ + bool, + typer.Option( + ..., + help=( + "Replace a page's PDF text with OCR when its text layer is broken, " + "e.g. from a font whose glyphs were not mapped to real characters." + ), + ), + ] = False, tables: Annotated[ bool, typer.Option( @@ -1577,12 +1587,18 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions] "mode": "--ocr-mode", "lang": "--ocr-lang", "psm": "--psm", + "replace_broken_text_layer": "--replace-broken-text-layer", } # `--ocr-mode` always has a value; count it as set only when it differs # from the default, so `--ocr-option mode=...` works on its own. explicit_ocr_fields = ( {"mode"} if force_ocr or ocr_mode != OcrMode.DEFAULT else set() ) + # Like `--ocr-mode`, count the flag as set only when it is on, so + # `--ocr-option replace_broken_text_layer=true` works on its own. + if replace_broken_text_layer: + ocr_kwargs["replace_broken_text_layer"] = True + explicit_ocr_fields.add("replace_broken_text_layer") ocr_lang_list = _split_list(ocr_lang) # `_split_list` returns None only when the option was not given, so an # explicitly empty value reaches the engine as `lang=[]`: "your default". diff --git a/docling/datamodel/base_models.py b/docling/datamodel/base_models.py index c33a4bc024..00a89f2e18 100644 --- a/docling/datamodel/base_models.py +++ b/docling/datamodel/base_models.py @@ -581,6 +581,9 @@ class Page(BaseModel): # conversion results. _shape_lines: list[BoundingBox] | None = PrivateAttr(default=None) _shape_bounding_boxes: list[BoundingBox] | None = PrivateAttr(default=None) + # Whether the page's PDF text layer is broken; read by the OCR stage to replace + # it (OcrOptions.replace_broken_text_layer). + _broken_text_layer: bool = PrivateAttr(default=False) _default_image_scale: float = 1.0 # Default image scale for external usage. _image_cache: dict[ float, Image diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 26ea1caee4..a2e13abf14 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -260,6 +260,23 @@ class OcrOptions(BaseOptions): ), ] = 3.0 + replace_broken_text_layer: Annotated[ + bool, + Field( + description=( + "Replace a broken PDF text layer with OCR, page by page. When enabled," + " Docling checks the visible text layer of each page. A page whose" + " text layer is broken, for example from a font whose glyphs were not" + " mapped to real characters, is read with full-page OCR, and its OCR" + " text is used instead of its PDF text for that page only. Its" + " `parse_score` drops to 0 and keeps describing the PDF text layer. A" + " low `parse_score` alone does not replace a page. Other pages, and" + " pages without a text layer, keep the usual behavior." + ), + examples=[False, True], + ), + ] = False + model_config = ConfigDict( validate_assignment=True, validate_default=True, diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index 640be66351..00307699b3 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -118,7 +118,7 @@ class _MergeCellsPriority(str, Enum): _PDF_CELLS_POST_PROCESSING_SEGREGATION = True -def _segregate_by_visibility( +def segregate_by_visibility( cells: Iterable[TextCell], ) -> tuple[list[TextCell], list[TextCell]]: """Split cells into (visible, invisible), preserving the order within each group. @@ -129,16 +129,32 @@ def _segregate_by_visibility( visible: list[TextCell] = [] invisible: list[TextCell] = [] for cell in cells: - if ( - isinstance(cell, PdfTextCell) - and cell.rendering_mode in _INVISIBLE_RENDERING_MODES - ): + if is_invisible(cell): invisible.append(cell) else: visible.append(cell) return visible, invisible +def is_invisible(cell: TextCell) -> bool: + """Whether a PDF cell is drawn with a text rendering mode that paints no ink.""" + return ( + isinstance(cell, PdfTextCell) + and cell.rendering_mode in _INVISIBLE_RENDERING_MODES + ) + + +def _ocr_full_page(options: OcrOptions, page: Page) -> bool: + """Whether a page is OCR'd in full and its PDF cells dropped. + + True in `OcrMode.FULL_PAGE`, and, with `replace_broken_text_layer`, for a page + whose text layer page preprocessing found broken. + """ + if options.mode == OcrMode.FULL_PAGE: + return True + return options.replace_broken_text_layer and page._broken_text_layer + + class BaseOcrModel(BasePageModel, BaseModelWithOptions): r""" Base class for all OCR models. @@ -243,15 +259,13 @@ def get_ocr_rects(self, page: Page) -> list[BoundingBox]: # Compute the OCR rects according to the mode ocr_rects: list[BoundingBox] - # Both DEFAULT and PDF_AWARE_LAYOUT_REGIONS make OCR input as layout detections eliminated by PDF cells - if ( - self.options.mode == OcrMode.DEFAULT - or self.options.mode == OcrMode.PDF_AWARE_LAYOUT_REGIONS - ): - ocr_rects = self._find_pdf_aware_layout_ocr_rects(page) - elif self.options.mode == OcrMode.LAYOUT_REGIONS: - ocr_rects = self._find_layout_ocr_rects(page) - elif self.options.mode == OcrMode.FULL_PAGE: + if _ocr_full_page(self.options, page): + if self.options.mode != OcrMode.FULL_PAGE: + _log.info( + "Page %s: the text layer is broken;" + " replacing it with full-page OCR", + page.page_no, + ) # A big bbox covering the entire page ocr_rects = [ BoundingBox( @@ -262,6 +276,14 @@ def get_ocr_rects(self, page: Page) -> list[BoundingBox]: coord_origin=CoordOrigin.TOPLEFT, ) ] + # Both DEFAULT and PDF_AWARE_LAYOUT_REGIONS make OCR input as layout detections eliminated by PDF cells + elif ( + self.options.mode == OcrMode.DEFAULT + or self.options.mode == OcrMode.PDF_AWARE_LAYOUT_REGIONS + ): + ocr_rects = self._find_pdf_aware_layout_ocr_rects(page) + elif self.options.mode == OcrMode.LAYOUT_REGIONS: + ocr_rects = self._find_layout_ocr_rects(page) return ocr_rects def _find_layout_ocr_rects(self, page: Page) -> list[BoundingBox]: @@ -604,7 +626,10 @@ def post_process_cells( r""" Post-process the OCR cells and update the page object according to the algorithm: - - If FULL_PAGE: Any existing PDF cells are ignored and only the OCR cells are used. + - If FULL_PAGE, or with `replace_broken_text_layer` the page's text layer is + broken (`is_broken_text_layer`): + Any existing PDF cells are ignored and only the OCR cells are used. + A page replaced for `replace_broken_text_layer` keeps its invisible cells. - If LAYOUT_REGIONS or PDF_AWARE_LAYOUT_REGIONS and the priority parameter is None, the priority is auto-selected based on the OcrMode: - OCR_FIRST when LAYOUT_REGIONS @@ -615,8 +640,16 @@ def post_process_cells( existing_cells = page.cells # Combine existing and OCR cells with overlap filtering - if self.options.mode == OcrMode.FULL_PAGE: + full_page = _ocr_full_page(self.options, page) + # A replaced broken text layer keeps its invisible cells, as in the merge + keep_invisible = full_page and self.options.mode != OcrMode.FULL_PAGE + if full_page: final_cells = ocr_cells + if keep_invisible: + _, invisible_cells = segregate_by_visibility(existing_cells) + final_cells = self._merge_cells_by_priority( + final_cells, invisible_cells + ) else: if priority is None: priority = ( @@ -645,12 +678,16 @@ def post_process_cells( page.parsed_page.has_lines = len(final_cells) > 0 # In OcrMode.FULL_PAGE, PDF-extracted word/char cells are unreliable. Keep only OCR cells - if self.options.mode == OcrMode.FULL_PAGE: + if full_page: page.parsed_page.word_cells = [ - c for c in page.parsed_page.word_cells if c.from_ocr + c + for c in page.parsed_page.word_cells + if c.from_ocr or (keep_invisible and is_invisible(c)) ] page.parsed_page.char_cells = [ - c for c in page.parsed_page.char_cells if c.from_ocr + c + for c in page.parsed_page.char_cells + if c.from_ocr or (keep_invisible and is_invisible(c)) ] page.parsed_page.has_words = len(page.parsed_page.word_cells) > 0 page.parsed_page.has_chars = len(page.parsed_page.char_cells) > 0 @@ -678,7 +715,7 @@ def _merge_ocr_and_pdf_cells( visible_cells: list[TextCell] invisible_cells: list[TextCell] if segregate_pdf_cells: - visible_cells, invisible_cells = _segregate_by_visibility(pdf_cells) + visible_cells, invisible_cells = segregate_by_visibility(pdf_cells) else: visible_cells, invisible_cells = list(pdf_cells), [] diff --git a/docling/models/stages/ocr/auto_ocr_model.py b/docling/models/stages/ocr/auto_ocr_model.py index 3ff1151f5c..000a6ce2cd 100644 --- a/docling/models/stages/ocr/auto_ocr_model.py +++ b/docling/models/stages/ocr/auto_ocr_model.py @@ -74,6 +74,9 @@ def __init__( options=OcrMacOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -99,6 +102,9 @@ def __init__( options=NemotronOcrOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -130,6 +136,9 @@ def __init__( backend="onnxruntime", mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -159,6 +168,9 @@ def __init__( options=EasyOcrOptions( mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) @@ -184,6 +196,9 @@ def __init__( backend="torch", mode=self.options.mode, lang=self.options.lang, + replace_broken_text_layer=( + self.options.replace_broken_text_layer + ), ), accelerator_options=accelerator_options, ) diff --git a/docling/models/stages/page_preprocessing/page_preprocessing_model.py b/docling/models/stages/page_preprocessing/page_preprocessing_model.py index 26a188fb7e..23cb7154e1 100644 --- a/docling/models/stages/page_preprocessing/page_preprocessing_model.py +++ b/docling/models/stages/page_preprocessing/page_preprocessing_model.py @@ -19,7 +19,9 @@ from docling.datamodel.pipeline_options import OcrMode, PdfPipelineOptions from docling.datamodel.settings import settings from docling.models.base_model import BasePageModel +from docling.models.base_ocr_model import segregate_by_visibility from docling.utils.profiling import TimeRecorder +from docling.utils.text_quality import is_broken_text_layer def resolve_skip_cell_extraction(pipeline_options: PdfPipelineOptions) -> bool: @@ -42,6 +44,8 @@ class PagePreprocessingOptions(BaseModel): False # Skip text cell extraction for VLM-only processing ) capture_reading_order_separators: bool = False + # Score a broken text layer as poor, for OcrOptions.replace_broken_text_layer. + detect_broken_text_layer: bool = False class PagePreprocessingModel(BasePageModel): @@ -114,12 +118,23 @@ def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page: warnings.filterwarnings( "ignore", "Mean of empty slice", RuntimeWarning, "numpy" ) - conv_res.confidence.pages[page.page_no].parse_score = float( + parse_score = float( np.nanquantile( text_scores, q=0.10 ) # To emphasise problems in the parse_score, we take the 10% percentile score of all text cells. ) + # Some broken text layers only show across the whole page, such as an + # unmapped font encoding. Judged on visible text, only when the OCR may + # replace the layer. Only this check replaces a page: a low per-cell score + # also comes from a few unmapped glyphs, such as bullets, on a good page. + if self.options.detect_broken_text_layer and not np.isnan(parse_score): + visible_cells, _ = segregate_by_visibility(page.cells) + if is_broken_text_layer("\n".join(c.text for c in visible_cells)): + page._broken_text_layer = True + parse_score = 0.0 + conv_res.confidence.pages[page.page_no].parse_score = parse_score + def draw_cell_boxes( image: Image, text_cells: Iterable[TextCell], diff --git a/docling/pipeline/standard_pdf_pipeline.py b/docling/pipeline/standard_pdf_pipeline.py index f5480a3b2a..c84bb0eb48 100644 --- a/docling/pipeline/standard_pdf_pipeline.py +++ b/docling/pipeline/standard_pdf_pipeline.py @@ -646,6 +646,10 @@ def _init_models(self) -> None: capture_reading_order_separators=( self.pipeline_options.use_reading_order_separators ), + detect_broken_text_layer=( + self.pipeline_options.do_ocr + and self.pipeline_options.ocr_options.replace_broken_text_layer + ), ) ) self.ocr_model = self._make_ocr_model(art_path) diff --git a/docling/utils/text_quality.py b/docling/utils/text_quality.py new file mode 100644 index 0000000000..3009c992ea --- /dev/null +++ b/docling/utils/text_quality.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Detect a PDF text layer that does not match the text drawn on the page. + +The checks look only at patterns that broken font mappings produce. Two of them +apply to every script: characters that do not belong in text, on the page or +in many of its lines. Two apply to Latin letters only, and use no language +profile: accented letters that stand for another script, and too few vowels. +""" + +import re +import unicodedata + +_MIN_CHARS = 20 # Less text than this is left alone. +_JUNK_SHARE = 0.03 # Control, U+FFFD or private-use characters on the page. +_JUNK_LINE_SHARE = 0.10 # Lines with U+FFFD or a control character... +_MIN_JUNK_LINES = 3 # ...and at least this many. + +# Latin letters: only judged when there are enough of them. +_MIN_LATIN_LETTERS = 50 +_ACCENTED_LATIN_SHARE = 0.5 # Real text in Latin script stays far below. +_MIN_WORD_LETTERS = 200 +_MIN_VOWEL_SHARE = 0.25 # Real text in Latin script stays above about 0.3. +# Shorter tokens are math variables; longer ones are base64 or hashes. +_WORD_LENGTH = range(3, 21) +# Base vowels, with æ, œ, ø, dotless i and schwa, which have no base letter. +_VOWELS = frozenset("aeiouy\u00e6\u0153\u00f8\u0131\u0259") +_GLYPH_PLACEHOLDER = re.compile(r"GLYPH<[^>]*>") # A parser's unmapped glyph. + + +def _is_junk(ch: str) -> bool: + """Whether a character is a control, surrogate, private-use or U+FFFD.""" + category = unicodedata.category(ch) + return ( + ch == "�" or category in ("Cs", "Co") or (category == "Cc" and not ch.isspace()) + ) + + +def _has_replacement(line: str) -> bool: + """Whether a line has U+FFFD or a control character after its first character. + + Private-use characters are not counted: icon fonts put them in many lines. + """ + return any( + ch == "�" or (unicodedata.category(ch) == "Cc" and not ch.isspace()) + for ch in line.lstrip()[1:] + ) + + +def _without_bullet(line: str) -> str: + """Return the line without a lone junk character at its start. + + An unmapped bullet glyph is a lone junk character before the item text; a + broken font puts junk inside words and numbers. + """ + stripped = line.lstrip() + if stripped[:1] and _is_junk(stripped[0]) and stripped[1:2].isspace(): + return stripped[1:] + if len(stripped) == 1 and _is_junk(stripped): + return "" + return line + + +def _is_latin(ch: str) -> bool: + return ch.isalpha() and "LATIN" in unicodedata.name(ch, "") + + +def _is_wrong_latin(text: str) -> bool: + """Whether the Latin letters stand for another script or were moved. + + A non-Latin 8-bit font read as Latin-1 gives mostly accented letters + (`Îò÷ò`). A broken ToUnicode map that moves letters, or a legacy font that + puts another script on ASCII letters, leaves few vowels. Accented vowels + count by their base letter, so no Latin-script language is penalized. + """ + text = _GLYPH_PLACEHOLDER.sub(" ", text) + latin = [ch for ch in text if _is_latin(ch)] + if len(latin) < _MIN_LATIN_LETTERS: + return False + # 8-bit code pages decode to U+00C0-U+024F; IPA and Vietnamese tones lie above. + accented = sum("À" <= ch <= "ɏ" for ch in latin) + if accented >= _ACCENTED_LATIN_SHARE * len(latin): + return True + words = [] + for token in text.split(): + letters = [ch for ch in token if _is_latin(ch)] + if len(letters) in _WORD_LENGTH and any(ch.islower() for ch in letters): + words.extend(letters) + if len(words) < _MIN_WORD_LETTERS: + return False + vowels = sum(unicodedata.normalize("NFD", ch)[0].lower() in _VOWELS for ch in words) + return vowels < _MIN_VOWEL_SHARE * len(words) + + +def is_broken_text_layer(text: str) -> bool: + """Whether a page's PDF text does not match the text drawn on the page. + + Args: + text: All text of one page's PDF text layer, cells joined by newlines. + + Returns: + ``True`` only for text a broken layer produces. When unsure it returns + ``False``, so pages keep their PDF text. + """ + stripped = text.strip() + if len(stripped) < _MIN_CHARS: + return False + lines = [_without_bullet(line) for line in stripped.splitlines()] + chars = [ch for line in lines for ch in line if not ch.isspace()] + if not chars: + return False + + # A font whose glyphs were mapped to control, private-use or no code points. + if sum(map(_is_junk, chars)) >= _JUNK_SHARE * len(chars): + return True + # The same in many lines, such as unmapped ligatures (`publica�on`). + junk_lines = sum(map(_has_replacement, lines)) + if junk_lines >= max(_MIN_JUNK_LINES, _JUNK_LINE_SHARE * len(lines)): + return True + + return _is_wrong_latin(stripped) diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 8ddcdfad60..32e2774cad 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -151,6 +151,49 @@ Engines that run one language at a time (RapidOCR, Nemotron-OCR) take the **firs about the rest. The KServe client also sends only the first entry. +## Replace a broken PDF text layer + +Some PDFs look correct on screen, but their text copies out as garbage, for example `8VceZWZTReV` +instead of `Certificate`. This happens when a font does not map its glyphs to real characters. With +`replace_broken_text_layer`, Docling detects these pages and reads them with OCR instead. Other +pages keep their PDF text. + +```python +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import PdfPipelineOptions, RapidOcrOptions +from docling.document_converter import DocumentConverter, PdfFormatOption + +pipeline_options = PdfPipelineOptions( + do_ocr=True, + ocr_options=RapidOcrOptions(replace_broken_text_layer=True), +) +doc_converter = DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options) + } +) +``` + +The option is off by default and works with every OCR engine. Pick an engine and `lang` that can +read the language of the document. A replaced page keeps its low `parse_score` in the +[confidence scores](confidence_scores.md), because the score describes the PDF text layer. + +```bash +docling report.pdf --replace-broken-text-layer +``` + +Docling detects a broken page from characters that do not belong in text, such as control +characters, `�`, or private-use characters, on the page or in many of its lines. These checks work +for every script. For Latin letters, it also detects accented letters that stand for another +script, for example `Îò÷¸ò ïîêàçûâàåò` for Russian text, and letters with too few vowels, as in +the example above. The vowel check uses no language profile, so it works the same for every +Latin-script language. One broken cell on an otherwise good page does not replace the page. + +Docling cannot detect a page whose text uses the right alphabet but the wrong letters, or the right +letters in the wrong order, for example `Cartifecita` instead of `Certificate`. Such pages keep +their PDF text. To read every page with OCR, set `mode=OcrMode.FULL_PAGE` (`--ocr-mode full_page`). + + ## RapidOCR The engine's own vocabulary, in its own codes, is listed in diff --git a/tests/test_cli.py b/tests/test_cli.py index ba04140ef5..783ea2ef15 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1731,7 +1731,9 @@ def test_cli_deprecated_psm_still_sets_tesseract_psm(tmp_path, monkeypatch): assert ocr_options.psm == 4 -def test_cli_options_lists_engine_fields(): +def test_cli_options_lists_engine_fields(monkeypatch): + # Rich reads the table width from `COLUMNS`, not from `terminal_width`. + monkeypatch.setenv("COLUMNS", "200") result = runner.invoke(app, ["options", "ocr", "rapidocr"], terminal_width=200) assert result.exit_code == 0, result.output assert "model_size" in result.output diff --git a/tests/test_replace_broken_text_layer.py b/tests/test_replace_broken_text_layer.py new file mode 100644 index 0000000000..62042604e2 --- /dev/null +++ b/tests/test_replace_broken_text_layer.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Replacing a broken PDF text layer with OCR (OcrOptions.replace_broken_text_layer). + +The test page shows "Proxy Statement" as an image, over a text layer that either +matches it or is the output of a font decoded as accented Latin letters. +""" + +import ctypes +from pathlib import Path + +import pypdfium2 as pdfium +import pypdfium2.raw as pdfium_c +import pytest +from PIL import Image, ImageDraw, ImageFont + +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import ConversionResult +from docling.datamodel.pipeline_options import ( + OcrAutoOptions, + OcrOptions, + PdfPipelineOptions, + TesseractCliOcrOptions, +) +from docling.document_converter import DocumentConverter, PdfFormatOption + +pytestmark = pytest.mark.ml_ocr + +_SHOWN = "Proxy Statement" +_BROKEN_LAYER = " ".join( + [ + "\N{LATIN SMALL LETTER A WITH ACUTE}\N{LATIN SMALL LETTER A WITH DIAERESIS}" + "\N{LATIN SMALL LETTER C WITH CEDILLA}\N{LATIN SMALL LETTER E WITH GRAVE}" + "\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}" + ] + * 12 +) + + +def _page_pdf( + path: Path, + text_layer: str, + *, + invisible: bool = False, + hidden_text: str | None = None, +) -> Path: + """One page: `_SHOWN` drawn as an image, under a text layer of `text_layer`. + + `invisible` draws the layer with no ink; `hidden_text` adds an invisible line. + """ + width, height = 400, 200 + image = Image.new("RGB", (width * 3, height * 3), "white") + font = ImageFont.load_default(size=110) + ImageDraw.Draw(image).text((60, 230), _SHOWN, fill="black", font=font) + + pdf = pdfium.PdfDocument.new() + page = pdf.new_page(width, height) + picture = pdfium.PdfImage.new(pdf) + picture.set_bitmap(pdfium.PdfBitmap.from_pil(image)) + picture.set_matrix(pdfium.PdfMatrix().scale(width, height)) + page.insert_obj(picture) + + _add_text(pdf, page, text_layer, y=112, invisible=invisible) + if hidden_text is not None: + _add_text(pdf, page, hidden_text, y=20, invisible=True) + page.gen_content() + pdf.save(path) + return path + + +def _add_text( + pdf: pdfium.PdfDocument, + page: pdfium.PdfPage, + text_layer: str, + *, + y: float, + invisible: bool, +) -> None: + helvetica = pdfium_c.FPDFText_LoadStandardFont(pdf.raw, b"Helvetica") + text = pdfium_c.FPDFPageObj_CreateTextObj(pdf.raw, helvetica, 9.0) + utf16 = (text_layer + "\0").encode("utf-16-le") + pdfium_c.FPDFText_SetText( + text, ctypes.cast(ctypes.c_char_p(utf16), ctypes.POINTER(pdfium_c.FPDF_WCHAR)) + ) + if invisible: + pdfium_c.FPDFTextObj_SetTextRenderMode( + text, pdfium_c.FPDF_TEXTRENDERMODE_INVISIBLE + ) + else: + # A broken layer still draws the right glyphs; white ink keeps this one + # from adding marks of its own to the image the OCR reads. + pdfium_c.FPDFPageObj_SetFillColor(text, 255, 255, 255, 255) + pdfium_c.FPDFPageObj_Transform(text, 1, 0, 0, 1, 20, y) + pdfium_c.FPDFPage_InsertObject(page.raw, text) + + +def _convert( + path: Path, *, replace: bool, ocr: OcrOptions | None = None +) -> ConversionResult: + if ocr is None: + ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=replace) + options = PdfPipelineOptions(do_ocr=True, do_table_structure=False, ocr_options=ocr) + converter = DocumentConverter( + format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} + ) + return converter.convert(path) + + +def test_broken_text_layer_is_replaced_by_ocr(tmp_path: Path) -> None: + result = _convert(_page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=True) + + assert result.confidence.pages[1].parse_score == 0.0 + markdown = result.document.export_to_markdown() + assert _SHOWN in markdown + assert _BROKEN_LAYER[:5] not in markdown + + +def test_option_off_keeps_the_text_layer_and_its_score(tmp_path: Path) -> None: + result = _convert(_page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=False) + + assert result.confidence.pages[1].parse_score == 1.0 + assert _BROKEN_LAYER[:5] in result.document.export_to_markdown() + + +def test_good_text_layer_is_kept(tmp_path: Path) -> None: + layer = "Annual Proxy Statement of the Company" + result = _convert(_page_pdf(tmp_path / "good.pdf", layer), replace=True) + + assert result.confidence.pages[1].parse_score == 1.0 + assert layer in result.document.export_to_markdown() + + +def test_poor_cell_score_alone_keeps_the_text_layer(tmp_path: Path) -> None: + """An unmapped bullet glyph grades the page poor, but its text is good.""" + layer = "GLYPH<81> Annual Report of the Company" + result = _convert(_page_pdf(tmp_path / "bullet.pdf", layer), replace=True) + + assert result.confidence.pages[1].parse_score == 0.0 + markdown = result.document.export_to_markdown() + assert "Annual Report of the Company" in markdown + assert _SHOWN not in markdown + + +def test_auto_ocr_engine_replaces_the_broken_layer(tmp_path: Path) -> None: + """The engine OcrAutoOptions selects gets the option too.""" + ocr = OcrAutoOptions(replace_broken_text_layer=True) + result = _convert( + _page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER), replace=True, ocr=ocr + ) + + assert result.confidence.pages[1].parse_score == 0.0 + assert _BROKEN_LAYER[:5] not in result.document.export_to_markdown() + + +def test_invisible_text_does_not_grade_the_layer(tmp_path: Path) -> None: + """Only visible text is judged.""" + pdf = _page_pdf(tmp_path / "hidden.pdf", _BROKEN_LAYER, invisible=True) + result = _convert(pdf, replace=True) + + assert result.confidence.pages[1].parse_score == 1.0 + + +def test_replaced_page_keeps_its_invisible_text(tmp_path: Path) -> None: + hidden = "Accessible summary of the statement" + pdf = _page_pdf(tmp_path / "broken.pdf", _BROKEN_LAYER, hidden_text=hidden) + ocr = TesseractCliOcrOptions(lang=["eng"], replace_broken_text_layer=True) + options = PdfPipelineOptions( + do_ocr=True, + do_table_structure=False, + generate_parsed_pages=True, + ocr_options=ocr, + ) + converter = DocumentConverter( + format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} + ) + result = converter.convert(pdf) + + assert result.confidence.pages[1].parse_score == 0.0 + parsed = result.pages[0].parsed_page + assert parsed is not None + texts = [c.text for c in parsed.textline_cells] + assert hidden in texts + assert not any(_BROKEN_LAYER[:5] in t for t in texts) diff --git a/tests/test_text_quality.py b/tests/test_text_quality.py new file mode 100644 index 0000000000..55720d1038 --- /dev/null +++ b/tests/test_text_quality.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Detection of PDF text layers that do not match the text drawn on the page.""" + +import pytest + +from docling.utils.text_quality import is_broken_text_layer + + +def _words(first: int, last: int, count: int = 40) -> str: + """Words made of the letters in a Unicode range, as a stand-in for a script.""" + letters = [chr(cp) for cp in range(first, last + 1) if chr(cp).isalpha()] + return " ".join( + "".join(letters[(i + j) % len(letters)] for j in range(5)) for i in range(count) + ) + + +_PROSE = ( + "The quarterly report shows that revenue increased across all regions, " + "driven by strong demand for cloud services and improved margins. " +) * 3 + + +def _shift(text: str, offset: int) -> str: + """Letters moved by a constant, as a broken ToUnicode map does.""" + return "".join(chr(ord(ch) + offset) if ch.isalpha() else ch for ch in text) + + +# Ordinary text layers: never broken. +ORDINARY = { + "latex": r"The loss $L = \frac{1}{n}\sum_{i=1}^{n} (y_i - \hat{y}_i)^2$ with " + r"\alpha \in [0,1] and \beta^{2} \leq \gamma_{k}. " * 3, + "shell": "export PATH=$HOME/bin:$PATH; cd ${USER}/work && ls ~/docs/*.md " + "| grep -v '#draft' > ~/out/${DATE}.log; echo $? " * 3, + "python": "def f(x):\n return {k: v**2 for k, v in x.items() if v > 0}\n" + "print(f({'a': 1, 'b': -2}))\n" * 3, + "json": '{"id": 42, "tags": ["a", "b"], "meta": {"ok": true}, "path": "/v/a.log"}\n' + * 3, + "markdown_table": "| Metric | Value |\n|---|---|\n| F1 | 0.91 |\n**Note:** see " + "`config.yaml` and [docs](https://x.y/z).\n" * 3, + "dot_leaders": "Net revenues . . . . . . $ 12,345 $ 11,234 (1)\n" + "Operating income . . . . 2,345 (1,987)\n" * 5, + "caption": "Q3 2024: $12,345 (+5.2%) vs 11,734 in the prior year", + "units": "RDS(ON) < 8.7 m\N{OHM SIGN} at VGS=10V; TJ = \N{MINUS SIGN}55 to 150 " + "\N{DEGREE SIGN}C; t \N{LESS-THAN OR EQUAL TO} 10 \N{MICRO SIGN}s. " * 3, + "bullets_and_checkboxes": "\N{BULLET} Dallas (DFW4)\n\N{BLACK SMALL SQUARE} " + "Chicago\n\N{BALLOT BOX} No \N{BALLOT BOX WITH X} Yes\n______________\n" * 4, + "icon_font_glyphs": "".join(chr(0xF002) + " Brabant Konzession, Teil von Arriva. ") + * 4, + "accented_european": "L\N{RIGHT SINGLE QUOTATION MARK}\N{LATIN CAPITAL LETTER E WITH ACUTE}cole " + "doctorale accueille aujourd\N{RIGHT SINGLE QUOTATION MARK}hui des \N{LATIN SMALL LETTER E WITH ACUTE}tudiants " + "de la r\N{LATIN SMALL LETTER E WITH ACUTE}gion. " * 4, + "greek": _words(0x03B1, 0x03C9), + "cyrillic": _words(0x0430, 0x044F), + "hebrew": _words(0x05D0, 0x05EA), + "arabic": _words(0x0627, 0x064A), + "thai": _words(0x0E01, 0x0E2E), + "devanagari": _words(0x0915, 0x0939), + "cjk": _words(0x4E00, 0x4E40), + # Long enough for the letter-frequency check. + "english_prose": _PROSE, + "czech": "Vl\N{LATIN SMALL LETTER A WITH ACUTE}da schv\N{LATIN SMALL LETTER A WITH ACUTE}lila " + "n\N{LATIN SMALL LETTER A WITH ACUTE}vrh z\N{LATIN SMALL LETTER A WITH ACUTE}kona, kter\N{LATIN SMALL LETTER Y WITH ACUTE} " + "zjednodu\N{LATIN SMALL LETTER S WITH CARON}uje stavebn\N{LATIN SMALL LETTER I WITH ACUTE} " + "\N{LATIN SMALL LETTER R WITH CARON}\N{LATIN SMALL LETTER I WITH ACUTE}zen\N{LATIN SMALL LETTER I WITH ACUTE} " + "a podporuje v\N{LATIN SMALL LETTER Y WITH ACUTE}stavbu nov\N{LATIN SMALL LETTER Y WITH ACUTE}ch byt\N{LATIN SMALL LETTER U WITH RING ABOVE} " + "ve velk\N{LATIN SMALL LETTER Y WITH ACUTE}ch m\N{LATIN SMALL LETTER E WITH CARON}stech. " + * 3, + "welsh": "Mae'r llywodraeth wedi cyhoeddi cynllun newydd i gefnogi busnesau bach " + "yng nghefn gwlad Cymru dros y blynyddoedd nesaf. " * 3, + "vietnamese": "\N{LATIN CAPITAL LETTER D WITH STROKE}\N{LATIN SMALL LETTER U WITH HORN AND GRAVE}\N{LATIN SMALL LETTER O WITH HORN}ng " + "\N{LATIN SMALL LETTER D WITH STROKE}\N{LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE}n t\N{LATIN SMALL LETTER U WITH HORN}\N{LATIN SMALL LETTER O WITH HORN}ng " + "lai c\N{LATIN SMALL LETTER U WITH HOOK ABOVE}a nh\N{LATIN SMALL LETTER U WITH HORN AND TILDE}ng ng\N{LATIN SMALL LETTER U WITH HORN}\N{LATIN SMALL LETTER O WITH HORN AND GRAVE}i " + "d\N{LATIN SMALL LETTER A WITH CIRCUMFLEX}n Vi\N{LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW}t Nam " + "r\N{LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE}t d\N{LATIN SMALL LETTER A WITH GRAVE}i. " + * 4, + "c_code": "for (int i = 0; i < n; i++) { ptr->buf[i] = xmm_mul(cfg.k, src[i]); " + "if (!chk(ptr)) return -EINVAL; } " * 4, + "part_numbers": "STM32F407VGT6 LQFP100 GPIO PB12 SPI2_NSS PB13 SPI2_SCK TIM1_CH1N " + "HCLK PCLK1 PCLK2 RCC_CFGR SYSCLK PLLM PLLN\n" * 4, + "base64": "TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcmsuIFRoZSBxdWljayBicm93biBmb3gganVtcHM=\n" + * 5, + "dna": "ACGTTGCAAGCTTGCATGCCTGCAGGTCGACTCTAGAGGATCCCCGGGTACCGAGCTCGAATTC\n" * 5, + # Icon fonts repeat a few private-use code points, also in a row. + "icon_rating_and_footer": _PROSE + + "\n" + + chr(0xF0AB) * 4 + + chr(0xF0AA) + + "\n" + + " ".join(chr(0xF09A + i) for i in range(5)), + # One U+FFFD from a math font, on a full page. + "single_replacement_character": _PROSE + "\nx \N{REPLACEMENT CHARACTER} y\n", + # pdfium can mark one end-of-line hyphen with a control character. + "hyphen_control": _PROSE.replace("regions", "re\x02\ngions", 1), + # A figure whose axis labels a parser could not map, on a good page. + "glyph_placeholders": _PROSE + "\nGLYPHGLYPHGLYPH" * 60, + # Formulas: many one-letter variables and few vowels. + "math_variables": ( + "(p, q, k) and (p, k, q)\n" + "PP[(q)] = 5 e2 [(k2 g - k k) - (q2 g - q q)]\n" + "= 2 e sin(p ^ q) B (p, q, k) = i2 g sin(k ^ q) (p, q, k) ln(s) ln(p) mn pq\n" + ) + * 6, + "dotted_capital_i": _PROSE + + "\nThe office in \N{LATIN CAPITAL LETTER I WITH DOT ABOVE}stanbul opened.\n", + # The 25 vowels of an IPA vowel chart, in three rows: phonetic symbols, not mojibake. + "ipa_vowel_chart": "\n".join( + [ + "\u026a \u028f \u028a e \u00f8 \u0259 \u0250 \u00e6 a \u0251 \u0252 \u028c " + "\u0254 \u0153 \u025c \u025e \u0264 o \u0258 \u0275 \u026f \u0268 \u0289 y u" + ] + * 3 + ), + # Unmapped list bullets: a lone U+FFFD before good item text, in 3 of 25 lines. + "bullet_replacement_markers": "\n".join( + [f"\N{REPLACEMENT CHARACTER} Revenue grew in region {n}" for n in range(3)] + + [f"Line {n}: the regional teams reported stable demand." for n in range(22)] + ), + # A short slide: a title and 3 items, each bullet in its own cell. + "bullet_slide": "Quarterly results\n" + + "\n".join( + f"\N{REPLACEMENT CHARACTER}\n{item}" + for item in ("Revenue grew by 5%", "Costs fell", "Margins improved") + ), + # An agenda slide: 5 short items, each a bullet, a space and the item text. + "bullet_agenda": "Agenda\n" + + "\n".join( + f"\N{REPLACEMENT CHARACTER} {item}" + for item in ("Welcome", "Results", "Outlook", "Questions", "Close") + ), + # One cell in a broken font: OCR would replace the good text of the whole page. + "one_private_use_cell": _PROSE * 2 + + "\n" + + "".join(chr(0xF041 + i) for i in range(6)) + + "\n", + "one_control_code_cell": _PROSE * 2 + + "\nTotal " + + "".join(chr(0x10 + i) for i in range(6)) + + "\n", +} + +# Text layers from fonts whose glyphs were not mapped to real characters. +BROKEN = { + "glyphs_as_control_codes": "Net assets 2,635 2,849\n" + + "".join(chr(0x10 + i % 15) for i in range(40)) + + "\nTotal 3,004\n", + "replacement_characters": "Annual report \N{REPLACEMENT CHARACTER}" + "\N{REPLACEMENT CHARACTER} 2024 " * 6, + "glyphs_as_private_use": " ".join( + "".join(chr(0xF021 + (i + j) % 60) for j in range(4)) for i in range(30) + ), + "glyphs_as_accented_latin": _words(0x0100, 0x017F), + "glyphs_as_shifted_letters": _shift(_PROSE, 3), + "glyphs_as_letters_and_digits": _shift(_PROSE, -11), + # Not only English: the vowel check uses no language profile. + "welsh_as_shifted_letters": _shift(ORDINARY["welsh"], 3), + # Below the share the control check needs, but in several cells. + "replacement_characters_in_many_cells": _PROSE + + "\n" + + "\n".join("Item " + "\N{REPLACEMENT CHARACTER}" * n for n in (4, 3, 3)), + # One U+FFFD in each of many lines: a small share of the characters. + "one_replacement_in_many_lines": "\n".join( + "BIB@QOLKF@ PB@QOFQFBP QO>AFKD\N{REPLACEMENT CHARACTER} IBKAFKD >KA M>VJBKQ" + " PBOSF@BP CLO @LJM>KFBP >KA FKAFSFA§>IP" + for _ in range(8) + ), +} + + +@pytest.mark.parametrize("name", sorted(ORDINARY)) +def test_ordinary_text_layer_is_kept(name: str) -> None: + assert not is_broken_text_layer(ORDINARY[name]) + + +@pytest.mark.parametrize("name", sorted(BROKEN)) +def test_unmapped_font_is_detected(name: str) -> None: + assert is_broken_text_layer(BROKEN[name]) + + +def test_short_or_empty_text_is_left_alone() -> None: + assert not is_broken_text_layer("") + assert not is_broken_text_layer("\x01\x02\x03") # too little text to judge