fix: allow full-page OCR after native text decode errors

docling/models/stages/page_preprocessing/page_preprocessing_model.py

@@ -1,10 +1,18 @@
+import logging
 import re
 import warnings
 from collections.abc import Iterable
 from pathlib import Path
 from typing import Literal, Optional
 
 import numpy as np
+from docling_core.types.doc import BoundingBox, CoordOrigin
+from docling_core.types.doc.page import (
+    BoundingRectangle,
+    PdfPageBoundaryType,
+    PdfPageGeometry,
+    SegmentedPdfPage,
+)
 from PIL import ImageDraw
 from pydantic import BaseModel
 
@@ -14,12 +22,15 @@
 from docling.models.base_model import BasePageModel
 from docling.utils.profiling import TimeRecorder
 
+_log = logging.getLogger(__name__)
+
 
 class PagePreprocessingOptions(BaseModel):
     images_scale: Optional[float]
     skip_cell_extraction: bool = (
         False  # Skip text cell extraction for VLM-only processing
     )
+    allow_empty_cells_on_decode_error: bool = False
 
 
 class PagePreprocessingModel(BasePageModel):
@@ -69,7 +80,40 @@ def _populate_page_images(self, page: Page) -> Page:
     def _parse_page_cells(self, conv_res: ConversionResult, page: Page) -> Page:
         assert page._backend is not None
 
-        page.parsed_page = page._backend.get_segmented_page()
+        try:
+            page.parsed_page = page._backend.get_segmented_page()
+        except UnicodeDecodeError:
+            if not self.options.allow_empty_cells_on_decode_error:
+                raise
+
+            assert page.size is not None
+            bbox = BoundingBox(
+                l=0.0,
+                t=0.0,
+                r=float(page.size.width),
+                b=float(page.size.height),
+                coord_origin=CoordOrigin.BOTTOMLEFT,
+            )
+            page.parsed_page = SegmentedPdfPage(
+                dimension=PdfPageGeometry(
+                    angle=0.0,
+                    rect=BoundingRectangle.from_bounding_box(bbox),
+                    boundary_type=PdfPageBoundaryType.CROP_BOX,
+                    art_bbox=bbox,
+                    bleed_bbox=bbox,
+                    crop_bbox=bbox,
+                    media_bbox=bbox,
+                    trim_bbox=bbox,
+                ),
+                char_cells=[],
+                word_cells=[],
+                textline_cells=[],
+            )
+            _log.warning(
+                "Native text decoding failed on page %s; continuing with "
+                "an empty text layer for full-page OCR",
+                page.page_no,
+            )
         assert page.parsed_page is not None
 
         # Rate the text quality from the PDF parser, and aggregate on page

docling/models/stages/table_structure/table_structure_model.py

@@ -225,8 +225,8 @@ def predict_tables(
                 }
 
                 for table_cluster, tbl_box in in_tables:
-                    # Check if word-level cells are available from backend:
-                    sp = page._backend.get_segmented_page()
+                    # Use the parsed page because OCR may have replaced backend cells.
+                    sp = page.parsed_page
                     if sp is not None:
                         tcells = sp.get_cells_in_bbox(
                             cell_unit=TextCellUnit.WORD,

docling/pipeline/legacy_standard_pdf_pipeline.py

@@ -15,6 +15,7 @@
     LayoutObjectDetectionOptions,
     LayoutOptions,
     LayoutPostprocessorOptions,
+    OcrMode,
     PdfPipelineOptions,
 )
 from docling.datamodel.settings import settings
@@ -114,6 +115,10 @@ def __init__(self, pipeline_options: PdfPipelineOptions):
             PagePreprocessingModel(
                 options=PagePreprocessingOptions(
                     images_scale=pipeline_options.images_scale,
+                    allow_empty_cells_on_decode_error=(
+                        pipeline_options.do_ocr
+                        and pipeline_options.ocr_options.mode == OcrMode.FULL_PAGE
+                    ),
                 )
             ),
             # OCR

docling/pipeline/standard_pdf_pipeline.py

@@ -50,6 +50,7 @@
     LayoutObjectDetectionOptions,
     LayoutOptions,
     LayoutPostprocessorOptions,
+    OcrMode,
     ThreadedPdfPipelineOptions,
 )
 from docling.datamodel.settings import settings
@@ -597,7 +598,11 @@ def _init_models(self) -> None:
         )
         self.preprocessing_model = PagePreprocessingModel(
             options=PagePreprocessingOptions(
-                images_scale=self.pipeline_options.images_scale
+                images_scale=self.pipeline_options.images_scale,
+                allow_empty_cells_on_decode_error=(
+                    self.pipeline_options.do_ocr
+                    and self.pipeline_options.ocr_options.mode == OcrMode.FULL_PAGE
+                ),
             )
         )
         self.ocr_model = self._make_ocr_model(art_path)

tests/test_page_preprocessing_model.py

@@ -0,0 +1,58 @@
+from types import SimpleNamespace
+from typing import cast
+
+import pytest
+from docling_core.types.doc import Size
+
+from docling.backend.pdf_backend import PdfPageBackend
+from docling.datamodel.base_models import Page
+from docling.datamodel.document import ConversionResult
+from docling.models.stages.page_preprocessing.page_preprocessing_model import (
+    PagePreprocessingModel,
+    PagePreprocessingOptions,
+)
+
+
+class _InvalidTextBackend:
+    def get_segmented_page(self):
+        raise UnicodeDecodeError("utf-8", b"\xdf", 0, 1, "invalid start byte")
+
+
+def _page() -> Page:
+    page = Page(page_no=1, size=Size(width=595, height=842))
+    page._backend = cast(PdfPageBackend, _InvalidTextBackend())
+    return page
+
+
+def _conversion_result() -> ConversionResult:
+    return cast(
+        ConversionResult,
+        SimpleNamespace(
+            confidence=SimpleNamespace(
+                pages={1: SimpleNamespace(parse_score=None)},
+            )
+        ),
+    )
+
+
+def test_native_text_decode_error_falls_back_to_empty_page() -> None:
+    model = PagePreprocessingModel(
+        PagePreprocessingOptions(
+            images_scale=None,
+            allow_empty_cells_on_decode_error=True,
+        )
+    )
+
+    page = model._parse_page_cells(_conversion_result(), _page())
+
+    assert page.parsed_page is not None
+    assert page.parsed_page.dimension.width == 595
+    assert page.parsed_page.dimension.height == 842
+    assert page.cells == []
+
+
+def test_native_text_decode_error_is_not_hidden_by_default() -> None:
+    model = PagePreprocessingModel(PagePreprocessingOptions(images_scale=None))
+
+    with pytest.raises(UnicodeDecodeError):
+        model._parse_page_cells(_conversion_result(), _page())