diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 910b11cca4..ac22ddfc99 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -422,6 +422,17 @@ class RapidOcrOptions(OcrOptions): description="Enable text direction classification stage. If None, uses RapidOCR default behavior." ), ] = None + detect_orientation: Annotated[ + bool, + Field( + description=( + "Read an OCR region again at the other three orientations when fewer than half of its " + "lines are read with confidence, and keep the first orientation that reads well. Unlike " + "`use_cls`, which flips single text lines by 180 degrees, this also fixes pages turned " + "by 90 or 270 degrees." + ) + ), + ] = False use_rec: Annotated[ bool | None, Field( diff --git a/docling/models/stages/ocr/rapid_ocr_model.py b/docling/models/stages/ocr/rapid_ocr_model.py index 481ed78754..7af64078b2 100644 --- a/docling/models/stages/ocr/rapid_ocr_model.py +++ b/docling/models/stages/ocr/rapid_ocr_model.py @@ -10,7 +10,8 @@ import numpy from docling_core.types.doc import BoundingBox, CoordOrigin -from docling_core.types.doc.page import BoundingRectangle, TextCell +from docling_core.types.doc.page import TextCell +from PIL import Image from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions from docling.datamodel.base_models import Page @@ -32,6 +33,7 @@ OcrLanguageResolver, OcrLanguageSupport, ) +from docling.utils.orientation import ocr_box_to_bounding_rectangle from docling.utils.profiling import TimeRecorder from docling.utils.utils import download_url_with_progress, safe_version @@ -57,6 +59,14 @@ # annotation so the two cannot drift. _RAPIDOCR_BACKENDS: frozenset[str] = frozenset(get_args(RapidOcrBackend)) +# detect_orientation: a line with at least this score is read with confidence, and a +# region with fewer lines is too small to judge its orientation. +_ORIENTATION_CONFIDENCE_SCORE = 0.8 +_ORIENTATION_MIN_LINES = 3 + +# A line read by RapidOCR: its box as four corner points, its text and its score +_OcrLine = tuple[list[list[float]], str, float] + # Canonical BCP-47 to PP-OCR recognizer codes. The PP-OCRv6 set comes from the # installed `rapidocr`; the v4/v5 sets below have no upstream equivalent to read, # so they mirror the PP-OCR release notes summarised in `docs/concepts/OCR.md`. @@ -462,6 +472,20 @@ def _validate_rapidocr_model_size( ) from err +def _reads_well(lines: list[_OcrLine]) -> bool: + """Whether enough lines were found and at least half are read with confidence.""" + confident = sum( + 1 for _, _, score in lines if score >= _ORIENTATION_CONFIDENCE_SCORE + ) + return len(lines) >= _ORIENTATION_MIN_LINES and 2 * confident >= len(lines) + + +def _confident_chars(lines: list[_OcrLine]) -> int: + return sum( + len(text) for _, text, score in lines if score >= _ORIENTATION_CONFIDENCE_SCORE + ) + + class RapidOcrModel(BaseOcrModel): _model_repo_folder = "RapidOcr" @@ -622,7 +646,11 @@ def __init__( params = { # Global settings (these are still correct) - "Global.text_score": self.options.text_score, + # detect_orientation judges a reading by all its lines, so it applies + # text_score itself, after that. + "Global.text_score": 0.0 + if self.options.detect_orientation + else self.options.text_score, "Global.font_path": font_path, "Global.log_level": "info" if self.options.print_verbose else "warning", # Engine-level ONNXRuntime settings @@ -769,22 +797,17 @@ def __call__( high_res_image = page._backend.get_page_image( scale=self.scale, cropbox=ocr_rect ) - im = numpy.asarray(high_res_image) - result = self.reader( - im, - use_det=self.options.use_det, - use_cls=self.options.use_cls, - use_rec=self.options.use_rec, - ) - if result is None or result.boxes is None: + result = self._read(high_res_image) + if result is None: _log.warning("RapidOCR returned empty result!") continue - result = list( - zip(result.boxes.tolist(), result.txts, result.scores) - ) + orientation, im_size = 0, high_res_image.size + if self.options.detect_orientation: + result, orientation, im_size = self._read_upright( + high_res_image, result + ) del high_res_image - del im if result is not None: cells = [ @@ -794,20 +817,22 @@ def __call__( orig=line[1], confidence=line[2], from_ocr=True, - rect=BoundingRectangle.from_bounding_box( + rect=ocr_box_to_bounding_rectangle( + # Corners 0 and 2 of RapidOCR's quadrilateral + # span the line box BoundingBox.from_tuple( coord=( - (line[0][0][0] / self.scale) - + ocr_rect.l, - (line[0][0][1] / self.scale) - + ocr_rect.t, - (line[0][2][0] / self.scale) - + ocr_rect.l, - (line[0][2][1] / self.scale) - + ocr_rect.t, + line[0][0][0], + line[0][0][1], + line[0][2][0], + line[0][2][1], ), origin=CoordOrigin.TOPLEFT, - ) + ), + original_offset=ocr_rect, + scale=self.scale, + orientation=orientation, + im_size=im_size, ), ) for ix, line in enumerate(result) @@ -823,6 +848,44 @@ def __call__( yield page + def _read(self, image: Image.Image) -> list[_OcrLine] | None: + result = self.reader( + numpy.asarray(image), + use_det=self.options.use_det, + use_cls=self.options.use_cls, + use_rec=self.options.use_rec, + ) + if result is None or result.boxes is None: + return None + return list(zip(result.boxes.tolist(), result.txts, result.scores)) + + def _read_upright( + self, image: Image.Image, lines: list[_OcrLine] + ) -> tuple[list[_OcrLine], int, tuple[int, int]]: + """Read a poorly read region again at the other orientations. + + Keeps the first orientation that reads well with at least twice the confidently + read characters. Returns the lines that score at least `text_score`, the + orientation they were read at (the image turned with ``rotate(-orientation)``, + as in the Tesseract engines) and the size of the image they were read on. + """ + orientation, im_size = 0, image.size + if len(lines) >= _ORIENTATION_MIN_LINES and not _reads_well(lines): + needed = 2 * _confident_chars(lines) + # RapidOCR turns tall line crops a quarter turn counter-clockwise, so it + # reads a region turned counter-clockwise upside down: try that fix first. + for turn in (90, 180, 270): + turned = image.rotate(-turn, expand=True) + turned_lines = self._read(turned) or [] + if ( + _reads_well(turned_lines) + and _confident_chars(turned_lines) >= needed + ): + lines, orientation, im_size = turned_lines, turn, turned.size + break + lines = [line for line in lines if line[2] >= self.options.text_score] + return lines, orientation, im_size + @classmethod def get_options_type(cls) -> Type[OcrOptions]: return RapidOcrOptions diff --git a/docling/models/stages/ocr/tesseract_ocr_cli_model.py b/docling/models/stages/ocr/tesseract_ocr_cli_model.py index 2ea2c78137..faa4913363 100644 --- a/docling/models/stages/ocr/tesseract_ocr_cli_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_cli_model.py @@ -32,13 +32,13 @@ language_to_tesseract_code, osd_script_to_tesseract_code, parse_tesseract_orientation, - tesseract_box_to_bounding_rectangle, tesseract_vocabulary, ) from docling.utils.ocr_language import ( OcrLanguage, OcrLanguageSupport, ) +from docling.utils.orientation import ocr_box_to_bounding_rectangle from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) @@ -406,7 +406,7 @@ def __call__( b=bottom, coord_origin=CoordOrigin.TOPLEFT, ) - rect = tesseract_box_to_bounding_rectangle( + rect = ocr_box_to_bounding_rectangle( bbox, original_offset=ocr_rect, scale=self.scale, diff --git a/docling/models/stages/ocr/tesseract_ocr_model.py b/docling/models/stages/ocr/tesseract_ocr_model.py index 19e0ea2d3c..50e3d1e73f 100644 --- a/docling/models/stages/ocr/tesseract_ocr_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_model.py @@ -25,13 +25,13 @@ language_to_tesseract_code, osd_script_to_tesseract_code, parse_tesseract_orientation, - tesseract_box_to_bounding_rectangle, tesseract_vocabulary, ) from docling.utils.ocr_language import ( OcrLanguage, OcrLanguageSupport, ) +from docling.utils.orientation import ocr_box_to_bounding_rectangle from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) @@ -269,7 +269,7 @@ def __call__( b=bottom, coord_origin=CoordOrigin.TOPLEFT, ) - rect = tesseract_box_to_bounding_rectangle( + rect = ocr_box_to_bounding_rectangle( bbox, original_offset=ocr_rect, scale=self.scale, diff --git a/docling/models/stages/ocr/tesseract_utils.py b/docling/models/stages/ocr/tesseract_utils.py index 447a0e2d61..d97ac6e26d 100644 --- a/docling/models/stages/ocr/tesseract_utils.py +++ b/docling/models/stages/ocr/tesseract_utils.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT -"""Tessdata names, orientation and box geometry, shared by both Tesseract models. +"""Tessdata names and orientation, shared by both Tesseract models. `tesseract_ocr_model.py` (the tesserocr bindings) and `tesseract_ocr_cli_model.py` drive the same engine through different front-ends, so everything that is about @@ -9,18 +9,15 @@ """ from collections.abc import Sequence -from typing import Optional, Tuple import langcodes -from docling_core.types.doc import BoundingBox, CoordOrigin -from docling_core.types.doc.page import BoundingRectangle from docling.utils.ocr_language import ( OcrLanguage, OcrLanguageResolver, OcrLanguageSupport, ) -from docling.utils.orientation import CLIPPED_ORIENTATIONS, rotate_bounding_box +from docling.utils.orientation import CLIPPED_ORIENTATIONS # Tessdata files that are not recognizers: `osd` is the orientation-and-script # detector, `equ` the equation model. Neither is a language anyone can ask for. @@ -129,40 +126,3 @@ def parse_tesseract_orientation(orientation: str) -> int: parsed = -parsed parsed %= 360 return parsed - - -def tesseract_box_to_bounding_rectangle( - bbox: BoundingBox, - *, - original_offset: Optional[BoundingBox] = None, - scale: float, - orientation: int, - im_size: Tuple[int, int], -) -> BoundingRectangle: - # box is in the top, left, height, width format, top left coordinates - rect = rotate_bounding_box(bbox, angle=orientation, im_size=im_size) - rect = BoundingRectangle( - r_x0=rect.r_x0 / scale, - r_y0=rect.r_y0 / scale, - r_x1=rect.r_x1 / scale, - r_y1=rect.r_y1 / scale, - r_x2=rect.r_x2 / scale, - r_y2=rect.r_y2 / scale, - r_x3=rect.r_x3 / scale, - r_y3=rect.r_y3 / scale, - coord_origin=CoordOrigin.TOPLEFT, - ) - if original_offset is not None: - if original_offset.coord_origin is not CoordOrigin.TOPLEFT: - msg = f"expected coordinate origin to be {CoordOrigin.TOPLEFT.value}" - raise ValueError(msg) - if original_offset is not None: - rect.r_x0 += original_offset.l - rect.r_x1 += original_offset.l - rect.r_x2 += original_offset.l - rect.r_x3 += original_offset.l - rect.r_y0 += original_offset.t - rect.r_y1 += original_offset.t - rect.r_y2 += original_offset.t - rect.r_y3 += original_offset.t - return rect diff --git a/docling/utils/orientation.py b/docling/utils/orientation.py index af54b56d1d..c304a27784 100644 --- a/docling/utils/orientation.py +++ b/docling/utils/orientation.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT -from typing import Tuple +from typing import Optional, Tuple from docling_core.types.doc import BoundingBox, CoordOrigin from docling_core.types.doc.page import BoundingRectangle @@ -66,3 +66,40 @@ def rotate_bounding_box( coord_origin=CoordOrigin.TOPLEFT, ) return rectangle + + +def ocr_box_to_bounding_rectangle( + bbox: BoundingBox, + *, + original_offset: Optional[BoundingBox] = None, + scale: float, + orientation: int, + im_size: Tuple[int, int], +) -> BoundingRectangle: + # box is in the top, left, height, width format, top left coordinates + rect = rotate_bounding_box(bbox, angle=orientation, im_size=im_size) + rect = BoundingRectangle( + r_x0=rect.r_x0 / scale, + r_y0=rect.r_y0 / scale, + r_x1=rect.r_x1 / scale, + r_y1=rect.r_y1 / scale, + r_x2=rect.r_x2 / scale, + r_y2=rect.r_y2 / scale, + r_x3=rect.r_x3 / scale, + r_y3=rect.r_y3 / scale, + coord_origin=CoordOrigin.TOPLEFT, + ) + if original_offset is not None: + if original_offset.coord_origin is not CoordOrigin.TOPLEFT: + msg = f"expected coordinate origin to be {CoordOrigin.TOPLEFT.value}" + raise ValueError(msg) + if original_offset is not None: + rect.r_x0 += original_offset.l + rect.r_x1 += original_offset.l + rect.r_x2 += original_offset.l + rect.r_x3 += original_offset.l + rect.r_y0 += original_offset.t + rect.r_y1 += original_offset.t + rect.r_y2 += original_offset.t + rect.r_y3 += original_offset.t + return rect diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 8ddcdfad60..0a4450ed90 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -244,6 +244,24 @@ to every `--rapidocr-backend-lang` pair given (or to the default pair, if none a docling-tools models download rapidocr --rapidocr-backend-lang onnxruntime:en --rapidocr-model-size tiny ``` +### RapidOCR page orientation + +RapidOCR turns a tall text line a quarter turn counter-clockwise before it reads the line, and its +line classifier (`use_cls`) only flips single lines by 180 degrees. A scanned page that is turned +counter-clockwise or upside down is read as garbage, or only in part. + +With `detect_orientation`, RapidOCR first reads each OCR region as it is. If it finds at least 3 +lines and fewer than half of them score 0.8 or more, it reads the region again at the other three +orientations. It keeps the first orientation in which at least half of the lines score 0.8 or more +and the text with such scores is at least twice as long. The text boxes are mapped back onto the +page. A region that reads well is read only once, and no extra package or model is needed. + +```python +from docling.datamodel.pipeline_options import RapidOcrOptions + +options = RapidOcrOptions(detect_orientation=True) +``` + ## EasyOCR diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d4e58f81c..6cf6d97f33 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1744,7 +1744,13 @@ def test_cli_deprecated_psm_still_sets_tesseract_psm(tmp_path, monkeypatch): def test_cli_options_lists_engine_fields(): - result = runner.invoke(app, ["options", "ocr", "rapidocr"], terminal_width=200) + # terminal_width sets Click's width only; Rich sizes the table from COLUMNS. + result = runner.invoke( + app, + ["options", "ocr", "rapidocr"], + terminal_width=200, + env={"COLUMNS": "200"}, + ) assert result.exit_code == 0, result.output assert "model_size" in result.output assert "[tiny|small|medium]" in result.output diff --git a/tests/test_e2e_ocr_conversion.py b/tests/test_e2e_ocr_conversion.py index 136e44d2ab..4f516542e1 100644 --- a/tests/test_e2e_ocr_conversion.py +++ b/tests/test_e2e_ocr_conversion.py @@ -103,6 +103,12 @@ def test_e2e_conversions(): ) ) + # Orientation detection must not change the upright pages + configs.append((RapidOcrOptions(detect_orientation=True), False)) + configs.append( + (RapidOcrOptions(detect_orientation=True, mode=OcrMode.FULL_PAGE), False) + ) + # only works on mac if "darwin" == sys.platform: configs.append((OcrMacOptions(), True)) diff --git a/tests/test_e2e_rotated_ocr.py b/tests/test_e2e_rotated_ocr.py index b163006aa8..9a279abb4a 100644 --- a/tests/test_e2e_rotated_ocr.py +++ b/tests/test_e2e_rotated_ocr.py @@ -48,6 +48,7 @@ import importlib.util import shutil +from functools import partial from pathlib import Path import pytest @@ -156,6 +157,28 @@ def test_rotated_page_ocr_text_reaches_default_export(angle, ocr_options_cls): ) +@_SKIP_NO_RAPIDOCR +@pytest.mark.parametrize( + ("angle", "edge"), [(90, "left"), (180, "bottom"), (270, "right")] +) +def test_rapidocr_detect_orientation_reads_rotated_text(angle, edge): + """With `detect_orientation`, RapidOCR reads every line of a turned page, and the + text stays where it is on the stored page: in the top margin, turned to `edge`.""" + doc = _convert(_FIXTURES[angle], partial(RapidOcrOptions, detect_orientation=True)) + + items = [item for item in doc.texts if _MARKER in item.text] + assert sum(item.text.count(_MARKER) for item in items) == 3, ( + f"RapidOCR should read all three lines at {angle}°" + ) + page_size = doc.pages[1].size + for item in items: + box = item.prov[0].bbox.to_top_left_origin(page_size.height) + x = (box.l + box.r) / 2 / page_size.width + y = (box.t + box.b) / 2 / page_size.height + on_edge = {"left": x < 0.1, "bottom": y > 0.9, "right": x > 0.9}[edge] + assert on_edge, f"text at {angle}° should be on the {edge} edge of the page" + + if __name__ == "__main__": for _angle in _FIXTURES: _generate_fixture(_angle)