From 25a0eeadd752a349a2f707024d931b80ce1ebbc0 Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 12 Oct 2026 21:48:00 +0200 Subject: [PATCH 1/7] feat: add LightOnOCR-3 presets and grounded output parsing Signed-off-by: Pierre Mesure --- .../.agents/skills/docling/references/cli.md | 1 + docling/datamodel/pipeline_options.py | 3 + .../datamodel/pipeline_options_vlm_model.py | 1 + docling/datamodel/stage_model_specs.py | 44 ++++ docling/pipeline/vlm_pipeline.py | 10 + docling/utils/lightonocr_utils.py | 132 +++++++++++ docs/usage/vision_models.md | 36 +++ tests/test_lightonocr3_vlm.py | 216 ++++++++++++++++++ 8 files changed, 443 insertions(+) create mode 100644 docling/utils/lightonocr_utils.py create mode 100644 tests/test_lightonocr3_vlm.py diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index 7b8ebbb31c..3021e5eabb 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -52,6 +52,7 @@ docling report.pdf --pipeline vlm --vlm-model granite_docling --output /tmp/ docling report.pdf --pipeline vlm --vlm-model smoldocling --output /tmp/ docling report.pdf --pipeline vlm --vlm-model nemotron_parse_v2 --output /tmp/ docling report.pdf --pipeline vlm --vlm-model nemotron_parse_v3_5 --trust-remote-code --output /tmp/ +docling report.pdf --pipeline vlm --vlm-model lightonocr3_08b --output /tmp/ docling chart.png --from image --to dclx --pipeline vlm --enrich-chart-extraction docling report.pdf --pipeline native --from pdf --output /tmp/ ``` diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 910b11cca4..36f6ea95b5 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -1232,6 +1232,9 @@ class CodeFormulaVlmOptions(StagePresetMixin, VlmEngineOptionsMixin, BaseModel): VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_DOLPHIN) VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_GLMOCR) VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR) +VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_08B) +VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_1B) +VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_4B) VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_FALCON_OCR) VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_CHANDRA_OCR2) VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_UNLIMITED_OCR) diff --git a/docling/datamodel/pipeline_options_vlm_model.py b/docling/datamodel/pipeline_options_vlm_model.py index 5a3f4b918c..dfe01d1946 100644 --- a/docling/datamodel/pipeline_options_vlm_model.py +++ b/docling/datamodel/pipeline_options_vlm_model.py @@ -107,6 +107,7 @@ class ResponseFormat(str, Enum): MARKDOWN = "markdown" DEEPSEEKOCR_MARKDOWN = "deepseekocr_markdown" UNLIMITED_OCR_MARKDOWN = "unlimited_ocr_markdown" + LIGHTONOCR_MARKDOWN = "lightonocr_markdown" HTML = "html" OTSL = "otsl" PLAINTEXT = "plaintext" diff --git a/docling/datamodel/stage_model_specs.py b/docling/datamodel/stage_model_specs.py index 26fe32c1a3..ffe12d1c03 100644 --- a/docling/datamodel/stage_model_specs.py +++ b/docling/datamodel/stage_model_specs.py @@ -1697,6 +1697,50 @@ def from_preset( default_engine_type=VlmEngineType.AUTO_INLINE, ) + +def _lightonocr3_preset(size: str, preset_id: str) -> StageModelPreset: + name = f"LightOnOCR-3-{size}" + repo_id = f"lightonai/{name}" + return StageModelPreset( + preset_id=preset_id, + name=name, + description=f"LightOnOCR-3 OCR with block grounding ({size} parameters)", + model_spec=VlmModelSpec( + name=name, + default_repo_id=repo_id, + prompt="grounding", + response_format=ResponseFormat.LIGHTONOCR_MARKDOWN, + max_new_tokens=4096, + engine_overrides={ + VlmEngineType.TRANSFORMERS: EngineModelConfig( + min_engine_version="5.5.4", + torch_dtype="bfloat16", + extra_config={ + "transformers_model_type": TransformersModelType.AUTOMODEL_IMAGETEXTTOTEXT, + "transformers_prompt_style": TransformersPromptStyle.CHAT, + "extra_generation_config": {"skip_special_tokens": True}, + }, + ), + }, + api_overrides={ + VlmEngineType.API: ApiModelConfig( + params={"model": repo_id, "max_tokens": 4096} + ), + VlmEngineType.API_OPENAI: ApiModelConfig( + params={"model": repo_id, "max_tokens": 4096} + ), + }, + ), + scale=2.0, + max_size=1540, + default_engine_type=VlmEngineType.AUTO_INLINE, + ) + + +VLM_CONVERT_LIGHTONOCR_3_08B = _lightonocr3_preset("0.8B", "lightonocr3_08b") +VLM_CONVERT_LIGHTONOCR_3_1B = _lightonocr3_preset("1B", "lightonocr3_1b") +VLM_CONVERT_LIGHTONOCR_3_4B = _lightonocr3_preset("4B", "lightonocr3_4b") + # ----------------------------------------------------------------------------- # PICTURE_DESCRIPTION PRESETS (for image captioning/description) # ----------------------------------------------------------------------------- diff --git a/docling/pipeline/vlm_pipeline.py b/docling/pipeline/vlm_pipeline.py index 0bf6cc0bba..b0573e9e88 100644 --- a/docling/pipeline/vlm_pipeline.py +++ b/docling/pipeline/vlm_pipeline.py @@ -559,6 +559,16 @@ def _finalize_page_document( page_image=page.image, normalize_heading_levels=False, ) + elif response_format == ResponseFormat.LIGHTONOCR_MARKDOWN: + from docling.utils.lightonocr_utils import parse_lightonocr_markdown + + document = parse_lightonocr_markdown( + content=predicted_text, + original_page_size=page.size, + page_no=page.page_no, + filename=conv_res.input.file.name or "file", + page_image=page.image, + ) elif response_format == ResponseFormat.MARKDOWN: document = self._convert_text_page( conv_res, page, InputFormat.MD, MarkdownDocumentBackend diff --git a/docling/utils/lightonocr_utils.py b/docling/utils/lightonocr_utils.py new file mode 100644 index 0000000000..5226b3c268 --- /dev/null +++ b/docling/utils/lightonocr_utils.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Parse LightOnOCR-3 Markdown with normalized block bounding boxes.""" + +from __future__ import annotations + +import html +import re +from collections.abc import Iterator +from pathlib import Path + +import marko +from docling_core.types.doc import ( + ContentLayer, + DocItemLabel, + DoclingDocument, + DocumentOrigin, + ImageRef, + Size, +) +from PIL import Image as PILImage + +from docling.utils.chandra_utils import _add_html_fragment + +_MARKER = re.compile( + r"^ {0,3}!\[([a-z_]+)\+?\]\(([^)\n]*,[^)\n]*,[^)\n]*,[^)\n]*)\)[ \t]*(.*)$" +) +_LABELS = { + "title": "Title", + "header": "Page-Header", + "footer": "Page-Footer", + "page_number": "Page-Footer", + "footnote": "Footnote", + "caption": "Caption", + "formula": "Equation-Block", + "code": "Code-Block", + "table": "Table", + "list": "List-Group", + "image": "Image", + "chart": "Figure", + "header_image": "Image", + "footer_image": "Image", +} + + +def _blocks(content: str) -> Iterator[tuple[str, str, str]]: + label, bbox = "text", "" + lines: list[str] = [] + fence = "" + for line in content.splitlines(keepends=True): + fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})", line) + if fence_match: + delimiter = fence_match[1] + if not fence: + fence = delimiter + elif ( + delimiter[0] == fence[0] + and len(delimiter) >= len(fence) + and not line[fence_match.end() :].strip() + ): + fence = "" + lines.append(line) + continue + marker = _MARKER.fullmatch(line.rstrip("\r\n")) if not fence else None + if marker is not None: + if lines or bbox: + yield label, bbox, "".join(lines) + label, bbox, inline_body = marker.groups() + lines = [inline_body + "\n"] if inline_body else [] + else: + lines.append(line) + if lines or bbox: + yield label, bbox, "".join(lines) + + +def parse_lightonocr_markdown( + content: str, + original_page_size: Size, + page_no: int, + filename: str = "file", + page_image: PILImage.Image | None = None, +) -> DoclingDocument: + """Preserve Markdown/HTML content and assign each block bbox at most once. + + Continuation labels retain their base class. Separate blocks are not merged + or associated by proximity. Plain transcription is recovered without boxes. + """ + doc = DoclingDocument( + name=Path(filename).stem, + origin=DocumentOrigin( + filename=filename, mimetype="text/markdown", binary_hash=0 + ), + ) + dpi = ( + max(1, round(72 * page_image.width / original_page_size.width)) + if page_image is not None + else 72 + ) + doc.add_page( + page_no=page_no, + size=original_page_size, + image=ImageRef.from_pil(image=page_image, dpi=dpi) if page_image else None, + ) + markdown = marko.Markdown(extensions=["gfm"]) + for label, bbox, body in _blocks(content): + if not body.strip() and label not in { + "image", + "chart", + "header_image", + "footer_image", + }: + continue + rendered = markdown.convert(body) + if label in {"image", "header_image", "footer_image"}: + rendered = f'{html.escape(body.strip(), quote=True)}' + attrs = ( + f'data-bbox="{html.escape(bbox.replace(",", " "), quote=True)}"' + if bbox + else "" + ) + if bbox: + attrs += f' data-label="{_LABELS.get(label, "Text")}"' + fragment = f"
{rendered}
" + picture_start = len(doc.pictures) + _add_html_fragment(doc, fragment, label=DocItemLabel.TEXT) + for picture in doc.pictures[picture_start:]: + if picture.meta is not None and picture.meta.description is not None: + picture.meta.description.created_by = "lightonocr" + if label in {"header_image", "footer_image"}: + picture.content_layer = ContentLayer.FURNITURE + return doc diff --git a/docs/usage/vision_models.md b/docs/usage/vision_models.md index f021fb9fb6..67d2fd01a1 100644 --- a/docs/usage/vision_models.md +++ b/docs/usage/vision_models.md @@ -121,6 +121,42 @@ converter = DocumentConverter( doc = converter.convert(source="FILE").document ``` +### LightOnOCR-3 + +LightOnOCR-3 has three presets: `lightonocr3_08b`, `lightonocr3_1b`, and +`lightonocr3_4b`. They use the official +[LightOnOCR-3 models](https://huggingface.co/lightonai/LightOnOCR-3-1B) +and require `transformers>=5.5.4` for local inference. The existing `lightonocr` +preset continues to use LightOnOCR-2-1B. + +```bash +docling contract.pdf --pipeline vlm --vlm-model lightonocr3_08b +``` + +```python +from docling.datamodel.pipeline_options import VlmConvertOptions, VlmPipelineOptions + +pipeline_options = VlmPipelineOptions( + vlm_options=VlmConvertOptions.from_preset("lightonocr3_1b"), +) +``` + +Pass these options to the `PdfFormatOption` in the example above. The presets +use the `grounding` prompt. Docling parses the block labels and normalized +bounding boxes, Markdown text, HTML tables, image descriptions, and chart tables. +Coordinates belong to blocks; derived text and table cells do not receive +separate boxes. Continuation blocks keep their base labels without being merged. +Lists keep their structure, without assigning the block box to individual items. +Separate captions are not associated with pictures or tables by proximity. + +For plain transcription, set `pipeline_options.vlm_options.model_spec.prompt = ""`. +The parser also accepts that Markdown output, without bounding boxes. Use only +the empty prompt or `grounding`, as specified in the model cards. + +Local inference uses Transformers. No MLX export is configured for these presets. +An OpenAI-compatible server can also serve the official model repositories; +configure its URL and engine options as described below. + ### Other models Other models can be configured by directly providing the Hugging Face `repo_id`, the prompt and a few more options. diff --git a/tests/test_lightonocr3_vlm.py b/tests/test_lightonocr3_vlm.py new file mode 100644 index 0000000000..74524d50b9 --- /dev/null +++ b/tests/test_lightonocr3_vlm.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""LightOnOCR-3 preset and grounded response tests, without model downloads.""" + +from pathlib import PurePath +from types import SimpleNamespace + +import pytest +from docling_core.types.doc import ContentLayer, DocItemLabel, Size +from PIL import Image + +from docling.datamodel.base_models import Page, PagePredictions, VlmPrediction +from docling.datamodel.pipeline_options import VlmConvertOptions, VlmPipelineOptions +from docling.datamodel.pipeline_options_vlm_model import ( + ResponseFormat, + TransformersModelType, + TransformersPromptStyle, +) +from docling.models.inference_engines.vlm.base import VlmEngineType +from docling.pipeline.vlm_pipeline import VlmPipeline +from docling.utils.lightonocr_utils import parse_lightonocr_markdown + +PAGE_SIZE = Size(width=600, height=800) + + +@pytest.mark.parametrize("size, suffix", [("0.8B", "08b"), ("1B", "1b"), ("4B", "4b")]) +def test_preset_runtime_configuration(size: str, suffix: str): + options = VlmConvertOptions.from_preset(f"lightonocr3_{suffix}") + spec = options.model_spec + config = spec.get_engine_config(VlmEngineType.TRANSFORMERS) + runtime = spec.get_runtime_input_extra_config(VlmEngineType.TRANSFORMERS) + assert config.repo_id == f"lightonai/LightOnOCR-3-{size}" + assert config.min_engine_version == "5.5.4" + assert config.torch_dtype == "bfloat16" + assert ( + config.extra_config["transformers_model_type"] + == TransformersModelType.AUTOMODEL_IMAGETEXTTOTEXT + ) + assert runtime["transformers_prompt_style"] == TransformersPromptStyle.CHAT + assert runtime["skip_special_tokens"] is True + assert "extra_generation_config" not in runtime + assert spec.prompt == "grounding" + assert spec.response_format == ResponseFormat.LIGHTONOCR_MARKDOWN + assert options.max_size == 1540 + for engine in (VlmEngineType.API, VlmEngineType.API_OPENAI): + assert spec.get_api_params(engine) == { + "model": config.repo_id, + "max_tokens": 4096, + } + assert not spec.has_explicit_engine_export(VlmEngineType.MLX) + + +def test_grounded_text_and_continuation(): + content = """![title](100,50,900,100) +# Contract + +![text](100,150,900,400) +First line +and its continuation. + +A second paragraph. + +![text+](100,450,900,500) +Continued block. +""" + doc = parse_lightonocr_markdown( + content, PAGE_SIZE, page_no=3, filename="contract.pdf" + ) + assert doc.name == "contract" + assert [t.text for t in doc.texts] == [ + "Contract", + "First line and its continuation.", + "A second paragraph.", + "Continued block.", + ] + assert doc.texts[0].label == DocItemLabel.TITLE + assert doc.texts[0].prov[0].page_no == 3 + bbox = doc.texts[0].prov[0].bbox + assert (bbox.l, bbox.t, bbox.r, bbox.b) == (60, 40, 540, 80) + assert doc.texts[1].prov + assert not doc.texts[2].prov + assert doc.texts[3].prov + + +def test_inline_markers_from_model_prediction(): + content = ( + "![title](51,110,448,180) # LEASE AGREEMENT\n\n" + "![text](51,297,527,542) Tenant: Example AB Monthly rent: SEK 1000 " + "Start date: 2026-10-01" + ) + doc = parse_lightonocr_markdown(content, PAGE_SIZE, page_no=1) + assert not doc.pictures + assert doc.texts[0].text == "LEASE AGREEMENT" + assert doc.texts[0].label == DocItemLabel.TITLE + assert doc.texts[0].prov[0].bbox.l == pytest.approx(30.6) + assert ( + doc.texts[1].text + == "Tenant: Example AB Monthly rent: SEK 1000 Start date: 2026-10-01" + ) + assert doc.texts[1].prov + + +def test_tables_images_and_charts(): + content = """![table](0,0,1000,250) +
Rent
1000
+ +![image](0,300,400,600) +A building with a sign saying "Office". + +![chart](500,300,1000,600) +
YearRent
20261000
+ +![caption](500,650,1000,700) +Annual rent. +""" + doc = parse_lightonocr_markdown(content, PAGE_SIZE, page_no=1) + assert len(doc.tables) == 2 + assert doc.tables[0].data.grid[1][0].text == "1000" + assert len(doc.pictures) == 2 + description = doc.pictures[0].meta.description + assert description.text == 'A building with a sign saying "Office".' + assert description.created_by == "lightonocr" + assert doc.tables[1].parent == doc.pictures[1].get_ref() + assert doc.pictures[1].prov + assert not doc.tables[1].prov + assert doc.texts[-1].label == DocItemLabel.CAPTION + assert not doc.pictures[1].captions + + +def test_furniture_and_invalid_coordinates(): + content = """![header](0,0,1000,100) +Lease +![page_number](400,900,600,1000) +3 +![header_image](0,0,100,100) +Company logo. +![text](900,200,100,300) +Retain this despite the reversed box. +""" + doc = parse_lightonocr_markdown(content, PAGE_SIZE, page_no=1) + assert doc.texts[0].label == DocItemLabel.PAGE_HEADER + assert doc.texts[1].label == DocItemLabel.PAGE_FOOTER + assert all(t.content_layer == ContentLayer.FURNITURE for t in doc.texts[:2]) + assert doc.pictures[0].content_layer == ContentLayer.FURNITURE + assert doc.texts[-1].text == "Retain this despite the reversed box." + assert not doc.texts[-1].prov + + +def test_plain_transcription_and_code_markers(): + content = """# Heading + +| Field | Value | +| --- | --- | +| Rent | 1000 | + +![code](0,200,1000,500) +```markdown +![text](0,0,1000,1000) +``` +""" + doc = parse_lightonocr_markdown(content, PAGE_SIZE, page_no=1) + assert doc.texts[0].text == "Heading" + assert not doc.texts[0].prov + assert len(doc.tables) == 1 + assert doc.texts[-1].label == DocItemLabel.CODE + assert doc.texts[-1].text == "![text](0,0,1000,1000)" + assert doc.texts[-1].prov + + +def test_empty_page_preserves_image(): + image = Image.new("RGB", (600, 800), "white") + doc = parse_lightonocr_markdown("", PAGE_SIZE, page_no=2, page_image=image) + assert doc.pages[2].image.pil_image.size == (600, 800) + assert not (doc.texts or doc.tables or doc.pictures) + + +def test_lists_and_empty_image_blocks(): + content = """![list](100,100,900,300) +1. First condition +2. Second condition + +![image](100,400,900,700) +""" + doc = parse_lightonocr_markdown(content, PAGE_SIZE, page_no=1) + assert [t.text for t in doc.texts] == ["First condition", "Second condition"] + assert all(t.label == DocItemLabel.LIST_ITEM for t in doc.texts) + assert all(not t.prov for t in doc.texts) + assert len(doc.pictures) == 1 + assert doc.pictures[0].prov + + +def test_pipeline_dispatch_and_page_concatenation(): + pipeline = VlmPipeline.__new__(VlmPipeline) + pipeline.pipeline_options = VlmPipelineOptions( + vlm_options=VlmConvertOptions.from_preset("lightonocr3_08b") + ) + pipeline.force_backend_text = False + page = Page( + page_no=5, + size=PAGE_SIZE, + predictions=PagePredictions( + vlm_response=VlmPrediction(text="![text](100,200,900,300)\nRent: 1000") + ), + ) + result = SimpleNamespace( + input=SimpleNamespace(file=PurePath("contract.pdf")), errors=[] + ) + page_document = pipeline._finalize_page_document(result, page) + document = pipeline._concatenate_page_documents([(5, page_document)]) + assert sorted(document.pages) == [5] + assert document.texts[0].text == "Rent: 1000" + assert document.texts[0].prov[0].page_no == 5 + bbox = document.texts[0].prov[0].bbox + assert (bbox.l, bbox.t, bbox.r, bbox.b) == (60, 160, 540, 240) + assert not result.errors From 889bf7f40b2d13c58e5cdcc9e89ba6a837766534 Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 19 Oct 2026 07:50:12 +0200 Subject: [PATCH 2/7] test: accept LightOnOCR response format in preset checks Include LIGHTONOCR_MARKDOWN in the explicit valid-format list so the LightOnOCR-3 presets pass the cross-stage consistency check. Signed-off-by: Pierre Mesure --- tests/test_vlm_presets_and_runtime_options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_vlm_presets_and_runtime_options.py b/tests/test_vlm_presets_and_runtime_options.py index 0aa109bcdc..28cd7216d6 100644 --- a/tests/test_vlm_presets_and_runtime_options.py +++ b/tests/test_vlm_presets_and_runtime_options.py @@ -737,6 +737,7 @@ def test_response_format_consistency(self): ResponseFormat.MARKDOWN, ResponseFormat.DEEPSEEKOCR_MARKDOWN, ResponseFormat.UNLIMITED_OCR_MARKDOWN, + ResponseFormat.LIGHTONOCR_MARKDOWN, ResponseFormat.PLAINTEXT, ResponseFormat.CHANDRA_HTML, ResponseFormat.DOTS_JSON, From 7cdf10c91461c475f9ad115615cd0b67c3af2743 Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 19 Oct 2026 21:11:40 +0200 Subject: [PATCH 3/7] fix: guard optional Markdown dependency in LightOnOCR parser Defer the marko import until parsing and report the format-markdown extra when it is missing. Add a cold-import regression test and document the requirement for slim installations. Signed-off-by: Pierre Mesure --- docling/utils/lightonocr_utils.py | 10 +++++++++- docs/usage/vision_models.md | 6 ++++++ tests/test_backend_optional_dependencies.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docling/utils/lightonocr_utils.py b/docling/utils/lightonocr_utils.py index 5226b3c268..e3517c3449 100644 --- a/docling/utils/lightonocr_utils.py +++ b/docling/utils/lightonocr_utils.py @@ -10,7 +10,6 @@ from collections.abc import Iterator from pathlib import Path -import marko from docling_core.types.doc import ( ContentLayer, DocItemLabel, @@ -86,6 +85,15 @@ def parse_lightonocr_markdown( Continuation labels retain their base class. Separate blocks are not merged or associated by proximity. Plain transcription is recovered without boxes. """ + # Markdown support is optional in docling-slim installations. + try: + import marko + except ImportError as exc: + raise ImportError( + "LightOnOCR Markdown parsing requires " + "`pip install 'docling-slim[format-markdown]'`." + ) from exc + doc = DoclingDocument( name=Path(filename).stem, origin=DocumentOrigin( diff --git a/docs/usage/vision_models.md b/docs/usage/vision_models.md index 67d2fd01a1..ce66d4eb2e 100644 --- a/docs/usage/vision_models.md +++ b/docs/usage/vision_models.md @@ -129,6 +129,12 @@ LightOnOCR-3 has three presets: `lightonocr3_08b`, `lightonocr3_1b`, and and require `transformers>=5.5.4` for local inference. The existing `lightonocr` preset continues to use LightOnOCR-2-1B. +For `docling-slim` installations, the parser also requires the Markdown extra: + +```bash +pip install 'docling-slim[format-markdown]' +``` + ```bash docling contract.pdf --pipeline vlm --vlm-model lightonocr3_08b ``` diff --git a/tests/test_backend_optional_dependencies.py b/tests/test_backend_optional_dependencies.py index 626431bf25..ce49f65344 100644 --- a/tests/test_backend_optional_dependencies.py +++ b/tests/test_backend_optional_dependencies.py @@ -197,6 +197,26 @@ def test_backend_reports_missing_dependency_with_install_hint( assert "actionable-error-raised" in result.stdout +def test_lightonocr_parser_reports_missing_markdown_dependency() -> None: + result = _run_with_blocked_module( + "marko", + "from docling.utils.lightonocr_utils import parse_lightonocr_markdown\n" + "from docling_core.types.doc import Size\n" + "try:\n" + " parse_lightonocr_markdown(\n" + " '![text](0,0,1000,1000)\\nRent: 1000',\n" + " Size(width=600, height=800), page_no=1,\n" + " )\n" + "except ImportError as exc:\n" + " assert \"pip install 'docling-slim[format-markdown]'\" in str(exc), str(exc)\n" + " print('actionable-error-raised')\n" + "else:\n" + " raise AssertionError('expected ImportError when marko is missing')\n", + ) + assert result.returncode == 0, result.stderr + assert "actionable-error-raised" in result.stdout + + def test_service_client_imports_without_pdf_pipeline_dependency() -> None: # The service client reaches docling.utils.pdf_outline (via noop_backend -> # datamodel.document, which needs only the _PdfOutlineItem model), so a From 80ccc0c17fc8499569eabe4a3b52d2fda284ae23 Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 19 Oct 2026 22:16:13 +0200 Subject: [PATCH 4/7] fix: preserve LightOnOCR math, headings, and fenced code Keep LaTeX literal during Markdown parsing and store formulas without export delimiters. Preserve section heading levels using Docling's title-relative levels, and track fences that start on grounding-marker lines. Add regression cases for formula escapes and multiline content, heading exports and coordinates, inline math versus code, and both fence styles. Keep the shared Chandra mapper unchanged. Signed-off-by: Pierre Mesure --- docling/utils/lightonocr_utils.py | 113 ++++++++++++++++++++++++------ docs/usage/vision_models.md | 1 + tests/test_lightonocr3_vlm.py | 99 +++++++++++++++++++++++++- 3 files changed, 190 insertions(+), 23 deletions(-) diff --git a/docling/utils/lightonocr_utils.py b/docling/utils/lightonocr_utils.py index e3517c3449..b2db419001 100644 --- a/docling/utils/lightonocr_utils.py +++ b/docling/utils/lightonocr_utils.py @@ -9,6 +9,7 @@ import re from collections.abc import Iterator from pathlib import Path +from typing import TYPE_CHECKING from docling_core.types.doc import ( ContentLayer, @@ -16,15 +17,24 @@ DoclingDocument, DocumentOrigin, ImageRef, + SectionHeaderItem, Size, ) from PIL import Image as PILImage from docling.utils.chandra_utils import _add_html_fragment +if TYPE_CHECKING: + import marko + _MARKER = re.compile( r"^ {0,3}!\[([a-z_]+)\+?\]\(([^)\n]*,[^)\n]*,[^)\n]*,[^)\n]*)\)[ \t]*(.*)$" ) +_MATH = re.compile( + r"(? Iterator[tuple[str, str, str]]: lines: list[str] = [] fence = "" for line in content.splitlines(keepends=True): + marker = _MARKER.fullmatch(line.rstrip("\r\n")) if not fence else None + if marker is not None: + if lines or bbox: + yield label, bbox, "".join(lines) + label, bbox, inline_body = marker.groups() + lines = [] + line = inline_body + "\n" if inline_body else "" + + # Check the block body too: a fence can start on the marker's line. fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})", line) if fence_match: delimiter = fence_match[1] @@ -59,20 +78,62 @@ def _blocks(content: str) -> Iterator[tuple[str, str, str]]: and not line[fence_match.end() :].strip() ): fence = "" - lines.append(line) - continue - marker = _MARKER.fullmatch(line.rstrip("\r\n")) if not fence else None - if marker is not None: - if lines or bbox: - yield label, bbox, "".join(lines) - label, bbox, inline_body = marker.groups() - lines = [inline_body + "\n"] if inline_body else [] - else: - lines.append(line) + lines.append(line) if lines or bbox: yield label, bbox, "".join(lines) +def _math_html(text: str, *, display: bool) -> str: + """Keep LaTeX literal and let Docling's serializers supply its delimiters.""" + text = text.strip() + for opening, closing in (("$$", "$$"), (r"\[", r"\]"), (r"\(", r"\)"), ("$", "$")): + if ( + len(text) >= len(opening) + len(closing) + and text.startswith(opening) + and text.endswith(closing) + ): + text = text[len(opening) : -len(closing)].strip() + break + attrs = ' display="block"' if display else "" + return f"{html.escape(text)}" + + +def _markdown_parser() -> marko.Markdown: + # Markdown support is optional in docling-slim installations. + try: + import marko + except ImportError as exc: + raise ImportError( + "LightOnOCR Markdown parsing requires " + "`pip install 'docling-slim[format-markdown]'`." + ) from exc + from marko.helpers import MarkoExtension + + class Math(marko.inline.InlineElement): + children: str + pattern = _MATH + priority = 8 + parse_group = 0 + + class CodeSpan(marko.inline.CodeSpan): + override = True + priority = 9 + + class MathRenderer: + def render_math(self, element: Math) -> str: + return _math_html( + element.children, + display=element.children.startswith(("$$", r"\[")), + ) + + return marko.Markdown( + extensions=[ + "gfm", + MarkoExtension(elements=[Math, CodeSpan], renderer_mixins=[MathRenderer]), + ] + ) + + def parse_lightonocr_markdown( content: str, original_page_size: Size, @@ -85,14 +146,9 @@ def parse_lightonocr_markdown( Continuation labels retain their base class. Separate blocks are not merged or associated by proximity. Plain transcription is recovered without boxes. """ - # Markdown support is optional in docling-slim installations. - try: - import marko - except ImportError as exc: - raise ImportError( - "LightOnOCR Markdown parsing requires " - "`pip install 'docling-slim[format-markdown]'`." - ) from exc + markdown = _markdown_parser() + # The parser above checks this optional dependency before its types are used. + from marko.block import Heading, SetextHeading doc = DoclingDocument( name=Path(filename).stem, @@ -110,7 +166,6 @@ def parse_lightonocr_markdown( size=original_page_size, image=ImageRef.from_pil(image=page_image, dpi=dpi) if page_image else None, ) - markdown = marko.Markdown(extensions=["gfm"]) for label, bbox, body in _blocks(content): if not body.strip() and label not in { "image", @@ -119,16 +174,26 @@ def parse_lightonocr_markdown( "footer_image", }: continue - rendered = markdown.convert(body) - if label in {"image", "header_image", "footer_image"}: + layout_label = _LABELS.get(label, "Text") + if label == "formula": + rendered = _math_html(body, display=True) + elif label in {"image", "header_image", "footer_image"}: rendered = f'{html.escape(body.strip(), quote=True)}' + else: + parsed = markdown.parse(body) + rendered = markdown.render(parsed) + if label == "title" and any( + isinstance(node, (Heading, SetextHeading)) and node.level > 1 + for node in parsed.children + ): + layout_label = "Section-Header" attrs = ( f'data-bbox="{html.escape(bbox.replace(",", " "), quote=True)}"' if bbox else "" ) if bbox: - attrs += f' data-label="{_LABELS.get(label, "Text")}"' + attrs += f' data-label="{layout_label}"' fragment = f"
{rendered}
" picture_start = len(doc.pictures) _add_html_fragment(doc, fragment, label=DocItemLabel.TEXT) @@ -137,4 +202,8 @@ def parse_lightonocr_markdown( picture.meta.description.created_by = "lightonocr" if label in {"header_image", "footer_image"}: picture.content_layer = ContentLayer.FURNITURE + # The HTML mapper uses h1-h6 levels; Docling reserves the top level for titles. + for item in doc.texts: + if isinstance(item, SectionHeaderItem) and item.level > 1: + item.level -= 1 return doc diff --git a/docs/usage/vision_models.md b/docs/usage/vision_models.md index ce66d4eb2e..10750ef9c0 100644 --- a/docs/usage/vision_models.md +++ b/docs/usage/vision_models.md @@ -150,6 +150,7 @@ pipeline_options = VlmPipelineOptions( Pass these options to the `PdfFormatOption` in the example above. The presets use the `grounding` prompt. Docling parses the block labels and normalized bounding boxes, Markdown text, HTML tables, image descriptions, and chart tables. +Markdown section heading levels and LaTeX math are preserved. Coordinates belong to blocks; derived text and table cells do not receive separate boxes. Continuation blocks keep their base labels without being merged. Lists keep their structure, without assigning the block box to individual items. diff --git a/tests/test_lightonocr3_vlm.py b/tests/test_lightonocr3_vlm.py index 74524d50b9..8d4ace8d3b 100644 --- a/tests/test_lightonocr3_vlm.py +++ b/tests/test_lightonocr3_vlm.py @@ -7,7 +7,14 @@ from types import SimpleNamespace import pytest -from docling_core.types.doc import ContentLayer, DocItemLabel, Size +from docling_core.types.doc import ( + CodeItem, + ContentLayer, + DocItemLabel, + FormulaItem, + SectionHeaderItem, + Size, +) from PIL import Image from docling.datamodel.base_models import Page, PagePredictions, VlmPrediction @@ -214,3 +221,93 @@ def test_pipeline_dispatch_and_page_concatenation(): bbox = document.texts[0].prov[0].bbox assert (bbox.l, bbox.t, bbox.r, bbox.b) == (60, 160, 540, 240) assert not result.errors + + +@pytest.mark.parametrize( + "heading, level", + [("## 2.2 LightOnOCR-1", 2), ("### Subsection", 3), ("Section\n-------", 2)], +) +def test_grounded_section_heading_preserves_level(heading: str, level: int): + doc = parse_lightonocr_markdown( + f"![title](100,100,900,200) {heading}", PAGE_SIZE, page_no=3 + ) + item = doc.texts[0] + assert isinstance(item, SectionHeaderItem) + assert item.level == level - 1 + assert item.prov[0].page_no == 3 + assert item.prov[0].bbox.t == 80 + assert doc.export_to_markdown() == f"{'#' * level} {item.text}" + + +@pytest.mark.parametrize( + "body, latex", + [ + ("$$\nx^2 + y^2 = z^2\n$$", "x^2 + y^2 = z^2"), + (r"\[\left\{x : x > 0\right\}\]", r"\left\{x : x > 0\right\}"), + (r"\(x_1 * y_2 * z_3\)", r"x_1 * y_2 * z_3"), + (r"$x_1$", r"x_1"), + (r"\frac{a}{b}", r"\frac{a}{b}"), + ( + "$$\n\\begin{aligned}\na &= b \\\\\nc &= d\n\\end{aligned}\n$$", + "\\begin{aligned}\na &= b \\\\\nc &= d\n\\end{aligned}", + ), + ], +) +def test_grounded_formula_preserves_latex(body: str, latex: str): + doc = parse_lightonocr_markdown( + f"![formula](100,100,900,300)\n{body}", PAGE_SIZE, page_no=2 + ) + assert len(doc.texts) == 1 + formula = doc.texts[0] + assert isinstance(formula, FormulaItem) + assert formula.text == latex + assert formula.prov[0].page_no == 2 + assert formula.prov[0].bbox.l == 60 + assert doc.export_to_markdown() == f"$${latex}$$" + + +@pytest.mark.parametrize("marker", ["", "![text](100,100,900,300)\n"]) +def test_inline_math_preserves_latex_and_code(marker: str): + latex = r"\left\{x_1 * y_2 * z_3 < 2\right\}" + source = marker + f"**Value** ${latex}$ and `literal $x_1$`. Cost: \\$5." + doc = parse_lightonocr_markdown(source, PAGE_SIZE, page_no=1) + formulas = [item for item in doc.texts if isinstance(item, FormulaItem)] + assert [formula.text for formula in formulas] == [latex] + assert all(not formula.prov for formula in formulas) + assert [item.text for item in doc.texts if isinstance(item, CodeItem)] == [ + "literal $x_1$" + ] + exported = doc.export_to_markdown() + assert f"${latex}$" in exported + assert "`literal $x_1$`" in exported + assert "**Value**" in exported + assert "$5" in exported + + +@pytest.mark.parametrize("fence", ["```", "~~~~"]) +@pytest.mark.parametrize("code", ["print(1)", "![text](0,0,1000,1000)"]) +def test_inline_code_fence_preserves_following_block(fence: str, code: str): + source = ( + f"![code](0,0,1000,500) {fence}python\n{code}\n{fence}\n" + "![text](0,500,1000,600) After code.\n" + ) + doc = parse_lightonocr_markdown(source, PAGE_SIZE, page_no=1) + assert [(item.label, item.text) for item in doc.texts] == [ + (DocItemLabel.CODE, code), + (DocItemLabel.TEXT, "After code."), + ] + assert not doc.pictures + assert doc.texts[0].prov[0].bbox.b == 400 + assert doc.texts[1].prov[0].bbox.t == 400 + assert doc.export_to_markdown() == f"```\n{code}\n```\n\nAfter code." + + +def test_formatted_section_heading_owns_its_coordinates_once(): + doc = parse_lightonocr_markdown( + "![title](100,100,900,200) ## **Rent** and terms", PAGE_SIZE, page_no=1 + ) + assert isinstance(doc.texts[0], SectionHeaderItem) + assert doc.texts[0].level == 1 + assert doc.texts[0].prov + assert all(not item.prov for item in doc.texts[1:]) + assert doc.export_to_markdown() == "## **Rent** and terms" From 6999ddf1fccd7cd034e63f6dd43d89d3a14f5c93 Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 19 Oct 2026 22:56:48 +0200 Subject: [PATCH 5/7] refactor: reuse shared VLM math and heading handling Move Markdown math rendering and delimiter normalization into shared helpers. Correct heading depth in the shared HTML mapper and remove the LightOn-only compensation. Coordinate with parser issues #4902, #4903, and #4904. Signed-off-by: Pierre Mesure --- docling/utils/lightonocr_utils.py | 71 ++-------------------- docling/utils/markdown_math.py | 97 +++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 67 deletions(-) create mode 100644 docling/utils/markdown_math.py diff --git a/docling/utils/lightonocr_utils.py b/docling/utils/lightonocr_utils.py index b2db419001..79b455cda6 100644 --- a/docling/utils/lightonocr_utils.py +++ b/docling/utils/lightonocr_utils.py @@ -9,7 +9,6 @@ import re from collections.abc import Iterator from pathlib import Path -from typing import TYPE_CHECKING from docling_core.types.doc import ( ContentLayer, @@ -17,24 +16,17 @@ DoclingDocument, DocumentOrigin, ImageRef, - SectionHeaderItem, Size, ) from PIL import Image as PILImage from docling.utils.chandra_utils import _add_html_fragment - -if TYPE_CHECKING: - import marko +from docling.utils.markdown_math import create_markdown_parser +from docling.utils.vlm_utils import strip_math_delimiters _MARKER = re.compile( r"^ {0,3}!\[([a-z_]+)\+?\]\(([^)\n]*,[^)\n]*,[^)\n]*,[^)\n]*)\)[ \t]*(.*)$" ) -_MATH = re.compile( - r"(? Iterator[tuple[str, str, str]]: yield label, bbox, "".join(lines) -def _math_html(text: str, *, display: bool) -> str: - """Keep LaTeX literal and let Docling's serializers supply its delimiters.""" - text = text.strip() - for opening, closing in (("$$", "$$"), (r"\[", r"\]"), (r"\(", r"\)"), ("$", "$")): - if ( - len(text) >= len(opening) + len(closing) - and text.startswith(opening) - and text.endswith(closing) - ): - text = text[len(opening) : -len(closing)].strip() - break - attrs = ' display="block"' if display else "" - return f"{html.escape(text)}" - - -def _markdown_parser() -> marko.Markdown: - # Markdown support is optional in docling-slim installations. - try: - import marko - except ImportError as exc: - raise ImportError( - "LightOnOCR Markdown parsing requires " - "`pip install 'docling-slim[format-markdown]'`." - ) from exc - from marko.helpers import MarkoExtension - - class Math(marko.inline.InlineElement): - children: str - pattern = _MATH - priority = 8 - parse_group = 0 - - class CodeSpan(marko.inline.CodeSpan): - override = True - priority = 9 - - class MathRenderer: - def render_math(self, element: Math) -> str: - return _math_html( - element.children, - display=element.children.startswith(("$$", r"\[")), - ) - - return marko.Markdown( - extensions=[ - "gfm", - MarkoExtension(elements=[Math, CodeSpan], renderer_mixins=[MathRenderer]), - ] - ) - - def parse_lightonocr_markdown( content: str, original_page_size: Size, @@ -146,7 +87,7 @@ def parse_lightonocr_markdown( Continuation labels retain their base class. Separate blocks are not merged or associated by proximity. Plain transcription is recovered without boxes. """ - markdown = _markdown_parser() + markdown = create_markdown_parser() # The parser above checks this optional dependency before its types are used. from marko.block import Heading, SetextHeading @@ -176,7 +117,7 @@ def parse_lightonocr_markdown( continue layout_label = _LABELS.get(label, "Text") if label == "formula": - rendered = _math_html(body, display=True) + rendered = f'{html.escape(strip_math_delimiters(body))}' elif label in {"image", "header_image", "footer_image"}: rendered = f'{html.escape(body.strip(), quote=True)}' else: @@ -202,8 +143,4 @@ def parse_lightonocr_markdown( picture.meta.description.created_by = "lightonocr" if label in {"header_image", "footer_image"}: picture.content_layer = ContentLayer.FURNITURE - # The HTML mapper uses h1-h6 levels; Docling reserves the top level for titles. - for item in doc.texts: - if isinstance(item, SectionHeaderItem) and item.level > 1: - item.level -= 1 return doc diff --git a/docling/utils/markdown_math.py b/docling/utils/markdown_math.py new file mode 100644 index 0000000000..4ae1f6f4ba --- /dev/null +++ b/docling/utils/markdown_math.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Math-aware Markdown rendering for native VLM response adapters.""" + +from __future__ import annotations + +import html +import re +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import marko + +_MATH = re.compile( + r"(? marko.Markdown: + """Render LaTeX as literal math nodes and keep code and currency literal. + + ``inline_only`` leaves the enclosing native block type in control instead + of interpreting its text as new Markdown lists, headings, or tables. + """ + # Markdown is an optional dependency for slim installations. + try: + import marko + except ImportError as exc: + raise ImportError( + "Markdown parsing requires `pip install 'docling-slim[format-markdown]'`." + ) from exc + from marko.helpers import MarkoExtension + + class Math(marko.inline.InlineElement): + children: str + pattern = _MATH + priority = 8 + parse_group = 0 + + class CodeSpan(marko.inline.CodeSpan): + override = True + priority = 9 + + class LiteralCode(marko.inline.InlineElement): + children: str + pattern = re.compile(r"]*>[\s\S]*?", re.IGNORECASE) + priority = 10 + parse_group = 0 + + class MathRenderer: + def render_math(self, element: Math) -> str: + match = _MATH.fullmatch(element.children) + assert match is not None + latex = next(group for group in match.groups() if group is not None) + display = match.group(1) is not None or match.group(4) is not None + attrs = ' display="block"' if display else "" + return f"{html.escape(latex.strip())}" + + def render_literal_code(self, element: LiteralCode) -> str: + return element.children + + class Paragraph(marko.block.Paragraph): + override = True + + @classmethod + def break_paragraph( + cls, source: marko.source.Source, lazy: bool = False + ) -> bool: + return False + + class InlineParser(marko.Parser): + def __init__(self) -> None: + super().__init__() + self.block_elements = { + name: element + for name, element in self.block_elements.items() + if name in {"Document", "Paragraph", "BlankLine"} + } + + return marko.Markdown( + parser=InlineParser if inline_only else marko.Parser, + extensions=[ + "gfm", + MarkoExtension( + elements=[ + Math, + CodeSpan, + LiteralCode, + *([Paragraph] if inline_only else []), + ], + renderer_mixins=[MathRenderer], + ), + ], + ) From 41c954f0d1abf9aa48ecb7a335d6bac5aad201db Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Mon, 19 Oct 2026 23:08:01 +0200 Subject: [PATCH 6/7] fix: keep shared inline math parser free of GFM side effects Signed-off-by: Pierre Mesure --- docling/utils/markdown_math.py | 4 +++- tests/test_markdown_math.py | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/test_markdown_math.py diff --git a/docling/utils/markdown_math.py b/docling/utils/markdown_math.py index 4ae1f6f4ba..9612ad78c6 100644 --- a/docling/utils/markdown_math.py +++ b/docling/utils/markdown_math.py @@ -83,7 +83,9 @@ def __init__(self) -> None: return marko.Markdown( parser=InlineParser if inline_only else marko.Parser, extensions=[ - "gfm", + # GFM adds block elements after InlineParser filters them, and its + # strikethrough and autolinks change literal native block content. + *([] if inline_only else ["gfm"]), MarkoExtension( elements=[ Math, diff --git a/tests/test_markdown_math.py b/tests/test_markdown_math.py new file mode 100644 index 0000000000..037fad62fc --- /dev/null +++ b/tests/test_markdown_math.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +import pytest + +from docling.utils.markdown_math import create_markdown_parser + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + "about ~5 m and ~10 m, where $x$ is the length", + "about ~5 m and ~10 m, where x is the length", + ), + ( + "| A | B |\n| --- | --- |\n| $x$ | y |", + "| A | B |\n| --- | --- |\n| x | y |", + ), + ("> [!NOTE]\n> Value $x$.", "> [!NOTE]\n> Value x."), + ( + "See www.example.com and $x$.", + "See www.example.com and x.", + ), + ], +) +def test_inline_math_preserves_native_text_syntax(source: str, expected: str) -> None: + assert create_markdown_parser(inline_only=True).convert(source) == ( + f"

{expected}

\n" + ) + + +def test_full_markdown_still_parses_tables_with_math() -> None: + rendered = create_markdown_parser().convert("| A | B |\n| --- | --- |\n| $x$ | y |") + assert "" in rendered + assert "" in rendered From 7173aab5085d55395cce0bacf7d7bb13ebdb47dd Mon Sep 17 00:00:00 2001 From: Pierre Mesure Date: Thu, 22 Oct 2026 19:39:23 +0200 Subject: [PATCH 7/7] fix: normalize LightOnOCR heading levels after joining pages Signed-off-by: Pierre Mesure --- docling/pipeline/vlm_pipeline.py | 2 + docling/utils/lightonocr_utils.py | 11 +++- docling/utils/markdown_math.py | 7 ++- docs/usage/vision_models.md | 3 +- tests/test_lightonocr3_heading_levels.py | 68 ++++++++++++++++++++++++ tests/test_markdown_math.py | 4 ++ 6 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/test_lightonocr3_heading_levels.py diff --git a/docling/pipeline/vlm_pipeline.py b/docling/pipeline/vlm_pipeline.py index b0573e9e88..e83954b077 100644 --- a/docling/pipeline/vlm_pipeline.py +++ b/docling/pipeline/vlm_pipeline.py @@ -79,6 +79,7 @@ ResponseFormat.CHANDRA_HTML, ResponseFormat.DEEPSEEKOCR_MARKDOWN, ResponseFormat.DOTS_JSON, + ResponseFormat.LIGHTONOCR_MARKDOWN, ResponseFormat.NEMOTRON_PARSE_V2, ResponseFormat.NEMOTRON_PARSE_V3_5, ResponseFormat.UNLIMITED_OCR_MARKDOWN, @@ -568,6 +569,7 @@ def _finalize_page_document( page_no=page.page_no, filename=conv_res.input.file.name or "file", page_image=page.image, + normalize_heading_levels=False, ) elif response_format == ResponseFormat.MARKDOWN: document = self._convert_text_page( diff --git a/docling/utils/lightonocr_utils.py b/docling/utils/lightonocr_utils.py index 79b455cda6..ac529d763b 100644 --- a/docling/utils/lightonocr_utils.py +++ b/docling/utils/lightonocr_utils.py @@ -22,7 +22,10 @@ from docling.utils.chandra_utils import _add_html_fragment from docling.utils.markdown_math import create_markdown_parser -from docling.utils.vlm_utils import strip_math_delimiters +from docling.utils.vlm_utils import ( + normalize_section_header_levels, + strip_math_delimiters, +) _MARKER = re.compile( r"^ {0,3}!\[([a-z_]+)\+?\]\(([^)\n]*,[^)\n]*,[^)\n]*,[^)\n]*)\)[ \t]*(.*)$" @@ -81,11 +84,15 @@ def parse_lightonocr_markdown( page_no: int, filename: str = "file", page_image: PILImage.Image | None = None, + *, + normalize_heading_levels: bool = True, ) -> DoclingDocument: """Preserve Markdown/HTML content and assign each block bbox at most once. Continuation labels retain their base class. Separate blocks are not merged or associated by proximity. Plain transcription is recovered without boxes. + Heading depths normalize per page unless the pipeline defers normalization + until all pages are joined with ``normalize_heading_levels=False``. """ markdown = create_markdown_parser() # The parser above checks this optional dependency before its types are used. @@ -143,4 +150,6 @@ def parse_lightonocr_markdown( picture.meta.description.created_by = "lightonocr" if label in {"header_image", "footer_image"}: picture.content_layer = ContentLayer.FURNITURE + if normalize_heading_levels: + normalize_section_header_levels(doc) return doc diff --git a/docling/utils/markdown_math.py b/docling/utils/markdown_math.py index 9612ad78c6..4241c664a2 100644 --- a/docling/utils/markdown_math.py +++ b/docling/utils/markdown_math.py @@ -14,7 +14,8 @@ _MATH = re.compile( r"(? bool: return False + @staticmethod + def is_setext_heading(line: str) -> bool: + return False + class InlineParser(marko.Parser): def __init__(self) -> None: super().__init__() diff --git a/docs/usage/vision_models.md b/docs/usage/vision_models.md index 10750ef9c0..d44a3cb505 100644 --- a/docs/usage/vision_models.md +++ b/docs/usage/vision_models.md @@ -150,7 +150,8 @@ pipeline_options = VlmPipelineOptions( Pass these options to the `PdfFormatOption` in the example above. The presets use the `grounding` prompt. Docling parses the block labels and normalized bounding boxes, Markdown text, HTML tables, image descriptions, and chart tables. -Markdown section heading levels and LaTeX math are preserved. +Section headings use the document-wide depth normalization described below. +LaTeX math is preserved. Coordinates belong to blocks; derived text and table cells do not receive separate boxes. Continuation blocks keep their base labels without being merged. Lists keep their structure, without assigning the block box to individual items. diff --git a/tests/test_lightonocr3_heading_levels.py b/tests/test_lightonocr3_heading_levels.py new file mode 100644 index 0000000000..8154df16dd --- /dev/null +++ b/tests/test_lightonocr3_heading_levels.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +from pathlib import PurePath +from types import SimpleNamespace + +import pytest +from docling_core.types.doc import SectionHeaderItem, Size + +from docling.datamodel.base_models import Page, PagePredictions, VlmPrediction +from docling.datamodel.pipeline_options import VlmConvertOptions, VlmPipelineOptions +from docling.pipeline.vlm_pipeline import VlmPipeline +from docling.utils.lightonocr_utils import parse_lightonocr_markdown + +_SIZE = Size(width=600, height=800) + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_standalone_heading_levels_keep_source_depths_distinct(depth: int) -> None: + source = ( + f"![text](100,100,900,200) {'#' * depth} Parent\n" + f"![text](100,300,900,400) {'#' * (depth + 1)} Child\n" + ) + doc = parse_lightonocr_markdown(source, _SIZE, page_no=2) + assert [ + item.level for item in doc.texts if isinstance(item, SectionHeaderItem) + ] == [ + 1, + 2, + ] + assert doc.export_to_markdown() == "## Parent\n\n### Child" + assert all(item.prov[0].page_no == 2 for item in doc.texts) + + +@pytest.mark.parametrize("has_depth_one", [False, True]) +def test_pipeline_normalizes_heading_levels_once_across_pages( + has_depth_one: bool, +) -> None: + pipeline = VlmPipeline.__new__(VlmPipeline) + pipeline.pipeline_options = VlmPipelineOptions( + vlm_options=VlmConvertOptions.from_preset("lightonocr3_08b") + ) + pipeline.force_backend_text = False + conv_res = SimpleNamespace( + input=SimpleNamespace(file=PurePath("test.pdf")), errors=[] + ) + first = "![text](100,100,900,200) # Chapter\n" if has_depth_one else "" + pages = [] + for page_no, source in ( + (1, first + "![title](100,300,900,400) ## Section"), + (2, "![title](100,100,900,200) ## Next section"), + ): + page = Page( + page_no=page_no, + size=_SIZE, + predictions=PagePredictions(vlm_response=VlmPrediction(text=source)), + ) + pages.append((page_no, pipeline._finalize_page_document(conv_res, page))) + document = pipeline._join_page_documents(pages) + headings = [item for item in document.texts if isinstance(item, SectionHeaderItem)] + expected = [(1, 1), (1, 2), (2, 2)] if has_depth_one else [(1, 1), (2, 1)] + assert [(item.prov[0].page_no, item.level) for item in headings] == expected + assert document.export_to_markdown() == ( + "## Chapter\n\n### Section\n\n### Next section" + if has_depth_one + else "## Section\n\n## Next section" + ) + assert not conv_res.errors diff --git a/tests/test_markdown_math.py b/tests/test_markdown_math.py index 037fad62fc..68a6b9668c 100644 --- a/tests/test_markdown_math.py +++ b/tests/test_markdown_math.py @@ -9,6 +9,10 @@ @pytest.mark.parametrize( ("source", "expected"), [ + ( + "Price $5-$10 and $x$; the answer is $5$ or $2+2$.", + "Price $5-$10 and x; the answer is 5 or 2+2.", + ), ( "about ~5 m and ~10 m, where $x$ is the length", "about ~5 m and ~10 m, where x is the length",
x