feat: add LightOnOCR-3 presets and grounded output parsing

docling/.agents/skills/docling/references/cli.md

@@ -52,6 +52,7 @@ docling report.pdf --pipeline vlm --vlm-model granite_docling --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model smoldocling --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model nemotron_parse_v2 --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model nemotron_parse_v3_5 --trust-remote-code --output /tmp/
+docling report.pdf --pipeline vlm --vlm-model lightonocr3_08b --output /tmp/
 docling chart.png --from image --to dclx --pipeline vlm --enrich-chart-extraction
 docling report.pdf --pipeline native --from pdf --output /tmp/
 ```

docling/datamodel/pipeline_options.py

@@ -1232,6 +1232,9 @@ class CodeFormulaVlmOptions(StagePresetMixin, VlmEngineOptionsMixin, BaseModel):
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_DOLPHIN)
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_GLMOCR)
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR)
+VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_08B)
+VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_1B)
+VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_LIGHTONOCR_3_4B)
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_FALCON_OCR)
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_CHANDRA_OCR2)
 VlmConvertOptions.register_preset(stage_model_specs.VLM_CONVERT_UNLIMITED_OCR)

docling/datamodel/pipeline_options_vlm_model.py

@@ -107,6 +107,7 @@ class ResponseFormat(str, Enum):
     MARKDOWN = "markdown"
     DEEPSEEKOCR_MARKDOWN = "deepseekocr_markdown"
     UNLIMITED_OCR_MARKDOWN = "unlimited_ocr_markdown"
+    LIGHTONOCR_MARKDOWN = "lightonocr_markdown"
     HTML = "html"
     OTSL = "otsl"
     PLAINTEXT = "plaintext"

docling/datamodel/stage_model_specs.py

@@ -1697,6 +1697,50 @@ def from_preset(
     default_engine_type=VlmEngineType.AUTO_INLINE,
 )
 
+
+def _lightonocr3_preset(size: str, preset_id: str) -> StageModelPreset:
+    name = f"LightOnOCR-3-{size}"
+    repo_id = f"lightonai/{name}"
+    return StageModelPreset(
+        preset_id=preset_id,
+        name=name,
+        description=f"LightOnOCR-3 OCR with block grounding ({size} parameters)",
+        model_spec=VlmModelSpec(
+            name=name,
+            default_repo_id=repo_id,
+            prompt="grounding",
+            response_format=ResponseFormat.LIGHTONOCR_MARKDOWN,
+            max_new_tokens=4096,
+            engine_overrides={
+                VlmEngineType.TRANSFORMERS: EngineModelConfig(
+                    min_engine_version="5.5.4",
+                    torch_dtype="bfloat16",
+                    extra_config={
+                        "transformers_model_type": TransformersModelType.AUTOMODEL_IMAGETEXTTOTEXT,
+                        "transformers_prompt_style": TransformersPromptStyle.CHAT,
+                        "extra_generation_config": {"skip_special_tokens": True},
+                    },
+                ),
+            },
+            api_overrides={
+                VlmEngineType.API: ApiModelConfig(
+                    params={"model": repo_id, "max_tokens": 4096}
+                ),
+                VlmEngineType.API_OPENAI: ApiModelConfig(
+                    params={"model": repo_id, "max_tokens": 4096}
+                ),
+            },
+        ),
+        scale=2.0,
+        max_size=1540,
+        default_engine_type=VlmEngineType.AUTO_INLINE,
+    )
+
+
+VLM_CONVERT_LIGHTONOCR_3_08B = _lightonocr3_preset("0.8B", "lightonocr3_08b")
+VLM_CONVERT_LIGHTONOCR_3_1B = _lightonocr3_preset("1B", "lightonocr3_1b")
+VLM_CONVERT_LIGHTONOCR_3_4B = _lightonocr3_preset("4B", "lightonocr3_4b")
+
 # -----------------------------------------------------------------------------
 # PICTURE_DESCRIPTION PRESETS (for image captioning/description)
 # -----------------------------------------------------------------------------

docling/pipeline/vlm_pipeline.py

@@ -79,6 +79,7 @@
         ResponseFormat.CHANDRA_HTML,
         ResponseFormat.DEEPSEEKOCR_MARKDOWN,
         ResponseFormat.DOTS_JSON,
+        ResponseFormat.LIGHTONOCR_MARKDOWN,
         ResponseFormat.NEMOTRON_PARSE_V2,
         ResponseFormat.NEMOTRON_PARSE_V3_5,
         ResponseFormat.UNLIMITED_OCR_MARKDOWN,
@@ -559,6 +560,17 @@ def _finalize_page_document(
                 page_image=page.image,
                 normalize_heading_levels=False,
             )
+        elif response_format == ResponseFormat.LIGHTONOCR_MARKDOWN:
+            from docling.utils.lightonocr_utils import parse_lightonocr_markdown
+
+            document = parse_lightonocr_markdown(
+                content=predicted_text,
+                original_page_size=page.size,
+                page_no=page.page_no,
+                filename=conv_res.input.file.name or "file",
+                page_image=page.image,
+                normalize_heading_levels=False,
+            )
         elif response_format == ResponseFormat.MARKDOWN:
             document = self._convert_text_page(
                 conv_res, page, InputFormat.MD, MarkdownDocumentBackend

docling/utils/lightonocr_utils.py

@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: The Docling Contributors
+# SPDX-License-Identifier: MIT
+
+"""Parse LightOnOCR-3 Markdown with normalized block bounding boxes."""
+
+from __future__ import annotations
+
+import html
+import re
+from collections.abc import Iterator
+from pathlib import Path
+
+from docling_core.types.doc import (
+    ContentLayer,
+    DocItemLabel,
+    DoclingDocument,
+    DocumentOrigin,
+    ImageRef,
+    Size,
+)
+from PIL import Image as PILImage
+
+from docling.utils.chandra_utils import _add_html_fragment
+from docling.utils.markdown_math import create_markdown_parser
+from docling.utils.vlm_utils import (
+    normalize_section_header_levels,
+    strip_math_delimiters,
+)
+
+_MARKER = re.compile(
+    r"^ {0,3}!\[([a-z_]+)\+?\]\(([^)\n]*,[^)\n]*,[^)\n]*,[^)\n]*)\)[ \t]*(.*)$"
+)
+_LABELS = {
+    "title": "Title",
+    "header": "Page-Header",
+    "footer": "Page-Footer",
+    "page_number": "Page-Footer",
+    "footnote": "Footnote",
+    "caption": "Caption",
+    "formula": "Equation-Block",
+    "code": "Code-Block",
+    "table": "Table",
+    "list": "List-Group",
+    "image": "Image",
+    "chart": "Figure",
+    "header_image": "Image",
+    "footer_image": "Image",
+}
+
+
+def _blocks(content: str) -> Iterator[tuple[str, str, str]]:
+    label, bbox = "text", ""
+    lines: list[str] = []
+    fence = ""
+    for line in content.splitlines(keepends=True):
+        marker = _MARKER.fullmatch(line.rstrip("\r\n")) if not fence else None
+        if marker is not None:
+            if lines or bbox:
+                yield label, bbox, "".join(lines)
+            label, bbox, inline_body = marker.groups()
+            lines = []
+            line = inline_body + "\n" if inline_body else ""
+
+        # Check the block body too: a fence can start on the marker's line.
+        fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})", line)
+        if fence_match:
+            delimiter = fence_match[1]
+            if not fence:
+                fence = delimiter
+            elif (
+                delimiter[0] == fence[0]
+                and len(delimiter) >= len(fence)
+                and not line[fence_match.end() :].strip()
+            ):
+                fence = ""
+        lines.append(line)
+    if lines or bbox:
+        yield label, bbox, "".join(lines)
+
+
+def parse_lightonocr_markdown(
+    content: str,
+    original_page_size: Size,
+    page_no: int,
+    filename: str = "file",
+    page_image: PILImage.Image | None = None,
+    *,
+    normalize_heading_levels: bool = True,
+) -> DoclingDocument:
+    """Preserve Markdown/HTML content and assign each block bbox at most once.
+
+    Continuation labels retain their base class. Separate blocks are not merged
+    or associated by proximity. Plain transcription is recovered without boxes.
+    Heading depths normalize per page unless the pipeline defers normalization
+    until all pages are joined with ``normalize_heading_levels=False``.
+    """
+    markdown = create_markdown_parser()
+    # The parser above checks this optional dependency before its types are used.
+    from marko.block import Heading, SetextHeading
+
+    doc = DoclingDocument(
+        name=Path(filename).stem,
+        origin=DocumentOrigin(
+            filename=filename, mimetype="text/markdown", binary_hash=0
+        ),
+    )
+    dpi = (
+        max(1, round(72 * page_image.width / original_page_size.width))
+        if page_image is not None
+        else 72
+    )
+    doc.add_page(
+        page_no=page_no,
+        size=original_page_size,
+        image=ImageRef.from_pil(image=page_image, dpi=dpi) if page_image else None,
+    )
+    for label, bbox, body in _blocks(content):
+        if not body.strip() and label not in {
+            "image",
+            "chart",
+            "header_image",
+            "footer_image",
+        }:
+            continue
+        layout_label = _LABELS.get(label, "Text")
+        if label == "formula":
+            rendered = f'<math display="block">{html.escape(strip_math_delimiters(body))}</math>'
+        elif label in {"image", "header_image", "footer_image"}:
+            rendered = f'<img alt="{html.escape(body.strip(), quote=True)}">'
+        else:
+            parsed = markdown.parse(body)
+            rendered = markdown.render(parsed)
+            if label == "title" and any(
+                isinstance(node, (Heading, SetextHeading)) and node.level > 1
+                for node in parsed.children
+            ):
+                layout_label = "Section-Header"
+        attrs = (
+            f'data-bbox="{html.escape(bbox.replace(",", " "), quote=True)}"'
+            if bbox
+            else ""
+        )
+        if bbox:
+            attrs += f' data-label="{layout_label}"'
+        fragment = f"<div {attrs}>{rendered}</div>"
+        picture_start = len(doc.pictures)
+        _add_html_fragment(doc, fragment, label=DocItemLabel.TEXT)
+        for picture in doc.pictures[picture_start:]:
+            if picture.meta is not None and picture.meta.description is not None:
+                picture.meta.description.created_by = "lightonocr"
+            if label in {"header_image", "footer_image"}:
+                picture.content_layer = ContentLayer.FURNITURE
+    if normalize_heading_levels:
+        normalize_section_header_levels(doc)
+    return doc

docling/utils/markdown_math.py

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: The Docling Contributors
+# SPDX-License-Identifier: MIT
+
+"""Math-aware Markdown rendering for native VLM response adapters."""
+
+from __future__ import annotations
+
+import html
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import marko
+
+_MATH = re.compile(
+    r"(?<!\\)(?:\$\$([\s\S]*?)(?<!\\)\$\$"
+    # A dollar before a digit starts an amount, not a closing math delimiter.
+    r"|(?<!\$)\$(?![\s$])([^\n$]*?)(?<![\\\s])\$(?![\d$])"
+    r"|\\\(([\s\S]*?)\\\)|\\\[([\s\S]*?)\\\])"
+)
+
+
+def create_markdown_parser(*, inline_only: bool = False) -> marko.Markdown:
+    """Render LaTeX as literal math nodes and keep code and currency literal.
+
+    ``inline_only`` leaves the enclosing native block type in control instead
+    of interpreting its text as new Markdown lists, headings, or tables.
+    """
+    # Markdown is an optional dependency for slim installations.
+    try:
+        import marko
+    except ImportError as exc:
+        raise ImportError(
+            "Markdown parsing requires `pip install 'docling-slim[format-markdown]'`."
+        ) from exc
+    from marko.helpers import MarkoExtension
+
+    class Math(marko.inline.InlineElement):
+        children: str
+        pattern = _MATH
+        priority = 8
+        parse_group = 0
+
+    class CodeSpan(marko.inline.CodeSpan):
+        override = True
+        priority = 9
+
+    class LiteralCode(marko.inline.InlineElement):
+        children: str
+        pattern = re.compile(r"<code\b[^>]*>[\s\S]*?</code>", re.IGNORECASE)
+        priority = 10
+        parse_group = 0
+
+    class MathRenderer:
+        def render_math(self, element: Math) -> str:
+            match = _MATH.fullmatch(element.children)
+            assert match is not None
+            latex = next(group for group in match.groups() if group is not None)
+            display = match.group(1) is not None or match.group(4) is not None
+            attrs = ' display="block"' if display else ""
+            return f"<math{attrs}>{html.escape(latex.strip())}</math>"
+
+        def render_literal_code(self, element: LiteralCode) -> str:
+            return element.children
+
+    class Paragraph(marko.block.Paragraph):
+        override = True
+
+        @classmethod
+        def break_paragraph(
+            cls, source: marko.source.Source, lazy: bool = False
+        ) -> bool:
+            return False
+
+        @staticmethod
+        def is_setext_heading(line: str) -> bool:
+            return False
+
+    class InlineParser(marko.Parser):
+        def __init__(self) -> None:
+            super().__init__()
+            self.block_elements = {
+                name: element
+                for name, element in self.block_elements.items()
+                if name in {"Document", "Paragraph", "BlankLine"}
+            }
+
+    return marko.Markdown(
+        parser=InlineParser if inline_only else marko.Parser,
+        extensions=[
+            # GFM adds block elements after InlineParser filters them, and its
+            # strikethrough and autolinks change literal native block content.
+            *([] if inline_only else ["gfm"]),
+            MarkoExtension(
+                elements=[
+                    Math,
+                    CodeSpan,
+                    LiteralCode,
+                    *([Paragraph] if inline_only else []),
+                ],
+                renderer_mixins=[MathRenderer],
+            ),
+        ],
+    )