fix(latex): backend updates for better parsing of arxiv papers

.gitignore

@@ -1,6 +1,10 @@
 model_artifacts/
 scratch/
 ds_convert_models/
+/latex-failures.jsonl
+/latex-manifest.jsonl
+/latex-failures/
+/latex-success/
 output/
 
 # Created by https://www.toptal.com/developers/gitignore/api/python,macos,virtualenv,pycharm,visualstudiocode,emacs,vim,jupyternotebooks
@@ -11,6 +15,7 @@ output/
 *~
 *.dclx
 *.dclg
+!tests/data/latex/groundtruth/*.dclg
 *.xml
 *.zip
 \#*\#

docling/backend/latex/backend.py

@@ -3,22 +3,26 @@
 
 import logging
 import os
+import re
 import threading
 from io import BytesIO
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union, cast
 
-from docling_core.types.doc import DocItemLabel, DoclingDocument, NodeItem
+from docling_core.types.doc import ContentLayer, DocItemLabel, DoclingDocument, NodeItem
 from docling_core.types.doc.document import Formatting
 
 from docling.backend.abstract_backend import DeclarativeDocumentBackend
+from docling.backend.latex.constants import MACROS_IGNORED
 from docling.backend.latex.handlers.environments import EnvironmentHandlerMixin
 from docling.backend.latex.handlers.macros import MacroHandlerMixin
-from docling.backend.latex.handlers.math import MathHandlerMixin
+from docling.backend.latex.handlers.math import InlineFormula, MathHandlerMixin
+from docling.backend.latex.handlers.metadata import LatexMetadataHandler
 from docling.backend.latex.utils.encoding import decode_latex_content
 from docling.backend.latex.utils.latex_context import LATEX_CONTEXT_DB
 from docling.backend.latex.utils.table import TableHelperMixin
-from docling.backend.latex.utils.text import TextHelperMixin
+from docling.backend.latex.utils.text import TextHelperMixin, iter_child_nodelists
+from docling.backend.latex.utils.whitespace import collapse_source_layout_whitespace
 from docling.datamodel.backend_options import LatexBackendOptions
 from docling.datamodel.base_models import InputFormat
 from docling.datamodel.document import InputDocument
@@ -36,6 +40,7 @@
         LatexMathNode,
         LatexWalker,
     )
+    from pylatexenc.macrospec import MacroSpec
 
     _PYLATEXENC_AVAILABLE = True
 except ImportError as e:  # pragma: no cover - import-time guard
@@ -82,6 +87,9 @@ def __init__(
         self.labels: dict[str, bool] = {}
         self._custom_macros: dict[str, str] = {}
         self._custom_macro_num_args: dict[str, int] = {}
+        self._metadata_handler = LatexMetadataHandler(self)
+        self._masked_macro_spans: list[tuple[int, int]] = []
+        self._suppress_image_path_caption = False
         self._input_stack: set[str] = set()
         self._tectonic_engine: TectonicEngine | None = None
         self._tikz_executor: concurrent.futures.ThreadPoolExecutor | None = None
@@ -104,7 +112,6 @@ def supported_formats(cls) -> set[InputFormat]:
 
     def _do_parse_and_process(self, doc: DoclingDocument) -> DoclingDocument:
         import concurrent.futures
-        import re
 
         preprocessed_text = self._preprocess_custom_macros(self.latex_text)
 
@@ -121,8 +128,18 @@ def _do_parse_and_process(self, doc: DoclingDocument) -> DoclingDocument:
         else:
             self.latex_preamble = ""
 
+        parser_context = LATEX_CONTEXT_DB.filter_context()
+        zero_arg_macros = [
+            MacroSpec(name, "")
+            for name, count in self._custom_macro_num_args.items()
+            if count == 0
+        ]
+        if zero_arg_macros:
+            parser_context.add_context_category(
+                "docling-source-macros", macros=zero_arg_macros, prepend=True
+            )
         walker = LatexWalker(
-            preprocessed_text, tolerant_parsing=True, latex_context=LATEX_CONTEXT_DB
+            preprocessed_text, tolerant_parsing=True, latex_context=parser_context
         )
 
         try:
@@ -132,13 +149,15 @@ def _do_parse_and_process(self, doc: DoclingDocument) -> DoclingDocument:
             doc.add_text(label=DocItemLabel.TEXT, text=self.latex_text)
             return doc
 
+        self._preserve_source_commands(nodes, doc)
+
         workers = max(1, (os.cpu_count() or 2) - 1)
         self._tikz_executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
         self._tikz_futures = []  # type: ignore
 
         try:
             self._extract_custom_macros(nodes)
-            self._extract_preamble_metadata(nodes, doc)
+            self._metadata_handler.process_preamble(nodes, doc)
 
             doc_node = self._find_document_env(nodes)
 
@@ -157,13 +176,49 @@ def _do_parse_and_process(self, doc: DoclingDocument) -> DoclingDocument:
 
         return doc
 
+    def _preserve_source_commands(self, nodes, doc: DoclingDocument) -> None:
+        """Keep source commands in a layer supported by DocLang."""
+        commands = [
+            (start, self.latex_text[start:end])
+            for start, end in self._masked_macro_spans
+        ]
+
+        def visit(items, in_preamble: bool) -> None:
+            for node in items:
+                if (
+                    isinstance(node, LatexMacroNode)
+                    and node.macroname in MACROS_IGNORED
+                ):
+                    if in_preamble or node.macroname.startswith("new"):
+                        commands.append((node.pos, node.latex_verbatim()))
+                if (
+                    isinstance(node, LatexEnvironmentNode)
+                    and node.envname == "document"
+                ):
+                    visit(node.nodelist or [], False)
+                    in_preamble = False
+                elif not (
+                    isinstance(node, LatexEnvironmentNode)
+                    and node.envname in {"verbatim", "lstlisting", "minted"}
+                ):
+                    for children in iter_child_nodelists(node):
+                        visit(children, in_preamble)
+
+        visit(nodes, True)
+        for _, command in sorted(commands):
+            doc.add_text(
+                label=DocItemLabel.TEXT,
+                text=command,
+                content_layer=ContentLayer.FURNITURE,
+            )
+
     def convert(self) -> DoclingDocument:
         doc = DoclingDocument(name=self.file.stem)
         opts = cast(LatexBackendOptions, self.options)
         timeout = opts.parse_timeout
 
         if timeout is None:
-            return self._do_parse_and_process(doc)
+            return self._normalize_document_text(self._do_parse_and_process(doc))
 
         result_container: list[DoclingDocument] = []
         error_container: list[Exception] = []
@@ -187,14 +242,31 @@ def _worker_fn():
             )
             fallback = DoclingDocument(name=self.file.stem)
             fallback.add_text(label=DocItemLabel.TEXT, text=self.latex_text)
-            return fallback
+            return self._normalize_document_text(fallback)
 
         if error_container:
             _log.error(f"Error during LaTeX parsing: {error_container[0]}")
-            return doc
+            return self._normalize_document_text(doc)
 
         if result_container:
-            return result_container[0]
+            return self._normalize_document_text(result_container[0])
+
+        return self._normalize_document_text(doc)
+
+    @staticmethod
+    def _normalize_document_text(doc: DoclingDocument) -> DoclingDocument:
+        for item in doc.texts:
+            if item.content_layer == ContentLayer.FURNITURE or item.label in {
+                DocItemLabel.FORMULA,
+                DocItemLabel.CODE,
+            }:
+                continue
+            item.text = collapse_source_layout_whitespace(item.text)
+            item.orig = collapse_source_layout_whitespace(item.orig)
+
+        for table in doc.tables:
+            for cell in table.data.table_cells:
+                cell.text = collapse_source_layout_whitespace(cell.text)
 
         return doc
 
@@ -205,22 +277,86 @@ def _process_nodes(
         parent: NodeItem | None = None,
         formatting: Formatting | None = None,
         text_label: DocItemLabel | None = None,
+        heading_level: int | None = None,
     ):
         if nodes is None:
             return
 
-        text_buffer: list[str] = []
+        text_buffer: list[str | InlineFormula] = []
 
         def flush_text_buffer():
             if text_buffer:
-                combined_text = "".join(text_buffer).strip()
-                if combined_text:
-                    doc.add_text(
-                        parent=parent,
-                        label=text_label or DocItemLabel.TEXT,
-                        text=combined_text,
-                        formatting=formatting,
-                    )
+                if any(isinstance(part, InlineFormula) for part in text_buffer):
+                    if heading_level is not None:
+                        paragraph = doc.add_heading(
+                            parent=parent,
+                            text="",
+                            level=heading_level,
+                            formatting=formatting,
+                        )
+                    else:
+                        paragraph = doc.add_text(
+                            parent=parent,
+                            label=text_label or DocItemLabel.TEXT,
+                            text="",
+                            formatting=formatting,
+                        )
+                    inline_group = doc.add_inline_group(parent=paragraph)
+                    parts: list[str | InlineFormula] = []
+                    for part in text_buffer:
+                        if (
+                            isinstance(part, str)
+                            and parts
+                            and isinstance(parts[-1], str)
+                        ):
+                            parts[-1] += part
+                        else:
+                            parts.append(part)
+                    if isinstance(parts[0], str):
+                        parts[0] = parts[0].lstrip()
+                    if isinstance(parts[-1], str):
+                        parts[-1] = parts[-1].rstrip()
+                    text_run = ""
+                    for part in parts:
+                        if isinstance(part, InlineFormula):
+                            if text_run:
+                                doc.add_text(
+                                    parent=inline_group,
+                                    label=DocItemLabel.TEXT,
+                                    text=text_run,
+                                )
+                                text_run = ""
+                            doc.add_formula(
+                                parent=inline_group,
+                                text=collapse_source_layout_whitespace(
+                                    part.text
+                                ).strip(),
+                            )
+                        else:
+                            text_run += part
+                    if text_run:
+                        doc.add_text(
+                            parent=inline_group,
+                            label=DocItemLabel.TEXT,
+                            text=text_run,
+                        )
+                else:
+                    combined_text = "".join(text_buffer).strip()
+                    if combined_text:
+                        if heading_level is not None:
+                            doc.add_heading(
+                                parent=parent,
+                                text=combined_text,
+                                level=heading_level,
+                                formatting=formatting,
+                            )
+                        else:
+                            doc.add_text(
+                                parent=parent,
+                                label=text_label or DocItemLabel.TEXT,
+                                text=combined_text,
+                                formatting=formatting,
+                            )
                 text_buffer.clear()
 
         idx = 0
@@ -239,21 +375,45 @@ def flush_text_buffer():
                         flush_text_buffer,
                     )
 
+                elif isinstance(node, InlineFormula):
+                    text_buffer.append(node)
+
                 elif isinstance(node, LatexMacroNode):
-                    consumed_following = self._process_macro_node_inline(
-                        node,
-                        doc,
-                        parent,
-                        formatting,
-                        text_label,
-                        text_buffer,
-                        flush_text_buffer,
-                        nodes[idx + 1 :],
-                    )
+                    inline_args = self._inline_math_argument_nodes(node)
+                    if inline_args is not None:
+                        self._append_inline_nodes(
+                            inline_args,
+                            doc,
+                            parent,
+                            formatting,
+                            text_label,
+                            text_buffer,
+                            flush_text_buffer,
+                        )
+                    else:
+                        consumed_following = self._process_macro_node_inline(
+                            node,
+                            doc,
+                            parent,
+                            formatting,
+                            text_label,
+                            text_buffer,
+                            flush_text_buffer,
+                            nodes[idx + 1 :],
+                        )
 
                 elif isinstance(node, LatexEnvironmentNode):
-                    flush_text_buffer()
-                    self._process_environment(node, doc, parent, formatting, text_label)
+                    if node.envname == "math":
+                        math_text = self._clean_math(
+                            node.latex_verbatim(), node.envname
+                        )
+                        if math_text:
+                            text_buffer.append(InlineFormula(math_text))
+                    else:
+                        flush_text_buffer()
+                        self._process_environment(
+                            node, doc, parent, formatting, text_label
+                        )
 
                 elif isinstance(node, LatexMathNode):
                     self._process_math_node(