feat: make inline serializers concatenate runs faithfully

docling_core/compat.py

@@ -179,7 +179,9 @@ def project_to(doc: DoclingDocument, target_version: str) -> DoclingDocument:
     # The current SDK's version validator normalises the version field to
     # CURRENT_VERSION on success.  We restore the target version afterwards so
     # callers and downstream serialisation correctly reflect the projected schema.
-    result = DoclingDocument.model_validate(data)
+    # Validate under the source stamp: the projected stamp would trigger load-time upgrades
+    # (the inline separator migration) on data that is already current.
+    result = DoclingDocument.model_validate({**data, "version": doc.version})
     object.__setattr__(result, "version", data["version"])
     return result
 
@@ -198,6 +200,49 @@ def list_projectors() -> list[tuple[int, int]]:
 # the serialised dict without error.  Full semantic fidelity is not always
 # possible; the goal is a document that validates rather than crashes.
 
+# --- 1.11 → 1.10 ----------------------------------------------------------
+# Schema 1.11 (docling-core v2.105.0): InlineGroup runs carry their own
+# significant whitespace and serializers concatenate them faithfully.
+#
+# The 1.11 and 1.10 schemas are structurally identical -- no new fields,
+# labels, or lists.  The only difference is *semantic*: a 1.11 run's text
+# contains its boundary whitespace, whereas a 1.10 client re-inserts a
+# single " " between runs at serialization time.  Left as-is, a 1.10 client
+# renders every boundary twice and keeps the space inside the markup
+# (``**bold **``, which CommonMark does not render as bold).  The projector
+# therefore strips inline runs to the shape old producers emitted; the old
+# join puts one space back.  1.10 cannot represent a tight run boundary, so
+# ``H2O`` comes back as the old ``H 2 O``.
+# ---------------------------------------------------------------------------
+
+
+@register_projector(from_minor=11, to_minor=10)
+def _project_1_11_to_1_10(data: dict) -> dict:
+    """Downgrade a schema 1.11 document dict to schema 1.10.
+
+    1.11 and 1.10 share an identical structure; the difference is only in how
+    inline-run whitespace is represented. A 1.10 client joins runs with ``" "``,
+    so the text of each inline-group child is stripped. A whitespace-only run
+    becomes ``""``, which old serializers skip, so no refs need renumbering.
+    ``orig`` is left alone.
+    """
+    data = dict(data)
+    data["version"] = "1.10.0"
+    inline_runs = {
+        child.get("$ref", child.get("cref"))
+        for group in data.get("groups", [])
+        if group.get("label") == "inline"
+        for child in group.get("children", [])
+    }
+    data["texts"] = [
+        {**item, "text": item["text"].strip()}
+        if item.get("self_ref") in inline_runs and isinstance(item.get("text"), str)
+        else item
+        for item in data.get("texts", [])
+    ]
+    return data
+
+
 # --- 1.10 → 1.9 -----------------------------------------------------------
 # Schema 1.10 (docling-core v2.69.0, PR #519):
 #   • Added FieldRegionItem, FieldHeadingItem, FieldItem, FieldValueItem

docling_core/transforms/deserializer/doclang.py

@@ -147,6 +147,22 @@ def _repair_unescaped_xml_text(text: str) -> str:
     return "".join(parts)
 
 
+# A whitespace run touching a newline is pretty-print indentation; one without is content.
+_PRETTY_INDENT = re.compile(r"\A\s*\n\s*|\s*\n\s*\Z")
+
+
+def _normalize_bare_text(data: str) -> str:
+    """Drop pretty-print indentation from a bare XML text node, keep significant whitespace.
+
+    Standard XML heuristic: whitespace containing a newline is layout noise, whitespace
+    without one is meaningful. Serialized DocLang uses ``<content>`` for whitespace-bearing
+    runs, so this only governs hand-written and VLM-emitted input.
+    """
+    if not data.strip():
+        return "" if "\n" in data else data
+    return _PRETTY_INDENT.sub("", data)
+
+
 def _enforce_doclang_dom_budgets(
     root: Element,
     *,
@@ -521,7 +537,8 @@ def _parse_text_like(self, *, doc: DoclingDocument, el: Element, parent: NodeIte
 
         prov_list = self._extract_provenance(doc=doc, el=el)
         content_layer = self._extract_layer(el=el)
-        text, formatting = self._extract_text_with_formatting(el)
+        # A run inside an inline group owns its edge whitespace; a block host does not.
+        text, formatting = self._extract_text_with_formatting(el, trim=not isinstance(parent, InlineGroup))
         if not text and not is_empty:
             if (
                 thread_id
@@ -676,17 +693,12 @@ def _extract_code_content_and_language(self, el: Element) -> tuple[str, CodeLang
                     parts.append(node.data)
             elif isinstance(node, Element):
                 nm_child = node.tagName
-                if nm_child in {
-                    DocLangToken.LOCATION.value,
-                    DocLangToken.LAYER.value,
-                    DocLangToken.LABEL.value,
-                    DocLangToken.CAPTION.value,
-                }:
+                if self._is_element_head_tag(node):
                     continue
                 elif nm_child == DocLangToken.BR.value:
                     parts.append("\n")
                 else:
-                    parts.append(self._get_text(node))
+                    parts.append(self._collect_text(node))
 
         return "".join(parts), lang_label
 
@@ -1524,15 +1536,29 @@ def _parse_inline_runs(
         """Add text runs for ``nodes``, carrying ``formatting`` through nested formatting tags."""
         for node in nodes:
             if isinstance(node, Element):
-                if node.tagName in _FORMATTING_TAG_VALUES:
+                if node.tagName == DocLangToken.CONTENT.value:
+                    # <content> is the explicit whitespace-preserving text wrapper: it is a run,
+                    # not a nested item, so it inherits the formatting accumulated on the way in.
+                    # Dispatching it would drop that formatting (<bold><content>x </content></bold>
+                    # would come back unformatted).
+                    text_content = self._collect_text(node)
+                    if text_content:
+                        doc.add_text(
+                            label=DocItemLabel.TEXT,
+                            text=text_content,
+                            parent=parent,
+                            formatting=formatting,
+                        )
+                elif node.tagName in _FORMATTING_TAG_VALUES:
                     # Accumulate this tag's style and keep descending, so e.g. <bold>a
                     # <italic>b</italic></bold> gives "a" bold and "b" bold+italic.
                     merged = self._merge_formatting_tag(node.tagName, formatting)
                     self._parse_inline_runs(doc=doc, nodes=node.childNodes, parent=parent, formatting=merged)
                 else:
                     self._dispatch_element(doc=doc, el=node, parent=parent)
             elif isinstance(node, Text):
-                text_content = node.data.strip()
+                # A run owns its boundary whitespace; only pretty-print indentation is dropped.
+                text_content = _normalize_bare_text(node.data)
                 if text_content:
                     doc.add_text(
                         label=DocItemLabel.TEXT,
@@ -2377,20 +2403,23 @@ def _parse_otsl_table_content(
             table_cells=table_cells,
         )
 
-    def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting | None]:
+    def _extract_text_with_formatting(self, el: Element, *, trim: bool = True) -> tuple[str, Formatting | None]:
         """Extract text content and formatting from an element.
 
         If the element contains a single formatting child (bold, italic, etc.),
         recursively extract the text and build up the Formatting object.
 
+        Args:
+            el: the element to extract from.
+            trim: trim the outer boundary. True for a block-level host, False when ``el`` is a
+                run inside an inline group, whose edge whitespace is significant.
+
         Returns:
             Tuple of (text_content, formatting_object or None)
         """
-        # Get non-whitespace, non-location child elements
+        # Body child elements only -- element-head tags carry metadata, not text.
         child_elements = [
-            node
-            for node in el.childNodes
-            if isinstance(node, Element) and node.tagName not in {DocLangToken.LOCATION.value}
+            node for node in el.childNodes if isinstance(node, Element) and not self._is_element_head_tag(node)
         ]
 
         # Check if we have a single child that is a formatting tag
@@ -2411,7 +2440,7 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting |
 
             if tag_name in format_tags:
                 # Recursively extract text and formatting from the child
-                text, child_formatting = self._extract_text_with_formatting(child)
+                text, child_formatting = self._extract_text_with_formatting(child, trim=trim)
 
                 # Build up the formatting object
                 if child_formatting is None:
@@ -2433,25 +2462,57 @@ def _extract_text_with_formatting(self, el: Element) -> tuple[str, Formatting |
 
                 return text, child_formatting
 
-        # No formatting found, just extract plain text
-        return self._get_text(el), None
+        # No formatting found, just extract plain text.
+        return (self._trimmed_text(el) if trim else self._collect_text(el)), None
 
     def _get_text(self, el: Element) -> str:
-        out: list[str] = []
+        """Extract the text of a block-level element, trimming the block boundary only."""
+        return self._trimmed_text(el)
+
+    def _collect_text(self, el: Element) -> str:
+        """Concatenate descendant text verbatim, per the inline whitespace contract.
+
+        Fragments are joined faithfully -- stripping each one is what collapsed
+        `Advanced <bold>Topics</bold>` into `AdvancedTopics`.
+        """
+        return "".join(text for text, _ in self._collect_fragments(el))
+
+    def _trimmed_text(self, el: Element) -> str:
+        """Concatenate descendant text, trimming the outer block boundary only.
+
+        Whitespace delivered through ``<content>`` is protected: that tag is the format's
+        explicit "this whitespace is significant" marker, so a block edge never strips it.
+        Same shape as docling's ``_normalize_odf_text_runs``.
+        """
+        frags = self._collect_fragments(el)
+        for idx, strip in ((0, str.lstrip), (-1, str.rstrip)):
+            while frags and not frags[idx][1]:
+                if trimmed := strip(frags[idx][0]):
+                    frags[idx] = (trimmed, False)
+                    break
+                frags.pop(idx)
+        return "".join(text for text, _ in frags)
+
+    def _collect_fragments(self, el: Element) -> list[tuple[str, bool]]:
+        """Return ``(text, protected)`` fragments; protected text came from ``<content>``."""
+        out: list[tuple[str, bool]] = []
         for node in el.childNodes:
             if isinstance(node, Text):
-                # Skip pure indentation/pretty-print whitespace
-                if node.data.strip():
-                    out.append(node.data if el.tagName == DocLangToken.CONTENT.value else node.data.strip())
+                if el.tagName == DocLangToken.CONTENT.value:
+                    # `<content>` is the lossless whitespace channel: take it verbatim.
+                    out.append((node.data, True))
+                else:
+                    out.append((_normalize_bare_text(node.data), False))
             elif isinstance(node, Element):
-                nm = node.tagName
-                if nm in {DocLangToken.LOCATION.value, DocLangToken.CUSTOM.value}:
+                # Element-head children (<caption>, <description>, <summary>, <custom>, ...) are
+                # metadata; they are modelled in item.meta and must not bleed into body text.
+                if self._is_element_head_tag(node):
                     continue
-                if nm == DocLangToken.BR.value:
-                    out.append("\n")
+                if node.tagName == DocLangToken.BR.value:
+                    out.append(("\n", False))
                 else:
-                    out.append(self._get_text(node))
-        return "".join(out)
+                    out.extend(self._collect_fragments(node))
+        return out
 
     # --------- Location helpers ---------
     def _ensure_page_exists(self, *, doc: DoclingDocument, page_no: int, resolution: int | None = None) -> None:

docling_core/transforms/serializer/common.py

@@ -8,7 +8,7 @@
 from collections.abc import Iterable
 from functools import cached_property
 from pathlib import Path, PurePath
-from typing import Annotated, Any, Final, Optional, Union
+from typing import Annotated, Any, ClassVar, Final, Optional, Union
 from urllib.parse import quote, urlsplit, urlunsplit
 
 from pydantic import (
@@ -408,6 +408,11 @@ class DocSerializer(BaseModel, BaseDocSerializer):
 
     params: CommonParams = CommonParams()
 
+    # Move whitespace at the edges of a run outside the formatting/hyperlink markers.
+    # Off for formats whose own whitespace channel must keep it inside the markup
+    # (DocLang wraps it in `<content>`), on where adjacency breaks the markup (Markdown).
+    hoist_decoration_whitespace: ClassVar[bool] = False
+
     _excluded_refs_cache: dict[str, set[str]] = {}
 
     @computed_field  # type: ignore[misc]
@@ -755,7 +760,23 @@ def post_process(
     ) -> str:
         """Apply some text post-processing steps."""
         params = self.params.merge_with_patch(patch=kwargs)
+
+        lead = trail = ""
         res = text
+        if self.hoist_decoration_whitespace:
+            # Inline runs carry their own significant whitespace, which may sit *inside* a
+            # formatted run (common in DOCX/HTML). Decorating it verbatim produces
+            # `**bold ** tail`, which CommonMark renders as literal asterisks. Hoist the edge
+            # whitespace around the *complete* decoration stack -- hoisting inside an
+            # individual formatter would let the hyperlink capture it again.
+            core = text.strip()
+            if not core:
+                # Whitespace-only (or empty) run: no markers, no empty hyperlink.
+                return text
+            lead = text[: len(text) - len(text.lstrip())]
+            trail = text[len(text.rstrip()) :]
+            res = core
+
         if params.include_formatting and formatting:
             if formatting.bold:
                 res = self.serialize_bold(text=res, **kwargs)
@@ -774,7 +795,7 @@ def post_process(
                 res = self.serialize_hyperlink(text=res, hyperlink=hyperlink, **kwargs)
             else:
                 _logger.warning("Dropping hyperlink with unsafe scheme; keeping the link text only.")
-        return res
+        return f"{lead}{res}{trail}"
 
     @override
     def serialize_bold(self, text: str, **kwargs: Any) -> str:

docling_core/transforms/serializer/doclang.py

@@ -1865,17 +1865,23 @@ def serialize(
                         yres=params.ysize,
                     )
                     parts.append(create_ser_result(text=loc_str))
-        parts.extend(
-            doc_serializer.get_parts(
-                item=item,
-                list_level=list_level,
-                is_inline_scope=True,
-                visited=my_visited,
-                **{**kwargs, **params.model_dump()},
-            )
+        head_parts = list(parts)
+        inline_parts = doc_serializer.get_parts(
+            item=item,
+            list_level=list_level,
+            is_inline_scope=True,
+            visited=my_visited,
+            **{**kwargs, **params.model_dump()},
         )
+        parts.extend(inline_parts)
         delim = _get_delim(params=params)
-        text_res = delim.join([p.text for p in parts if p.text])
+        # Inline runs carry their own significant whitespace; concatenate them faithfully.
+        # The delimiter separates the element head from the body, it is not a whitespace
+        # channel — `<content>` is (see `_escape_text`).
+        segments = [p.text for p in head_parts if p.text]
+        if inline_text := "".join([p.text for p in inline_parts if p.text]):
+            segments.append(inline_text)
+        text_res = delim.join(segments)
         if text_res:
             text_res = f"{text_res}{delim}"
 

docling_core/transforms/serializer/html.py

@@ -8,7 +8,7 @@
 from html.parser import HTMLParser
 from io import BytesIO
 from pathlib import Path, PurePath
-from typing import Any, Optional, Union
+from typing import Any, ClassVar, Optional, Union
 from xml.etree.ElementTree import tostring
 
 import latex2mathml.converter
@@ -926,8 +926,8 @@ def serialize(
             **kwargs,
         )
 
-        # Join all parts without separators
-        inline_html = " ".join([p.text for p in parts if p.text])
+        # Inline runs carry their own significant whitespace; concatenate faithfully.
+        inline_html = "".join([p.text for p in parts if p.text])
 
         # Wrap in span if needed
         if inline_html:
@@ -1114,6 +1114,8 @@ class HTMLDocSerializer(DocSerializer):
 
     params: HTMLParams = HTMLParams()
 
+    hoist_decoration_whitespace: ClassVar[bool] = True
+
     @override
     def _item_wraps_meta(self, item: NodeItem) -> bool:
         """PictureItem meta is wrapped inside the figure element."""