fix(docx): read moved text, bidirectional spans and block-level custom XML

docling/backend/msword_backend.py

@@ -1274,9 +1274,13 @@ def _in_textbox(elem):
             # This must come before the image branches: they are computed with
             # descendant XPaths, so a control holding a picture anywhere inside
             # would match there and its paragraphs would never be walked.
-            elif tag_name == "sdt":
-                sdt_content = element.find(
-                    "./w:sdtContent", namespaces=_OOXML_NAMESPACES
+            # Custom XML markup wraps paragraphs and tables the same way, with no
+            # content element of its own in between.
+            elif tag_name in ("sdt", "customXml"):
+                sdt_content = (
+                    element.find("./w:sdtContent", namespaces=_OOXML_NAMESPACES)
+                    if tag_name == "sdt"
+                    else element
                 )
                 # Recursively walk the SDT content to catch textboxes, tables, and nested structures
                 added_elements.extend(self._walk_optional_container(sdt_content, doc))
@@ -2272,7 +2276,15 @@ def _iter_paragraph_content(
         def _get_children_recursive(node):
             for child in node:
                 tag_name = etree.QName(child).localname
-                if tag_name in {"smartTag", "customXml", "ins", "fldSimple"}:
+                if tag_name in {
+                    "smartTag",
+                    "customXml",
+                    "ins",
+                    "moveTo",
+                    "fldSimple",
+                    "dir",
+                    "bdo",
+                }:
                     yield from _get_children_recursive(child)
                 else:
                     yield child

tests/test_backend_msword.py

@@ -24,7 +24,7 @@
 from docx import Document
 from docx.enum.style import WD_STYLE_TYPE
 from docx.opc.constants import RELATIONSHIP_TYPE as RT
-from docx.oxml import OxmlElement
+from docx.oxml import OxmlElement, parse_xml
 from docx.oxml.ns import qn
 from docx.shared import Inches
 from lxml import etree
@@ -910,6 +910,60 @@ def _append_citation(paragraph, text: str):
     assert "(Standalone citation)" in markdown
 
 
+_W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'
+
+
+def _w_run(text: str) -> str:
+    return f'<w:r><w:t xml:space="preserve">{text}</w:t></w:r>'
+
+
+def _append_to_body(document, xml: str) -> None:
+    body = document.element.body
+    body.insert(len(body) - 1, parse_xml(xml))
+
+
+def test_runs_in_moves_and_bidirectional_spans_are_read(tmp_path):
+    """Text moved with track changes on, and text in w:dir or w:bdo, belongs to its paragraph."""
+    revision = 'w:author="Reviewer" w:date="2024-01-01T00:00:00Z"'
+    d = Document()
+    for content in (
+        f'<w:moveFrom w:id="1" {revision}>{_w_run("Moved away")}</w:moveFrom>'
+        f'<w:moveTo w:id="2" {revision}>{_w_run("Moved here")}</w:moveTo>',
+        _w_run("Before ") + f'<w:dir w:val="rtl">{_w_run("embedded")}</w:dir>',
+        _w_run("Before ") + f'<w:bdo w:val="rtl">{_w_run("overridden")}</w:bdo>',
+    ):
+        _append_to_body(d, f"<w:p {_W_NS}>{content}</w:p>")
+
+    doc = _convert_built(d, tmp_path)
+
+    assert [item.text for item in doc.texts] == [
+        "Moved here",
+        "Before embedded",
+        "Before overridden",
+    ]
+
+
+def test_paragraphs_and_tables_in_block_custom_xml_are_read(tmp_path):
+    """Custom XML markup around body paragraphs and tables does not hide them."""
+    d = Document()
+    d.add_paragraph("Before")
+    _append_to_body(
+        d,
+        f'<w:customXml {_W_NS} w:element="clause">'
+        f"<w:p>{_w_run('In custom XML')}</w:p>"
+        "<w:tbl><w:tblPr/><w:tblGrid><w:gridCol/><w:gridCol/></w:tblGrid><w:tr>"
+        f"<w:tc><w:p>{_w_run('Name')}</w:p></w:tc>"
+        f"<w:tc><w:p>{_w_run('Value')}</w:p></w:tc>"
+        "</w:tr></w:tbl></w:customXml>",
+    )
+    d.add_paragraph("After")
+
+    doc = _convert_built(d, tmp_path)
+
+    assert [item.text for item in doc.texts] == ["Before", "In custom XML", "After"]
+    assert [cell.text for cell in doc.tables[0].data.table_cells] == ["Name", "Value"]
+
+
 def test_block_sdt_tables_are_extracted():
     """Test tables wrapped in block-level SDT content controls."""
     docx_path = Path("./tests/data/docx/sources/docx_rich_tables_01.docx")