fix(docx): keep XML comments in NOTES, drop processing instructions

docling/backend/msword_backend.py

@@ -67,6 +67,7 @@
     from docx.document import Document as DocxDocument
     from docx.enum.style import WD_STYLE_TYPE
     from docx.opc.constants import RELATIONSHIP_TYPE as DOCX_RT
+    from docx.opc.part import XmlPart
     from docx.oxml import parse_xml as docx_parse_xml
     from docx.oxml.simpletypes import ST_Merge
     from docx.oxml.table import CT_Tc
@@ -572,6 +573,39 @@ def _remove_fragment_only_rels(content: bytes) -> bytes:
     return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True)
 
 
+def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> list[str]:
+    """Remove XML comment nodes and processing instructions from every XML part.
+
+    Word comments (``w:comment`` elements) are not affected. Returns the text of
+    the XML comments found in the main document part, in document order.
+
+    Some producers write them into the document body (docx4j adds
+    ``<!-- Created by docx4j ... -->``). The backend walks element children and
+    reads their tag, which a comment does not have, so a single one fails the
+    conversion or drops the content of the table it sits in. Text following a
+    removed node is kept.
+    """
+    texts: list[str] = []
+    for part in docx_obj.part.package.iter_parts():
+        if not isinstance(part, XmlPart):
+            continue
+        for node in list(part.element.iter(etree.Comment, etree.ProcessingInstruction)):
+            parent = node.getparent()
+            if parent is None:
+                continue
+            if part is docx_obj.part and node.tag is etree.Comment and node.text:
+                if text := node.text.strip():
+                    texts.append(text)
+            if node.tail:
+                previous = node.getprevious()
+                if previous is not None:
+                    previous.tail = (previous.tail or "") + node.tail
+                else:
+                    parent.text = (parent.text or "") + node.tail
+            parent.remove(node)
+    return texts
+
+
 def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO:
     """Rewrite a DOCX archive in memory, removing fragment-only relationship targets.
 
@@ -822,6 +856,7 @@ def __init__(
         self.docx_to_pdf_converter_init = False
         self.display_drawingml_warning = True
         self._empty_docx_template: DocxDocument | None = None
+        self._xml_comments: list[str] = []
 
         for i in range(-1, self.max_levels):
             self.parents[i] = None
@@ -877,6 +912,7 @@ def __init__(
             path_or_stream=self.path_or_stream, document_hash=self.document_hash
         )
         if self.docx_obj:
+            self._xml_comments = _remove_xml_comment_nodes(self.docx_obj)
             self.valid = True
             self.current_part = self.docx_obj.part
             self._default_paragraph_style = self.docx_obj.styles.default(
@@ -931,6 +967,12 @@ def convert(self) -> DoclingDocument:
             self._add_header_footer(self.docx_obj, doc)
             # Add comments and link them to annotated paragraphs
             self._add_comments(self.docx_obj, doc)
+            for text in self._xml_comments:
+                doc.add_text(
+                    label=DocItemLabel.TEXT,
+                    text=text,
+                    content_layer=ContentLayer.NOTES,
+                )
             # Add footnotes and endnotes (their body text lives in a separate part;
             # the in-body reference is otherwise silently empty, see docstring below)
             self._add_footnotes_and_endnotes(self.docx_obj, doc)

tests/test_backend_msword.py

@@ -2640,3 +2640,101 @@ def test_body_level_alternate_content_with_no_usable_branch_adds_nothing(tmp_pat
         "Paragraph before AlternateContent",
         "Paragraph after AlternateContent",
     ]
+
+
+def _docx_with_xml_node(insert):
+    """Build a DOCX with a heading, a two-run paragraph and a 1x2 table, then let
+    ``insert`` add an XML comment or processing instruction to word/document.xml."""
+    import zipfile
+    from io import BytesIO
+
+    doc = Document()
+    doc.add_heading("Title", 1)
+    paragraph = doc.add_paragraph("first run ")
+    paragraph.add_run("second run")
+    table = doc.add_table(rows=1, cols=2)
+    table.cell(0, 0).text = "cell A"
+    table.cell(0, 1).text = "cell B"
+    buf = BytesIO()
+    doc.save(buf)
+    buf.seek(0)
+
+    out = BytesIO()
+    with zipfile.ZipFile(buf) as src, zipfile.ZipFile(out, "w") as dst:
+        for info in src.infolist():
+            data = src.read(info.filename)
+            if info.filename == "word/document.xml":
+                changed = insert(data)
+                assert changed != data
+                data = changed
+            dst.writestr(info, data)
+    out.seek(0)
+    return out
+
+
+_COMMENT = b"<!-- Created by docx4j 8.2.8 (Apache licensed) -->"
+
+
+@pytest.mark.parametrize(
+    "insert",
+    [
+        pytest.param(
+            lambda x: x.replace(b"<w:body>", b"<w:body>" + _COMMENT, 1), id="body"
+        ),
+        pytest.param(
+            lambda x: re.sub(rb"(<w:p>|<w:p [^>]*>)", rb"\1" + _COMMENT, x, count=1),
+            id="paragraph",
+        ),
+        pytest.param(lambda x: x.replace(b"<w:r>", b"<w:r>" + _COMMENT, 1), id="run"),
+        pytest.param(
+            lambda x: x.replace(b"<w:tc>", b"<w:tc>" + _COMMENT, 1), id="table-cell"
+        ),
+        pytest.param(
+            lambda x: re.sub(
+                rb"(<w:tr[ >][^>]*>|<w:tr>)", rb"\1" + _COMMENT, x, count=1
+            ),
+            id="table-row",
+        ),
+        pytest.param(
+            lambda x: x.replace(b"<w:body>", b"<w:body><?producer note?>", 1),
+            id="processing-instruction",
+        ),
+    ],
+)
+def test_xml_comments_and_processing_instructions_keep_body_text(insert):
+    """Regression test: an XML comment or processing instruction in the document
+    part (docx4j writes one at the start of the body) used to fail the whole
+    conversion with ``ValueError: Invalid input tag``, or drop the text of the
+    table it was in."""
+    stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert))
+    converter = DocumentConverter(allowed_formats=[InputFormat.DOCX])
+    result = converter.convert(stream, raises_on_error=True)
+    markdown = result.document.export_to_markdown()
+    for text in ("Title", "first run", "second run", "cell A", "cell B"):
+        assert text in markdown
+
+
+def test_xml_comment_text_goes_to_notes():
+    """XML comments in the document part are kept in the NOTES layer, out of the
+    body; processing instructions are dropped."""
+
+    def insert(data):
+        data = data.replace(b"<w:body>", b"<w:body>" + _COMMENT, 1)
+        data = data.replace(b">first run <", b">first <!-- inline -->run <", 1)
+        return data.replace(b"<w:tc>", b"<w:tc><?producer note?>", 1)
+
+    stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert))
+    converter = DocumentConverter(allowed_formats=[InputFormat.DOCX])
+    doc = converter.convert(stream, raises_on_error=True).document
+
+    notes = [
+        item.text
+        for item, _ in doc.iterate_items(included_content_layers={ContentLayer.NOTES})
+        if isinstance(item, TextItem)
+    ]
+    assert notes == ["Created by docx4j 8.2.8 (Apache licensed)", "inline"]
+    markdown = doc.export_to_markdown()
+    assert "first run second run" in markdown
+    assert "docx4j" not in markdown
+    assert "inline" not in markdown
+    assert "producer" not in markdown