From bccbfc00c8da4772ae986d3b104cd1434197112f Mon Sep 17 00:00:00 2001 From: loickouao Date: Mon, 5 Oct 2026 14:13:07 +0200 Subject: [PATCH 1/4] fix(docx): ignore XML comments and processing instructions docx4j writes an XML comment at the start of the document body. The backend reads the tag of every child it walks, which a comment does not have, so lxml raised "ValueError: Invalid input tag" and the whole conversion failed. A comment inside a table cell or row dropped the table's text instead. Remove comments and processing instructions from every XML part once the document is loaded, keeping any text that follows them. Signed-off-by: loickouao --- docling/backend/msword_backend.py | 27 ++++++++++++ tests/test_backend_msword.py | 72 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index c0324f6fb1..8a9054c0e0 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -572,6 +572,32 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) +def _remove_xml_comments(docx_obj: DocxDocument) -> None: + """Remove XML comments and processing instructions from every XML part. + + Some producers write them into the document body (docx4j adds + ````). The backend walks element children and + reads their tag, which a comment does not have, so a single one fails the + conversion or drops the content of the table it sits in. Text following a + removed node is kept. + """ + for part in docx_obj.part.package.iter_parts(): + element = getattr(part, "_element", None) + if element is None: + continue + for node in list(element.iter(etree.Comment, etree.ProcessingInstruction)): + parent = node.getparent() + if parent is None: + continue + if node.tail: + previous = node.getprevious() + if previous is not None: + previous.tail = (previous.tail or "") + node.tail + else: + parent.text = (parent.text or "") + node.tail + parent.remove(node) + + def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: """Rewrite a DOCX archive in memory, removing fragment-only relationship targets. @@ -877,6 +903,7 @@ def __init__( path_or_stream=self.path_or_stream, document_hash=self.document_hash ) if self.docx_obj: + _remove_xml_comments(self.docx_obj) self.valid = True self.current_part = self.docx_obj.part self._default_paragraph_style = self.docx_obj.styles.default( diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index 314fa1c6f3..f9d7d41577 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -2640,3 +2640,75 @@ def test_body_level_alternate_content_with_no_usable_branch_adds_nothing(tmp_pat "Paragraph before AlternateContent", "Paragraph after AlternateContent", ] + + +def _docx_with_xml_node(insert): + """Build a DOCX with a heading, a two-run paragraph and a 1x2 table, then let + ``insert`` add an XML comment or processing instruction to word/document.xml.""" + import zipfile + from io import BytesIO + + doc = Document() + doc.add_heading("Title", 1) + paragraph = doc.add_paragraph("first run ") + paragraph.add_run("second run") + table = doc.add_table(rows=1, cols=2) + table.cell(0, 0).text = "cell A" + table.cell(0, 1).text = "cell B" + buf = BytesIO() + doc.save(buf) + buf.seek(0) + + out = BytesIO() + with zipfile.ZipFile(buf) as src, zipfile.ZipFile(out, "w") as dst: + for info in src.infolist(): + data = src.read(info.filename) + if info.filename == "word/document.xml": + changed = insert(data) + assert changed != data + data = changed + dst.writestr(info, data) + out.seek(0) + return out + + +_COMMENT = b"" + + +@pytest.mark.parametrize( + "insert", + [ + pytest.param( + lambda x: x.replace(b"", b"" + _COMMENT, 1), id="body" + ), + pytest.param( + lambda x: re.sub(rb"(|]*>)", rb"\1" + _COMMENT, x, count=1), + id="paragraph", + ), + pytest.param(lambda x: x.replace(b"", b"" + _COMMENT, 1), id="run"), + pytest.param( + lambda x: x.replace(b"", b"" + _COMMENT, 1), id="table-cell" + ), + pytest.param( + lambda x: re.sub( + rb"(][^>]*>|)", rb"\1" + _COMMENT, x, count=1 + ), + id="table-row", + ), + pytest.param( + lambda x: x.replace(b"", b"", 1), + id="processing-instruction", + ), + ], +) +def test_xml_comments_and_processing_instructions_are_ignored(insert): + """Regression test: an XML comment or processing instruction in the document + part (docx4j writes one at the start of the body) used to fail the whole + conversion with ``ValueError: Invalid input tag``, or drop the text of the + table it was in.""" + stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert)) + converter = DocumentConverter(allowed_formats=[InputFormat.DOCX]) + result = converter.convert(stream, raises_on_error=True) + markdown = result.document.export_to_markdown() + for text in ("Title", "first run", "second run", "cell A", "cell B"): + assert text in markdown From add855a49c4a4f68071b56e33a81e555fad0d2d9 Mon Sep 17 00:00:00 2001 From: loickouao Date: Tue, 6 Oct 2026 11:42:01 +0200 Subject: [PATCH 2/4] refactor(docx): name the helper after XML comment nodes The helper only removes nodes and processing instructions. Word comments (w:comment) are elements and still go to the NOTES layer. Signed-off-by: loickouao --- docling/backend/msword_backend.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index 8a9054c0e0..ac3d9ff1ca 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -572,8 +572,10 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) -def _remove_xml_comments(docx_obj: DocxDocument) -> None: - """Remove XML comments and processing instructions from every XML part. +def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> None: + """Remove XML comment nodes and processing instructions from every XML part. + + Word comments (``w:comment`` elements) are not affected. Some producers write them into the document body (docx4j adds ````). The backend walks element children and @@ -903,7 +905,7 @@ def __init__( path_or_stream=self.path_or_stream, document_hash=self.document_hash ) if self.docx_obj: - _remove_xml_comments(self.docx_obj) + _remove_xml_comment_nodes(self.docx_obj) self.valid = True self.current_part = self.docx_obj.part self._default_paragraph_style = self.docx_obj.styles.default( From d330ef9bb779d15f5a34edbe43a7e3e818d5c5aa Mon Sep 17 00:00:00 2001 From: loickouao Date: Tue, 6 Oct 2026 14:45:21 +0200 Subject: [PATCH 3/4] fix(docx): keep XML comment text in the NOTES layer XML comments in the main document part are no longer dropped: their text is added to the NOTES layer, after the body. Processing instructions and comments in the other parts are still removed. Signed-off-by: loickouao --- docling/backend/msword_backend.py | 19 ++++++++++++++++--- tests/test_backend_msword.py | 28 +++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index ac3d9ff1ca..88f5fc0973 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -572,10 +572,11 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) -def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> None: +def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> list[str]: """Remove XML comment nodes and processing instructions from every XML part. - Word comments (``w:comment`` elements) are not affected. + Word comments (``w:comment`` elements) are not affected. Returns the text of + the XML comments found in the main document part, in document order. Some producers write them into the document body (docx4j adds ````). The backend walks element children and @@ -583,6 +584,7 @@ def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> None: conversion or drops the content of the table it sits in. Text following a removed node is kept. """ + texts: list[str] = [] for part in docx_obj.part.package.iter_parts(): element = getattr(part, "_element", None) if element is None: @@ -591,6 +593,9 @@ def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> None: parent = node.getparent() if parent is None: continue + if part is docx_obj.part and node.tag is etree.Comment and node.text: + if text := node.text.strip(): + texts.append(text) if node.tail: previous = node.getprevious() if previous is not None: @@ -598,6 +603,7 @@ def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> None: else: parent.text = (parent.text or "") + node.tail parent.remove(node) + return texts def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: @@ -850,6 +856,7 @@ def __init__( self.docx_to_pdf_converter_init = False self.display_drawingml_warning = True self._empty_docx_template: DocxDocument | None = None + self._xml_comments: list[str] = [] for i in range(-1, self.max_levels): self.parents[i] = None @@ -905,7 +912,7 @@ def __init__( path_or_stream=self.path_or_stream, document_hash=self.document_hash ) if self.docx_obj: - _remove_xml_comment_nodes(self.docx_obj) + self._xml_comments = _remove_xml_comment_nodes(self.docx_obj) self.valid = True self.current_part = self.docx_obj.part self._default_paragraph_style = self.docx_obj.styles.default( @@ -960,6 +967,12 @@ def convert(self) -> DoclingDocument: self._add_header_footer(self.docx_obj, doc) # Add comments and link them to annotated paragraphs self._add_comments(self.docx_obj, doc) + for text in self._xml_comments: + doc.add_text( + label=DocItemLabel.TEXT, + text=text, + content_layer=ContentLayer.NOTES, + ) # Add footnotes and endnotes (their body text lives in a separate part; # the in-body reference is otherwise silently empty, see docstring below) self._add_footnotes_and_endnotes(self.docx_obj, doc) diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index f9d7d41577..91e8b1a348 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -2701,7 +2701,7 @@ def _docx_with_xml_node(insert): ), ], ) -def test_xml_comments_and_processing_instructions_are_ignored(insert): +def test_xml_comments_and_processing_instructions_keep_body_text(insert): """Regression test: an XML comment or processing instruction in the document part (docx4j writes one at the start of the body) used to fail the whole conversion with ``ValueError: Invalid input tag``, or drop the text of the @@ -2712,3 +2712,29 @@ def test_xml_comments_and_processing_instructions_are_ignored(insert): markdown = result.document.export_to_markdown() for text in ("Title", "first run", "second run", "cell A", "cell B"): assert text in markdown + + +def test_xml_comment_text_goes_to_notes(): + """XML comments in the document part are kept in the NOTES layer, out of the + body; processing instructions are dropped.""" + + def insert(data): + data = data.replace(b"", b"" + _COMMENT, 1) + data = data.replace(b">first run <", b">first run <", 1) + return data.replace(b"", b"", 1) + + stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert)) + converter = DocumentConverter(allowed_formats=[InputFormat.DOCX]) + doc = converter.convert(stream, raises_on_error=True).document + + notes = [ + item.text + for item, _ in doc.iterate_items(included_content_layers={ContentLayer.NOTES}) + if isinstance(item, TextItem) + ] + assert notes == ["Created by docx4j 8.2.8 (Apache licensed)", "inline"] + markdown = doc.export_to_markdown() + assert "first run second run" in markdown + assert "docx4j" not in markdown + assert "inline" not in markdown + assert "producer" not in markdown From 543c6c2211bd4ea74f4d1ad92bff28f66b532711 Mon Sep 17 00:00:00 2001 From: loickouao Date: Wed, 14 Oct 2026 15:44:14 +0200 Subject: [PATCH 4/4] refactor(docx): find XML parts with isinstance(XmlPart) Use the public XmlPart.element instead of probing for the private _element attribute, as AGENTS.md asks. In python-docx only XmlPart parts have an element, so the same parts are cleaned. Signed-off-by: loickouao --- docling/backend/msword_backend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index 88f5fc0973..fc9535c611 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -67,6 +67,7 @@ from docx.document import Document as DocxDocument from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as DOCX_RT + from docx.opc.part import XmlPart from docx.oxml import parse_xml as docx_parse_xml from docx.oxml.simpletypes import ST_Merge from docx.oxml.table import CT_Tc @@ -586,10 +587,9 @@ def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> list[str]: """ texts: list[str] = [] for part in docx_obj.part.package.iter_parts(): - element = getattr(part, "_element", None) - if element is None: + if not isinstance(part, XmlPart): continue - for node in list(element.iter(etree.Comment, etree.ProcessingInstruction)): + for node in list(part.element.iter(etree.Comment, etree.ProcessingInstruction)): parent = node.getparent() if parent is None: continue