diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index c0324f6fb1..fc9535c611 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -67,6 +67,7 @@ from docx.document import Document as DocxDocument from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as DOCX_RT + from docx.opc.part import XmlPart from docx.oxml import parse_xml as docx_parse_xml from docx.oxml.simpletypes import ST_Merge from docx.oxml.table import CT_Tc @@ -572,6 +573,39 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) +def _remove_xml_comment_nodes(docx_obj: DocxDocument) -> list[str]: + """Remove XML comment nodes and processing instructions from every XML part. + + Word comments (``w:comment`` elements) are not affected. Returns the text of + the XML comments found in the main document part, in document order. + + Some producers write them into the document body (docx4j adds + ````). The backend walks element children and + reads their tag, which a comment does not have, so a single one fails the + conversion or drops the content of the table it sits in. Text following a + removed node is kept. + """ + texts: list[str] = [] + for part in docx_obj.part.package.iter_parts(): + if not isinstance(part, XmlPart): + continue + for node in list(part.element.iter(etree.Comment, etree.ProcessingInstruction)): + parent = node.getparent() + if parent is None: + continue + if part is docx_obj.part and node.tag is etree.Comment and node.text: + if text := node.text.strip(): + texts.append(text) + if node.tail: + previous = node.getprevious() + if previous is not None: + previous.tail = (previous.tail or "") + node.tail + else: + parent.text = (parent.text or "") + node.tail + parent.remove(node) + return texts + + def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: """Rewrite a DOCX archive in memory, removing fragment-only relationship targets. @@ -822,6 +856,7 @@ def __init__( self.docx_to_pdf_converter_init = False self.display_drawingml_warning = True self._empty_docx_template: DocxDocument | None = None + self._xml_comments: list[str] = [] for i in range(-1, self.max_levels): self.parents[i] = None @@ -877,6 +912,7 @@ def __init__( path_or_stream=self.path_or_stream, document_hash=self.document_hash ) if self.docx_obj: + self._xml_comments = _remove_xml_comment_nodes(self.docx_obj) self.valid = True self.current_part = self.docx_obj.part self._default_paragraph_style = self.docx_obj.styles.default( @@ -931,6 +967,12 @@ def convert(self) -> DoclingDocument: self._add_header_footer(self.docx_obj, doc) # Add comments and link them to annotated paragraphs self._add_comments(self.docx_obj, doc) + for text in self._xml_comments: + doc.add_text( + label=DocItemLabel.TEXT, + text=text, + content_layer=ContentLayer.NOTES, + ) # Add footnotes and endnotes (their body text lives in a separate part; # the in-body reference is otherwise silently empty, see docstring below) self._add_footnotes_and_endnotes(self.docx_obj, doc) diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index 314fa1c6f3..91e8b1a348 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -2640,3 +2640,101 @@ def test_body_level_alternate_content_with_no_usable_branch_adds_nothing(tmp_pat "Paragraph before AlternateContent", "Paragraph after AlternateContent", ] + + +def _docx_with_xml_node(insert): + """Build a DOCX with a heading, a two-run paragraph and a 1x2 table, then let + ``insert`` add an XML comment or processing instruction to word/document.xml.""" + import zipfile + from io import BytesIO + + doc = Document() + doc.add_heading("Title", 1) + paragraph = doc.add_paragraph("first run ") + paragraph.add_run("second run") + table = doc.add_table(rows=1, cols=2) + table.cell(0, 0).text = "cell A" + table.cell(0, 1).text = "cell B" + buf = BytesIO() + doc.save(buf) + buf.seek(0) + + out = BytesIO() + with zipfile.ZipFile(buf) as src, zipfile.ZipFile(out, "w") as dst: + for info in src.infolist(): + data = src.read(info.filename) + if info.filename == "word/document.xml": + changed = insert(data) + assert changed != data + data = changed + dst.writestr(info, data) + out.seek(0) + return out + + +_COMMENT = b"" + + +@pytest.mark.parametrize( + "insert", + [ + pytest.param( + lambda x: x.replace(b"", b"" + _COMMENT, 1), id="body" + ), + pytest.param( + lambda x: re.sub(rb"(|]*>)", rb"\1" + _COMMENT, x, count=1), + id="paragraph", + ), + pytest.param(lambda x: x.replace(b"", b"" + _COMMENT, 1), id="run"), + pytest.param( + lambda x: x.replace(b"", b"" + _COMMENT, 1), id="table-cell" + ), + pytest.param( + lambda x: re.sub( + rb"(][^>]*>|)", rb"\1" + _COMMENT, x, count=1 + ), + id="table-row", + ), + pytest.param( + lambda x: x.replace(b"", b"", 1), + id="processing-instruction", + ), + ], +) +def test_xml_comments_and_processing_instructions_keep_body_text(insert): + """Regression test: an XML comment or processing instruction in the document + part (docx4j writes one at the start of the body) used to fail the whole + conversion with ``ValueError: Invalid input tag``, or drop the text of the + table it was in.""" + stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert)) + converter = DocumentConverter(allowed_formats=[InputFormat.DOCX]) + result = converter.convert(stream, raises_on_error=True) + markdown = result.document.export_to_markdown() + for text in ("Title", "first run", "second run", "cell A", "cell B"): + assert text in markdown + + +def test_xml_comment_text_goes_to_notes(): + """XML comments in the document part are kept in the NOTES layer, out of the + body; processing instructions are dropped.""" + + def insert(data): + data = data.replace(b"", b"" + _COMMENT, 1) + data = data.replace(b">first run <", b">first run <", 1) + return data.replace(b"", b"", 1) + + stream = DocumentStream(name="xml_comment.docx", stream=_docx_with_xml_node(insert)) + converter = DocumentConverter(allowed_formats=[InputFormat.DOCX]) + doc = converter.convert(stream, raises_on_error=True).document + + notes = [ + item.text + for item, _ in doc.iterate_items(included_content_layers={ContentLayer.NOTES}) + if isinstance(item, TextItem) + ] + assert notes == ["Created by docx4j 8.2.8 (Apache licensed)", "inline"] + markdown = doc.export_to_markdown() + assert "first run second run" in markdown + assert "docx4j" not in markdown + assert "inline" not in markdown + assert "producer" not in markdown