From c3282c3d81890889f55d3306a2144f03a37b3259 Mon Sep 17 00:00:00 2001 From: Michael Ghaly Date: Thu, 22 Oct 2026 00:01:30 +0200 Subject: [PATCH 1/3] fix(docx): leave hidden text out of the default export Text with Word's Hidden font property (w:vanish) was emitted as body content, although Word does not show or print it. A fully hidden paragraph now goes to ContentLayer.INVISIBLE, the same treatment a hidden PPTX slide (#4647) or shape (#4733) gets. A hidden run inside a shown paragraph or a table cell cannot have its own layer, so it is dropped from the text. The hidden state is resolved from the run, then its character style chain, then the paragraph style chain. Documents with no w:vanish skip the checks, so conversion speed is unchanged. Signed-off-by: Michael Ghaly --- docling/backend/msword_backend.py | 113 ++++++++++++++++++++++++---- tests/test_backend_msword_hidden.py | 80 ++++++++++++++++++++ 2 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 tests/test_backend_msword_hidden.py diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index 942704e369..8bcbdda7b4 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -69,6 +69,7 @@ from docx.document import Document as DocxDocument from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as DOCX_RT + from docx.opc.part import XmlPart from docx.oxml import parse_xml as docx_parse_xml from docx.oxml.ns import qn from docx.oxml.simpletypes import ST_Merge @@ -895,6 +896,9 @@ def __init__( self._pending_code_blank_lines: int = 0 # The document default style's font is not evidence of code. self._default_paragraph_style: BaseStyle | None = None + # Whether any part uses w:vanish; most documents do not, and then the + # per-run hidden checks are skipped. + self._uses_hidden_text = False self.docx_obj = self.load_msword_file( path_or_stream=self.path_or_stream, document_hash=self.document_hash @@ -903,6 +907,11 @@ def __init__( self.valid = True self.current_part = self.docx_obj.part _normalize_list_style_types(self.docx_obj) + self._uses_hidden_text = any( + part.element.xpath(".//w:vanish") + for part in self.docx_obj.part.package.iter_parts() + if isinstance(part, XmlPart) + ) self._default_paragraph_style = self.docx_obj.styles.default( WD_STYLE_TYPE.PARAGRAPH ) @@ -2337,10 +2346,51 @@ def _get_hyperlink_target(self, hyperlink: Hyperlink) -> AnyUrl | Path | None: return None + def _is_hidden_run( + self, run: Run, paragraph_style_chain: Sequence[CharacterStyle] + ) -> bool: + """Return True if Word hides ``run`` (the Hidden font property, ``w:vanish``). + + The first explicit setting wins: the run's own formatting, then its + character style chain, then the paragraph style chain. + """ + if not self._uses_hidden_text: + return False + fonts = ( + run.font, + *(style.font for style in self._get_style_chain(run.style)), + *(style.font for style in paragraph_style_chain), + ) + for font in fonts: + if font.hidden is not None: + return font.hidden + return False + + def _is_hidden_paragraph(self, paragraph: Paragraph) -> bool: + """Return True if ``paragraph`` has text and Word hides all of it.""" + if not self._uses_hidden_text: + return False + runs = [ + Run(r_el, paragraph) + for r_el in paragraph._p.iter(f"{_W_NS_CLARK}r") + if r_el.find(f"{_W_NS_CLARK}t") is not None + ] + return bool(runs) and all( + self._is_hidden_run(run, self._get_style_chain(paragraph.style)) + for run in runs + ) + def _iter_paragraph_content( self, paragraph: Paragraph ) -> list[tuple[str, Formatting | None, AnyUrl | Path | None]]: content: list[tuple[str, Formatting | None, AnyUrl | Path | None]] = [] + # A fully hidden paragraph keeps its text; it goes to the invisible layer. + keep_hidden = self._is_hidden_paragraph(paragraph) + + def _is_shown(run: Run) -> bool: + return keep_hidden or not self._is_hidden_run( + run, self._get_style_chain(paragraph.style) + ) def _get_children_recursive(node): for child in node: @@ -2354,24 +2404,24 @@ def _get_children_recursive(node): tag_name = etree.QName(child).localname if tag_name == "sdt": - text = "".join( - child.xpath( - ".//w:sdtContent//w:t/text()", - namespaces=_OOXML_NAMESPACES, + runs = [ + run + for run in ( + Run(r_el, paragraph) + for r_el in child.xpath( + ".//w:sdtContent//w:r", + namespaces=_OOXML_NAMESPACES, + ) ) - ) + if _is_shown(run) + ] + text = "".join(t for run in runs for t in run._r.xpath("./w:t/text()")) if len(text) == 0: continue - runs = child.xpath( - ".//w:sdtContent//w:r", - namespaces=_OOXML_NAMESPACES, - ) fmt = ( self._get_format_from_run( - Run(runs[0], paragraph), - paragraph, - self._get_style_chain(paragraph.style), + runs[0], paragraph, self._get_style_chain(paragraph.style) ) if runs else None @@ -2392,11 +2442,14 @@ def _get_children_recursive(node): # Hyperlink.text and Hyperlink.runs only read the direct w:r # children, so also read the runs nested in the same wrappers # as above (e.g. a tracked insertion inside the link). - runs = [ + all_runs = [ Run(r_el, paragraph) for r_el in _get_children_recursive(child) if r_el.tag == f"{_W_NS_CLARK}r" ] + runs = [run for run in all_runs if _is_shown(run)] + if all_runs and not runs: + continue # a fully hidden link content.append( ( "".join(run.text for run in runs), @@ -2412,7 +2465,7 @@ def _get_children_recursive(node): self._get_hyperlink_target(item), ) ) - elif isinstance(item, Run): + elif _is_shown(item): content.append( ( item.text, @@ -2933,6 +2986,17 @@ def _handle_text_elements( ) -> list[RefItem]: elem_ref: list[RefItem] = [] paragraph = Paragraph(element, self.docx_obj) + if self.content_layer != ContentLayer.INVISIBLE and self._is_hidden_paragraph( + paragraph + ): + # Word does not show it: keep it, like a hidden slide or sheet, + # but out of the default exports. + previous_layer = self.content_layer + self.content_layer = ContentLayer.INVISIBLE + try: + return self._handle_text_elements(element, doc, skip_empty_text) + finally: + self.content_layer = previous_layer paragraph_elements = self._get_paragraph_elements(paragraph) text, equations = self._handle_equations_in_text( element=element, text=self._get_paragraph_text(paragraph) @@ -3870,7 +3934,26 @@ def _handle_tables( element=cell._element, text=cell.text ) if len(equations) == 0: - text = cell.text + # cell.text includes hidden runs, so rebuild it from the + # paragraphs' visible content when the cell has any. A cell + # has no layer of its own, so fully hidden paragraphs are + # left out. + has_hidden = any( + self._is_hidden_run( + Run(r_el, p), self._get_style_chain(p.style) + ) + for p in cell.paragraphs + for r_el in p._p.xpath(".//w:r") + ) + text = ( + "\n".join( + self._get_paragraph_text(p) + for p in cell.paragraphs + if not self._is_hidden_paragraph(p) + ) + if has_hidden + else cell.text + ) else: text = text.replace("", "$").replace("", "$") diff --git a/tests/test_backend_msword_hidden.py b/tests/test_backend_msword_hidden.py new file mode 100644 index 0000000000..0f804bee85 --- /dev/null +++ b/tests/test_backend_msword_hidden.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tests for text with Word's Hidden font property (``w:vanish``).""" + +from pathlib import Path + +from docling_core.types.doc import ContentLayer, TextItem +from docx import Document +from docx.enum.style import WD_STYLE_TYPE + +from docling.backend.msword_backend import MsWordDocumentBackend +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import DoclingDocument, InputDocument + + +def _convert(document, tmp_path: Path) -> DoclingDocument: + path = tmp_path / "hidden.docx" + document.save(str(path)) + in_doc = InputDocument( + path_or_stream=path, format=InputFormat.DOCX, backend=MsWordDocumentBackend + ) + return in_doc._backend.convert() + + +def _layers_by_text(doc: DoclingDocument) -> dict[str, str]: + return { + item.text: item.content_layer.value + for item, _ in doc.iterate_items(included_content_layers=set(ContentLayer)) + if isinstance(item, TextItem) + } + + +def test_docx_hidden_text_is_left_out_of_the_default_export(tmp_path: Path): + """Word does not show or print hidden text. + + A fully hidden paragraph goes to ``ContentLayer.INVISIBLE``, the same + treatment a hidden PPTX slide or shape gets. A hidden run in a shown + paragraph or a table cell cannot have its own layer, so it is dropped. + """ + document = Document() + para = document.add_paragraph("Shown start. ") + para.add_run("Hidden run. ").font.hidden = True + para.add_run("Shown end.") + document.add_paragraph().add_run("Hidden paragraph.").font.hidden = True + table = document.add_table(rows=1, cols=2) + table.cell(0, 0).text = "Shown cell" + table.cell(0, 1).paragraphs[0].add_run("Hidden cell").font.hidden = True + + doc = _convert(document, tmp_path) + + layers = _layers_by_text(doc) + assert layers["Shown start. Shown end."] == "body" + assert layers["Hidden paragraph."] == "invisible" + assert [c.text for c in doc.tables[0].data.table_cells] == ["Shown cell", ""] + assert "Hidden" not in doc.export_to_markdown() + + +def test_docx_hidden_text_follows_style_inheritance(tmp_path: Path): + """Hidden can come from a character or paragraph style, and an explicit + ``w:vanish w:val="0"`` on a run overrides a hidden style.""" + document = Document() + char_style = document.styles.add_style("Hidden Chars", WD_STYLE_TYPE.CHARACTER) + char_style.font.hidden = True + para_style = document.styles.add_style("Hidden Para", WD_STYLE_TYPE.PARAGRAPH) + para_style.font.hidden = True + + document.add_paragraph("Shown. ").add_run("Hidden by character style.", char_style) + document.add_paragraph("Hidden by paragraph style.", para_style) + mixed = document.add_paragraph(style=para_style) + mixed.add_run("Hidden by paragraph style too. ") + mixed.add_run("Shown by override.").font.hidden = False + + doc = _convert(document, tmp_path) + + layers = _layers_by_text(doc) + assert layers["Shown."] == "body" + assert layers["Hidden by paragraph style."] == "invisible" + assert layers["Shown by override."] == "body" + assert "Hidden" not in doc.export_to_markdown() From 7e13581bb686cf3760d17864bc2fbfe9b4844c4d Mon Sep 17 00:00:00 2001 From: Michael Ghaly Date: Thu, 22 Oct 2026 17:00:14 +0200 Subject: [PATCH 2/3] fix(docx): hide hidden headings, list items and note text too Address the review of the first commit: - Headings and list items take their own add calls, which did not pass the content layer. They now go to ContentLayer.INVISIBLE when hidden. List groups stay on the surrounding layer, so a hidden first item no longer hides the shown items after it or makes the list loose. - The equation parser read the w:t of hidden runs, so its rebuilt text did not match and the equation was lost. It now skips hidden runs. A table cell with an equation also kept its hidden text; it does not now. - A paragraph with an equation is never fully hidden, and an equation with only hidden text beside it is a standalone formula again. - python-docx loads footnotes and endnotes as plain parts, so w:vanish only there was not found. A hidden paragraph in a note is now left out. - Documents without w:vanish skip the style chain lookups for every run and table cell run. Signed-off-by: Michael Ghaly --- docling/backend/msword_backend.py | 134 ++++++++++++++++++---------- tests/test_backend_msword.py | 25 ++++++ tests/test_backend_msword_hidden.py | 84 +++++++++++++++++ 3 files changed, 195 insertions(+), 48 deletions(-) diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index 8bcbdda7b4..e51126ed47 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -52,7 +52,7 @@ get_docx_to_pdf_converter, get_pil_from_dml_docx, ) -from docling.backend.docx.latex.omml import oMath2Latex +from docling.backend.docx.latex.omml import OMML_NS, oMath2Latex from docling.backend.utils.image import normalize_image_for_png from docling.datamodel.backend_options import MsWordBackendOptions from docling.datamodel.base_models import FormatToMimeType @@ -899,6 +899,9 @@ def __init__( # Whether any part uses w:vanish; most documents do not, and then the # per-run hidden checks are skipped. self._uses_hidden_text = False + # The layer around a hidden paragraph while it is added. List groups + # stay on it, so a hidden item does not hide the shown items with it. + self._layer_around_hidden: ContentLayer | None = None self.docx_obj = self.load_msword_file( path_or_stream=self.path_or_stream, document_hash=self.document_hash @@ -909,8 +912,10 @@ def __init__( _normalize_list_style_types(self.docx_obj) self._uses_hidden_text = any( part.element.xpath(".//w:vanish") - for part in self.docx_obj.part.package.iter_parts() if isinstance(part, XmlPart) + # python-docx loads footnotes and endnotes as plain parts. + else part.content_type.endswith("+xml") and b"vanish" in part.blob + for part in self.docx_obj.part.package.iter_parts() ) self._default_paragraph_style = self.docx_obj.styles.default( WD_STYLE_TYPE.PARAGRAPH @@ -1124,7 +1129,7 @@ def _get_or_create_list_group( list_gr = doc.add_list_group( name="list", parent=parent, - content_layer=self.content_layer, + content_layer=self._layer_around_hidden or self.content_layer, ) elem_ref.append(list_gr.get_ref()) @@ -2354,8 +2359,6 @@ def _is_hidden_run( The first explicit setting wins: the run's own formatting, then its character style chain, then the paragraph style chain. """ - if not self._uses_hidden_text: - return False fonts = ( run.font, *(style.font for style in self._get_style_chain(run.style)), @@ -2367,28 +2370,31 @@ def _is_hidden_run( return False def _is_hidden_paragraph(self, paragraph: Paragraph) -> bool: - """Return True if ``paragraph`` has text and Word hides all of it.""" - if not self._uses_hidden_text: + """Return True if ``paragraph`` has text and Word hides all of it. + + An equation counts as shown: its runs are math runs, not ``w:r``. + """ + if not self._uses_hidden_text or ( + paragraph._p.find(f".//{OMML_NS}oMath") is not None + ): return False runs = [ Run(r_el, paragraph) for r_el in paragraph._p.iter(f"{_W_NS_CLARK}r") if r_el.find(f"{_W_NS_CLARK}t") is not None ] - return bool(runs) and all( - self._is_hidden_run(run, self._get_style_chain(paragraph.style)) - for run in runs - ) + style_chain = self._get_style_chain(paragraph.style) + return bool(runs) and all(self._is_hidden_run(run, style_chain) for run in runs) def _iter_paragraph_content( self, paragraph: Paragraph ) -> list[tuple[str, Formatting | None, AnyUrl | Path | None]]: content: list[tuple[str, Formatting | None, AnyUrl | Path | None]] = [] # A fully hidden paragraph keeps its text; it goes to the invisible layer. - keep_hidden = self._is_hidden_paragraph(paragraph) + show_all = not self._uses_hidden_text or self._is_hidden_paragraph(paragraph) def _is_shown(run: Run) -> bool: - return keep_hidden or not self._is_hidden_run( + return show_all or not self._is_hidden_run( run, self._get_style_chain(paragraph.style) ) @@ -2877,10 +2883,31 @@ def _handle_textbox_content( self.parents[level] = original_parent return elem_ref + def _get_hidden_text_elements( + self, element: BaseOxmlElement + ) -> set[etree._Element]: + """Return the ``w:t`` elements of the hidden runs in ``element``. + + Only an element with an equation needs them, so for any other element + the result is empty. + """ + hidden: set[etree._Element] = set() + if not self._uses_hidden_text or element.find(f".//{OMML_NS}oMath") is None: + return hidden + for p_el in element.iter(f"{_W_NS_CLARK}p"): + paragraph = Paragraph(p_el, self.docx_obj) + style_chain = self._get_style_chain(paragraph.style) + for r_el in p_el.iter(f"{_W_NS_CLARK}r"): + if self._is_hidden_run(Run(r_el, paragraph), style_chain): + hidden.update(r_el.iter(f"{_W_NS_CLARK}t")) + return hidden + def _handle_equations_in_text(self, element, text): only_texts = [] only_equations = [] texts_and_equations = [] + # ``text`` has no hidden text, so leave it out here too. + hidden_texts = self._get_hidden_text_elements(element) # Collect oMath elements and text runs from the paragraph. # Use direct children iteration first; fall back to deep iteration @@ -2908,7 +2935,11 @@ def _handle_equations_in_text(self, element, text): # Collect text from non-math children (e.g. runs) for t_elem in child.iter(): t_tag = etree.QName(t_elem).localname - if t_tag == "t" and "math" not in t_elem.tag: + if ( + t_tag == "t" + and "math" not in t_elem.tag + and t_elem not in hidden_texts + ): if isinstance(t_elem.text, str): only_texts.append(t_elem.text) texts_and_equations.append(t_elem.text) @@ -2917,7 +2948,11 @@ def _handle_equations_in_text(self, element, text): # inside oMathPara or other wrapper elements). for subt in element.iter(): tag_name = etree.QName(subt).localname - if tag_name == "t" and "math" not in subt.tag: + if ( + tag_name == "t" + and "math" not in subt.tag + and subt not in hidden_texts + ): if isinstance(subt.text, str): only_texts.append(subt.text) texts_and_equations.append(subt.text) @@ -2991,12 +3026,13 @@ def _handle_text_elements( ): # Word does not show it: keep it, like a hidden slide or sheet, # but out of the default exports. - previous_layer = self.content_layer + self._layer_around_hidden = self.content_layer self.content_layer = ContentLayer.INVISIBLE try: return self._handle_text_elements(element, doc, skip_empty_text) finally: - self.content_layer = previous_layer + self.content_layer = self._layer_around_hidden + self._layer_around_hidden = None paragraph_elements = self._get_paragraph_elements(paragraph) text, equations = self._handle_equations_in_text( element=element, text=self._get_paragraph_text(paragraph) @@ -3122,9 +3158,8 @@ def _handle_text_elements( elem_ref.extend(h1) elif len(equations) > 0: - if (paragraph.text is None or len(paragraph.text.strip()) == 0) and len( - text - ) > 0: + # The paragraph's text without equations or hidden runs. + if len(self._get_paragraph_text(paragraph).strip()) == 0 and len(text) > 0: # Standalone equation(s) — emit each as a separate formula level = self._get_level() parent = self.parents[level - 1] @@ -3372,6 +3407,7 @@ def _add_heading( parent=self.parents[parent_level], text=text, level=add_level, + content_layer=self.content_layer, ) self.parents[current_level] = hd elem_ref.append(hd.get_ref()) @@ -3405,6 +3441,7 @@ def _add_formatted_list_item( text=text, formatting=format, hyperlink=hyperlink, + content_layer=self.content_layer, ) else: new_item = doc.add_list_item( @@ -3412,8 +3449,11 @@ def _add_formatted_list_item( enumerated=enumerated, parent=self.parents[level], text="", + content_layer=self.content_layer, + ) + new_parent = doc.add_inline_group( + parent=new_item, content_layer=self.content_layer ) - new_parent = doc.add_inline_group(parent=new_item) for text, format, hyperlink in elements: if text: doc.add_text( @@ -3641,7 +3681,7 @@ def _manage_list_structure( list_gr1 = doc.add_list_group( name="list", parent=self._sub_list_parent(doc, self.parents.get(i - 1)), - content_layer=self.content_layer, + content_layer=self._layer_around_hidden or self.content_layer, ) self.parents[i] = list_gr1 elem_ref.append(list_gr1.get_ref()) @@ -3791,6 +3831,7 @@ def _add_list_item_with_equations( enumerated=is_numbered, parent=self.parents[use_level], text="", + content_layer=self.content_layer, ) inline_group = doc.add_inline_group( @@ -3929,31 +3970,27 @@ def _handle_tables( cell = _Cell(tc, table) + # cell.text includes hidden runs, so rebuild it from the + # paragraphs' visible content when the cell has any. A cell has + # no layer of its own, so fully hidden paragraphs are left out. + cell_text = cell.text + if self._uses_hidden_text and any( + self._is_hidden_run(Run(r_el, p), self._get_style_chain(p.style)) + for p in cell.paragraphs + for r_el in p._p.xpath(".//w:r") + ): + cell_text = "\n".join( + self._get_paragraph_text(p) + for p in cell.paragraphs + if not self._is_hidden_paragraph(p) + ) + # Detect equations in cell text text, equations = self._handle_equations_in_text( - element=cell._element, text=cell.text + element=cell._element, text=cell_text ) if len(equations) == 0: - # cell.text includes hidden runs, so rebuild it from the - # paragraphs' visible content when the cell has any. A cell - # has no layer of its own, so fully hidden paragraphs are - # left out. - has_hidden = any( - self._is_hidden_run( - Run(r_el, p), self._get_style_chain(p.style) - ) - for p in cell.paragraphs - for r_el in p._p.xpath(".//w:r") - ) - text = ( - "\n".join( - self._get_paragraph_text(p) - for p in cell.paragraphs - if not self._is_hidden_paragraph(p) - ) - if has_hidden - else cell.text - ) + text = cell_text else: text = text.replace("", "$").replace("", "$") @@ -4845,14 +4882,15 @@ def _add_footnotes_and_endnotes( for note in root.findall(f"{_W_NS_CLARK}{tag_name}"): if note.get(f"{_W_NS_CLARK}type") in skip_types: continue + # A note has no layer per paragraph, so hidden ones are left out. texts = [ text - for p_elm in note.findall(f".//{_W_NS_CLARK}p") - if ( - text := self._get_paragraph_text( - Paragraph(p_elm, docx_obj) - ).strip() + for paragraph in ( + Paragraph(p_elm, docx_obj) + for p_elm in note.findall(f".//{_W_NS_CLARK}p") ) + if not self._is_hidden_paragraph(paragraph) + and (text := self._get_paragraph_text(paragraph).strip()) ] if not texts: continue diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index 2054dc189e..0b1781bfed 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -2546,6 +2546,31 @@ def test_footnote_body_text_is_not_dropped_outside_plain_paragraphs( assert expected in footnote_items +def test_hidden_footnote_paragraph_is_left_out(): + """A hidden paragraph in a footnote is left out of the footnote text. + + python-docx loads the footnotes part as a plain part, so this also checks + that ``w:vanish`` there is found when the body has none. + """ + note_body = ( + "Shown note." + "Hidden note." + ) + stream = DocumentStream( + name="notes.docx", stream=_docx_with_notes(footnote_body=note_body) + ) + result = DocumentConverter(allowed_formats=[InputFormat.DOCX]).convert( + stream, raises_on_error=True + ) + + footnote_items = [ + item.text + for item in result.document.texts + if item.label == DocItemLabel.FOOTNOTE + ] + assert footnote_items == ["Shown note.", "An endnote body."] + + _MC_BODY_BLOCK = """