diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index ae708d4195..c09afd274a 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -52,7 +52,7 @@ get_docx_to_pdf_converter, get_pil_from_dml_docx, ) -from docling.backend.docx.latex.omml import oMath2Latex +from docling.backend.docx.latex.omml import OMML_NS, oMath2Latex from docling.backend.utils.archive_limits import ( ArchiveLimits, check_archive_limits, @@ -74,6 +74,7 @@ from docx.document import Document as DocxDocument from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as DOCX_RT + from docx.opc.part import XmlPart from docx.oxml import parse_xml as docx_parse_xml from docx.oxml.ns import qn from docx.oxml.simpletypes import ST_Merge @@ -890,6 +891,12 @@ def __init__( self._pending_code_blank_lines: int = 0 # The document default style's font is not evidence of code. self._default_paragraph_style: BaseStyle | None = None + # Whether any part uses w:vanish; most documents do not, and then the + # per-run hidden checks are skipped. + self._uses_hidden_text = False + # The layer around a hidden paragraph while it is added. List groups + # stay on it, so a hidden item does not hide the shown items with it. + self._layer_around_hidden: ContentLayer | None = None self.docx_obj = self.load_msword_file( path_or_stream=self.path_or_stream, @@ -900,6 +907,13 @@ def __init__( self.valid = True self.current_part = self.docx_obj.part _normalize_list_style_types(self.docx_obj) + self._uses_hidden_text = any( + part.element.xpath(".//w:vanish") + if isinstance(part, XmlPart) + # python-docx loads footnotes and endnotes as plain parts. + else part.content_type.endswith("+xml") and b"vanish" in part.blob + for part in self.docx_obj.part.package.iter_parts() + ) self._default_paragraph_style = self.docx_obj.styles.default( WD_STYLE_TYPE.PARAGRAPH ) @@ -1127,7 +1141,7 @@ def _get_or_create_list_group( list_gr = doc.add_list_group( name="list", parent=parent, - content_layer=self.content_layer, + content_layer=self._layer_around_hidden or self.content_layer, ) elem_ref.append(list_gr.get_ref()) @@ -2349,10 +2363,52 @@ def _get_hyperlink_target(self, hyperlink: Hyperlink) -> AnyUrl | Path | None: return None + def _is_hidden_run( + self, run: Run, paragraph_style_chain: Sequence[CharacterStyle] + ) -> bool: + """Return True if Word hides ``run`` (the Hidden font property, ``w:vanish``). + + The first explicit setting wins: the run's own formatting, then its + character style chain, then the paragraph style chain. + """ + fonts = ( + run.font, + *(style.font for style in self._get_style_chain(run.style)), + *(style.font for style in paragraph_style_chain), + ) + for font in fonts: + if font.hidden is not None: + return font.hidden + return False + + def _is_hidden_paragraph(self, paragraph: Paragraph) -> bool: + """Return True if ``paragraph`` has text and Word hides all of it. + + An equation counts as shown: its runs are math runs, not ``w:r``. + """ + if not self._uses_hidden_text or ( + paragraph._p.find(f".//{OMML_NS}oMath") is not None + ): + return False + runs = [ + Run(r_el, paragraph) + for r_el in paragraph._p.iter(f"{_W_NS_CLARK}r") + if r_el.find(f"{_W_NS_CLARK}t") is not None + ] + style_chain = self._get_style_chain(paragraph.style) + return bool(runs) and all(self._is_hidden_run(run, style_chain) for run in runs) + def _iter_paragraph_content( self, paragraph: Paragraph ) -> list[tuple[str, Formatting | None, AnyUrl | Path | None]]: content: list[tuple[str, Formatting | None, AnyUrl | Path | None]] = [] + # A fully hidden paragraph keeps its text; it goes to the invisible layer. + show_all = not self._uses_hidden_text or self._is_hidden_paragraph(paragraph) + + def _is_shown(run: Run) -> bool: + return show_all or not self._is_hidden_run( + run, self._get_style_chain(paragraph.style) + ) def _get_children_recursive(node): for child in node: @@ -2366,24 +2422,24 @@ def _get_children_recursive(node): tag_name = etree.QName(child).localname if tag_name == "sdt": - text = "".join( - child.xpath( - ".//w:sdtContent//w:t/text()", - namespaces=_OOXML_NAMESPACES, + runs = [ + run + for run in ( + Run(r_el, paragraph) + for r_el in child.xpath( + ".//w:sdtContent//w:r", + namespaces=_OOXML_NAMESPACES, + ) ) - ) + if _is_shown(run) + ] + text = "".join(t for run in runs for t in run._r.xpath("./w:t/text()")) if len(text) == 0: continue - runs = child.xpath( - ".//w:sdtContent//w:r", - namespaces=_OOXML_NAMESPACES, - ) fmt = ( self._get_format_from_run( - Run(runs[0], paragraph), - paragraph, - self._get_style_chain(paragraph.style), + runs[0], paragraph, self._get_style_chain(paragraph.style) ) if runs else None @@ -2404,11 +2460,14 @@ def _get_children_recursive(node): # Hyperlink.text and Hyperlink.runs only read the direct w:r # children, so also read the runs nested in the same wrappers # as above (e.g. a tracked insertion inside the link). - runs = [ + all_runs = [ Run(r_el, paragraph) for r_el in _get_children_recursive(child) if r_el.tag == f"{_W_NS_CLARK}r" ] + runs = [run for run in all_runs if _is_shown(run)] + if all_runs and not runs: + continue # a fully hidden link content.append( ( "".join(run.text for run in runs), @@ -2424,7 +2483,7 @@ def _get_children_recursive(node): self._get_hyperlink_target(item), ) ) - elif isinstance(item, Run): + elif _is_shown(item): content.append( ( item.text, @@ -2836,10 +2895,31 @@ def _handle_textbox_content( self.parents[level] = original_parent return elem_ref + def _get_hidden_text_elements( + self, element: BaseOxmlElement + ) -> set[etree._Element]: + """Return the ``w:t`` elements of the hidden runs in ``element``. + + Only an element with an equation needs them, so for any other element + the result is empty. + """ + hidden: set[etree._Element] = set() + if not self._uses_hidden_text or element.find(f".//{OMML_NS}oMath") is None: + return hidden + for p_el in element.iter(f"{_W_NS_CLARK}p"): + paragraph = Paragraph(p_el, self.docx_obj) + style_chain = self._get_style_chain(paragraph.style) + for r_el in p_el.iter(f"{_W_NS_CLARK}r"): + if self._is_hidden_run(Run(r_el, paragraph), style_chain): + hidden.update(r_el.iter(f"{_W_NS_CLARK}t")) + return hidden + def _handle_equations_in_text(self, element, text): only_texts = [] only_equations = [] texts_and_equations = [] + # ``text`` has no hidden text, so leave it out here too. + hidden_texts = self._get_hidden_text_elements(element) # Collect oMath elements and text runs from the paragraph. # Use direct children iteration first; fall back to deep iteration @@ -2867,7 +2947,11 @@ def _handle_equations_in_text(self, element, text): # Collect text from non-math children (e.g. runs) for t_elem in child.iter(): t_tag = etree.QName(t_elem).localname - if t_tag == "t" and "math" not in t_elem.tag: + if ( + t_tag == "t" + and "math" not in t_elem.tag + and t_elem not in hidden_texts + ): if isinstance(t_elem.text, str): only_texts.append(t_elem.text) texts_and_equations.append(t_elem.text) @@ -2876,7 +2960,11 @@ def _handle_equations_in_text(self, element, text): # inside oMathPara or other wrapper elements). for subt in element.iter(): tag_name = etree.QName(subt).localname - if tag_name == "t" and "math" not in subt.tag: + if ( + tag_name == "t" + and "math" not in subt.tag + and subt not in hidden_texts + ): if isinstance(subt.text, str): only_texts.append(subt.text) texts_and_equations.append(subt.text) @@ -2945,6 +3033,18 @@ def _handle_text_elements( ) -> list[RefItem]: elem_ref: list[RefItem] = [] paragraph = Paragraph(element, self.docx_obj) + if self.content_layer != ContentLayer.INVISIBLE and self._is_hidden_paragraph( + paragraph + ): + # Word does not show it: keep it, like a hidden slide or sheet, + # but out of the default exports. + self._layer_around_hidden = self.content_layer + self.content_layer = ContentLayer.INVISIBLE + try: + return self._handle_text_elements(element, doc, skip_empty_text) + finally: + self.content_layer = self._layer_around_hidden + self._layer_around_hidden = None paragraph_elements = self._get_paragraph_elements(paragraph) text, equations = self._handle_equations_in_text( element=element, text=self._get_paragraph_text(paragraph) @@ -3070,9 +3170,8 @@ def _handle_text_elements( elem_ref.extend(h1) elif len(equations) > 0: - if (paragraph.text is None or len(paragraph.text.strip()) == 0) and len( - text - ) > 0: + # The paragraph's text without equations or hidden runs. + if len(self._get_paragraph_text(paragraph).strip()) == 0 and len(text) > 0: # Standalone equation(s) — emit each as a separate formula level = self._get_level() parent = self.parents[level - 1] @@ -3320,6 +3419,7 @@ def _add_heading( parent=self.parents[parent_level], text=text, level=add_level, + content_layer=self.content_layer, ) self.parents[current_level] = hd elem_ref.append(hd.get_ref()) @@ -3353,6 +3453,7 @@ def _add_formatted_list_item( text=text, formatting=format, hyperlink=hyperlink, + content_layer=self.content_layer, ) else: new_item = doc.add_list_item( @@ -3360,8 +3461,11 @@ def _add_formatted_list_item( enumerated=enumerated, parent=self.parents[level], text="", + content_layer=self.content_layer, + ) + new_parent = doc.add_inline_group( + parent=new_item, content_layer=self.content_layer ) - new_parent = doc.add_inline_group(parent=new_item) for text, format, hyperlink in elements: if text: doc.add_text( @@ -3589,7 +3693,7 @@ def _manage_list_structure( list_gr1 = doc.add_list_group( name="list", parent=self._sub_list_parent(doc, self.parents.get(i - 1)), - content_layer=self.content_layer, + content_layer=self._layer_around_hidden or self.content_layer, ) self.parents[i] = list_gr1 elem_ref.append(list_gr1.get_ref()) @@ -3739,6 +3843,7 @@ def _add_list_item_with_equations( enumerated=is_numbered, parent=self.parents[use_level], text="", + content_layer=self.content_layer, ) inline_group = doc.add_inline_group( @@ -3877,12 +3982,27 @@ def _handle_tables( cell = _Cell(tc, table) + # cell.text includes hidden runs, so rebuild it from the + # paragraphs' visible content when the cell has any. A cell has + # no layer of its own, so fully hidden paragraphs are left out. + cell_text = cell.text + if self._uses_hidden_text and any( + self._is_hidden_run(Run(r_el, p), self._get_style_chain(p.style)) + for p in cell.paragraphs + for r_el in p._p.xpath(".//w:r") + ): + cell_text = "\n".join( + self._get_paragraph_text(p) + for p in cell.paragraphs + if not self._is_hidden_paragraph(p) + ) + # Detect equations in cell text text, equations = self._handle_equations_in_text( - element=cell._element, text=cell.text + element=cell._element, text=cell_text ) if len(equations) == 0: - text = cell.text + text = cell_text else: text = text.replace("", "$").replace("", "$") @@ -4777,14 +4897,15 @@ def _add_footnotes_and_endnotes( for note in root.findall(f"{_W_NS_CLARK}{tag_name}"): if note.get(f"{_W_NS_CLARK}type") in skip_types: continue + # A note has no layer per paragraph, so hidden ones are left out. texts = [ text - for p_elm in note.findall(f".//{_W_NS_CLARK}p") - if ( - text := self._get_paragraph_text( - Paragraph(p_elm, docx_obj) - ).strip() + for paragraph in ( + Paragraph(p_elm, docx_obj) + for p_elm in note.findall(f".//{_W_NS_CLARK}p") ) + if not self._is_hidden_paragraph(paragraph) + and (text := self._get_paragraph_text(paragraph).strip()) ] if not texts: continue diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index 71dd664d30..c0245ed6ce 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -2563,6 +2563,31 @@ def test_footnote_body_text_is_not_dropped_outside_plain_paragraphs( assert expected in footnote_items +def test_hidden_footnote_paragraph_is_left_out(): + """A hidden paragraph in a footnote is left out of the footnote text. + + python-docx loads the footnotes part as a plain part, so this also checks + that ``w:vanish`` there is found when the body has none. + """ + note_body = ( + "Shown note." + "Hidden note." + ) + stream = DocumentStream( + name="notes.docx", stream=_docx_with_notes(footnote_body=note_body) + ) + result = DocumentConverter(allowed_formats=[InputFormat.DOCX]).convert( + stream, raises_on_error=True + ) + + footnote_items = [ + item.text + for item in result.document.texts + if item.label == DocItemLabel.FOOTNOTE + ] + assert footnote_items == ["Shown note.", "An endnote body."] + + _MC_BODY_BLOCK = """