diff --git a/docling/backend/afp_backend.py b/docling/backend/afp_backend.py index f714d70789..f4a4311e9c 100644 --- a/docling/backend/afp_backend.py +++ b/docling/backend/afp_backend.py @@ -33,6 +33,7 @@ DeclarativeDocumentBackend, PaginatedDocumentBackend, ) +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -216,8 +217,9 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, + options: BackendOptions | None = None, ) -> None: - super().__init__(in_doc, path_or_stream) + super().__init__(in_doc, path_or_stream, options) self.page_range = in_doc.limits.page_range try: self.content = ( diff --git a/docling/backend/asciidoc_backend.py b/docling/backend/asciidoc_backend.py index 8d362b42e6..3b2009f289 100644 --- a/docling/backend/asciidoc_backend.py +++ b/docling/backend/asciidoc_backend.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from io import BytesIO from pathlib import Path -from typing import Final, Optional, Union +from typing import Final, Optional, TypedDict, Union from docling_core.types.doc import ( DocItemLabel, @@ -45,6 +45,15 @@ _LIST_ITEM_PATTERN: Final = r"^(\s*)(\*+|-|\.+|\d+\.|\w+\.)\s+(.*)" +class _ParsedBlock(TypedDict, total=False): + type: str + text: str + level: int + indent: int + marker: str + numbered: bool + + @dataclass(frozen=True) class _LiteralBlock: text: str @@ -121,7 +130,7 @@ def convert(self) -> DoclingDocument: return doc - def _parse(self, doc: DoclingDocument): + def _parse(self, doc: DoclingDocument) -> DoclingDocument: """Orchestrate parsing and populate `doc` from the source lines. Handles titles, section headers, text paragraphs, lists, tables, @@ -151,9 +160,9 @@ def _parse(self, doc: DoclingDocument): list_continuation = False # parents: dict[int, Union[DocItem, GroupItem, None]] = {} - parents: dict[int, Union[GroupItem, None]] = {} + parents: dict[int, NodeItem | None] = {} # indents: dict[int, Union[DocItem, GroupItem, None]] = {} - indents: dict[int, Union[GroupItem, None]] = {} + indents: dict[int, int | None] = {} for i in range(10): parents[i] = None @@ -177,10 +186,12 @@ def _parse(self, doc: DoclingDocument): ) caption: Optional[TextItem] = None if caption_data: - caption = doc.add_text( + caption_item = doc.add_text( text=" ".join(caption_data), label=DocItemLabel.CAPTION, ) + assert isinstance(caption_item, TextItem) + caption = caption_item caption_data = [] code_language = ( detect_code_language(block.text, hint=block.language) @@ -250,6 +261,7 @@ def _parse(self, doc: DoclingDocument): level = self._get_current_level(parents) + current_indent = indents[level] if not in_list: in_list = True # GroupItem has no caption slot; emit the pending block @@ -268,14 +280,18 @@ def _parse(self, doc: DoclingDocument): ) indents[level + 1] = item["indent"] - elif in_list and item["indent"] > indents[level]: + elif current_indent is not None and item["indent"] > current_indent: parents[level + 1] = doc.add_group( parent=parents[level], name="list", label=GroupLabel.LIST ) indents[level + 1] = item["indent"] - elif in_list and item["indent"] < indents[level]: - while level > 0 and item["indent"] < indents[level]: + elif current_indent is not None and item["indent"] < current_indent: + while ( + level > 0 + and current_indent is not None + and item["indent"] < current_indent + ): # Only pop the current level if there is an outer group # to fall back to; otherwise keep it as the list root. if indents[level - 1] is None: @@ -283,13 +299,16 @@ def _parse(self, doc: DoclingDocument): parents[level] = None indents[level] = None level -= 1 + current_indent = indents[level] - last_list_item = doc.add_list_item( + list_item = doc.add_list_item( item["text"], enumerated=item["numbered"], marker=(item["marker"] if item["marker"][:-1].isdigit() else None), parent=self._get_current_parent(parents), ) + assert isinstance(list_item, ListItem) + last_list_item = list_item list_continuation = False elif in_list and stripped_line == "+": @@ -383,7 +402,7 @@ def _parse(self, doc: DoclingDocument): return doc @staticmethod - def _get_current_level(parents): + def _get_current_level(parents: dict[int, NodeItem | None]) -> int: for k, v in parents.items(): if v is None and k > 0: return k - 1 @@ -391,14 +410,16 @@ def _get_current_level(parents): return 0 @staticmethod - def _get_current_parent(parents): + def _get_current_parent(parents: dict[int, NodeItem | None]) -> NodeItem | None: for k, v in parents.items(): if v is None and k > 0: return parents[k - 1] return None - _SOURCE_ATTR_RE = re.compile(r"^\[source(?:,\s*([\w+#.-]+))?[^\]]*\]$") + _SOURCE_ATTR_RE: re.Pattern[str] = re.compile( + r"^\[source(?:,\s*([\w+#.-]+))?[^\]]*\]$" + ) _CONTENT_BLOCK_DELIMITERS = ("====", "****", "____", "--", "+++") @staticmethod @@ -464,7 +485,9 @@ def _iter_blocks(cls, lines: list[str]) -> Iterator[str | _LiteralBlock]: if source_attr is not None and i + 1 < n and lines[i + 1].strip() == "----": block_data = [] block_delimiter = "----" - block_language = source_attr.group(1) + language = source_attr.group(1) + assert language is None or isinstance(language, str) + block_language = language i += 2 continue @@ -480,7 +503,7 @@ def _close_list_if_needed( *, line: str, in_list: bool, - parents: dict[int, GroupItem | None], + parents: dict[int, NodeItem | None], last_list_item: ListItem | None, list_continuation: bool, is_continuation_block: bool, @@ -502,7 +525,7 @@ def _flush_text_data( *, doc: DoclingDocument, text_data: list[str], - parent: GroupItem | None, + parent: NodeItem | None, ) -> list[str]: if len(text_data) > 0: doc.add_text( @@ -514,22 +537,24 @@ def _flush_text_data( # ========= Title @staticmethod - def _is_title(line): + def _is_title(line: str) -> re.Match[str] | None: return re.match(r"^= ", line) @staticmethod - def _parse_title(line): + def _parse_title(line: str) -> _ParsedBlock: return {"type": "title", "text": line[2:].strip(), "level": 0} # ========= Section headers @staticmethod - def _is_section_header(line): + def _is_section_header(line: str) -> re.Match[str] | None: return re.match(r"^==+\s+", line) @staticmethod - def _parse_section_header(line): + def _parse_section_header(line: str) -> _ParsedBlock: match = re.match(r"^(=+)\s+(.*)", line) + if match is None: + raise ValueError(f"Invalid AsciiDoc section header: {line!r}") marker = match.group(1) # The list marker (e.g., "*", "-", "1.") text = match.group(2) # The actual text of the list item @@ -542,11 +567,11 @@ def _parse_section_header(line): # ========= Lists @staticmethod - def _is_list_item(line): + def _is_list_item(line: str) -> re.Match[str] | None: return re.match(_LIST_ITEM_PATTERN, line) @staticmethod - def _parse_list_item(line): + def _parse_list_item(line: str) -> _ParsedBlock: """Extract the item marker (number or bullet symbol) and the text of the item.""" match = re.match(_LIST_ITEM_PATTERN, line) @@ -586,11 +611,11 @@ def _parse_list_item(line): # ========= Tables @staticmethod - def _is_table_line(line): + def _is_table_line(line: str) -> re.Match[str] | None: return re.match(rf"^{_CELL_SPEC}\|.*\|", line) @staticmethod - def _parse_table_line(line): + def _parse_table_line(line: str) -> list[str]: # Drop cell specifiers glued to a "|" (e.g. "^.^h"); anchored to # whitespace so content ending in a style letter (e.g. "Eth") survives. line = re.sub(rf"(^|\s){_CELL_SPEC}(?=\|)", r"\1", line) @@ -619,7 +644,7 @@ def _add_table_if_nonempty( doc.add_table(data=data, parent=parent, caption=caption) @staticmethod - def _populate_table_as_grid(table_data): + def _populate_table_as_grid(table_data: list[list[str]]) -> TableData: num_rows = len(table_data) # Adjust the table data into a grid format @@ -651,19 +676,22 @@ def _populate_table_as_grid(table_data): # ========= Pictures @staticmethod - def _is_picture(line): + def _is_picture(line: str) -> re.Match[str] | None: return re.match(r"^image::", line) @staticmethod - def _parse_picture(line): + def _parse_picture(line: str) -> dict[str, str]: """ Parse an image macro, extracting its path and attributes. Syntax: image::path/to/image.png[Alt Text, width=200, height=150, align=center] """ mtch = re.match(r"^image::(.+)\[(.*)\]$", line) if mtch: - picture_path = mtch.group(1).strip() - attributes = mtch.group(2).split(",") + path = mtch.group(1) + attrs = mtch.group(2) + assert isinstance(path, str) and isinstance(attrs, str) + picture_path = path.strip() + attributes = attrs.split(",") picture_info = {"type": "picture", "uri": picture_path} # Extract optional attributes (alt text, width, height, alignment) @@ -683,11 +711,11 @@ def _parse_picture(line): # ========= Captions @staticmethod - def _is_caption(line): + def _is_caption(line: str) -> re.Match[str] | None: return re.match(r"^\.(\S.*)", line) @staticmethod - def _parse_caption(line): + def _parse_caption(line: str) -> _ParsedBlock: mtch = re.match(r"^\.(.+)", line) if mtch: text = mtch.group(1) @@ -697,5 +725,5 @@ def _parse_caption(line): # ========= Plain text @staticmethod - def _parse_text(line): + def _parse_text(line: str) -> _ParsedBlock: return {"type": "text", "text": line.strip()} diff --git a/docling/backend/boxnote_backend.py b/docling/backend/boxnote_backend.py index d0e5b412a4..38d32c8fb7 100644 --- a/docling/backend/boxnote_backend.py +++ b/docling/backend/boxnote_backend.py @@ -29,6 +29,7 @@ clamp_span, table_width, ) +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -61,8 +62,13 @@ class BoxNoteDocumentBackend(DeclarativeDocumentBackend): """ @override - def __init__(self, in_doc: InputDocument, path_or_stream: BytesIO | Path): - super().__init__(in_doc, path_or_stream) + def __init__( + self, + in_doc: InputDocument, + path_or_stream: BytesIO | Path, + options: BackendOptions | None = None, + ): + super().__init__(in_doc, path_or_stream, options) self.data: dict[str, Any] = {} # utf-8-sig drops a leading BOM. Kept, it reaches json.loads, which @@ -122,7 +128,7 @@ def convert(self) -> DoclingDocument: return doc def _add_blocks( - self, nodes: list[dict], doc: DoclingDocument, parent: NodeItem | None + self, nodes: list[dict[str, Any]], doc: DoclingDocument, parent: NodeItem | None ) -> None: for node in nodes: self._add_block(node, doc, parent) @@ -170,7 +176,10 @@ def _add_block( self._add_blocks(content, doc, parent) def _add_paragraph( - self, content: list[dict], doc: DoclingDocument, parent: NodeItem | None + self, + content: list[dict[str, Any]], + doc: DoclingDocument, + parent: NodeItem | None, ) -> None: runs = self._runs(content) if not runs: @@ -198,7 +207,7 @@ def _add_paragraph( def _add_list( self, list_type: str, - items: list[dict], + items: list[dict[str, Any]], doc: DoclingDocument, parent: NodeItem | None, ) -> None: @@ -253,14 +262,14 @@ def _add_check_item( def _split_item( self, item: dict[str, Any] - ) -> tuple[str, Formatting | None, AnyUrl | None, list[dict]]: + ) -> tuple[str, Formatting | None, AnyUrl | None, list[dict[str, Any]]]: """Split a list item into its own text and any nested blocks. The leading paragraph becomes the item text; everything else (nested lists, extra paragraphs) is returned to be added as children. """ text, formatting, hyperlink = "", None, None - nested: list[dict] = [] + nested: list[dict[str, Any]] = [] for child in item.get("content", []): if not text and child.get("type") == "paragraph": text, formatting, hyperlink = self._collapse(child.get("content", [])) @@ -269,7 +278,7 @@ def _split_item( return text, formatting, hyperlink, nested def _add_table( - self, rows: list[dict], doc: DoclingDocument, parent: NodeItem | None + self, rows: list[dict[str, Any]], doc: DoclingDocument, parent: NodeItem | None ) -> None: rows = [row for row in rows if row.get("type") == "table_row"] if not rows: @@ -359,7 +368,7 @@ def _add_table( table.data.num_cols = num_cols - def _cell_is_rich(self, blocks: list[dict]) -> bool: + def _cell_is_rich(self, blocks: list[dict[str, Any]]) -> bool: """A cell is rich when it cannot be one plain-text TableCell. That covers images, multiple blocks, any non-paragraph block (lists, @@ -391,7 +400,7 @@ def _add_image( caption = doc.add_text(label=DocItemLabel.CAPTION, text=label) doc.add_picture(caption=caption, parent=parent) - def _collapse(self, content: list[dict]) -> _Run: + def _collapse(self, content: list[dict[str, Any]]) -> _Run: """Reduce inline content to a single run. Headings and list items take one formatting and one hyperlink, so a @@ -403,7 +412,7 @@ def _collapse(self, content: list[dict]) -> _Run: return runs[0] return "".join(text for text, _, _ in runs), None, None - def _runs(self, content: list[dict]) -> list[_Run]: + def _runs(self, content: list[dict[str, Any]]) -> list[_Run]: runs: list[_Run] = [] for node in content or []: node_type = node.get("type") @@ -416,7 +425,9 @@ def _runs(self, content: list[dict]) -> list[_Run]: runs.append((" ", None, None)) return runs - def _marks(self, marks: list[dict]) -> tuple[Formatting | None, AnyUrl | None]: + def _marks( + self, marks: list[dict[str, Any]] + ) -> tuple[Formatting | None, AnyUrl | None]: formatting: Formatting | None = None hyperlink: AnyUrl | None = None for mark in marks or []: @@ -439,7 +450,7 @@ def _marks(self, marks: list[dict]) -> tuple[Formatting | None, AnyUrl | None]: hyperlink = self._as_url(href) return formatting, hyperlink - def _plain_text(self, nodes: list[dict]) -> str: + def _plain_text(self, nodes: list[dict[str, Any]]) -> str: parts: list[str] = [] for node in nodes or []: node_type = node.get("type") @@ -451,7 +462,7 @@ def _plain_text(self, nodes: list[dict]) -> str: parts.append(self._plain_text(node["content"])) return "".join(parts) - def _cell_text(self, blocks: list[dict]) -> str: + def _cell_text(self, blocks: list[dict[str, Any]]) -> str: texts = (self._plain_text(block.get("content", [])).strip() for block in blocks) return " ".join(text for text in texts if text) diff --git a/docling/backend/docling_parse_backend.py b/docling/backend/docling_parse_backend.py index 729efb761d..f95b927eb2 100644 --- a/docling/backend/docling_parse_backend.py +++ b/docling/backend/docling_parse_backend.py @@ -180,7 +180,7 @@ def get_segmented_page(self) -> Optional[SegmentedPdfPage]: def get_text_cells(self) -> Iterable[TextCell]: segmented_page = self.get_segmented_page() if segmented_page is None: - return [] + return list[TextCell]() return segmented_page.textline_cells @override @@ -194,7 +194,7 @@ def get_visible_text_cells(self) -> Optional[list[TextCell]]: def get_bitmap_rects(self, scale: float = 1) -> Iterable[BoundingBox]: segmented_page = self.get_segmented_page() if segmented_page is None: - return [] + return list[BoundingBox]() page_height = self.get_size().height cropboxes: list[BoundingBox] = [] @@ -343,7 +343,7 @@ def __init__( if isinstance(self.path_or_stream, BytesIO): self.path_or_stream.seek(0) self.doc_key = self.parser.load( - self.path_or_stream, + path_or_stream, password=password, page_range=in_doc.limits.page_range, ) diff --git a/docling/backend/docx/drawingml/utils.py b/docling/backend/docx/drawingml/utils.py index 1646552be6..707968d705 100644 --- a/docling/backend/docx/drawingml/utils.py +++ b/docling/backend/docx/drawingml/utils.py @@ -142,7 +142,7 @@ def _build_soffice_command( ] -def _kill_soffice_process_group(proc: subprocess.Popen) -> None: +def _kill_soffice_process_group(proc: subprocess.Popen[bytes]) -> None: """Best-effort SIGKILL of the whole ``soffice`` process group. ``soffice`` is a thin wrapper that forks ``soffice.bin``; killing only @@ -328,7 +328,7 @@ def convert_to_modern_format( shutil.rmtree(tmp_dir, ignore_errors=True) -def get_docx_to_pdf_converter() -> Optional[Callable]: +def get_docx_to_pdf_converter() -> Optional[Callable[[Path, Path], None]]: """ Detects the best available DOCX to PDF tool and returns a conversion function. The returned function accepts (input_path, output_path). @@ -411,7 +411,7 @@ def crop_whitespace(image: Image.Image, bg_color=None, padding=0) -> Image.Image def get_pil_from_dml_docx( - docx: Document, converter: Optional[Callable] + docx: Document, converter: Optional[Callable[[Path, Path], None]] ) -> Optional[Image.Image]: if converter is None: return None diff --git a/docling/backend/docx/latex/omml.py b/docling/backend/docx/latex/omml.py index 4fdad3d0a3..880857d8f0 100644 --- a/docling/backend/docx/latex/omml.py +++ b/docling/backend/docx/latex/omml.py @@ -110,7 +110,7 @@ def escape_latex(strs: str) -> str: return BLANK.join(new_chr) -def get_val(key: str | None, default: str, store: dict | None = CHR) -> str: +def get_val(key: str | None, default: str, store: dict[str, str] | None = CHR) -> str: """Get a value from a dictionary store or return the key/default. When default is provided, this function always returns a non-None string. @@ -146,7 +146,9 @@ def call_method(self, elm: _Element, stag: str | None = None) -> Any | None: """ getmethod = self.tag2meth.get if stag is None: - stag = elm.tag.replace(OMML_NS, "") + tag = elm.tag + assert isinstance(tag, str) + stag = str(tag).replace(OMML_NS, "") method = getmethod(stag) if method: return method(self, elm) @@ -232,7 +234,7 @@ class Pr(Tag2Method): text: str __val_tags: tuple[str, ...] - __innerdict: dict[str, Any] + __innerdict: dict[str, str | None] def __init__(self, elm: _Element): """Initialize properties processor. @@ -253,7 +255,7 @@ def __unicode__(self) -> str: """Return unicode representation.""" return self.__str__() - def __getattr__(self, name: str) -> Any | None: + def __getattr__(self, name: str) -> str | None: """Get attribute from internal dictionary. Args: @@ -387,6 +389,7 @@ def do_bar(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm) pr = c_dict["barPr"] + assert isinstance(pr, Pr) latex_s = get_val(pr.pos, default=POS_DEFAULT["BAR_VAL"], store=POS) # If latex_s contains %s, format it; otherwise return as-is (unmapped character) if "%s" in latex_s: @@ -405,6 +408,7 @@ def do_d(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm) pr = c_dict["dPr"] + assert isinstance(pr, Pr) null = D_DEFAULT["null"] s_val = get_val(pr.begChr, default=D_DEFAULT["left"], store=T) @@ -470,10 +474,11 @@ def do_ssub(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm, include=("e", "sub", "sSubPr")) base = c_dict.get("e", "").rstrip() + assert isinstance(base, str) sub = self._unwrap_script(c_dict.get("sub", ""), "_") if self._needs_grouping(base): base = "{" + base + "}" - return base + (SUB % sub) + return f"{base}_{{{sub}}}" def do_ssup(self, elm: _Element) -> str: """Process superscript element. @@ -486,10 +491,11 @@ def do_ssup(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm, include=("e", "sup", "sSupPr")) base = c_dict.get("e", "").rstrip() + assert isinstance(base, str) sup = self._unwrap_script(c_dict.get("sup", ""), "^") if self._needs_grouping(base): base = "{" + base + "}" - return base + (SUP % sup) + return f"{base}^{{{sup}}}" def do_ssubsup(self, elm: _Element) -> str: """Process combined sub-superscript element. @@ -504,11 +510,12 @@ def do_ssubsup(self, elm: _Element) -> str: elm, include=("e", "sub", "sup", "sSubSupPr") ) base = c_dict.get("e", "").rstrip() + assert isinstance(base, str) sub = self._unwrap_script(c_dict.get("sub", ""), "_") sup = self._unwrap_script(c_dict.get("sup", ""), "^") if self._needs_grouping(base): base = "{" + base + "}" - return base + (SUB % sub) + (SUP % sup) + return f"{base}_{{{sub}}}^{{{sup}}}" def do_sub(self, elm: _Element) -> str: """Process standalone subscript content. @@ -551,6 +558,7 @@ def do_f(self, elm: _Element) -> str: "num": c_dict.get("num"), "den": c_dict.get("den"), } + assert isinstance(pr, Pr) latex_s = get_val(pr.type, default=F_DEFAULT, store=F) return pr.text + ( latex_s % {"num": c_dict.get("num"), "den": c_dict.get("den")} @@ -567,6 +575,7 @@ def do_func(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm) func_name = c_dict.get("fName", "") + assert isinstance(func_name, str) return func_name.replace(FUNC_PLACE, c_dict.get("e", "")) def do_fname(self, elm: _Element) -> str: @@ -606,6 +615,7 @@ def do_groupchr(self, elm: _Element) -> str: """ c_dict = self.process_children_dict(elm) pr = c_dict["groupChrPr"] + assert isinstance(pr, Pr) latex_s = get_val(pr.chr, default=CHR_DEFAULT["GROUPCHR_VAL"], store=CHR) # If latex_s contains %s, format it; otherwise return as-is (unmapped character) if "%s" in latex_s: @@ -784,7 +794,7 @@ def process_unicode(self, s: str) -> str: if s in self._MATH_CHAR_MAP: return self._MATH_CHAR_MAP[s] - out_latex_str = self.u.unicode_to_latex(s) + out_latex_str = str(self.u.unicode_to_latex(s)) if ( s.startswith("{") is False diff --git a/docling/backend/ebcdic_backend.py b/docling/backend/ebcdic_backend.py index 5642083064..8f9862a357 100644 --- a/docling/backend/ebcdic_backend.py +++ b/docling/backend/ebcdic_backend.py @@ -32,6 +32,7 @@ from docling.backend.abstract_backend import DeclarativeDocumentBackend from docling.datamodel.backend_options import ( + BaseBackendOptions, EbcdicBackendOptions, EbcdicField, EbcdicFieldType, @@ -221,10 +222,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path], - options: Union[EbcdicBackendOptions, None] = None, + options: Union[BaseBackendOptions, None] = None, ) -> None: if options is None: options = EbcdicBackendOptions() + if not isinstance(options, EbcdicBackendOptions): + raise TypeError("Expected EbcdicBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.layout = self._resolve_layout() diff --git a/docling/backend/epub_backend.py b/docling/backend/epub_backend.py index ec8d89b1ed..dc1a8dde30 100644 --- a/docling/backend/epub_backend.py +++ b/docling/backend/epub_backend.py @@ -28,6 +28,8 @@ class EpubDocumentBackend(DeclarativeDocumentBackend): + options: EpubBackendOptions + """Backend for converting EPUB files to DoclingDocument format. EPUB files are essentially ZIP archives containing XHTML content files. @@ -458,7 +460,7 @@ def convert(self) -> DoclingDocument: if self.temp_dir: dummy_file = self.temp_dir / "combined.html" - html_options.source_uri = str(dummy_file) + html_options.source_uri = dummy_file _log.debug(f"Set HTML backend source_uri to: {html_options.source_uri}") in_doc = InputDocument( diff --git a/docling/backend/html_backend.py b/docling/backend/html_backend.py index 640a3d58df..982e77ea09 100644 --- a/docling/backend/html_backend.py +++ b/docling/backend/html_backend.py @@ -24,6 +24,7 @@ import requests import urllib3 +import urllib3.exceptions from docling_core.types.doc import ( BoundingBox, CodeLanguageLabel, @@ -39,6 +40,8 @@ GraphLinkLabel, GroupItem, GroupLabel, + ListItem, + NodeItem, PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, @@ -56,7 +59,7 @@ from docling_core.types.doc.document import ContentLayer, Formatting, ImageRef, Script from PIL import Image, UnidentifiedImageError from pydantic import AnyUrl, BaseModel, ValidationError -from typing_extensions import Self, override +from typing_extensions import override from docling.backend.abstract_backend import ( DeclarativeDocumentBackend, @@ -68,7 +71,7 @@ clamp_span, table_width, ) -from docling.datamodel.backend_options import HTMLBackendOptions +from docling.datamodel.backend_options import BaseBackendOptions, HTMLBackendOptions from docling.datamodel.base_models import FormatToMimeType, InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError, OperationNotAllowed @@ -322,7 +325,7 @@ class AnnotatedText(BaseModel): source_tag_id: Optional[str] = None -class AnnotatedTextList(list): +class AnnotatedTextList(list[AnnotatedText]): def to_single_text_element(self) -> AnnotatedText: current_h = None current_text = "" @@ -420,7 +423,7 @@ def simplify_text_elements(self) -> AnnotatedTextList: ) return simplified - def split_by_newline(self) -> list[Self]: + def split_by_newline(self) -> list[AnnotatedTextList]: """Split text elements on multiple consecutive line breaks (from
tags). Single
tags are converted to newline characters (\n) within the same paragraph. @@ -428,7 +431,7 @@ def split_by_newline(self) -> list[Self]: Regular newlines from HTML source formatting have already been normalized to spaces during text extraction. """ - super_list: list[Self] = [] + super_list: list[AnnotatedTextList] = [] active_annotated_text_list = AnnotatedTextList() double_sentinel = _BR_SENTINEL + _BR_SENTINEL @@ -464,35 +467,40 @@ def __init__( self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path], - options: Optional[HTMLBackendOptions] = None, + options: Optional[BaseBackendOptions] = None, ): if not _BS4_AVAILABLE: raise ImportError(_INSTALL_HINT) from _BS4_IMPORT_ERROR if options is None: options = HTMLBackendOptions() - super().__init__(in_doc, path_or_stream, options) + typed_options = cast(HTMLBackendOptions, options) + super().__init__(in_doc, path_or_stream, typed_options) self.options: HTMLBackendOptions self.soup: Optional[BeautifulSoup] = None - self.path_or_stream: Union[BytesIO, Path] = path_or_stream + self.path_or_stream: Union[BytesIO, Path, None] = path_or_stream configured_base_path: Optional[str] = ( - str(options.source_uri) if options.source_uri is not None else None + str(typed_options.source_uri) + if typed_options.source_uri is not None + else None ) self.base_path = configured_base_path self._mhtml_resources: dict[str, bytes] | None = None self._image_loader = ImageResourceLoader( - enable_local_fetch=options.enable_local_fetch, - enable_remote_fetch=options.enable_remote_fetch, - max_image_data_base64_bytes=options.max_image_data_base64_bytes, - max_remote_image_bytes=options.max_remote_image_bytes, - max_redirects=options.max_redirects, - headers=options.headers, - header_origins=self._get_header_origins(options, configured_base_path), + enable_local_fetch=typed_options.enable_local_fetch, + enable_remote_fetch=typed_options.enable_remote_fetch, + max_image_data_base64_bytes=typed_options.max_image_data_base64_bytes, + max_remote_image_bytes=typed_options.max_remote_image_bytes, + max_redirects=typed_options.max_redirects, + headers=typed_options.headers, + header_origins=self._get_header_origins( + typed_options, configured_base_path + ), ) # Initialize the parents for the hierarchy self.max_levels = 10 self.level = 0 - self.parents: dict[int, Optional[Union[DocItem, GroupItem]]] = {} + self.parents: dict[int, NodeItem | None] = {} self.ctx = _Context() self._disable_inline_group_depth: int = 0 for i in range(self.max_levels): @@ -519,7 +527,7 @@ def __init__( else Path(path_or_stream).read_bytes() ) if self.input_format == InputFormat.MHTML: - if options.render_page: + if typed_options.render_page: raise DocumentLoadError( "Browser rendering is not supported for MHTML input." ) @@ -875,13 +883,13 @@ def convert(self) -> DoclingDocument: ) doc = DoclingDocument(name=self.file.stem or "file", origin=origin) - if cast(HTMLBackendOptions, self.options).render_page: + if self.options.render_page: self._render_with_browser() if self._rendered_html: self.soup = BeautifulSoup(self._rendered_html, "html.parser") if self._rendered_page_images and self._rendered_page_size: - render_dpi = cast(HTMLBackendOptions, self.options).render_dpi + render_dpi = self.options.render_dpi for page_no, page_image in enumerate(self._rendered_page_images, start=1): doc.add_page( page_no=page_no, @@ -944,6 +952,7 @@ def _get_header_origins( options: HTMLBackendOptions, source_uri: Optional[str] ) -> list[str]: """Return the origins that receive ``options.headers``.""" + origins: list[str] if options.headers_allowed_origins is not None: origins = list(options.headers_allowed_origins) elif source_uri is not None and ImageResourceLoader.is_remote_url(source_uri): @@ -955,7 +964,7 @@ def _get_header_origins( return origins def _get_render_page_size(self) -> tuple[int, int]: - options = cast(HTMLBackendOptions, self.options) + options = self.options width = options.render_page_width height = options.render_page_height if options.render_page_orientation == "landscape": @@ -1076,12 +1085,12 @@ def _pad_image(self, image: Image.Image, width: int, height: int) -> Image.Image return canvas def _render_with_browser(self) -> None: - options = cast(HTMLBackendOptions, self.options) + options = self.options if not options.render_page: return try: - from playwright.sync_api import sync_playwright # type: ignore + from playwright.sync_api import sync_playwright except ImportError as exc: raise RuntimeError( "Playwright is required for HTML rendering. " @@ -1310,7 +1319,7 @@ def _route_request(route, request) -> None: def _capture_page_images( self, page, - render_data: dict, + render_data: dict[str, Any], page_width: int, page_height: int, full_page: bool, @@ -1346,7 +1355,7 @@ def _capture_page_images( return page_images def _build_bbox_mapping( - self, render_data: dict, page_height: int, full_page: bool + self, render_data: dict[str, Any], page_height: int, full_page: bool ) -> dict[str, _RenderedBBox]: boxes = render_data.get("boxes", {}) or {} scroll_height = float(render_data.get("scrollHeight", page_height)) @@ -1459,8 +1468,6 @@ def _has_rendered_presence(self, tag: Tag) -> bool: has_visible_descendant = False for descendant in tag.find_all(True): - if not isinstance(descendant, Tag): - continue if ( self._get_rendered_text_bbox_for_tag(descendant) is not None or self._get_rendered_bbox_for_tag(descendant) is not None @@ -1527,8 +1534,6 @@ def _make_text_prov( if render_box is None and isinstance(tag, Tag): descendant_boxes: list[_RenderedBBox] = [] for descendant in [tag, *tag.find_all(True)]: - if not isinstance(descendant, Tag): - continue descendant_box = self._get_rendered_text_bbox_for_tag(descendant) if descendant_box is not None: descendant_boxes.append(descendant_box) @@ -1620,7 +1625,7 @@ def _are_source_tag_ids_inline_neighbors( horizontal_gap = right_box.l - left_box.r max_gap = max(8.0, 1.5 * max_height) - min_gap = -0.5 * min(left_box.width, right_box.width) + min_gap = -0.5 * float(min(left_box.width, right_box.width)) return min_gap <= horizontal_gap <= max_gap def _compact_adjacent_single_char_parts( @@ -1734,8 +1739,11 @@ def _start_para(): nonlocal current_p if current_p is None: current_p = soup.new_tag("p") - if p.get(_DATA_DOCLING_ID_ATTR): - current_p[_DATA_DOCLING_ID_ATTR] = p.get(_DATA_DOCLING_ID_ATTR) + source_id = HTMLDocumentBackend._get_attr_as_string( + p, _DATA_DOCLING_ID_ATTR + ) + if source_id: + current_p[_DATA_DOCLING_ID_ATTR] = source_id new_nodes.append(current_p) def _flush_para_if_empty(): @@ -1825,10 +1833,10 @@ def group_cell_elements( group_element.children.append(prov) pr_item = prov.resolve(doc) item_parent = pr_item.parent.resolve(doc) - if pr_item.get_ref() in item_parent.children: - item_parent.children.remove(pr_item.get_ref()) - pr_item.parent = group_element.get_ref() - ref_for_rich_cell = group_element.get_ref() + if RefItem(cref=pr_item.self_ref) in item_parent.children: + item_parent.children.remove(RefItem(cref=pr_item.self_ref)) + pr_item.parent = RefItem(cref=group_element.self_ref) + ref_for_rich_cell = RefItem(cref=group_element.self_ref) return ref_for_rich_cell @staticmethod @@ -1874,12 +1882,10 @@ def _is_rich_table_cell(self, table_cell: Tag) -> bool: direct_block_text_children = [ child for child in table_cell.find_all(recursive=False) - if isinstance(child, Tag) and child.name in {"p", "div", "li"} + if child.name in {"p", "div", "li"} ] has_nested_form_semantic_id = any( - self._is_form_semantic_tag(child) - for child in children - if isinstance(child, Tag) + self._is_form_semantic_tag(child) for child in children ) if has_nested_form_semantic_id: return True @@ -1924,7 +1930,9 @@ def parse_table_data( t.unwrap() _log.debug(f"The table has {num_rows} rows and {num_cols} cols.") - grid: list = [[None for _ in range(num_cols)] for _ in range(num_rows)] + grid: list[list[str | None]] = [ + [None for _ in range(num_cols)] for _ in range(num_rows) + ] data = TableData(num_rows=num_rows, num_cols=num_cols, table_cells=[]) # Iterate over the rows in the table @@ -1933,8 +1941,6 @@ def parse_table_data( # We don't want this recursive to support nested tables for row in element("tr", recursive=False): - if not isinstance(row, Tag): - continue row_classes = { class_name.lower() for class_name in self._get_tag_classes(row) } @@ -1946,13 +1952,12 @@ def parse_table_data( col_header = True row_header = True for html_cell in cells: - if isinstance(html_cell, Tag): - _, row_span = HTMLDocumentBackend._get_cell_spans(html_cell) - if html_cell.name == "td": - col_header = False - row_header = False - elif row_span == 1: - row_header = False + _, row_span = HTMLDocumentBackend._get_cell_spans(html_cell) + if html_cell.name == "td": + col_header = False + row_header = False + elif row_span == 1: + row_header = False if not row_header: row_idx += 1 start_row_span = 0 @@ -1962,8 +1967,6 @@ def parse_table_data( # Extract the text content of each cell col_idx = 0 for html_cell in cells: - if not isinstance(html_cell, Tag): - continue cell_classes = { class_name.lower() for class_name in self._get_tag_classes(html_cell) @@ -1978,6 +1981,7 @@ def parse_table_data( formula.replace_with(NavigableString(math_formula)) provs_in_cell: list[RefItem] = [] + ref_for_rich_cell: RefItem | None = None rich_table_cell = self._is_rich_table_cell(html_cell) if rich_table_cell: # Parse table cell sub-tree for Rich Cells content: @@ -2018,6 +2022,7 @@ def parse_table_data( grid[row_idx + r][col_idx + c] = text if rich_table_cell: + assert ref_for_rich_cell is not None rich_cell = RichTableCell( text=text, bbox=cell_bbox, @@ -2106,7 +2111,9 @@ def _flush_buffer() -> None: prov=prov, ) if inline_ref is None: - added_refs.append(docling_code2.get_ref()) + added_refs.append( + RefItem(cref=docling_code2.self_ref) + ) else: prov = self._make_text_prov_for_source_tag_ids( text=seg_clean, @@ -2124,7 +2131,9 @@ def _flush_buffer() -> None: prov=prov, ) if inline_ref is None: - added_refs.append(docling_text2.get_ref()) + added_refs.append( + RefItem(cref=docling_text2.self_ref) + ) if inline_ref is not None: added_refs.append(inline_ref) @@ -2436,7 +2445,7 @@ def _get_tag_name_for_docling_id(self, source_tag_id: str) -> Optional[str]: return tag_name or None if self.soup is None: return None - tag = self.soup.find(attrs={_DATA_DOCLING_ID_ATTR: source_tag_id}) + tag = self.soup.find(True, attrs={_DATA_DOCLING_ID_ATTR: source_tag_id}) tag_name = tag.name if isinstance(tag, Tag) else "" self._tag_name_by_docling_id_cache[source_tag_id] = tag_name return tag_name or None @@ -2505,7 +2514,7 @@ def _use_inline_group( self.parents[self.level + 1] = inline_fmt self.level += 1 try: - yield inline_fmt.get_ref() + yield RefItem(cref=inline_fmt.self_ref) finally: self.parents[self.level] = None self.level -= 1 @@ -2588,7 +2597,7 @@ def _use_table_cell_context(self): self.parents = original_parents @contextmanager - def _use_list_item_context(self, parent_item: RefItem | None) -> Iterator[None]: + def _use_list_item_context(self, parent_item: NodeItem | None) -> Iterator[None]: """Set up context for processing nested content within a list item. Args: @@ -2610,7 +2619,7 @@ def _use_list_item_context(self, parent_item: RefItem | None) -> Iterator[None]: yield def _handle_heading(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: - added_ref = [] + added_ref: list[RefItem] = [] tag_name = tag.name.lower() # set default content layer to BODY as soon as we encounter a heading self.content_layer = ContentLayer.BODY @@ -2639,7 +2648,7 @@ def _handle_heading(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: ) p1 = self.parents[self.level + 1] if p1 is not None: - added_ref = [p1.get_ref()] + added_ref = [RefItem(cref=p1.self_ref)] # the other levels need to be lowered by 1 if a title was set else: level -= 1 @@ -2673,13 +2682,12 @@ def _handle_heading(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: ) p2 = self.parents[self.level + 1] if p2 is not None: - added_ref = [p2.get_ref()] + added_ref = [RefItem(cref=p2.self_ref)] self.level += 1 for img_tag in tag("img"): - if isinstance(img_tag, Tag): - im_ref = self._emit_image(img_tag, doc) - if im_ref: - added_ref.append(im_ref) + im_ref = self._emit_image(img_tag, doc) + if im_ref: + added_ref.append(im_ref) return added_ref def _has_list_ancestor(self, elem: Tag, boundary: Tag) -> bool: @@ -2694,7 +2702,7 @@ def _has_list_ancestor(self, elem: Tag, boundary: Tag) -> bool: """ parent = elem.parent while parent and parent != boundary: - if isinstance(parent, Tag) and parent.name in {"ul", "ol", "dl"}: + if parent.name in {"ul", "ol", "dl"}: return True parent = parent.parent return False @@ -2704,7 +2712,7 @@ def _process_nested_element( elem, li: Tag, doc: DoclingDocument, - processed_elements: set, + processed_elements: set[int], ) -> None: """Process a single nested element within a list item. @@ -2743,7 +2751,7 @@ def _process_list_item_nested_content( self, li: Tag, doc: DoclingDocument, - processed_elements: set, + processed_elements: set[int], ) -> None: """Process nested content (images, lists, etc.) within a list item in DOM order. @@ -2759,11 +2767,11 @@ def _add_list_item_with_content( self, tag: Tag, doc: DoclingDocument, - parent: RefItem, + parent: NodeItem, enumerated: bool = False, marker: str = "", extra_formatting: Optional[Formatting] = None, - ) -> Optional[RefItem]: + ) -> Optional[ListItem]: """Helper method to add a list item with its content. Handles both simple and complex content with inline groups. @@ -2839,6 +2847,7 @@ def _add_list_item_with_content( self.parents[self.level] = None self.level -= 1 + assert isinstance(list_item, ListItem) return list_item else: # Simple content - single text element @@ -2870,6 +2879,7 @@ def _add_list_item_with_content( hyperlink=annotated_text.hyperlink, prov=prov, ) + assert isinstance(list_item, ListItem) return list_item def _emit_task_list_inputs( @@ -2877,8 +2887,7 @@ def _emit_task_list_inputs( ) -> None: """Emit the checkbox items of a task-list
  • under the list group.""" for input_tag in inputs_in_li: - if isinstance(input_tag, Tag): - self._emit_input(input_tag, doc) + self._emit_input(input_tag, doc) def _is_task_list_item( self, li: Tag, inputs_in_li: list[Tag], custom_checkboxes_in_li: list[Tag] @@ -2897,7 +2906,7 @@ def _description_list_children(dl: Tag) -> list[PageElement]: (HTML allows wrapping each group of
    and
    elements in a
    ).""" children: list[PageElement] = [] for child in dl.find_all(["dt", "dd", "div"], recursive=False): - if isinstance(child, Tag) and child.name == "div": + if child.name == "div": children.extend(child.find_all(["dt", "dd"], recursive=False)) else: children.append(child) @@ -3004,13 +3013,10 @@ def _handle_list(self, tag: Tag, doc: DoclingDocument) -> RefItem: self.parents[self.level + 1] = None self.level -= 1 - return list_group.get_ref() + return RefItem(cref=list_group.self_ref) # For each top-level
  • in this list (ul/ol) for li in tag.find_all({"li", "ul", "ol"}, recursive=False): - if not isinstance(li, Tag): - continue - # sub-list items should be indented under main list items, but temporarily # addressing invalid HTML (docling-core/issues/357) if li.name in {"ul", "ol"}: @@ -3073,11 +3079,9 @@ def _handle_list(self, tag: Tag, doc: DoclingDocument) -> RefItem: # Handle inputs and checkboxes if inputs_in_li or custom_checkboxes_in_li: for input_tag in inputs_in_li: - if isinstance(input_tag, Tag): - self._emit_input(input_tag, doc) + self._emit_input(input_tag, doc) for checkbox_tag in custom_checkboxes_in_li: - if isinstance(checkbox_tag, Tag): - self._emit_custom_checkbox(checkbox_tag, doc) + self._emit_custom_checkbox(checkbox_tag, doc) # 4) Process nested content (images, lists, etc.) in DOM order processed_elements = set() @@ -3087,16 +3091,14 @@ def _handle_list(self, tag: Tag, doc: DoclingDocument) -> RefItem: else: # No content, but check for nested lists (including those wrapped in divs) for sublist in li({"ul", "ol", "dl"}): - if isinstance(sublist, Tag): - # Check if this list has a ul/ol/dl ancestor within the current li - has_list_ancestor = self._has_list_ancestor(sublist, li) + has_list_ancestor = self._has_list_ancestor(sublist, li) - if not has_list_ancestor: - self._handle_block(sublist, doc) + if not has_list_ancestor: + self._handle_block(sublist, doc) self.parents[self.level + 1] = None self.level -= 1 - return list_group.get_ref() + return RefItem(cref=list_group.self_ref) @staticmethod def get_html_table_row_col(tag: Tag) -> tuple[int, int]: @@ -3108,12 +3110,8 @@ def get_html_table_row_col(tag: Tag) -> tuple[int, int]: for row in tag("tr", recursive=False): cell_spans: list[tuple[int, int]] = [] is_row_header = True - if not isinstance(row, Tag): - continue for cell in row(["td", "th"], recursive=False): - if not isinstance(row, Tag): - continue - cell_tag = cast(Tag, cell) + cell_tag = cell col_span, row_span = HTMLDocumentBackend._get_cell_spans(cell_tag) cell_spans.append((col_span, row_span)) if cell_tag.name == "td" or row_span == 1: @@ -3164,7 +3162,7 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq if cap_item is not None and added_refs: first_item = added_refs[0].resolve(doc) if isinstance(first_item, TableItem): - first_item.captions.append(cap_item.get_ref()) + first_item.captions.append(RefItem(cref=cap_item.self_ref)) elif tag_name in {"h1", "h2", "h3", "h4", "h5", "h6"}: heading_refs = self._handle_heading(tag, doc) @@ -3201,7 +3199,9 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq prov=prov, ) if inline_ref is None: - added_refs.append(docling_code.get_ref()) + added_refs.append( + RefItem(cref=docling_code.self_ref) + ) else: prov = self._make_text_prov_for_source_tag_ids( text=seg_clean, @@ -3219,27 +3219,26 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq prov=prov, ) if inline_ref is None: - added_refs.append(docling_text.get_ref()) + added_refs.append( + RefItem(cref=docling_text.self_ref) + ) if inline_ref is not None: added_refs.append(inline_ref) for img_tag in tag("img"): - if isinstance(img_tag, Tag): - self._emit_image(img_tag, doc) + self._emit_image(img_tag, doc) for input_tag in tag("input"): - if isinstance(input_tag, Tag): - input_ref = self._emit_input(input_tag, doc) - if input_ref is not None: - added_refs.append(input_ref) + input_ref = self._emit_input(input_tag, doc) + if input_ref is not None: + added_refs.append(input_ref) for checkbox_tag in tag.find_all( lambda item: ( isinstance(item, Tag) and self._is_custom_checkbox_tag(item) ) ): - if isinstance(checkbox_tag, Tag): - checkbox_ref = self._emit_custom_checkbox(checkbox_tag, doc) - if checkbox_ref is not None: - added_refs.append(checkbox_ref) + checkbox_ref = self._emit_custom_checkbox(checkbox_tag, doc) + if checkbox_ref is not None: + added_refs.append(checkbox_ref) elif tag_name == "table": num_rows, num_cols = self.get_html_table_row_col(tag) @@ -3258,17 +3257,18 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq content_layer=self.content_layer, caption=cap_item, ) - added_refs.append(docling_table.get_ref()) + added_refs.append(RefItem(cref=docling_table.self_ref)) self.parse_table_data(tag, doc, docling_table, num_rows, num_cols) elif tag_name in {"stamp", "signature"}: _class_name = PictureClassificationLabel.STAMP.value if tag_name == "signature": _class_name = PictureClassificationLabel.SIGNATURE.value - placeholder: PictureItem = doc.add_picture( + placeholder = doc.add_picture( parent=self.parents[self.level], content_layer=self.content_layer, ) + assert isinstance(placeholder, PictureItem) placeholder.meta = PictureMeta( classification=PictureClassificationMetaField( predictions=[ @@ -3315,7 +3315,7 @@ def _handle_block(self, tag: Tag, doc: DoclingDocument) -> list[RefItem]: # noq prov=prov, ) if inline_ref is None: - added_refs.append(docling_code2.get_ref()) + added_refs.append(RefItem(cref=docling_code2.self_ref)) if inline_ref is not None: added_refs.append(inline_ref) @@ -3338,7 +3338,7 @@ def _is_form_container(tag: Tag) -> bool: def _nearest_form_container_ancestor(self, tag: Tag) -> Optional[Tag]: for parent in tag.parents: - if isinstance(parent, Tag) and self._is_form_container(parent): + if self._is_form_container(parent): return parent return None @@ -3381,7 +3381,7 @@ def _emit_flattened_text_tag( content_layer=self.content_layer, prov=prov, ) - return text_item.get_ref() + return RefItem(cref=text_item.self_ref) def _ensure_tag_html_id(self, tag: Tag) -> str: existing = self._get_html_id(tag) @@ -3410,13 +3410,9 @@ def _dom_distance_between_tags(left_tag: Tag, right_tag: Tag) -> int: return 0 left_chain: list[Tag] = [left_tag] - left_chain.extend( - parent for parent in left_tag.parents if isinstance(parent, Tag) - ) + left_chain.extend(parent for parent in left_tag.parents) right_chain: list[Tag] = [right_tag] - right_chain.extend( - parent for parent in right_tag.parents if isinstance(parent, Tag) - ) + right_chain.extend(parent for parent in right_tag.parents) left_positions = {id(tag): idx for idx, tag in enumerate(left_chain)} best_distance: Optional[int] = None @@ -3461,14 +3457,7 @@ def _select_form_value_entries( ranked_entries = sorted( entries, key=lambda entry: ( - ( - 0 - if ( - key_tag is not None - and self._is_value_in_key_scope(key_tag, entry[2]) - ) - else 1 - ) + (0 if (self._is_value_in_key_scope(key_tag, entry[2])) else 1) if key_tag is not None else 0, ( @@ -3628,7 +3617,7 @@ def _get_tag_classes(tag: Tag) -> set[str]: return set() if isinstance(classes, str): return {classes} - return {str(value) for value in classes if isinstance(value, str)} + return set(classes) @staticmethod def _code_language_hint(tag: Tag) -> str | None: @@ -3656,7 +3645,7 @@ def _has_inline_display_style(tag: Tag) -> bool: display_match = re.search(r"display\s*:\s*([^;]+)", style_attr, flags=re.I) if display_match is None: return False - display_value = display_match.group(1).strip().lower() + display_value = str(display_match.group(1)).strip().lower() return display_value.startswith("inline") or display_value == "contents" def _should_buffer_tag_text_inline(self, tag: Tag) -> bool: @@ -3735,9 +3724,7 @@ def _extract(node: PageElement) -> list[str]: @staticmethod def _has_direct_checkbox_like_child(tag: Tag) -> bool: for child in tag.find_all(recursive=False): - if isinstance(child, Tag) and HTMLDocumentBackend._is_checkbox_like_tag( - child - ): + if HTMLDocumentBackend._is_checkbox_like_tag(child): return True return False @@ -3833,8 +3820,7 @@ def _label_texts(tags: list[Tag]) -> list[str]: for label_tag in self.soup.find_all( "label", attrs={"for": input_id} ): - if isinstance(label_tag, Tag): - _add_label_tag(label_tag) + _add_label_tag(label_tag) # Backward-compatible local search (same parent container). if parent is not None: for sibling in parent.find_all("label", recursive=False): @@ -3869,7 +3855,7 @@ def _label_texts(tags: list[Tag]) -> list[str]: if "checkbox-container" in parent_classes: label_texts: list[str] = [] for sibling in parent.find_all(recursive=False): - if not isinstance(sibling, Tag) or sibling is checkbox_tag: + if sibling is checkbox_tag: continue if "checkbox-label" not in self._get_tag_classes(sibling): continue @@ -3892,7 +3878,7 @@ def _label_texts(tags: list[Tag]) -> list[str]: for child in parent.contents ) for sibling in parent.find_all(recursive=False): - if not isinstance(sibling, Tag) or sibling is checkbox_tag: + if sibling is checkbox_tag: continue if self._is_checkbox_like_tag(sibling): continue @@ -4088,7 +4074,7 @@ def _emit_custom_checkbox( hyperlink=self.hyperlink, prov=prov, ) - return checkbox_item.get_ref() + return RefItem(cref=checkbox_item.self_ref) @contextmanager def _suppress_tag_ids(self, tag_ids: set[str]): @@ -4274,9 +4260,7 @@ def _is_simple_extra_text_candidate(self, tag: Tag) -> bool: ): return False - direct_tag_children = [ - child for child in tag.find_all(recursive=False) if isinstance(child, Tag) - ] + direct_tag_children = list(tag.find_all(recursive=False)) has_direct_text = any( isinstance(child, NavigableString) and bool(str(child).strip()) for child in tag.contents @@ -4296,7 +4280,7 @@ def _add_field_item_from_extracted( self, field: _ExtractedFormField, doc: DoclingDocument, - parent: Optional[Union[DocItem, GroupItem]], + parent: NodeItem | None, ) -> list[RefItem]: refs: list[RefItem] = [] doc_with_fields = cast(Any, doc) @@ -4304,7 +4288,7 @@ def _add_field_item_from_extracted( parent=parent, content_layer=self.content_layer, ) - refs.append(field_item.get_ref()) + refs.append(RefItem(cref=field_item.self_ref)) parts: list[tuple[int, Literal["key", "marker", "value", "text"], Any]] = [] if field.key_tag is not None and field.key_text: @@ -4325,7 +4309,7 @@ def _add_field_item_from_extracted( parent=field_item, content_layer=self.content_layer, ) - refs.append(field_key.get_ref()) + refs.append(RefItem(cref=field_key.self_ref)) elif part_type == "marker": marker = cast(_ExtractedFormMarker, payload) marker_item = doc.add_text( @@ -4336,7 +4320,7 @@ def _add_field_item_from_extracted( parent=field_item, content_layer=self.content_layer, ) - refs.append(marker_item.get_ref()) + refs.append(RefItem(cref=marker_item.self_ref)) elif part_type == "text": extra_text = cast(_ExtractedFormText, payload) text_item = doc.add_text( @@ -4347,7 +4331,7 @@ def _add_field_item_from_extracted( parent=field_item, content_layer=self.content_layer, ) - refs.append(text_item.get_ref()) + refs.append(RefItem(cref=text_item.self_ref)) else: value = cast(_ExtractedFormValue, payload) if value.checkbox_label is not None: @@ -4359,7 +4343,7 @@ def _add_field_item_from_extracted( parent=field_item, content_layer=self.content_layer, ) - refs.append(checkbox_item.get_ref()) + refs.append(RefItem(cref=checkbox_item.self_ref)) else: field_value = doc_with_fields.add_field_value( text=value.text, @@ -4369,7 +4353,7 @@ def _add_field_item_from_extracted( content_layer=self.content_layer, kind=value.kind, ) - refs.append(field_value.get_ref()) + refs.append(RefItem(cref=field_value.self_ref)) return refs @@ -4427,10 +4411,9 @@ def _extract_form_region(self, form_tag: Tag) -> Optional[_ExtractedFormRegion]: table_bboxes: list[BoundingBox] = [] if self._rendered_bbox_by_id: for table_tag in form_tag.find_all("table"): - if isinstance(table_tag, Tag): - rendered = self._get_rendered_bbox_for_tag(table_tag) - if rendered is not None: - table_bboxes.append(rendered.bbox) + rendered = self._get_rendered_bbox_for_tag(table_tag) + if rendered is not None: + table_bboxes.append(rendered.bbox) key_ids_in_order = sorted( first_order_by_key.keys(), key=lambda key_id: first_order_by_key[key_id] @@ -4634,11 +4617,7 @@ def _extract_form_region(self, form_tag: Tag) -> Optional[_ExtractedFormRegion]: for parent_tag in unique_parent_tags_to_scan: component_direct_child_indices: list[int] = [] - direct_children = [ - child - for child in parent_tag.find_all(recursive=False) - if isinstance(child, Tag) - ] + direct_children = list(parent_tag.find_all(recursive=False)) for idx, child in enumerate(direct_children): child_obj_id = id(child) if child_obj_id in component_tag_obj_ids: @@ -4914,7 +4893,7 @@ def _handle_form_container(self, tag: Tag, doc: DoclingDocument) -> list[RefItem parent=self.parents[self.level], ) field_region.content_layer = self.content_layer - added_refs.append(field_region.get_ref()) + added_refs.append(RefItem(cref=field_region.self_ref)) with self._use_form_container(field_region): with self._use_form_fields_by_key_id(fields_by_key_id): @@ -4936,7 +4915,7 @@ def _handle_form_container(self, tag: Tag, doc: DoclingDocument) -> list[RefItem parent=self.parents[self.level], ) form_item.content_layer = self.content_layer - added_refs.append(form_item.get_ref()) + added_refs.append(RefItem(cref=form_item.self_ref)) if form_graph is not None: kv_item = doc.add_key_values( @@ -4945,7 +4924,7 @@ def _handle_form_container(self, tag: Tag, doc: DoclingDocument) -> list[RefItem parent=form_item, ) kv_item.content_layer = self.content_layer - added_refs.append(kv_item.get_ref()) + added_refs.append(RefItem(cref=kv_item.self_ref)) with self._use_form_container(form_item): if tag.name.lower() == "table": @@ -4971,7 +4950,7 @@ def _emit_caption( tag=caption_tag, source_tag_id=cap_anno.source_tag_id, ) - return doc.add_text( + caption = doc.add_text( label=DocItemLabel.CAPTION, text=cap_text, orig=cap_anno.text, @@ -4982,6 +4961,9 @@ def _emit_caption( parent=self.parents[self.level], ) + assert isinstance(caption, TextItem) + return caption + def _emit_image(self, img_tag: Tag, doc: DoclingDocument) -> Optional[RefItem]: figure = img_tag.find_parent("figure") caption: AnnotatedTextList = AnnotatedTextList() @@ -4999,7 +4981,7 @@ def get_img_hyperlink(img_tag): return None if img_hyperlink := get_img_hyperlink(img_tag): - img_text = img_tag.get("alt") or "" + img_text = self._get_attr_as_string(img_tag, "alt") caption.append(AnnotatedText(text=img_text, hyperlink=img_hyperlink)) caption_prov_tag = img_tag @@ -5011,7 +4993,9 @@ def get_img_hyperlink(img_tag): ) caption_prov_tag = caption_tag if not caption and img_tag.get("alt"): - caption = AnnotatedTextList([AnnotatedText(text=img_tag.get("alt"))]) + caption = AnnotatedTextList( + [AnnotatedText(text=self._get_attr_as_string(img_tag, "alt"))] + ) caption_prov_tag = img_tag caption_anno_text = caption.to_single_text_element() @@ -5026,7 +5010,7 @@ def get_img_hyperlink(img_tag): tag=caption_prov_tag or img_tag, source_tag_id=caption_anno_text.source_tag_id, ) - caption_item = doc.add_text( + emitted_caption = doc.add_text( label=DocItemLabel.CAPTION, text=text_clean, orig=caption_anno_text.text, @@ -5036,17 +5020,21 @@ def get_img_hyperlink(img_tag): prov=prov, ) + assert isinstance(emitted_caption, TextItem) + caption_item = emitted_caption + src_loc: str = self._get_attr_as_string(img_tag, "src") pic_prov = self._make_prov(text="", tag=img_tag) - if not cast(HTMLBackendOptions, self.options).fetch_images or not src_loc: + if not self.options.fetch_images or not src_loc: # Do not fetch the image, just add a placeholder - placeholder: PictureItem = doc.add_picture( + placeholder = doc.add_picture( caption=caption_item, parent=parent, content_layer=self.content_layer, prov=pic_prov, ) - return placeholder.get_ref() + assert isinstance(placeholder, PictureItem) + return RefItem(cref=placeholder.self_ref) src_loc = self._resolve_relative_path(src_loc) img_ref = self._create_image_ref(src_loc) @@ -5058,7 +5046,7 @@ def get_img_hyperlink(img_tag): content_layer=self.content_layer, prov=pic_prov, ) - return docling_pic.get_ref() + return RefItem(cref=docling_pic.self_ref) def _emit_input(self, input_tag: Tag, doc: DoclingDocument) -> Optional[RefItem]: if self._is_suppressed_tag(input_tag): @@ -5068,6 +5056,7 @@ def _emit_input(self, input_tag: Tag, doc: DoclingDocument) -> Optional[RefItem] return None label = DocItemLabel.TEXT + checkbox_label_tags: list[Tag] = [] checkbox_label = self._get_checkbox_label_for_tag(input_tag) if checkbox_label is not None: label = checkbox_label @@ -5117,7 +5106,7 @@ def _emit_input(self, input_tag: Tag, doc: DoclingDocument) -> Optional[RefItem] hyperlink=self.hyperlink, prov=prov, ) - return input_item.get_ref() + return RefItem(cref=input_item.self_ref) def _create_image_ref(self, src_url: str) -> Optional[ImageRef]: if self._mhtml_resources is not None: diff --git a/docling/backend/image_backend.py b/docling/backend/image_backend.py index 7308939738..4adb7b4a1f 100644 --- a/docling/backend/image_backend.py +++ b/docling/backend/image_backend.py @@ -142,7 +142,7 @@ def get_segmented_page(self) -> SegmentedPdfPage: @override def get_text_cells(self) -> Iterable[TextCell]: # No text cells on raw images - return [] + return list[TextCell]() @override def get_bitmap_rects(self, scale: float = 1) -> Iterable[BoundingBox]: @@ -248,7 +248,7 @@ def __init__( self._frames: list[Image.Image] = [] self._frame_dpi: list[tuple[float, float]] = [] try: - with Image.open(self.path_or_stream) as img: # type: ignore[arg-type] + with Image.open(path_or_stream) as img: # Handle multi-frame and single-frame images # - multiframe formats: TIFF, GIF, ICO # - singleframe formats: JPEG (.jpg, .jpeg), PNG (.png), BMP, WEBP (unless animated), HEIC diff --git a/docling/backend/iwork/cells.py b/docling/backend/iwork/cells.py index 1915c97695..85dd77e6c4 100644 --- a/docling/backend/iwork/cells.py +++ b/docling/backend/iwork/cells.py @@ -263,7 +263,7 @@ def iwa_current_cell(storage: bytes, start: int, values: CellValues) -> Cell | N decoded = decoded._replace( text=values.strings.get(read_uint32(storage, offset)) ) - elif flag == CELL_FLAG_RICH_TEXT: + else: decoded = decoded._replace( text=values.rich_text.get(read_uint32(storage, offset)) ) diff --git a/docling/backend/iwork/legacy.py b/docling/backend/iwork/legacy.py index 117bab6434..f8c6fe44d6 100644 --- a/docling/backend/iwork/legacy.py +++ b/docling/backend/iwork/legacy.py @@ -653,7 +653,7 @@ def legacy_formatting(style: Element) -> Formatting | None: label = SF_PROPERTY_LABELS.get(element.tag) if label is not None: active.add(label) - elif element.tag == SF_SUPERSCRIPT and number is not None: + elif element.tag == SF_SUPERSCRIPT: script = SCRIPTS.get(as_int(number)) return build_formatting(active, script) diff --git a/docling/backend/iwork/numbers_xml.py b/docling/backend/iwork/numbers_xml.py index 95b4a50ae5..1f43363d58 100644 --- a/docling/backend/iwork/numbers_xml.py +++ b/docling/backend/iwork/numbers_xml.py @@ -17,6 +17,7 @@ import logging import plistlib import zipfile +from typing import cast from xml.etree.ElementTree import Element from docling.backend.iwork.content import Chart, ChartKind, ChartSeries @@ -177,7 +178,9 @@ def read_content( return sheets -def read_chart_shares(archive: zipfile.ZipFile, max_file_bytes: int) -> dict[str, dict]: +def read_chart_shares( + archive: zipfile.ZipFile, max_file_bytes: int +) -> dict[str, dict[str, object]]: """Read the cached data of every chart in the container. Args: @@ -189,7 +192,7 @@ def read_chart_shares(archive: zipfile.ZipFile, max_file_bytes: int) -> dict[str is missing or unreadable is left out rather than failing the document, since the chart still has its names without it. """ - shares: dict[str, dict] = {} + shares: dict[str, dict[str, object]] = {} for info in archive.infolist(): if not info.filename.endswith(CHART_SHARE_SUFFIX): continue @@ -206,7 +209,9 @@ def read_chart_shares(archive: zipfile.ZipFile, max_file_bytes: int) -> dict[str _log.debug("Could not read chart data '%s'.", info.filename) continue if isinstance(share, dict): - shares[info.filename[: -len(CHART_SHARE_SUFFIX)]] = share + shares[info.filename[: -len(CHART_SHARE_SUFFIX)]] = cast( + dict[str, object], share + ) return shares @@ -426,7 +431,9 @@ def formula_text(cell: Element) -> str | None: return None -def read_chart(info: Element, shares: dict[str, dict]) -> PlacedChart | None: +def read_chart( + info: Element, shares: dict[str, dict[str, object]] +) -> PlacedChart | None: """Build one chart from an ``sf:chart-info``, and say where it sits. A chart bound to a table leaves ``sf:chart-data`` empty and keeps the values @@ -456,10 +463,18 @@ def read_chart(info: Element, shares: dict[str, dict]) -> PlacedChart | None: if share is not None: # The share is what the chart last drew, so where the two disagree — a # stale name left behind in the element, say — the share wins. - shared_names = [str(name) for name in share.get(SHARE_COLUMNS_KEY, [])] + columns = share.get(SHARE_COLUMNS_KEY, []) + shared_names = ( + [str(name) for name in columns] if isinstance(columns, list) else [] + ) if shared_names: names = shared_names - rows = share.get(SHARE_ROWS_KEY) or [] + raw_rows = share.get(SHARE_ROWS_KEY) or [] + rows = ( + [cast(dict[str, object], row) for row in raw_rows if isinstance(row, dict)] + if isinstance(raw_rows, list) + else [] + ) if rows: categories = [str(row.get(SHARE_ROW_NAME_KEY, "")) for row in rows] by_category = [points(row) for row in rows] @@ -508,13 +523,16 @@ def _point_of( return row[series] if series < len(row) else None -def points(row: dict) -> list[float | None]: +def points(row: dict[str, object]) -> list[float | None]: """Read one category's plotted values, leaving the gaps empty.""" + values = row.get(SHARE_ROW_VALUES_KEY, []) + if not isinstance(values, list): + return [] return [ float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None - for value in row.get(SHARE_ROW_VALUES_KEY, []) + for value in values ] diff --git a/docling/backend/iwork_backend.py b/docling/backend/iwork_backend.py index dde1023bd5..1e47e8eba9 100644 --- a/docling/backend/iwork_backend.py +++ b/docling/backend/iwork_backend.py @@ -86,7 +86,7 @@ ) from docling.backend.iwork.iwa import is_encrypted from docling.backend.iwork.keynote_content import Presentation, Slide -from docling.datamodel.backend_options import IWorkBackendOptions +from docling.datamodel.backend_options import BaseBackendOptions, IWorkBackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -292,10 +292,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: IWorkBackendOptions | None = None, + options: BaseBackendOptions | None = None, ): if options is None: options = IWorkBackendOptions() + if not isinstance(options, IWorkBackendOptions): + raise TypeError("Expected IWorkBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.options: IWorkBackendOptions = options @@ -422,10 +424,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: IWorkBackendOptions | None = None, + options: BaseBackendOptions | None = None, ): if options is None: options = IWorkBackendOptions() + if not isinstance(options, IWorkBackendOptions): + raise TypeError("Expected IWorkBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.options: IWorkBackendOptions = options @@ -924,14 +928,18 @@ def _add_paragraph( return _add_list_item(doc, paragraph, paragraph.list_label, lists, prov) if paragraph.label == DocItemLabel.TITLE: - return doc.add_title(text=paragraph.text, parent=parent, prov=prov) + item = doc.add_title(text=paragraph.text, parent=parent, prov=prov) + assert isinstance(item, TextItem) + return item if paragraph.label == DocItemLabel.SECTION_HEADER: - return doc.add_heading( + item = doc.add_heading( text=paragraph.text, level=paragraph.level or 1, parent=parent, prov=prov, ) + assert isinstance(item, TextItem) + return item return _add_runs(doc, paragraph, paragraph.label, parent=parent, prov=prov) @@ -966,7 +974,7 @@ def _add_runs( runs = [run for run in paragraph.runs if run.text] if _uniform(runs): first = runs[0] if runs else Run("", None) - return doc.add_text( + item = doc.add_text( label=label, text=paragraph.text, formatting=first.formatting, @@ -975,6 +983,8 @@ def _add_runs( parent=parent, prov=prov, ) + assert isinstance(item, TextItem) + return item group = doc.add_inline_group(content_layer=content_layer, parent=parent) items = [ @@ -989,7 +999,9 @@ def _add_runs( ) for run in runs ] - return items[0] + first_item = items[0] + assert isinstance(first_item, TextItem) + return first_item def _add_list_item( @@ -1003,7 +1015,7 @@ def _add_list_item( group = lists.group_for(label.depth) runs = [run for run in paragraph.runs if run.text] uniform = runs[0] if runs and _uniform(runs) else Run("", None) - return doc.add_list_item( + item = doc.add_list_item( text=paragraph.text, enumerated=label.enumerated, marker=label.marker, @@ -1012,6 +1024,8 @@ def _add_list_item( hyperlink=_hyperlink(uniform.hyperlink), prov=prov, ) + assert isinstance(item, TextItem) + return item def _uniform(runs: list[Run]) -> bool: @@ -1087,10 +1101,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: IWorkBackendOptions | None = None, + options: BaseBackendOptions | None = None, ): if options is None: options = IWorkBackendOptions() + if not isinstance(options, IWorkBackendOptions): + raise TypeError("Expected IWorkBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.options: IWorkBackendOptions = options self.page_range = in_doc.limits.page_range @@ -1203,7 +1219,7 @@ def convert(self) -> DoclingDocument: for drawable in sorted(drawn, key=numbers_content.sheet_order): if isinstance(drawable, numbers_content.Table): _add_sheet_table(doc, drawable, parent=group, page_no=index) - elif isinstance(drawable, numbers_content.PlacedChart): + else: _add_chart( doc, drawable.chart, diff --git a/docling/backend/latex/backend.py b/docling/backend/latex/backend.py index 830a9a1aab..69875728cc 100644 --- a/docling/backend/latex/backend.py +++ b/docling/backend/latex/backend.py @@ -79,7 +79,7 @@ def __init__( self._input_stack: set[str] = set() self._tectonic_engine: TectonicEngine | None = None self._tikz_executor: concurrent.futures.ThreadPoolExecutor | None = None - self._tikz_futures: list[concurrent.futures.Future] = [] + self._tikz_futures: list[concurrent.futures.Future[None]] = [] self.latex_text = decode_latex_content(self.path_or_stream) @override @@ -131,7 +131,7 @@ def _do_parse_and_process(self, doc: DoclingDocument) -> DoclingDocument: workers = max(1, (os.cpu_count() or 2) - 1) self._tikz_executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) - self._tikz_futures = [] # type: ignore + self._tikz_futures = [] try: self._extract_custom_macros(nodes) diff --git a/docling/backend/latex/handlers/environments.py b/docling/backend/latex/handlers/environments.py index b9d72f0d01..2cf808b05d 100644 --- a/docling/backend/latex/handlers/environments.py +++ b/docling/backend/latex/handlers/environments.py @@ -32,7 +32,7 @@ ) try: # pragma: no cover - import-time guard - from pylatexenc.latexwalker import LatexEnvironmentNode, LatexMacroNode + from pylatexenc.latexwalker import LatexEnvironmentNode, LatexMacroNode, LatexNode except ImportError: pass # guarded by LatexDocumentBackend.__init__ @@ -250,7 +250,7 @@ def set_code_fallback(): ) def _extract_tikzpicture_atomic(self, node: LatexEnvironmentNode) -> str | None: - raw = node.latex_verbatim() + raw = str(node.latex_verbatim()) if _TIKZ_END_PATTERN.search(raw) is None: return None if not self._validate_tikz_nodelist(node.nodelist, 0): @@ -265,7 +265,7 @@ def _validate_tikz_nodelist(self, nodes, depth: int = 0) -> bool: for node in nodes: if isinstance(node, LatexEnvironmentNode) and node.envname == "tikzpicture": - nested_raw = node.latex_verbatim() + nested_raw = str(node.latex_verbatim()) if _TIKZ_END_PATTERN.search(nested_raw) is None: return False @@ -309,7 +309,7 @@ def _process_list( list_group = doc.add_group(parent=parent, name="list", label=GroupLabel.LIST) items = [] - current_item: list = [] + current_item: list[LatexNode] = [] if node.nodelist is not None: for n in node.nodelist: @@ -347,7 +347,7 @@ def _process_bibliography( ) items = [] - current_item: list = [] + current_item: list[LatexNode] = [] current_key = "" if node.nodelist is not None: diff --git a/docling/backend/latex/handlers/macros.py b/docling/backend/latex/handlers/macros.py index cc6de91b96..148821b030 100644 --- a/docling/backend/latex/handlers/macros.py +++ b/docling/backend/latex/handlers/macros.py @@ -47,6 +47,8 @@ LatexEnvironmentNode, LatexGroupNode, LatexMacroNode, + LatexMathNode, + LatexNode, LatexWalker, LatexWalkerParseError, ) @@ -544,7 +546,7 @@ def _extract_macro_arg(self, node: LatexMacroNode) -> str: if arg: if hasattr(arg, "nodelist"): return self._nodes_to_text(arg.nodelist) - return arg.latex_verbatim().strip("{} ") + return str(arg.latex_verbatim()).strip("{} ") return "" def _extract_macro_arg_by_index(self, node: LatexMacroNode, index: int) -> str: @@ -554,15 +556,27 @@ def _extract_macro_arg_by_index(self, node: LatexMacroNode, index: int) -> str: if arg: if hasattr(arg, "nodelist"): return self._nodes_to_text(arg.nodelist) - return arg.latex_verbatim().strip("{} ") + return str(arg.latex_verbatim()).strip("{} ") return "" - def _extract_macro_arg_nodes(self, node: LatexMacroNode, index: int) -> list: + def _extract_macro_arg_nodes( + self, node: LatexMacroNode, index: int + ) -> list[LatexNode]: if node.nodeargd and node.nodeargd.argnlist: if 0 <= index < len(node.nodeargd.argnlist): arg = node.nodeargd.argnlist[index] - if arg and hasattr(arg, "nodelist"): - return arg.nodelist + if isinstance( + arg, (LatexGroupNode, LatexEnvironmentNode, LatexMathNode) + ): + nodes = arg.nodelist + if nodes is None: + return [] + assert isinstance(nodes, list) + typed_nodes: list[LatexNode] = [] + for child in nodes: + assert isinstance(child, LatexNode) + typed_nodes.append(child) + return typed_nodes return [] def _extract_all_macro_args_inline(self, node: LatexMacroNode) -> str: diff --git a/docling/backend/latex/handlers/math.py b/docling/backend/latex/handlers/math.py index 5e0c76e741..97ed923aea 100644 --- a/docling/backend/latex/handlers/math.py +++ b/docling/backend/latex/handlers/math.py @@ -50,7 +50,7 @@ def _clean_math(self, latex_str: str, env_name: str) -> str: pattern = rf"\\begin\{{{re.escape(env_name)}\}}(.*?)\\end\{{{re.escape(env_name)}\}}" match = re.search(pattern, latex_str, re.DOTALL) if match: - latex_str = match.group(1) + latex_str = str(match.group(1)) latex_str = latex_str.strip() diff --git a/docling/backend/latex/utils/encoding.py b/docling/backend/latex/utils/encoding.py index 1d7bd39214..6646269fb6 100644 --- a/docling/backend/latex/utils/encoding.py +++ b/docling/backend/latex/utils/encoding.py @@ -26,7 +26,7 @@ def decode_latex_content(path_or_stream: Union[BytesIO, Path]) -> str: if not latex_text: _log.warning("Failed to decode LaTeX content, using replacement mode") latex_text = raw_bytes.decode("utf-8", errors="replace") - elif isinstance(path_or_stream, Path): + else: for encoding in _ENCODINGS: try: with open(path_or_stream, encoding=encoding) as f: diff --git a/docling/backend/latex/utils/table.py b/docling/backend/latex/utils/table.py index ed23f64b87..fcc230a70a 100644 --- a/docling/backend/latex/utils/table.py +++ b/docling/backend/latex/utils/table.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Callable, List, Optional from docling_core.types.doc.document import TableCell, TableData +from pydantic import PrivateAttr from docling.backend.latex.constants import ( MACROS_ESCAPED, @@ -22,6 +23,7 @@ LatexCharsNode, LatexEnvironmentNode, LatexMacroNode, + LatexNode, LatexWalker, LatexWalkerParseError, ) @@ -29,6 +31,12 @@ pass # guarded by LatexDocumentBackend.__init__ +class _ParsedTableCell(TableCell): + _col_span: int = PrivateAttr(1) + _row_span: int = PrivateAttr(1) + _is_placeholder: bool = PrivateAttr(False) + + class TableHelperMixin: if TYPE_CHECKING: @@ -118,11 +126,11 @@ def _process_table_macro_node( def _parse_table(self, node: LatexEnvironmentNode) -> TableData | None: rows = [] current_row = [] - current_cell_nodes: list = [] + current_cell_nodes: list[LatexNode] = [] source_latex = node.latex_verbatim() - def parse_brace_args(text: str) -> list: + def parse_brace_args(text: str) -> list[str]: args = [] i = 0 while i < len(text): @@ -143,27 +151,27 @@ def parse_brace_args(text: str) -> list: def finish_cell(col_span: int = 1, row_span: int = 1): text = self._nodes_to_text(current_cell_nodes).strip() - cell = TableCell( + cell = _ParsedTableCell( text=text, start_row_offset_idx=0, end_row_offset_idx=0, start_col_offset_idx=0, end_col_offset_idx=0, ) - cell._col_span = col_span # type: ignore[attr-defined] - cell._row_span = row_span # type: ignore[attr-defined] + cell._col_span = col_span + cell._row_span = row_span current_row.append(cell) current_cell_nodes.clear() for _ in range(col_span - 1): - placeholder = TableCell( + placeholder = _ParsedTableCell( text="", start_row_offset_idx=0, end_row_offset_idx=0, start_col_offset_idx=0, end_col_offset_idx=0, ) - placeholder._is_placeholder = True # type: ignore[attr-defined] + placeholder._is_placeholder = True current_row.append(placeholder) def finish_row(): @@ -216,10 +224,10 @@ def finish_row(): for j in range(num_cols): if j < len(row): cell = row[j] - if getattr(cell, "_is_placeholder", False): + if cell._is_placeholder: continue else: - cell = TableCell( + cell = _ParsedTableCell( text="", start_row_offset_idx=0, end_row_offset_idx=0, @@ -230,8 +238,8 @@ def finish_row(): cell.start_row_offset_idx = i cell.start_col_offset_idx = j - col_span = getattr(cell, "_col_span", 1) - row_span = getattr(cell, "_row_span", 1) + col_span = cell._col_span + row_span = cell._row_span cell.end_row_offset_idx = i + row_span cell.end_col_offset_idx = j + col_span diff --git a/docling/backend/latex/utils/text.py b/docling/backend/latex/utils/text.py index 113ad75671..3cb390d110 100644 --- a/docling/backend/latex/utils/text.py +++ b/docling/backend/latex/utils/text.py @@ -117,7 +117,9 @@ def _extract_verbatim_content(self, latex_str: str, env_name: str) -> str: return re.sub(r"\A(?:[ \t]*\r?\n)+", "", match.group(1)).rstrip() return latex_str - def _macro_node_to_text(self, node: LatexMacroNode, following_nodes) -> tuple: + def _macro_node_to_text( + self, node: LatexMacroNode, following_nodes + ) -> tuple[str, int]: """Return ``(text, consumed_following)`` for a single macro node.""" consumed = 0 if node.macroname in (MACROS_TEXT_FORMATTING | MACROS_TEXT_STYLE): @@ -130,7 +132,7 @@ def _macro_node_to_text(self, node: LatexMacroNode, following_nodes) -> tuple: return (self._nodes_to_text(text_arg.nodelist), consumed) return ("", consumed) if node.macroname in MACROS_CITATION: - return (node.latex_verbatim(), consumed) + return (str(node.latex_verbatim()), consumed) if node.macroname == "\\": return ("\n", consumed) if node.macroname in ["~"]: @@ -143,7 +145,7 @@ def _macro_node_to_text(self, node: LatexMacroNode, following_nodes) -> tuple: return (f"{opt_text}: ", consumed) return ("", consumed) if node.macroname in MACROS_ESCAPED: - return (node.macroname, consumed) + return (str(node.macroname), consumed) if node.macroname in self._custom_macros: expansion, consumed = self._expand_custom_macro_invocation( node, following_nodes diff --git a/docling/backend/md_backend.py b/docling/backend/md_backend.py index c148d62c76..97c2e20ac0 100644 --- a/docling/backend/md_backend.py +++ b/docling/backend/md_backend.py @@ -36,6 +36,7 @@ from docling.backend.html_backend import HTMLDocumentBackend from docling.backend.utils.image_resource_loader import ImageResourceLoader from docling.datamodel.backend_options import ( + BaseBackendOptions, HTMLBackendOptions, MarkdownBackendOptions, ) @@ -102,7 +103,7 @@ class _ListItemCreationPayload(BaseModel): ] -def _only_plain_line_breaks(children: list) -> bool: +def _only_plain_line_breaks(children: list[marko.element.Element]) -> bool: """Return True when children consist solely of RawText/Literal runs separated by LineBreak nodes (soft or hard), with at least one break present. @@ -280,7 +281,7 @@ def __init__( self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path], - options: Optional[MarkdownBackendOptions] = None, + options: Optional[BaseBackendOptions] = None, ): # Raised first so a missing optional dependency gives an actionable # message rather than a NameError when marko is dereferenced below. @@ -288,6 +289,8 @@ def __init__( raise ImportError(_INSTALL_HINT) from _MARKO_IMPORT_ERROR if options is None: options = MarkdownBackendOptions() + if not isinstance(options, MarkdownBackendOptions): + raise TypeError("Expected MarkdownBackendOptions.") super().__init__(in_doc, path_or_stream, options) _log.debug("Starting MarkdownDocumentBackend...") @@ -416,7 +419,7 @@ def _create_list_item( marker: str = "", formatting: Optional[Formatting] = None, hyperlink: Optional[Union[AnyUrl, Path]] = None, - ): + ) -> ListItem: item = doc.add_list_item( text=text, enumerated=enumerated, @@ -425,6 +428,7 @@ def _create_list_item( formatting=formatting, hyperlink=hyperlink, ) + assert isinstance(item, ListItem) return item def _create_heading_item( @@ -435,7 +439,7 @@ def _create_heading_item( level: int, formatting: Optional[Formatting] = None, hyperlink: Optional[Union[AnyUrl, Path]] = None, - ): + ) -> TextItem: if level == 1: item = doc.add_title( text=text, @@ -451,6 +455,7 @@ def _create_heading_item( formatting=formatting, hyperlink=hyperlink, ) + assert isinstance(item, TextItem) return item def _flush_creation_stack( @@ -487,12 +492,12 @@ def _flush_creation_stack( hyperlink=hyperlink, ) if parent_ref: - list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) + list_last_item_by_ref[parent_ref] = parent_item list_item_counter_by_ref[parent_ref] = ( list_item_counter_by_ref.get(parent_ref, 0) + 1 ) - elif isinstance(to_create, _HeadingCreationPayload): + else: # Not keeping as parent_item as logic for correctly tracking # that not implemented yet (section components not captured # as heading children in marko) @@ -621,7 +626,7 @@ def _iterate_elements( # noqa: C901 hyperlink=hyperlink, ) if parent_ref: - list_last_item_by_ref[parent_ref] = cast(ListItem, parent_item) + list_last_item_by_ref[parent_ref] = parent_item list_item_counter_by_ref[parent_ref] = ( list_item_counter_by_ref.get(parent_ref, 0) + 1 ) @@ -637,13 +642,16 @@ def _iterate_elements( # noqa: C901 fig_caption: Optional[TextItem] = None if element.title is not None and element.title != "": title = unescape(element.title) - fig_caption = doc.add_text( + caption_item = doc.add_text( label=DocItemLabel.CAPTION, text=title, formatting=formatting, hyperlink=hyperlink, ) + assert isinstance(caption_item, TextItem) + fig_caption = caption_item + image_ref = self._load_image_ref(element.dest) doc.add_picture(parent=parent_item, image=image_ref, caption=fig_caption) @@ -659,9 +667,9 @@ def _iterate_elements( # noqa: C901 elif isinstance(element, marko.inline.Link): _log.debug(" - Link: %s", element.children) - hyperlink = TypeAdapter(Optional[Union[AnyUrl, Path]]).validate_python( - element.dest - ) + hyperlink = TypeAdapter[AnyUrl | Path | None]( + AnyUrl | Path | None + ).validate_python(element.dest) elif isinstance(element, marko.inline.RawText | marko.inline.Literal): _log.debug(" - RawText/Literal: %s", element.children) @@ -860,10 +868,10 @@ def _iterate_elements( # noqa: C901 ) # Iterate through the element's children (if any) - if hasattr(element, "children") and not isinstance( + if isinstance(element_children, list) and not isinstance( element, processed_block_types ): - for child in element.children: + for child in element_children: if ( isinstance(element, marko.block.ListItem) and isinstance(child, marko.block.List) diff --git a/docling/backend/mets_gbs_backend.py b/docling/backend/mets_gbs_backend.py index bbf8a1b1c8..a73730540f 100644 --- a/docling/backend/mets_gbs_backend.py +++ b/docling/backend/mets_gbs_backend.py @@ -531,7 +531,6 @@ def _parse_page( textline_cells=line_cells, char_cells=[], word_cells=word_cells, - has_textlines=True, has_words=True, has_chars=False, ) diff --git a/docling/backend/msexcel_backend.py b/docling/backend/msexcel_backend.py index 0743b6687b..a878800aec 100644 --- a/docling/backend/msexcel_backend.py +++ b/docling/backend/msexcel_backend.py @@ -52,7 +52,7 @@ crop_whitespace, get_docx_to_pdf_converter, ) -from docling.datamodel.backend_options import MsExcelBackendOptions +from docling.datamodel.backend_options import BaseBackendOptions, MsExcelBackendOptions from docling.datamodel.base_models import FormatToMimeType, InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -225,14 +225,14 @@ def __init__(self, sheet: Worksheet) -> None: ), ) min_row = ( - merged_range.min_row + int(merged_range.min_row) if min_row is None - else min(min_row, merged_range.min_row) + else min(min_row, int(merged_range.min_row)) ) min_col = ( - merged_range.min_col + int(merged_range.min_col) if min_col is None - else min(min_col, merged_range.min_col) + else min(min_col, int(merged_range.min_col)) ) max_row = max(max_row, merged_range.max_row) max_col = max(max_col, merged_range.max_col) @@ -337,7 +337,7 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: MsExcelBackendOptions | None = None, + options: BaseBackendOptions | None = None, ) -> None: """Initialize the MsExcelDocumentBackend object. @@ -355,6 +355,8 @@ def __init__( path_or_stream = convert_to_modern_format(path_or_stream, "xls", "xlsx") if options is None: options = MsExcelBackendOptions() + if not isinstance(options, MsExcelBackendOptions): + raise TypeError("Expected MsExcelBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.page_range = in_doc.limits.page_range @@ -363,7 +365,7 @@ def __init__( self.parent: GroupItem | None = None # Lazy-initialized LibreOffice converter for EMF/WMF images - self.xlsx_to_pdf_converter: Callable | None = None + self.xlsx_to_pdf_converter: Callable[[Path, Path], None] | None = None self.xlsx_to_pdf_converter_init: bool = False self.workbook = None @@ -448,6 +450,8 @@ def _parse_threaded_comments( except Exception as e: _log.debug(f"Could not parse person.xml: {e}") + if self.workbook is None: + return {} sheet_num = next( ( i @@ -659,8 +663,7 @@ def _convert_sheet( doc = self._find_tables_in_sheet(doc, sheet, page_no) doc = self._find_images_in_sheet(doc, sheet, page_no) # Charts can be on both Worksheet and Chartsheet objects - if isinstance(sheet, (Worksheet, Chartsheet)): - doc = self._find_chart_in_sheet(doc, sheet, page_no) + doc = self._find_chart_in_sheet(doc, sheet, page_no) self._sort_sheet_children_by_position(doc, page_no) return doc @@ -691,7 +694,7 @@ def _top_row(ref: Any) -> float: return float("inf") for prov in getattr(item, "prov", []): if prov.page_no == page_no: - return prov.bbox.t + return float(prov.bbox.t) return float("inf") sheet_group.children.sort(key=_top_row) @@ -811,7 +814,9 @@ def _find_tables_in_sheet( # Extract comments and link them to cells for (row, col), thread in comment_map.items(): cell_item = self._find_cell_item(doc, page_no, row, col) - targets = [cell_item] if cell_item else None + targets: list[DocItem | tuple[DocItem, tuple[int, int]]] | None = ( + [cell_item] if cell_item else None + ) comment_group = doc.add_group( label=GroupLabel.COMMENT_SECTION, @@ -1209,7 +1214,7 @@ def _anchor_to_tuple(anchor: Any) -> tuple[int, int, int, int]: ) return (0, 0, 0, 0) - def _get_libreoffice_converter(self) -> Callable | None: + def _get_libreoffice_converter(self) -> Callable[[Path, Path], None] | None: """Lazily initialize and return a LibreOffice converter callable. The converter accepts ``(input_path: Path, output_path: Path)`` and @@ -1289,10 +1294,11 @@ def _find_unsupported_images_in_sheet( Returns: The updated DoclingDocument. """ - # drawing rels are stored on the worksheet object by openpyxl + # openpyxl initializes these real private fields; its stubs omit them. + # Drawing relationships are stored on the worksheet object. drawing_paths: list[str] = [ rel.target - for rel in sheet._rels.find(SpreadsheetDrawing._rel_type) # type: ignore[attr-defined] + for rel in sheet._rels.find(SpreadsheetDrawing._rel_type) # ty: ignore[unresolved-attribute] ] if not drawing_paths: return doc @@ -1302,8 +1308,11 @@ def _find_unsupported_images_in_sheet( if isinstance(self.path_or_stream, BytesIO): self.path_or_stream.seek(0) + path_or_stream = self.path_or_stream + if path_or_stream is None: + return doc try: - with ZipFile(self.path_or_stream, "r") as zf: + with ZipFile(path_or_stream, "r") as zf: if _has_unsafe_zip_paths(zf.namelist()): _log.warning( "Skipping EMF/WMF scan: XLSX archive contains unsafe ZIP paths" @@ -1353,7 +1362,10 @@ def _process_drawing_for_unsupported_images( return doc deps = get_dependents(zf, rels_path) - for rel in drawing._blip_rels: + if drawing is None: + return doc + # openpyxl exposes this property; its stubs omit it. + for rel in drawing._blip_rels: # ty: ignore[unresolved-attribute] dep = deps.get(rel.embed) if dep.Type != IMAGE_NS or dep.target not in zf.namelist(): continue @@ -1410,13 +1422,15 @@ def _find_images_in_sheet( """ if self.workbook is not None: content_layer = self._get_sheet_content_layer(sheet) - # Images that PIL can read are already loaded by openpyxl into sheet._images - for item in sheet._images: # type: ignore[attr-defined] + # openpyxl initializes _images; its stubs omit this real field. + # These images can already be read by PIL. + for item in sheet._images: # ty: ignore[unresolved-attribute] try: image: Image = cast(Image, item) ref = image.ref + # Pillow also accepts read-only streams, which its IO stubs omit. pil_image = ( - ref if isinstance(ref, PILImage.Image) else PILImage.open(ref) + ref if isinstance(ref, PILImage.Image) else PILImage.open(ref) # ty: ignore[invalid-argument-type] ) doc.add_picture( parent=self.parent, @@ -1467,7 +1481,8 @@ def _find_chart_in_sheet( return doc content_layer = self._get_sheet_content_layer(sheet) - charts = sheet._charts # type: ignore[attr-defined] + # openpyxl initializes _charts on both sheet types; its stubs omit it. + charts = sheet._charts # ty: ignore[unresolved-attribute] if not charts: return doc @@ -1863,10 +1878,10 @@ def _ref_formula(data_source: Any) -> str | None: return None num_ref = getattr(data_source, "numRef", None) if num_ref is not None and num_ref.f: - return num_ref.f + return str(num_ref.f) str_ref = getattr(data_source, "strRef", None) if str_ref is not None and str_ref.f: - return str_ref.f + return str(str_ref.f) return None def _resolve_reference(self, ref: str) -> list[str]: @@ -1897,7 +1912,10 @@ def _resolve_reference(self, ref: str) -> list[str]: sheet_part = sheet_part[1:-1].replace("''", "'") sheet_name = sheet_part else: - sheet_name, cell_range = self.workbook.active.title, ref + active_sheet = self.workbook.active + if active_sheet is None: + return [] + sheet_name, cell_range = active_sheet.title, ref if sheet_name not in self.workbook.sheetnames: _log.debug("Chart references unknown sheet %r", sheet_name) return [] @@ -1910,6 +1928,8 @@ def _resolve_reference(self, ref: str) -> list[str]: _log.debug("Could not parse chart range %r", cell_range) return [] + if min_row is None or max_row is None or min_col is None or max_col is None: + return [] values: list[str] = [] for row in range(min_row, max_row + 1): for col in range(min_col, max_col + 1): @@ -1977,7 +1997,7 @@ def _find_cell_item( return None @staticmethod - def _get_sheet_content_layer(sheet: Worksheet) -> ContentLayer | None: + def _get_sheet_content_layer(sheet: Worksheet | Chartsheet) -> ContentLayer | None: return ( None if sheet.sheet_state == Worksheet.SHEETSTATE_VISIBLE diff --git a/docling/backend/mspowerpoint_backend.py b/docling/backend/mspowerpoint_backend.py index 1bd2eebbb4..a909cd6b0d 100644 --- a/docling/backend/mspowerpoint_backend.py +++ b/docling/backend/mspowerpoint_backend.py @@ -10,7 +10,7 @@ from io import BytesIO from pathlib import Path from tempfile import mkdtemp -from typing import Any, Callable, Final, Iterable, Iterator, Optional, Union +from typing import Any, Callable, Final, Iterable, Iterator, Optional, TypedDict, Union from docling_core.types.doc import ( BoundingBox, @@ -22,6 +22,7 @@ ImageRef, ListGroup, ListItem, + NodeItem, PictureClassificationLabel, PictureClassificationMetaField, PictureClassificationPrediction, @@ -46,11 +47,22 @@ crop_whitespace, get_docx_to_pdf_converter, ) -from docling.datamodel.backend_options import MsPowerpointBackendOptions +from docling.datamodel.backend_options import ( + BaseBackendOptions, + MsPowerpointBackendOptions, +) from docling.datamodel.base_models import FormatToMimeType, InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError + +class _ListMarker(TypedDict): + is_list: bool | None + kind: str | None + detail: str | None + level: int + + _log = logging.getLogger(__name__) _PPTX_AVAILABLE: bool = False @@ -201,7 +213,7 @@ def __init__( self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path], - options: Optional[MsPowerpointBackendOptions] = None, + options: BaseBackendOptions | None = None, ) -> None: if not _PPTX_AVAILABLE: raise ImportError(_INSTALL_HINT) from _PPTX_IMPORT_ERROR @@ -209,11 +221,13 @@ def __init__( path_or_stream = convert_to_modern_format(path_or_stream, "ppt", "pptx") if options is None: options = MsPowerpointBackendOptions() + if not isinstance(options, MsPowerpointBackendOptions): + raise TypeError("MsPowerpointBackendOptions are required.") super().__init__(in_doc, path_or_stream, options) - self.path_or_stream: Union[BytesIO, Path] = path_or_stream + self.path_or_stream: BytesIO | Path | None = path_or_stream self.page_range = in_doc.limits.page_range - self.pptx_to_pdf_converter: Optional[Callable] = None + self.pptx_to_pdf_converter: Callable[[Path, Path], None] | None = None self.pptx_to_pdf_converter_init: bool = False self._render_charts: bool = False self._metafile_hint_emitted: bool = False @@ -221,10 +235,10 @@ def __init__( self.pptx_obj: Optional[presentation.Presentation] = None self.valid: bool = False try: - if isinstance(self.path_or_stream, BytesIO): - self.pptx_obj = Presentation(self.path_or_stream) - elif isinstance(self.path_or_stream, Path): - self.pptx_obj = Presentation(str(self.path_or_stream)) + if isinstance(path_or_stream, BytesIO): + self.pptx_obj = Presentation(path_or_stream) + else: + self.pptx_obj = Presentation(str(path_or_stream)) self.valid = True except Exception as e: @@ -301,7 +315,7 @@ def _generate_prov( top = 0 width = slide_size.width height = slide_size.height - shape_bbox = [left, top, left + width, top + height] + shape_bbox = (float(left), float(top), float(left + width), float(top + height)) # python-pptx reports left and top as EMU from the slide's top-left, with # y growing downward, so the tuple above is in TOPLEFT order. Tagging it # BOTTOMLEFT does not convert it: BoundingBox.from_tuple unpacks @@ -388,12 +402,14 @@ def _parse_bullet_from_paragraph_properties( # Bullet character buChar = pPr.find("a:buChar", namespaces=self.NAMESPACES) if buChar is not None: - return (True, "buChar", buChar.get("char")) + detail = buChar.get("char") + return (True, "buChar", str(detail) if detail is not None else None) # Auto numbering buAuto = pPr.find("a:buAutoNum", namespaces=self.NAMESPACES) if buAuto is not None: - return (True, "buAutoNum", buAuto.get("type")) + detail = buAuto.get("type") + return (True, "buAutoNum", str(detail) if detail is not None else None) # Picture bullet buBlip = pPr.find("a:buBlip", namespaces=self.NAMESPACES) @@ -573,7 +589,7 @@ def _is_list_item(self, paragraph) -> tuple[bool, str]: else: return (False, "None") - def _get_effective_list_marker(self, shape, paragraph) -> dict: + def _get_effective_list_marker(self, shape, paragraph) -> _ListMarker: """Return a dictionary describing the effective list marker for a paragraph. List marker information can come from several sources: direct paragraph @@ -621,7 +637,7 @@ def _get_effective_list_marker(self, shape, paragraph) -> dict: } # 3) Layout placeholder lstStyle (if this is a placeholder) - layout_result = None + layout_result: _ListMarker | None = None if shape.is_placeholder: idx = shape.placeholder_format.idx layout = shape.part.slide.slide_layout @@ -1068,7 +1084,7 @@ def _chart_title_text(chart: Any) -> Optional[str]: """ if not chart.has_title: return None - text = chart.chart_title.text_frame.text.strip() + text = str(chart.chart_title.text_frame.text).strip() return text or None @staticmethod @@ -1255,7 +1271,7 @@ def _handle_chart(self, shape, parent_slide, slide_ind, doc, slide_size): ) return - def _get_libreoffice_converter(self) -> Optional[Callable]: + def _get_libreoffice_converter(self) -> Callable[[Path, Path], None] | None: """Lazily initialize and return a LibreOffice converter callable. The converter accepts ``(input_path, output_path)`` and converts the @@ -1455,6 +1471,7 @@ def _walk_linear( # Units of size in PPTX by default are EMU units (English Metric Units) slide_width = pptx_obj.slide_width slide_height = pptx_obj.slide_height + assert slide_width is not None and slide_height is not None self._render_charts = ( isinstance(self.options, MsPowerpointBackendOptions) @@ -1465,7 +1482,7 @@ def _walk_linear( self._render_charts = False max_levels = 10 - parents = {} # type: ignore + parents: dict[int, NodeItem | None] = {} for i in range(max_levels): parents[i] = None diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index dfcb50c99c..8ce618545a 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -11,7 +11,7 @@ from copy import deepcopy from io import BytesIO from pathlib import Path -from typing import Any, Callable, Final +from typing import Any, Callable, Final, TypedDict from urllib.parse import urlparse from docling_core.types.doc import ( @@ -50,7 +50,7 @@ get_pil_from_dml_docx, ) from docling.backend.docx.latex.omml import oMath2Latex -from docling.datamodel.backend_options import MsWordBackendOptions +from docling.datamodel.backend_options import BaseBackendOptions, MsWordBackendOptions from docling.datamodel.base_models import FormatToMimeType from docling.datamodel.document import InputDocument, InputFormat from docling.exceptions import DocumentLoadError, SecurityError @@ -566,7 +566,11 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: parent = rel.getparent() if parent is not None: parent.remove(rel) - return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) + serialized = etree.tostring( + root, xml_declaration=True, encoding="UTF-8", standalone=True + ) + assert isinstance(serialized, bytes) + return serialized def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: @@ -646,6 +650,13 @@ def _normalize_strict_ooxml(archive: zipfile.ZipFile) -> BytesIO: return normalized +class _ListHistory(TypedDict): + names: list[str | None] + levels: list[int | None] + numids: list[int | None] + indents: list[int | None] + + class MsWordDocumentBackend(DeclarativeDocumentBackend): """Backend for parsing Word documents (DOCX and DOC files). @@ -771,12 +782,14 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: MsWordBackendOptions | None = None, + options: BaseBackendOptions | None = None, ) -> None: if not _DOCX_AVAILABLE: raise ImportError(_INSTALL_HINT) from _DOCX_IMPORT_ERROR if options is None: options = MsWordBackendOptions() + if not isinstance(options, MsWordBackendOptions): + raise TypeError("Expected MsWordBackendOptions.") if in_doc.format in {InputFormat.DOC, InputFormat.RTF}: path_or_stream = convert_to_modern_format( path_or_stream, in_doc.format.value, "docx" @@ -789,7 +802,7 @@ def __init__( ) # self.initialise(path_or_stream) # Word file: - self.path_or_stream: BytesIO | Path = path_or_stream + self.path_or_stream: BytesIO | Path | None = path_or_stream self.valid: bool = False # Initialise the parents for the hierarchy self.max_levels: int = 10 @@ -800,7 +813,7 @@ def __init__( self.equation_bookends: str = "{EQ}" # Track processed textbox elements to avoid duplication self.processed_textbox_elements: set[etree._Element] = set() - self.docx_to_pdf_converter: Callable | None = None + self.docx_to_pdf_converter: Callable[[Path, Path], None] | None = None self.docx_to_pdf_converter_init = False self.display_drawingml_warning = True self._empty_docx_template: DocxDocument | None = None @@ -824,7 +837,7 @@ def __init__( # Set starting content layer self.content_layer = ContentLayer.BODY - self.history: dict[str, Any] = { + self.history: _ListHistory = { "names": [None], "levels": [None], "numids": [None], @@ -849,7 +862,7 @@ def __init__( self._default_paragraph_style: BaseStyle | None = None self.docx_obj = self.load_msword_file( - path_or_stream=self.path_or_stream, document_hash=self.document_hash + path_or_stream=path_or_stream, document_hash=self.document_hash ) if self.docx_obj: self.valid = True @@ -931,7 +944,7 @@ def load_msword_file( if has_fragment_rels: return Document(_sanitize_docx(archive)) return Document(str(path_or_stream)) - elif isinstance(path_or_stream, BytesIO): + else: with zipfile.ZipFile(path_or_stream) as archive: is_strict = _is_strict_ooxml(archive) has_fragment_rels = not is_strict and _has_fragment_only_rels( @@ -943,8 +956,6 @@ def load_msword_file( return Document(_sanitize_docx(archive)) path_or_stream.seek(0) return Document(path_or_stream) - else: - return None except SecurityError: raise except Exception as e: @@ -1014,21 +1025,22 @@ def _get_or_create_list_group( Returns: The list group to use (either reused or newly created). """ - if self._can_reuse_list_group(numid, parent): + cached_group = self.last_list_group + if cached_group is not None and self._can_reuse_list_group(numid, parent): # Reuse only if nothing but empty paragraphs (added when the list was # closed) follows the cached group in its parent. Otherwise the new # items would be placed before the intervening content, e.g. a table. container = parent if parent is not None else doc.body - trailing_empty: list[TextItem] = [] + trailing_empty: list[NodeItem] = [] for ref in reversed(container.children): item = ref.resolve(doc) if isinstance(item, TextItem) and not item.text.strip(): trailing_empty.append(item) continue - if item.self_ref == self.last_list_group.self_ref: + if item.self_ref == cached_group.self_ref: if trailing_empty: doc.delete_items(node_items=trailing_empty) - return self.last_list_group + return cached_group break list_gr = doc.add_list_group( @@ -1036,7 +1048,7 @@ def _get_or_create_list_group( parent=parent, content_layer=self.content_layer, ) - elem_ref.append(list_gr.get_ref()) + elem_ref.append(RefItem(cref=list_gr.self_ref)) # Update cache for potential future reuse self.last_list_group = list_gr @@ -1071,7 +1083,7 @@ def _isolated_list_context(self): This ensures that lists in different table cells are treated independently, even when they share the same numId. """ - saved_history = { + saved_history: _ListHistory = { "names": self.history["names"].copy(), "levels": self.history["levels"].copy(), "numids": self.history["numids"].copy(), @@ -1166,7 +1178,9 @@ def _in_textbox(elem): name="shape-text", content_layer=self.content_layer, ) - added_elements.append(shape_group.get_ref()) + added_elements.append( + RefItem(cref=shape_group.self_ref) + ) doc.add_text( label=DocItemLabel.TEXT, parent=shape_group, @@ -1291,15 +1305,16 @@ def _is_invisible_spacer(self, pil_image: Image.Image | None) -> bool: try: extrema = pil_image.getextrema() - if extrema is not None: - if pil_image.mode in ("RGBA", "LA"): + if pil_image.mode in ("RGBA", "LA"): + alpha = extrema[-1] + if isinstance(alpha, tuple): # extrema[-1] is the Alpha channel. If max alpha is 0, it is 100% invisible. - if extrema[-1][1] == 0: - return True - elif pil_image.mode == "RGB": - # If all channels are exactly 255, it is a pure white spacing box. - if extrema == ((255, 255), (255, 255), (255, 255)): + if alpha[1] == 0: return True + elif pil_image.mode == "RGB": + # If all channels are exactly 255, it is a pure white spacing box. + if extrema == ((255, 255), (255, 255), (255, 255)): + return True except Exception: pass # pragma: no cover @@ -1376,16 +1391,19 @@ def _style_numbering( if numId is None: numId_elem = style_numPr.find(f"{_W_NS_CLARK}numId") if numId_elem is not None: - numId = numId_elem.get(self.XML_KEY) + value = numId_elem.get(self.XML_KEY) + numId = str(value) if value is not None else None if ilvl is None: ilvl_elem = style_numPr.find(f"{_W_NS_CLARK}ilvl") if ilvl_elem is not None: - ilvl = ilvl_elem.get(self.XML_KEY) + value = ilvl_elem.get(self.XML_KEY) + ilvl = str(value) if value is not None else None if numId is not None and ilvl is not None: break # A malformed basedOn chain can hop to a style type that lacks # base_style; getattr keeps the walk safe. - style = getattr(style, "base_style", None) + base_style = getattr(style, "base_style", None) + style = base_style if isinstance(base_style, ParagraphStyle) else None depth += 1 # If numId is found but ilvl is not specified, default to level 0 @@ -1393,7 +1411,7 @@ def _style_numbering( ilvl = "0" return numId, ilvl - def _get_level_element(self, numid: int, ilvl: int) -> BaseOxmlElement | None: + def _get_level_element(self, numid: int, ilvl: int) -> etree._Element | None: """Find the level element from the numbering XML for a given numId and ilvl.""" try: numbering_part = None @@ -1431,9 +1449,11 @@ def _get_level_element(self, numid: int, ilvl: int) -> BaseOxmlElement | None: if abstract_num_element is None: return None - return abstract_num_element.find( + level = abstract_num_element.find( f".//w:lvl[@w:ilvl='{ilvl}']", namespaces=namespaces ) + assert level is None or isinstance(level, etree._Element) + return level except Exception as e: _log.debug(f"Error finding level element: {e}") return None @@ -1447,7 +1467,8 @@ def _get_level_num_fmt(self, numid: int, ilvl: int) -> str | None: num_fmt_element = lvl_element.find(".//w:numFmt", namespaces=namespaces) if num_fmt_element is None: return None - return num_fmt_element.get(self.XML_KEY) + value = num_fmt_element.get(self.XML_KEY) + return str(value) if value is not None else None def _get_start_value(self, numid: int, ilvl: int) -> int: """Read the start value from the abstractNum definition.""" @@ -1633,7 +1654,9 @@ def _last_child_item( container = parent if parent is not None else doc.body if not container.children: return None - return container.children[-1].resolve(doc) + child = container.children[-1].resolve(doc) + assert isinstance(child, NodeItem) + return child def _is_code_style(self, style: ParagraphStyle | None) -> bool: """Return True if a style marks its paragraphs as code. @@ -1657,7 +1680,8 @@ def _is_code_style(self, style: ParagraphStyle | None) -> bool: # A malformed basedOn chain can hop to a style type (e.g. a # numbering style) that lacks this attribute; getattr keeps # the walk safe. - style = getattr(style, "base_style", None) + base_style = getattr(style, "base_style", None) + style = base_style if isinstance(base_style, ParagraphStyle) else None depth += 1 return False @@ -1687,11 +1711,12 @@ def _effective_style_font(self, style: ParagraphStyle | None) -> str: return "" font_name = style.font.name if font_name: - return font_name.strip().lower() + return str(font_name).strip().lower() # A malformed basedOn chain can hop to a style type (e.g. a # numbering style) that lacks base_style; getattr keeps the # walk safe. - style = getattr(style, "base_style", None) + base_style = getattr(style, "base_style", None) + style = base_style if isinstance(base_style, ParagraphStyle) else None depth += 1 return "" @@ -1812,13 +1837,15 @@ def _get_label_and_level(self, paragraph: Paragraph) -> tuple[str, int | None]: if style is None: return "Normal", None - label: str = style.style_id - name: str = style.name or "" + label = str(style.style_id) + name = str(style.name or "") base_style_label: str | None = None base_style_name: str | None = None if isinstance(base_style := getattr(style, "base_style", None), ParagraphStyle): - base_style_label = base_style.style_id - base_style_name = base_style.name + base_style_label = str(base_style.style_id) + base_style_name = ( + str(base_style.name) if base_style.name is not None else None + ) if not label: return "Normal", None @@ -1928,16 +1955,15 @@ def _get_format_from_run( if not is_bold: try: - if run._element is not None: - b_tags = run._element.xpath(".//w:b") - for b in b_tags: - val = b.get(f"{_W_NS_CLARK}val") - if val not in ["0", "false"]: - is_bold = True - break + b_tags = run._element.xpath(".//w:b") + for b in b_tags: + val = b.get(f"{_W_NS_CLARK}val") + if val not in ["0", "false"]: + is_bold = True + break - if not is_bold and run._parent._element is not None: - pPr_b = run._parent._element.xpath("./w:pPr/w:rPr/w:b") + if not is_bold and isinstance(run._parent, Paragraph): + pPr_b = run._parent._p.xpath("./w:pPr/w:rPr/w:b") for b in pPr_b: val = b.get(f"{_W_NS_CLARK}val") if val not in ["0", "false"]: @@ -2067,7 +2093,7 @@ def _get_children_recursive(node): self._get_hyperlink_target(item), ) ) - elif isinstance(item, Run): + else: content.append( (item.text, self._get_format_from_run(item, paragraph), None) ) @@ -2157,7 +2183,7 @@ def _is_checkbox_checked(self, element: BaseOxmlElement) -> bool: if checked_elem is not None: val = checked_elem.get(f"{{{w14_ns}}}val") - return val == "1" + return str(val) == "1" return False @@ -2335,7 +2361,7 @@ def _collect_textbox_paragraphs( def _handle_textbox_content( self, - textbox_elements: list, + textbox_elements: list[etree._Element], doc: DoclingDocument, ) -> list[RefItem]: """Process textbox content and add it to the document structure.""" @@ -2348,7 +2374,7 @@ def _handle_textbox_content( name="textbox", content_layer=self.content_layer, ) - elem_ref.append(textbox_group.get_ref()) + elem_ref.append(RefItem(cref=textbox_group.self_ref)) # Set this as the current parent to ensure textbox content # is properly nested in document structure original_parent = self.parents[level] @@ -2508,7 +2534,7 @@ def _create_or_reuse_parent( *, doc: DoclingDocument, prev_parent: NodeItem | None, - paragraph_elements: list, + paragraph_elements: list[tuple[str, Formatting | None, AnyUrl | Path | None]], ) -> NodeItem | None: return ( doc.add_inline_group(parent=prev_parent, content_layer=self.content_layer) @@ -2629,7 +2655,7 @@ def _handle_text_elements( content_layer=self.content_layer, ) self.parents[0] = te - elem_ref.append(te.get_ref()) + elem_ref.append(RefItem(cref=te.self_ref)) elif "Heading" in p_style_id: # _add_heading clears the parents tail; reset list context so the # next list item opens fresh under this heading. @@ -2639,9 +2665,7 @@ def _handle_text_elements( elem_ref.extend(h1) elif len(equations) > 0: - if (paragraph.text is None or len(paragraph.text.strip()) == 0) and len( - text - ) > 0: + if len(paragraph.text.strip()) == 0 and len(text) > 0: # Standalone equation(s) — emit each as a separate formula level = self._get_level() parent = self.parents[level - 1] @@ -2655,7 +2679,7 @@ def _handle_text_elements( text=eq_text, content_layer=self.content_layer, ) - elem_ref.append(t1.get_ref()) + elem_ref.append(RefItem(cref=t1.self_ref)) else: t1 = doc.add_text( label=DocItemLabel.FORMULA, @@ -2663,14 +2687,14 @@ def _handle_text_elements( text=text.replace("", "").replace("", ""), content_layer=self.content_layer, ) - elem_ref.append(t1.get_ref()) + elem_ref.append(RefItem(cref=t1.self_ref)) else: # Inline equation level = self._get_level() inline_equation = doc.add_inline_group( parent=self.parents[level - 1], content_layer=self.content_layer ) - elem_ref.append(inline_equation.get_ref()) + elem_ref.append(RefItem(cref=inline_equation.self_ref)) self._add_inline_equations_to_parent( doc=doc, @@ -2713,7 +2737,7 @@ def _handle_text_elements( # Buffered: written only if more code follows, so a # block never ends in blank lines. self._pending_code_blank_lines += 1 - elem_ref.append(merge_target.get_ref()) + elem_ref.append(RefItem(cref=merge_target.self_ref)) self._force_new_code_block = False elif text: # Start a new block, but never on a leading blank paragraph. @@ -2725,7 +2749,7 @@ def _handle_text_elements( content_layer=self.content_layer, code_language=detect_code_language(code_text), ) - elem_ref.append(code_item.get_ref()) + elem_ref.append(RefItem(cref=code_item.self_ref)) self._force_new_code_block = False # A blank that neither starts nor extends a block leaves the # barrier armed. @@ -2756,7 +2780,7 @@ def _handle_text_elements( hyperlink=hyperlink, content_layer=self.content_layer, ) - elem_ref.append(text_item.get_ref()) + elem_ref.append(RefItem(cref=text_item.self_ref)) self._update_history(p_style_id, p_level, numid, ilevel) @@ -2786,7 +2810,7 @@ def _add_heading( label=GroupLabel.SECTION, name=f"header-{i}", ) - elem_ref.append(gr1.get_ref()) + elem_ref.append(RefItem(cref=gr1.self_ref)) self.parents[i] = gr1 elif curr_level < level: @@ -2836,13 +2860,13 @@ def _add_heading( level=add_level, ) self.parents[current_level] = hd - elem_ref.append(hd.get_ref()) + elem_ref.append(RefItem(cref=hd.self_ref)) return elem_ref def _add_formatted_list_item( self, doc: DoclingDocument, - elements: list, + elements: list[tuple[str, Formatting | None, AnyUrl | Path | None]], marker: str, enumerated: bool, level: int, @@ -2891,7 +2915,7 @@ def _add_formatted_list_item( def _add_list_item_with_marker( self, doc: DoclingDocument, - elements: list, + elements: list[tuple[str, Formatting | None, AnyUrl | Path | None]], numid: int, ilevel: int, is_numbered: bool, @@ -2946,7 +2970,7 @@ def _add_inline_equations_to_parent( content_layer=self.content_layer, ) if elem_ref is not None: - elem_ref.append(e1.get_ref()) + elem_ref.append(RefItem(cref=e1.self_ref)) e2 = doc.add_text( label=DocItemLabel.FORMULA, @@ -2955,7 +2979,7 @@ def _add_inline_equations_to_parent( content_layer=self.content_layer, ) if elem_ref is not None: - elem_ref.append(e2.get_ref()) + elem_ref.append(RefItem(cref=e2.self_ref)) if len(text_tmp) > 0: e3 = doc.add_text( @@ -2965,7 +2989,7 @@ def _add_inline_equations_to_parent( content_layer=self.content_layer, ) if elem_ref is not None: - elem_ref.append(e3.get_ref()) + elem_ref.append(RefItem(cref=e3.self_ref)) def _slot_for(self, word_ilevel: int) -> int: """Map a Word ``w:ilvl`` value to the internal parents-slot index. @@ -2980,6 +3004,8 @@ def _slot_for(self, word_ilevel: int) -> int: sits at level 1 and later returns to level 0) are clamped to the list base so they stay inside the current list instead of mapping below it. """ + assert self.level_at_new_list is not None + assert self.level_start_ilevel is not None return self.level_at_new_list + max(0, word_ilevel - self.level_start_ilevel) def _manage_list_structure( @@ -3057,7 +3083,7 @@ def _manage_list_structure( content_layer=self.content_layer, ) self.parents[i] = list_gr1 - elem_ref.append(list_gr1.get_ref()) + elem_ref.append(RefItem(cref=list_gr1.self_ref)) use_level = self._slot_for(ilevel) elif ( @@ -3118,7 +3144,7 @@ def _add_list_item( doc: DoclingDocument, numid: int, ilevel: int, - elements: list, + elements: list[tuple[str, Formatting | None, AnyUrl | Path | None]], is_numbered: bool = False, ) -> list[RefItem]: """Add a regular list item without inline equations. @@ -3234,10 +3260,10 @@ def _group_cell_elements( group_element.children.append(prov) pr_item = prov.resolve(doc) item_parent = pr_item.parent.resolve(doc) - if pr_item.get_ref() in item_parent.children: - item_parent.children.remove(pr_item.get_ref()) - pr_item.parent = group_element.get_ref() - ref_for_rich_cell = group_element.get_ref() + if RefItem(cref=pr_item.self_ref) in item_parent.children: + item_parent.children.remove(RefItem(cref=pr_item.self_ref)) + pr_item.parent = RefItem(cref=group_element.self_ref) + ref_for_rich_cell = RefItem(cref=group_element.self_ref) return ref_for_rich_cell @staticmethod @@ -3317,7 +3343,7 @@ def _handle_tables( docling_table = doc.add_table( data=data, parent=self.parents[level - 1], content_layer=self.content_layer ) - elem_ref.append(docling_table.get_ref()) + elem_ref.append(RefItem(cref=docling_table.self_ref)) open_cells: dict[int, TableCell] = {} for row_idx, row in enumerate(table.rows): @@ -3460,10 +3486,9 @@ def _is_rich_table_cell(self, cell: _Cell) -> bool: runs = list(para.iterchildren(f"{_W_NS_CLARK}r")) for rn in runs: item: Run = Run(rn, self.docx_obj) - if item is not None: - fm = MsWordDocumentBackend._get_format_from_run(item) - if fm != Formatting(): - return True + fm = MsWordDocumentBackend._get_format_from_run(item) + if fm != Formatting(): + return True # Walk a non-empty code-styled cell as rich content so it can emit a # CodeItem; the font fallback never fires inside cells. @@ -3502,7 +3527,9 @@ def _get_image_from_relationship( return None # Access the image part using the relationship ID image_part = rel.target_part - image_data = image_part.blob # Get the binary image data + blob = image_part.blob # Get the binary image data + assert isinstance(blob, bytes) + image_data = blob return image_data def _add_picture_to_doc( @@ -3537,7 +3564,7 @@ def _add_picture_to_doc( caption=None, content_layer=target_layer, ) - return p.get_ref() + return RefItem(cref=p.self_ref) def _convert_elements_via_docx( self, elements: Any | list[Any], element_tag: str | list[str] | None = None @@ -3570,6 +3597,7 @@ def _convert_elements_via_docx( try: # Loaded once per backend instance; del body[:] clears cheaply on every call. if self._empty_docx_template is None: + assert self.path_or_stream is not None self._empty_docx_template = self.load_msword_file( self.path_or_stream, self.document_hash ) @@ -3727,7 +3755,7 @@ def _handle_vml_pictures( ) pil_image = None - if pil_image is None and image_data is not None: + if pil_image is None: pil_image = self._convert_elements_via_docx( imagedata, ["object", "pict"] ) @@ -3914,7 +3942,7 @@ def _chart_title_text(self, chart_root: Any) -> str | None: if not text: cached = self._read_chart_cache(title) text = cached[0].strip() if cached else "" - return text or None + return str(text) if text else None def _chart_to_table_data(self, chart_root: Any) -> TableData | None: """Reconstruct a chart's underlying data grid as a TableData. @@ -4088,7 +4116,7 @@ def _handle_chart(self, doc: DoclingDocument, chart_el: Any) -> RefItem | None: else None ), ) - return picture.get_ref() + return RefItem(cref=picture.self_ref) def _add_header_footer(self, docx_obj: DocxDocument, doc: DoclingDocument) -> None: """Add section headers and footers. diff --git a/docling/backend/opendocument_backend.py b/docling/backend/opendocument_backend.py index 877118a52f..1b45862ce0 100644 --- a/docling/backend/opendocument_backend.py +++ b/docling/backend/opendocument_backend.py @@ -25,7 +25,7 @@ from dataclasses import dataclass from io import BytesIO from pathlib import Path -from typing import Any, cast +from typing import Any, TypedDict, cast from docling_core.types.doc import ( BoundingBox, @@ -61,7 +61,11 @@ PaginatedDocumentBackend, ) from docling.backend.utils.image_resource_loader import ImageResourceLoader -from docling.datamodel.backend_options import OdsBackendOptions +from docling.datamodel.backend_options import ( + BackendOptions, + BaseBackendOptions, + OdsBackendOptions, +) from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument @@ -77,7 +81,9 @@ ListItem, Paragraph, Section, + Style, Table as OdfTable, + XmlPart, ) _ODFDO_AVAILABLE = True @@ -149,6 +155,23 @@ def _load_odf_document( ) from e +class _OdsTableRegion(TypedDict): + bounds: tuple[int, int, int, int] + data: TableData + + +class _OdfCellData(TypedDict): + text: str + row_span: int + col_span: int + start_row_offset_idx: int + end_row_offset_idx: int + start_col_offset_idx: int + end_col_offset_idx: int + column_header: bool + row_header: bool + + class _OdfBaseBackend(DeclarativeDocumentBackend): """Shared loading / validation logic for ODT, ODS and ODP backends.""" @@ -159,12 +182,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: OdsBackendOptions | None = None, + options: BackendOptions | None = None, ) -> None: if not _ODFDO_AVAILABLE: raise ImportError(_INSTALL_HINT) from _ODFDO_IMPORT_ERROR super().__init__(in_doc, path_or_stream, options) - self.path_or_stream: BytesIO | Path = path_or_stream + self.path_or_stream: BytesIO | Path | None = path_or_stream self.page_range = in_doc.limits.page_range self.valid: bool = False self.odf_obj: OdfDocument = _load_odf_document( @@ -315,7 +338,7 @@ def _formatting_from_odf_text_style( return _formatting_or_none(formatting) style = odf_obj.get_style("text", style_name) - if style is None: + if not isinstance(style, Style): return _formatting_or_none(formatting) props = style.get_properties() or {} @@ -331,7 +354,7 @@ def _formatting_from_odf_text_style( ), None, ) - if font_weight is not None: + if isinstance(font_weight, str): formatting.bold = _is_bold_weight(font_weight) font_style = next( @@ -360,7 +383,7 @@ def _formatting_from_odf_text_style( formatting.strikethrough = line_through != "none" text_position = props.get("style:text-position") - if text_position is not None: + if isinstance(text_position, str): if text_position.startswith("super"): formatting.script = Script.SUPER elif text_position.startswith("sub"): @@ -480,7 +503,7 @@ def _add_odf_text_runs( if not runs: return None if len(runs) == 1: - return doc.add_text( + item = doc.add_text( label=label, parent=parent, text=runs[0].text, @@ -489,6 +512,9 @@ def _add_odf_text_runs( hyperlink=runs[0].hyperlink, ) + assert isinstance(item, NodeItem) + return item + inline_group = doc.add_inline_group(parent=parent, content_layer=content_layer) for run in runs: doc.add_text( @@ -770,7 +796,7 @@ def _odf_list_level_style( return None style = odf_obj.get_style("list", style_name) - if style is None: + if not isinstance(style, Style): return None return style.get_level_style(level) @@ -785,7 +811,7 @@ def _odf_list_level_is_enumerated( level_style = _odf_list_level_style(odf_obj, odf_list, level) if level_style is None: return fallback - return level_style.tag == "text:list-level-style-number" + return bool(level_style.tag == "text:list-level-style-number") def _odf_list_start_value( @@ -901,11 +927,15 @@ def _image_ref_from_odf_image( image_data: bytes | None = None get_data = getattr(image, "get_data", None) if callable(get_data): - image_data = get_data() + data = get_data() + if isinstance(data, bytes): + image_data = data if image_data is None and odf_obj is not None and image_url: try: - image_data = odf_obj.get_part(image_url) + part = odf_obj.get_part(image_url) + if isinstance(part, bytes): + image_data = part except Exception: image_data = None @@ -923,15 +953,16 @@ def _image_ref_from_odf_image( def _odf_image_href(image: Any) -> str | None: - return getattr(image, "url", None) or getattr(image, "attributes", {}).get( + href = getattr(image, "url", None) or getattr(image, "attributes", {}).get( "xlink:href" ) + return str(href) if href is not None else None def _odf_image_can_be_bitmap(image: Any, image_url: str | None) -> bool: mime_type = getattr(image, "attributes", {}).get("draw:mime-type") if mime_type is not None: - return mime_type.startswith("image/") and mime_type != "image/svg+xml" + return bool(mime_type.startswith("image/") and mime_type != "image/svg+xml") if image_url is None: return True @@ -974,7 +1005,7 @@ def _strip_odf_image_reference_text(text: str, images: list[Any]) -> str: def _add_odf_images( doc: DoclingDocument, images: list[Any], - parent: NodeItem, + parent: NodeItem | None, content_layer: ContentLayer | None, odf_obj: OdfDocument | None, *, @@ -1143,7 +1174,7 @@ def _odf_chart_title(chart_content: Any) -> str | None: for title in chart_content.get_elements("descendant::chart:title"): text = title.text_content.strip() if text: - return text + return str(text) return None @@ -1163,6 +1194,8 @@ def _chart_data_from_frame( try: chart_content = odf_obj.get_part(_embedded_odf_content_path(object_href)) + if not isinstance(chart_content, XmlPart): + return None # odfdo resolves XML parts lazily, so a reference to a part that is missing # from the package only fails once the part is actually read. chart_classification = _odf_chart_classification(chart_content) @@ -1238,7 +1271,7 @@ def _add_odf_charts( def _add_odf_list( doc: DoclingDocument, odf_list: OdfList, - parent: NodeItem, + parent: NodeItem | None, content_layer: ContentLayer | None, odf_obj: OdfDocument | None, enumerated: bool = False, @@ -1283,7 +1316,7 @@ def _add_odf_list( ) return None - if should_continue and continued_state is not None: + if should_continue: list_group = continued_state.group current_enumerated = continued_state.enumerated counter = continued_state.counter @@ -1392,7 +1425,7 @@ def _add_odf_list( def _add_rich_cell_children( doc: DoclingDocument, cell: Any, - parent: NodeItem, + parent: NodeItem | None, content_layer: ContentLayer | None, odf_obj: OdfDocument | None, image_loader: ImageResourceLoader | None = None, @@ -1460,7 +1493,7 @@ def _add_table_from_odf( adjusted_row = row_idx - min_row adjusted_col = col_idx - min_col text = _odf_cell_text(cell) - cell_kwargs = { + cell_kwargs: _OdfCellData = { "text": text, "row_span": row_span, "col_span": col_span, @@ -1493,6 +1526,7 @@ def _add_table_from_odf( doc.add_table_cell(table_item=table_item, cell=table_cell) + assert isinstance(table_item, TableItem) return table_item @@ -1741,8 +1775,8 @@ def _is_slide_title_element(element: Any, is_first_text_content: bool) -> bool: attrs = getattr(element, "attributes", {}) if attrs.get("presentation:class") == "title": return True - return is_first_text_content and getattr(element, "tag", None) == ( - "draw:custom-shape" + return is_first_text_content and bool( + getattr(element, "tag", None) == "draw:custom-shape" ) def _walk_slide_frame( @@ -1762,6 +1796,8 @@ def _walk_slide_frame( ) for tbl in frame.get_elements("descendant::table:table"): + if not isinstance(tbl, OdfTable): + continue _add_table_from_odf( doc, tbl, @@ -1843,10 +1879,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: OdsBackendOptions | None = None, + options: BaseBackendOptions | None = None, ) -> None: if options is None: options = OdsBackendOptions() + if not isinstance(options, OdsBackendOptions): + raise TypeError("Expected OdsBackendOptions.") super().__init__(in_doc, path_or_stream, options) @classmethod @@ -2004,9 +2042,7 @@ def _convert_sheet_table( odf_obj=self.odf_obj, ) - def _find_data_tables_in_sheet( - self, table: OdfTable - ) -> list[dict[str, tuple[int, int, int, int] | TableData]]: + def _find_data_tables_in_sheet(self, table: OdfTable) -> list[_OdsTableRegion]: """Find all disconnected data tables in an ODS sheet using flood-fill. Returns a list of dictionaries, each containing: @@ -2032,7 +2068,7 @@ def _find_data_tables_in_sheet( return [] GAP_TOLERANCE = cast(OdsBackendOptions, self.options).gap_tolerance - tables: list[dict[str, tuple[int, int, int, int] | TableData]] = [] + tables: list[_OdsTableRegion] = [] visited: set[tuple[int, int]] = set() # Build a map of cell contents for quick lookup @@ -2126,15 +2162,15 @@ def _find_images_in_sheet( # Get all images in the table images = table.get_images() for img in images: + # Rich cells own their images; this pass handles sheet drawings. + if img.get_elements("ancestor::table:table-cell"): + continue try: # Get the image data - image_data = img.get_data() - if image_data: - # Convert to PIL Image - from io import BytesIO - - pil_image = PILImage.open(BytesIO(image_data)) - + image_ref = _image_ref_from_odf_image( + self.odf_obj, img, image_loader=self._image_loader + ) + if image_ref is not None: # Try to get position information # ODF images are typically anchored to cells # For now, use a default position @@ -2142,7 +2178,7 @@ def _find_images_in_sheet( doc.add_picture( parent=parent, - image=ImageRef.from_pil(image=pil_image, dpi=72), + image=image_ref, caption=None, prov=ProvenanceItem( page_no=page_no, @@ -2170,8 +2206,6 @@ def _find_page_size(doc: DoclingDocument, page_no: int) -> tuple[float, float]: if not isinstance(item, DocItem): continue for provenance in item.prov: - if provenance.bbox is None: - continue bbox = provenance.bbox left = min(left, bbox.l) if left != -1 else bbox.l right = max(right, bbox.r) if right != -1 else bbox.r diff --git a/docling/backend/pypdfium2_backend.py b/docling/backend/pypdfium2_backend.py index cb9f1fd806..ff3431e524 100644 --- a/docling/backend/pypdfium2_backend.py +++ b/docling/backend/pypdfium2_backend.py @@ -454,7 +454,7 @@ def _get_object_bucket(self, obj_type: int) -> List[tuple[BoundingBox, bool]]: Returns: The cached ``(bbox, invisible_text)`` list for ``obj_type``. """ - bucket = self._object_index.get(obj_type) + bucket: list[tuple[BoundingBox, bool]] | None = self._object_index.get(obj_type) if bucket is not None: return bucket @@ -484,6 +484,7 @@ def _get_object_bucket(self, obj_type: int) -> List[tuple[BoundingBox, bool]]: bbox = BoundingBox.from_tuple( pos, origin=CoordOrigin.BOTTOMLEFT ).to_top_left_origin(page_height=page_size.height) + assert isinstance(bbox, BoundingBox) bucket.append((bbox, invisible)) self._object_index[obj_type] = bucket @@ -592,6 +593,7 @@ def get_text_in_rect(self, bbox: BoundingBox) -> str: with pypdfium2_lock: text_piece = self.text_page.get_text_bounded(*pdf_rect) + assert isinstance(text_piece, str) return text_piece @override @@ -613,7 +615,6 @@ def get_segmented_page(self) -> Optional[SegmentedPdfPage]: textline_cells=text_cells, char_cells=[], word_cells=[], - has_textlines=len(text_cells) > 0, has_words=False, has_chars=False, ) @@ -658,6 +659,7 @@ def get_page_image( size=(round(cropbox.width * scale), round(cropbox.height * scale)) ) + assert isinstance(image, Image.Image) return image @override @@ -705,11 +707,15 @@ def __init__( @override def page_count(self) -> int: with pypdfium2_lock: + if self._pdoc is None: + raise RuntimeError("PDF document is unloaded.") return len(self._pdoc) @override def load_page(self, page_no: int) -> PyPdfiumPageBackend: with pypdfium2_lock: + if self._pdoc is None: + raise RuntimeError("PDF document is unloaded.") return PyPdfiumPageBackend(self._pdoc, self.document_hash, page_no) @override diff --git a/docling/backend/utils/image_resource_loader.py b/docling/backend/utils/image_resource_loader.py index 5d498761a9..6d701500cd 100644 --- a/docling/backend/utils/image_resource_loader.py +++ b/docling/backend/utils/image_resource_loader.py @@ -27,6 +27,7 @@ import certifi import requests import urllib3 +import urllib3.exceptions from docling_core.types.doc.document import ImageRef from PIL import Image, UnidentifiedImageError from pydantic import ValidationError @@ -206,6 +207,7 @@ def _open_direct(url: str, headers: dict[str, str]) -> Iterator[urllib3.HTTPResp ) as e: errors.append(e) continue + assert isinstance(response, urllib3.HTTPResponse) with response: yield response return @@ -224,7 +226,9 @@ def _open_with_proxy( timeout=(5, 30), allow_redirects=False, ) as response: - yield response.raw + raw = response.raw + assert isinstance(raw, urllib3.HTTPResponse) + yield raw def url_origin(url: str) -> Optional[tuple[str, str, Optional[int]]]: diff --git a/docling/backend/webvtt_backend.py b/docling/backend/webvtt_backend.py index 2a0aa462f3..05a0d5bf62 100644 --- a/docling/backend/webvtt_backend.py +++ b/docling/backend/webvtt_backend.py @@ -27,6 +27,7 @@ from typing_extensions import override from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -40,7 +41,7 @@ class AnnotatedText: voice: str | None = None formatting: Formatting | None = None - def copy_meta(self, text): + def copy_meta(self, text: str) -> "AnnotatedText": return AnnotatedText( text=text, voice=self.voice, @@ -64,8 +65,13 @@ class WebVTTDocumentBackend(DeclarativeDocumentBackend): """ @override - def __init__(self, in_doc: InputDocument, path_or_stream: BytesIO | Path): - super().__init__(in_doc, path_or_stream) + def __init__( + self, + in_doc: InputDocument, + path_or_stream: BytesIO | Path, + options: BackendOptions | None = None, + ): + super().__init__(in_doc, path_or_stream, options) self.content: str = "" # utf-8-sig drops a leading BOM. The WebVTT file body grammar allows an diff --git a/docling/backend/xml/doclang_archive_backend.py b/docling/backend/xml/doclang_archive_backend.py index 742c5255ec..8ef3075531 100644 --- a/docling/backend/xml/doclang_archive_backend.py +++ b/docling/backend/xml/doclang_archive_backend.py @@ -11,6 +11,7 @@ from typing_extensions import override from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument @@ -20,9 +21,12 @@ class DocLangArchiveBackend(DeclarativeDocumentBackend): @override def __init__( - self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path] + self, + in_doc: InputDocument, + path_or_stream: Union[BytesIO, Path], + options: BackendOptions | None = None, ) -> None: - super().__init__(in_doc, path_or_stream) + super().__init__(in_doc, path_or_stream, options) self._temp_dir: Path | None = None self._doc_or_err = self._get_doc_or_err() diff --git a/docling/backend/xml/doclang_backend.py b/docling/backend/xml/doclang_backend.py index 356c6a2254..b911ba3a52 100644 --- a/docling/backend/xml/doclang_backend.py +++ b/docling/backend/xml/doclang_backend.py @@ -10,6 +10,7 @@ from typing_extensions import override from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument @@ -17,9 +18,12 @@ class DocLangDocumentBackend(DeclarativeDocumentBackend): @override def __init__( - self, in_doc: InputDocument, path_or_stream: Union[BytesIO, Path] + self, + in_doc: InputDocument, + path_or_stream: Union[BytesIO, Path], + options: BackendOptions | None = None, ) -> None: - super().__init__(in_doc, path_or_stream) + super().__init__(in_doc, path_or_stream, options) self._doc_or_err = self._get_doc_or_err() @override diff --git a/docling/backend/xml/jats_backend.py b/docling/backend/xml/jats_backend.py index aebf0b82e2..388ab4472a 100644 --- a/docling/backend/xml/jats_backend.py +++ b/docling/backend/xml/jats_backend.py @@ -51,7 +51,7 @@ from docling.backend.html_backend import HTMLDocumentBackend from docling.backend.utils.image_resource_loader import ImageResourceLoader from docling.backend.utils.table_spans import table_width -from docling.datamodel.backend_options import JatsBackendOptions +from docling.datamodel.backend_options import BaseBackendOptions, JatsBackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -187,12 +187,14 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: JatsBackendOptions | None = None, + options: BaseBackendOptions | None = None, ) -> None: if not _BS4_AVAILABLE: raise ImportError(_INSTALL_HINT) from _BS4_IMPORT_ERROR if options is None: options = JatsBackendOptions() + if not isinstance(options, JatsBackendOptions): + raise TypeError("Expected JatsBackendOptions.") super().__init__(in_doc, path_or_stream, options) self.options: JatsBackendOptions self.path_or_stream = path_or_stream @@ -266,14 +268,14 @@ def supported_formats(cls) -> set[InputFormat]: @override def convert(self) -> DoclingDocument: + # Create the document before the recoverable parse block. + origin = DocumentOrigin( + filename=self.file.name or "file", + mimetype="application/xml", + binary_hash=self.document_hash, + ) + doc = DoclingDocument(name=self.file.stem or "file", origin=origin) try: - # Create empty document - origin = DocumentOrigin( - filename=self.file.name or "file", - mimetype="application/xml", - binary_hash=self.document_hash, - ) - doc = DoclingDocument(name=self.file.stem or "file", origin=origin) self.hlevel = 0 # Get metadata XML components @@ -300,7 +302,7 @@ def convert(self) -> DoclingDocument: def _get_text(node: etree._Element, sep: str | None = None) -> str: skip_tags = ["term", "disp-formula", "inline-formula"] text: str = ( - node.text.replace("\n", " ") + str(node.text).replace("\n", " ") if (node.tag not in skip_tags and node.text) else "" ) @@ -310,7 +312,7 @@ def _get_text(node: etree._Element, sep: str | None = None) -> str: text += JatsDocumentBackend._get_text(child, sep) if sep: text = text.rstrip(sep) + sep - text += child.tail.replace("\n", " ") if child.tail else "" + text += str(child.tail).replace("\n", " ") if child.tail else "" return text @@ -552,11 +554,12 @@ def _add_abstract( for section in sections: section_title = section["title"] if section_title: - section_parent: NodeItem = doc.add_heading( + section_parent = doc.add_heading( parent=abstract_heading, text=section_title, level=self.hlevel + 2, ) + assert isinstance(section_parent, NodeItem) else: section_parent = abstract_heading for paragraph in section["paragraphs"]: @@ -578,9 +581,9 @@ def _add_abstract( def _add_authors(self, doc: DoclingDocument, xml_components: XMLComponents) -> None: # TODO: once docling supports text formatting, add affiliation reference to # author names through superscripts - authors: list = [item["name"] for item in xml_components["authors"]] + authors: list[str] = [item["name"] for item in xml_components["authors"]] authors_str = ", ".join(authors) - affiliations: list = [ + affiliations: list[str] = [ item for author in xml_components["authors"] for item in author["affiliation_names"] @@ -659,19 +662,19 @@ def _parse_element_citation(self, node: etree._Element) -> str: ) # Journal, year, publisher name, publisher location, volume, elocation - fields: list[str] = [ - "source", - "year", - "publisher-name", - "publisher-loc", - "volume", - ] - for item in fields: - item_node = node.xpath(item) - if len(item_node) > 0: - citation[item.replace("-", "_")] = ( # type: ignore[literal-required] - JatsDocumentBackend._normalize_whitespace(item_node[0].text) - ) + def field_text(tag: str) -> str: + nodes = node.xpath(tag) + return ( + JatsDocumentBackend._normalize_whitespace(nodes[0].text) + if nodes + else "" + ) + + citation["source"] = field_text("source") + citation["year"] = field_text("year") + citation["publisher_name"] = field_text("publisher-name") + citation["publisher_loc"] = field_text("publisher-loc") + citation["volume"] = field_text("volume") # Publication identifier if len(node.xpath("pub-id")) > 0: @@ -751,7 +754,7 @@ def _add_equation( def _extract_tex_math(node: etree._Element) -> str | None: if not node.text: return None - text = node.text.strip() + text = str(node.text).strip() for delimiter in ("$$", "$"): if ( len(text) > 2 * len(delimiter) @@ -796,7 +799,7 @@ def _walk_inline_formula( current = JatsDocumentBackend._merge_formatting(formatting, node.tag) segments: list[InlineSegment] = [] if node.text: - text = node.text.replace("\n", " ") + text = str(node.text).replace("\n", " ") if text: segments.append( InlineSegment( @@ -826,7 +829,7 @@ def _walk_inline_formula( JatsDocumentBackend._walk_inline_formula(child, current, hyperlink) ) if child.tail: - tail = child.tail.replace("\n", " ") + tail = str(child.tail).replace("\n", " ") if tail: segments.append( InlineSegment( @@ -922,12 +925,14 @@ def _add_figure_captions( # TODO: format label vs caption once styling is supported fig_text: str = f"{label}{' ' if label and caption else ''}{caption}" - fig_caption: TextItem | None = ( + fig_caption = ( doc.add_text(label=DocItemLabel.CAPTION, text=fig_text) if fig_text else None ) + assert fig_caption is None or isinstance(fig_caption, TextItem) + doc.add_picture( parent=parent, caption=fig_caption, @@ -1034,12 +1039,8 @@ def parse_table_data(element: Tag) -> TableData | None: for row in element("tr"): cell_spans: list[tuple[int, int]] = [] is_row_header = True - if not isinstance(row, Tag): - continue for cell in row(["td", "th"]): - if not isinstance(row, Tag): - continue - cell_tag = cast(Tag, cell) + cell_tag = cell col_span, row_span = HTMLDocumentBackend._get_cell_spans(cell_tag) cell_spans.append((col_span, row_span)) if cell_tag.name == "td" or row_span == 1: @@ -1051,7 +1052,9 @@ def parse_table_data(element: Tag) -> TableData | None: _log.debug(f"The table has {num_rows} rows and {num_cols} cols.") - grid: list = [[None for _ in range(num_cols)] for _ in range(num_rows)] + grid: list[list[str | None]] = [ + [None for _ in range(num_cols)] for _ in range(num_rows) + ] data = TableData(num_rows=num_rows, num_cols=num_cols, table_cells=[]) @@ -1059,9 +1062,6 @@ def parse_table_data(element: Tag) -> TableData | None: start_row_span = 0 row_idx = -1 for row in element("tr"): - if not isinstance(row, Tag): - continue - # For each row, find all the column cells (both and ) cells = row(["td", "th"]) @@ -1069,13 +1069,12 @@ def parse_table_data(element: Tag) -> TableData | None: col_header = True row_header = True for html_cell in cells: - if isinstance(html_cell, Tag): - _, row_span = HTMLDocumentBackend._get_cell_spans(html_cell) - if html_cell.name == "td": - col_header = False - row_header = False - elif row_span == 1: - row_header = False + _, row_span = HTMLDocumentBackend._get_cell_spans(html_cell) + if html_cell.name == "td": + col_header = False + row_header = False + elif row_span == 1: + row_header = False if not row_header: row_idx += 1 start_row_span = 0 @@ -1085,9 +1084,6 @@ def parse_table_data(element: Tag) -> TableData | None: # Extract the text content of each cell col_idx = 0 for html_cell in cells: - if not isinstance(html_cell, Tag): - continue - # extract inline formulas for formula in html_cell("inline-formula"): math_parts = formula.text.split("$$") @@ -1142,11 +1138,13 @@ def _add_table( label = table_xml_component["label"] caption = table_xml_component["caption"] table_text: str = f"{label}{' ' if label and caption else ''}{caption}" - table_caption: TextItem | None = ( + table_caption = ( doc.add_text(label=DocItemLabel.CAPTION, text=table_text) if table_text else None ) + assert table_caption is None or isinstance(table_caption, TextItem) + if data is not None: doc.add_table(data=data, parent=parent, caption=table_caption) @@ -1245,7 +1243,7 @@ def _walk_linear( ) -> list[InlineSegment]: skip_tags = ["term"] flush_tags = ["ack", "sec", "list", "boxed-text", "disp-formula", "fig"] - new_parent: NodeItem = parent + new_parent = parent current = JatsDocumentBackend._merge_formatting(formatting, node.tag) current_hyperlink = hyperlink if node.tag == "ext-link": @@ -1267,6 +1265,7 @@ def _walk_linear( inline_segments = [] # add elements and decide whether to stop walking + section_has_heading = False if child.tag in ("sec", "ack"): header = child.xpath("title|label") text: str | None = None @@ -1275,14 +1274,17 @@ def _walk_linear( elif child.tag == "ack": text = DEFAULT_HEADER_ACKNOWLEDGMENTS if text: + section_has_heading = True self.hlevel += 1 new_parent = doc.add_heading( text=text, parent=parent, level=self.hlevel ) + assert isinstance(new_parent, NodeItem) elif child.tag == "list": new_parent = doc.add_group( label=GroupLabel.LIST, name="list", parent=parent ) + assert isinstance(new_parent, NodeItem) elif child.tag == "list-item": # TODO: address non-paragraph, non-list content inside list-item # (e.g. disp-formula, fig, table-wrap) @@ -1302,6 +1304,7 @@ def _walk_linear( text=text, parent=parent, ) + assert isinstance(new_parent, NodeItem) for nested in nested_lists: nested_group = doc.add_group( @@ -1334,9 +1337,11 @@ def _walk_linear( else DEFAULT_HEADER_REFERENCES ) new_parent = doc.add_heading(text=text, parent=parent) + assert isinstance(new_parent, NodeItem) new_parent = doc.add_group( parent=new_parent, label=GroupLabel.LIST, name="list" ) + assert isinstance(new_parent, NodeItem) elif child.tag == "element-citation": text = self._parse_element_citation(child) self._add_citation(doc, parent, text) @@ -1367,7 +1372,7 @@ def _walk_linear( JatsDocumentBackend._extend_segments( inline_segments, child_segments ) - if child.tag in ("sec", "ack") and text: + if section_has_heading: self.hlevel -= 1 # pick up the tail text diff --git a/docling/backend/xml/uspto_backend.py b/docling/backend/xml/uspto_backend.py index 16054ab1a0..60c8b3aabb 100644 --- a/docling/backend/xml/uspto_backend.py +++ b/docling/backend/xml/uspto_backend.py @@ -67,6 +67,7 @@ from typing_extensions import Self, TypedDict, override from docling.backend.abstract_backend import DeclarativeDocumentBackend +from docling.datamodel.backend_options import BackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError @@ -132,22 +133,27 @@ class PatentHeading(Enum): CLAIMS = "CLAIMS", 2 @override - def __new__(cls, value: str, _) -> Self: + def __new__(cls, value: object, _: object = None) -> Self: obj = object.__new__(cls) obj._value_ = value return obj @override - def __init__(self, _, level: LevelNumber) -> None: + def __init__(self, _: object = None, level: LevelNumber = 2) -> None: self.level: LevelNumber = level class PatentUsptoDocumentBackend(DeclarativeDocumentBackend): @override - def __init__(self, in_doc: InputDocument, path_or_stream: BytesIO | Path) -> None: + def __init__( + self, + in_doc: InputDocument, + path_or_stream: BytesIO | Path, + options: BackendOptions | None = None, + ) -> None: if not _BS4_AVAILABLE: raise ImportError(_INSTALL_HINT) from _BS4_IMPORT_ERROR - super().__init__(in_doc, path_or_stream) + super().__init__(in_doc, path_or_stream, options) self.patent_content: str = "" self.parser: PatentUspto | None = None @@ -336,13 +342,13 @@ class Element(Enum): MATHS = "maths", False # to avoid keeping formulas @override - def __new__(cls, value: str, _) -> Self: + def __new__(cls, value: object, _: object = None) -> Self: obj = object.__new__(cls) obj._value_ = value return obj @override - def __init__(self, _, is_text: bool) -> None: + def __init__(self, _: object = None, is_text: bool = False) -> None: self.is_text: bool = is_text @override @@ -365,13 +371,15 @@ def __init__(self) -> None: self.style_html = HtmlEntity() @override - def startElement(self, tag, attributes): + def startElement(self, name: str, attrs: AttributesImpl) -> None: """Signal the start of an element. Args: tag: The element tag. attributes: The element attributes. """ + tag = name + attributes = attrs if tag in ( self.APP_DOC_ELEMENT, self.GRANT_DOC_ELEMENT, @@ -415,12 +423,13 @@ def skippedEntity(self, name): self.text += unescaped @override - def endElement(self, tag): + def endElement(self, name: str) -> None: """Signal the end of an element. Args: tag: The element tag. """ + tag = name if tag in ( self.APP_DOC_ELEMENT, self.GRANT_DOC_ELEMENT, @@ -687,13 +696,13 @@ class Element(Enum): TABLE = ("table", False) # to keep track of table positions @override - def __new__(cls, value: str, _) -> Self: + def __new__(cls, value: object, _: object = None) -> Self: obj = object.__new__(cls) obj._value_ = value return obj @override - def __init__(self, _, is_text: bool) -> None: + def __init__(self, _: object = None, is_text: bool = False) -> None: self.is_text: bool = is_text @override @@ -716,13 +725,15 @@ def __init__(self) -> None: self.style_html = HtmlEntity() @override - def startElement(self, tag, attributes): + def startElement(self, name: str, attrs: AttributesImpl) -> None: """Signal the start of an element. Args: tag: The element tag. attributes: The element attributes. """ + tag = name + attributes = attrs if tag == self.GRANT_DOC_ELEMENT: self.doc = DoclingDocument(name="file") self.text = "" @@ -763,12 +774,13 @@ def skippedEntity(self, name): self.text += unescaped @override - def endElement(self, tag): + def endElement(self, name: str) -> None: """Signal the end of an element. Args: tag: The element tag. """ + tag = name if tag == self.GRANT_DOC_ELEMENT: self._clean_data() self._end_registered_element(tag) @@ -1019,9 +1031,7 @@ def get_last_text_item(self) -> TextItem | None: else: return None text_list: list[TextItem] = [ - item - for item in self.doc.texts - if isinstance(item, TextItem) and item.get_ref() in children + item for item in self.doc.texts if item.get_ref() in children ] if text_list: @@ -1153,21 +1163,21 @@ def parse(self, patent_content: str) -> DoclingDocument | None: value: str = "" line_num = 0 for line in patent_content.splitlines(): - cols = re.split("\\s{2,}", line, maxsplit=1) + cols = re.split(r"\s{2,}", line, maxsplit=1) if key and value and (len(cols) == 1 or (len(cols) == 2 and cols[0])): self.store_content(section, key, value) key = "" value = "" if len(cols) == 1: # section title - section = cols[0] + section = str(cols[0]) self.store_section(section) _log.debug(f"Parsing section {section}") elif len(cols) == 2: # key value if cols[0]: # key present - key = cols[0] - value = cols[1] + key = str(cols[0]) + value = str(cols[1]) elif not re.match(r"^##STR\d+##$", cols[1]): # line continues - value += " " + cols[1] + value += " " + str(cols[1]) line_num += 1 if key and value: self.store_content(section, key, value) @@ -1262,13 +1272,13 @@ class Element(Enum): MATH = "math-cwu", False @override - def __new__(cls, value: str, _) -> Self: + def __new__(cls, value: object, _: object = None) -> Self: obj = object.__new__(cls) obj._value_ = value return obj @override - def __init__(self, _, is_text: bool) -> None: + def __init__(self, _: object = None, is_text: bool = False) -> None: self.is_text: bool = is_text @override @@ -1291,13 +1301,15 @@ def __init__(self) -> None: self.style_html = HtmlEntity() @override - def startElement(self, tag, attributes): + def startElement(self, name: str, attrs: AttributesImpl) -> None: """Signal the start of an element. Args: tag: The element tag. attributes: The element attributes. """ + tag = name + attributes = attrs if tag == self.APP_DOC_ELEMENT: self.doc = DoclingDocument(name="file") self.text = "" @@ -1338,12 +1350,13 @@ def skippedEntity(self, name): self.text += unescaped @override - def endElement(self, tag): + def endElement(self, name: str) -> None: """Signal the end of an element. Args: tag: The element tag. """ + tag = name if tag == self.APP_DOC_ELEMENT: self._clean_data() self._end_registered_element(tag) @@ -1535,11 +1548,11 @@ class XmlTable: class ColInfo(TypedDict): ncols: int - colinfo: list[dict] + colinfo: list[dict[str, str | list[str] | None]] class MinColInfoType(TypedDict): - offset: list[int] - colwidth: list[int] + offset: list[float] + colwidth: list[float] class ColInfoType(MinColInfoType): cell_range: list[int] @@ -1583,9 +1596,12 @@ def _create_tg_range(self, tgs: list[ColInfo]) -> dict[int, ColInfoType]: "cell_range": [], "cell_offst": [0], } - offst = 0 + offst: float = 0 for info in tg["colinfo"]: - cw = info["colwidth"] + width = info["colwidth"] + if not isinstance(width, str): + raise ValueError("Column width must be text.") + cw = width cw = re.sub("pt", "", cw, flags=re.I) cw = re.sub("mm", "", cw, flags=re.I) try: @@ -1671,8 +1687,6 @@ def _parse_table(self, table: Tag) -> TableData: tg_secs = table("tgroup") if tg_secs: for tg_sec in tg_secs: - if not isinstance(tg_sec, Tag): - continue col_val = tg_sec.get("cols") ncols = ( int(col_val) @@ -1683,8 +1697,6 @@ def _parse_table(self, table: Tag) -> TableData: cs_secs = tg_sec("colspec") if cs_secs: for cs_sec in cs_secs: - if not isinstance(cs_sec, Tag): - continue colname = cs_sec.get("colname") colwidth = cs_sec.get("colwidth") tg_align["colinfo"].append( @@ -1710,15 +1722,11 @@ def _parse_table(self, table: Tag) -> TableData: tg_secs = table("tgroup") if tg_secs: for itg, tg_sec in enumerate(tg_secs): - if not isinstance(tg_sec, Tag): - continue tg_range = tgs_range[itg] row_secs = tg_sec(["row", "tr"]) if row_secs: for row_sec in row_secs: - if not isinstance(row_sec, Tag): - continue entry_secs = row_sec(["entry", "td"]) is_header: bool = ( row_sec.parent is not None @@ -1731,8 +1739,6 @@ def _parse_table(self, table: Tag) -> TableData: if entry_secs: wrong_nbr_cols = False for ientry, entry_sec in enumerate(entry_secs): - if not isinstance(entry_sec, Tag): - continue text = entry_sec.get_text().strip() # start-end diff --git a/docling/backend/xml/xbrl_backend.py b/docling/backend/xml/xbrl_backend.py index c2c92039ce..25b3d3db94 100644 --- a/docling/backend/xml/xbrl_backend.py +++ b/docling/backend/xml/xbrl_backend.py @@ -42,7 +42,11 @@ from docling.backend.abstract_backend import DeclarativeDocumentBackend from docling.backend.html_backend import HTMLDocumentBackend -from docling.datamodel.backend_options import HTMLBackendOptions, XBRLBackendOptions +from docling.datamodel.backend_options import ( + BaseBackendOptions, + HTMLBackendOptions, + XBRLBackendOptions, +) from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument from docling.exceptions import DocumentLoadError, OperationNotAllowed @@ -50,10 +54,10 @@ _XBRL_AVAILABLE: bool = False _XBRL_IMPORT_ERROR: ImportError | None = None try: - from arelle import Cntlr # type: ignore - from arelle.ModelDocument import Type # type: ignore - from arelle.ModelDtsObject import ModelConcept # type: ignore - from arelle.ModelXbrl import ModelXbrl # type: ignore + from arelle import Cntlr + from arelle.ModelDocument import Type + from arelle.ModelDtsObject import ModelConcept + from arelle.ModelXbrl import ModelXbrl _XBRL_AVAILABLE = True except ImportError as e: @@ -87,10 +91,12 @@ def __init__( self, in_doc: InputDocument, path_or_stream: BytesIO | Path, - options: XBRLBackendOptions | None = None, + options: BaseBackendOptions | None = None, ) -> None: if options is None: options = XBRLBackendOptions() + if not isinstance(options, XBRLBackendOptions): + raise TypeError("Expected XBRLBackendOptions.") # Check if arelle is available before proceeding if not _XBRL_AVAILABLE: raise ImportError( @@ -210,14 +216,16 @@ def _get_hierarchy_cell( concept: ModelConcept, ) -> int: """Get existing or create new cell for a concept node.""" - qname_str = str(concept.qname) + qname = concept.qname + assert qname is not None + qname_str = str(qname) if qname_str not in self._hierarchy_cell_ids: cell_id = self._kv_idx self._cells.append( GraphCell( label=GraphCellLabel.KEY, cell_id=cell_id, - text=concept.qname.localName, + text=qname.localName, orig=qname_str, ) ) @@ -274,7 +282,10 @@ def convert(self) -> DoclingDocument: if fact.qname.localName == "DocumentPeriodEndDate" and fact.value: doc_period = fact.value title = f"{doc_type} {doc_org} {doc_period}".strip() - title = title if title else self.model_xbrl.modelDocument.basename + model_document = self.model_xbrl.modelDocument + title = title or ( + model_document.basename if model_document is not None else self.file.name + ) doc.add_title(text=title) # Text blocks (as HTML) @@ -318,18 +329,21 @@ def convert(self) -> DoclingDocument: # period period_text = "" if fact.context is not None: - if fact.context.isInstantPeriod: - period_text = str(fact.context.instantDatetime.date()) - elif fact.context.isStartEndPeriod: - period_text = f"{fact.context.startDatetime.date()} - {fact.context.endDatetime.date()}" + instant = fact.context.instantDatetime + start = fact.context.startDatetime + end = fact.context.endDatetime + if fact.context.isInstantPeriod and instant is not None: + period_text = str(instant.date()) + elif ( + fact.context.isStartEndPeriod + and start is not None + and end is not None + ): + period_text = f"{start.date()} - {end.date()}" # unit unit_text = "" - if ( - fact.unit is not None - and fact.unit.measures - and fact.unit.measures[0] - ): + if fact.unit is not None and fact.unit.measures[0]: unit_text = fact.unit.measures[0][0].localName # decimals diff --git a/tests/test_backend_opendocument.py b/tests/test_backend_opendocument.py index f8512b4a5a..358a651e0d 100644 --- a/tests/test_backend_opendocument.py +++ b/tests/test_backend_opendocument.py @@ -1040,6 +1040,38 @@ def test_odt_table_cell_image_creates_rich_cell_picture(tmp_path: Path): assert pictures[0].image is not None +@pytest.mark.parametrize("in_cell", [False, True]) +def test_ods_embedded_image_is_emitted_once(tmp_path: Path, in_cell: bool): + image_path = tmp_path / "picture.png" + Image.new("RGB", (2, 3), "red").save(image_path) + path = tmp_path / "image.ods" + source = OdfDocument("spreadsheet") + source.body.clear() + table = Table("Images", width=1, height=1) + href = source.add_file(str(image_path)) + frame = Frame.image_frame(href, size=("1cm", "1cm")) + if in_cell: + cell = table.get_cell("A1") + cell.set_value("Image cell") + cell.append(frame) + table.set_cell("A1", cell) + else: + table.append(frame) + source.body.append(table) + source.save(str(path)) + + result = DocumentConverter(allowed_formats=[InputFormat.ODS]).convert(path) + pictures = result.document.pictures + assert len(pictures) == 1 + assert pictures[0].image is not None + pixels = pictures[0].image.pil_image + assert pixels is not None + assert pixels.size == (2, 3) + assert pixels.getpixel((0, 0)) == (255, 0, 0) + html = result.document.export_to_html(image_mode=ImageRefMode.EMBEDDED) + assert html.count("data:image/png;base64,") == 1 + + def _rich_cell_descendants(doc: DoclingDocument, cell: RichTableCell): group = cell.ref.resolve(doc) return [