diff --git a/docling/.agents/skills/docling/references/python-sdk.md b/docling/.agents/skills/docling/references/python-sdk.md index e44ae5fd95..254f59f383 100644 --- a/docling/.agents/skills/docling/references/python-sdk.md +++ b/docling/.agents/skills/docling/references/python-sdk.md @@ -485,6 +485,28 @@ converter = DocumentConverter( format_options={InputFormat.HTML: HTMLFormatOption(backend_options=html_options)} ) ``` +## DOCX, XLSX and EPUB size limits + +These formats are ZIP archives. Docling checks the decompressed sizes before it +parses them and fails the file with `ArchiveLimitError` (a `DocumentLoadError`) +when one is too large. Defaults: DOCX/XLSX 128 MiB per member, 512 MiB in total, +10,000 members; EPUB content documents 10 MB each, 100 MB in total. Raise them +for large trusted files with `max_file_bytes`, `max_total_bytes` and +`max_member_count` on `MsWordBackendOptions`, `MsExcelBackendOptions` or +`EpubBackendOptions`: + +```python +from docling.datamodel.backend_options import MsWordBackendOptions +from docling.datamodel.base_models import InputFormat +from docling.document_converter import DocumentConverter, WordFormatOption + +converter = DocumentConverter(format_options={ + InputFormat.DOCX: WordFormatOption( + backend_options=MsWordBackendOptions(max_file_bytes=512 * 1024 * 1024) + ), +}) +``` + ## Offline / air-gapped models Pre-download model artifacts, then point conversions at them so no network is diff --git a/docling/backend/epub_backend.py b/docling/backend/epub_backend.py index ef64c8c0fe..207da16bb1 100644 --- a/docling/backend/epub_backend.py +++ b/docling/backend/epub_backend.py @@ -19,6 +19,12 @@ from docling.backend.abstract_backend import DeclarativeDocumentBackend from docling.backend.html_backend import HTMLDocumentBackend +from docling.backend.utils.archive_limits import ( + ArchiveLimitError, + ArchiveLimits, + check_archive_limits, + read_member, +) from docling.datamodel.backend_options import EpubBackendOptions, HTMLBackendOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.document import InputDocument @@ -35,10 +41,11 @@ class EpubDocumentBackend(DeclarativeDocumentBackend): for processing the XHTML structure. Note: - This module enforces limits when extracting EPUB archives for image processing. - These limits prevent malicious EPUBs from exhausting disk space or memory - through decompression bombs. Limits can be configured via - EpubBackendOptions. + This module limits the decompressed size of the archive members it reads + (container, package document and content documents) and of the + members it extracts for image processing. These limits keep the + memory and disk use proportional to the configured values. Limits + can be configured via EpubBackendOptions. Known Limitations: Internal anchor links (e.g., footnote references) are converted but @@ -81,6 +88,9 @@ def __init__( self.valid = True _log.debug(f"Found {len(self.content_files)} content files in EPUB") + except ArchiveLimitError as e: + _log.error(f"EPUB archive exceeds the configured limits: {e}") + raise except Exception as e: _log.error(f"Failed to initialize EPUB backend: {e}") raise DocumentLoadError( @@ -94,7 +104,10 @@ def _parse_epub_structure(self): # Read container.xml to find the content.opf file try: - container_data = self.epub_zip.read("META-INF/container.xml") + max_file_bytes = self.options.max_file_bytes + container_data = read_member( + self.epub_zip, "META-INF/container.xml", max_file_bytes + ) container_root = ET.fromstring(container_data) # Find the content.opf path @@ -109,7 +122,7 @@ def _parse_epub_structure(self): raise ValueError("Could not find full-path in rootfile") # Parse content.opf to get the reading order - opf_data = self.epub_zip.read(opf_path) + opf_data = read_member(self.epub_zip, opf_path, max_file_bytes) opf_root = ET.fromstring(opf_data) # Extract metadata @@ -146,6 +159,7 @@ def _parse_epub_structure(self): self.content_files.append(full_path) _log.debug(f"Content files in reading order: {self.content_files}") + self._check_content_sizes() except DefusedXmlException as e: _log.error(f"Security issue detected while parsing EPUB XML: {e}") @@ -156,6 +170,26 @@ def _parse_epub_structure(self): _log.error(f"Error parsing EPUB structure: {e}") raise + def _check_content_sizes(self) -> None: + """Check the declared sizes of the content documents before they are read. + + Raises: + ArchiveLimitError: If a content document is larger than + ``max_file_bytes``, or all of them together are larger than + ``max_total_bytes``, or there are more than ``max_member_count``. + """ + if not self.epub_zip: + return + names = set(self.epub_zip.namelist()) + members = [ + self.epub_zip.getinfo(name) + for name in dict.fromkeys(self.content_files) + if name in names + ] + check_archive_limits( + self.epub_zip, ArchiveLimits.from_options(self.options), members + ) + def _safe_extract_epub(self, target_dir: Path) -> bool: """Safely extract EPUB contents with zip-bomb protection. @@ -404,7 +438,9 @@ def convert(self) -> DoclingDocument: _log.debug(f"Reading content file: {content_file}") # Read the XHTML content - xhtml_data = self.epub_zip.read(content_file) + xhtml_data = read_member( + self.epub_zip, content_file, self.options.max_file_bytes + ) xhtml_text = self._decode_content_file(xhtml_data) # Extract the body content from the XHTML @@ -429,6 +465,8 @@ def convert(self) -> DoclingDocument: combined_html_parts.append(body_content) + except ArchiveLimitError: + raise except Exception as e: _log.warning(f"Failed to read content file {content_file}: {e}") continue diff --git a/docling/backend/msexcel_backend.py b/docling/backend/msexcel_backend.py index 766f3355c2..37e9be5eb7 100644 --- a/docling/backend/msexcel_backend.py +++ b/docling/backend/msexcel_backend.py @@ -53,6 +53,12 @@ crop_whitespace, get_docx_to_pdf_converter, ) +from docling.backend.utils.archive_limits import ( + ArchiveLimitError, + ArchiveLimits, + check_archive_limits, + read_member, +) from docling.backend.utils.charts import ( MAX_CHART_TABLE_CELLS, build_chart_table, @@ -570,8 +576,8 @@ class MsExcelDocumentBackend(DeclarativeDocumentBackend, PaginatedDocumentBacken parsing, so threaded comments are not available for that format. """ - # Maximum uncompressed byte sizes accepted when reading members from the XLSX zip. - # These caps guard against decompression-bomb payloads in drawing XML / image files. + # Maximum decompressed byte sizes accepted when this backend reads drawing XML + # and image members from the XLSX zip itself (openpyxl reads the rest). _MAX_DRAWING_BYTES: Final[int] = 10 * 1024 * 1024 # 10 MB _MAX_IMAGE_BYTES: Final[int] = 50 * 1024 * 1024 # 50 MB @@ -599,6 +605,7 @@ def __init__( if options is None: options = MsExcelBackendOptions() super().__init__(in_doc, path_or_stream, options) + self._archive_limits = ArchiveLimits.from_options(options) self.page_range = in_doc.limits.page_range @@ -614,6 +621,7 @@ def __init__( self.workbook = None try: + self._check_archive_limits() # Suppress the openpyxl warning for WMF/EMF images being dropped: # those formats are handled separately via LibreOffice conversion. with warnings.catch_warnings(): @@ -635,6 +643,9 @@ def __init__( self.workbook = _load_workbook(str(self.path_or_stream)) self.valid = self.workbook is not None + except ArchiveLimitError: + self.valid = False + raise except Exception as e: self.valid = False @@ -642,6 +653,20 @@ def __init__( f"MsExcelDocumentBackend could not load document with hash {self.document_hash}" ) from e + def _check_archive_limits(self) -> None: + """Check the decompressed sizes the XLSX archive declares. + + openpyxl decompresses the workbook members while it loads them, so the + check runs before ``load_workbook``. + + Raises: + ArchiveLimitError: If the archive exceeds the configured limits. + """ + with ZipFile(self.path_or_stream, "r") as zip_file: + check_archive_limits(zip_file, self._archive_limits) + if isinstance(self.path_or_stream, BytesIO): + self.path_or_stream.seek(0) + def _parse_threaded_comments( self, sheet_name: str ) -> dict[str, list[tuple[str, str, datetime | None]]]: @@ -680,7 +705,11 @@ def _parse_threaded_comments( person_map: dict[str, str] = {} try: - person_xml = zip_file.read("xl/persons/person.xml") + person_xml = read_member( + zip_file, + "xl/persons/person.xml", + self._archive_limits.max_file_bytes, + ) person_tree = etree.fromstring(person_xml, parser=_SAFE_XML_PARSER) person_map = { person.get("id"): person.get("displayName") @@ -703,7 +732,9 @@ def _parse_threaded_comments( threaded_file = f"xl/threadedComments/threadedComment{sheet_num}.xml" try: - threaded_xml = zip_file.read(threaded_file) + threaded_xml = read_member( + zip_file, threaded_file, self._archive_limits.max_file_bytes + ) threaded_tree = etree.fromstring( threaded_xml, parser=_SAFE_XML_PARSER ) @@ -1584,7 +1615,12 @@ def _process_drawing_for_unsupported_images( if drawing_path not in zf.namelist(): return doc - tree = etree.fromstring(zf.read(drawing_path), parser=_SAFE_XML_PARSER) + try: + drawing_xml = read_member(zf, drawing_path, self._MAX_DRAWING_BYTES) + except ArchiveLimitError as exc: + _log.warning("Skipping EMF/WMF scan of %s: %s", drawing_path, exc) + return doc + tree = etree.fromstring(drawing_xml, parser=_SAFE_XML_PARSER) try: drawing = SpreadsheetDrawing.from_tree(tree) except TypeError: @@ -1600,7 +1636,11 @@ def _process_drawing_for_unsupported_images( if dep.Type != IMAGE_NS or dep.target not in zf.namelist(): continue - image_bytes = zf.read(dep.target) + try: + image_bytes = read_member(zf, dep.target, self._MAX_IMAGE_BYTES) + except ArchiveLimitError as exc: + _log.warning("Skipping image %s: %s", dep.target, exc) + continue # Skip images PIL can already handle — openpyxl already added those. try: diff --git a/docling/backend/msword_backend.py b/docling/backend/msword_backend.py index a8d8c429ae..ae708d4195 100644 --- a/docling/backend/msword_backend.py +++ b/docling/backend/msword_backend.py @@ -53,6 +53,11 @@ get_pil_from_dml_docx, ) from docling.backend.docx.latex.omml import oMath2Latex +from docling.backend.utils.archive_limits import ( + ArchiveLimits, + check_archive_limits, + read_member, +) from docling.backend.utils.image import normalize_image_for_png from docling.datamodel.backend_options import MsWordBackendOptions from docling.datamodel.base_models import FormatToMimeType @@ -181,11 +186,10 @@ _MAX_ROOT_RELS_SIZE: Final[int] = 64 * 1024 """Read cap (64 KiB) for ``_rels/.rels`` during Strict detection (the part is typically ~500 bytes).""" -_MAX_MEMBER_UNCOMPRESSED_SIZE: Final[int] = 512 * 1024 * 1024 -"""Per-member uncompressed size cap (512 MiB) applied during Strict-to-Transitional rewriting.""" - -_MAX_TOTAL_UNCOMPRESSED_SIZE: Final[int] = 2 * 1024 * 1024 * 1024 -"""Total uncompressed size cap (2 GiB) for all members during Strict-to-Transitional rewriting.""" +_DEFAULT_ARCHIVE_LIMITS: Final[ArchiveLimits] = ArchiveLimits.from_options( + MsWordBackendOptions() +) +"""Decompressed size limits for a .docx package, from the ``MsWordBackendOptions`` defaults.""" _STRICT_OOXML_NS_OVERRIDES: Final[dict[str, str]] = { "http://purl.oclc.org/ooxml/descriptions/base": "http://descriptions.openxmlformats.org/description/base", @@ -528,7 +532,7 @@ def _is_safe_zip_member(name: str) -> bool: return not any(part == ".." for part in normalized.split("/")) -def _has_fragment_only_rels(archive: zipfile.ZipFile) -> bool: +def _has_fragment_only_rels(archive: zipfile.ZipFile, limits: ArchiveLimits) -> bool: """Return True if any ``*.rels`` part contains a fragment-only relationship target. A fragment-only target (e.g. ``Target="#_Procédures_spéciales"``) is an @@ -540,8 +544,8 @@ def _has_fragment_only_rels(archive: zipfile.ZipFile) -> bool: for info in archive.infolist(): if not info.filename.endswith(".rels"): continue + content = read_member(archive, info, limits.max_file_bytes) try: - content = archive.read(info.filename) root = etree.fromstring(content, _SAFE_XML_PARSER) for rel in root.iter(rels_tag): target = rel.get("Target", "") @@ -575,7 +579,7 @@ def _remove_fragment_only_rels(content: bytes) -> bytes: return etree.tostring(root, xml_declaration=True, encoding="UTF-8", standalone=True) -def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: +def _sanitize_docx(archive: zipfile.ZipFile, limits: ArchiveLimits) -> BytesIO: """Rewrite a DOCX archive in memory, removing fragment-only relationship targets. Fragment-only targets (``Target="#anchor"``) are internal bookmark references @@ -583,25 +587,18 @@ def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: offending ``Relationship`` elements from every ``*.rels`` part so that ``python-docx`` can load the document without raising ``KeyError``. - The archive is validated against zip-slip and zip-bomb attacks while it is - read. Non-``.rels`` members are copied through unchanged. + The archive is validated against zip-slip and checked against the + decompressed size ``limits`` before it is read. Non-``.rels`` members are + copied through unchanged. """ + for info in archive.infolist(): + if not _is_safe_zip_member(info.filename): + raise SecurityError(f"ZIP slip attempt: {info.filename}") + check_archive_limits(archive, limits) sanitized = BytesIO() - total_uncompressed = 0 with zipfile.ZipFile(sanitized, "w", zipfile.ZIP_DEFLATED) as target: for info in archive.infolist(): - if not _is_safe_zip_member(info.filename): - raise SecurityError(f"ZIP slip attempt: {info.filename}") - if info.file_size > _MAX_MEMBER_UNCOMPRESSED_SIZE: - raise SecurityError( - f"Refusing to expand oversized OOXML part: {info.filename}" - ) - total_uncompressed += info.file_size - if total_uncompressed > _MAX_TOTAL_UNCOMPRESSED_SIZE: - raise SecurityError( - "Refusing to expand OOXML package exceeding the uncompressed size limit" - ) - content = archive.read(info.filename) + content = read_member(archive, info, limits.max_file_bytes) if info.filename.endswith(".rels"): content = _remove_fragment_only_rels(content) target.writestr(info, content) @@ -609,35 +606,28 @@ def _sanitize_docx(archive: zipfile.ZipFile) -> BytesIO: return sanitized -def _normalize_strict_ooxml(archive: zipfile.ZipFile) -> BytesIO: +def _normalize_strict_ooxml(archive: zipfile.ZipFile, limits: ArchiveLimits) -> BytesIO: """Rewrite an open Strict OOXML package to Transitional namespaces in memory. Only XML/relationship parts that actually carry a Strict namespace are decoded and rewritten; every other member (images, fonts, ...) is copied through with its original compression, avoiding a needless decode pass. Each member is decompressed exactly once. The archive is validated against - zip-slip and zip-bomb attacks while it is read. + zip-slip and checked against the decompressed size ``limits`` before it is + read. Fragment-only relationship targets are also removed in this pass (see ``_remove_fragment_only_rels``), so a combined Strict + fragment-only document is handled in a single archive traversal. """ + for info in archive.infolist(): + if not _is_safe_zip_member(info.filename): + raise SecurityError(f"ZIP slip attempt: {info.filename}") + check_archive_limits(archive, limits) normalized = BytesIO() - total_uncompressed = 0 with zipfile.ZipFile(normalized, "w", zipfile.ZIP_DEFLATED) as target: for info in archive.infolist(): - if not _is_safe_zip_member(info.filename): - raise SecurityError(f"ZIP slip attempt: {info.filename}") - if info.file_size > _MAX_MEMBER_UNCOMPRESSED_SIZE: - raise SecurityError( - f"Refusing to expand oversized OOXML part: {info.filename}" - ) - total_uncompressed += info.file_size - if total_uncompressed > _MAX_TOTAL_UNCOMPRESSED_SIZE: - raise SecurityError( - "Refusing to expand OOXML package exceeding the uncompressed size limit" - ) - content = archive.read(info.filename) + content = read_member(archive, info, limits.max_file_bytes) if info.filename.endswith((".xml", ".rels")) and ( _STRICT_OOXML_MARKER in content ): @@ -820,6 +810,7 @@ def __init__( path_or_stream, in_doc.format.value, "docx" ) super().__init__(in_doc, path_or_stream, options) + self._archive_limits = ArchiveLimits.from_options(options) self.XML_KEY = f"{_W_NS_CLARK}val" self.blip_xpath_expr = etree.XPath(".//a:blip", namespaces=_OOXML_NAMESPACES) self.vml_imagedata_xpath_expr = etree.XPath( @@ -901,7 +892,9 @@ def __init__( self._default_paragraph_style: BaseStyle | None = None self.docx_obj = self.load_msword_file( - path_or_stream=self.path_or_stream, document_hash=self.document_hash + path_or_stream=self.path_or_stream, + document_hash=self.document_hash, + limits=self._archive_limits, ) if self.docx_obj: self.valid = True @@ -978,35 +971,50 @@ def convert(self) -> DoclingDocument: @staticmethod def load_msword_file( - path_or_stream: BytesIO | Path, document_hash: str + path_or_stream: BytesIO | Path, + document_hash: str, + limits: ArchiveLimits = _DEFAULT_ARCHIVE_LIMITS, ) -> DocxDocument: + """Load a .docx package with python-docx. + + The package is checked against the decompressed size ``limits`` before + any member is read. Strict OOXML packages and packages with + fragment-only relationship targets are rewritten in memory first. + + Raises: + ArchiveLimitError: If the package exceeds the ``limits``. + SecurityError: If a rewritten package has an unsafe member name. + DocumentLoadError: If python-docx cannot load the package. + """ try: if isinstance(path_or_stream, Path): with zipfile.ZipFile(path_or_stream) as archive: + check_archive_limits(archive, limits) is_strict = _is_strict_ooxml(archive) has_fragment_rels = not is_strict and _has_fragment_only_rels( - archive + archive, limits ) if is_strict: - return Document(_normalize_strict_ooxml(archive)) + return Document(_normalize_strict_ooxml(archive, limits)) if has_fragment_rels: - return Document(_sanitize_docx(archive)) + return Document(_sanitize_docx(archive, limits)) return Document(str(path_or_stream)) elif isinstance(path_or_stream, BytesIO): with zipfile.ZipFile(path_or_stream) as archive: + check_archive_limits(archive, limits) is_strict = _is_strict_ooxml(archive) has_fragment_rels = not is_strict and _has_fragment_only_rels( - archive + archive, limits ) if is_strict: - return Document(_normalize_strict_ooxml(archive)) + return Document(_normalize_strict_ooxml(archive, limits)) if has_fragment_rels: - return Document(_sanitize_docx(archive)) + return Document(_sanitize_docx(archive, limits)) path_or_stream.seek(0) return Document(path_or_stream) else: return None - except SecurityError: + except (SecurityError, DocumentLoadError): raise except Exception as e: raise DocumentLoadError( @@ -4111,7 +4119,9 @@ def _convert_elements_via_docx( # Loaded once per backend instance; del body[:] clears cheaply on every call. if self._empty_docx_template is None: self._empty_docx_template = self.load_msword_file( - self.path_or_stream, self.document_hash + self.path_or_stream, + self.document_hash, + self._archive_limits, ) render_doc = self._empty_docx_template body = render_doc._element.body diff --git a/docling/backend/utils/archive_limits.py b/docling/backend/utils/archive_limits.py new file mode 100644 index 0000000000..b117737ddd --- /dev/null +++ b/docling/backend/utils/archive_limits.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Limits on how much data a backend decompresses from a ZIP-based input. + +Office Open XML and EPUB files are ZIP archives. A small file can hold members +that decompress to a very large size, and the libraries that parse them keep the +decompressed data, and the trees built from it, in memory. These helpers check +the sizes an archive declares before any member is read, and stop a read once +the decompressed bytes go above the limit. +""" + +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Final, Protocol +from zipfile import ZipFile, ZipInfo + +from docling.exceptions import DocumentLoadError + +_READ_CHUNK_BYTES: Final[int] = 1024 * 1024 + + +class ArchiveLimitError(DocumentLoadError): + """An archive declares, or decompresses to, more data than the limits allow.""" + + +class _ArchiveLimitOptions(Protocol): + """Backend options that carry the archive limit fields.""" + + max_file_bytes: int + max_total_bytes: int + max_member_count: int + + +@dataclass(frozen=True) +class ArchiveLimits: + """Limits on the decompressed content of a ZIP archive. + + Attributes: + max_file_bytes: Largest decompressed size of one member. + max_total_bytes: Largest decompressed size of all checked members. + max_member_count: Largest number of checked members. + """ + + max_file_bytes: int + max_total_bytes: int + max_member_count: int + + @classmethod + def from_options(cls, options: _ArchiveLimitOptions) -> "ArchiveLimits": + """Build the limits from backend options with the same field names.""" + return cls( + max_file_bytes=options.max_file_bytes, + max_total_bytes=options.max_total_bytes, + max_member_count=options.max_member_count, + ) + + +def check_archive_limits( + archive: ZipFile, + limits: ArchiveLimits, + members: Iterable[ZipInfo] | None = None, +) -> None: + """Check the declared sizes of archive members against the limits. + + No member is decompressed. ``ZipFile`` never returns more bytes for a member + than its central directory entry declares, so this check also bounds what + later reads of these members return. + + Args: + archive: The open archive. + limits: The limits to apply. + members: The members to check. All members of the archive when None. + + Raises: + ArchiveLimitError: If the members exceed one of the limits. + """ + infos = archive.infolist() if members is None else list(members) + if len(infos) > limits.max_member_count: + raise ArchiveLimitError( + f"Archive has {len(infos)} members, exceeding the max_member_count " + f"limit of {limits.max_member_count}." + ) + total_bytes = 0 + for info in infos: + if info.file_size > limits.max_file_bytes: + raise ArchiveLimitError( + f"Archive member {info.filename} decompresses to {info.file_size} " + f"bytes, exceeding the max_file_bytes limit of " + f"{limits.max_file_bytes}." + ) + total_bytes += info.file_size + if total_bytes > limits.max_total_bytes: + raise ArchiveLimitError( + f"Archive decompresses to {total_bytes} bytes, exceeding the " + f"max_total_bytes limit of {limits.max_total_bytes}." + ) + + +def read_member(archive: ZipFile, member: str | ZipInfo, max_bytes: int) -> bytes: + """Read one archive member, decompressing at most ``max_bytes`` bytes. + + The declared size is checked first, so an oversized member is rejected + without decompression. The read then counts the bytes it actually gets and + stops once they go above the limit. + + Args: + archive: The open archive. + member: The member name or entry. + max_bytes: The largest decompressed size to accept. + + Returns: + The decompressed content of the member. + + Raises: + KeyError: If the archive has no member with this name. + ArchiveLimitError: If the member decompresses to more than ``max_bytes``. + """ + info = member if isinstance(member, ZipInfo) else archive.getinfo(member) + if info.file_size > max_bytes: + raise ArchiveLimitError( + f"Archive member {info.filename} decompresses to {info.file_size} " + f"bytes, exceeding the limit of {max_bytes}." + ) + chunks: list[bytes] = [] + size = 0 + with archive.open(info) as stream: + while chunk := stream.read(min(_READ_CHUNK_BYTES, max_bytes - size + 1)): + size += len(chunk) + if size > max_bytes: + raise ArchiveLimitError( + f"Archive member {info.filename} decompresses to more than " + f"{max_bytes} bytes." + ) + chunks.append(chunk) + return b"".join(chunks) diff --git a/docling/datamodel/backend_options.py b/docling/datamodel/backend_options.py index 8ef0fddc13..41564b2dbc 100644 --- a/docling/datamodel/backend_options.py +++ b/docling/datamodel/backend_options.py @@ -287,13 +287,20 @@ class EpubBackendOptions(BaseBackendOptions): max_total_bytes: Annotated[ PositiveInt, Field( - description="Maximum cumulative size in bytes of all data extracted from the EPUB archive during processing" + description=( + "Maximum cumulative size in bytes of the content documents read " + "from the EPUB archive, and of all data extracted from it for " + "image processing" + ) ), ] = 100 * 1024 * 1024 # 100 MB max_file_bytes: Annotated[ PositiveInt, Field( - description="Maximum size in bytes for any single file extracted from the EPUB archive" + description=( + "Maximum size in bytes for any single file read or extracted " + "from the EPUB archive" + ) ), ] = 10 * 1024 * 1024 # 10 MB max_member_count: Annotated[ @@ -506,6 +513,28 @@ class MsExcelBackendOptions(BaseBackendOptions): "Set to None (default) to include all sheets." ), ) + max_total_bytes: Annotated[ + PositiveInt, + Field( + description=( + "Maximum cumulative decompressed size in bytes of all members of " + "the XLSX archive. The archive is checked before it is parsed." + ) + ), + ] = 512 * 1024 * 1024 + max_file_bytes: Annotated[ + PositiveInt, + Field( + description=( + "Maximum decompressed size in bytes of any single member of the " + "XLSX archive." + ) + ), + ] = 128 * 1024 * 1024 + max_member_count: Annotated[ + PositiveInt, + Field(description="Maximum number of members in the XLSX archive"), + ] = 10_000 class MsPowerpointBackendOptions(BaseBackendOptions): @@ -558,6 +587,28 @@ class MsWordBackendOptions(BaseBackendOptions): ) ), ] = True + max_total_bytes: Annotated[ + PositiveInt, + Field( + description=( + "Maximum cumulative decompressed size in bytes of all members of " + "the DOCX archive. The archive is checked before it is parsed." + ) + ), + ] = 512 * 1024 * 1024 + max_file_bytes: Annotated[ + PositiveInt, + Field( + description=( + "Maximum decompressed size in bytes of any single member of the " + "DOCX archive." + ) + ), + ] = 128 * 1024 * 1024 + max_member_count: Annotated[ + PositiveInt, + Field(description="Maximum number of members in the DOCX archive"), + ] = 10_000 class OdsBackendOptions(BaseBackendOptions): diff --git a/docs/usage/advanced_options.md b/docs/usage/advanced_options.md index b2ffac8f35..6c65ff703d 100644 --- a/docs/usage/advanced_options.md +++ b/docs/usage/advanced_options.md @@ -661,6 +661,47 @@ converter = DocumentConverter( ) ``` +### Limits on the decompressed size of DOCX, XLSX and EPUB files + +DOCX, XLSX and EPUB files are ZIP archives, so a small file can decompress to a +much larger size. Docling checks the decompressed sizes that the archive declares +before it parses the file, and stops each read once the decompressed bytes go +above the limit. A file that exceeds a limit fails to convert with an +`ArchiveLimitError`, which is a `DocumentLoadError`. + +| Option | `MsWordBackendOptions` / `MsExcelBackendOptions` | `EpubBackendOptions` | +| --- | --- | --- | +| `max_file_bytes` | 128 MiB per member | 10 MB per content document | +| `max_total_bytes` | 512 MiB for all members | 100 MB for all content documents | +| `max_member_count` | 10,000 members | 1,000 content documents | + +The DOCX and XLSX limits apply to every member of the package, because +python-docx and openpyxl load the package members into memory. The EPUB limits +apply to the container, the package document and the content documents in the +spine, and also to the members extracted for image processing. Raise the limits +for large trusted files: + +```python +from docling.datamodel.backend_options import MsExcelBackendOptions, MsWordBackendOptions +from docling.datamodel.base_models import InputFormat +from docling.document_converter import ( + DocumentConverter, + ExcelFormatOption, + WordFormatOption, +) + +doc_converter = DocumentConverter( + format_options={ + InputFormat.DOCX: WordFormatOption( + backend_options=MsWordBackendOptions(max_file_bytes=512 * 1024 * 1024) + ), + InputFormat.XLSX: ExcelFormatOption( + backend_options=MsExcelBackendOptions(max_total_bytes=2 * 1024**3) + ), + } +) +``` + ## Convert from binary PDF streams You can convert PDFs from a binary stream instead of from the filesystem as follows: diff --git a/tests/test_backend_archive_limits.py b/tests/test_backend_archive_limits.py new file mode 100644 index 0000000000..7a66ea570f --- /dev/null +++ b/tests/test_backend_archive_limits.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Decompressed size limits of the ZIP-based DOCX, XLSX and EPUB backends.""" + +import struct +import tracemalloc +import zipfile +from io import BytesIO +from pathlib import Path + +import pytest + +from docling.backend.epub_backend import EpubDocumentBackend +from docling.backend.msexcel_backend import MsExcelDocumentBackend +from docling.backend.msword_backend import MsWordDocumentBackend +from docling.backend.utils.archive_limits import ( + ArchiveLimitError, + ArchiveLimits, + check_archive_limits, + read_member, +) +from docling.datamodel.backend_options import ( + BaseBackendOptions, + EpubBackendOptions, + MsExcelBackendOptions, + MsWordBackendOptions, +) +from docling.datamodel.base_models import ConversionStatus, DocumentStream, InputFormat +from docling.datamodel.document import InputDocument, get_input_rejection_cause +from docling.document_converter import ( + DocumentConverter, + EpubFormatOption, + ExcelFormatOption, + WordFormatOption, +) + +_DATA = Path("./tests/data") +_MIB = 1024 * 1024 +# The inflated member: tens of MiB of zeros that compress to a few tens of KiB. +_INFLATED_BYTES = 32 * _MIB +_LIMIT_BYTES = 1 * _MIB + +_DOCX = _DATA / "docx/sources/Transitional.docx" +_STRICT_DOCX = _DATA / "docx/sources/Strict.docx" +_XLSX = _DATA / "xlsx/sources/xlsx_01.xlsx" +_EPUB = _DATA / "epub/sources/epub_purvis_poetry.epub" + +_FORMATS = { + InputFormat.DOCX: (MsWordDocumentBackend, MsWordBackendOptions, WordFormatOption), + InputFormat.XLSX: ( + MsExcelDocumentBackend, + MsExcelBackendOptions, + ExcelFormatOption, + ), + InputFormat.EPUB: (EpubDocumentBackend, EpubBackendOptions, EpubFormatOption), +} + +_CASES = [ + pytest.param(InputFormat.DOCX, _DOCX, "word/document.xml", id="docx-document"), + pytest.param( + InputFormat.DOCX, + _DOCX, + "word/_rels/document.xml.rels", + id="docx-relationships", + ), + pytest.param(InputFormat.DOCX, _STRICT_DOCX, "word/document.xml", id="docx-strict"), + pytest.param(InputFormat.XLSX, _XLSX, "xl/worksheets/sheet1.xml", id="xlsx-sheet"), + pytest.param(InputFormat.XLSX, _XLSX, "xl/media/padding.bin", id="xlsx-extra"), + pytest.param(InputFormat.EPUB, _EPUB, "epub/text/poetry.xhtml", id="epub-content"), + pytest.param( + InputFormat.EPUB, _EPUB, "META-INF/container.xml", id="epub-container" + ), +] + + +def _inflate(source: Path, member: str, target: Path) -> Path: + """Copy ``source`` and make ``member`` decompress to ``_INFLATED_BYTES`` zeros.""" + zeros = bytes(_MIB) + with ( + zipfile.ZipFile(source) as original, + zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED) as inflated, + ): + for info in original.infolist(): + if info.filename != member: + inflated.writestr(info, original.read(info)) + with inflated.open(member, "w") as stream: + for _ in range(_INFLATED_BYTES // _MIB): + stream.write(zeros) + return target + + +def _limits(input_format: InputFormat) -> BaseBackendOptions: + options_cls = _FORMATS[input_format][1] + return options_cls(max_file_bytes=_LIMIT_BYTES) + + +@pytest.mark.parametrize(("input_format", "source", "member"), _CASES) +@pytest.mark.parametrize("as_stream", [False, True], ids=["path", "stream"]) +def test_oversized_member_is_rejected_before_it_is_decompressed( + tmp_path: Path, + input_format: InputFormat, + source: Path, + member: str, + as_stream: bool, +) -> None: + inflated = _inflate(source, member, tmp_path / f"inflated{source.suffix}") + path_or_stream = BytesIO(inflated.read_bytes()) if as_stream else inflated + backend_cls = _FORMATS[input_format][0] + + tracemalloc.start() + try: + in_doc = InputDocument( + path_or_stream=path_or_stream, + format=input_format, + backend=backend_cls, + backend_options=_limits(input_format), + filename=inflated.name, + ) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert not in_doc.valid + cause = get_input_rejection_cause(in_doc) + assert isinstance(cause, ArchiveLimitError) + assert member in str(cause) + assert peak < _INFLATED_BYTES // 4 + + +@pytest.mark.parametrize("input_format", list(_FORMATS)) +def test_converter_reports_the_exceeded_limit( + tmp_path: Path, input_format: InputFormat +) -> None: + source, member = { + InputFormat.DOCX: (_DOCX, "word/document.xml"), + InputFormat.XLSX: (_XLSX, "xl/worksheets/sheet1.xml"), + InputFormat.EPUB: (_EPUB, "epub/text/poetry.xhtml"), + }[input_format] + inflated = _inflate(source, member, tmp_path / f"inflated{source.suffix}") + format_option_cls = _FORMATS[input_format][2] + converter = DocumentConverter( + allowed_formats=[input_format], + format_options={ + input_format: format_option_cls(backend_options=_limits(input_format)) + }, + ) + + result = converter.convert( + DocumentStream(name=inflated.name, stream=BytesIO(inflated.read_bytes())), + raises_on_error=False, + ) + + assert result.status == ConversionStatus.FAILURE + assert any("max_file_bytes" in error.error_message for error in result.errors) + + +def test_ooxml_total_and_member_count_limits_apply_to_the_whole_package() -> None: + with zipfile.ZipFile(_DOCX) as archive: + infos = archive.infolist() + total = sum(info.file_size for info in infos) + largest = max(info.file_size for info in infos) + + with pytest.raises(ArchiveLimitError, match="max_total_bytes"): + MsWordDocumentBackend.load_msword_file( + _DOCX, "hash", ArchiveLimits(largest, total - 1, len(infos)) + ) + with pytest.raises(ArchiveLimitError, match="max_member_count"): + MsWordDocumentBackend.load_msword_file( + _DOCX, "hash", ArchiveLimits(largest, total, len(infos) - 1) + ) + assert ( + MsWordDocumentBackend.load_msword_file( + _DOCX, "hash", ArchiveLimits(largest, total, len(infos)) + ) + is not None + ) + + +def _understate_size(archive_bytes: bytes, member: str, declared: int) -> bytes: + """Rewrite the central directory so ``member`` declares ``declared`` bytes.""" + data = bytearray(archive_bytes) + offset = 0 + while True: + offset = data.index(b"PK\x01\x02", offset) + name_length = struct.unpack_from(" None: + """A member that declares fewer bytes than it holds cannot be read in full. + + The declared sizes are what ``check_archive_limits`` adds up, so this is the + property that makes the declared-size check bound the actual reads. + """ + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("part.xml", bytes(4 * _MIB)) + patched = _understate_size(buffer.getvalue(), "part.xml", 1024) + + with zipfile.ZipFile(BytesIO(patched)) as archive: + limits = ArchiveLimits( + max_file_bytes=4096, max_total_bytes=4096, max_member_count=1 + ) + check_archive_limits(archive, limits) + with pytest.raises(zipfile.BadZipFile): + read_member(archive, "part.xml", limits.max_file_bytes) + + +def test_read_member_rejects_a_declared_size_over_the_limit() -> None: + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("part.xml", bytes(4097)) + + with zipfile.ZipFile(buffer) as archive: + assert read_member(archive, "part.xml", 4097) == bytes(4097) + with pytest.raises(ArchiveLimitError, match=r"part\.xml"): + read_member(archive, "part.xml", 4096) diff --git a/tests/test_backend_msword.py b/tests/test_backend_msword.py index 3ae66d440d..71dd664d30 100644 --- a/tests/test_backend_msword.py +++ b/tests/test_backend_msword.py @@ -34,6 +34,7 @@ import docling.backend.msword_backend as msword_backend_module from docling.backend.docx.drawingml.utils import get_libreoffice_cmd from docling.backend.msword_backend import MsWordDocumentBackend +from docling.backend.utils.archive_limits import ArchiveLimitError, ArchiveLimits from docling.datamodel.backend_options import MsWordBackendOptions from docling.datamodel.base_models import DocumentStream, InputFormat from docling.datamodel.document import ( @@ -838,27 +839,26 @@ def test_strict_ooxml_rejects_zip_slip(evil_name): with zipfile.ZipFile(buffer) as archive: assert msword_backend_module._is_strict_ooxml(archive) is True with pytest.raises(SecurityError, match="ZIP slip"): - msword_backend_module._normalize_strict_ooxml(archive) + msword_backend_module._normalize_strict_ooxml( + archive, msword_backend_module._DEFAULT_ARCHIVE_LIMITS + ) @pytest.mark.parametrize( - ("cap_attr", "message"), + ("limits", "message"), [ - ("_MAX_TOTAL_UNCOMPRESSED_SIZE", "uncompressed size"), - ("_MAX_MEMBER_UNCOMPRESSED_SIZE", "oversized OOXML part"), + (ArchiveLimits(4096, 1024, 10), "max_total_bytes"), + (ArchiveLimits(1024, 1 << 20, 10), "max_file_bytes"), ], ) -def test_strict_ooxml_rejects_zip_bomb(monkeypatch, cap_attr, message): +def test_strict_ooxml_rejects_oversized_package(limits, message): """A Strict package exceeding the per-member or total budget is rejected.""" import zipfile - from docling.exceptions import SecurityError - - monkeypatch.setattr(msword_backend_module, cap_attr, 128) buffer = _make_strict_zip({"word/document.xml": b"A" * 4096}) with zipfile.ZipFile(buffer) as archive: - with pytest.raises(SecurityError, match=message): - msword_backend_module._normalize_strict_ooxml(archive) + with pytest.raises(ArchiveLimitError, match=message): + msword_backend_module._normalize_strict_ooxml(archive, limits) def test_load_msword_file_propagates_security_error():