fix: decode image XObject samples only when a page draws them

src/parse/pdf_resources/page_xobject_image.h

@@ -196,8 +196,8 @@ namespace pdflib
     std::shared_ptr<Buffer>  get_decoded_stream_data() const;
     bool                     has_soft_mask_data() const;
     std::shared_ptr<std::vector<uint8_t>> get_soft_mask_data() const;
-    int get_soft_mask_width() const { return soft_mask_width; }
-    int get_soft_mask_height() const { return soft_mask_height; }
+    int get_soft_mask_width() const { load_pixels(); return soft_mask_width; }
+    int get_soft_mask_height() const { load_pixels(); return soft_mask_height; }
 
     // Determine file extension from filters (e.g. ".jpg", ".jp2", ".jb2", ".bin")
     std::string pick_extension() const;
@@ -216,19 +216,24 @@ namespace pdflib
 
     void init_filters();
 
-    void init_stream_data();
-    void init_codec_stream_data();
-    void init_soft_mask_data();
+    // Defilters the samples and the soft mask on first use of a pixel getter.
+    void load_pixels() const;
+
+    void init_stream_data() const;
+    void init_codec_stream_data() const;
+    void init_soft_mask_data() const;
 
     // Resolves a /Separation or /DeviceN /ColorSpace array into a colour space
     // resource, keeping it only when its tint transform is usable.
     void resolve_tint_colorspace(QPDFObjectHandle qpdf_cs);
 
   private:
 
-    QPDFObjectHandle qpdf_xobject;
+    // mutable: QPDF's stream readers are non-const, and load_pixels() reads
+    // the stream from the const getters.
+    mutable QPDFObjectHandle qpdf_xobject;
 
-    QPDFObjectHandle qpdf_xobject_dict;
+    mutable QPDFObjectHandle qpdf_xobject_dict;
     nlohmann::json   json_xobject_dict;
 
     std::string xobject_key;
@@ -258,21 +263,23 @@ namespace pdflib
     std::string      intent;
     std::vector<std::string> image_filters;
 
-    // Stream data
-    std::shared_ptr<Buffer> raw_stream_data;
+    // Stream data. Loaded lazily by load_pixels(): pages that share one
+    // /Resources dictionary list every image of the document, but draw few.
+    mutable std::shared_ptr<Buffer> raw_stream_data;
 
     // The raw stream with the transport filters that precede the image codec
     // undone -- the bytes the codec was handed at encode time. Aliases
     // raw_stream_data when the codec is the first filter in the chain.
-    std::shared_ptr<Buffer> codec_stream_data;
+    mutable std::shared_ptr<Buffer> codec_stream_data;
 
-    std::shared_ptr<Buffer> decoded_stream_data;
-    std::shared_ptr<std::vector<uint8_t>> soft_mask_data;
+    mutable std::shared_ptr<Buffer> decoded_stream_data;
+    mutable std::shared_ptr<std::vector<uint8_t>> soft_mask_data;
 
-    // set() argument, kept so parse() can consult it
+    // set() argument, kept so load_pixels() can consult it
     bool extract_pixels_ = true;
-    int soft_mask_width = 0;
-    int soft_mask_height = 0;
+    mutable bool pixels_loaded_ = false;
+    mutable int soft_mask_width = 0;
+    mutable int soft_mask_height = 0;
 
     // PDF image semantics
     std::vector<double> decode_array; // length 2*ncomp when present
@@ -326,6 +333,7 @@ namespace pdflib
     xobject_key  = xobject_key_;
     qpdf_xobject = qpdf_xobject_;
     extract_pixels_ = extract_pixels;
+    pixels_loaded_ = false;
 
     parse();
 
@@ -349,15 +357,23 @@ namespace pdflib
 
     init_filters();
     init_image_properties();
+  }
 
+  void pdf_resource<PAGE_XOBJECT_IMAGE>::load_pixels() const
+  {
     // Defiltering the samples and building the alpha plane is the whole cost
-    // of an image XObject; everything above comes from the dictionary. A
+    // of an image XObject; everything parse() reads comes from the
+    // dictionary. Deferring it to the first `Do` keeps a page from holding
+    // the pixels of images it only lists in a shared /Resources (#424). A
     // caller that only wants to know where the bitmaps are skips both.
-    if(extract_pixels_)
+    if(pixels_loaded_ or not extract_pixels_)
       {
-        init_stream_data();
-        init_soft_mask_data();
+        return;
       }
+    pixels_loaded_ = true;
+
+    init_stream_data();
+    init_soft_mask_data();
   }
 
   void pdf_resource<PAGE_XOBJECT_IMAGE>::init_image_properties()
@@ -1081,7 +1097,7 @@ namespace pdflib
       }
   }
 
-  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_stream_data()
+  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_stream_data() const
   {
     LOG_S(INFO) << __FUNCTION__;
 
@@ -1116,7 +1132,7 @@ namespace pdflib
     init_codec_stream_data();
   }
 
-  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_codec_stream_data()
+  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_codec_stream_data() const
   {
     // QPDF declares a whole /Filter chain unfilterable as soon as one of its
     // filters is an image codec it will not invert, so `decoded_stream_data`
@@ -1163,7 +1179,7 @@ namespace pdflib
       }
   }
 
-  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_soft_mask_data()
+  void pdf_resource<PAGE_XOBJECT_IMAGE>::init_soft_mask_data() const
   {
     soft_mask_data.reset();
 
@@ -1581,41 +1597,49 @@ namespace pdflib
 
   bool pdf_resource<PAGE_XOBJECT_IMAGE>::has_raw_stream_data() const
   {
+    load_pixels();
     return (raw_stream_data != nullptr && raw_stream_data->getSize() > 0);
   }
 
   std::shared_ptr<Buffer> pdf_resource<PAGE_XOBJECT_IMAGE>::get_raw_stream_data() const
   {
+    load_pixels();
     return raw_stream_data;
   }
 
   bool pdf_resource<PAGE_XOBJECT_IMAGE>::has_codec_stream_data() const
   {
+    load_pixels();
     return (codec_stream_data != nullptr && codec_stream_data->getSize() > 0);
   }
 
   std::shared_ptr<Buffer> pdf_resource<PAGE_XOBJECT_IMAGE>::get_codec_stream_data() const
   {
+    load_pixels();
     return codec_stream_data;
   }
 
   bool pdf_resource<PAGE_XOBJECT_IMAGE>::has_decoded_stream_data() const
   {
+    load_pixels();
     return (decoded_stream_data != nullptr && decoded_stream_data->getSize() > 0);
   }
 
   std::shared_ptr<Buffer> pdf_resource<PAGE_XOBJECT_IMAGE>::get_decoded_stream_data() const
   {
+    load_pixels();
     return decoded_stream_data;
   }
 
   bool pdf_resource<PAGE_XOBJECT_IMAGE>::has_soft_mask_data() const
   {
+    load_pixels();
     return (soft_mask_data != nullptr and not soft_mask_data->empty());
   }
 
   std::shared_ptr<std::vector<uint8_t>> pdf_resource<PAGE_XOBJECT_IMAGE>::get_soft_mask_data() const
   {
+    load_pixels();
     return soft_mask_data;
   }
 

tests/test_unit_shared_resource_images.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+"""Images listed in a /Resources dictionary that pages share (#424).
+
+LibreOffice points every page at one /Resources dictionary that names every
+image of the document. Each page decoder defiltered all of them up front, so
+every page result held the pixels of images it never draws: a 100-page report
+with a handful of Flate photos cost gigabytes as soon as a caller kept its
+pages. Only an image a page actually paints with `Do` may cost that page its
+pixels.
+"""
+
+from __future__ import annotations
+
+import zlib
+from io import BytesIO
+from pathlib import Path
+
+import pytest
+
+from tests.pdf_builder import build_pdf, content_stream, stream_object
+
+STATUS = Path("/proc/self/status")
+
+N_PAGES = 8
+IMAGE_W, IMAGE_H = 4000, 3000
+DECODED_MB = IMAGE_W * IMAGE_H * 3 / 2**20  # ~34 MB
+
+
+def _rss_mb() -> float:
+    for line in STATUS.read_text().splitlines():
+        if line.startswith("VmRSS:"):
+            return int(line.split()[1]) / 1024
+    raise AssertionError("no VmRSS in /proc/self/status")
+
+
+def _shared_resources_pdf() -> bytes:
+    """Pages 1..N share one /Resources naming one large image; page 1 draws it."""
+    pixels = bytes([200, 40, 40]) * (IMAGE_W * IMAGE_H)
+    image = stream_object(
+        f"/Type /XObject /Subtype /Image /Width {IMAGE_W} /Height {IMAGE_H} "
+        "/ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /FlateDecode",
+        zlib.compress(pixels, 9),
+    )
+    text = "BT /F1 12 Tf 20 170 Td (shared resources) Tj ET"
+    draw = "q 160 0 0 120 20 20 cm /Im0 Do Q"
+
+    # 1 catalog, 2 pages, 3 font, 4 image, 5 resources, then page + content pairs
+    objects: list[str | bytes] = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "",
+        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+        image,
+        "<< /Font << /F1 3 0 R >> /XObject << /Im0 4 0 R >> >>",
+    ]
+    kids = []
+    for page in range(N_PAGES):
+        page_obj = len(objects) + 1
+        kids.append(f"{page_obj} 0 R")
+        objects.append(
+            f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            f"/Resources 5 0 R /Contents {page_obj + 1} 0 R >>"
+        )
+        objects.append(content_stream(text + ("\n" + draw if page == 0 else "")))
+    objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {N_PAGES} >>"
+
+    return build_pdf(objects)
+
+
+@pytest.mark.skipif(not STATUS.exists(), reason="reads VmRSS from /proc")
+def test_pages_that_do_not_draw_a_shared_image_do_not_hold_its_pixels():
+    from docling_parse.pdf_parser import (
+        DecodeConfig,
+        DoclingThreadedPdfParser,
+        ThreadedPdfParserConfig,
+    )
+
+    parser = DoclingThreadedPdfParser(
+        parser_config=ThreadedPdfParserConfig(
+            loglevel="fatal",
+            threads=1,
+            max_concurrent_results=N_PAGES,
+            render_config=None,
+        ),
+        decode_config=DecodeConfig(keep_qpdf_warnings=False),
+    )
+    parser.load(BytesIO(_shared_resources_pdf()))
+
+    before = _rss_mb()
+    try:
+        kept = list(parser.iterate_results())
+        grown = _rss_mb() - before
+
+        assert [r.success for r in kept] == [True] * N_PAGES
+        drawn = {r.page_number: len(r.get_page().bitmap_resources) for r in kept}
+        assert drawn == {1: 1, **dict.fromkeys(range(2, N_PAGES + 1), 0)}
+
+        # Page 1 holds the image once (as decoded samples); the other pages
+        # hold nothing of it. Decoding it on every page grows by N_PAGES times
+        # that.
+        assert grown < 3 * DECODED_MB, (
+            f"keeping {N_PAGES} page results grew RSS by {grown:.0f} MB; "
+            f"one decoded copy of the shared image is {DECODED_MB:.0f} MB"
+        )
+    finally:
+        parser.unload_all()