Every page decodes all Flate images of a shared /Resources dictionary (~150 MB per page result for LibreOffice PDFs) #424

# pip install docling-parse==7.23.0 pillow
# python repro_shared_resources.py
"""Each page holds every Flate image of a shared /Resources dictionary.

Writes two 100-page PDFs with identical content: text on every page and 8
Flate-compressed 1600x1200 RGB images drawn only on page 1. In shared.pdf all
pages point at ONE /Resources dictionary listing the 8 images (as LibreOffice
writes it); in unshared.pdf only page 1 lists them. Then it parses each file
with DoclingThreadedPdfParser, keeps every PageParseResult (as a caller that
needs the pages later does) and reports RSS.
"""

import io
import random
import resource
import zlib

from PIL import Image, ImageFilter
from docling_parse.pdf_parser import DoclingThreadedPdfParser, RenderConfig, ThreadedPdfParserConfig

W, H = 595.28, 841.89


def image_bytes(seed, size=(1600, 1200)):
    rnd = random.Random(seed)
    small = Image.new("RGB", (40, 30))
    small.putdata([tuple(rnd.randrange(256) for _ in range(3)) for _ in range(40 * 30)])
    return small.resize(size, Image.BICUBIC).filter(ImageFilter.GaussianBlur(4)).tobytes()


def write_pdf(path, shared, n_pages=100, n_images=8):
    objs = []

    def add(body):
        objs.append(body)
        return len(objs)

    font = add(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
    imgs = []
    for k in range(n_images):
        data = zlib.compress(image_bytes(k))
        imgs.append(add(b"<< /Type /XObject /Subtype /Image /Width 1600 /Height 1200 /ColorSpace /DeviceRGB "
                        b"/BitsPerComponent 8 /Filter /FlateDecode /Length %d >>\nstream\n" % len(data) + data + b"\nendstream"))
    xobj = b" ".join(b"/Im%d %d 0 R" % (k, r) for k, r in enumerate(imgs))
    with_images = add(b"<< /Font << /F1 %d 0 R >> /XObject << %s >> >>" % (font, xobj))
    text_only = with_images if shared else add(b"<< /Font << /F1 %d 0 R >> >>" % font)
    draw = b"\n".join(b"q 120 0 0 90 %d %d cm /Im%d Do Q" % (40 + 130 * (k % 4), 600 - 100 * (k // 4), k) for k in range(n_images))
    text = b"BT /F1 10 Tf 12 TL 50 800 Td " + b" ".join(b"(Line %d of some body text on this page.) Tj T*" % i for i in range(60)) + b" ET"
    pages_id = len(objs) + 1 + 2 * n_pages
    kids = []
    for i in range(n_pages):
        content = zlib.compress(text + (b"\n" + draw if i == 0 else b""))
        cid = add(b"<< /Length %d /Filter /FlateDecode >>\nstream\n" % len(content) + content + b"\nendstream")
        kids.append(add(b"<< /Type /Page /Parent %d 0 R /MediaBox [0 0 %.2f %.2f] /Resources %d 0 R /Contents %d 0 R >>"
                        % (pages_id, W, H, with_images if i == 0 else text_only, cid)))
    add(b"<< /Type /Pages /Kids [%s] /Count %d >>" % (b" ".join(b"%d 0 R" % k for k in kids), n_pages))
    root = add(b"<< /Type /Catalog /Pages %d 0 R >>" % pages_id)
    out, offsets = bytearray(b"%PDF-1.7\n"), []
    for i, body in enumerate(objs, 1):
        offsets.append(len(out))
        out += b"%d 0 obj\n" % i + body + b"\nendobj\n"
    xref = len(out)
    out += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objs) + 1) + b"".join(b"%010d 00000 n \n" % o for o in offsets)
    out += b"trailer\n<< /Size %d /Root %d 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objs) + 1, root, xref)
    open(path, "wb").write(out)


def rss_mb():
    with open("/proc/self/status") as f:
        return next(int(l.split()[1]) // 1024 for l in f if l.startswith("VmRSS:"))


if __name__ == "__main__":
    import sys
    if len(sys.argv) == 1:
        import subprocess
        write_pdf("shared.pdf", shared=True)
        write_pdf("unshared.pdf", shared=False)
        for pdf in ("shared.pdf", "unshared.pdf"):
            for render in ("render", "no-render"):
                subprocess.run([sys.executable, __file__, pdf, render], check=False)
    else:
        pdf, mode = sys.argv[1], sys.argv[2]
        cfg = ThreadedPdfParserConfig(render_config=RenderConfig() if mode == "render" else None)
        parser = DoclingThreadedPdfParser(parser_config=cfg)
        parser.load(pdf)
        before = rss_mb()
        kept = [r for r in parser.iterate_results()]
        print(f"{pdf:13s} {mode:9s} pages={len(kept)} rss before={before} MB, after keeping all results={rss_mb()} MB, "
              f"peak={resource.getrusage(resource.RUSAGE_SELF).ru_maxrss // 1024} MB")