fix: keep the /AcroForm JSON export out of the document behind a signature

src/parse/pdf_decoders/document.h

@@ -125,6 +125,10 @@ namespace pdflib
     std::string load_error_category;
     std::string load_error_message;
 
+    // Built on the first call to get_annotations(), get_meta_xml() or
+    // get_table_of_contents(), never during load: a conversion that reads
+    // only the pages pays nothing for the form, the metadata stream and the
+    // outline. Only ever touched under thread_safe_buffer_mutex.
     nlohmann::json json_annots;
     bool annots_loaded;
 
@@ -177,6 +181,11 @@ namespace pdflib
 
   void pdf_decoder<DOCUMENT>::ensure_annots_loaded()
   {
+    // Reads qpdf_document, which the page-buffer writer below also uses:
+    // with the threaded parser a worker may be copying a page out while the
+    // main thread asks for the annotations.
+    std::lock_guard<std::mutex> lock(thread_safe_buffer_mutex);
+
     if(annots_loaded)
       {
         return;
@@ -339,8 +348,6 @@ namespace pdflib
         LOG_S(ERROR) << "could not process buffer by qpdf: " << exc.what();
         return false;
       }
-
-    ensure_annots_loaded();
 
     timings.add_timing(pdf_timings::KEY_PROCESS_DOCUMENT_FROM_BYTESIO, timer.get_time());
 

src/parse/pdf_resources/page_font.h

@@ -1587,8 +1587,8 @@ namespace pdflib
     font_program.base_font = base_font;
     font_program.font_name = font_name;
     font_program.from_descendant_font = from_descendant_font;
-    font_program.descriptor_json = to_json(descriptor_obj, {}, 0, 2);
-    font_program.stream_dict_json = to_json(stream_obj, {}, 0, 2);
+    font_program.descriptor_json = to_json(descriptor_obj, 0, 2);
+    font_program.stream_dict_json = to_json(stream_obj, 0, 2);
     // Disabled: dumping the full stream dictionary per font floods the logs.
     // LOG_S(INFO) << __FUNCTION__
     //             << ": stream-dict-json=\n"

src/parse/qpdf/annots.h

@@ -15,41 +15,26 @@
 
 namespace pdflib
 {
-  // FIXME: add a begin time to cap the max time spent in this routine
-  nlohmann::json extract_annots_in_json(QPDFObjectHandle obj,
-                                        std::unordered_set<std::string> prev_objs={},
-                                        int level=0, int max_level=16)
+  nlohmann::json extract_annots_in_json(QPDFObjectHandle obj)
   {
     LOG_S(INFO) << __FUNCTION__;
 
-    /*
-    if(obj.isDictionary())
-      {
-        for(auto key : obj.getKeys())
-          {
-            LOG_S(INFO) << "key: " << key;
-          }
-      }
-    */
-
     nlohmann::json result;
 
-    if(level==0 and obj.isDictionary() and
-       obj.hasKey("/Annot"))
+    if(obj.isDictionary() and obj.hasKey("/Annot"))
       {
         QPDFObjectHandle annot = obj.getKey("/Annot");
         if(not annot.isNull())
           {
-            result = to_json(annot, prev_objs, level, max_level);
+            result = to_json(annot);
           }
       }
-    else if(level==0 and obj.isDictionary() and
-            obj.hasKey("/Annots"))
+    else if(obj.isDictionary() and obj.hasKey("/Annots"))
       {
         QPDFObjectHandle annots = obj.getKey("/Annots");
         if(not annots.isNull())
           {
-            result = to_json(annots, prev_objs, level, max_level);
+            result = to_json(annots);
           }
       }
 
@@ -69,7 +54,7 @@ namespace pdflib
 
 	try
 	  {
-	    result = to_json(root.getKey("/AcroForm"), {}, 0, 16);
+	    result = to_json(root.getKey("/AcroForm"));
 	  }
 	catch(const std::exception& exc)
 	  {

src/parse/qpdf/to_json.h

@@ -6,6 +6,7 @@
 #include <sstream>
 #include <iostream>
 #include <iomanip>
+#include <set>
 #include <unordered_set>
 
 #include <nlohmann/json.hpp>
@@ -80,34 +81,67 @@ namespace pdflib
     return {false, ""};
   }
 
-  // FIXME: add a begin time to cap the max time spent in this routine
-  nlohmann::json to_json(QPDFObjectHandle obj,
-			 std::unordered_set<std::string> prev_objs={},
-                         int level=0,
-			 int max_level=32)
+  // Dictionaries that are the document itself rather than annotation or
+  // form content. Followed from an annotation, each of them pulls the
+  // document into the export: a signature's /Reference[*]/Data points back
+  // at the catalog (Acrobat and iText write it for DocMDP and FieldMDP), a
+  // link destination names a page, a structure element names its /Pg. They
+  // are written out as references instead, like /Parent and /P already are.
+  bool is_document_structure(QPDFObjectHandle obj)
   {
-    nlohmann::json result;
+    if(not obj.isDictionary())
+      {
+        return false;
+      }
 
-    if(obj.isDictionary() or obj.isArray())
+    QPDFObjectHandle type = obj.getKey("/Type");
+    if(not type.isName())
       {
-        std::string this_obj = obj.unparse();
+        return false;
+      }
 
-        if(not utf8::is_valid(this_obj.begin(), this_obj.end()))
-          {
-            utf8::replace_invalid(this_obj.begin(), this_obj.end(),
-                                  std::back_inserter(this_obj));
-            LOG_S(WARNING) << "fixed already encountered object: " << this_obj;
-          }
+    const static std::unordered_set<std::string> names = {
+      "/Catalog", "/Pages", "/Page", "/StructTreeRoot", "/StructElem"};
+
+    return names.count(type.getName())==1;
+  }
 
-        if(prev_objs.count(this_obj)==1)
+  // `path` holds the ids of the indirect dictionaries and arrays that are
+  // being expanded on the current path, so that a cycle is written as a
+  // reference instead of recursing. Direct objects are inline and cannot
+  // close a cycle, so they are not tracked.
+  //
+  // Until docling-parse#399 the guard was the unparsed text of every
+  // dictionary and array, direct ones included, collected into a set that
+  // was copied into every recursive call; for a direct object that text is
+  // its whole subtree, serialised once per level of nesting. That overhead
+  // only became visible once an export escaped into the document through a
+  // signature's /Reference[0]/Data (see is_document_structure above): the
+  // structure tree is a DAG that names each page from every element on it,
+  // and each of those paths expanded the page again.
+  nlohmann::json to_json_impl(QPDFObjectHandle obj,
+                              std::set<QPDFObjGen>& path,
+                              int level,
+                              int max_level)
+  {
+    nlohmann::json result;
+
+    if(level>0 and is_document_structure(obj))
+      {
+        std::string ref = obj.isIndirect() ? " '"+obj.unparse()+"'" : "";
+        result = "[skipping "+obj.getKey("/Type").getName()+" object"+ref+"]";
+        return result;
+      }
+
+    bool on_path = false;
+    if(obj.isIndirect() and (obj.isDictionary() or obj.isArray()))
+      {
+        if(not path.insert(obj.getObjGen()).second)
           {
-            result = "[ref to previous object: '"+this_obj+"']";
+            result = "[ref to previous object: '"+obj.unparse()+"']";
             return result;
           }
-        else
-          {
-            prev_objs.insert(this_obj);
-          }
+        on_path = true;
       }
 
     if(level<max_level)
@@ -116,36 +150,25 @@ namespace pdflib
 
         if(obj.isDictionary())
           {
-            /*
-              LOG_S(INFO) << "detected dict: ";
-              for(auto key : obj.getKeys())
-              {
-              LOG_S(INFO) << " -> key: " << key;
-              }
-            */
-
             for(auto key : obj.getKeys())
               {
-                //LOG_S(INFO) << "key: " << key;
-
                 if(keys_to_be_skipped.count(key)==1)
                   {
                     result[key] = "[skipping " + key + "]";
                   }
                 else
                   {
-                    result[key] = to_json(obj.getKey(key), prev_objs, level+1);
+                    result[key] = to_json_impl(obj.getKey(key), path, level+1, max_level);
                   }
               }
           }
         else if(obj.isArray())
           {
-            //LOG_S(INFO) << "array: " << obj.getArrayNItems();
             for(int l=0; l<obj.getArrayNItems(); l++)
               {
                 QPDFObjectHandle new_obj = obj.getArrayItem(l);
 
-                auto item = to_json(new_obj, prev_objs, level+1);
+                auto item = to_json_impl(new_obj, path, level+1, max_level);
                 result.push_back(item);
               }
           }
@@ -262,9 +285,20 @@ namespace pdflib
         result = "[exceeding recursion]";
       }
 
+    if(on_path)
+      {
+        path.erase(obj.getObjGen());
+      }
+
     return result;
   }
 
+  nlohmann::json to_json(QPDFObjectHandle obj, int level=0, int max_level=32)
+  {
+    std::set<QPDFObjGen> path;
+    return to_json_impl(obj, path, level, max_level);
+  }
+
   void print_obj(QPDFObjectHandle obj, int level = 0)
   {
     if(obj.isDictionary())

tests/test_unit_signed_form_json.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""The /AcroForm export of a signed, tagged document stays inside the form.
+
+A certification signature carries /Reference[0]/Data, which Acrobat and iText
+point back at the catalog. Serialising the form used to follow that reference
+into the whole document, and from the structure tree back into every page
+once per element that named it, so loading a 90-page signed file took minutes
+and gigabytes (docling-parse#399). The fixture is the smallest document with
+that shape: one signature field, one structure element, one page with a font.
+"""
+
+import json
+from io import BytesIO
+
+from docling_parse.pdf_parser import (
+    DoclingPdfParser,
+    DoclingThreadedPdfParser,
+    ThreadedPdfParserConfig,
+)
+from tests.pdf_builder import build_pdf, content_stream
+
+
+def _signed_tagged_pdf() -> bytes:
+    return build_pdf(
+        [
+            # 1: the form, next to the structure tree it must not drag in
+            "<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> "
+            "/StructTreeRoot 7 0 R /AcroForm << /Fields [5 0 R] /SigFlags 3 >> >>",
+            # 2
+            "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+            # 3
+            "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R "
+            "/Resources << /Font << /F1 9 0 R >> >> /Annots [5 0 R] /StructParents 0 >>",
+            # 4
+            content_stream("BT /F1 12 Tf 20 100 Td (signed) Tj ET"),
+            # 5: the signature field, merged with its widget
+            "<< /FT /Sig /T (Signature1) /Type /Annot /Subtype /Widget /F 4 "
+            "/Rect [0 0 0 0] /P 3 0 R /V 6 0 R >>",
+            # 6: the signature; /Data is the catalog, as Acrobat and iText write it
+            "<< /Type /Sig /Filter /Adobe.PPKLite /SubFilter /adbe.pkcs7.detached "
+            "/ByteRange [0 0 0 0] /Contents <3030> /M (D:20261016120000Z) "
+            "/Reference [<< /Type /SigRef /TransformMethod /DocMDP "
+            "/TransformParams << /Type /TransformParams /P 2 /V /1.2 >> "
+            "/DigestMethod /SHA256 /Data 1 0 R >>] >>",
+            # 7
+            "<< /Type /StructTreeRoot /K [8 0 R] /ParentTree << /Nums [0 [8 0 R]] >> >>",
+            # 8
+            "<< /Type /StructElem /S /P /P 7 0 R /Pg 3 0 R /K [0] >>",
+            # 9
+            "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+        ]
+    )
+
+
+def _assert_form_stays_inside_the_form(form: dict) -> None:
+    (field,) = form["/Fields"]
+    assert field["/FT"] == "/Sig"
+    assert field["/T"] == "Signature1"
+    assert field["/P"] == "[skipping /P]"
+
+    signature = field["/V"]
+    assert signature["/Filter"] == "/Adobe.PPKLite"
+    assert signature["/ByteRange"] == [0, 0, 0, 0]
+
+    (reference,) = signature["/Reference"]
+    assert reference["/TransformMethod"] == "/DocMDP"
+    # /P is skipped wherever it occurs, so the permission level is not exported
+    assert reference["/TransformParams"]["/V"] == "/1.2"
+    assert reference["/Data"] == "[skipping /Catalog object '1 0 R']"
+
+    # nothing from behind the catalog: no page tree, no page resources, no
+    # structure tree
+    dumped = json.dumps(form)
+    for key in ("/Pages", "/Kids", "/Resources", "/BaseFont", "/StructTreeRoot", "/Pg"):
+        assert key not in dumped, key
+
+
+def test_signature_reference_does_not_expand_the_document():
+    pdf_doc = DoclingPdfParser(loglevel="fatal").load(
+        path_or_stream=BytesIO(_signed_tagged_pdf()), lazy=True
+    )
+    annotations = pdf_doc.get_annotations()
+    assert annotations is not None
+
+    _assert_form_stays_inside_the_form(annotations.form)
+
+    # the page behind the signature still decodes as before
+    page = pdf_doc.get_page(1)
+    assert [cell.text for cell in page.textline_cells] == ["signed"]
+    pdf_doc.unload()
+
+
+def test_threaded_parser_exports_the_same_form():
+    parser = DoclingThreadedPdfParser(
+        parser_config=ThreadedPdfParserConfig(
+            loglevel="fatal", threads=1, max_concurrent_results=2
+        )
+    )
+    doc_key = parser.load(BytesIO(_signed_tagged_pdf()), page_numbers=[1])
+
+    # ask for the form after the workers have started: it is built on request now
+    result = next(parser.iterate_results())
+    assert result.success, result.error_message
+
+    annotations = parser.get_annotations(doc_key)
+    assert annotations is not None
+    _assert_form_stays_inside_the_form(annotations.form)
+    parser.unload_all()