From daea6d8fae3803d514937cc79f0c0e05659af279 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 29 Oct 2026 02:41:46 +0000 Subject: [PATCH 1/2] feat(pdf): extract layout and tables with optional local OCR --- .github/workflows/ci.yml | 8 + .github/workflows/python.yml | 2 + AGENTS.md | 4 + CHANGELOG.md | 25 + Cargo.lock | 599 +++++- Cargo.toml | 5 +- README.md | 84 +- .../bench-pdf-layout-ship-1793241589.json | 522 +++++ .../bench-pdf-layout-ship-1793241589.md | 50 + crates/markkit-cli/Cargo.toml | 4 +- crates/markkit-cli/src/main.rs | 53 +- crates/markkit-cli/tests/cli_integration.rs | 51 + crates/markkit-eval/Cargo.toml | 4 +- crates/markkit-eval/data/pdf-layout.json | 1720 +++++++++++++++++ .../results/hayro-2026-10-29.json | 431 +++++ .../markkit-eval/results/ocrs-2026-10-28.json | 488 +++++ .../results/pdf-layout-2026-10-28.json | 522 +++++ crates/markkit-eval/src/lib.rs | 1 + crates/markkit-eval/src/main.rs | 5 + crates/markkit-eval/src/pdf.rs | 263 +++ crates/markkit-js/Cargo.toml | 5 +- crates/markkit-js/README.md | 50 +- crates/markkit-js/__test__/options.test.cjs | 36 + crates/markkit-js/src/lib.rs | 168 ++ crates/markkit-python/Cargo.toml | 6 +- crates/markkit-python/README.md | 46 + crates/markkit-python/src/lib.rs | 46 + crates/markkit-python/tests/test_documents.py | 43 + crates/markkit-wasm/Cargo.toml | 3 +- crates/markkit-wasm/README.md | 24 +- .../__test__/markkit-wasm.test.mjs | 33 +- crates/markkit-wasm/js/index.d.ts | 43 + crates/markkit-wasm/src/lib.rs | 30 + crates/markkit/Cargo.toml | 10 + crates/markkit/src/config.rs | 76 + crates/markkit/src/extract.rs | 17 + crates/markkit/src/ir.rs | 3 + crates/markkit/src/layout.rs | 116 ++ crates/markkit/src/lib.rs | 6 +- crates/markkit/src/limits.rs | 2 +- crates/markkit/src/pdf/analyze.rs | 1128 +++++++++++ crates/markkit/src/pdf/font.rs | 8 + crates/markkit/src/pdf/guard.rs | 401 +++- crates/markkit/src/pdf/layout.rs | 299 +++ crates/markkit/src/pdf/mod.rs | 588 +++--- crates/markkit/src/pdf/ocr.rs | 406 ++++ crates/markkit/src/pdf/ocr/rust.rs | 131 ++ crates/markkit/src/pdf/tags.rs | 109 ++ crates/markkit/tests/pdf_layout.rs | 285 +++ crates/markkit/tests/pdf_ocr.rs | 226 +++ crates/markkit/tests/pdf_structure.rs | 259 +++ docs/getting-started.md | 27 + justfile | 12 + knowledge/formats/format-support.md | 90 +- knowledge/foundations/architecture.md | 10 + knowledge/foundations/ir.md | 21 +- knowledge/log.md | 10 + knowledge/operations/eval.md | 63 +- knowledge/operations/testing.md | 9 + knowledge/runtimes/node-package.md | 10 +- knowledge/runtimes/python-package.md | 15 + knowledge/runtimes/wasm-package.md | 7 +- knowledge/security/index.md | 2 + knowledge/security/native-dependencies.md | 78 + knowledge/security/threat-model.md | 55 +- schema/ir.schema.json | 9 + scripts/compare_pdf_ocr.py | 174 ++ scripts/ocr_models.json | 23 + scripts/prepare_ocr_models.py | 59 + scripts/tests/test_compare_pdf_ocr.py | 25 + scripts/tests/test_prepare_ocr_models.py | 38 + 71 files changed, 9797 insertions(+), 384 deletions(-) create mode 100644 crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json create mode 100644 crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md create mode 100644 crates/markkit-eval/data/pdf-layout.json create mode 100644 crates/markkit-eval/results/hayro-2026-10-29.json create mode 100644 crates/markkit-eval/results/ocrs-2026-10-28.json create mode 100644 crates/markkit-eval/results/pdf-layout-2026-10-28.json create mode 100644 crates/markkit-eval/src/pdf.rs create mode 100644 crates/markkit-python/tests/test_documents.py create mode 100644 crates/markkit/src/layout.rs create mode 100644 crates/markkit/src/pdf/analyze.rs create mode 100644 crates/markkit/src/pdf/layout.rs create mode 100644 crates/markkit/src/pdf/ocr.rs create mode 100644 crates/markkit/src/pdf/ocr/rust.rs create mode 100644 crates/markkit/src/pdf/tags.rs create mode 100644 crates/markkit/tests/pdf_layout.rs create mode 100644 crates/markkit/tests/pdf_ocr.rs create mode 100644 crates/markkit/tests/pdf_structure.rs create mode 100644 knowledge/security/native-dependencies.md create mode 100644 scripts/compare_pdf_ocr.py create mode 100644 scripts/ocr_models.json create mode 100644 scripts/prepare_ocr_models.py create mode 100644 scripts/tests/test_compare_pdf_ocr.py create mode 100644 scripts/tests/test_prepare_ocr_models.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd1a3b0..5cc8e24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,14 @@ jobs: run: cargo test --workspace - name: Run tests (all features) run: cargo test --workspace --all-features + - name: Rust OCR model integration + env: + MARKKIT_OCR_MODEL_DIR: ${{ github.workspace }}/target/ocr-models + RTEN_NUM_THREADS: '1' + RAYON_NUM_THREADS: '1' + run: | + python3 scripts/prepare_ocr_models.py + cargo test --release -p markkit --features pdf-ocr-ocrs --test pdf_ocr ocrs_recognizes_scanned_text_with_source_word_geometry -- --include-ignored - name: Run tests (office formats, no HTTP client) run: | if cargo tree -p markkit -e normal | grep -E 'reqwest|hyper'; then diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f11ef5c..b53d802 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -35,3 +35,5 @@ jobs: run: | pip install target/wheels/*.whl python -c 'from importlib.metadata import metadata; import markkit; assert metadata("markkit")["Name"] == "markkit"; assert markkit.extract_html_sync("

Smoke

").strip() == "# Smoke"; print("Distribution and import OK")' + - name: Test structured documents and PDF layout + run: python -m unittest discover -s crates/markkit-python/tests -p 'test_*.py' diff --git a/AGENTS.md b/AGENTS.md index edeca4c..06a311c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | formats/format-support | Formats, bytes-in API, detection order, DOCX + spreadsheet mapping | | integrations/tool-contract | `MarkkitTool` LLM tool (`tool` feature), Everruns toolkit contract | | security/security-model | One-page threat/invariant summary | +| security/native-dependencies | Hayro/ocrs provenance, model licensing, native library and host-runtime inventory | | security/fuzzing | cargo-fuzz targets, S17 oracle, nightly workflow, crash triage | | security/threat-model | Threats T1-T9, mitigations, test map, resource limits, invariants S1-S21 | | runtimes/python-package | PyO3 `markkit` module, maturin, PyPI | @@ -81,6 +82,9 @@ just bench-criterion # Criterion micro-benches (crates/markkit/benches) just eval-public-setup # Pinned Mira + isolated Python scorer environment just eval-public-prepare # Download and verify public corpora (PR slice) just eval-public # Release WCXB + AgentDocBench runs and regression gate +just eval-pdf # Authored PDF text/layout/table expectations +just ocr-models # Verify optional OCR model assets and attribution +just eval-ocr # Tesseract/ocrs quality, geometry, timing and memory comparison just pre-pr # Pre-PR checks ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a610a..0b98d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- PDF stream guards honor escaped dictionary keys and bound compressed prefixes + before image filters, including when OCR is disabled. + - Footnote bodies cannot become unsafe reference-link destinations in Markdown readers without footnote support. Bodies start after a blank line and remain valid footnotes in readers that support them (nightly renderer fuzz regression). ### Added +- Optional Rust OCR backend (`pdf-ocr-ocrs`) using ocrs/RTen and explicit local + ONNX model paths. CLI `--ocr-backend ocrs`, Node `pdfOcrBackend`, and Python + `pdf_ocr_backend` select it; Tesseract remains the default. Pinned model setup + and repeated scan comparisons report text quality, geometry, latency and memory. + **Breaking:** exhaustive Rust `MarkkitBuilder` literals also need + `pdf_ocr_backend`, `pdf_ocr_detection_model` and `pdf_ocr_recognition_model`. + +- PDF layout, reading order and table extraction: glyph-positioned words, + column reconstruction, ruled tables with merged cells, conservative borderless + tables and wrapped cells; bold/italic, numbered lists, tagged heading roles and + `ActualText`; optional repeated header/footer removal. Layout JSON includes + source runs/words and semantic blocks/lines/cells linked to IR provenance. +- Opt-in native PDF OCR (`pdf-ocr`, CLI `--ocr`) with Rust Hayro rendering and installed Tesseract, + bounded raster size/output/deadline and configurable languages. WASM now supports + PDF text/layout/tables; Node and Python expose layout and native OCR options. + Python gains structured `extract_document_sync`. +- `just eval-pdf`: 20 authored PDF scenes across default/layout modes, including + column, table, wrapped-cell, tagging and list expectations. + **Breaking:** exhaustive Rust `MarkkitBuilder` literals must include `pdf_layout`, + `pdf_remove_repeated_headers`, `pdf_ocr`, and `pdf_ocr_language`; builder methods + and `..Default::default()` remain valid. + - Public WCXB and AgentDocBench evaluations through Mira (`just eval-public`), with pinned/verified corpora, a release-build baseline regression gate, per-type/format quality and conversion timings. PR/push runs use a fixed slice; nightly runs cover diff --git a/Cargo.lock b/Cargo.lock index 237374f..6876796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "alloca" version = "0.4.0" @@ -88,6 +103,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.92" @@ -199,18 +220,65 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -371,6 +439,15 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -511,7 +588,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf", + "phf 0.14.0", "smallvec", ] @@ -638,6 +715,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "fast-float2" version = "0.2.4" @@ -650,6 +736,27 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" + +[[package]] +name = "fearless_simd" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629c9d6b6f30b5604089e0980f4d924a013aacb1654a6cdf5d3d852224f68651" + [[package]] name = "find-msvc-tools" version = "0.1.15" @@ -662,6 +769,8 @@ version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -671,6 +780,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "font-types" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d" +dependencies = [ + "bytemuck", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -810,6 +934,32 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glifo" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1218dedd5348c1b166ec3b8b3cbca49d2f527bd43b3fa072ea5123c523ed3f" +dependencies = [ + "bytemuck", + "foldhash", + "hashbrown", + "log", + "peniko", + "png", + "skrifa 0.44.0", + "smallvec", + "vello_common", +] + +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + [[package]] name = "half" version = "2.7.1" @@ -826,6 +976,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashify" @@ -839,18 +992,64 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "hayro" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4675ff5acd7ca41c0c34ccd2a3a154f5c86d8b5ec92bef0a114bfb194969e2af" +dependencies = [ + "bytemuck", + "fearless_simd 1.2.0", + "hayro-interpret", + "image", + "kurbo", + "pic-scale", + "rustc-hash", + "vello_cpu", +] + [[package]] name = "hayro-ccitt" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2866f0d976e76497480b853d5e258dab6e4b89ecfe991124e08f15dfa8fd03ad" +[[package]] +name = "hayro-cmap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d285dc30731c8485de5fa732fbdf2b3affdf01e4da7c2135022ca6fa664bf6" +dependencies = [ + "brotli", + "hayro-postscript", +] + +[[package]] +name = "hayro-interpret" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9df074ff163a63864cec9b0bf640ccbff8abf4e44481df7369ce0ac172ffbe" +dependencies = [ + "bitflags", + "hayro-cmap", + "hayro-syntax", + "kurbo", + "moxcms", + "phf 0.13.1", + "rustc-hash", + "siphasher", + "skrifa 0.46.2", + "smallvec", + "yoke", +] + [[package]] name = "hayro-jbig2" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "000e025850f4592ddcaee85640db70704151f2bafd43b6f431aa5b6a2c6a35a9" dependencies = [ + "fearless_simd 1.2.0", "hayro-ccitt", ] @@ -859,6 +1058,15 @@ name = "hayro-jpeg2000" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "122c5f834c37a2260b9c851bb8be685cf539b2a8b94b704a931ef7d2525d5835" +dependencies = [ + "fearless_simd 1.2.0", +] + +[[package]] +name = "hayro-postscript" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "885c5ef0654933139a9b9546fc2c69e18d37f38aa2520f079092b1be18f1fcaa" [[package]] name = "hayro-syntax" @@ -866,8 +1074,11 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cdc818d8d265dcb42081d488f7599df4f73fb79e9d347ecedcd3c23809d2026" dependencies = [ + "flate2", + "hayro-ccitt", "hayro-jbig2", "hayro-jpeg2000", + "memchr", "rustc-hash", "smallvec", "zune-jpeg", @@ -879,6 +1090,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "html5ever" version = "0.40.1" @@ -1091,6 +1308,19 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -1207,6 +1437,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "libc" version = "0.2.191" @@ -1223,6 +1465,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1276,14 +1524,17 @@ dependencies = [ "criterion", "ego-tree", "futures-core", + "hayro", "hayro-syntax", "html5ever", "mail-parser", - "miniz_oxide", + "miniz_oxide 0.9.1", + "ocrs", "proptest", "pulldown-cmark", "quick-xml", "reqwest", + "rten", "schemars", "scraper", "serde", @@ -1351,6 +1602,7 @@ dependencies = [ "markkit", "pyo3", "pyo3-async-runtimes", + "serde_json", ] [[package]] @@ -1383,6 +1635,16 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -1390,6 +1652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1430,6 +1693,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -1530,6 +1803,31 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "ocrs" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e4192a7dbc7570398ef97ab2606799703b6ea11da53f637d705d44546614435" +dependencies = [ + "anyhow", + "rayon", + "rten", + "rten-imageproc", + "rten-tensor", + "thiserror", + "wasm-bindgen", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1587,20 +1885,43 @@ dependencies = [ "windows-link", ] +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros 0.13.1", + "phf_shared 0.13.1", +] + [[package]] name = "phf" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.14.0", + "phf_shared 0.14.0", "serde", ] @@ -1610,8 +1931,18 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.14.0", + "phf_shared 0.14.0", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", ] [[package]] @@ -1621,7 +1952,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" dependencies = [ "fastrand", - "phf_shared", + "phf_shared 0.14.0", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1630,13 +1974,22 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.14.0", + "phf_shared 0.14.0", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.14.0" @@ -1646,6 +1999,16 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pic-scale" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb0ff645d35accaec252de081d55984a0d6c952cb1e5dc2b9afa7c78f07288" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1686,6 +2049,28 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -1756,6 +2141,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "pyo3" version = "0.30.1" @@ -1970,6 +2361,28 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types", + "once_cell", +] + +[[package]] +name = "read-fonts" +version = "0.43.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005c8acf251756c478b0bf402885bfd88a1476020c4c7e6060edc1aa68da38ea" +dependencies = [ + "bytemuck", + "font-types", + "once_cell", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2079,6 +2492,111 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rten" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7741649977427aa13009537ac9a8fe1a503b0aa58784069ce8f117e47eb6ab" +dependencies = [ + "num_cpus", + "rayon", + "rten-base", + "rten-gemm", + "rten-onnx", + "rten-parallel", + "rten-shape-inference", + "rten-simd", + "rten-tensor", + "rten-vecmath", + "rustc-hash", + "smallvec", + "typeid", + "wasm-bindgen", +] + +[[package]] +name = "rten-base" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413dc975c7a3d0e6da0b146cd0387e3d389cceab0fc48328bfa716c7f2ae1b1e" + +[[package]] +name = "rten-gemm" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8efc07c654d4b3f8d81e2efd8b13aea7475568395289e5bb35393411aa6a451e" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "rten-simd", + "rten-tensor", +] + +[[package]] +name = "rten-imageproc" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18726eae9614e23a400560eb3a5e3fdb06526914c31aadaf767f14afde7680dd" +dependencies = [ + "rten-tensor", +] + +[[package]] +name = "rten-onnx" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96944374568990b8d1bbc5bae5957585866f9dd8256130460405730948bfd42e" + +[[package]] +name = "rten-parallel" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba9075b130f81df7be82496ea3a9bbf20fe9e75f106a22b0ece8700850f8f664" +dependencies = [ + "rayon", + "rten-base", +] + +[[package]] +name = "rten-shape-inference" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89289c26b6a5248fccf4d67627596848a03ed31888e6ce4ba6f666c689258122" +dependencies = [ + "rten-tensor", + "smallvec", +] + +[[package]] +name = "rten-simd" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0c01294eb782adca256aa130b48a04006d03763073a68f45a9c33b982ea556" + +[[package]] +name = "rten-tensor" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf72556ba231d7af50238d26dd264856a2149c6369694eb79b22644c8280c81b" +dependencies = [ + "rayon", + "rten-base", + "rten-parallel", + "smallvec", + "typeid", +] + +[[package]] +name = "rten-vecmath" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1007f05160f9ea2ce4e9c67324e5bafb27ae1e5302d0c6b1ea4db0d35fed78ee" +dependencies = [ + "rten-base", + "rten-simd", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2427,6 +2945,26 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts 0.41.0", +] + +[[package]] +name = "skrifa" +version = "0.46.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4febebda507fb889c545a37a135517769d1391941013561292897961d1887d6" +dependencies = [ + "bytemuck", + "read-fonts 0.43.3", +] + [[package]] name = "slab" version = "0.4.12" @@ -2463,7 +3001,7 @@ checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" dependencies = [ "new_debug_unreachable", "parking_lot", - "phf_shared", + "phf_shared 0.14.0", "precomputed-hash", ] @@ -2473,8 +3011,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.14.0", + "phf_shared 0.14.0", "proc-macro2", "quote", ] @@ -2762,6 +3300,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "unarray" version = "0.1.4" @@ -2832,6 +3376,35 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vello_common" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c2fbe53548bcd25c8ff42f75a93770a8611a16b37bee56d7ef7b44e36027f1" +dependencies = [ + "bytemuck", + "fearless_simd 0.7.0", + "guillotiere", + "log", + "peniko", + "png", + "smallvec", + "thiserror", +] + +[[package]] +name = "vello_cpu" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1fec424fbba6acf6cbc78bb526447eb77590d95f40419d598870fe37535212" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown", + "png", + "vello_common", +] + [[package]] name = "wait-timeout" version = "0.2.1" @@ -2948,7 +3521,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" dependencies = [ - "phf", + "phf 0.14.0", "phf_codegen", "string_cache", "string_cache_codegen", diff --git a/Cargo.toml b/Cargo.toml index d8cfd0e..8c47a25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,10 @@ calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS) zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip quick-xml = "0.42" # DOCX/ODS XML parsing cfb = "0.15" # XLS (OLE2) container pre-scan -hayro-syntax = { version = "0.8", default-features = false, features = ["std"] } # PDF object/content parsing (no image decoders) +hayro-syntax = { version = "0.8", default-features = false, features = ["std"] } # PDF object/content parsing; OCR also enables image decoding +hayro = { version = "0.8", default-features = false, features = ["embed-fonts", "embed-cmaps"] } +ocrs = { version = "0.13.1", default-features = false, features = ["onnx"] } +rten = { version = "0.26", default-features = false, features = ["onnx_format"] } miniz_oxide = "0.9" # Bounded FlateDecode for PDF streams pulldown-cmark = { version = "0.13", default-features = false } # Markdown input; output-safety test oracle mail-parser = { version = "0.11", default-features = false } # EML input (RFC 5322 / MIME) diff --git a/README.md b/README.md index 9602b1a..f92a7d6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ package for browsers and edge runtimes. - **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits - **Safe by default** – No code execution (scripts, macros, formulas), bounded decompression, depth and size limits, [threat model](knowledge/security/threat-model.md) with a test per mitigation -- **PDF** – Text layer to paragraphs, headings and bullet lists (pure Rust, no OCR; encrypted PDFs rejected) +- **PDF** – Text/layout, columns and tables; optional native OCR (encrypted PDFs rejected) | Format | Status | Feature | |---|---|---| @@ -47,9 +47,81 @@ package for browsers and edge runtimes. | Jupyter notebooks (ipynb) | Supported | always on | | Email (EML) | Supported | `email` | | Markdown | Supported | `markdown` | -| PDF (text layer) | Supported | `pdf` | +| PDF (text, layout and tables) | Supported | `pdf` | +| PDF OCR | Supported with Tesseract/language data | `pdf-ocr` | | DOC, PPT | Detected, rejected explicitly | – | +### PDF source layout + +```rust +let parsed = markkit::Markkit::builder().pdf_layout(true).build() + .parse_document_sync(pdf_bytes, markkit::DocumentHint::file_name("report.pdf"))?; +let layout = parsed.layout.as_ref().unwrap(); // pages, dimensions, positioned text spans +``` + +CLI: `markkit extract --file report.pdf --json --layout`. +Node: `extractDocumentSync(bytes, { fileName: "report.pdf" }, { pdfLayout: true })`. +WASM: `new Markkit({ pdfLayout: true }).extractDocument(bytes, { fileName: "report.pdf" })`. + +Layout v1 includes source runs and words, plus semantic blocks, lines and table +cells in reading order. Coordinates are display points from the cropped/rotated +page top-left (x right, y down). Source runs retain drawing order. Each block has +an ID, type, bounds and source span IDs; table cells include row/column positions, +rowspan/colspan and bounds. IR `Source.page` + `Source.layout_block` link blocks +when layout is requested. Font bounds approximate an em box; OCR bounds come from +the recognizer. Layout is omitted by default, and does not change Markdown. + +PDF extraction reconstructs columns, handles ruled and borderless tables, preserves +bold/italic and numbered lists, and uses tagged-PDF heading roles and `ActualText`. +Ambiguous alignments remain prose. Enable repeated margin-header/footer suppression +with `.pdf_remove_repeated_headers(true)` or CLI `--remove-pdf-headers`. + +Native OCR uses Rust Hayro rendering for pages without text. Install **Tesseract** +plus language data. Rust requires the `pdf-ocr` feature; the CLI, +Node and Python packages compile support in. WASM extracts existing text layers; +OCR helpers run in native hosts. + +```bash +markkit extract --file scanned.pdf --ocr --ocr-language eng --timeout 300 +``` + +Rust: `.pdf_ocr(true).pdf_ocr_language("eng+deu")`; Node: `{ pdfOcr: true, +pdfOcrLanguage: "eng", timeout: 300 }`. Python: `extract_document_sync(data, +file_name="report.pdf", pdf_layout=True, pdf_ocr=True, limits=Limits(timeout=300))`. +The same structured Python API returns metadata and table counts for all formats. + +For Rust-only OCR, build with the optional `pdf-ocr-ocrs` feature and supply local +ONNX models. Tesseract remains the default backend: + +```bash +python3 scripts/prepare_ocr_models.py # verifies pinned model sizes and SHA-256 +cargo build --release -p markkit-cli --features pdf-ocr-ocrs +target/release/markkit extract --file scanned.pdf --ocr --ocr-backend ocrs \ + --ocr-detection-model target/ocr-models/text-detection.onnx \ + --ocr-recognition-model target/ocr-models/text-recognition.onnx --timeout 300 +``` + +Rust: `.pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs)` plus +`.pdf_ocr_detection_model(path)` and `.pdf_ocr_recognition_model(path)`. +Extraction never downloads models. The pinned weights are CC-BY-SA 4.0, separate +from the MIT/Apache library code; the setup script writes their attribution and +license to `target/ocr-models/NOTICE.json`. The default model supports the Latin +alphabet; `ocrs` currently accepts `eng` only. Models are trusted deployment +assets. RTen inference checks deadlines between phases and batches; host quotas +provide hard CPU/memory limits. `RTEN_NUM_THREADS` and `RAYON_NUM_THREADS` configure +inference/postprocessing thread pools at process startup. Markkit's WASM binding +does not yet expose this native OCR path. + +PDF geometry/rules count against `max_nodes`; retained geometry and analysis work +are bounded by `max_pdf_decoded_size`. Table grids obey row/column/grid limits. +Compact layout JSON and Markdown are independently bounded by `max_output_size` +(excluding JSON wrapper/pretty-print overhead), including parse-only callers. +Hayro checks image/raster budgets before rendering and the deadline before/after +its non-interruptible render step. Tesseract uses private files, output caps and +a killable deadline; no shell is used. OCR currently supports JPEG, JPEG2000 and +generic-filter images without DecodeParms. Inline, JBIG2 and CCITT OCR is refused +until bounded decoding is available. Host isolation provides hard CPU/memory quotas. + ## Quick Start ### Rust Library @@ -180,7 +252,7 @@ npm install @everruns/markkit-wasm ``` Any JS runtime (browsers, Cloudflare Workers, Deno, Bun, Node); no special headers. -HTML, DOCX and spreadsheets; no URL fetching. See +HTML, PDF text/layout, DOCX and spreadsheets; no URL fetching. See [`crates/markkit-wasm/README.md`](crates/markkit-wasm/README.md). ## Configuration @@ -206,7 +278,9 @@ let markkit = Markkit::builder() | `docx` | No | DOCX conversion (zip + quick-xml) | | `pptx` | No | PPTX conversion (zip + quick-xml) | | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | -| `pdf` | No | PDF text-layer extraction (hayro-syntax, pure Rust) | +| `pdf` | No | PDF text/layout, reading order and tables (pure Rust) | +| `pdf-ocr` | No | Native OCR: Rust Hayro rendering + installed Tesseract/language data | +| `pdf-ocr-ocrs` | No | Optional Rust OCR via ocrs/RTen and caller-supplied local ONNX models | | `markdown` | No | Markdown normalization (pulldown-cmark) | | `email` | No | EML conversion (mail-parser) | | `odt` | No | ODT conversion (zip + quick-xml) | @@ -238,7 +312,7 @@ names the field that was exceeded. | `max_output_size` | 128 MiB | Markdown output | | `timeout` | 60 s | Fetch + parse + render, per call | | `max_nesting_depth` | 256 (hard max 512) | HTML flattens deeper content; deeper DOCX XML is rejected | -| `max_nodes` | 5 000 000 | HTML DOM nodes, DOCX XML nodes | +| `max_nodes` | 5 000 000 | HTML DOM nodes, DOCX XML nodes; retained PDF layout pages/runs | | `max_archive_entries` | 10 000 | ZIP entries (DOCX/XLSX/ODS) | | `max_archive_entry_size` | 64 MiB | Decompressed bytes per ZIP entry | | `max_archive_total_size` | 256 MiB | Decompressed bytes per ZIP | diff --git a/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json b/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json new file mode 100644 index 0000000..5427ef0 --- /dev/null +++ b/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json @@ -0,0 +1,522 @@ +{ + "timestamp": 1793241589, + "system": { + "moniker": "pdf-layout-ship", + "hostname": "75f200628c94", + "os": "linux", + "arch": "x86_64", + "cpus": 4, + "markkit_version": "0.1.1", + "git_commit": "7a4e62f0497d", + "profile": "release" + }, + "iterations": 20, + "warmup": 3, + "peak_rss_bytes": 36528128, + "results": [ + { + "name": "html_fixture_article", + "category": "html", + "description": "Golden fixture full_article.html", + "iterations": 20, + "input_bytes": 881, + "output_bytes": 315, + "times_ns": [ + 46813, + 45517, + 44921, + 45307, + 76701, + 54951, + 71166, + 58878, + 45694, + 46939, + 46516, + 46741, + 46165, + 46069, + 46055, + 46691, + 46043, + 45640, + 46513, + 46194 + ], + "mean_ns": 49975.7, + "median_ns": 46353.5, + "stddev_ns": 8691.929072996394, + "min_ns": 44921, + "max_ns": 76701, + "ops_per_sec": 20009.724726216944, + "throughput_mib_s": 16.81191204433167, + "allocations": 158, + "allocated_bytes": 35167, + "peak_heap_bytes": 22099, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_blog_post", + "category": "html", + "description": "Article with nav/footer/scripts, 20 sections", + "iterations": 20, + "input_bytes": 15685, + "output_bytes": 12030, + "times_ns": [ + 584145, + 560689, + 590203, + 615323, + 554728, + 572420, + 632011, + 587417, + 570463, + 555608, + 590713, + 557842, + 572445, + 638679, + 576898, + 687518, + 605808, + 645123, + 690913, + 562267 + ], + "mean_ns": 597560.65, + "median_ns": 585781.0, + "stddev_ns": 40568.211151436044, + "min_ns": 554728, + "max_ns": 690913, + "ops_per_sec": 1673.470299625653, + "throughput_mib_s": 25.03240742647969, + "allocations": 2505, + "allocated_bytes": 345159, + "peak_heap_bytes": 247097, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_docs_page", + "category": "html", + "description": "Long docs page, 400 sections, no chrome", + "iterations": 20, + "input_bytes": 306808, + "output_bytes": 240691, + "times_ns": [ + 11838625, + 11731597, + 11996133, + 11940130, + 11592701, + 12180716, + 12856740, + 12282815, + 13014072, + 11896949, + 12132685, + 11935568, + 12049239, + 12619906, + 13433711, + 16066508, + 11794425, + 12493820, + 12032603, + 12379253 + ], + "mean_ns": 12413409.8, + "median_ns": 12090962.0, + "stddev_ns": 953061.4497135847, + "min_ns": 11592701, + "max_ns": 16066508, + "ops_per_sec": 80.55804296414995, + "throughput_mib_s": 23.570873304123797, + "allocations": 49342, + "allocated_bytes": 6969184, + "peak_heap_bytes": 4961744, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_tables", + "category": "html", + "description": "50 tables x 40 rows x 6 cols", + "iterations": 20, + "input_bytes": 262766, + "output_bytes": 174139, + "times_ns": [ + 14735436, + 16687640, + 14065107, + 14533317, + 14416169, + 16439791, + 14082688, + 13832175, + 15854774, + 14867655, + 13669925, + 14364735, + 15349995, + 14755229, + 15297555, + 14154002, + 14512379, + 14644408, + 18767273, + 15121870 + ], + "mean_ns": 15007606.15, + "median_ns": 14689922.0, + "stddev_ns": 1164190.9415011902, + "min_ns": 13669925, + "max_ns": 18767273, + "ops_per_sec": 66.63287868865082, + "throughput_mib_s": 16.697745324613592, + "allocations": 33635, + "allocated_bytes": 7111395, + "peak_heap_bytes": 5315858, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_boilerplate", + "category": "html", + "description": "Mostly script/style/nav to strip, little content", + "iterations": 20, + "input_bytes": 46312, + "output_bytes": 191, + "times_ns": [ + 676915, + 663864, + 650786, + 716977, + 697255, + 666960, + 636731, + 662572, + 646993, + 656426, + 680715, + 638981, + 655176, + 664130, + 621318, + 638250, + 674455, + 608304, + 624358, + 663264 + ], + "mean_ns": 657221.5, + "median_ns": 659499.0, + "stddev_ns": 25186.64614731386, + "min_ns": 608304, + "max_ns": 716977, + "ops_per_sec": 1521.5570397499168, + "throughput_mib_s": 67.20194780816855, + "allocations": 66, + "allocated_bytes": 264364, + "peak_heap_bytes": 257958, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_deep_nesting", + "category": "html", + "description": "Divs nested past the depth limit (1000 levels)", + "iterations": 20, + "input_bytes": 11011, + "output_bytes": 5, + "times_ns": [ + 414513, + 424869, + 408592, + 415082, + 428873, + 415024, + 420195, + 404296, + 524581, + 429155, + 415840, + 434755, + 416972, + 451074, + 427698, + 433901, + 446099, + 424945, + 463847, + 393449 + ], + "mean_ns": 429688.0, + "median_ns": 424907.0, + "stddev_ns": 26923.541089537237, + "min_ns": 393449, + "max_ns": 524581, + "ops_per_sec": 2327.270019176705, + "throughput_mib_s": 24.43844812503309, + "allocations": 551, + "allocated_bytes": 93075, + "peak_heap_bytes": 53331, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_memo", + "category": "docx", + "description": "Short memo: heading, 10 paragraphs, 1 list", + "iterations": 20, + "input_bytes": 1429, + "output_bytes": 636, + "times_ns": [ + 74549, + 74061, + 67981, + 66127, + 65961, + 64760, + 94431, + 67657, + 65758, + 66838, + 64828, + 65212, + 64832, + 64341, + 64532, + 64527, + 65138, + 64512, + 64358, + 64128 + ], + "mean_ns": 67726.55, + "median_ns": 65175.0, + "stddev_ns": 6765.156963995734, + "min_ns": 64128, + "max_ns": 94431, + "ops_per_sec": 14765.258233292556, + "throughput_mib_s": 20.12210275208956, + "allocations": 487, + "allocated_bytes": 207703, + "peak_heap_bytes": 60705, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_report", + "category": "docx", + "description": "Report: 400 sections with lists and tables", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 268569, + "times_ns": [ + 16865670, + 17711098, + 16676757, + 16425247, + 17725293, + 18049829, + 17715314, + 17153265, + 18034253, + 17427472, + 18586530, + 16984482, + 16799437, + 18725013, + 17430906, + 16814605, + 18693276, + 16410731, + 16872208, + 18157874 + ], + "mean_ns": 17462963.0, + "median_ns": 17429189.0, + "stddev_ns": 721536.3248924478, + "min_ns": 16410731, + "max_ns": 18725013, + "ops_per_sec": 57.26405077992778, + "throughput_mib_s": 0.5078300554299816, + "allocations": 136855, + "allocated_bytes": 12535866, + "peak_heap_bytes": 4096773, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_small", + "category": "spreadsheet", + "description": "1 sheet, 50 rows x 5 cols", + "iterations": 20, + "input_bytes": 2865, + "output_bytes": 2723, + "times_ns": [ + 313170, + 250837, + 246779, + 246161, + 266631, + 242644, + 245446, + 243012, + 268626, + 242709, + 242335, + 241710, + 264529, + 244240, + 241025, + 262054, + 266596, + 242300, + 241587, + 240093 + ], + "mean_ns": 252624.2, + "median_ns": 244843.0, + "stddev_ns": 17000.194077127475, + "min_ns": 240093, + "max_ns": 313170, + "ops_per_sec": 3958.448953029836, + "throughput_mib_s": 10.815578699522476, + "allocations": 1165, + "allocated_bytes": 666307, + "peak_heap_bytes": 132231, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_large", + "category": "spreadsheet", + "description": "3 sheets, 5000 rows x 10 cols each (150k cells)", + "iterations": 20, + "input_bytes": 784216, + "output_bytes": 1844036, + "times_ns": [ + 131643926, + 131020753, + 135643173, + 132828999, + 140105438, + 131592771, + 128157132, + 136450013, + 132633156, + 127601147, + 127653114, + 130115468, + 129416760, + 131294725, + 128673412, + 131169409, + 132208996, + 128963111, + 131482277, + 131642690 + ], + "mean_ns": 131514823.5, + "median_ns": 131388501.0, + "stddev_ns": 3012957.9636681704, + "min_ns": 127601147, + "max_ns": 140105438, + "ops_per_sec": 7.603705600532552, + "throughput_mib_s": 5.6867099678299295, + "allocations": 512786, + "allocated_bytes": 39383548, + "peak_heap_bytes": 22510370, + "errors": 0, + "error_messages": [] + }, + { + "name": "ods_medium", + "category": "spreadsheet", + "description": "1 sheet, 2000 rows x 8 cols", + "iterations": 20, + "input_bytes": 71489, + "output_bytes": 166354, + "times_ns": [ + 19708481, + 19426236, + 19350839, + 19756707, + 19792602, + 18707106, + 18796882, + 19939690, + 20128203, + 21413218, + 19285174, + 19979224, + 19820926, + 19490273, + 19663430, + 19214942, + 19423201, + 19930702, + 21699598, + 20581212 + ], + "mean_ns": 19805432.3, + "median_ns": 19732594.0, + "stddev_ns": 721589.5909737127, + "min_ns": 18707106, + "max_ns": 21699598, + "ops_per_sec": 50.491197811420655, + "throughput_mib_s": 3.442349663105632, + "allocations": 72498, + "allocated_bytes": 9342742, + "peak_heap_bytes": 3180660, + "errors": 0, + "error_messages": [] + }, + { + "name": "detect_docx_magic", + "category": "detect", + "description": "Sniff DOCX from bytes without a hint", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 0, + "times_ns": [ + 1831, + 1812, + 1740, + 1735, + 1738, + 1732, + 1732, + 1731, + 1729, + 1739, + 1741, + 1733, + 1726, + 1732, + 1725, + 1766, + 1729, + 1728, + 1730, + 1732 + ], + "mean_ns": 1743.05, + "median_ns": 1732.0, + "stddev_ns": 27.639600214185442, + "min_ns": 1725, + "max_ns": 1831, + "ops_per_sec": 573707.0078311006, + "throughput_mib_s": 5087.75850851193, + "allocations": 0, + "allocated_bytes": 0, + "peak_heap_bytes": 0, + "errors": 0, + "error_messages": [] + } + ] +} diff --git a/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md b/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md new file mode 100644 index 0000000..5c6da26 --- /dev/null +++ b/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md @@ -0,0 +1,50 @@ +# Markkit Benchmark Report + +## System Information + +- **Moniker**: `pdf-layout-ship` +- **OS / Arch**: linux / x86_64 +- **CPUs**: 4 +- **markkit**: 0.1.1 (`7a4e62f0497d`) +- **Profile**: release +- **Timestamp**: 1793241589 +- **Iterations**: 20 (warmup 3) +- **Peak RSS**: 34.84 MiB + +## Summary + +12 cases, 198.261 ms total mean time per pass, 0 errors. + +Allocations, allocated and peak heap are per single operation. Throughput is input bytes over mean time. + +### HTML + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| html_fixture_article | 881 B | 315 B | 49.98 µs | 46.35 µs | ±8.69 µs | 20010 | 16.8 | 158 | 34.3 KiB | 21.6 KiB | - | +| html_blog_post | 15.3 KiB | 11.7 KiB | 597.56 µs | 585.78 µs | ±40.57 µs | 1673 | 25.0 | 2505 | 337.1 KiB | 241.3 KiB | - | +| html_docs_page | 299.6 KiB | 235.0 KiB | 12.41 ms | 12.09 ms | ±953.06 µs | 81 | 23.6 | 49342 | 6.65 MiB | 4.73 MiB | - | +| html_tables | 256.6 KiB | 170.1 KiB | 15.01 ms | 14.69 ms | ±1.16 ms | 67 | 16.7 | 33635 | 6.78 MiB | 5.07 MiB | - | +| html_boilerplate | 45.2 KiB | 191 B | 657.22 µs | 659.50 µs | ±25.19 µs | 1522 | 67.2 | 66 | 258.2 KiB | 251.9 KiB | - | +| html_deep_nesting | 10.8 KiB | 5 B | 429.69 µs | 424.91 µs | ±26.92 µs | 2327 | 24.4 | 551 | 90.9 KiB | 52.1 KiB | - | + +### DOCX + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| docx_memo | 1.4 KiB | 636 B | 67.73 µs | 65.17 µs | ±6.77 µs | 14765 | 20.1 | 487 | 202.8 KiB | 59.3 KiB | - | +| docx_report | 9.1 KiB | 262.3 KiB | 17.46 ms | 17.43 ms | ±721.54 µs | 57 | 0.5 | 136855 | 11.96 MiB | 3.91 MiB | - | + +### Spreadsheet + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| xlsx_small | 2.8 KiB | 2.7 KiB | 252.62 µs | 244.84 µs | ±17.00 µs | 3958 | 10.8 | 1165 | 650.7 KiB | 129.1 KiB | - | +| xlsx_large | 765.8 KiB | 1.76 MiB | 131.51 ms | 131.39 ms | ±3.01 ms | 8 | 5.7 | 512786 | 37.56 MiB | 21.47 MiB | - | +| ods_medium | 69.8 KiB | 162.5 KiB | 19.81 ms | 19.73 ms | ±721.59 µs | 50 | 3.4 | 72498 | 8.91 MiB | 3.03 MiB | - | + +### Detect + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| detect_docx_magic | 9.1 KiB | 0 B | 1.74 µs | 1.73 µs | ±28 ns | 573707 | 5087.8 | 0 | 0 B | 0 B | - | diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 9d07f64..ec5cfea 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -19,9 +19,11 @@ path = "src/main.rs" doc = false [features] -default = ["all_formats"] +default = ["all_formats", "pdf-ocr"] all_formats = ["markkit/all_formats"] pdf = ["markkit/pdf"] +pdf-ocr = ["pdf", "markkit/pdf-ocr"] +pdf-ocr-ocrs = ["pdf-ocr", "markkit/pdf-ocr-ocrs"] docx = ["markkit/docx"] pptx = ["markkit/pptx"] spreadsheet = ["markkit/spreadsheet"] diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 7f74cbc..3ec22ac 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -51,6 +51,29 @@ enum Commands { #[arg(long)] json: bool, + /// Include PDF page dimensions and positioned text runs in JSON output + #[arg(long, requires = "json")] + layout: bool, + + /// OCR textless PDF pages using the selected local backend + #[arg(long)] + ocr: bool, + /// Installed Tesseract language codes, e.g. eng+deu + #[arg(long, requires = "ocr")] + ocr_language: Option, + /// OCR implementation: tesseract (default) or ocrs (optional feature) + #[arg(long, requires = "ocr", value_parser = ["tesseract", "ocrs"])] + ocr_backend: Option, + /// Local ONNX detection model for the ocrs backend + #[arg(long, requires = "ocr")] + ocr_detection_model: Option, + /// Local ONNX recognition model for the ocrs backend + #[arg(long, requires = "ocr")] + ocr_recognition_model: Option, + /// Suppress headers/footers repeated in PDF page margins + #[arg(long)] + remove_pdf_headers: bool, + /// Resolve relative links and images in files / stdin against this URL /// (URLs use the final response URL) #[arg(long)] @@ -102,7 +125,7 @@ struct LimitArgs { /// Maximum element nesting, hard-capped at 512 [default: 256] #[arg(long, value_name = "N")] max_nesting_depth: Option, - /// Maximum parsed HTML DOM / DOCX XML nodes [default: 5000000] + /// Maximum parsed nodes / retained PDF layout pages and runs [default: 5000000] #[arg(long, value_name = "N")] max_nodes: Option, /// Maximum entries in a ZIP container [default: 10000] @@ -200,6 +223,13 @@ async fn main() -> Result<()> { limits, rules, json: output_json, + layout, + ocr, + ocr_language, + ocr_backend, + ocr_detection_model, + ocr_recognition_model, + remove_pdf_headers, base_url, main_content, } => { @@ -211,8 +241,22 @@ async fn main() -> Result<()> { let mut builder = Markkit::builder() .limits(limits.apply(Limits::default())) + .pdf_layout(layout) + .pdf_ocr(ocr) + .pdf_ocr_language(ocr_language.unwrap_or_else(|| "eng".into())) + .pdf_remove_repeated_headers(remove_pdf_headers) .main_content(main_content); + if let Some(backend) = ocr_backend { + builder = builder.pdf_ocr_backend(backend.parse().map_err(anyhow::Error::msg)?); + } + if let Some(path) = ocr_detection_model { + builder = builder.pdf_ocr_detection_model(path); + } + if let Some(path) = ocr_recognition_model { + builder = builder.pdf_ocr_recognition_model(path); + } + for sel in &include_selectors { builder = builder.include_selector(sel); } @@ -251,13 +295,18 @@ async fn main() -> Result<()> { }; if output_json { - let output = serde_json::json!({ + let mut output = serde_json::json!({ "source": source_label, "markdown": extraction.markdown, "format": extraction.format, "title": extraction.title, "metadata": extraction.metadata, + "stats": extraction.stats, + "warnings": extraction.warnings, }); + if let Some(layout) = extraction.layout { + output["layout"] = serde_json::to_value(layout)?; + } println!("{}", serde_json::to_string_pretty(&output)?); } else { print!("{}", extraction.markdown); diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 3d3cdd9..dfab278 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -168,6 +168,44 @@ fn extract_json_output() { assert!(parsed["markdown"].as_str().unwrap().contains("JSON Test")); } +#[test] +fn pdf_layout_requires_json_and_is_returned_for_pdf() { + let invalid = markkit_bin() + .args(["extract", "--stdin", "--layout"]) + .output() + .unwrap(); + assert!(!invalid.status.success()); + assert!(String::from_utf8_lossy(&invalid.stderr).contains("--json")); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("layout.pdf"); + // A repairable PDF with one page and no text, no external fixture needed. + std::fs::write(&path, b"%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF").unwrap(); + let output = markkit_bin() + .args([ + "extract", + "--file", + path.to_str().unwrap(), + "--json", + "--layout", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(value["layout"]["pages"][0]["width"], 300.0); + assert_eq!(value["layout"]["pages"][0]["height"], 400.0); + assert!( + value["layout"]["pages"][0]["spans"] + .as_array() + .unwrap() + .is_empty() + ); +} + #[test] fn extract_with_include_selector() { let mut child = markkit_bin() @@ -302,3 +340,16 @@ fn extract_with_bad_rules_file_fails() { assert!(!output.status.success()); assert!(String::from_utf8_lossy(&output.stderr).contains("invalid rules file")); } + +#[test] +fn ocr_backend_flags_validate_names_and_require_ocr() { + for args in [ + vec!["extract", "--stdin", "--ocr-backend", "ocrs"], + vec!["extract", "--stdin", "--ocr", "--ocr-backend", "unknown"], + vec!["extract", "--stdin", "--ocr-detection-model", "model.onnx"], + ] { + let output = markkit_bin().args(args).output().unwrap(); + assert!(!output.status.success()); + assert_eq!(output.status.code(), Some(2)); + } +} diff --git a/crates/markkit-eval/Cargo.toml b/crates/markkit-eval/Cargo.toml index 8a8f84b..fc48b3e 100644 --- a/crates/markkit-eval/Cargo.toml +++ b/crates/markkit-eval/Cargo.toml @@ -23,8 +23,8 @@ path = "src/main.rs" # Mira eval framework (crates.io), imported as `mira`. `macros` provides the # #[eval] discovery attribute. In-process subject only: no mira-everruns. mira-eval = { version = "0.5", features = ["macros"] } -# HTML only, no HTTP client: pages are embedded, markkit never fetches. -markkit = { path = "../markkit", default-features = false, features = ["html"] } +# Embedded HTML and generated PDF scenes; markkit never fetches. +markkit = { path = "../markkit", default-features = false, features = ["html", "pdf"] } serde.workspace = true serde_json.workspace = true # LLM evals (markkit_qa) call Anthropic / OpenRouter. Eval crate only: the diff --git a/crates/markkit-eval/data/pdf-layout.json b/crates/markkit-eval/data/pdf-layout.json new file mode 100644 index 0000000..15b85af --- /dev/null +++ b/crates/markkit-eval/data/pdf-layout.json @@ -0,0 +1,1720 @@ +[ + { + "id": "simple", + "category": "text", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 97.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 97.0, + 94.0 + ] + } + ] + } + ] + } + }, + { + "id": "cropped", + "category": "crop", + "pages": [ + { + "attrs": "/CropBox [50 100 450 700] /UserUnit 2", + "content": "BT /F1 10 Tf 100 200 Td (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 800, + "height": 1200, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 20, + "angle": 0, + "baseline": [ + [ + 100, + 1000 + ], + [ + 150.0, + 1000 + ] + ], + "bbox": [ + 100, + 984.0, + 150.0, + 1004.0 + ] + } + ] + } + ] + } + }, + { + "id": "rotate-90", + "category": "rotation", + "pages": [ + { + "attrs": "/Rotate 90", + "content": "BT /F1 10 Tf 72 700 Td (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 792, + "height": 612, + "rotation": 90, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 10, + "angle": 90, + "baseline": [ + [ + 700, + 72 + ], + [ + 700, + 97 + ] + ], + "bbox": [ + 698, + 72, + 708, + 97 + ] + } + ] + } + ] + } + }, + { + "id": "rotate-180", + "category": "rotation", + "pages": [ + { + "attrs": "/Rotate 180", + "content": "BT /F1 10 Tf 72 700 Td (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 180, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 10, + "angle": 180, + "baseline": [ + [ + 540, + 700 + ], + [ + 515, + 700 + ] + ], + "bbox": [ + 515, + 698, + 540, + 708 + ] + } + ] + } + ] + } + }, + { + "id": "rotate-270", + "category": "rotation", + "pages": [ + { + "attrs": "/Rotate 270", + "content": "BT /F1 10 Tf 72 700 Td (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 792, + "height": 612, + "rotation": 270, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 10, + "angle": 270, + "baseline": [ + [ + 92, + 540 + ], + [ + 92, + 515 + ] + ], + "bbox": [ + 84, + 515, + 94, + 540 + ] + } + ] + } + ] + } + }, + { + "id": "text-rotated", + "category": "rotation", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 0 1 -1 0 100 200 Tm (Hello) Tj ET" + } + ], + "markdown": "Hello\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Hello", + "font": "Helvetica", + "size": 10, + "angle": 270, + "baseline": [ + [ + 100, + 592 + ], + [ + 100, + 567 + ] + ], + "bbox": [ + 92, + 567, + 102, + 592 + ] + } + ] + } + ] + } + }, + { + "id": "tj-spacing", + "category": "spacing", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td [(Hi) -500 (there)] TJ ET" + } + ], + "markdown": "Hi there\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Hi", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 82.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 82.0, + 94.0 + ] + }, + { + "id": 1, + "text": "there", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 87, + 92 + ], + [ + 112.0, + 92 + ] + ], + "bbox": [ + 87, + 84.0, + 112.0, + 94.0 + ] + } + ] + } + ] + } + }, + { + "id": "scaled", + "category": "matrix", + "pages": [ + { + "attrs": "", + "content": "q 2 0 0 2 10 20 cm BT /F1 10 Tf 10 100 Td (Hi) Tj ET Q" + } + ], + "markdown": "Hi\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Hi", + "font": "Helvetica", + "size": 20, + "angle": 0, + "baseline": [ + [ + 30, + 572 + ], + [ + 50.0, + 572 + ] + ], + "bbox": [ + 30, + 556.0, + 50.0, + 576.0 + ] + } + ] + } + ] + } + }, + { + "id": "empty", + "category": "empty", + "pages": [ + { + "attrs": "", + "content": "" + } + ], + "markdown": "", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [] + } + ] + } + }, + { + "id": "shuffled-columns", + "category": "columns", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Left first) Tj ET\nBT /F1 10 Tf 300 700 Td (Right first) Tj ET\nBT /F1 10 Tf 72 686 Td (Left second) Tj ET\nBT /F1 10 Tf 300 686 Td (Right second) Tj ET" + } + ], + "markdown": "Left first Left second\n\nRight first Right second\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Left first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 122.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 122.0, + 94.0 + ] + }, + { + "id": 1, + "text": "Right first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 92 + ], + [ + 355.0, + 92 + ] + ], + "bbox": [ + 300, + 84.0, + 355.0, + 94.0 + ] + }, + { + "id": 2, + "text": "Left second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 106 + ], + [ + 127.0, + 106 + ] + ], + "bbox": [ + 72, + 98.0, + 127.0, + 108.0 + ] + }, + { + "id": 3, + "text": "Right second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 106 + ], + [ + 360.0, + 106 + ] + ], + "bbox": [ + 300, + 98.0, + 360.0, + 108.0 + ] + } + ] + } + ] + } + }, + { + "id": "ruled-table", + "category": "table", + "pages": [ + { + "attrs": "", + "content": "72 660 200 60 re S\n72 680 m 272 680 l S\n72 700 m 272 700 l S\n172 660 m 172 720 l S\nBT /F1 10 Tf 80 705 Td (Item) Tj ET\nBT /F1 10 Tf 180 705 Td (Qty) Tj ET\nBT /F1 10 Tf 80 685 Td (Apples) Tj ET\nBT /F1 10 Tf 180 685 Td (3) Tj ET\nBT /F1 10 Tf 80 665 Td (Pears) Tj ET\nBT /F1 10 Tf 180 665 Td (2) Tj ET" + } + ], + "markdown": "| Item | Qty |\n| --- | --- |\n| Apples | 3 |\n| Pears | 2 |\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Item", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 80, + 87 + ], + [ + 100.0, + 87 + ] + ], + "bbox": [ + 80, + 79.0, + 100.0, + 89.0 + ] + }, + { + "id": 1, + "text": "Qty", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 180, + 87 + ], + [ + 195.0, + 87 + ] + ], + "bbox": [ + 180, + 79.0, + 195.0, + 89.0 + ] + }, + { + "id": 2, + "text": "Apples", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 80, + 107 + ], + [ + 110.0, + 107 + ] + ], + "bbox": [ + 80, + 99.0, + 110.0, + 109.0 + ] + }, + { + "id": 3, + "text": "3", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 180, + 107 + ], + [ + 185.0, + 107 + ] + ], + "bbox": [ + 180, + 99.0, + 185.0, + 109.0 + ] + }, + { + "id": 4, + "text": "Pears", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 80, + 127 + ], + [ + 105.0, + 127 + ] + ], + "bbox": [ + 80, + 119.0, + 105.0, + 129.0 + ] + }, + { + "id": 5, + "text": "2", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 180, + 127 + ], + [ + 185.0, + 127 + ] + ], + "bbox": [ + 180, + 119.0, + 185.0, + 129.0 + ] + } + ] + } + ] + } + }, + { + "id": "multiple-pages", + "category": "pages", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (One) Tj ET" + }, + { + "attrs": "", + "content": "" + }, + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Three) Tj ET" + } + ], + "markdown": "One\n\nThree\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "One", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 87.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 87.0, + 94.0 + ] + } + ] + }, + { + "page": 2, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [] + }, + { + "page": 3, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Three", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 97.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 97.0, + 94.0 + ] + } + ] + } + ] + } + }, + { + "id": "borderless-table", + "category": "table", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Item) Tj ET\nBT /F1 10 Tf 300 700 Td (Qty) Tj ET\nBT /F1 10 Tf 72 684 Td (Apples) Tj ET\nBT /F1 10 Tf 300 684 Td (3) Tj ET\nBT /F1 10 Tf 72 668 Td (Pears) Tj ET\nBT /F1 10 Tf 300 668 Td (2) Tj ET\n" + } + ], + "markdown": "| Item | Qty |\n| --- | --- |\n| Apples | 3 |\n| Pears | 2 |\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Item", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 92.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 92.0, + 94.0 + ] + }, + { + "id": 1, + "text": "Qty", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 92 + ], + [ + 315.0, + 92 + ] + ], + "bbox": [ + 300, + 84.0, + 315.0, + 94.0 + ] + }, + { + "id": 2, + "text": "Apples", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 108 + ], + [ + 102.0, + 108 + ] + ], + "bbox": [ + 72, + 100.0, + 102.0, + 110.0 + ] + }, + { + "id": 3, + "text": "3", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 108 + ], + [ + 305.0, + 108 + ] + ], + "bbox": [ + 300, + 100.0, + 305.0, + 110.0 + ] + }, + { + "id": 4, + "text": "Pears", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 124 + ], + [ + 97.0, + 124 + ] + ], + "bbox": [ + 72, + 116.0, + 97.0, + 126.0 + ] + }, + { + "id": 5, + "text": "2", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 124 + ], + [ + 305.0, + 124 + ] + ], + "bbox": [ + 300, + 116.0, + 305.0, + 126.0 + ] + } + ], + "blocks": [ + { + "id": 0, + "kind": "table", + "cells": [ + { + "row": 0, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "row": 0, + "column": 1, + "rowspan": 1, + "colspan": 1 + }, + { + "row": 1, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "row": 1, + "column": 1, + "rowspan": 1, + "colspan": 1 + }, + { + "row": 2, + "column": 0, + "rowspan": 1, + "colspan": 1 + }, + { + "row": 2, + "column": 1, + "rowspan": 1, + "colspan": 1 + } + ] + } + ] + } + ] + } + }, + { + "id": "wrapped-cell", + "category": "table", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Item) Tj ET\nBT /F1 10 Tf 300 700 Td (Qty) Tj ET\nBT /F1 10 Tf 72 684 Td (Long description) Tj ET\nBT /F1 10 Tf 300 684 Td (3) Tj ET\nBT /F1 10 Tf 72 672 Td (continued) Tj ET\nBT /F1 10 Tf 72 652 Td (Pears) Tj ET\nBT /F1 10 Tf 300 652 Td (2) Tj ET\n" + } + ], + "markdown": "| Item | Qty |\n| --- | --- |\n| Long description continued | 3 |\n| Pears | 2 |\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Item", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 92.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 92.0, + 94.0 + ] + }, + { + "id": 1, + "text": "Qty", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 92 + ], + [ + 315.0, + 92 + ] + ], + "bbox": [ + 300, + 84.0, + 315.0, + 94.0 + ] + }, + { + "id": 2, + "text": "Long description", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 108 + ], + [ + 152.0, + 108 + ] + ], + "bbox": [ + 72, + 100.0, + 152.0, + 110.0 + ] + }, + { + "id": 3, + "text": "3", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 108 + ], + [ + 305.0, + 108 + ] + ], + "bbox": [ + 300, + 100.0, + 305.0, + 110.0 + ] + }, + { + "id": 4, + "text": "continued", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 120 + ], + [ + 117.0, + 120 + ] + ], + "bbox": [ + 72, + 112.0, + 117.0, + 122.0 + ] + }, + { + "id": 5, + "text": "Pears", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 140 + ], + [ + 97.0, + 140 + ] + ], + "bbox": [ + 72, + 132.0, + 97.0, + 142.0 + ] + }, + { + "id": 6, + "text": "2", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 140 + ], + [ + 305.0, + 140 + ] + ], + "bbox": [ + 300, + 132.0, + 305.0, + 142.0 + ] + } + ], + "blocks": [ + { + "id": 0, + "kind": "table" + } + ] + } + ] + } + }, + { + "id": "tagged-heading", + "category": "tags", + "pages": [ + { + "attrs": "", + "content": "/H2 BMC BT /F1 10 Tf 72 700 Td (Section) Tj ET EMC\nBT /F1 10 Tf 72 670 Td (Body) Tj ET" + } + ], + "markdown": "## Section\n\nBody\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Section", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 107.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 107.0, + 94.0 + ] + }, + { + "id": 1, + "text": "Body", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 122 + ], + [ + 92.0, + 122 + ] + ], + "bbox": [ + 72, + 114.0, + 92.0, + 124.0 + ] + } + ], + "blocks": [ + { + "id": 0, + "kind": "heading" + }, + { + "id": 1, + "kind": "paragraph" + } + ] + } + ] + } + }, + { + "id": "ordered-list", + "category": "lists", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 14 TL 72 700 Td (3. first) Tj T* (4. second) Tj ET" + } + ], + "markdown": "3. first\n4. second\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "3. first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 112.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 112.0, + 94.0 + ] + }, + { + "id": 1, + "text": "4. second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 106 + ], + [ + 117.0, + 106 + ] + ], + "bbox": [ + 72, + 98.0, + 117.0, + 108.0 + ] + } + ], + "blocks": [ + { + "id": 0, + "kind": "list" + } + ] + } + ] + } + }, + { + "id": "actual-text", + "category": "tags", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td /Span << /ActualText (Accessible text) >> BDC (junk) Tj (junk2) Tj EMC ET" + } + ], + "markdown": "Accessible text\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Accessible text", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 117, + 92 + ] + ], + "bbox": [ + 72, + 84, + 117, + 94 + ] + } + ] + } + ] + } + }, + { + "id": "spanning-heading", + "category": "columns", + "pages": [ + { + "attrs": "", + "content": "BT /F1 20 Tf 72 740 Td (Title across columns) Tj ET\nBT /F1 10 Tf 72 700 Td (Left first) Tj ET\nBT /F1 10 Tf 300 700 Td (Right first) Tj ET\nBT /F1 10 Tf 72 686 Td (Left second) Tj ET\nBT /F1 10 Tf 300 686 Td (Right second) Tj ET\n" + } + ], + "markdown": "# Title across columns\n\nLeft first Left second\n\nRight first Right second\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Title across columns", + "font": "Helvetica", + "size": 20, + "angle": 0, + "baseline": [ + [ + 72, + 52 + ], + [ + 272.0, + 52 + ] + ], + "bbox": [ + 72, + 36.0, + 272.0, + 56.0 + ] + }, + { + "id": 1, + "text": "Left first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 122.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 122.0, + 94.0 + ] + }, + { + "id": 2, + "text": "Right first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 92 + ], + [ + 355.0, + 92 + ] + ], + "bbox": [ + 300, + 84.0, + 355.0, + 94.0 + ] + }, + { + "id": 3, + "text": "Left second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 106 + ], + [ + 127.0, + 106 + ] + ], + "bbox": [ + 72, + 98.0, + 127.0, + 108.0 + ] + }, + { + "id": 4, + "text": "Right second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 300, + 106 + ], + [ + 360.0, + 106 + ] + ], + "bbox": [ + 300, + 98.0, + 360.0, + 108.0 + ] + } + ] + } + ] + } + }, + { + "id": "narrow-gutter", + "category": "columns", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 700 Td (Long left line filling the space completely.) Tj ET\nBT /F1 10 Tf 302 703 Td (Right first) Tj ET\nBT /F1 10 Tf 72 686 Td (Another left line filling the space fully.) Tj ET\nBT /F1 10 Tf 302 689 Td (Right second) Tj ET" + } + ], + "markdown": "Long left line filling the space completely. Another left line filling the space fully.\n\nRight first Right second\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Long left line filling the space completely.", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 292.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 292.0, + 94.0 + ] + }, + { + "id": 1, + "text": "Right first", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 302, + 89 + ], + [ + 357.0, + 89 + ] + ], + "bbox": [ + 302, + 81.0, + 357.0, + 91.0 + ] + }, + { + "id": 2, + "text": "Another left line filling the space fully.", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 106 + ], + [ + 282.0, + 106 + ] + ], + "bbox": [ + 72, + 98.0, + 282.0, + 108.0 + ] + }, + { + "id": 3, + "text": "Right second", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 302, + 103 + ], + [ + 362.0, + 103 + ] + ], + "bbox": [ + 302, + 95.0, + 362.0, + 105.0 + ] + } + ] + } + ] + } + }, + { + "id": "shuffled-lines", + "category": "order", + "pages": [ + { + "attrs": "", + "content": "BT /F1 10 Tf 72 686 Td (Second line) Tj ET\nBT /F1 10 Tf 72 700 Td (First line) Tj ET" + } + ], + "markdown": "First line Second line\n", + "layout": { + "version": 1, + "pages": [ + { + "page": 1, + "width": 612, + "height": 792, + "rotation": 0, + "spans": [ + { + "id": 0, + "text": "Second line", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 106 + ], + [ + 127.0, + 106 + ] + ], + "bbox": [ + 72, + 98.0, + 127.0, + 108.0 + ] + }, + { + "id": 1, + "text": "First line", + "font": "Helvetica", + "size": 10, + "angle": 0, + "baseline": [ + [ + 72, + 92 + ], + [ + 122.0, + 92 + ] + ], + "bbox": [ + 72, + 84.0, + 122.0, + 94.0 + ] + } + ] + } + ] + } + } +] diff --git a/crates/markkit-eval/results/hayro-2026-10-29.json b/crates/markkit-eval/results/hayro-2026-10-29.json new file mode 100644 index 0000000..7005a64 --- /dev/null +++ b/crates/markkit-eval/results/hayro-2026-10-29.json @@ -0,0 +1,431 @@ +{ + "date": "2026-10-29", + "scope": "Replace optional PDF OCR Poppler rasterization with Rust Hayro; retain layout/tables and Tesseract recognition; audit remaining native dependencies.", + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "source_dirty": true, + "candidate_binary_sha256": "54ac90378f5c89ecad827836306e363641c3bfe79585365c00d2de6b34d8db47", + "provenance_review": { + "project": "https://github.com/LaurenzV/hayro", + "registry_release": "0.8.0", + "registry_checksum": "4675ff5acd7ca41c0c34ccd2a3a154f5c86d8b5ec92bef0a114bfb194969e2af", + "registry_owners": [ + "LaurenzV" + ], + "maintainer_public_name": "Laurenz Stampfl", + "maintainer_public_location": "Vienna, Austria", + "earliest_surviving_default_branch_commit": "38efa5f7d08e68ebca2d1a209d4921c1deffeaf3", + "earliest_commit_date": "2025-04-22", + "finding": "No Russian organizational ownership found in public repository/registry records. Does not certify every contributor, historical affiliation or transitive dependency author." + }, + "public_default_regression": { + "cases": 178, + "regressions": [], + "quality": { + "agentdocbench/category/cjk": 0.9335, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.49095, + "agentdocbench/category/form": 0.8377666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8956333333333333, + "agentdocbench/category/report": 0.8705, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.75135, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7586666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.6553260869565217, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8660333333333333, + "agentdocbench/overall": 0.7938638888888889, + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565 + }, + "provenance": { + "binary_sha256": "54ac90378f5c89ecad827836306e363641c3bfe79585365c00d2de6b34d8db47", + "build_profile": "release", + "command": [ + "/workspace/markkit/target/public-benchmarks/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/work/hayro-review/public/mira.json" + ], + "core_diff_sha256": "d558790fae49d8a697de670706a70621ac0b787bfe2a903cfa51e2d1b46dfd50", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": true, + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + } + }, + "pdf_scenes": { + "cases": 40, + "passed": 40, + "samples": 20, + "modes": [ + "default", + "layout" + ] + }, + "public_pdf_geometry": { + "documents": 23, + "pages": 243, + "cases": [ + { + "id": "paper-evsched", + "pages": 23, + "spans": 12924, + "blocks": 310, + "markdown_unchanged": true + }, + { + "id": "paper-rankone", + "pages": 27, + "spans": 35955, + "blocks": 2302, + "markdown_unchanged": true + }, + { + "id": "paper-ulam", + "pages": 11, + "spans": 21304, + "blocks": 287, + "markdown_unchanged": true + }, + { + "id": "twocol-bygpt5", + "pages": 16, + "spans": 16572, + "blocks": 542, + "markdown_unchanged": true + }, + { + "id": "twocol-smile", + "pages": 13, + "spans": 11879, + "blocks": 733, + "markdown_unchanged": true + }, + { + "id": "twocol-uscore", + "pages": 17, + "spans": 16510, + "blocks": 502, + "markdown_unchanged": true + }, + { + "id": "report-crs-dod-approps", + "pages": 12, + "spans": 10842, + "blocks": 397, + "markdown_unchanged": true + }, + { + "id": "report-plz", + "pages": 5, + "spans": 5502, + "blocks": 52, + "markdown_unchanged": true + }, + { + "id": "fed-h8", + "pages": 22, + "spans": 10785, + "blocks": 2841, + "markdown_unchanged": true + }, + { + "id": "fin-bls-realer", + "pages": 5, + "spans": 6722, + "blocks": 571, + "markdown_unchanged": true + }, + { + "id": "fin-census-income", + "pages": 10, + "spans": 7424, + "blocks": 743, + "markdown_unchanged": true + }, + { + "id": "invoice-borderless", + "pages": 2, + "spans": 113, + "blocks": 42, + "markdown_unchanged": true + }, + { + "id": "form-1040es", + "pages": 12, + "spans": 5334, + "blocks": 631, + "markdown_unchanged": true + }, + { + "id": "form-w4", + "pages": 4, + "spans": 1739, + "blocks": 189, + "markdown_unchanged": true + }, + { + "id": "irs-w9", + "pages": 6, + "spans": 1074, + "blocks": 199, + "markdown_unchanged": true + }, + { + "id": "scan-nasa-budget", + "pages": 3, + "spans": 0, + "blocks": 0, + "markdown_unchanged": true + }, + { + "id": "scan-nasa-notes", + "pages": 3, + "spans": 0, + "blocks": 0, + "markdown_unchanged": true + }, + { + "id": "scan-oklahoma-address", + "pages": 7, + "spans": 0, + "blocks": 0, + "markdown_unchanged": true + }, + { + "id": "cjk-jp-stat-elderly", + "pages": 4, + "spans": 2797, + "blocks": 190, + "markdown_unchanged": true + }, + { + "id": "cjk-tw-archives", + "pages": 4, + "spans": 2385, + "blocks": 15, + "markdown_unchanged": true + }, + { + "id": "cjk-tw-dgbas-cpi", + "pages": 6, + "spans": 3614, + "blocks": 368, + "markdown_unchanged": true + }, + { + "id": "slides-nasa-clarreo", + "pages": 15, + "spans": 5715, + "blocks": 160, + "markdown_unchanged": true + }, + { + "id": "slides-nist-pqc-update", + "pages": 16, + "spans": 4054, + "blocks": 190, + "markdown_unchanged": true + } + ] + }, + "ocr": { + "documents": 3, + "poppler_mean_document_score": 0.5233666666666666, + "hayro_mean_document_score": 0.5604666666666667, + "cases": [ + { + "id": "scan-nasa-budget", + "scores": { + "document_score": 0.3253, + "sentences": 0.2, + "order": 0.6667, + "table_cells_f1": 0.1091 + }, + "seconds": 5.4309253960000206, + "spans": 839, + "binary_sha256": "54ac90378f5c89ecad827836306e363641c3bfe79585365c00d2de6b34d8db47", + "baseline_score": 0.353, + "score_delta": -0.0277 + }, + { + "id": "scan-nasa-notes", + "scores": { + "document_score": 0.4394, + "sentences": 0.5, + "order": 0.8182, + "table_cells_f1": 0.0 + }, + "seconds": 5.464721544998611, + "spans": 785, + "binary_sha256": "54ac90378f5c89ecad827836306e363641c3bfe79585365c00d2de6b34d8db47", + "baseline_score": 0.3838, + "score_delta": 0.0556 + }, + { + "id": "scan-oklahoma-address", + "scores": { + "document_score": 0.9167, + "sentences": 0.8333333333333334, + "order": 1.0 + }, + "seconds": 14.05723030000081, + "spans": 3174, + "binary_sha256": "54ac90378f5c89ecad827836306e363641c3bfe79585365c00d2de6b34d8db47", + "baseline_score": 0.8333, + "score_delta": 0.0834 + } + ], + "regressed_cases": [ + "scan-nasa-budget" + ], + "configuration": "--ocr --timeout 300 --json --layout; 1800px white-backed grayscale PGM; interpolation forced; explicit page DPI; Tesseract eng with one OpenMP thread; PATH contains only Tesseract (no Poppler)", + "timing_note": "Observed end-to-end wall time only; concurrent build activity and cache differences prevent a controlled performance comparison." + }, + "validation": { + "workspace_all_features": { + "passed": 820, + "failed": 0, + "ignored": 1 + }, + "ocr_integration_explicit": { + "passed": 4, + "failed": 0, + "ignored": 0 + }, + "pdf_only_tests": { + "passed": 134, + "failed": 0, + "ignored": 0 + }, + "format": "passed", + "knowledge": "OKF v0.2 conformant", + "clippy": "passed", + "clippy-pdf-only": "passed", + "workspace-build": "passed", + "wasm-build": "passed", + "rustdoc": "passed", + "node": { + "passed": 10, + "failed": 0 + }, + "wasm": { + "passed": 23, + "failed": 0 + }, + "python": { + "passed": 2, + "failed": 0 + } + }, + "native_dependencies": { + "production": [ + "Tesseract C++ and Leptonica C, host-installed for optional OCR; distribution-specific codec libraries", + "AWS-LC C/assembly for optional http_client TLS", + "CPython C / Node V8 C++ host runtimes for bindings", + "OS APIs and platform certificate stores" + ], + "development_only": [ + "alloca C shim via Criterion" + ], + "not_enabled": [ + "ring is in lockfile but not the active reviewed all-features graph" + ], + "renderer": "Hayro and its rendering/codec/font/color libraries are Rust; no Poppler executable required." + }, + "limits_and_tradeoffs": [ + "OCR mean improves, but NASA Budget falls from 0.3530 to 0.3253; per-case regression remains visible.", + "Hayro rendering is non-interruptible with cooperative deadline checks before/after; host isolation provides hard CPU/memory quotas.", + "JPEG, header-bounded JPEG2000 and generic-filter image OCR are supported. Inline images, DecodeParms, JBIG2 and CCITT OCR fail explicitly until bounded decoder support exists.", + "Per-image working estimate is 16 bytes per pixel; cumulative RGBA estimate is four bytes per pixel. These estimates are not a hard heap quota.", + "All existing layout/table inference limitations remain; no corpus/baseline/scorer changes." + ], + "changed_source_sha256": { + "Cargo.lock": "5f00819f5dbc9ed5dbb2ca5fe1152b592af34ef6d6f125aa8a40861c0b780b1b", + "Cargo.toml": "16622c0ddc29a8556ec93743b0c05cfa73974679590f70029dc615e364e4ffa6", + "crates/markkit-cli/Cargo.toml": "00ca964fa09dbb701e54dd43895927bd193877dccb85664352e6f15f68c61de0", + "crates/markkit-cli/src/main.rs": "13a93ed3ad55e1244eb0e285aa343e56e2e9720383cd1fcdb06795c825f2b696", + "crates/markkit-cli/tests/cli_integration.rs": "7cecfb9131c1cc446bdc3438a448cd630b1b0d440e7fa26641dcb47177ce5aad", + "crates/markkit-eval/Cargo.toml": "903b131f5716708491924ea56af529e681a980c499223d909cab1b93e43fee7d", + "crates/markkit-eval/data/pdf-layout.json": "38f5ac20041441fad3f6efcbe5a2f8ca82d408339c953971ccf036ebf856d42b", + "crates/markkit-eval/src/lib.rs": "ee1e9ac8ec5113fbb2b0a2cea6948e07905d9cc0d021c49ee212fc3f51ddf843", + "crates/markkit-eval/src/main.rs": "feb406d2f02b3fc35efd7a0647780dc7c66656852da1219e24b17f848f70720e", + "crates/markkit-eval/src/pdf.rs": "43bc3ee3b2fbd358068361bdce611710859ada341a1c631d566f367ebe6d6319", + "crates/markkit-js/Cargo.toml": "8c1465370c370169f46de4d689ee7a65d49cc2a6d021ae9f24df4b75aaa773e6", + "crates/markkit-js/__test__/options.test.cjs": "551806085f99d27176e798eeb224c6d99828fd446573ca2641b2896b7df5851f", + "crates/markkit-js/src/lib.rs": "d95f076585da9f14ef29d8e622ddb3566ce6db46655c7570208807bd6275b316", + "crates/markkit-python/Cargo.toml": "60a08f0895531f9f4e47b9df5058c1f48b202449256f587c6f2b802d3295b86f", + "crates/markkit-python/src/lib.rs": "bed7b5ba5ec910e4561117599f88e83a9ff054d6555cc15012e28d0c3548326c", + "crates/markkit-python/tests/test_documents.py": "8bad0e0698412f3f26c0af583b16e2dbfb23f04aeaf5c2892875f21183b1b243", + "crates/markkit-wasm/Cargo.toml": "6b59cdb571c528ce4f362e5d0713e2ef0aab29eb928cd44579855cef2c9095cc", + "crates/markkit-wasm/__test__/markkit-wasm.test.mjs": "6edd60dae6a6bc165b6c78e9ef1d5134b21a5ab10f4509fe5e0953468612209b", + "crates/markkit-wasm/js/index.d.ts": "3f6c9853e1dd753ef940fa667827bff52d34a2707526ac243e6360082528c404", + "crates/markkit-wasm/src/lib.rs": "85791162e8d7862019c1b7da621a28ee4b2bb7789fafa04ccac1004eb72ad1d3", + "crates/markkit/Cargo.toml": "ac7cf6249887b6fc346206506097acefae230fd605224380ec0ab5eae48f5436", + "crates/markkit/src/config.rs": "829c85678fc0eee0b8912af173a8d175f125763af4a8ad90eca1f13ed97e1a69", + "crates/markkit/src/extract.rs": "87b796ff60a4f995e6c6c308e480d132281f53983b9a76a66da021b935e2ed5e", + "crates/markkit/src/ir.rs": "b7a121efe1e498697698176bf2b779631f76a5daf511aebd74b6fa97c0b368b2", + "crates/markkit/src/layout.rs": "1979d867083314b2d5550bfe80b2c1a85f7da3a3bc5056af13379080b40bb623", + "crates/markkit/src/lib.rs": "78db7fa50930b8d60e123ac0c199b27376667b360b93da34dc7a19cc3b1f3f63", + "crates/markkit/src/limits.rs": "a197331637943ef907791697f1cb1c3270e35a6b5f7aed8480464bd01b00e0cc", + "crates/markkit/src/pdf/analyze.rs": "f27c781852039c574b72a6cc71594ac04562d39134794311b97e8dfb6252aa4d", + "crates/markkit/src/pdf/font.rs": "32c2569858d14ffb1cc6caf6419d79c25dce9c57bb33e1741a8714a728fdf02e", + "crates/markkit/src/pdf/guard.rs": "489d3bf291c63656ecd0181318dd4fffd7ac300e89ad029428814f3f2660d07e", + "crates/markkit/src/pdf/layout.rs": "8c9be884482158a6832989dab75936381cea955419d122e710e5b195478e88c6", + "crates/markkit/src/pdf/mod.rs": "95c0f19f28f49569c7f554166b67b99aa335eadc88c129b140cc7a1a75a3ada2", + "crates/markkit/src/pdf/ocr.rs": "40fb7b6774646653d1a67a5a2a5dfe56ec6f15350bfec6d24128a4afa830a70f", + "crates/markkit/src/pdf/tags.rs": "9e925845654113cdc2fecdf5a96f55377c8a23bd8443f9b70494a7dbde71cced", + "crates/markkit/tests/pdf_layout.rs": "9db16686713b59ae187bd315c8c47416bd7f67925750f80011c88f6f6cd2964a", + "crates/markkit/tests/pdf_ocr.rs": "824fe669525deee5043420cfb13c0d91923b5bf1fa373d6cc000de021d040304", + "crates/markkit/tests/pdf_structure.rs": "fd941762bbee191421f3eb790906e25a9b7d6ce49e5f55f8641595a0a5c54ca8", + "schema/ir.schema.json": "3763054cb70c7ce75c86b0ec2a70948cbee17d2e60613256ec63d697e8fe1860" + } +} diff --git a/crates/markkit-eval/results/ocrs-2026-10-28.json b/crates/markkit-eval/results/ocrs-2026-10-28.json new file mode 100644 index 0000000..3ae763e --- /dev/null +++ b/crates/markkit-eval/results/ocrs-2026-10-28.json @@ -0,0 +1,488 @@ +{ + "scope": "Same three pinned public scans, shared Hayro rendering and Markkit layout/Markdown; OCR backend is the variable.", + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "source_dirty": true, + "binary_sha256": "d77b9d69bc0de0964607f02ef4cdd126ab686d4af8442e7ad647b0806fa3961d", + "model_pins": { + "source": "https://github.com/robertknight/ocrs/blob/main/ocrs/examples/download-models.sh", + "revision": "Official example model pair, content pinned by SHA256", + "license": "CC-BY-SA-4.0", + "license_url": "https://huggingface.co/robertknight/ocrs/blob/df0edd170279ab971b53e094c627255a87e1a503/README.md", + "attribution": "Ocrs models by Robert Knight; trained on HierText (CC-BY-SA 4.0).", + "models": [ + { + "name": "text-detection.onnx", + "url": "https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.onnx", + "sha256": "a917b23dbd9524b465df7e922641b3ff2981623df4ded5a0234004ef2fee7cfe", + "size": 2499479, + "last_modified": "Sun, 13 Sep 2026 08:22:54 GMT" + }, + { + "name": "text-recognition.onnx", + "url": "https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.onnx", + "sha256": "86c145c2edb96c8caed5b1ebb8f44d706408922211451c309c625157dd6061c5", + "size": 9713177, + "last_modified": "Sun, 13 Sep 2026 08:22:52 GMT" + } + ] + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "versions": { + "ocrs": "0.13.1", + "rten": "0.26.0", + "tesseract": [ + "tesseract 5.5.0", + " leptonica-1.84.1" + ] + }, + "method": { + "profile": "release", + "warmups": 1, + "runs": 3, + "thread_environment": { + "OMP_THREAD_LIMIT": "1", + "RTEN_NUM_THREADS": "1", + "RAYON_NUM_THREADS": "1" + }, + "timing": "Fresh process end-to-end, including model loading and page rendering; scoring outside boundary.", + "memory": "Peak sum of process-tree resident bytes, sampled every 10ms on Linux. Shared pages may be counted multiple times; inference buffers are included.", + "timeout_seconds": 300, + "ocr_language": "eng", + "raster_max_dimension": 1800, + "ocrs_max_lines_per_batch": 20 + }, + "pure_rust_ocr_helper_check": "All ocrs executions used an empty PATH.", + "cases": [ + { + "id": "scan-nasa-budget", + "input_sha256": "f0637cdd4026471afa0cf1ab199a4030844ee619f0a6dd727ceab5f29717e847", + "engines": { + "tesseract": { + "scores": { + "document_score": 0.3253, + "sentences": 0.2, + "order": 0.6667, + "table_cells_f1": 0.1091 + }, + "geometry": { + "pages": 3, + "words": 839, + "valid": true, + "tables": 1 + }, + "median_seconds": 4.277916884999286, + "median_peak_process_tree_rss_bytes": 37949440, + "measurements": [ + { + "seconds": 3.9396111330006534, + "peak_process_tree_rss_bytes": 37814272 + }, + { + "seconds": 4.277916884999286, + "peak_process_tree_rss_bytes": 37949440 + }, + { + "seconds": 4.448720090998904, + "peak_process_tree_rss_bytes": 37949440 + } + ] + }, + "ocrs": { + "scores": { + "document_score": 0.3167, + "sentences": 0.2, + "order": 0.75, + "table_cells_f1": 0.0 + }, + "geometry": { + "pages": 3, + "words": 840, + "valid": true, + "tables": 0 + }, + "median_seconds": 12.777047276998928, + "median_peak_process_tree_rss_bytes": 238678016, + "measurements": [ + { + "seconds": 12.566175017000205, + "peak_process_tree_rss_bytes": 229179392 + }, + { + "seconds": 12.777047276998928, + "peak_process_tree_rss_bytes": 238678016 + }, + { + "seconds": 13.25562496999919, + "peak_process_tree_rss_bytes": 238919680 + } + ] + } + }, + "geometry_agreement": { + "exact_text_matches": 712, + "mean_matched_word_bbox_iou": 0.7946531947851007, + "interpretation": "Cross-engine agreement only; these scans have no ground-truth word boxes." + } + }, + { + "id": "scan-nasa-notes", + "input_sha256": "0d0ae5bc4a51949159beea4455d198852087e7fd6b95e9d6730a6248700975bf", + "engines": { + "tesseract": { + "scores": { + "document_score": 0.4394, + "sentences": 0.5, + "order": 0.8182, + "table_cells_f1": 0.0 + }, + "geometry": { + "pages": 3, + "words": 785, + "valid": true, + "tables": 0 + }, + "median_seconds": 3.745943695999813, + "median_peak_process_tree_rss_bytes": 36679680, + "measurements": [ + { + "seconds": 4.393917142000646, + "peak_process_tree_rss_bytes": 36679680 + }, + { + "seconds": 3.701504170001499, + "peak_process_tree_rss_bytes": 36880384 + }, + { + "seconds": 3.745943695999813, + "peak_process_tree_rss_bytes": 36614144 + } + ] + }, + "ocrs": { + "scores": { + "document_score": 0.4141, + "sentences": 0.3333333333333333, + "order": 0.9091, + "table_cells_f1": 0.0 + }, + "geometry": { + "pages": 3, + "words": 787, + "valid": true, + "tables": 0 + }, + "median_seconds": 12.50714061600047, + "median_peak_process_tree_rss_bytes": 228642816, + "measurements": [ + { + "seconds": 13.82876325000143, + "peak_process_tree_rss_bytes": 228769792 + }, + { + "seconds": 12.50714061600047, + "peak_process_tree_rss_bytes": 228642816 + }, + { + "seconds": 12.194090405999304, + "peak_process_tree_rss_bytes": 228585472 + } + ] + } + }, + "geometry_agreement": { + "exact_text_matches": 589, + "mean_matched_word_bbox_iou": 0.812589517515257, + "interpretation": "Cross-engine agreement only; these scans have no ground-truth word boxes." + } + }, + { + "id": "scan-oklahoma-address", + "input_sha256": "f9da22937bda1617e919bb4a3b902551187bb484a07af254ccff22647ae81848", + "engines": { + "tesseract": { + "scores": { + "document_score": 0.9167, + "sentences": 0.8333333333333334, + "order": 1.0 + }, + "geometry": { + "pages": 7, + "words": 3174, + "valid": true, + "tables": 0 + }, + "median_seconds": 13.509221518999766, + "median_peak_process_tree_rss_bytes": 58048512, + "measurements": [ + { + "seconds": 13.509221518999766, + "peak_process_tree_rss_bytes": 58048512 + }, + { + "seconds": 14.530162234999807, + "peak_process_tree_rss_bytes": 58155008 + }, + { + "seconds": 13.428984150999895, + "peak_process_tree_rss_bytes": 57786368 + } + ] + }, + "ocrs": { + "scores": { + "document_score": 0.9167, + "sentences": 0.8333333333333334, + "order": 1.0 + }, + "geometry": { + "pages": 7, + "words": 3165, + "valid": true, + "tables": 0 + }, + "median_seconds": 54.33059063300061, + "median_peak_process_tree_rss_bytes": 255877120, + "measurements": [ + { + "seconds": 55.482910818000164, + "peak_process_tree_rss_bytes": 255877120 + }, + { + "seconds": 54.33059063300061, + "peak_process_tree_rss_bytes": 256438272 + }, + { + "seconds": 52.57790281199959, + "peak_process_tree_rss_bytes": 255811584 + } + ] + } + }, + "geometry_agreement": { + "exact_text_matches": 2600, + "mean_matched_word_bbox_iou": 0.7583267686781607, + "interpretation": "Cross-engine agreement only; these scans have no ground-truth word boxes." + } + } + ], + "engines": { + "tesseract": { + "mean_document_score": 0.5604666666666667, + "median_document_seconds": 4.277916884999286, + "median_peak_process_tree_rss_bytes": 37949440 + }, + "ocrs": { + "mean_document_score": 0.5491666666666667, + "median_document_seconds": 12.777047276998928, + "median_peak_process_tree_rss_bytes": 238678016 + } + }, + "date": "2026-10-28", + "timezone": "America/Chicago", + "validation": { + "workspace_all_features": { + "passed": 824, + "failed": 0, + "ignored": 2 + }, + "ocr_models_and_tesseract_explicit": { + "passed": 8, + "failed": 0, + "ignored": 0 + }, + "tesseract_feature_without_ocrs": { + "passed": 5, + "failed": 0, + "ignored": 1 + }, + "node": { + "passed": 11, + "failed": 0 + }, + "wasm": { + "passed": 23, + "failed": 0 + }, + "python": { + "passed": 3, + "failed": 0 + }, + "clippy": "passed", + "ocrs-only-clippy": "passed", + "rustdoc": "passed", + "repo_scripts": { + "run": 38, + "skipped": 2, + "failed": 0 + }, + "pdf_authored_scenes": { + "passed": 40, + "failed": 0 + }, + "public_default": { + "cases": 178, + "regressions": [] + }, + "format": "passed", + "knowledge": "OKF v0.2 conformant" + }, + "public_default_regression": { + "cases": 178, + "regressions": [], + "quality": { + "agentdocbench/category/cjk": 0.9335, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.49095, + "agentdocbench/category/form": 0.8377666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8956333333333333, + "agentdocbench/category/report": 0.8705, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.75135, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7586666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.6553260869565217, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8660333333333333, + "agentdocbench/overall": 0.7938638888888889, + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565 + }, + "provenance": { + "binary_sha256": "7df6b8dee165fbb3b57b69392a1ac2604441feb6f19f4aa85bf17dbbe47d7ec8", + "build_profile": "release", + "command": [ + "/workspace/markkit/target/public-benchmarks/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/work/ocrs/public/mira.json" + ], + "core_diff_sha256": "03ee134984a9996a4fc4352921c640d5d821acb6099a5f188a907a3721351443", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": true, + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + }, + "build_configuration": "Default CLI features; optional ocrs disabled. Backend comparison uses a separately built CLI with pdf-ocr-ocrs enabled." + }, + "geometry_scope": "Authored model-backed fixture checks three known words in their independent expected source region. Public scans check geometry validity and cross-engine agreement; no ground-truth word boxes exist in this corpus.", + "dependency_provenance": { + "ocrs_owner": "robertknight", + "owner_public_name": "Robert Knight", + "owner_public_location": "London, United Kingdom", + "registry_owner_source": "https://crates.io/api/v1/crates/ocrs/owners", + "owner_source": "https://github.com/robertknight", + "finding": "No Russian organizational owner found in public records; does not establish every contributor's background or affiliations.", + "runtime": "Ocrs and RTen implement inference in Rust. Pure Rust OCR production dependency tree, with http_client disabled, adds no cc/cmake/native ML runtime dependency; Tesseract is not invoked on the ocrs path." + }, + "interpretation": { + "default_backend": "Tesseract remains the default; ocrs is explicitly selected and optional at build time.", + "recommendation": "Use ocrs for a Rust-only OCR deployment when its language, accuracy, CPU and memory tradeoffs fit. Keep Tesseract as default for this scan corpus.", + "limits": [ + "Only the configured default Latin model/eng selector is supported by this integration.", + "Combined local model file bytes are bounded; inference activation memory requires host quotas.", + "Deadlines are cooperative around non-interruptible RTen operations and between batches of 20 lines.", + "Model weights are separate CC-BY-SA 4.0 deployment assets and are not embedded in library/binding packages.", + "Same shared PDF rendering restrictions apply to both engines.", + "These three scanned documents are a small evaluation set, not a general OCR ranking." + ] + }, + "regressed_cases": [ + "scan-nasa-budget", + "scan-nasa-notes" + ], + "changed_source_sha256": { + ".github/workflows/ci.yml": "c406f40a0712ae1332fbe5673faa17337d38dc86cb31c630e02d1556cd1f29f2", + ".github/workflows/python.yml": "3eff41f9546a45d783194cf79a69506322ed00f7fa9d37de1ca63fb6cc725430", + "Cargo.lock": "f6a928ad608c83cf1de580b323a7eb9cbe5fc3f2d6a888e5cb754bd94803729e", + "Cargo.toml": "3dd578d6897285ad735650d56ab74afdede2b7cd96fb8f3ceb66ea6f1f592e1d", + "crates/markkit-cli/Cargo.toml": "2270315eda96b6b9858c775f8fa39d12dc09e28961e71923da523b9aa1bf3a16", + "crates/markkit-cli/src/main.rs": "eca24db09752599a5eb22c50866e5bcdb2f0f637e53a1d63bb91e6918e444f85", + "crates/markkit-cli/tests/cli_integration.rs": "6d654357450176589000bcd356720404307c59ca26faddfa47ca4f8310cef304", + "crates/markkit-eval/Cargo.toml": "903b131f5716708491924ea56af529e681a980c499223d909cab1b93e43fee7d", + "crates/markkit-eval/data/pdf-layout.json": "38f5ac20041441fad3f6efcbe5a2f8ca82d408339c953971ccf036ebf856d42b", + "crates/markkit-eval/src/lib.rs": "ee1e9ac8ec5113fbb2b0a2cea6948e07905d9cc0d021c49ee212fc3f51ddf843", + "crates/markkit-eval/src/main.rs": "feb406d2f02b3fc35efd7a0647780dc7c66656852da1219e24b17f848f70720e", + "crates/markkit-eval/src/pdf.rs": "43bc3ee3b2fbd358068361bdce611710859ada341a1c631d566f367ebe6d6319", + "crates/markkit-js/Cargo.toml": "91fe5bd89dff4161f5d4a0c4251f897c989e4040d532453d200837f212c611dc", + "crates/markkit-js/__test__/options.test.cjs": "2ed01b68f702eedb9f3ae1a11127d41e1ad5bbff60251f86ae1e34deb9907f9a", + "crates/markkit-js/src/lib.rs": "298880fbe8928703393fbd0f91c6e42985546aa42e6129aa250a21d500643dff", + "crates/markkit-python/Cargo.toml": "85c464e50e25902ec70d956cf883e877ab705637fc84c9214fd05d49989f7f82", + "crates/markkit-python/src/lib.rs": "a9a0f8e182986a6ac2fc9e8b5db2cbf4965c90fc7d353d0d4c19b9f05de8b248", + "crates/markkit-python/tests/test_documents.py": "55c8fa60dd6662f8fc21ddeef9e6097cf76e9c0c8599c604aa801fe378753571", + "crates/markkit-wasm/Cargo.toml": "6b59cdb571c528ce4f362e5d0713e2ef0aab29eb928cd44579855cef2c9095cc", + "crates/markkit-wasm/__test__/markkit-wasm.test.mjs": "6edd60dae6a6bc165b6c78e9ef1d5134b21a5ab10f4509fe5e0953468612209b", + "crates/markkit-wasm/js/index.d.ts": "3f6c9853e1dd753ef940fa667827bff52d34a2707526ac243e6360082528c404", + "crates/markkit-wasm/src/lib.rs": "85791162e8d7862019c1b7da621a28ee4b2bb7789fafa04ccac1004eb72ad1d3", + "crates/markkit/Cargo.toml": "a4781629501bfb9e8991c8ca5653e10945ebaa9a49d634f5cc2816dff3bae980", + "crates/markkit/src/config.rs": "3e90b014d24a086c45bbfcf237d23cf719f95648357dcd3091271a31d10bcb0a", + "crates/markkit/src/extract.rs": "87b796ff60a4f995e6c6c308e480d132281f53983b9a76a66da021b935e2ed5e", + "crates/markkit/src/ir.rs": "b7a121efe1e498697698176bf2b779631f76a5daf511aebd74b6fa97c0b368b2", + "crates/markkit/src/layout.rs": "1979d867083314b2d5550bfe80b2c1a85f7da3a3bc5056af13379080b40bb623", + "crates/markkit/src/lib.rs": "9340d6d23b82b95e179bc6a1d2c2d461793e91a2f0769a333ab948c988932b2b", + "crates/markkit/src/limits.rs": "a197331637943ef907791697f1cb1c3270e35a6b5f7aed8480464bd01b00e0cc", + "crates/markkit/src/pdf/analyze.rs": "f27c781852039c574b72a6cc71594ac04562d39134794311b97e8dfb6252aa4d", + "crates/markkit/src/pdf/font.rs": "32c2569858d14ffb1cc6caf6419d79c25dce9c57bb33e1741a8714a728fdf02e", + "crates/markkit/src/pdf/guard.rs": "489d3bf291c63656ecd0181318dd4fffd7ac300e89ad029428814f3f2660d07e", + "crates/markkit/src/pdf/layout.rs": "8c9be884482158a6832989dab75936381cea955419d122e710e5b195478e88c6", + "crates/markkit/src/pdf/mod.rs": "95c0f19f28f49569c7f554166b67b99aa335eadc88c129b140cc7a1a75a3ada2", + "crates/markkit/src/pdf/ocr.rs": "2a60a407cc0fedee60681c6844a32983c19ff36c24b6f6ec759322c066b58a67", + "crates/markkit/src/pdf/ocr/rust.rs": "9b0ac6113b8720a38efd20894197e3e1853ac26d191823e3f445774c0de2f442", + "crates/markkit/src/pdf/tags.rs": "9e925845654113cdc2fecdf5a96f55377c8a23bd8443f9b70494a7dbde71cced", + "crates/markkit/tests/pdf_layout.rs": "9db16686713b59ae187bd315c8c47416bd7f67925750f80011c88f6f6cd2964a", + "crates/markkit/tests/pdf_ocr.rs": "dfab4ee65512422496b87166b64b54a6f46d72c28ba65aa15256e9fbabb3807c", + "crates/markkit/tests/pdf_structure.rs": "fd941762bbee191421f3eb790906e25a9b7d6ce49e5f55f8641595a0a5c54ca8", + "schema/ir.schema.json": "3763054cb70c7ce75c86b0ec2a70948cbee17d2e60613256ec63d697e8fe1860", + "scripts/compare_pdf_ocr.py": "69bd7d19bda3389eeeea8cb66f5606c10f56ee117b8ed195eb7ab02bb46b30e6", + "scripts/ocr_models.json": "8f093a6cf9ffcfd9c175f004e51340f513208674fa15fe6762ec0c48b9701179", + "scripts/prepare_ocr_models.py": "0b9b6735696983d6934ad29aedd87003dfe025bf13406a93c9e067cff43996df", + "scripts/tests/test_compare_pdf_ocr.py": "ba19316f39f1a92edbe0c841979a77d04d43eb3da620e989a34cd4cfd31956c3", + "scripts/tests/test_prepare_ocr_models.py": "35706f5905e587b045241c7eed7e9a7246e102936543d7a1f02517fd0cbd8025" + } +} diff --git a/crates/markkit-eval/results/pdf-layout-2026-10-28.json b/crates/markkit-eval/results/pdf-layout-2026-10-28.json new file mode 100644 index 0000000..02f2d7b --- /dev/null +++ b/crates/markkit-eval/results/pdf-layout-2026-10-28.json @@ -0,0 +1,522 @@ +{ + "date": "2026-10-28", + "scope": "PDF extraction: reading order, layout blocks/words/lines/cells, ruled/borderless/merged/wrapped tables, typography/lists, tagged roles/ActualText, repeated margin suppression, native OCR, all bindings", + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "source_dirty": true, + "baseline_binary_sha256": "0544cae4e9baed8c01a677d0e96fd7ed6f5221f743e0feddd493a65a1c0ca9b4", + "candidate_binary_sha256": "d36d3fe77edad5907b0038cfca0ee8f6c35ef6ce97f25bf91dcfa66f17a20919", + "public_regression": { + "cases": 178, + "regressions": [], + "before": { + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565, + "agentdocbench/overall": 0.7520402777777778, + "agentdocbench/no_ocr": 0.8204075757575757, + "agentdocbench/category/paper": 0.8231999999999999, + "agentdocbench/category/two-column": 0.7465666666666667, + "agentdocbench/category/report": 0.87095, + "agentdocbench/category/financial": 0.423325, + "agentdocbench/category/form": 0.7486666666666667, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/cjk": 0.749, + "agentdocbench/category/slides": 0.674775, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/html": 1.0, + "agentdocbench/format/pdf": 0.5835739130434783, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0 + }, + "after": { + "agentdocbench/category/cjk": 0.9335, + "agentdocbench/category/docx": 0.9988666666666667, + "agentdocbench/category/epub": 0.9891333333333333, + "agentdocbench/category/financial": 0.49095, + "agentdocbench/category/form": 0.8377666666666667, + "agentdocbench/category/html": 1.0, + "agentdocbench/category/paper": 0.8956333333333333, + "agentdocbench/category/report": 0.8705, + "agentdocbench/category/scanned": 0.0, + "agentdocbench/category/slides": 0.75135, + "agentdocbench/category/spreadsheet": 1.0, + "agentdocbench/category/two-column": 0.7586666666666667, + "agentdocbench/format/csv": 1.0, + "agentdocbench/format/docx": 0.9988666666666667, + "agentdocbench/format/epub": 0.9891333333333333, + "agentdocbench/format/html": 1.0, + "agentdocbench/format/pdf": 0.6553260869565217, + "agentdocbench/format/pptx": 0.9572, + "agentdocbench/format/xlsx": 1.0, + "agentdocbench/no_ocr": 0.8660333333333333, + "agentdocbench/overall": 0.7938638888888889, + "wcxb/default/overall": 0.6770940834789322, + "wcxb/default/type/article": 0.8593022688301003, + "wcxb/default/type/collection": 0.5618264586879532, + "wcxb/default/type/documentation": 0.849698642479727, + "wcxb/default/type/forum": 0.4559187546618804, + "wcxb/default/type/listing": 0.6660601098285603, + "wcxb/default/type/product": 0.5220558838570278, + "wcxb/default/type/service": 0.8247964660072756, + "wcxb/main_content/overall": 0.4999264653091694, + "wcxb/main_content/type/article": 0.9687139672138977, + "wcxb/main_content/type/collection": 0.484765233729096, + "wcxb/main_content/type/documentation": 0.7157843169028515, + "wcxb/main_content/type/forum": 0.10889386123735083, + "wcxb/main_content/type/listing": 0.33261257051933946, + "wcxb/main_content/type/product": 0.4767832465487646, + "wcxb/main_content/type/service": 0.41193206101288565 + }, + "pdf_before": 0.5835739130434783, + "pdf_after": 0.6553260869565217, + "provenance": { + "binary_sha256": "d36d3fe77edad5907b0038cfca0ee8f6c35ef6ce97f25bf91dcfa66f17a20919", + "build_profile": "release", + "command": [ + "/workspace/markkit/target/public-benchmarks/bin/mira", + "run", + "--study-python", + "scripts/public_benchmarks/study.py", + "--max-concurrent", + "1", + "--max-retries", + "0", + "--timeout", + "300", + "--group-by", + "category", + "--format", + "json", + "--out", + "/workspace/markkit/work/public-final/mira.json" + ], + "core_diff_sha256": "24fd2301f0fe1e96c824b3e12caaac5464aca8fdd0ad5a102948e1f68c477399", + "cpu_count": 5, + "machine": "x86_64", + "mira_version": "mira 0.5.0", + "pins": { + "agentdocbench": "e97ccc4a2be9c9a71289c4e000e60e953daccaa3", + "mira": "v0.5.0", + "mira_linux_archive_sha256": "b69d1a9af34d83aae87aad2453cf642739de9e0859e043b2b6571839ef7462fc", + "mira_sdk": "334506ef19a1d975dace0f2c51092548a10c7b3e", + "samples_per_type": 10, + "seed": "markkit-public-pilot-v1:", + "wcxb": "c039d5ee9f5a3a984a0e167e63aacd04e76e78a9" + }, + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "python": "3.12.14", + "rustc_version": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "source_dirty": true, + "source_revision": "7a4e62f0497deee864f39600e5a36a3d348260bf", + "timing_boundary": "median of 3 fresh CLI processes after 1 warm-up; sequential; includes startup" + } + }, + "pdf_scenes": { + "cases": 40, + "samples": 20, + "passed_cases": 40, + "modes": { + "default": { + "cases": 20, + "metrics": { + "text_fidelity": { + "passed": 20, + "scored": 20, + "score": 1.0 + }, + "layout_geometry": { + "passed": 0, + "scored": 0, + "score": null + }, + "markdown_parity": { + "passed": 20, + "scored": 20, + "score": 1.0 + } + } + }, + "layout": { + "cases": 20, + "metrics": { + "text_fidelity": { + "passed": 20, + "scored": 20, + "score": 1.0 + }, + "layout_geometry": { + "passed": 20, + "scored": 20, + "score": 1.0 + }, + "markdown_parity": { + "passed": 20, + "scored": 20, + "score": 1.0 + } + } + } + }, + "expected_exit_status": 0 + }, + "public_pdf_layout": { + "documents": 23, + "pages": 243, + "spans": 183244, + "blocks": 11264, + "cases": [ + { + "id": "paper-evsched", + "pages": 23, + "spans": 12924, + "blocks": 310, + "tables": 14, + "markdown_unchanged": true + }, + { + "id": "paper-rankone", + "pages": 27, + "spans": 35955, + "blocks": 2302, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "paper-ulam", + "pages": 11, + "spans": 21304, + "blocks": 287, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "twocol-bygpt5", + "pages": 16, + "spans": 16572, + "blocks": 542, + "tables": 3, + "markdown_unchanged": true + }, + { + "id": "twocol-smile", + "pages": 13, + "spans": 11879, + "blocks": 733, + "tables": 9, + "markdown_unchanged": true + }, + { + "id": "twocol-uscore", + "pages": 17, + "spans": 16510, + "blocks": 502, + "tables": 13, + "markdown_unchanged": true + }, + { + "id": "report-crs-dod-approps", + "pages": 12, + "spans": 10842, + "blocks": 397, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "report-plz", + "pages": 5, + "spans": 5502, + "blocks": 52, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "fed-h8", + "pages": 22, + "spans": 10785, + "blocks": 2841, + "tables": 11, + "markdown_unchanged": true + }, + { + "id": "fin-bls-realer", + "pages": 5, + "spans": 6722, + "blocks": 571, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "fin-census-income", + "pages": 10, + "spans": 7424, + "blocks": 743, + "tables": 6, + "markdown_unchanged": true + }, + { + "id": "invoice-borderless", + "pages": 2, + "spans": 113, + "blocks": 42, + "tables": 1, + "markdown_unchanged": true + }, + { + "id": "form-1040es", + "pages": 12, + "spans": 5334, + "blocks": 631, + "tables": 5, + "markdown_unchanged": true + }, + { + "id": "form-w4", + "pages": 4, + "spans": 1739, + "blocks": 189, + "tables": 11, + "markdown_unchanged": true + }, + { + "id": "irs-w9", + "pages": 6, + "spans": 1074, + "blocks": 199, + "tables": 2, + "markdown_unchanged": true + }, + { + "id": "scan-nasa-budget", + "pages": 3, + "spans": 0, + "blocks": 0, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "scan-nasa-notes", + "pages": 3, + "spans": 0, + "blocks": 0, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "scan-oklahoma-address", + "pages": 7, + "spans": 0, + "blocks": 0, + "tables": 0, + "markdown_unchanged": true + }, + { + "id": "cjk-jp-stat-elderly", + "pages": 4, + "spans": 2797, + "blocks": 190, + "tables": 6, + "markdown_unchanged": true + }, + { + "id": "cjk-tw-archives", + "pages": 4, + "spans": 2385, + "blocks": 15, + "tables": 3, + "markdown_unchanged": true + }, + { + "id": "cjk-tw-dgbas-cpi", + "pages": 6, + "spans": 3614, + "blocks": 368, + "tables": 1, + "markdown_unchanged": true + }, + { + "id": "slides-nasa-clarreo", + "pages": 15, + "spans": 5715, + "blocks": 160, + "tables": 3, + "markdown_unchanged": true + }, + { + "id": "slides-nist-pqc-update", + "pages": 16, + "spans": 4054, + "blocks": 190, + "tables": 3, + "markdown_unchanged": true + } + ] + }, + "ocr": { + "documents": 3, + "baseline_mean_document_score": 0.0, + "mean_document_score": 0.5233666666666666, + "cases": [ + { + "id": "scan-nasa-budget", + "scores": { + "document_score": 0.353, + "sentences": 0.2, + "order": 0.6667, + "table_cells_f1": 0.1923 + }, + "seconds": 21.30685543400068, + "spans": 833, + "binary_sha256": "d36d3fe77edad5907b0038cfca0ee8f6c35ef6ce97f25bf91dcfa66f17a20919" + }, + { + "id": "scan-nasa-notes", + "scores": { + "document_score": 0.3838, + "sentences": 0.3333333333333333, + "order": 0.8182, + "table_cells_f1": 0.0 + }, + "seconds": 6.632421336999869, + "spans": 767, + "binary_sha256": "d36d3fe77edad5907b0038cfca0ee8f6c35ef6ce97f25bf91dcfa66f17a20919" + }, + { + "id": "scan-oklahoma-address", + "scores": { + "document_score": 0.8333, + "sentences": 0.6666666666666666, + "order": 1.0 + }, + "seconds": 33.16324986499967, + "spans": 3175, + "binary_sha256": "d36d3fe77edad5907b0038cfca0ee8f6c35ef6ce97f25bf91dcfa66f17a20919" + } + ], + "configuration": "--ocr --timeout 300 --json --layout; installed Poppler/Tesseract eng; 1800px grayscale; one OCR thread" + }, + "validation": { + "workspace_all_features": { + "passed": 812, + "failed": 0, + "ignored": 1 + }, + "workspace_default": { + "passed": 810, + "failed": 0, + "ignored": 1 + }, + "core_no_default_features": { + "passed": 149, + "failed": 0, + "ignored": 0 + }, + "ocr_helpers_explicit": { + "passed": 2, + "failed": 0, + "ignored": 0 + }, + "node": { + "passed": 10, + "failed": 0 + }, + "wasm": { + "passed": 23, + "failed": 0 + }, + "python": { + "passed": 2, + "failed": 0 + }, + "clippy": "passed, workspace/all targets/all features and PDF-only no-default build, -D warnings", + "rustdoc": "passed with -D warnings", + "format": "passed", + "knowledge": "OKF v0.2 conformant", + "script_tests": { + "run": 34, + "skipped": 2, + "failed": 0 + } + }, + "local_test_environment": { + "python_library": "LIBRARY_PATH and LD_LIBRARY_PATH point to the installed bundled Python 3.12 library", + "storage": "CARGO_INCREMENTAL=0 CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 after discarding disposable debug caches; no source changes for the storage workaround" + }, + "limits_and_scope": [ + "Layout is optional; source runs remain in drawing order and semantic blocks use reading order.", + "Table inference is conservative; nonrectangular/irregular or ambiguous structures can remain text.", + "Native OCR is opt-in for textless pages; requires installed helpers/languages. WASM extracts text layers/layout/tables.", + "OCR helpers run without a shell/network, with private working directories, deadline/size checks and kill/reap on timeout. Host process isolation/memory quotas govern native decoders.", + "Geometry is approximate font em bounds or recognized OCR word bounds." + ], + "changed_source_sha256": { + "Cargo.lock": "0d236f92a2230972c197d11ff2498a42c058a69e855abdddcea96bf7b7e9e76c", + "crates/markkit-cli/Cargo.toml": "00ca964fa09dbb701e54dd43895927bd193877dccb85664352e6f15f68c61de0", + "crates/markkit-cli/src/main.rs": "7afe10c34394d332b051f59925b94581735de7504b5ca9ba83501b32789a2bd8", + "crates/markkit-cli/tests/cli_integration.rs": "7cecfb9131c1cc446bdc3438a448cd630b1b0d440e7fa26641dcb47177ce5aad", + "crates/markkit-eval/Cargo.toml": "903b131f5716708491924ea56af529e681a980c499223d909cab1b93e43fee7d", + "crates/markkit-eval/src/lib.rs": "ee1e9ac8ec5113fbb2b0a2cea6948e07905d9cc0d021c49ee212fc3f51ddf843", + "crates/markkit-eval/src/main.rs": "feb406d2f02b3fc35efd7a0647780dc7c66656852da1219e24b17f848f70720e", + "crates/markkit-js/Cargo.toml": "8c1465370c370169f46de4d689ee7a65d49cc2a6d021ae9f24df4b75aaa773e6", + "crates/markkit-js/__test__/options.test.cjs": "551806085f99d27176e798eeb224c6d99828fd446573ca2641b2896b7df5851f", + "crates/markkit-js/src/lib.rs": "d95f076585da9f14ef29d8e622ddb3566ce6db46655c7570208807bd6275b316", + "crates/markkit-python/Cargo.toml": "60a08f0895531f9f4e47b9df5058c1f48b202449256f587c6f2b802d3295b86f", + "crates/markkit-python/src/lib.rs": "bed7b5ba5ec910e4561117599f88e83a9ff054d6555cc15012e28d0c3548326c", + "crates/markkit-wasm/Cargo.toml": "6b59cdb571c528ce4f362e5d0713e2ef0aab29eb928cd44579855cef2c9095cc", + "crates/markkit-wasm/__test__/markkit-wasm.test.mjs": "6edd60dae6a6bc165b6c78e9ef1d5134b21a5ab10f4509fe5e0953468612209b", + "crates/markkit-wasm/js/index.d.ts": "3f6c9853e1dd753ef940fa667827bff52d34a2707526ac243e6360082528c404", + "crates/markkit-wasm/src/lib.rs": "85791162e8d7862019c1b7da621a28ee4b2bb7789fafa04ccac1004eb72ad1d3", + "crates/markkit/Cargo.toml": "a4b5c413cf6268a9d77061755b1ae434e5ce616838c396280359fd7c0eb1038f", + "crates/markkit/src/config.rs": "c478f07db574ac09174a5f38b631ae41dc406527686c7379726f7fc1ae4464b4", + "crates/markkit/src/extract.rs": "87b796ff60a4f995e6c6c308e480d132281f53983b9a76a66da021b935e2ed5e", + "crates/markkit/src/ir.rs": "b7a121efe1e498697698176bf2b779631f76a5daf511aebd74b6fa97c0b368b2", + "crates/markkit/src/lib.rs": "757a2db4474403bc2b86a4f005dcf542ffd7b6a76e7682343565d91dbae46361", + "crates/markkit/src/limits.rs": "a197331637943ef907791697f1cb1c3270e35a6b5f7aed8480464bd01b00e0cc", + "crates/markkit/src/pdf/font.rs": "32c2569858d14ffb1cc6caf6419d79c25dce9c57bb33e1741a8714a728fdf02e", + "crates/markkit/src/pdf/mod.rs": "0b6f4c278e0239164ba87e93608bf1944a03779d38cb38cbfe3a53be9b30a07f", + "schema/ir.schema.json": "3763054cb70c7ce75c86b0ec2a70948cbee17d2e60613256ec63d697e8fe1860", + "crates/markkit-eval/data/pdf-layout.json": "38f5ac20041441fad3f6efcbe5a2f8ca82d408339c953971ccf036ebf856d42b", + "crates/markkit-eval/src/pdf.rs": "43bc3ee3b2fbd358068361bdce611710859ada341a1c631d566f367ebe6d6319", + "crates/markkit-python/tests/test_documents.py": "8bad0e0698412f3f26c0af583b16e2dbfb23f04aeaf5c2892875f21183b1b243", + "crates/markkit/src/layout.rs": "1979d867083314b2d5550bfe80b2c1a85f7da3a3bc5056af13379080b40bb623", + "crates/markkit/src/pdf/analyze.rs": "f27c781852039c574b72a6cc71594ac04562d39134794311b97e8dfb6252aa4d", + "crates/markkit/src/pdf/layout.rs": "8c9be884482158a6832989dab75936381cea955419d122e710e5b195478e88c6", + "crates/markkit/src/pdf/ocr.rs": "193274365632f5ce4464a5d6e285a7f7f46ab532026c72697e6b0b8b3057763b", + "crates/markkit/src/pdf/tags.rs": "9e925845654113cdc2fecdf5a96f55377c8a23bd8443f9b70494a7dbde71cced", + "crates/markkit/tests/pdf_layout.rs": "9db16686713b59ae187bd315c8c47416bd7f67925750f80011c88f6f6cd2964a", + "crates/markkit/tests/pdf_ocr.rs": "b999410a30f5bb90613d58a6e34a2cdb0fc76d6ad67eaf0dcfb7174c7302a24e", + "crates/markkit/tests/pdf_structure.rs": "fd941762bbee191421f3eb790906e25a9b7d6ce49e5f55f8641595a0a5c54ca8" + }, + "commands": [ + "cargo test --workspace", + "cargo test --workspace --all-features", + "cargo test -p markkit --no-default-features", + "cargo test -p markkit --features pdf-ocr --test pdf_ocr -- --include-ignored", + "cargo clippy --workspace --all-targets --all-features -- -D warnings", + "cargo clippy -p markkit --no-default-features --features pdf --all-targets -- -D warnings", + "just eval-pdf --max-concurrent 1 --format json --out work/pdf-complete.json", + "target/public-benchmarks/venv/bin/python scripts/public_benchmarks/run.py --out work/public-final", + "target/public-benchmarks/venv/bin/python work/final_pdf_validation.py", + "node --test crates/markkit-wasm/__test__/*.test.mjs", + "cd crates/markkit-js && node --test __test__/*.test.cjs", + "python3 -m unittest discover -s crates/markkit-python/tests" + ] +} diff --git a/crates/markkit-eval/src/lib.rs b/crates/markkit-eval/src/lib.rs index d408210..c66b4f7 100644 --- a/crates/markkit-eval/src/lib.rs +++ b/crates/markkit-eval/src/lib.rs @@ -12,5 +12,6 @@ pub mod checks; pub mod dataset; pub mod mira_study; +pub mod pdf; pub mod provider; pub mod qa; diff --git a/crates/markkit-eval/src/main.rs b/crates/markkit-eval/src/main.rs index b1688a4..2f0385c 100644 --- a/crates/markkit-eval/src/main.rs +++ b/crates/markkit-eval/src/main.rs @@ -18,6 +18,11 @@ fn markkit_extract() -> Eval { mira_study::extract_eval() } +#[eval] +fn markkit_pdf() -> Eval { + markkit_eval::pdf::pdf_eval() +} + #[eval] fn markkit_qa() -> Eval { markkit_eval::qa::qa_eval() diff --git a/crates/markkit-eval/src/pdf.rs b/crates/markkit-eval/src/pdf.rs new file mode 100644 index 0000000..35f408c --- /dev/null +++ b/crates/markkit-eval/src/pdf.rs @@ -0,0 +1,263 @@ +//! PDF text, layout and table eval with generated, inspectable content streams. +//! +//! Decision: expected coordinates and text are authored in the dataset, not +//! computed by the parser. Both modes measure ideal text fidelity; `layout` +//! additionally measures geometry. Column/table cases check spatial reconstruction +//! and table structure. Missing/failed conversions score zero, never N/A. + +use std::time::Instant; + +use markkit::{DocumentHint, Markkit}; +use mira::subject::subject_fn; +use mira::{Eval, Sample, Score, Transcript}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct Fixture { + id: String, + category: String, + pages: Vec, + markdown: String, + layout: Value, +} +#[derive(Debug, Clone, Deserialize, Serialize)] +struct Page { + attrs: String, + content: String, +} + +const DATA: &str = include_str!("../data/pdf-layout.json"); + +fn fixtures() -> Vec { + serde_json::from_str(DATA).expect("tested embedded PDF dataset") +} + +// Minimal valid PDF writer, independent of hayro/markkit internals. Helvetica +// omits widths intentionally: its documented 0.5 em fallback makes scenes easy +// to inspect. Each page may override inherited MediaBox / Rotate / CropBox. +fn pdf(f: &Fixture) -> Vec { + let kids: Vec<_> = (0..f.pages.len()) + .map(|i| format!("{} 0 R", 4 + i * 2)) + .collect(); + let mut objects = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + format!( + "<< /Type /Pages /Kids [{}] /Count {} /MediaBox [0 0 612 792] >>", + kids.join(" "), + f.pages.len() + ), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + ]; + for (i, p) in f.pages.iter().enumerate() { + objects.push(format!("<< /Type /Page /Parent 2 0 R {} /Resources << /Font << /F1 3 0 R >> >> /Contents {} 0 R >>", p.attrs, 5 + i * 2)); + objects.push(format!( + "<< /Length {} >>\nstream\n{}\nendstream", + p.content.len(), + p.content + )); + } + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = vec![0]; + for (i, o) in objects.iter().enumerate() { + offsets.push(out.len()); + out.extend(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend(format!("xref\n0 {}\n0000000000 65535 f \n", offsets.len()).as_bytes()); + for offset in &offsets[1..] { + out.extend(format!("{offset:010} 00000 n \n").as_bytes()); + } + out.extend( + format!( + "trailer << /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", + offsets.len() + ) + .as_bytes(), + ); + out +} + +/// Match expected geometry fields with a 0.001 point tolerance. Array lengths +/// must match, so extra/missing pages or text runs cannot improve the score. +fn matches(expected: &Value, actual: &Value) -> bool { + match (expected, actual) { + (Value::Object(e), Value::Object(a)) => e + .iter() + .all(|(k, v)| a.get(k).is_some_and(|a| matches(v, a))), + (Value::Array(e), Value::Array(a)) => { + e.len() == a.len() && e.iter().zip(a).all(|(e, a)| matches(e, a)) + } + (Value::Number(e), Value::Number(a)) => e + .as_f64() + .zip(a.as_f64()) + .is_some_and(|(e, a)| (e - a).abs() < 0.001), + _ => expected == actual, + } +} + +fn transcript(f: &Fixture, layout: bool) -> Transcript { + let bytes = pdf(f); + let start = Instant::now(); + let result = Markkit::builder() + .pdf_layout(layout) + .build() + .extract_document_sync(&bytes, DocumentHint::default()); + let elapsed = start.elapsed(); + let e = match result { + Ok(e) => e, + Err(e) => return Transcript::failed(format!("PDF extraction failed: {e}")), + }; + let mut t = Transcript::response(serde_json::to_string(&e).expect("serializable extraction")); + t.record_metric("extract_us", elapsed.as_micros() as f64); + t.record_metric("input_bytes", bytes.len() as f64); + t.record_metric("output_bytes", t.final_response.len() as f64); + t.record_metric("layout_requested", if layout { 1.0 } else { 0.0 }); + t +} + +fn score(name: &str, sample: &Sample, t: &Transcript) -> Score { + if t.errored_infra() { + return Score::na(name, "subject infra error"); + } + let fixture = fixtures() + .into_iter() + .find(|f| f.id == sample.id) + .expect("sample in embedded dataset"); + let actual: Value = serde_json::from_str(&t.final_response).unwrap_or(Value::Null); + if name == "layout_geometry" + && t.error.is_none() + && actual["markdown"].is_string() + && t.metrics.get("layout_requested") == Some(&0.0) + { + return Score::na(name, "layout capture disabled in default mode"); + } + let passed = match name { + "text_fidelity" => actual["markdown"].as_str() == Some(fixture.markdown.as_str()), + "layout_geometry" => matches(&fixture.layout, &actual["layout"]), + "markdown_parity" => { + let plain = transcript(&fixture, false); + let plain: Value = serde_json::from_str(&plain.final_response).unwrap_or(Value::Null); + actual["markdown"].is_string() && actual["markdown"] == plain["markdown"] + } + _ => false, + }; + Score { + scorer: name.into(), + value: if passed { 1.0 } else { 0.0 }, + pass: passed, + na: false, + reason: if passed { + "matched authored expectation".into() + } else { + format!("{name} expectation failed for {}", fixture.id) + }, + } +} + +/// Offline Mira study: compare default extraction with optional layout capture. +pub fn pdf_eval() -> Eval { + let mut b = Eval::new("markkit_pdf") + .describe("PDF text fidelity, positioned geometry, table structure and Markdown parity") + .subject(subject_fn(|sample: Sample, cx| async move { + let mode = cx.param("mode").unwrap_or("default"); + if !["default", "layout"].contains(&mode) { + return Transcript::infra_error("unknown PDF mode"); + } + let Some(f) = fixtures().into_iter().find(|f| f.id == sample.id) else { + return Transcript::infra_error("missing PDF fixture"); + }; + transcript(&f, mode == "layout") + })) + .axis("mode", ["default", "layout"]) + .target(crate::mira_study::markkit_target()) + .max_turns(1); + for name in ["text_fidelity", "layout_geometry", "markdown_parity"] { + b = b.scorer(mira::scorer::scorer( + name, + move |s: &Sample, t: &Transcript| score(name, s, t), + )); + } + for f in fixtures() { + b = b.add_sample( + Sample::new(&f.id, format!("PDF {} scene", f.category)) + .tag("pdf") + .tag(&f.category) + .file("scene.json", serde_json::to_string_pretty(&f).unwrap()), + ); + } + b.build() +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn fixtures_are_valid_and_cover_geometry_and_known_gaps() { + let fs = fixtures(); + assert!(fs.len() >= 10); + let ids: std::collections::HashSet<_> = fs.iter().map(|f| &f.id).collect(); + assert_eq!(ids.len(), fs.len()); + for category in ["rotation", "crop", "spacing", "columns", "table", "empty"] { + assert!( + fs.iter().any(|f| f.category == category), + "missing {category}" + ); + } + let _ = pdf_eval(); + for f in fs { + let t = transcript(&f, true); + assert!(t.error.is_none(), "{}: {:?}", f.id, t.error); + let actual: Value = serde_json::from_str(&t.final_response).unwrap(); + assert!( + matches(&f.layout, &actual["layout"]), + "{} geometry mismatch: {}", + f.id, + actual["layout"] + ); + } + } + #[test] + fn geometry_scorer_rejects_missing_extra_wrong_and_non_numeric_values() { + let expected = + serde_json::json!({"pages":[{"width":612.0,"spans":[{"text":"Hi","bbox":[1,2,3,4]}]}]}); + assert!(matches(&expected, &expected)); + assert!(!matches(&expected, &Value::Null)); + for patch in [ + serde_json::json!(null), + serde_json::json!("612"), + serde_json::json!(613), + ] { + let mut actual = expected.clone(); + actual["pages"][0]["width"] = patch; + assert!(!matches(&expected, &actual)); + } + let mut actual = expected.clone(); + actual["pages"][0]["spans"] = serde_json::json!([]); + assert!(!matches(&expected, &actual)); + let mut actual = expected.clone(); + actual["pages"].as_array_mut().unwrap().push(Value::Null); + assert!(!matches(&expected, &actual)); + } + #[test] + fn extraction_failure_is_scored_zero() { + let sample = Sample::new(&fixtures()[0].id, "failure"); + for name in ["text_fidelity", "layout_geometry", "markdown_parity"] { + let s = score(name, &sample, &Transcript::failed("malformed")); + assert!(!s.na && !s.pass && s.value == 0.0); + } + } + + #[test] + fn disabled_layout_is_na_but_missing_requested_geometry_scores_zero() { + let f = &fixtures()[0]; + let sample = Sample::new(&f.id, "optional geometry"); + assert!(score("layout_geometry", &sample, &transcript(f, false)).na); + let mut requested = transcript(f, true); + let mut actual: Value = serde_json::from_str(&requested.final_response).unwrap(); + actual.as_object_mut().unwrap().remove("layout"); + requested.final_response = actual.to_string(); + let s = score("layout_geometry", &sample, &requested); + assert!(!s.na && !s.pass && s.value == 0.0); + } +} diff --git a/crates/markkit-js/Cargo.toml b/crates/markkit-js/Cargo.toml index cf99468..4c4147c 100644 --- a/crates/markkit-js/Cargo.toml +++ b/crates/markkit-js/Cargo.toml @@ -13,8 +13,11 @@ description = "Node.js bindings for Markkit markdown extraction via NAPI-RS" crate-type = ["cdylib"] doc = false +[features] +pdf-ocr-ocrs = ["markkit/pdf-ocr-ocrs"] + [dependencies] -markkit = { path = "../markkit", features = ["all_formats", "http_client"] } +markkit = { path = "../markkit", features = ["all_formats", "pdf-ocr", "http_client"] } napi.workspace = true napi-derive.workspace = true diff --git a/crates/markkit-js/README.md b/crates/markkit-js/README.md index 3014518..02491d7 100644 --- a/crates/markkit-js/README.md +++ b/crates/markkit-js/README.md @@ -1,6 +1,6 @@ # @everruns/markkit -Native Node.js bindings for Markkit: HTML, DOCX and spreadsheets to Markdown. +Native Node.js bindings for Markkit: HTML, PDF, DOCX and spreadsheets to Markdown. This npm release supports **Linux x64 with glibc**. It does not include binaries for macOS, Windows, ARM64 or musl Linux. Other platforms require a separate future @@ -28,17 +28,61 @@ await extractHtml(html, { includeSelectors: ['article'], excludeSelectors: ['.ad // URLs: non-public addresses are blocked by default. await extractUrl('https://docs.example.com/page', { allowedHosts: ['docs.example.com'], maxRedirects: 2 }); -// Documents with metadata (HTML, DOCX, PPTX, XLSX/XLSB/XLS/ODS). `warnings` are `{ code, detail }`. +// Documents with metadata (HTML, PDF, DOCX, PPTX, XLSX/XLSB/XLS/ODS). `warnings` are `{ code, detail }`. const doc = extractDocumentSync(fs.readFileSync('report.docx'), { fileName: 'report.docx' }, { maxSheetRows: 500 }); console.log(doc.title, doc.format, doc.warnings, doc.markdown); ``` Fields: `maxInputSize`, `maxOutputSize`, `timeout` (seconds, covers fetch, parsing and rendering; `maxProcessingSecs` is the whole-second form), `maxNodes`, `maxNestingDepth`, -`baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, +`baseUrl`, `mainContent`, `pdfLayout`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors` (added to the defaults), `maxArchiveEntries`, `maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, `maxSpreadsheetCells`, `includeHiddenSheets`, `allowPrivateNetworks` (trusted URLs only), `allowedHosts`, `maxRedirects`, `maxPdfPages`, `maxPdfStreamSize`, `maxPdfDecodedSize`. For browsers and edge runtimes use [`@everruns/markkit-wasm`](https://www.npmjs.com/package/@everruns/markkit-wasm). + +## PDF layout and tables + +PDF text extraction reconstructs columns, ruled and conservative borderless tables, +including merged and wrapped cells. Tagged headings/ActualText, bold/italic and +numbered lists are preserved. Set `pdfRemoveRepeatedHeaders: true` to suppress +repeated margin headers/footers. + +`pdfLayout: true` adds `layout: { version: 1, pages }` to document results. Source +`spans` retain drawing order and include glyph-positioned `words`. Semantic `blocks` +are in reading order, with `id`, `kind`, `bbox`, `spanIds`, `lines` and table `cells`. +Cells have row/column positions, rowspan/colspan and source references. Coordinates +are display points from the cropped/rotated top-left, x right and y down. Font +bounds approximate an em box. Layout is optional and does not change Markdown. + +Node and WASM use camelCase keys. `maxNodes` bounds primitives and geometry; +`maxPdfDecodedSize` bounds retained geometry and analysis work; table row/column/ +grid limits apply. `maxOutputSize` independently bounds compact layout JSON and +Markdown, excluding JSON wrapper/pretty-print overhead. + +Native OCR: install Tesseract plus language data. Use +`{ pdfOcr: true, pdfOcrLanguage: "eng", timeout: 300 }` on `extractDocumentSync` / +`extractDocument`. Only textless pages are recognized. OCR word bounds are returned +in layout; helpers use private files, bounded dimensions/output and the deadline. + +## Optional Rust OCR backend + +Source builds can enable `markkit-js`'s Cargo feature `pdf-ocr-ocrs`. Default npm +builds retain Tesseract. Prepare the pinned models from the repository with +`python3 scripts/prepare_ocr_models.py`, then configure: + +```js +extractDocumentSync(bytes, { fileName: "scan.pdf" }, { + pdfOcr: true, pdfOcrBackend: "ocrs", timeout: 300, + pdfOcrDetectionModel: "/models/text-detection.onnx", + pdfOcrRecognitionModel: "/models/text-recognition.onnx" +}); +``` + +Models load from explicit local paths only. The pinned weights use CC-BY-SA 4.0 +and carry attribution in the setup script's NOTICE.json. The current Latin +model accepts `pdfOcrLanguage: "eng"`. Missing build features/models and other +languages fail explicitly. Model files and inference need trusted deployment +configuration and host CPU/memory quotas; inference deadlines are cooperative. diff --git a/crates/markkit-js/__test__/options.test.cjs b/crates/markkit-js/__test__/options.test.cjs index 35eb814..12fe0b1 100644 --- a/crates/markkit-js/__test__/options.test.cjs +++ b/crates/markkit-js/__test__/options.test.cjs @@ -86,3 +86,39 @@ test("extractUrl blocks private networks unless allowed", async () => { srv.close(); } }); + +test("PDF source geometry is opt-in and obeys layout limits", () => { + const content = "BT /F1 10 Tf 10 100 Td (Hi) Tj ET"; + const source = `%PDF-1.7 +1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj +2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj +3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >> endobj +4 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj +5 0 obj << /Length ${content.length} >> stream +${content} +endstream endobj +trailer << /Root 1 0 R >> +%%EOF`; + const data = Buffer.from(source); + const hint = { fileName: "a.pdf" }; + const plain = m.extractDocumentSync(data, hint); + assert.equal(plain.markdown, "Hi\n"); + assert.ok(plain.layout == null); + const result = m.extractDocumentSync(data, hint, { pdfLayout: true }); + assert.equal(result.markdown, plain.markdown); + assert.equal(result.layout.pages[0].width, 300); + assert.equal(result.layout.pages[0].blocks[0].kind, "paragraph"); + assert.deepEqual(result.layout.pages[0].blocks[0].spanIds, [0]); + assert.equal(result.layout.pages[0].spans[0].words[0].text, "Hi"); + assert.equal(result.layout.pages[0].height, 400); + assert.equal(result.layout.pages[0].spans[0].text, "Hi"); + assert.deepEqual(result.layout.pages[0].spans[0].baseline, [[10, 300], [20, 300]]); + assert.throws(() => m.extractDocumentSync(data, hint, { pdfLayout: true, maxNodes: 1 }), /max_nodes/); + assert.throws(() => m.extractDocumentSync(data, hint, { pdfLayout: true, maxOutputSize: 50 }), /max_output_size/); +}); + + +test("OCR backend configuration validates names", () => { + assert.throws(() => m.extractHtmlSync("

Hi

", { pdfOcrBackend: "unknown" }), /tesseract or ocrs/); + assert.equal(m.extractHtmlSync("

Hi

", { pdfOcrBackend: "ocrs", pdfOcrDetectionModel: "unused.onnx", pdfOcrRecognitionModel: "unused.onnx" }), "Hi\n"); +}); diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 5116dce..d6e93ce 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -33,6 +33,14 @@ pub struct MarkkitOptions { pub base_url: Option, /// Keep only the detected main content (article body) of HTML pages. pub main_content: Option, + /// Retain PDF page dimensions and positioned text runs in `Extraction.layout`. + pub pdf_layout: Option, + pub pdf_remove_repeated_headers: Option, + pub pdf_ocr: Option, + pub pdf_ocr_language: Option, + pub pdf_ocr_backend: Option, + pub pdf_ocr_detection_model: Option, + pub pdf_ocr_recognition_model: Option, pub preserve_links: Option, pub preserve_images: Option, /// CSS selectors to include (empty = whole document). @@ -100,6 +108,7 @@ pub struct Warning { /// Markdown plus metadata. #[napi(object)] pub struct Extraction { + pub layout: Option, pub markdown: String, pub title: Option, /// Detected format (`html`, `docx`, `pptx`, `xlsx`, `xlsb`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`, `eml`, `odt`, `epub`). @@ -109,6 +118,75 @@ pub struct Extraction { pub metadata: Metadata, } +/// PDF source geometry; coordinates are display points, x right and y down. +#[napi(object)] +pub struct PdfLayout { + pub version: u32, + pub pages: Vec, +} + +#[napi(object)] +pub struct PdfPageLayout { + pub page: u32, + pub width: f64, + pub height: f64, + pub rotation: u32, + pub spans: Vec, + pub blocks: Vec, +} + +#[napi(object)] +pub struct PdfTextSpan { + pub id: u32, + pub text: String, + /// Approximate em bounds: [left, top, right, bottom]. + pub bbox: Vec, + pub baseline: Vec>, + pub font: Option, + pub size: f64, + pub angle: f64, + pub words: Vec, + pub bold: bool, + pub italic: bool, + pub role: Option, + pub ocr: bool, +} + +#[napi(object)] +pub struct PdfWord { + pub text: String, + pub bbox: Vec, + pub baseline: Vec>, +} +#[napi(object)] +pub struct PdfLine { + pub bbox: Vec, + pub span_ids: Vec, +} +#[napi(object)] +pub struct PdfCell { + pub row: u32, + pub column: u32, + pub rowspan: u32, + pub colspan: u32, + pub bbox: Vec, + pub span_ids: Vec, +} +#[napi(object)] +pub struct PdfBlock { + pub id: u32, + pub kind: String, + pub bbox: Vec, + pub span_ids: Vec, + pub lines: Vec, + pub cells: Vec, +} +fn js_ids(ids: Vec) -> Vec { + ids.into_iter() + .map(|n| u32::try_from(n).unwrap_or(u32::MAX)) + .collect() +} + fn invalid(field: &str) -> napi::Error { napi::Error::from_reason(format!( "invalid options: `{field}` must be a non-negative integer" @@ -164,6 +242,27 @@ fn markkit(options: Option) -> napi::Result { if let Some(v) = o.main_content { b = b.main_content(v); } + if let Some(v) = o.pdf_remove_repeated_headers { + b = b.pdf_remove_repeated_headers(v); + } + if let Some(v) = o.pdf_ocr { + b = b.pdf_ocr(v); + } + if let Some(v) = o.pdf_ocr_backend { + b = b.pdf_ocr_backend(v.parse().map_err(napi::Error::from_reason)?); + } + if let Some(v) = o.pdf_ocr_detection_model { + b = b.pdf_ocr_detection_model(v); + } + if let Some(v) = o.pdf_ocr_recognition_model { + b = b.pdf_ocr_recognition_model(v); + } + if let Some(v) = o.pdf_ocr_language { + b = b.pdf_ocr_language(v); + } + if let Some(v) = o.pdf_layout { + b = b.pdf_layout(v); + } if let Some(v) = o.preserve_links { b = b.preserve_links(v); } @@ -228,6 +327,75 @@ fn count(v: Option) -> Option { fn to_js(e: markkit::Extraction) -> Extraction { Extraction { + layout: e.layout.map(|l| PdfLayout { + version: l.version, + pages: l + .pages + .into_iter() + .map(|p| PdfPageLayout { + page: p.page, + width: p.width, + height: p.height, + rotation: p.rotation.into(), + blocks: p + .blocks + .into_iter() + .map(|b| PdfBlock { + id: b.id as u32, + kind: b.kind, + bbox: b.bbox.to_vec(), + span_ids: js_ids(b.span_ids), + lines: b + .lines + .into_iter() + .map(|l| PdfLine { + bbox: l.bbox.to_vec(), + span_ids: js_ids(l.span_ids), + }) + .collect(), + cells: b + .cells + .into_iter() + .map(|c| PdfCell { + row: c.row as u32, + column: c.column as u32, + rowspan: c.rowspan, + colspan: c.colspan, + bbox: c.bbox.to_vec(), + span_ids: js_ids(c.span_ids), + }) + .collect(), + }) + .collect(), + spans: p + .spans + .into_iter() + .map(|s| PdfTextSpan { + id: u32::try_from(s.id).unwrap_or(u32::MAX), + text: s.text, + bbox: s.bbox.to_vec(), + baseline: s.baseline.into_iter().map(|p| p.to_vec()).collect(), + font: s.font, + size: s.size, + angle: s.angle, + words: s + .words + .into_iter() + .map(|w| PdfWord { + text: w.text, + bbox: w.bbox.to_vec(), + baseline: w.baseline.into_iter().map(|p| p.to_vec()).collect(), + }) + .collect(), + bold: s.bold, + italic: s.italic, + role: s.role, + ocr: s.ocr, + }) + .collect(), + }) + .collect(), + }), markdown: e.markdown, title: e.title, format: e.format.as_str().to_string(), diff --git a/crates/markkit-python/Cargo.toml b/crates/markkit-python/Cargo.toml index 1d7300e..7b4d8d1 100644 --- a/crates/markkit-python/Cargo.toml +++ b/crates/markkit-python/Cargo.toml @@ -14,7 +14,11 @@ name = "markkit_python" crate-type = ["cdylib"] doc = false +[features] +pdf-ocr-ocrs = ["markkit/pdf-ocr-ocrs"] + [dependencies] -markkit = { path = "../markkit", features = ["all_formats", "http_client"] } +markkit = { path = "../markkit", features = ["all_formats", "pdf-ocr", "http_client"] } +serde_json.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true diff --git a/crates/markkit-python/README.md b/crates/markkit-python/README.md index 38e2621..afbcccf 100644 --- a/crates/markkit-python/README.md +++ b/crates/markkit-python/README.md @@ -37,3 +37,49 @@ PyPI API token. A dry run builds and verifies distributions without publishing. Python packaging is configured by the workspace-root `pyproject.toml`. Source archives retain the complete Cargo workspace and its unchanged security lockfile. + +## Structured documents, PDF layout and OCR + +```python +from markkit import extract_document_sync, Limits +result = extract_document_sync( + pdf_bytes, file_name="report.pdf", pdf_layout=True, + pdf_remove_repeated_headers=True, +) +print(result["markdown"], result["stats"]["tables"]) +print(result["layout"]["pages"][0]["blocks"]) +``` + +PDF extraction reconstructs reading order, columns and ruled/borderless tables, +with merged/wrapped cells, emphasis, numbered lists and tagged headings/ActualText. +Layout includes source runs/words and semantic block/line/cell geometry. Keys are +snake_case; coordinates are display points from the cropped/rotated top-left. + +For image-only pages, install Tesseract and your language data: +`extract_document_sync(pdf_bytes, file_name="scan.pdf", pdf_ocr=True, +pdf_ocr_language="eng", pdf_layout=True, limits=Limits(timeout=300))`. +OCR is opt-in and only processes pages without text. Resource limits also bound +geometry, grid expansion, raster dimensions/helper files and process deadlines. + +## Optional Rust OCR backend + +Source builds can enable Cargo feature `pdf-ocr-ocrs` (for example, +`maturin build --features pdf-ocr-ocrs`). Default wheels retain Tesseract. +Prepare the repository's pinned model pair with +`python3 scripts/prepare_ocr_models.py`, then use: + +```python +markkit.extract_document_sync( + data, file_name="scan.pdf", pdf_ocr=True, pdf_ocr_backend="ocrs", + pdf_ocr_detection_model="/models/text-detection.onnx", + pdf_ocr_recognition_model="/models/text-recognition.onnx", + limits=markkit.Limits(timeout=300), +) +``` + +Extraction never downloads models. The pinned weights are CC-BY-SA 4.0 assets +with attribution in the setup script's NOTICE.json. The current Latin model +accepts `pdf_ocr_language="eng"` only. Model files are trusted deployment assets; +inference deadlines are cooperative and hard CPU/memory limits require host +quotas. Invalid backend names raise ValueError; missing features/models and +unsupported languages raise RuntimeError. diff --git a/crates/markkit-python/src/lib.rs b/crates/markkit-python/src/lib.rs index 71b28a9..42ee8c8 100644 --- a/crates/markkit-python/src/lib.rs +++ b/crates/markkit-python/src/lib.rs @@ -251,10 +251,56 @@ fn extract_bytes<'py>( }) } +/// Extract document bytes to a Python dictionary containing Markdown, metadata +/// and optional PDF layout. OCR is local and opt-in; helpers must be installed. +#[pyfunction] +#[pyo3(signature = (data, *, file_name=None, content_type=None, pdf_layout=false, pdf_remove_repeated_headers=false, pdf_ocr=false, pdf_ocr_language="eng", pdf_ocr_backend="tesseract", pdf_ocr_detection_model=None, pdf_ocr_recognition_model=None, limits=None))] +#[allow(clippy::too_many_arguments)] +fn extract_document_sync<'py>( + py: Python<'py>, + data: Vec, + file_name: Option, + content_type: Option, + pdf_layout: bool, + pdf_remove_repeated_headers: bool, + pdf_ocr: bool, + pdf_ocr_language: &str, + pdf_ocr_backend: &str, + pdf_ocr_detection_model: Option, + pdf_ocr_recognition_model: Option, + limits: Option>, +) -> PyResult> { + let mut config = builder(limits.as_deref()) + .pdf_layout(pdf_layout) + .pdf_remove_repeated_headers(pdf_remove_repeated_headers) + .pdf_ocr(pdf_ocr) + .pdf_ocr_language(pdf_ocr_language) + .pdf_ocr_backend(pdf_ocr_backend.parse().map_err(PyValueError::new_err)?); + if let Some(path) = pdf_ocr_detection_model { + config = config.pdf_ocr_detection_model(path); + } + if let Some(path) = pdf_ocr_recognition_model { + config = config.pdf_ocr_recognition_model(path); + } + let e = config + .build() + .extract_document_sync( + &data, + markkit::DocumentHint { + file_name: file_name.as_deref(), + content_type: content_type.as_deref(), + }, + ) + .map_err(err)?; + let json = serde_json::to_string(&e).map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + py.import("json")?.call_method1("loads", (json,)) +} + #[pymodule] #[pyo3(name = "markkit")] fn markkit_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; + m.add_function(wrap_pyfunction!(extract_document_sync, m)?)?; m.add_function(wrap_pyfunction!(extract_html, m)?)?; m.add_function(wrap_pyfunction!(extract_html_sync, m)?)?; m.add_function(wrap_pyfunction!(extract_html_sync_with_options, m)?)?; diff --git a/crates/markkit-python/tests/test_documents.py b/crates/markkit-python/tests/test_documents.py new file mode 100644 index 0000000..0937094 --- /dev/null +++ b/crates/markkit-python/tests/test_documents.py @@ -0,0 +1,43 @@ +"""Structured byte extraction, including PDF tables and source geometry.""" +import unittest +import markkit + + +def pdf(content): + return ("%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n" + "2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n" + "3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >> endobj\n" + "4 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\n" + f"5 0 obj << /Length {len(content)} >> stream\n{content}\nendstream endobj\n" + "trailer << /Root 1 0 R >>\n%%EOF").encode() + + +class Documents(unittest.TestCase): + def test_pdf_layout_and_limits(self): + data = pdf("BT /F1 10 Tf 10 100 Td (Hi) Tj ET") + plain = markkit.extract_document_sync(data, file_name="a.pdf") + self.assertNotIn("layout", plain) + e = markkit.extract_document_sync(data, file_name="a.pdf", pdf_layout=True) + self.assertEqual(e["markdown"], plain["markdown"]) + self.assertEqual(e["layout"]["pages"][0]["blocks"][0]["kind"], "paragraph") + self.assertEqual(e["layout"]["pages"][0]["spans"][0]["baseline"], [[10, 300], [20, 300]]) + with self.assertRaisesRegex(RuntimeError, "max_nodes"): + markkit.extract_document_sync(data, file_name="a.pdf", pdf_layout=True, limits=markkit.Limits(max_nodes=1)) + + def test_ocr_backend_validation_and_lazy_models(self): + data = pdf("BT /F1 10 Tf 10 100 Td (Hi) Tj ET") + with self.assertRaisesRegex(ValueError, "tesseract or ocrs"): + markkit.extract_document_sync(data, pdf_ocr_backend="unknown") + result = markkit.extract_document_sync(data, pdf_ocr=True, pdf_ocr_backend="ocrs", pdf_ocr_detection_model="unused.onnx", pdf_ocr_recognition_model="unused.onnx") + self.assertEqual(result["markdown"], "Hi\n") + + def test_borderless_table(self): + content = "\n".join(f"BT /F1 10 Tf 10 {y} Td ({a}) Tj ET BT /F1 10 Tf 150 {y} Td ({b}) Tj ET" for y, a, b in [(300, "Item", "Qty"), (280, "Apples", "3"), (260, "Pears", "2")]) + e = markkit.extract_document_sync(pdf(content), file_name="a.pdf", pdf_layout=True) + self.assertEqual(e["stats"]["tables"], 1) + self.assertIn("| Apples | 3 |", e["markdown"]) + self.assertEqual(e["layout"]["pages"][0]["blocks"][0]["kind"], "table") + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/markkit-wasm/Cargo.toml b/crates/markkit-wasm/Cargo.toml index 857ae55..4c6d45e 100644 --- a/crates/markkit-wasm/Cargo.toml +++ b/crates/markkit-wasm/Cargo.toml @@ -5,7 +5,7 @@ # SharedArrayBuffer and no COOP/COEP headers, so it runs in browsers, edge # workers (Cloudflare, Vercel, Deno Deploy), Node, Deno and Bun. # -# Feature surface: html, docx, spreadsheet, tool. `http_client` is excluded: +# Feature surface includes pure-Rust PDF text/layout. `http_client` is excluded: # reqwest needs sockets; JS hosts fetch themselves and pass bytes in, which also # keeps SSRF policy with the host. See knowledge/runtimes/wasm-package.md. @@ -26,6 +26,7 @@ doc = false [dependencies] markkit = { path = "../markkit", default-features = false, features = [ "html", + "pdf", "docx", "spreadsheet", "markdown", diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index 05dc404..6936b4f 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -1,6 +1,6 @@ # @everruns/markkit-wasm -Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX, +Markkit compiled to WebAssembly: extract clean Markdown from HTML, PDF text/layout, DOCX, spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, Jupyter notebooks, plain text and Markdown in the **browser and any JavaScript runtime**: edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. @@ -67,3 +67,25 @@ try { ## License MIT + +## PDF layout and tables + +PDF text extraction reconstructs columns, ruled and conservative borderless tables, +including merged and wrapped cells. Tagged headings/ActualText, bold/italic and +numbered lists are preserved. Set `pdfRemoveRepeatedHeaders: true` to suppress +repeated margin headers/footers. + +`pdfLayout: true` adds `layout: { version: 1, pages }` to document results. Source +`spans` retain drawing order and include glyph-positioned `words`. Semantic `blocks` +are in reading order, with `id`, `kind`, `bbox`, `spanIds`, `lines` and table `cells`. +Cells have row/column positions, rowspan/colspan and source references. Coordinates +are display points from the cropped/rotated top-left, x right and y down. Font +bounds approximate an em box. Layout is optional and does not change Markdown. + +Node and WASM use camelCase keys. `maxNodes` bounds primitives and geometry; +`maxPdfDecodedSize` bounds retained geometry and analysis work; table row/column/ +grid limits apply. `maxOutputSize` independently bounds compact layout JSON and +Markdown, excluding JSON wrapper/pretty-print overhead. + +WASM reads existing PDF text layers. Native process OCR runs through the Rust, +Node or Python packages with installed Tesseract and language data. diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index ccb3f2e..f2e733e 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -105,7 +105,8 @@ test("extractDocument reads Jupyter notebooks", () => { test("extractBytes and unsupported formats", () => { const mk = new Markkit(); assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); - assert.throws(() => mk.extractDocument(new TextEncoder().encode("%PDF-1.7"), { fileName: "a.pdf" }), /Unsupported/); + assert.throws(() => mk.extractDocument(new TextEncoder().encode("%PDF-1.7"), { fileName: "a.pdf" }), /PDF extraction error/); + assert.throws(() => mk.extractDocument(new Uint8Array([0]), { fileName: "a.doc" }), /Unsupported/); }); test("tool metadata and definition", () => { @@ -182,3 +183,33 @@ test("tool converts base64 DOCX", async () => { assert.equal(out.format, "docx"); assert.match(out.markdown, /From tool/); }); + +test("PDF source geometry is opt-in and obeys layout limits", () => { + const content = "BT /F1 10 Tf 10 100 Td (Hi) Tj ET"; + const source = `%PDF-1.7 +1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj +2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj +3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >> endobj +4 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj +5 0 obj << /Length ${content.length} >> stream +${content} +endstream endobj +trailer << /Root 1 0 R >> +%%EOF`; + const data = new TextEncoder().encode(source); + const hint = { fileName: "a.pdf" }; + const plain = new Markkit().extractDocument(data, hint); + assert.equal(plain.markdown, "Hi\n"); + assert.ok(plain.layout == null); + const result = new Markkit({ pdfLayout: true }).extractDocument(data, hint); + assert.equal(result.markdown, plain.markdown); + assert.equal(result.layout.pages[0].width, 300); + assert.equal(result.layout.pages[0].blocks[0].kind, "paragraph"); + assert.deepEqual(result.layout.pages[0].blocks[0].spanIds, [0]); + assert.equal(result.layout.pages[0].spans[0].words[0].text, "Hi"); + assert.equal(result.layout.pages[0].height, 400); + assert.equal(result.layout.pages[0].spans[0].text, "Hi"); + assert.deepEqual(result.layout.pages[0].spans[0].baseline, [[10, 300], [20, 300]]); + assert.throws(() => new Markkit({ pdfLayout: true, maxNodes: 1 }).extractDocument(data, hint), /max_nodes/); + assert.throws(() => new Markkit({ pdfLayout: true, maxOutputSize: 50 }).extractDocument(data, hint), /max_output_size/); +}); diff --git a/crates/markkit-wasm/js/index.d.ts b/crates/markkit-wasm/js/index.d.ts index 8e69612..915e9ce 100644 --- a/crates/markkit-wasm/js/index.d.ts +++ b/crates/markkit-wasm/js/index.d.ts @@ -20,6 +20,12 @@ export interface MarkkitOptions { baseUrl?: string; /** Keep only the detected main content (article body) of HTML pages. */ mainContent?: boolean; + /** Include PDF source geometry in extractDocument results. */ + pdfLayout?: boolean; + pdfRemoveRepeatedHeaders?: boolean; + maxPdfPages?: number; + maxPdfStreamSize?: number; + maxPdfDecodedSize?: number; preserveLinks?: boolean; preserveImages?: boolean; /** CSS selectors to include (empty = whole document). */ @@ -68,6 +74,7 @@ export interface Warning { } export interface Extraction { + layout?: PdfLayout; markdown: string; title: string | null; format: string; @@ -76,6 +83,42 @@ export interface Extraction { metadata: Metadata; } +/** Source geometry in displayed crop-box points, x right and y down. */ +export interface PdfLayout { + version: number; + pages: { + page: number; + width: number; + height: number; + rotation: number; + blocks: PdfBlock[]; + spans: { + id: number; + text: string; + /** Approximate em bounds: [left, top, right, bottom]. */ + bbox: [number, number, number, number]; + baseline: [[number, number], [number, number]]; + font?: string; + size: number; + angle: number; + words: {text:string; bbox:[number,number,number,number]; baseline:[[number,number],[number,number]]}[]; + bold?: boolean; + italic?: boolean; + role?: string; + ocr?: boolean; + }[]; + }[]; +} + +export interface PdfBlock { + id:number; + kind:string; + bbox:[number,number,number,number]; + spanIds:number[]; + lines:{bbox:[number,number,number,number];spanIds:number[]}[]; + cells?:{row:number;column:number;rowspan:number;colspan:number;bbox:[number,number,number,number];spanIds:number[]}[]; +} + export class Markkit { constructor(options?: MarkkitOptions); extractHtml(html: string): string; diff --git a/crates/markkit-wasm/src/lib.rs b/crates/markkit-wasm/src/lib.rs index fe279da..75f907c 100644 --- a/crates/markkit-wasm/src/lib.rs +++ b/crates/markkit-wasm/src/lib.rs @@ -27,6 +27,11 @@ struct Options { /// Absolute URL relative links and images resolve against. base_url: Option, main_content: Option, + pdf_layout: Option, + pdf_remove_repeated_headers: Option, + max_pdf_pages: Option, + max_pdf_stream_size: Option, + max_pdf_decoded_size: Option, preserve_links: Option, preserve_images: Option, /// Added to the default selector set. @@ -57,6 +62,11 @@ impl Options { max_nesting_depth, base_url, main_content, + pdf_layout, + pdf_remove_repeated_headers, + max_pdf_pages, + max_pdf_stream_size, + max_pdf_decoded_size, preserve_links, preserve_images, max_archive_entries, @@ -139,6 +149,23 @@ fn to_js(value: &T) -> Result { .map_err(|e| JsError::new(&e.to_string())) } +fn camel_case_tree(value: &mut serde_json::Value) { + camel_case_keys(value); + match value { + serde_json::Value::Object(map) => { + for v in map.values_mut() { + camel_case_tree(v); + } + } + serde_json::Value::Array(list) => { + for v in list { + camel_case_tree(v); + } + } + _ => {} + } +} + /// Rust serializes `snake_case`; JS callers get `camelCase` (`canonicalUrl`). fn camel_case_keys(value: &mut serde_json::Value) { if let serde_json::Value::Object(map) = value { @@ -213,6 +240,9 @@ impl Markkit { .map_err(js_err)?; let mut value = serde_json::to_value(&extraction).map_err(js_err)?; camel_case_keys(&mut value["metadata"]); + if let Some(layout) = value.get_mut("layout") { + camel_case_tree(layout); + } to_js(&value) } diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 2cd2f2f..3a49378 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -17,6 +17,10 @@ readme = "../../README.md" default = ["html"] html = ["dep:scraper", "dep:html5ever", "dep:ego-tree", "dep:url"] pdf = ["dep:hayro-syntax", "dep:miniz_oxide"] +# Native OCR is opt-in; Rust rendering plus host-installed Tesseract. +pdf-ocr = ["pdf", "dep:tempfile", "dep:hayro"] +# Optional Rust OCR with caller-supplied, local ONNX models. +pdf-ocr-ocrs = ["pdf-ocr", "dep:ocrs", "dep:rten"] docx = ["dep:zip", "dep:quick-xml"] pptx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] @@ -33,7 +37,13 @@ tool = ["dep:tower", "dep:futures-core", "dep:base64", "dep:tokio", "tokio/sync" json-schema = ["dep:schemars"] all_formats = ["html", "pdf", "docx", "pptx", "spreadsheet", "markdown", "email", "odt", "epub"] +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +hayro = { workspace = true, optional = true } +ocrs = { workspace = true, optional = true } +rten = { workspace = true, optional = true } + [dependencies] +tempfile = { workspace = true, optional = true } # Async (optional): only URL fetching and the tool's output channel need tokio. tokio = { workspace = true, optional = true } diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index c1e34f5..7cb5650 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -3,6 +3,25 @@ use std::time::Duration; use crate::extract::Markkit; use crate::limits::Limits; +/// Local OCR implementation for textless PDF pages. Tesseract is the default. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum PdfOcrBackend { + #[default] + Tesseract, + /// Rust inference; requires `pdf-ocr-ocrs` and explicit local ONNX models. + Ocrs, +} +impl std::str::FromStr for PdfOcrBackend { + type Err = String; + fn from_str(value: &str) -> Result { + match value { + "tesseract" => Ok(Self::Tesseract), + "ocrs" => Ok(Self::Ocrs), + _ => Err("OCR backend must be tesseract or ocrs".into()), + } + } +} + /// Builder for configuring a Markkit instance. #[derive(Debug, Clone)] pub struct MarkkitBuilder { @@ -39,6 +58,21 @@ pub struct MarkkitBuilder { /// HTML element XPath, DOCX block index, spreadsheet sheet and range. /// Default: false (no cost, no `source` in IR JSON). pub track_sources: bool, + /// Retain PDF page dimensions and positioned text runs. Default: false. + /// Source geometry is separate from the semantic IR and Markdown output. + pub pdf_layout: bool, + /// Remove text repeated in page margins. Default: false. + pub pdf_remove_repeated_headers: bool, + /// OCR textless PDF pages with Hayro rendering and the selected OCR backend (`pdf-ocr` feature). + pub pdf_ocr: bool, + /// Installed Tesseract language(s), e.g. `eng` or `eng+deu`. + pub pdf_ocr_language: String, + /// OCR backend; requires `pdf_ocr(true)`. Default: Tesseract. + pub pdf_ocr_backend: PdfOcrBackend, + /// Local ONNX text detection model, required by the ocrs backend. + pub pdf_ocr_detection_model: Option, + /// Local ONNX text recognition model, required by the ocrs backend. + pub pdf_ocr_recognition_model: Option, /// IR passes run after parsing, before rendering (see [`crate::transform`]). pub transforms: crate::transform::TransformChain, /// Keep only the detected main content (article body) of HTML pages, @@ -74,6 +108,13 @@ impl Default for MarkkitBuilder { allowed_hosts: Vec::new(), base_url: None, track_sources: false, + pdf_layout: false, + pdf_remove_repeated_headers: false, + pdf_ocr: false, + pdf_ocr_language: "eng".into(), + pdf_ocr_backend: PdfOcrBackend::default(), + pdf_ocr_detection_model: None, + pdf_ocr_recognition_model: None, transforms: Default::default(), main_content: false, html_rules: Vec::new(), @@ -83,6 +124,41 @@ impl Default for MarkkitBuilder { } impl MarkkitBuilder { + /// Retain PDF source geometry in `ParsedDocument::layout` and + /// `Extraction::layout`. Has no effect on other formats or Markdown. + pub fn pdf_layout(mut self, enabled: bool) -> Self { + self.pdf_layout = enabled; + self + } + + pub fn pdf_remove_repeated_headers(mut self, enabled: bool) -> Self { + self.pdf_remove_repeated_headers = enabled; + self + } + + pub fn pdf_ocr(mut self, enabled: bool) -> Self { + self.pdf_ocr = enabled; + self + } + + pub fn pdf_ocr_language(mut self, language: impl Into) -> Self { + self.pdf_ocr_language = language.into(); + self + } + + pub fn pdf_ocr_backend(mut self, backend: PdfOcrBackend) -> Self { + self.pdf_ocr_backend = backend; + self + } + pub fn pdf_ocr_detection_model(mut self, path: impl Into) -> Self { + self.pdf_ocr_detection_model = Some(path.into()); + self + } + pub fn pdf_ocr_recognition_model(mut self, path: impl Into) -> Self { + self.pdf_ocr_recognition_model = Some(path.into()); + self + } + pub fn max_input_size(mut self, size: usize) -> Self { self.limits.max_input_size = size; self diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 61dea98..e225e03 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -10,6 +10,9 @@ use crate::limits::Budget; #[derive(Debug, Clone, PartialEq, Serialize)] #[non_exhaustive] pub struct Extraction { + /// Optional PDF source geometry (enable with `pdf_layout(true)`). + #[serde(skip_serializing_if = "Option::is_none")] + pub layout: Option, /// Rendered Markdown. Non-empty output always ends with exactly one `\n` /// (POSIX text convention, safe to concatenate); empty documents render `""`. pub markdown: String, @@ -176,6 +179,9 @@ pub struct ExtractionStats { #[derive(Debug, Clone, PartialEq, Serialize)] #[non_exhaustive] pub struct ParsedDocument { + /// Optional PDF source geometry, independent of semantic IR transforms. + #[serde(skip_serializing_if = "Option::is_none")] + pub layout: Option, /// Canonical IR (blocks carry [`crate::ir::Source`] when /// [`MarkkitBuilder::track_sources`] is enabled). pub document: Document, @@ -196,6 +202,7 @@ impl ParsedDocument { /// [`crate::registry::FormatParser`]s). pub fn new(document: Document, format: Format) -> Self { Self { + layout: None, document, title: None, format, @@ -223,6 +230,7 @@ impl ParsedDocument { fn into_extraction(self, markdown: String) -> Extraction { Extraction { + layout: self.layout, markdown, title: self.title, format: self.format, @@ -694,8 +702,17 @@ impl Markkit { crate::pdf::parse(bytes, &self.config, budget) })?; let mut p = ParsedDocument::new(out.doc, Format::Pdf); + p.layout = out.layout; + self.config.transforms.apply(&mut p.document)?; p.title = out.title; p.warnings = out.warnings; + p.stats.tables = Some( + p.document + .nodes + .iter() + .filter(|n| matches!(n, crate::ir::Node::Table { .. })) + .count(), + ); p.stats.pages = Some(out.pages); Ok(p) } diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs index e3ec9b3..a3f21ad 100644 --- a/crates/markkit/src/ir.rs +++ b/crates/markkit/src/ir.rs @@ -376,6 +376,9 @@ pub struct Source { /// Paged formats: 1-based page number (PPTX: slide number). #[serde(default, skip_serializing_if = "Option::is_none")] pub page: Option, + /// PDF: page-local semantic layout block ID. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layout_block: Option, } impl Source { diff --git a/crates/markkit/src/layout.rs b/crates/markkit/src/layout.rs new file mode 100644 index 0000000..c2cc4df --- /dev/null +++ b/crates/markkit/src/layout.rs @@ -0,0 +1,116 @@ +//! Optional PDF source geometry, separate from the semantic document IR. +//! +//! Decision: source runs stay in drawing order; words retain glyph advances. +//! Semantic blocks/lines/table cells are in reading order and link to IR via +//! `Source::page` + `Source::layout_block`. IDs are stable for identical input +//! and configuration. Coordinates are points (1/72 inch), relative to the +//! displayed crop box top-left, x right and y down after rotation/UserUnit. +//! Font bounds approximate 0.8 em ascent and 0.2 em descent, not ink outlines; +//! OCR bounds come from the recognizer. Off-page text is retained. +//! Geometry stays source data if an embedder transforms the semantic IR. + +use serde::{Deserialize, Serialize}; + +/// Positioned PDF text. Present only when `pdf_layout(true)` is requested. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfLayout { + /// Layout schema version, independent of the semantic IR version. + pub version: u32, + /// Extracted pages, including empty pages, up to `max_pdf_pages`. + pub pages: Vec, +} + +/// Display dimensions and positioned text for a PDF page. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfPageLayout { + /// One-based page number. + pub page: u32, + /// Display width in points, after cropping, rotation and UserUnit. + pub width: f64, + /// Display height in points. + pub height: f64, + /// Applied clockwise page rotation: 0, 90, 180 or 270 degrees. + pub rotation: u16, + /// Text-showing runs in drawing order. `TJ` strings are separate runs. + pub spans: Vec, + /// Semantic regions in reading order. Source spans retain drawing order. + #[serde(default)] + pub blocks: Vec, +} + +/// One text-showing run, before line joining or dehyphenation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfTextSpan { + /// Zero-based run index within its page (not a semantic IR node ID). + pub id: usize, + /// Decoded text; ligatures expanded and control characters removed. + pub text: String, + /// Approximate axis-aligned bounds: `[left, top, right, bottom]`. + pub bbox: [f64; 4], + /// Start and end of the text baseline, including advances and spacing. + pub baseline: [[f64; 2]; 2], + /// PDF BaseFont name, including any subset prefix, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub font: Option, + /// Length of the transformed vertical em vector in points. + pub size: f64, + /// Clockwise baseline direction in display coordinates, in [0, 360). + pub angle: f64, + #[serde(default)] + pub words: Vec, + /// True for a run recognized from a rasterized page. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub ocr: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub bold: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub italic: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, +} + +/// A word positioned using glyph advances, before paragraph normalization. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfWord { + pub text: String, + pub bbox: [f64; 4], + pub baseline: [[f64; 2]; 2], +} + +/// Semantic region in reading order. `(page, id)` links to `Source::layout_block`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfBlock { + pub id: usize, + /// `heading`, `paragraph`, `list`, `table`, `header` or `footer`. + pub kind: String, + pub bbox: [f64; 4], + pub span_ids: Vec, + pub lines: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub cells: Vec, +} + +/// Line geometry and references to source text runs. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfLine { + pub bbox: [f64; 4], + pub span_ids: Vec, +} + +/// A table cell, with zero-based row/column and authored/inferred spans. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PdfCell { + pub row: usize, + pub column: usize, + pub rowspan: u32, + pub colspan: u32, + pub bbox: [f64; 4], + pub span_ids: Vec, +} diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 0733327..b69b174 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -42,7 +42,8 @@ //! - **Jupyter notebooks** (always on): `.ipynb`, cells and text outputs //! - **CSV / TSV** (always on): `text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`, one table //! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR -//! - **PDF** (`pdf`): Text layer of PDF documents (no OCR); encrypted PDFs are +//! - **PDF** (`pdf`): Text, layout and tables; optional native OCR (`pdf-ocr`) +//! uses Rust Hayro rendering and installed Tesseract. Encrypted PDFs are //! rejected with [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, opt-in): Fetch and extract from URLs via reqwest //! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the @@ -77,6 +78,7 @@ pub mod format; #[cfg(feature = "html")] pub mod html; pub mod ir; +pub mod layout; pub mod limits; #[cfg(feature = "markdown")] mod markdown; @@ -108,7 +110,7 @@ pub mod transform; #[cfg_attr(not(feature = "docx"), allow(dead_code))] // DOCX uses every helper mod xml; -pub use config::MarkkitBuilder; +pub use config::{MarkkitBuilder, PdfOcrBackend}; pub use error::Error; pub use extract::{ Extraction, ExtractionStats, Markkit, Metadata, ParsedDocument, Warning, WarningCode, diff --git a/crates/markkit/src/limits.rs b/crates/markkit/src/limits.rs index 78e204a..f6425cd 100644 --- a/crates/markkit/src/limits.rs +++ b/crates/markkit/src/limits.rs @@ -126,7 +126,7 @@ pub struct Limits { /// [`HARD_MAX_NESTING_DEPTH`]. Default: 256. pub max_nesting_depth: usize, /// Maximum parsed nodes (HTML DOM nodes; XML elements and text nodes summed - /// over DOCX parts). Default: 5 000 000. + /// over DOCX parts; retained PDF layout pages/runs). Default: 5 000 000. pub max_nodes: usize, /// Maximum number of entries in a ZIP container (DOCX/XLSX/ODS). Default: 10 000. pub max_archive_entries: usize, diff --git a/crates/markkit/src/pdf/analyze.rs b/crates/markkit/src/pdf/analyze.rs new file mode 100644 index 0000000..b5b3121 --- /dev/null +++ b/crates/markkit/src/pdf/analyze.rs @@ -0,0 +1,1128 @@ +//! Deterministic geometry → regions → semantic IR. +//! +//! Decisions: ruling lines take precedence over alignment tables. Borderless +//! tables require repeated row/column alignment plus numeric data or explicit +//! header styling; a page of prose columns is not evidence of a table. XY cuts +//! order columns within bands separated by spanning headings/tables. All pair +//! comparisons and grid expansion are bounded before allocating a grid. + +use crate::ir::{ListItem, Node, Source, TableCell}; +use crate::layout::{PdfBlock, PdfCell, PdfLayout, PdfLine}; +use crate::limits::{Budget, LimitExceeded, LimitKind}; +use crate::{Error, MarkkitBuilder}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +#[derive(Clone, Copy, Debug)] +pub(super) struct Rule { + pub a: [f64; 2], + pub b: [f64; 2], +} +pub(super) fn union(a: [f64; 4], b: [f64; 4]) -> [f64; 4] { + [ + a[0].min(b[0]), + a[1].min(b[1]), + a[2].max(b[2]), + a[3].max(b[3]), + ] +} +fn bbox(points: impl Iterator) -> [f64; 4] { + points.reduce(union).unwrap_or([0.0; 4]) +} +fn cx(b: [f64; 4]) -> f64 { + (b[0] + b[2]) * 0.5 +} +fn cy(b: [f64; 4]) -> f64 { + (b[1] + b[3]) * 0.5 +} +fn inside(b: [f64; 4], x: f64, y: f64) -> bool { + x >= b[0] - 0.5 && x <= b[2] + 0.5 && y >= b[1] - 0.5 && y <= b[3] + 0.5 +} + +struct Work<'a> { + remaining: usize, + clock: &'a Budget, +} +impl Work<'_> { + fn tick(&mut self, n: usize) -> Result<(), Error> { + self.clock.check_time()?; + self.remaining = self.remaining.checked_sub(n).ok_or_else(|| { + LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "PDF geometry analysis exceeds max_pdf_decoded_size work budget", + ) + })?; + Ok(()) + } +} +#[derive(Clone, Copy)] +struct Coords(u16); +impl Coords { + fn point(self, p: [f64; 2]) -> [f64; 2] { + match self.0 { + 90 => [p[1], -p[0]], + 180 => [-p[0], -p[1]], + 270 => [-p[1], p[0]], + _ => p, + } + } + fn inverse(self) -> Self { + Self((360 - self.0) % 360) + } + fn rect(self, b: [f64; 4]) -> [f64; 4] { + bbox( + [[b[0], b[1]], [b[0], b[3]], [b[2], b[1]], [b[2], b[3]]] + .into_iter() + .map(|p| { + let [x, y] = self.point(p); + [x, y, x, y] + }), + ) + } +} +#[derive(Clone, Debug)] +struct Word { + text: String, + rect: [f64; 4], + display: [f64; 4], + y: f64, + size: f64, + span: usize, + bold: bool, + italic: bool, + role: Option, +} +#[derive(Clone, Debug)] +struct Line { + words: Vec, + rect: [f64; 4], + size: f64, + y: f64, +} +impl Line { + fn new(words: Vec) -> Self { + let rect = bbox(words.iter().map(|w| w.rect)); + let size = words.iter().map(|w| w.size).fold(0.1, f64::max); + let y = words.first().map_or(0.0, |w| w.y); + Self { + words, + rect, + size, + y, + } + } + fn text(&self) -> String { + plain(&self.words) + } + fn geometry(&self) -> PdfLine { + PdfLine { + bbox: bbox(self.words.iter().map(|w| w.display)), + span_ids: ids(&self.words), + } + } +} +fn ids(words: &[Word]) -> Vec { + words + .iter() + .map(|w| w.span) + .collect::>() + .into_iter() + .collect() +} +fn plain(words: &[Word]) -> String { + let mut s = String::new(); + let mut prev: Option<&Word> = None; + for w in words { + if let Some(p) = prev + && ((w.y - p.y).abs() > w.size * 0.4 || w.rect[0] - p.rect[2] > w.size * 0.15) + { + s.push(' '); + } + s.push_str(&w.text); + prev = Some(w); + } + s +} +fn inline(words: &[Word]) -> Vec { + let mut nodes = Vec::new(); + let mut buffer = String::new(); + let mut style = (false, false); + let mut prev: Option<&Word> = None; + let flush = |nodes: &mut Vec, buffer: &mut String, style: (bool, bool)| { + if buffer.is_empty() { + return; + } + let mut node = Node::Text(std::mem::take(buffer)); + if style.1 { + node = Node::Italic { + children: vec![node], + }; + } + if style.0 { + node = Node::Bold { + children: vec![node], + }; + } + nodes.push(node); + }; + for w in words { + let gap = prev.is_some_and(|p| { + (w.y - p.y).abs() > w.size * 0.4 || w.rect[0] - p.rect[2] > w.size * 0.15 + }); + if (w.bold, w.italic) != style { + flush(&mut nodes, &mut buffer, style); + if gap { + nodes.push(Node::Text(" ".into())); + } + style = (w.bold, w.italic); + } else if gap { + buffer.push(' '); + } + buffer.push_str(&w.text); + prev = Some(w); + } + flush(&mut nodes, &mut buffer, style); + nodes +} + +fn rows(mut words: Vec) -> Vec { + words.sort_by(|a, b| { + a.y.total_cmp(&b.y) + .then(a.rect[0].total_cmp(&b.rect[0])) + .then(a.span.cmp(&b.span)) + }); + let mut rows: Vec = Vec::new(); + for w in words { + if let Some(last) = rows.last_mut() + && (last.y - w.y).abs() <= 0.4 * last.size.max(w.size) + { + last.rect = union(last.rect, w.rect); + last.size = last.size.max(w.size); + last.words.push(w); + } else { + rows.push(Line::new(vec![w])); + } + } + for row in &mut rows { + row.words + .sort_by(|a, b| a.rect[0].total_cmp(&b.rect[0]).then(a.span.cmp(&b.span))); + } + rows +} +fn fragments(row: Line) -> Vec { + let mut out = Vec::new(); + let mut words = Vec::new(); + let mut right = None; + for w in row.words { + if right.is_some_and(|r| w.rect[0] - r >= 0.8 * row.size) && !words.is_empty() { + out.push(Line::new(std::mem::take(&mut words))); + } + right = Some(w.rect[2]); + words.push(w); + } + if !words.is_empty() { + out.push(Line::new(words)); + } + out +} +#[derive(Clone)] +struct Region { + rect: [f64; 4], + lines: Vec, + node: Option, + cells: Vec, + removed: Option<&'static str>, +} +impl Region { + fn line(line: Line) -> Self { + Self { + rect: line.rect, + lines: vec![line], + node: None, + cells: Vec::new(), + removed: None, + } + } +} + +pub(super) fn analyze( + layout: &mut PdfLayout, + rules: &[Vec], + cfg: &MarkkitBuilder, + clock: &Budget, +) -> Result, Error> { + let mut work = Work { + remaining: cfg.limits.max_pdf_decoded_size, + clock, + }; + let repeated = repeated_margins(layout); + let mut sizes: HashMap = HashMap::new(); + for p in &layout.pages { + for s in &p.spans { + *sizes.entry((s.size * 2.0).round() as i32).or_default() += s.text.chars().count(); + } + } + let body = sizes + .into_iter() + .max_by_key(|(s, n)| (*n, -*s)) + .map_or(10.0, |(s, _)| f64::from(s) * 0.5); + let mut heading_sizes = layout + .pages + .iter() + .flat_map(|p| &p.spans) + .filter(|s| s.size >= body * 1.2 && s.text.chars().count() <= 200) + .map(|s| (s.size * 2.0).round() as i32) + .collect::>(); + heading_sizes.sort_unstable_by(|a, b| b.cmp(a)); + heading_sizes.dedup(); + let mut nodes = Vec::new(); + for (pi, page) in layout.pages.iter_mut().enumerate() { + work.tick(1)?; + let mut angles = HashMap::::new(); + for s in &page.spans { + let a = ((s.angle / 90.0).round() as u16 % 4) * 90; + *angles.entry(a).or_default() += s.text.chars().count(); + } + let coords = Coords( + angles + .into_iter() + .max_by_key(|(a, n)| (*n, std::cmp::Reverse(*a))) + .map_or(0, |(a, _)| a), + ); + let words = page + .spans + .iter() + .flat_map(|s| { + s.words.iter().map(move |w| Word { + text: w.text.clone(), + rect: coords.rect(w.bbox), + display: w.bbox, + y: coords.point(w.baseline[0])[1], + size: s.size.max(0.1), + span: s.id, + bold: s.bold, + italic: s.italic, + role: s.role.clone(), + }) + }) + .collect::>(); + let mut removed = Vec::new(); + let mut active = Vec::new(); + for row in rows(words) { + let text = margin_key(&row.text()); + let margin = row.words.iter().all(|w| w.display[3] < page.height * 0.08) + || row.words.iter().all(|w| w.display[1] > page.height * 0.92); + if cfg.pdf_remove_repeated_headers && margin && repeated.contains(&text) { + let kind = if cy(row.words[0].display) < page.height * 0.5 { + "header" + } else { + "footer" + }; + let mut r = Region::line(row); + r.removed = Some(kind); + removed.push(r); + } else { + active.extend(row.words); + } + } + let mut used = vec![false; active.len()]; + let rs = rules + .get(pi) + .map_or(&[][..], Vec::as_slice) + .iter() + .map(|r| Rule { + a: coords.point(r.a), + b: coords.point(r.b), + }) + .collect::>(); + let mut regions = ruled_tables(&active, &mut used, &rs, coords, cfg, &mut work)?; + let remaining = active + .into_iter() + .enumerate() + .filter_map(|(i, w)| (!used[i]).then_some(w)) + .collect::>(); + let mut line_rows = rows(remaining) + .into_iter() + .map(fragments) + .collect::>(); + regions.extend(borderless_tables(&mut line_rows, coords, cfg, &mut work)?); + regions.extend(line_rows.into_iter().flatten().map(Region::line)); + regions.extend(removed); + let mut ordered = Vec::new(); + order(regions, body, 0, &mut ordered, &mut work)?; + let mut paras: Vec = Vec::new(); + let mut force = true; + for atom in ordered { + let Some(mut r) = atom else { + force = true; + continue; + }; + if r.node.is_some() || r.removed.is_some() { + paras.push(r); + force = true; + continue; + } + let line = &r.lines[0]; + let text = line.text(); + let item = list_prefix(&text).is_some(); + let heading = is_heading(line, body); + let join = !force + && !item + && !heading + && paras.last().is_some_and(|p| { + p.node.is_none() + && p.removed.is_none() + && p.lines.last().is_some_and(|prev| { + let d = line.y - prev.y; + d > 0.0 + && d <= 1.7 * line.size.max(prev.size) + && (prev.size - line.size).abs() < 0.15 * prev.size.max(line.size) + && !is_heading(prev, body) + }) + }); + if join { + let last = paras.last_mut().unwrap(); + last.rect = union(last.rect, r.rect); + last.lines.append(&mut r.lines); + } else { + paras.push(r); + } + force = false; + } + page.blocks.clear(); + for r in paras { + work.tick(1)?; + clock.add_nodes(1)?; + let all = r + .lines + .iter() + .flat_map(|l| l.words.iter().cloned()) + .collect::>(); + let mut block = PdfBlock { + id: page.blocks.len(), + kind: r.removed.unwrap_or("paragraph").into(), + bbox: bbox(all.iter().map(|w| w.display)), + span_ids: ids(&all), + lines: r.lines.iter().map(Line::geometry).collect(), + cells: r.cells, + }; + if !block.cells.is_empty() { + block.bbox = bbox(block.cells.iter().map(|c| c.bbox)); + } + if r.removed.is_some() { + page.blocks.push(block); + continue; + } + let mut node = if let Some(n) = r.node { + block.kind = "table".into(); + n + } else { + let mut children = Vec::new(); + let mut text = String::new(); + for l in &r.lines { + let value = l.text(); + if !children.is_empty() { + if text.ends_with('-') + && value.chars().next().is_some_and(char::is_lowercase) + { + trim_hyphen(&mut children); + } else { + children.push(Node::Text(" ".into())); + } + } + children.extend(inline(&l.words)); + text.push_str(&value); + } + if let Some((ordered, start, len)) = list_prefix(&text) { + block.kind = "list".into(); + remove_prefix(&mut children, len); + Node::List { + ordered, + start, + items: vec![ListItem::new(vec![Node::paragraph(children)])], + source: None, + } + } else if is_heading(&r.lines[0], body) { + block.kind = "heading".into(); + let level = r.lines[0] + .words + .iter() + .find_map(|w| { + w.role + .as_deref() + .and_then(|r| r.strip_prefix('H')) + .and_then(|n| n.parse::().ok()) + }) + .unwrap_or_else(|| { + (heading_sizes + .iter() + .position(|s| *s == (r.lines[0].size * 2.0).round() as i32) + .unwrap_or(0) + + 1) + .min(6) as u8 + }); + Node::heading(level, children) + } else { + Node::paragraph(children) + } + }; + if cfg.track_sources || cfg.pdf_layout { + let mut source = Source::default().with_page(page.page); + source.layout_block = cfg.pdf_layout.then_some(block.id); + node.set_source(source); + } + // Adjacent list items share one semantic/layout block, never across pages. + let merge = match (nodes.last(), &node) { + ( + Some(Node::List { + ordered: a, + start, + items, + .. + }), + Node::List { + ordered: b, + start: next, + .. + }, + ) => { + a == b + && (!a || next == &Some(start.unwrap_or(1) + items.len() as u64)) + && page.blocks.last().is_some_and(|b| b.kind == "list") + } + _ => false, + }; + if merge { + if let (Some(Node::List { items, .. }), Node::List { items: more, .. }) = + (nodes.last_mut(), node) + { + items.extend(more); + } + let prev = page.blocks.last_mut().unwrap(); + prev.bbox = union(prev.bbox, block.bbox); + prev.lines.extend(block.lines); + prev.span_ids.extend(block.span_ids); + prev.span_ids.sort_unstable(); + prev.span_ids.dedup(); + } else { + nodes.push(node); + page.blocks.push(block); + } + } + } + // Account for retained block/cell metadata before it is returned. Word/source + // text is separately bounded during interpretation; this checks added geometry. + let extra = layout + .pages + .iter() + .flat_map(|p| &p.blocks) + .try_fold(0usize, |n, b| { + n.checked_add( + std::mem::size_of::() + + b.span_ids.len() * std::mem::size_of::() + + b.lines + .iter() + .map(|l| { + std::mem::size_of::() + + l.span_ids.len() * std::mem::size_of::() + }) + .sum::() + + b.cells + .iter() + .map(|c| { + std::mem::size_of::() + + c.span_ids.len() * std::mem::size_of::() + }) + .sum::(), + ) + }); + if extra.is_none_or(|n| n > cfg.limits.max_pdf_decoded_size) { + return Err(LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "PDF region geometry exceeds max_pdf_decoded_size", + ) + .into()); + } + Ok(nodes) +} +fn is_heading(l: &Line, body: f64) -> bool { + l.words.iter().any(|w| { + w.role + .as_deref() + .is_some_and(|r| matches!(r, "H1" | "H2" | "H3" | "H4" | "H5" | "H6")) + }) || (l.size >= body * 1.2 && l.text().chars().count() <= 200) +} +fn trim_hyphen(nodes: &mut [Node]) { + if let Some(n) = nodes.last_mut() { + match n { + Node::Text(s) => { + if s.ends_with('-') { + s.pop(); + } + } + Node::Bold { children } | Node::Italic { children } => trim_hyphen(children), + _ => {} + } + } +} +fn remove_prefix(nodes: &mut Vec, mut bytes: usize) { + while bytes > 0 && !nodes.is_empty() { + if let Node::Text(s) = &mut nodes[0] { + if s.len() <= bytes { + bytes -= s.len(); + nodes.remove(0); + } else { + s.drain(..bytes); + bytes = 0; + } + } else { + break; + } + } +} +fn list_prefix(s: &str) -> Option<(bool, Option, usize)> { + if let Some(c) = s.chars().next() + && super::BULLETS.contains(&c) + && s[c.len_utf8()..].starts_with(' ') + { + return Some((false, None, c.len_utf8() + 1)); + } + let n = s.bytes().take_while(u8::is_ascii_digit).count(); + if (1..=10).contains(&n) && s.get(n..n + 2).is_some_and(|x| x == ". " || x == ") ") { + return Some((true, s[..n].parse().ok(), n + 2)); + } + None +} +fn margin_key(s: &str) -> String { + s.chars() + .map(|c| { + if c.is_ascii_digit() { + '#' + } else { + c.to_ascii_lowercase() + } + }) + .collect() +} +fn repeated_margins(layout: &PdfLayout) -> HashSet { + let mut counts = HashMap::::new(); + for p in &layout.pages { + let mut seen = HashSet::new(); + for s in &p.spans { + if s.bbox[3] < p.height * 0.08 || s.bbox[1] > p.height * 0.92 { + seen.insert(margin_key(&s.text)); + } + } + for key in seen { + *counts.entry(key).or_default() += 1; + } + } + counts + .into_iter() + .filter_map(|(k, n)| (n >= 2 && n * 5 >= layout.pages.len() * 3).then_some(k)) + .collect() +} + +// A bounded union-find serves both connected ruling components and merged cells. +struct Sets(Vec); +impl Sets { + fn new(n: usize) -> Self { + Self((0..n).collect()) + } + fn root(&mut self, mut i: usize) -> usize { + while self.0[i] != i { + self.0[i] = self.0[self.0[i]]; + i = self.0[i]; + } + i + } + fn join(&mut self, a: usize, b: usize) { + let a = self.root(a); + let b = self.root(b); + self.0[b] = a; + } +} +#[derive(Clone, Copy)] +struct Seg { + vertical: bool, + pos: f64, + lo: f64, + hi: f64, +} +fn segment(r: Rule) -> Option { + if (r.a[0] - r.b[0]).abs() < 0.75 && (r.a[1] - r.b[1]).abs() > 3.0 { + Some(Seg { + vertical: true, + pos: (r.a[0] + r.b[0]) * 0.5, + lo: r.a[1].min(r.b[1]), + hi: r.a[1].max(r.b[1]), + }) + } else if (r.a[1] - r.b[1]).abs() < 0.75 && (r.a[0] - r.b[0]).abs() > 3.0 { + Some(Seg { + vertical: false, + pos: (r.a[1] + r.b[1]) * 0.5, + lo: r.a[0].min(r.b[0]), + hi: r.a[0].max(r.b[0]), + }) + } else { + None + } +} +fn unique(mut vals: Vec) -> Vec { + vals.sort_by(f64::total_cmp); + vals.dedup_by(|a, b| (*a - *b).abs() < 0.75); + vals +} +fn covers( + segs: &[Seg], + vertical: bool, + pos: f64, + lo: f64, + hi: f64, + work: &mut Work<'_>, +) -> Result { + let mut parts = Vec::new(); + for s in segs { + work.tick(1)?; + if s.vertical == vertical && (s.pos - pos).abs() < 0.75 && s.hi >= lo && s.lo <= hi { + parts.push((s.lo, s.hi)); + } + } + parts.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut end = lo; + for (a, b) in parts { + if a > end + 0.75 { + return Ok(false); + } + end = end.max(b); + } + Ok(end >= hi - 0.75) +} +fn grid_limit(rows: usize, cols: usize, cfg: &MarkkitBuilder) -> Result<(), Error> { + if rows > cfg.limits.max_sheet_rows + || cols > cfg.limits.max_sheet_cols + || rows + .checked_mul(cols) + .is_none_or(|n| n as u64 > cfg.limits.max_spreadsheet_cells) + { + Err(LimitExceeded::new( + LimitKind::MaxSpreadsheetCells, + "PDF table exceeds row/column/grid limits", + ) + .into()) + } else { + Ok(()) + } +} +fn ruled_tables( + words: &[Word], + used: &mut [bool], + rules: &[Rule], + coords: Coords, + cfg: &MarkkitBuilder, + work: &mut Work<'_>, +) -> Result, Error> { + let segs = rules.iter().filter_map(|r| segment(*r)).collect::>(); + let mut sets = Sets::new(segs.len()); + for (i, a) in segs.iter().enumerate() { + for (j, b) in segs[..i].iter().enumerate() { + work.tick(1)?; + let touch = if a.vertical == b.vertical { + (a.pos - b.pos).abs() < 0.75 && a.lo <= b.hi + 0.75 && b.lo <= a.hi + 0.75 + } else { + let (v, h) = if a.vertical { (a, b) } else { (b, a) }; + v.pos >= h.lo - 0.75 + && v.pos <= h.hi + 0.75 + && h.pos >= v.lo - 0.75 + && h.pos <= v.hi + 0.75 + }; + if touch { + sets.join(i, j); + } + } + } + let mut groups = BTreeMap::>::new(); + for (i, s) in segs.into_iter().enumerate() { + groups.entry(sets.root(i)).or_default().push(s); + } + let mut out = Vec::new(); + for segs in groups.into_values() { + let xs = unique(segs.iter().filter(|s| s.vertical).map(|s| s.pos).collect()); + let ys = unique(segs.iter().filter(|s| !s.vertical).map(|s| s.pos).collect()); + if xs.len() < 3 || ys.len() < 3 { + continue; + } + let (nr, nc) = (ys.len() - 1, xs.len() - 1); + grid_limit(nr, nc, cfg)?; + work.clock.add_nodes(nr * nc)?; + let bounds = [xs[0], ys[0], *xs.last().unwrap(), *ys.last().unwrap()]; + if words + .iter() + .enumerate() + .filter(|(i, w)| !used[*i] && inside(bounds, cx(w.rect), cy(w.rect))) + .count() + < 2 + { + continue; + } + let mut cells = Sets::new(nr * nc); + for r in 0..nr { + for c in 0..nc { + if c + 1 < nc && !covers(&segs, true, xs[c + 1], ys[r], ys[r + 1], work)? { + cells.join(r * nc + c, r * nc + c + 1); + } + if r + 1 < nr && !covers(&segs, false, ys[r + 1], xs[c], xs[c + 1], work)? { + cells.join(r * nc + c, (r + 1) * nc + c); + } + } + } + let mut groups = BTreeMap::>::new(); + for r in 0..nr { + for c in 0..nc { + groups + .entry(cells.root(r * nc + c)) + .or_default() + .push((r, c)); + } + } + if groups.values().any(|slots| { + let r0 = slots.iter().map(|x| x.0).min().unwrap(); + let r1 = slots.iter().map(|x| x.0).max().unwrap(); + let c0 = slots.iter().map(|x| x.1).min().unwrap(); + let c1 = slots.iter().map(|x| x.1).max().unwrap(); + (r1 - r0 + 1) * (c1 - c0 + 1) != slots.len() + }) { + continue; + } + let mut assigned = BTreeMap::>::new(); + let mut selected = Vec::new(); + for (i, w) in words.iter().enumerate() { + work.tick(1)?; + if !used[i] && inside(bounds, cx(w.rect), cy(w.rect)) { + let c = xs + .partition_point(|x| *x <= cx(w.rect)) + .saturating_sub(1) + .min(nc - 1); + let r = ys + .partition_point(|y| *y <= cy(w.rect)) + .saturating_sub(1) + .min(nr - 1); + assigned + .entry(cells.root(r * nc + c)) + .or_default() + .push(w.clone()); + selected.push(i); + } + } + let mut geometry = Vec::new(); + let mut table_rows = vec![Vec::new(); nr]; + let mut all = Vec::new(); + let mut ordered = groups + .into_iter() + .map(|(root, slots)| { + let r0 = slots.iter().map(|x| x.0).min().unwrap(); + let r1 = slots.iter().map(|x| x.0).max().unwrap(); + let c0 = slots.iter().map(|x| x.1).min().unwrap(); + let c1 = slots.iter().map(|x| x.1).max().unwrap(); + (r0, c0, r1, c1, root) + }) + .collect::>(); + ordered.sort_unstable(); + for (r, c, r1, c1, root) in ordered { + let mut ws = assigned.remove(&root).unwrap_or_default(); + ws.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.rect[0].total_cmp(&b.rect[0]))); + let cs = (c1 - c + 1) as u32; + let rs = (r1 - r + 1) as u32; + table_rows[r].push(TableCell::new(inline(&ws)).with_span(cs, rs)); + geometry.push(PdfCell { + row: r, + column: c, + rowspan: rs, + colspan: cs, + bbox: coords + .inverse() + .rect([xs[c], ys[r], xs[c1 + 1], ys[r1 + 1]]), + span_ids: ids(&ws), + }); + all.extend(ws); + } + for i in selected { + used[i] = true; + } + let headers = table_rows.remove(0); + out.push(Region { + rect: bounds, + lines: rows(all), + node: Some(Node::Table { + alignments: Vec::new(), + headers, + rows: table_rows, + source: None, + }), + cells: geometry, + removed: None, + }); + } + Ok(out) +} +fn numeric(s: &str) -> bool { + s.chars().any(|c| c.is_ascii_digit()) + && s.chars() + .all(|c| c.is_ascii_digit() || c.is_whitespace() || ".,%$€£¥+-−()/:".contains(c)) +} +fn borderless_tables( + rows: &mut [Vec], + coords: Coords, + cfg: &MarkkitBuilder, + work: &mut Work<'_>, +) -> Result, Error> { + let mut out = Vec::new(); + let mut i = 0; + while i < rows.len() { + work.tick(1)?; + let cols = rows[i].len(); + if cols < 2 { + i += 1; + continue; + } + let anchors = rows[i].iter().map(|l| l.rect).collect::>(); + let size = rows[i][0].size; + let mut logical = vec![rows[i].iter().cloned().map(|l| vec![l]).collect::>()]; + let mut end = i + 1; + while end < rows.len() && !rows[end].is_empty() { + work.tick(1)?; + let gap = rows[end][0].y - rows[end - 1][0].y; + if gap > 3.0 * size { + break; + } + if rows[end].len() == cols + && rows[end].iter().zip(&anchors).all(|(l, a)| { + (l.rect[0] - a[0]).abs() <= 2.0 * size || (l.rect[2] - a[2]).abs() <= 2.0 * size + }) + { + logical.push(rows[end].iter().cloned().map(|l| vec![l]).collect()); + } else if logical.len() > 1 + && rows[end].len() == 1 + && gap < 1.7 * size + && (rows[end][0].rect[0] - anchors[0][0]).abs() < size + && !numeric(&rows[end][0].text()) + { + logical.last_mut().unwrap()[0].push(rows[end][0].clone()); + } else { + break; + } + end += 1; + } + let n = logical.len(); + let compact = logical + .iter() + .flat_map(|r| r.iter()) + .flatten() + .all(|l| l.words.len() <= 5); + let data = n >= 3 + && (0..cols).any(|c| { + logical[1..] + .iter() + .filter(|r| numeric(&r[c].iter().map(Line::text).collect::>().join(" "))) + .count() + * 2 + >= n - 1 + }); + let explicit = rows[i].iter().any(|l| { + l.words + .iter() + .any(|w| w.bold || w.role.as_deref() == Some("TH")) + }); + if n < 3 || !compact || (!data && !explicit) { + i += 1; + continue; + } + grid_limit(n, cols, cfg)?; + work.clock.add_nodes(n * cols)?; + let mut geometry = Vec::new(); + let mut table_rows = Vec::new(); + let mut lines = Vec::new(); + let mut bounds = [ + f64::INFINITY, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NEG_INFINITY, + ]; + for (r, row) in logical.into_iter().enumerate() { + let mut cells = Vec::new(); + for (c, cell_lines) in row.into_iter().enumerate() { + let ws = cell_lines + .iter() + .flat_map(|l| l.words.iter().cloned()) + .collect::>(); + let rect = bbox(ws.iter().map(|w| w.rect)); + bounds = union(bounds, rect); + cells.push(TableCell::new(inline(&ws))); + geometry.push(PdfCell { + row: r, + column: c, + rowspan: 1, + colspan: 1, + bbox: coords.inverse().rect(rect), + span_ids: ids(&ws), + }); + lines.extend(cell_lines); + } + table_rows.push(cells); + } + for row in &mut rows[i..end] { + row.clear(); + } + let headers = table_rows.remove(0); + out.push(Region { + rect: bounds, + lines, + node: Some(Node::Table { + headers, + rows: table_rows, + alignments: Vec::new(), + source: None, + }), + cells: geometry, + removed: None, + }); + i = end; + } + Ok(out) +} + +fn order( + mut regions: Vec, + body: f64, + depth: usize, + out: &mut Vec>, + work: &mut Work<'_>, +) -> Result<(), Error> { + if regions.is_empty() { + return Ok(()); + } + work.tick(regions.len())?; + regions.sort_by(|a, b| { + a.rect[1] + .total_cmp(&b.rect[1]) + .then(a.rect[0].total_cmp(&b.rect[0])) + }); + let mut cuts = Vec::new(); + for (i, a) in regions.iter().enumerate() { + // Only spatial neighbors can bound a gutter. Keeping every pair's + // midpoint would allocate O(n²) candidates on a hostile single row. + for b in regions[i + 1..].iter().take(8) { + work.tick(1)?; + let (left, right) = if a.rect[0] < b.rect[0] { + (a, b) + } else { + (b, a) + }; + if (cy(left.rect) - cy(right.rect)).abs() < body + && right.rect[0] - left.rect[2] >= 0.8 * body + { + cuts.push((left.rect[2] + right.rect[0]) * 0.5); + } + } + } + let mut best = None; + let mut best_score = 0; + if depth < 12 { + for cut in unique(cuts) { + let left = regions + .iter() + .filter(|r| r.rect[2] < cut) + .collect::>(); + let right = regions + .iter() + .filter(|r| r.rect[0] > cut) + .collect::>(); + work.tick(regions.len())?; + if left.len() < 2 + || right.len() < 2 + || (left.len() + right.len()) * 3 < regions.len() * 2 + { + continue; + } + let ly0 = left.iter().map(|r| r.rect[1]).fold(f64::INFINITY, f64::min); + let ly1 = left + .iter() + .map(|r| r.rect[3]) + .fold(f64::NEG_INFINITY, f64::max); + let ry0 = right + .iter() + .map(|r| r.rect[1]) + .fold(f64::INFINITY, f64::min); + let ry1 = right + .iter() + .map(|r| r.rect[3]) + .fold(f64::NEG_INFINITY, f64::max); + if ly0 > ry1 || ry0 > ly1 { + continue; + } + let y_range = |items: &[&Region]| { + let lo = items + .iter() + .map(|r| cy(r.rect)) + .fold(f64::INFINITY, f64::min); + let hi = items + .iter() + .map(|r| cy(r.rect)) + .fold(f64::NEG_INFINITY, f64::max); + hi - lo + }; + if y_range(&left) < body * 0.5 || y_range(&right) < body * 0.5 { + continue; + } + let score = left.len().min(right.len()); + if score > best_score { + best = Some(cut); + best_score = score; + } + } + } + if let Some(cut) = best { + let mut left = Vec::new(); + let mut right = Vec::new(); + for r in regions { + let spanning = r.rect[0] <= cut && r.rect[2] >= cut; + if spanning { + order(std::mem::take(&mut left), body, depth + 1, out, work)?; + out.push(None); + order(std::mem::take(&mut right), body, depth + 1, out, work)?; + out.push(None); + out.push(Some(r)); + out.push(None); + } else if r.rect[2] < cut { + left.push(r); + } else { + right.push(r); + } + } + order(left, body, depth + 1, out, work)?; + out.push(None); + order(right, body, depth + 1, out, work)?; + out.push(None); + } else { + let mut merged: Vec = Vec::new(); + for mut r in regions { + if let Some(last) = merged.last_mut() + && last.node.is_none() + && r.node.is_none() + && last.removed.is_none() + && r.removed.is_none() + && (last.lines[0].y - r.lines[0].y).abs() < 0.4 * body + { + last.lines[0].words.append(&mut r.lines[0].words); + last.lines[0] + .words + .sort_by(|a, b| a.rect[0].total_cmp(&b.rect[0])); + last.rect = union(last.rect, r.rect); + last.lines[0].rect = last.rect; + } else { + merged.push(r); + } + } + out.extend(merged.into_iter().map(Some)); + } + Ok(()) +} diff --git a/crates/markkit/src/pdf/font.rs b/crates/markkit/src/pdf/font.rs index 7af7dff..09ca5ab 100644 --- a/crates/markkit/src/pdf/font.rs +++ b/crates/markkit/src/pdf/font.rs @@ -36,6 +36,8 @@ pub(crate) struct Font { /// Composite font with no ToUnicode: text cannot be mapped. pub unmapped: bool, pub name: String, + pub bold: bool, + pub italic: bool, } impl Font { @@ -57,6 +59,10 @@ impl Font { cid_widths: Vec::new(), default_width: 0.5, unmapped: false, + bold: name.to_ascii_lowercase().contains("bold") + || name.to_ascii_lowercase().contains("black"), + italic: name.to_ascii_lowercase().contains("italic") + || name.to_ascii_lowercase().contains("oblique"), name, }; @@ -106,6 +112,8 @@ impl Font { default_width: 0.5, unmapped: false, name: String::new(), + bold: false, + italic: false, } } diff --git a/crates/markkit/src/pdf/guard.rs b/crates/markkit/src/pdf/guard.rs index 09dc22f..428c169 100644 --- a/crates/markkit/src/pdf/guard.rs +++ b/crates/markkit/src/pdf/guard.rs @@ -15,7 +15,10 @@ //! Content streams, form XObjects and ToUnicode CMaps are decoded through //! [`decode`] (same bounded decoders), never hayro's unbounded `decoded()`. //! Image filters (DCT, JPX, JBIG2, CCITT) end a chain: markkit never decodes -//! images, and hayro is built without its `images` feature. +//! images. OCR-enabled builds also enable Hayro image codecs: structural streams +//! cannot use image filters, and rendering gets a stricter image preflight. +//! JPEG, header-bounded JPEG2000 and generic image streams are supported; inline images, image +//! DecodeParms, JBIG2/CCITT are refused for OCR until bounded decoding is available. use hayro_syntax::Filter; @@ -92,6 +95,24 @@ pub(crate) fn prescan( bytes: &[u8], stream_cap: usize, total_cap: usize, +) -> Result<(), PrescanError> { + prescan_inner(bytes, stream_cap, total_cap, false) +} + +#[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] +pub(crate) fn preflight_render( + bytes: &[u8], + stream_cap: usize, + total_cap: usize, +) -> Result<(), PrescanError> { + prescan_inner(bytes, stream_cap, total_cap, true) +} + +fn prescan_inner( + bytes: &[u8], + stream_cap: usize, + total_cap: usize, + render: bool, ) -> Result<(), PrescanError> { if find_key(bytes, b"/Encrypt").is_some() { return Err(PrescanError::Encrypted); @@ -111,13 +132,79 @@ pub(crate) fn prescan( Ok(c) => c, Err(()) => return Err(PrescanError::Unverifiable), }; - if chain.iter().all(|k| matches!(k, Kind::Image)) { - continue; + // Enabling the renderer unifies hayro-syntax's image feature. Do not + // accept declared object/xref streams with image codecs. + #[cfg(feature = "pdf-ocr")] + if chain.contains(&Kind::Image) + && (direct_name(dict, b"/Subtype").as_deref() != Some(b"Image".as_slice()) + || find_key(dict, b"/N").is_some() + || find_key(dict, b"/First").is_some() + || matches!( + direct_name(dict, b"/Type").as_deref(), + Some(b"ObjStm" | b"XRef") + )) + { + return Err(PrescanError::Unverifiable); } let end = find(&bytes[data_start..], b"endstream") .map(|e| data_start + e) .unwrap_or(bytes.len()); let budget = stream_cap.min(total_cap.saturating_sub(total)); + if render && direct_name(dict, b"/Subtype").as_deref() == Some(b"Image".as_slice()) { + // Reserve conservatively for color conversion, masks and 16-bit + // samples, not merely the final 1800px destination raster. + let width = direct_uint(dict, b"/Width").ok_or(PrescanError::Unverifiable)?; + let height = direct_uint(dict, b"/Height").ok_or(PrescanError::Unverifiable)?; + let size = width.checked_mul(height).and_then(|n| n.checked_mul(16)); + let Some(mut size) = size.filter(|n| width > 0 && height > 0 && *n <= budget) else { + return Err(PrescanError::Limit(LimitExceeded::new( + if budget < stream_cap { + LimitKind::MaxPdfDecodedSize + } else { + LimitKind::MaxPdfStreamSize + }, + "OCR image working storage exceeds PDF decode budget", + ))); + }; + if find_key(dict, b"/DecodeParms").is_some() { + return Err(PrescanError::RenderUnsupported); + } + if chain.contains(&Kind::Image) { + match direct_name(dict, b"/Filter").as_deref() { + // JPEG patches oversized SOF dimensions to the PDF bounds. + Some(b"DCTDecode" | b"DCT") => {} + Some(b"JPXDecode") => { + let pixels = jpx_pixels(&bytes[data_start..end]) + .ok_or(PrescanError::Unverifiable)?; + let working = + pixels + .checked_mul(16) + .filter(|n| *n <= budget) + .ok_or_else(|| { + PrescanError::Limit(LimitExceeded::new( + if budget < stream_cap { + LimitKind::MaxPdfDecodedSize + } else { + LimitKind::MaxPdfStreamSize + }, + "JPEG2000 codestream exceeds OCR image working budget", + )) + })?; + size = size.max(working); + } + _ => return Err(PrescanError::RenderUnsupported), + } + // Working buffers are released page by page; charge cumulative + // decoded RGBA bytes once, not every codec scratch copy. + total += size / 4; + continue; + } + // Generic filters still pass through bounded decoding below. + total += size / 4; + } else if !chain.is_empty() && chain.iter().all(|kind| *kind == Kind::Image) { + continue; + } + let out = decode(&chain, &bytes[data_start..end], budget).map_err(|_| { if budget < stream_cap { PrescanError::Limit(LimitExceeded::new( @@ -131,7 +218,22 @@ pub(crate) fn prescan( )) } })?; - total += out.len(); + if render && direct_name(dict, b"/Subtype").as_deref() != Some(b"Image".as_slice()) { + let mut instructions = hayro_syntax::content::UntypedIter::new(&out); + while let Some(instruction) = instructions.next() { + let op: &[u8] = instruction.operator; + if op == b"BI" { + return Err(PrescanError::RenderUnsupported); + } + } + } + total = total.saturating_add(out.len()); + if total > total_cap { + return Err(PrescanError::Limit(LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "OCR decoded storage exceeds max_pdf_decoded_size", + ))); + } } Ok(()) } @@ -140,9 +242,110 @@ pub(crate) fn prescan( pub(crate) enum PrescanError { Encrypted, Unverifiable, + RenderUnsupported, Limit(LimitExceeded), } +/// Bound JPEG2000 reference grid, tile area, tile count and channel precision +/// directly from SIZ before any codec allocation. JP2 boxes are length-walked, +/// never searched through arbitrary payload bytes. +fn jpx_pixels(mut data: &[u8]) -> Option { + if !data.starts_with(b"\xff\x4f") { + loop { + let short = u32::from_be_bytes(data.get(..4)?.try_into().ok()?) as usize; + let kind = data.get(4..8)?; + let (len, header) = match short { + 0 => (data.len(), 8), + 1 => ( + usize::try_from(u64::from_be_bytes(data.get(8..16)?.try_into().ok()?)).ok()?, + 16, + ), + n => (n, 8), + }; + if len < header || len > data.len() { + return None; + } + if kind == b"jp2c" { + data = &data[header..len]; + break; + } + data = &data[len..]; + } + } + if !data.starts_with(b"\xff\x4f\xff\x51") { + return None; + } + let read = |i| Some(u32::from_be_bytes(data.get(i..i + 4)?.try_into().ok()?) as usize); + let (w, h, x, y, tw, th, tx, ty) = ( + read(8)?, + read(12)?, + read(16)?, + read(20)?, + read(24)?, + read(28)?, + read(32)?, + read(36)?, + ); + if x >= w || y >= h || tw == 0 || th == 0 || tx > x || ty > y { + return None; + } + let tiles = (w - tx).div_ceil(tw).checked_mul((h - ty).div_ceil(th))?; + if tiles > 4096 { + return None; + } + let channels = u16::from_be_bytes(data.get(40..42)?.try_into().ok()?) as usize; + if !(1..=4).contains(&channels) { + return None; + } + let samples = data.get(42..42 + channels * 3)?; + if samples + .as_chunks::<3>() + .0 + .iter() + .any(|c| (c[0] & 127) >= 16 || c[1] == 0 || c[2] == 0) + { + return None; + } + Some(w.checked_mul(h)?.max(tw.checked_mul(th)?)) +} + +/// Conservative direct dictionary scalars. Ambiguous/indirect values fail closed. +fn direct_value<'a>(dict: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let at = find_key(dict, key)?.end; + if find_key(&dict[at..], key).is_some() { + return None; + } + Some(skip_ws(&dict[at..])) +} +fn direct_name<'a>(dict: &'a [u8], key: &[u8]) -> Option> { + let rest = direct_value(dict, key)?.strip_prefix(b"/")?; + let n = rest + .iter() + .position(|b| !is_regular(*b)) + .unwrap_or(rest.len()); + let name = &rest[..n]; + Some(if name.contains(&b'#') { + std::borrow::Cow::Owned(unescape_name(name)) + } else { + std::borrow::Cow::Borrowed(name) + }) +} +fn direct_uint(dict: &[u8], key: &[u8]) -> Option { + let rest = direct_value(dict, key)?; + let n = rest + .iter() + .position(|b| !b.is_ascii_digit()) + .unwrap_or(rest.len()); + if n == 0 || rest.get(n).is_some_and(|b| is_regular(*b)) { + return None; + } + // An indirect "W 1 0 R" must not pass as width=1. + if skip_ws(&rest[n..]).first().is_some_and(u8::is_ascii_digit) { + return None; + } + std::str::from_utf8(&rest[..n]).ok()?.parse().ok() +} + fn find(hay: &[u8], needle: &[u8]) -> Option { hay.windows(needle.len()).position(|w| w == needle) } @@ -170,16 +373,40 @@ fn is_regular(b: u8) -> bool { ) } -/// `/Key` followed by a delimiter (so `/Encrypt` does not match `/EncryptMetadata`). -fn find_key(hay: &[u8], key: &[u8]) -> Option { +/// Match PDF name keys, including #xx escapes. Return the complete raw name +/// range so callers advance past escaped keys correctly. Comparison allocates +/// nothing, even for a hostile name as long as the input. +fn find_key(hay: &[u8], key: &[u8]) -> Option> { let mut from = 0; - while let Some(rel) = find(&hay[from..], key) { - let at = from + rel; - let next = hay.get(at + key.len()).copied().unwrap_or(b' '); - if !is_regular(next) { - return Some(at); + while let Some(rel) = hay[from..].iter().position(|b| *b == b'/') { + let start = from + rel; + let raw = &hay[start + 1..]; + let len = raw + .iter() + .position(|b| !is_regular(*b)) + .unwrap_or(raw.len()); + let raw = &raw[..len]; + let mut offset = 0; + let equal = key[1..].iter().all(|expected| { + let Some(mut value) = raw.get(offset).copied() else { + return false; + }; + if value == b'#' + && let (Some(hi), Some(lo)) = ( + raw.get(offset + 1).and_then(hex), + raw.get(offset + 2).and_then(hex), + ) + { + value = hi << 4 | lo; + offset += 2; + } + offset += 1; + value == *expected + }); + if equal && offset == raw.len() { + return Some(start..start + 1 + len); } - from = at + key.len(); + from = start + 1 + len; } None } @@ -213,13 +440,13 @@ fn dict_before(bytes: &[u8], kw: usize) -> &[u8] { /// Parse `/Filter` from raw dictionary text. `Err` when it cannot be verified. fn filter_chain(dict: &[u8]) -> Result, ()> { - let Some(at) = find_key(dict, b"/Filter") else { + let Some(range) = find_key(dict, b"/Filter") else { return Ok(Vec::new()); }; - if find_key(&dict[at + 7..], b"/Filter").is_some() { + if find_key(&dict[range.end..], b"/Filter").is_some() { return Err(()); } - let mut rest = skip_ws(&dict[at + 7..]); + let mut rest = skip_ws(&dict[range.end..]); let array = rest.first() == Some(&b'['); if array { rest = &rest[1..]; @@ -578,4 +805,148 @@ mod tests { // Words containing "stream" are not stream keywords. assert!(prescan(b"(upstream)\n", 10, 10).is_ok()); } + #[test] + fn unfiltered_streams_obey_caps() { + assert!(matches!( + prescan(b"1 0 obj << >>\nstream\n123456789\nendstream", 4, 100), + Err(PrescanError::Limit(_)) + )); + } + + #[cfg(feature = "pdf-ocr")] + #[test] + fn image_codec_cannot_be_used_as_structural_stream() { + assert!(matches!( + prescan( + b"1 0 obj << /Type /ObjStm /Filter /JBIG2Decode >>\nstream\nx\nendstream", + 100, + 100 + ), + Err(PrescanError::Unverifiable) + )); + } + + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + #[test] + fn render_preflight_bounds_images_and_refuses_unbounded_codecs() { + let stream = |dict: &str, data: &[u8]| { + let mut bytes = format!("1 0 obj << {dict} >>\nstream\n").into_bytes(); + bytes.extend_from_slice(data); + bytes.extend_from_slice(b"\nendstream"); + bytes + }; + let image = "/Type /XObject /Subtype /Image /Width 10 /Height 10"; + assert!(preflight_render(&stream(image, &[255; 100]), 10000, 20000).is_ok()); + for extra in [ + "/Filter /JBIG2Decode", + "/Filter /CCITTFaxDecode", + "/DecodeParms << /Columns 999999999 >>", + ] { + assert!(matches!( + preflight_render(&stream(&format!("{image} {extra}"), b"x"), 10000, 20000), + Err(PrescanError::RenderUnsupported) + )); + } + assert!(matches!( + preflight_render( + &stream("/Subtype /Image /Width 999999999 /Height 999999999", b"x"), + 10000, + 20000 + ), + Err(PrescanError::Limit(_)) + )); + assert!(matches!( + preflight_render( + &stream("/Subtype /Image /Width 10 0 R /Height 10", b"x"), + 10000, + 20000 + ), + Err(PrescanError::Unverifiable) + )); + assert!(matches!( + preflight_render( + &stream("", b"BI /W 1 /H 1 /BPC 8 /CS /G ID x EI"), + 10000, + 20000 + ), + Err(PrescanError::RenderUnsupported) + )); + } + #[test] + fn jpeg2000_bounds_use_codestream_not_pdf_dictionary() { + let mut code = b"\xff\x4f\xff\x51\x00\x29\x00\x00".to_vec(); + for n in [100_u32, 200, 0, 0, 100, 200, 0, 0] { + code.extend(n.to_be_bytes()); + } + code.extend([0, 1, 7, 1, 1]); + assert_eq!(jpx_pixels(&code), Some(20000)); + let mut boxed = ((code.len() + 8) as u32).to_be_bytes().to_vec(); + boxed.extend(b"jp2c"); + boxed.extend(&code); + assert_eq!(jpx_pixels(&boxed), Some(20000)); + code[24..28].copy_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!(jpx_pixels(&code), (u32::MAX as usize).checked_mul(200)); + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + { + let mut stream = + b"1 0 obj << /Subtype /Image /Width 10 /Height 10 /Filter /JPXDecode >>\nstream\n" + .to_vec(); + stream.extend(&code); + stream.extend(b"\nendstream"); + assert!(matches!( + preflight_render(&stream, 10000, 20000), + Err(PrescanError::Limit(_)) + )); + } + code[40..42].copy_from_slice(&500_u16.to_be_bytes()); + assert_eq!(jpx_pixels(&code), None); + assert_eq!(jpx_pixels(b"\x00\x00\x00\x01jp2c"), None); + } + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + #[test] + fn sequential_image_working_buffers_are_not_charged_as_retained_bytes() { + let image = b"1 0 obj << /Subtype /Image /Width 10 /Height 10 /Filter /DCTDecode >>\nstream\nx\nendstream\nendobj\n"; + assert!(preflight_render(&image.repeat(5), 2000, 6500).is_ok()); + assert!(matches!( + preflight_render(image, 1500, 6500), + Err(PrescanError::Limit(_)) + )); + } + #[test] + fn image_filter_prefixes_cannot_bypass_decoded_caps() { + let mut bytes = b"1 0 obj << /Subtype /Image /Width 1 /Height 1 /Filter [/FlateDecode /DCTDecode] >>\nstream\n".to_vec(); + bytes.extend(zlib(&vec![0; 2000])); + bytes.extend(b"\nendstream"); + assert!(matches!( + prescan(&bytes, 1000, 10000), + Err(PrescanError::Limit(_)) + )); + } + + #[test] + fn escaped_dictionary_keys_cannot_bypass_stream_guards() { + let mut bytes = + b"1 0 obj << /Type /ObjStm /N 1 /First 4 /Fil#74er /FlateDecode >>\nstream\n".to_vec(); + bytes.extend(zlib(&vec![0; 2000])); + bytes.extend(b"\nendstream"); + assert!(matches!( + prescan(&bytes, 1000, 10000), + Err(PrescanError::Limit(_)) + )); + assert!(matches!( + prescan(b"trailer << /En#63rypt 5 0 R >>", 1000, 10000), + Err(PrescanError::Encrypted) + )); + assert_eq!(filter_chain(b"<< /Filter /Fl /Fil#74er /AHx >>"), Err(())); + } + + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + #[test] + fn escaped_image_names_and_dimensions_are_preflighted() { + let bytes = b"1 0 obj << /Sub#74ype /Ima#67e /W#69dth 999999999 /Height 999999999 >>\nstream\nx\nendstream"; + assert!(matches!( + preflight_render(bytes, 10000, 20000), + Err(PrescanError::Limit(_)) + )); + } } diff --git a/crates/markkit/src/pdf/layout.rs b/crates/markkit/src/pdf/layout.rs new file mode 100644 index 0000000..3b0c2e6 --- /dev/null +++ b/crates/markkit/src/pdf/layout.rs @@ -0,0 +1,299 @@ +//! Capture source geometry without changing the text-to-IR heuristics. +//! +//! Retained pages/runs count against max_nodes; storage is charged before +//! allocation against max_pdf_decoded_size. Compact layout JSON is separately +//! bounded by max_output_size (including parse-only callers), using a counting +//! writer so validation never allocates another copy of the layout. + +use std::io::{self, Write}; + +use hayro_syntax::page::{Page, Rotation}; + +use crate::layout::{PdfLayout, PdfPageLayout, PdfTextSpan, PdfWord}; +use crate::limits::{Budget, LimitExceeded, LimitKind}; +use crate::{Error, MarkkitBuilder}; + +use super::Matrix; + +pub(super) struct Capture { + pub output: PdfLayout, + pub(super) space: Space, + remaining: usize, +} + +impl Capture { + pub fn new(cfg: &MarkkitBuilder) -> Self { + Self { + output: PdfLayout { + version: 1, + pages: Vec::new(), + }, + space: Space { + origin: [0.0; 2], + width: 0.0, + height: 0.0, + unit: 1.0, + rotation: 0, + }, + remaining: cfg.limits.max_pdf_decoded_size, + } + } + + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + pub(super) fn available(&self) -> usize { + self.remaining + } + + pub(super) fn charge(&mut self, bytes: usize) -> Result<(), Error> { + self.remaining = self.remaining.checked_sub(bytes).ok_or_else(|| { + LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "PDF layout storage exceeds max_pdf_decoded_size", + ) + })?; + Ok(()) + } + + pub fn page(&mut self, page: &Page<'_>, clock: &Budget) -> Result<(), Error> { + clock.add_nodes(1)?; + self.charge(std::mem::size_of::())?; + self.space = Space::new(page)?; + let (width, height) = if self.space.rotation == 90 || self.space.rotation == 270 { + (self.space.height, self.space.width) + } else { + (self.space.width, self.space.height) + }; + self.output.pages.push(PdfPageLayout { + page: self.output.pages.len() as u32 + 1, + width, + height, + rotation: self.space.rotation, + spans: Vec::new(), + blocks: Vec::new(), + }); + Ok(()) + } + + pub fn span( + &mut self, + start: Matrix, + end: Matrix, + text: &str, + info: TextInfo<'_>, + clock: &Budget, + ) -> Result<(), Error> { + clock.add_nodes(1)?; + self.charge(std::mem::size_of::())?; + self.charge(text.len())?; + self.charge(info.font.map_or(0, str::len))?; + self.charge(info.role.map_or(0, str::len))?; + let mut words = Vec::new(); + for word in info.words { + clock.add_nodes(1)?; + self.charge(std::mem::size_of::() + word.text.len())?; + let (bbox, baseline) = self.space.bounds(word.start, word.end); + if !bbox + .iter() + .chain(baseline.iter().flatten()) + .all(|v| v.is_finite()) + { + return Err(Error::PdfExtract("non-finite PDF word geometry".into())); + } + words.push(PdfWord { + text: word.text, + bbox, + baseline, + }); + } + // f64 arithmetic here avoids overflow from otherwise finite f32 PDF + // coordinates, transforms and UserUnit. Never publish NaN/Infinity. + let apply = |m: Matrix, y: f64| { + let [_, _, c, d, e, f] = m.0.map(f64::from); + self.space.point(e + c * y, f + d * y) + }; + let baseline = [apply(start, 0.0), apply(end, 0.0)]; + let corners = [ + apply(start, -0.2), + apply(end, -0.2), + apply(start, 0.8), + apply(end, 0.8), + ]; + let mut bbox = [ + f64::INFINITY, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NEG_INFINITY, + ]; + for p in corners { + bbox[0] = bbox[0].min(p[0]); + bbox[1] = bbox[1].min(p[1]); + bbox[2] = bbox[2].max(p[0]); + bbox[3] = bbox[3].max(p[1]); + } + let [a, b, c, d, _, _] = start.0.map(f64::from); + let [dx, dy] = self.space.vector(a, b); + let [vx, vy] = self.space.vector(c, d); + let size = vx.hypot(vy); + let angle = dy.atan2(dx).to_degrees().rem_euclid(360.0); + if !bbox + .iter() + .chain(baseline.iter().flatten()) + .chain([size, angle].iter()) + .all(|v| v.is_finite()) + { + return Err(Error::PdfExtract("non-finite PDF text geometry".into())); + } + let page = self + .output + .pages + .last_mut() + .expect("page initialized before interpretation"); + page.spans.push(PdfTextSpan { + id: page.spans.len(), + text: text.to_string(), + bbox, + baseline, + words, + ocr: false, + bold: info.bold, + italic: info.italic, + role: info.role.map(str::to_string), + font: info.font.filter(|f| !f.is_empty()).map(str::to_string), + size, + angle, + }); + Ok(()) + } + + pub fn finish(self, cfg: &MarkkitBuilder, clock: &Budget) -> Result { + let mut writer = Count { + remaining: cfg.limits.max_output_size, + clock, + }; + if serde_json::to_writer(&mut writer, &self.output).is_err() { + clock.check_time()?; + return Err(LimitExceeded::new( + LimitKind::MaxOutputSize, + "PDF layout JSON exceeds max_output_size", + ) + .into()); + } + Ok(self.output) + } +} + +struct Count<'a> { + remaining: usize, + clock: &'a Budget, +} +impl Write for Count<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.clock.check_time().map_err(io::Error::other)?; + self.remaining = self + .remaining + .checked_sub(bytes.len()) + .ok_or_else(|| io::Error::other("layout output limit"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(super) struct Space { + origin: [f64; 2], + width: f64, + height: f64, + unit: f64, + rotation: u16, +} +impl Space { + fn new(page: &Page<'_>) -> Result { + let crop = page.intersected_crop_box(); + let unit = f64::from(page.raw().get::(b"UserUnit").unwrap_or(1.0)); + let width = crop.width() * unit; + let height = crop.height() * unit; + if ![crop.x0, crop.y0, width, height, unit] + .iter() + .all(|v| v.is_finite()) + || !(0.0 < unit && unit <= 75_000.0) + || width <= 0.0 + || height <= 0.0 + { + return Err(Error::PdfExtract( + "invalid PDF page geometry for layout extraction".into(), + )); + } + let rotation = match page.rotation() { + Rotation::None => 0, + Rotation::Horizontal => 90, + Rotation::Flipped => 180, + Rotation::FlippedHorizontal => 270, + }; + Ok(Self { + origin: [crop.x0, crop.y0], + width, + height, + unit, + rotation, + }) + } + fn vector(&self, x: f64, y: f64) -> [f64; 2] { + let (x, y) = (x * self.unit, y * self.unit); + match self.rotation { + 90 => [y, x], + 180 => [-x, y], + 270 => [-y, -x], + _ => [x, -y], + } + } + pub(super) fn point(&self, x: f64, y: f64) -> [f64; 2] { + let [x, y] = self.vector(x - self.origin[0], y - self.origin[1]); + match self.rotation { + 90 => [x, y], + 180 => [x + self.width, y], + 270 => [x + self.height, y + self.width], + _ => [x, y + self.height], + } + } +} + +pub(super) struct RawWord { + pub text: String, + pub start: Matrix, + pub end: Matrix, +} +pub(super) struct TextInfo<'a> { + pub font: Option<&'a str>, + pub bold: bool, + pub italic: bool, + pub role: Option<&'a str>, + pub words: Vec, +} +impl Space { + fn bounds(&self, start: Matrix, end: Matrix) -> ([f64; 4], [[f64; 2]; 2]) { + let apply = |m: Matrix, y: f64| { + let [_, _, c, d, e, f] = m.0.map(f64::from); + self.point(e + c * y, f + d * y) + }; + let corners = [ + apply(start, -0.2), + apply(start, 0.8), + apply(end, -0.2), + apply(end, 0.8), + ]; + let mut bbox = [ + f64::INFINITY, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NEG_INFINITY, + ]; + for p in corners { + bbox[0] = bbox[0].min(p[0]); + bbox[1] = bbox[1].min(p[1]); + bbox[2] = bbox[2].max(p[0]); + bbox[3] = bbox[3].max(p[1]); + } + (bbox, [apply(start, 0.0), apply(end, 0.0)]) + } +} diff --git a/crates/markkit/src/pdf/mod.rs b/crates/markkit/src/pdf/mod.rs index 1263f82..aa73efe 100644 --- a/crates/markkit/src/pdf/mod.rs +++ b/crates/markkit/src/pdf/mod.rs @@ -6,28 +6,24 @@ //! checked by [`guard::prescan`] before hayro sees the file, and all streams //! markkit interprets go through [`guard::decode`] (see guard.rs). //! -//! Text layer only: no OCR, no image extraction. Best-effort layout (see -//! knowledge/formats/format-support.md): -//! - Text-showing operators are interpreted with the text/graphics state -//! (`Tm`, `CTM`, `Tc`, `Tw`, `Tz`, `TL`, `Ts`, `TJ` kerning) to place each -//! run in device space. Form XObjects are followed (depth 8, cycle-safe). -//! - Runs join into lines in content-stream order (same baseline, moving right); -//! gaps wider than ~0.15 em become spaces. -//! - Lines join into paragraphs; a vertical gap above ~1.3× the page's typical -//! line pitch, a move upwards (new column), or a font-size change starts a new -//! one. `letter-` + lowercase continuation is de-hyphenated. -//! - A line starting with a bullet glyph (`•`, `▪`, `–`, …) plus a space starts a -//! list item; consecutive items form one unordered list. -//! - Headings: short paragraphs at ≥ 1.2× the dominant body size; the largest -//! sizes map to `#`, `##`, … (max 6). +//! Glyph geometry, tagged roles and vector rulings feed a bounded spatial +//! analyzer: reading order, paragraphs, headings, styled/numbered lists, ruled +//! tables with spans and conservative borderless tables. Optional native OCR +//! (`pdf-ocr`) recognizes textless pages using Hayro and installed Tesseract. +//! `pdf_layout(true)` exposes source runs/words and semantic block/cell geometry. //! //! Limits: `max_pdf_pages` (truncate + `truncated_pdf` warning), //! `max_pdf_stream_size` / `max_pdf_decoded_size` for decoding, and the same //! total as an interpretation budget, so a form XObject reused on every page //! cannot multiply CPU time without bound. +mod analyze; mod font; pub(crate) mod guard; +mod layout; +#[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] +mod ocr; +mod tags; use std::collections::HashMap; @@ -38,7 +34,7 @@ use hayro_syntax::page::Resources; use crate::config::MarkkitBuilder; use crate::error::Error; -use crate::ir::{Document, ListItem, Node, Source}; +use crate::ir::Document; use crate::limits::Budget; use font::{Font, Glyph}; @@ -46,8 +42,24 @@ use guard::PrescanError; const MAX_FORM_DEPTH: usize = 8; +fn prescan_error(e: PrescanError) -> Error { + match e { + PrescanError::Encrypted => { + Error::UnsupportedFormat("encrypted PDFs are not supported".into()) + } + PrescanError::Unverifiable => Error::PdfExtract( + "stream filter declared indirectly or ambiguously; refusing to decode".into(), + ), + PrescanError::RenderUnsupported => Error::UnsupportedFormat( + "OCR rendering supports JPEG, header-bounded JPEG2000 and generic-filter images without DecodeParms; inline, JBIG2 and CCITT images require bounded decoder support".into(), + ), + PrescanError::Limit(m) => Error::ResourceLimit(m), + } +} + /// Result of parsing a PDF. pub(crate) struct PdfOutput { + pub layout: Option, pub doc: Document, pub title: Option, pub pages: usize, @@ -64,15 +76,7 @@ pub(crate) fn parse( cfg.limits.max_pdf_stream_size, cfg.limits.max_pdf_decoded_size, ) - .map_err(|e| match e { - PrescanError::Encrypted => { - Error::UnsupportedFormat("encrypted PDFs are not supported".into()) - } - PrescanError::Unverifiable => Error::PdfExtract( - "stream filter declared indirectly or ambiguously; refusing to decode".into(), - ), - PrescanError::Limit(m) => Error::ResourceLimit(m), - })?; + .map_err(prescan_error)?; clock.check_time()?; @@ -98,10 +102,15 @@ pub(crate) fn parse( cfg.limits.max_pdf_decoded_size, clock, ); - let mut page_lines = Vec::new(); + interp.layout = Some(layout::Capture::new(cfg)); + interp.tag_roles = tags::roles(&pdf, cfg, clock)?; + let mut page_rules = Vec::new(); for page in pages.iter().take(cfg.limits.max_pdf_pages) { clock.check_time()?; - interp.spans.clear(); + interp.current_page = page.raw().obj_id(); + if let Some(layout) = &mut interp.layout { + layout.page(page, clock)?; + } let res = page.resources().clone(); if let Some(contents) = page.raw().get::>(b"Contents") { let streams: Vec> = match contents { @@ -117,7 +126,16 @@ pub(crate) fn parse( } interp.run(&data, &res, Matrix::IDENTITY, 0)?; } - page_lines.push(build_lines(std::mem::take(&mut interp.spans))); + let space = &interp.layout.as_ref().unwrap().space; + page_rules.push( + std::mem::take(&mut interp.rules) + .into_iter() + .map(|r| analyze::Rule { + a: space.point(r.a[0], r.a[1]), + b: space.point(r.b[0], r.b[1]), + }) + .collect::>(), + ); } if pages.len() > cfg.limits.max_pdf_pages { warnings.push(crate::extract::Warning::new( @@ -138,14 +156,28 @@ pub(crate) fn parse( )); } - let nodes = build_nodes(page_lines, cfg.track_sources); + if cfg.pdf_ocr { + #[cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] + ocr::recognize(bytes, &pdf, interp.layout.as_mut().unwrap(), cfg, clock)?; + #[cfg(not(all(feature = "pdf-ocr", not(target_arch = "wasm32"))))] + return Err(Error::UnsupportedFormat( + "native PDF OCR requires the pdf-ocr feature".into(), + )); + } + let capture = interp.layout.as_mut().unwrap(); + let nodes = analyze::analyze(&mut capture.output, &page_rules, cfg, clock)?; if nodes.is_empty() && !pages.is_empty() { warnings.push(crate::extract::Warning::new( crate::extract::WarningCode::PdfNoText, - "no text layer found (scanned PDF? OCR is not supported)", + "no extractable text found (enable PDF OCR for scanned pages)", )); } Ok(PdfOutput { + layout: if cfg.pdf_layout { + Some(interp.layout.take().unwrap().finish(cfg, clock)?) + } else { + None + }, doc: Document::new(nodes), title, pages: pages.len().min(cfg.limits.max_pdf_pages), @@ -212,6 +244,7 @@ impl Matrix { #[derive(Clone)] struct State { ctm: Matrix, + role: Option, font: Option, size: f32, char_spacing: f32, @@ -225,6 +258,7 @@ impl State { fn new(ctm: Matrix) -> Self { State { ctm, + role: None, font: None, size: 0.0, char_spacing: 0.0, @@ -236,21 +270,14 @@ impl State { } } -/// A run of text placed in device space. -#[derive(Debug, Clone)] -struct Span { - x0: f32, - x1: f32, - y: f32, - size: f32, - text: String, -} - struct Interpreter<'c> { + layout: Option, + rules: Vec, + tag_roles: tags::Roles, + current_page: Option, fonts: Vec, font_ids: HashMap, unmapped: std::collections::BTreeSet, - spans: Vec, stream_cap: usize, budget: usize, form_stack: Vec, @@ -274,10 +301,13 @@ fn num(o: Option<&Object<'_>>) -> f32 { impl<'c> Interpreter<'c> { fn new(stream_cap: usize, budget: usize, clock: &'c Budget) -> Self { Interpreter { + layout: None, + rules: Vec::new(), + tag_roles: Default::default(), + current_page: None, fonts: Vec::new(), font_ids: HashMap::new(), unmapped: Default::default(), - spans: Vec::new(), stream_cap, budget, form_stack: Vec::new(), @@ -337,6 +367,10 @@ impl<'c> Interpreter<'c> { let mut glyphs = Vec::new(); let mut ops = UntypedIter::new(data); let mut count = 0usize; + let mut path: Vec = Vec::new(); + let mut current = None; + let mut first = None; + let mut marks: Vec<(Option, Option, usize)> = Vec::new(); while let Some(ins) = ops.next() { // Each operator costs at least one budget unit. count += 1; @@ -350,6 +384,140 @@ impl<'c> Interpreter<'c> { let a = |i: usize| num(args.get(i).copied()); let op: &[u8] = ins.operator; match op { + b"BMC" | b"BDC" => { + if marks.len() >= 256 { + return Err(Error::PdfExtract( + "marked-content nesting exceeds 256".into(), + )); + } + let mut role = args.first().and_then(|o| match o { + Object::Name(n) => Some(font::name_of(Some((*n).clone()))), + _ => None, + }); + let props = args.get(1).and_then(|o| match o { + Object::Dict(d) => Some((*d).clone()), + Object::Name(n) => res.properties.get::>(n.as_ref()), + _ => None, + }); + if let (Some(page), Some(mcid)) = ( + self.current_page, + props.as_ref().and_then(|d| d.get::(b"MCID")), + ) && let Some(mapped) = self.tag_roles.get(&(page, mcid)) + { + role = Some(mapped.clone()); + } + if role.as_ref().is_some_and(|r| r.len() > 256) { + return Err(Error::PdfExtract( + "PDF marked-content role exceeds 256 bytes".into(), + )); + } + let replacement = props + .and_then(|d| d.get::>(b"ActualText")) + .map(|s| pdf_text_string(s.as_bytes())); + let index = self + .layout + .as_ref() + .map_or(0, |l| l.output.pages.last().unwrap().spans.len()); + marks.push((st.role.clone(), replacement, index)); + st.role = role; + } + b"EMC" => { + if let Some((role, replacement, index)) = marks.pop() { + if let (Some(text), Some(capture)) = (replacement, self.layout.as_mut()) { + capture.charge( + text.len().saturating_mul(2) + + std::mem::size_of::(), + )?; + self.clock.add_nodes(1)?; + let spans = &mut capture.output.pages.last_mut().unwrap().spans; + if index < spans.len() { + let bbox = spans[index..] + .iter() + .map(|s| s.bbox) + .reduce(analyze::union) + .unwrap(); + let baseline = + [spans[index].baseline[0], spans.last().unwrap().baseline[1]]; + spans.truncate(index + 1); + spans[index].text = text.clone(); + spans[index].bbox = bbox; + spans[index].baseline = baseline; + spans[index].words = vec![crate::layout::PdfWord { + text, + bbox, + baseline, + }]; + } + } + st.role = role; + } + } + b"m" | b"l" => { + let (x, y) = st.ctm.apply(a(0), a(1)); + let p = [f64::from(x), f64::from(y)]; + if p.iter().all(|x| x.is_finite()) { + if op == b"l" + && let Some(prev) = current + { + self.clock.add_nodes(1)?; + self.charge(32)?; + path.push(analyze::Rule { a: prev, b: p }); + } else { + first = Some(p); + } + current = Some(p); + } + } + b"h" => { + if let (Some(a), Some(b)) = (current, first) { + self.clock.add_nodes(1)?; + self.charge(32)?; + path.push(analyze::Rule { a, b }); + current = Some(b); + } + } + b"re" => { + let pts = [ + (a(0), a(1)), + (a(0) + a(2), a(1)), + (a(0) + a(2), a(1) + a(3)), + (a(0), a(1) + a(3)), + ] + .map(|(x, y)| { + let (x, y) = st.ctm.apply(x, y); + [f64::from(x), f64::from(y)] + }); + if pts.iter().flatten().all(|x| x.is_finite()) { + self.clock.add_nodes(4)?; + self.charge(128)?; + for i in 0..4 { + path.push(analyze::Rule { + a: pts[i], + b: pts[(i + 1) % 4], + }); + } + } + } + b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*" | b"f" | b"f*" | b"F" => { + let fill_only = matches!(op, b"f" | b"f*" | b"F"); + let thin = path + .iter() + .map(|r| [r.a[0], r.a[1], r.b[0], r.b[1]]) + .reduce(analyze::union) + .is_some_and(|b| (b[2] - b[0]).abs() < 2.0 || (b[3] - b[1]).abs() < 2.0); + if !fill_only || thin { + self.rules.append(&mut path); + } else { + path.clear(); + } + current = None; + first = None; + } + b"n" => { + path.clear(); + current = None; + first = None; + } b"q" => { if stack.len() < 256 { stack.push(st.clone()); @@ -405,7 +573,7 @@ impl<'c> Interpreter<'c> { } b"Tj" => { if let Some(Object::String(s)) = args.first() { - self.show(&st, &mut tm, s.as_bytes(), &mut glyphs); + self.show(&st, &mut tm, s.as_bytes(), &mut glyphs)?; } } b"'" | b"\"" => { @@ -416,12 +584,12 @@ impl<'c> Interpreter<'c> { tlm = Matrix::translate(0.0, -st.leading).then(&tlm); tm = tlm; if let Some(Object::String(s)) = args.last() { - self.show(&st, &mut tm, s.as_bytes(), &mut glyphs); + self.show(&st, &mut tm, s.as_bytes(), &mut glyphs)?; } } b"TJ" => { if let Some(Object::Array(arr)) = args.first() { - self.show_array(&st, &mut tm, arr, &mut glyphs); + self.show_array(&st, &mut tm, arr, &mut glyphs)?; } } b"Do" if depth < MAX_FORM_DEPTH => { @@ -485,10 +653,10 @@ impl<'c> Interpreter<'c> { tm: &mut Matrix, arr: &Array<'_>, glyphs: &mut Vec, - ) { + ) -> Result<(), Error> { for item in arr.iter::>() { match item { - Object::String(s) => self.show(st, tm, s.as_bytes(), glyphs), + Object::String(s) => self.show(st, tm, s.as_bytes(), glyphs)?, Object::Number(n) => { let adj = n.as_f32(); if !adj.is_finite() { @@ -496,20 +664,20 @@ impl<'c> Interpreter<'c> { } let tx = -adj / 1000.0 * st.size * st.scale; *tm = Matrix::translate(tx, 0.0).then(tm); - // A large negative adjustment is a word gap. - if -adj > 200.0 - && let Some(last) = self.spans.last_mut() - && !last.text.ends_with(' ') - { - last.text.push(' '); - } } _ => {} } } + Ok(()) } - fn show(&mut self, st: &State, tm: &mut Matrix, bytes: &[u8], glyphs: &mut Vec) { + fn show( + &mut self, + st: &State, + tm: &mut Matrix, + bytes: &[u8], + glyphs: &mut Vec, + ) -> Result<(), Error> { glyphs.clear(); match st.font { Some(i) => { @@ -526,10 +694,10 @@ impl<'c> Interpreter<'c> { // No (resolvable) font selected: best effort as WinAnsi. None => Font::fallback().decode(bytes, glyphs), } - self.place(st, tm, glyphs); + self.place(st, tm, glyphs) } - fn place(&mut self, st: &State, tm: &mut Matrix, glyphs: &[Glyph]) { + fn place(&mut self, st: &State, tm: &mut Matrix, glyphs: &[Glyph]) -> Result<(), Error> { let trm = |tm: &Matrix| { Matrix([st.size * st.scale, 0.0, 0.0, st.size, 0.0, st.rise]) .then(tm) @@ -539,7 +707,28 @@ impl<'c> Interpreter<'c> { let (x0, y) = start.apply(0.0, 0.0); let size = (start.0[2].hypot(start.0[3])).abs(); let mut text = String::new(); + let mut words = Vec::new(); + let mut word = String::new(); + let mut word_start = start; for g in glyphs { + let before = trm(tm); + let whitespace = g + .text + .as_ref() + .is_some_and(|t| t.chars().all(char::is_whitespace)); + if whitespace && !word.is_empty() { + words.push(layout::RawWord { + text: std::mem::take(&mut word), + start: word_start, + end: before, + }); + } + if !whitespace && let Some(t) = &g.text { + if word.is_empty() { + word_start = before; + } + push_clean(&mut word, t); + } if let Some(t) = &g.text { push_clean(&mut text, t); } @@ -549,17 +738,32 @@ impl<'c> Interpreter<'c> { } *tm = Matrix::translate(tx * st.scale, 0.0).then(tm); } - let (x1, _) = trm(tm).apply(0.0, 0.0); + if !word.is_empty() { + words.push(layout::RawWord { + text: word, + start: word_start, + end: trm(tm), + }); + } if text.is_empty() || !x0.is_finite() || !y.is_finite() || !size.is_finite() { - return; + return Ok(()); } - self.spans.push(Span { - x0, - x1: x1.max(x0), - y, - size: size.max(0.1), - text, - }); + if let Some(layout) = &mut self.layout { + layout.span( + start, + trm(tm), + &text, + layout::TextInfo { + font: st.font.map(|i| self.fonts[i].name.as_str()), + bold: st.font.is_some_and(|i| self.fonts[i].bold), + italic: st.font.is_some_and(|i| self.fonts[i].italic), + role: st.role.as_deref(), + words, + }, + self.clock, + )?; + } + Ok(()) } } @@ -581,266 +785,12 @@ fn push_clean(out: &mut String, t: &str) { } } -// ─── Layout ───────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -struct Line { - y: f32, - size: f32, - x1: f32, - text: String, -} - -fn build_lines(spans: Vec) -> Vec { - let mut lines: Vec = Vec::new(); - for s in spans { - if let Some(line) = lines.last_mut() { - let tol = 0.5 * s.size.max(line.size); - let same_baseline = (s.y - line.y).abs() <= tol; - let gap = s.x0 - line.x1; - if same_baseline && gap > -0.5 * s.size { - if gap > 0.15 * s.size && !line.text.ends_with(' ') && !s.text.starts_with(' ') { - line.text.push(' '); - } - line.text.push_str(&s.text); - line.x1 = line.x1.max(s.x1); - // Superscripts / subscripts keep the line's baseline and size. - if s.size > line.size { - line.size = s.size; - } - continue; - } - } - lines.push(Line { - y: s.y, - size: s.size, - x1: s.x1, - text: s.text, - }); - } - for l in &mut lines { - l.text = l.text.split_whitespace().collect::>().join(" "); - } - lines.retain(|l| !l.text.is_empty()); - lines -} - -struct Para { - /// 1-based page number. - page: u32, - size: f32, - lines: usize, - text: String, - bullet: bool, -} - const BULLETS: &[char] = &['•', '●', '▪', '◦', '‣', '∙', '■', '–']; -/// `• text` → `Some("text")`. -fn strip_bullet(text: &str) -> Option<&str> { - let mut it = text.chars(); - let c = it.next()?; - let rest = it.as_str(); - (BULLETS.contains(&c) && rest.starts_with(' ')).then(|| rest.trim_start()) -} - -fn round_size(s: f32) -> i32 { - (s * 2.0).round() as i32 -} - -/// Lines per page → blocks; `track` records `Source::page` on each block. -fn build_nodes(pages: Vec>, track: bool) -> Vec { - let mut paras: Vec = Vec::new(); - for (page_idx, lines) in pages.into_iter().enumerate() { - let page = u32::try_from(page_idx + 1).unwrap_or(u32::MAX); - // Typical line pitch: most common downward step between lines (ties → - // smaller), so paragraph gaps stand out even when frequent. - let mut steps: HashMap = HashMap::new(); - for w in lines.windows(2) { - let d = w[0].y - w[1].y; - if d > 0.0 { - *steps.entry(round_size(d)).or_default() += 1; - } - } - let pitch = steps - .iter() - .max_by_key(|(d, n)| (**n, -**d)) - .map(|(d, _)| *d as f32 / 2.0); - - let mut prev: Option<&Line> = None; - for line in &lines { - let new_para = match prev { - None => true, - Some(p) => { - let step = p.y - line.y; - let size_change = (line.size - p.size).abs() > 0.15 * p.size.max(line.size); - let limit = pitch.unwrap_or(1.2 * p.size).max(p.size * 0.8) * 1.3; - size_change || step < -0.5 * line.size || step > limit - } - }; - let bullet = strip_bullet(&line.text); - if new_para || bullet.is_some() { - paras.push(Para { - page, - size: line.size, - lines: 1, - text: bullet.unwrap_or(&line.text).to_string(), - bullet: bullet.is_some(), - }); - } else if let Some(para) = paras.last_mut() { - join_line(&mut para.text, &line.text); - para.lines += 1; - } - prev = Some(line); - } - } - - // Dominant body size by character count. - let mut by_size: HashMap = HashMap::new(); - for p in ¶s { - *by_size.entry(round_size(p.size)).or_default() += p.text.chars().count(); - } - let body = by_size - .iter() - .max_by_key(|(s, n)| (**n, -**s)) - .map(|(s, _)| *s as f32 / 2.0) - .unwrap_or(0.0); - let is_heading = |p: &Para| { - body > 0.0 - && p.size >= body * 1.2 - && p.lines <= 3 - && p.text.chars().count() <= 200 - && p.text.chars().any(char::is_alphabetic) - }; - let mut heading_sizes: Vec = paras - .iter() - .filter(|p| is_heading(p)) - .map(|p| round_size(p.size)) - .collect(); - heading_sizes.sort_unstable_by(|a, b| b.cmp(a)); - heading_sizes.dedup(); - - let mut nodes = Vec::new(); - for p in ¶s { - let children = vec![Node::Text(p.text.clone())]; - let source = track.then(|| Box::new(Source::default().with_page(p.page))); - if p.bullet { - let item = ListItem::new(vec![Node::Paragraph { - children, - source: None, - }]); - if let Some(Node::List { items, .. }) = nodes.last_mut() { - items.push(item); - } else { - nodes.push(Node::List { - ordered: false, - start: None, - items: vec![item], - source, - }); - } - } else if is_heading(p) { - let rank = heading_sizes - .iter() - .position(|s| *s == round_size(p.size)) - .unwrap_or(0); - nodes.push(Node::Heading { - level: (rank + 1).min(6) as u8, - children, - source, - }); - } else { - nodes.push(Node::Paragraph { children, source }); - } - } - nodes -} - -/// Append a line to a paragraph, undoing end-of-line hyphenation. -fn join_line(para: &mut String, line: &str) { - let mut tail = para.chars().rev(); - let dehyphenate = tail.next() == Some('-') - && tail.next().is_some_and(char::is_alphabetic) - && line.chars().next().is_some_and(char::is_lowercase); - if dehyphenate || para.ends_with('\u{AD}') { - para.pop(); - } else { - para.push(' '); - } - para.push_str(line); -} - #[cfg(test)] mod tests { use super::*; - fn line(y: f32, size: f32, text: &str) -> Line { - Line { - y, - size, - x1: 0.0, - text: text.into(), - } - } - - #[test] - fn spans_join_into_lines_with_spaces() { - let spans = vec![ - Span { - x0: 0.0, - x1: 20.0, - y: 100.0, - size: 10.0, - text: "Hello".into(), - }, - Span { - x0: 23.0, - x1: 50.0, - y: 100.0, - size: 10.0, - text: "world".into(), - }, - Span { - x0: 50.0, - x1: 60.0, - y: 100.0, - size: 10.0, - text: "!".into(), - }, - Span { - x0: 0.0, - x1: 20.0, - y: 88.0, - size: 10.0, - text: "Next".into(), - }, - ]; - let lines = build_lines(spans); - assert_eq!(lines.len(), 2); - assert_eq!(lines[0].text, "Hello world!"); - } - - #[test] - fn paragraphs_headings_and_hyphenation() { - let page = vec![ - line(700.0, 20.0, "Title"), - line(660.0, 10.0, "First para-"), - line(648.0, 10.0, "graph goes on."), - line(636.0, 10.0, "Still first."), - line(610.0, 10.0, "Second paragraph."), - ]; - let nodes = build_nodes(vec![page], false); - assert_eq!(nodes.len(), 3); - assert!(matches!(&nodes[0], Node::Heading { level: 1, .. })); - assert_eq!( - nodes[1], - Node::Paragraph { - children: vec![Node::Text("First paragraph goes on. Still first.".into())], - source: None, - } - ); - } - #[test] fn text_strings() { assert_eq!(pdf_text_string(b"\xFE\xFF\x00H\x00i"), "Hi"); diff --git a/crates/markkit/src/pdf/ocr.rs b/crates/markkit/src/pdf/ocr.rs new file mode 100644 index 0000000..d7ec34b --- /dev/null +++ b/crates/markkit/src/pdf/ocr.rs @@ -0,0 +1,406 @@ +//! Optional OCR: Hayro renders in Rust; Tesseract or ocrs emits positioned words. +//! Decision: no Poppler executable. Render onto white at most 1800×1800, +//! export bounded grayscale PGM, keep files private. Preflight guards image +//! allocations/codecs. Hayro is non-interruptible: deadlines are checked before +//! and after rendering; hosts still need memory/CPU isolation for hostile PDFs. +//! Tesseract is killed and reaped on deadline/file-size violations. + +#[cfg(feature = "pdf-ocr-ocrs")] +mod rust; + +use super::layout::Capture; +use crate::layout::{PdfTextSpan, PdfWord}; +use crate::limits::{Budget, LimitExceeded, LimitKind}; +use crate::{Error, MarkkitBuilder, PdfOcrBackend}; +use std::fs; +use std::io::Read; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +fn run( + command: &mut Command, + output: &Path, + cap: usize, + kind: LimitKind, + clock: &Budget, +) -> Result<(), Error> { + clock.check_time()?; + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| { + Error::PdfExtract(format!( + "OCR helper unavailable: {e}; install tesseract and language data" + )) + })?; + loop { + let error = clock.check_time().err().or_else(|| { + fs::metadata(output) + .ok() + .filter(|m| m.len() > cap as u64) + .map(|_| { + LimitExceeded::new(kind, format!("OCR helper output exceeds {kind}")).into() + }) + }); + if let Some(e) = error { + let _ = child.kill(); + let _ = child.wait(); + return Err(e); + } + match child.try_wait() { + Ok(Some(s)) => { + return if s.success() { + Ok(()) + } else { + Err(Error::PdfExtract(format!( + "OCR helper failed with {s}; check installed Tesseract languages" + ))) + }; + } + Ok(None) => thread::sleep(Duration::from_millis(10)), + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::PdfExtract(format!("OCR helper failed: {e}"))); + } + } + } +} +fn read_bounded(path: &Path, cap: usize, kind: LimitKind) -> Result, Error> { + let file = fs::File::open(path) + .map_err(|e| Error::PdfExtract(format!("OCR output unavailable: {e}")))?; + let mut bytes = Vec::new(); + file.take((cap as u64).saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|e| Error::PdfExtract(format!("OCR output unreadable: {e}")))?; + if bytes.len() > cap { + return Err(LimitExceeded::new(kind, format!("OCR output exceeds {kind}")).into()); + } + Ok(bytes) +} + +fn raster_dimensions(w: f32, h: f32) -> Result<(u32, u32), Error> { + if !w.is_finite() || !h.is_finite() || w <= 0.0 || h <= 0.0 { + return Err(Error::PdfExtract("invalid OCR page dimensions".into())); + } + // f32 multiplication can round 1800 up past the limit before ceil. + let scale = 1800.0 / f64::from(w.max(h)); + let width = (f64::from(w) * scale).ceil().min(1800.0) as u32; + let height = (f64::from(h) * scale).ceil().min(1800.0) as u32; + // Hayro allocates from f32-scaled dimensions. An extreme aspect ratio can + // make a scale infinite even though the f64 target dimensions are bounded. + let sx = (width as f32 + 0.001) / w; + let sy = (height as f32 + 0.001) / h; + if !sx.is_finite() || !sy.is_finite() || (w * sx) as u32 != width || (h * sy) as u32 != height { + return Err(Error::PdfExtract("invalid OCR raster scale".into())); + } + Ok((width, height)) +} + +fn rasterize( + page: &hayro_syntax::page::Page<'_>, + cap: usize, + clock: &Budget, +) -> Result<(Vec, u32, u32), Error> { + use hayro::{ + PixmapSettings, RenderCache, RenderSettings, hayro_interpret::InterpreterSettings, + }; + clock.check_time()?; + let (w, h) = page.render_dimensions(); + let (width, height) = raster_dimensions(w, h)?; + let header = format!("P5\n{width} {height}\n255\n"); + let len = width as usize * height as usize + header.len(); + // Charge the RGBA raster as well as its grayscale export before rendering. + if (width as usize * height as usize * 4).max(len) > cap { + return Err(LimitExceeded::new( + LimitKind::MaxPdfStreamSize, + "OCR raster exceeds max_pdf_stream_size", + ) + .into()); + } + let pixmap = hayro::render( + page, + &RenderCache::new(), + &InterpreterSettings { + render_annotations: false, + ..Default::default() + }, + &RenderSettings { + force_image_interpolation: true, + }, + &PixmapSettings { + x_scale: (width as f32 + 0.001) / w, + y_scale: (height as f32 + 0.001) / h, + bg_color: hayro::vello_cpu::color::palette::css::WHITE, + }, + ); + clock.check_time()?; + let mut pgm = Vec::with_capacity(len); + pgm.extend_from_slice(header.as_bytes()); + for rgba in pixmap.data_as_u8_slice().as_chunks::<4>().0 { + pgm.push( + ((299 * u32::from(rgba[0]) + 587 * u32::from(rgba[1]) + 114 * u32::from(rgba[2]) + 500) + / 1000) as u8, + ); + } + Ok((pgm, width, height)) +} + +pub(super) fn recognize( + bytes: &[u8], + pdf: &hayro_syntax::Pdf, + capture: &mut Capture, + cfg: &MarkkitBuilder, + clock: &Budget, +) -> Result<(), Error> { + if cfg.pdf_ocr_language.is_empty() + || cfg.pdf_ocr_language.len() > 128 + || !cfg + .pdf_ocr_language + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_+".contains(&b)) + { + return Err(Error::PdfExtract( + "invalid OCR language; use installed language codes such as eng+deu".into(), + )); + } + if capture.output.pages.iter().all(|p| !p.spans.is_empty()) { + return Ok(()); + } + if cfg.pdf_ocr_backend == PdfOcrBackend::Ocrs { + #[cfg(not(feature = "pdf-ocr-ocrs"))] + return Err(Error::UnsupportedFormat( + "ocrs requires the pdf-ocr-ocrs feature".into(), + )); + #[cfg(feature = "pdf-ocr-ocrs")] + if cfg.pdf_ocr_language != "eng" { + return Err(Error::UnsupportedFormat( + "ocrs uses its Latin-alphabet model; pdf_ocr_language must be eng".into(), + )); + } + } + super::guard::preflight_render( + bytes, + cfg.limits.max_pdf_stream_size, + cfg.limits.max_pdf_decoded_size, + ) + .map_err(super::prescan_error)?; + #[cfg(feature = "pdf-ocr-ocrs")] + let engine = if cfg.pdf_ocr_backend == PdfOcrBackend::Ocrs { + Some(rust::Engine::load(cfg, clock)?) + } else { + None + }; + let dir = + if cfg.pdf_ocr_backend == PdfOcrBackend::Tesseract { + Some(tempfile::tempdir().map_err(|e| { + Error::PdfExtract(format!("cannot create private OCR directory: {e}")) + })?) + } else { + None + }; + for index in 0..capture.output.pages.len() { + if !capture.output.pages[index].spans.is_empty() { + continue; + } + clock.check_time()?; + let (raster, width, height) = + rasterize(&pdf.pages()[index], cfg.limits.max_pdf_stream_size, clock)?; + let spans = if cfg.pdf_ocr_backend == PdfOcrBackend::Ocrs { + #[cfg(feature = "pdf-ocr-ocrs")] + { + let page = &capture.output.pages[index]; + engine.as_ref().unwrap().recognize( + &raster[raster.len() - width as usize * height as usize..], + (width, height), + (page.width, page.height), + capture.available(), + clock, + )? + } + #[cfg(not(feature = "pdf-ocr-ocrs"))] + return Err(Error::UnsupportedFormat( + "ocrs requires the pdf-ocr-ocrs feature".into(), + )); + } else { + let base = dir.as_ref().unwrap().path().join("page"); + let pgm = base.with_extension("pgm"); + let tsv = base.with_extension("tsv"); + fs::write(&pgm, &raster) + .map_err(|e| Error::PdfExtract(format!("cannot stage OCR raster: {e}")))?; + // PGM has no resolution metadata; preserve the displayed page DPI + // rather than relying on Tesseract's fallback/estimation. + let page = &capture.output.pages[index]; + let dpi = (72.0 * f64::from(width) / page.width) + .round() + .clamp(70.0, 2400.0) + .to_string(); + run( + Command::new("tesseract") + .current_dir(dir.as_ref().unwrap().path()) + .env("OMP_THREAD_LIMIT", "1") + .arg(&pgm) + .arg(&base) + .arg("-l") + .arg(&cfg.pdf_ocr_language) + .args(["--dpi", &dpi]) + .arg("tsv"), + &tsv, + cfg.limits.max_pdf_decoded_size, + LimitKind::MaxPdfDecodedSize, + clock, + )?; + let data = read_bounded( + &tsv, + cfg.limits.max_pdf_decoded_size, + LimitKind::MaxPdfDecodedSize, + )?; + let (w, h) = { + let p = &capture.output.pages[index]; + (p.width, p.height) + }; + let spans = parse_tsv( + &data, + w / f64::from(width), + h / f64::from(height), + capture.available(), + clock, + )?; + let _ = fs::remove_file(pgm); + let _ = fs::remove_file(tsv); + spans + }; + for span in spans { + capture.charge( + std::mem::size_of::() + + std::mem::size_of::() + + span.text.len() * 2, + )?; + capture.output.pages[index].spans.push(span); + } + } + Ok(()) +} +fn word_span(id: usize, text: String, bbox: [f64; 4]) -> PdfTextSpan { + let baseline = [[bbox[0], bbox[3]], [bbox[2], bbox[3]]]; + PdfTextSpan { + id, + words: vec![PdfWord { + text: text.clone(), + bbox, + baseline, + }], + text, + bbox, + baseline, + font: None, + size: bbox[3] - bbox[1], + angle: 0.0, + bold: false, + italic: false, + role: None, + ocr: true, + } +} + +fn parse_tsv( + data: &[u8], + sx: f64, + sy: f64, + mut storage: usize, + clock: &Budget, +) -> Result, Error> { + let mut spans = Vec::new(); + for line in String::from_utf8_lossy(data).lines().skip(1) { + clock.check_time()?; + let fields = line.splitn(12, '\t').collect::>(); + if fields.len() != 12 || fields[0] != "5" { + continue; + } + let values = fields[6..10] + .iter() + .map(|v| v.parse::()) + .collect::, _>>(); + let Ok(v) = values else { + continue; + }; + if v.iter().any(|v| !v.is_finite() || *v < 0.0 || *v > 1800.0) || v[2] == 0.0 || v[3] == 0.0 + { + continue; + } + let mut text = String::new(); + super::push_clean(&mut text, fields[11]); + if text.trim().is_empty() { + continue; + } + storage = storage + .checked_sub( + std::mem::size_of::() + + std::mem::size_of::() + + text.len().saturating_mul(2), + ) + .ok_or_else(|| { + LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "OCR word storage exceeds max_pdf_decoded_size", + ) + })?; + clock.add_nodes(2)?; + let bbox = [v[0] * sx, v[1] * sy, (v[0] + v[2]) * sx, (v[1] + v[3]) * sy]; + let mut span = word_span(spans.len(), text, bbox); + span.size = v[3] * sy; + spans.push(span); + } + Ok(spans) +} +#[cfg(test)] +mod tests { + #[test] + fn fractional_page_dimensions_stay_inside_raster_limit() { + for (w, h) in [(583.68, 760.32), (576.0, 737.28), (760.32, 583.68)] { + let (x, y) = super::raster_dimensions(w, h).unwrap(); + assert_eq!(x.max(y), 1800); + assert!(x > 0 && y > 0); + } + assert!(super::raster_dimensions(f32::NAN, 100.0).is_err()); + assert!(super::raster_dimensions(0.0, 100.0).is_err()); + assert!(super::raster_dimensions(f32::from_bits(1), 100.0).is_err()); + } + + use super::*; + #[test] + fn tsv_words_are_scaled_and_malformed_rows_skipped() { + let b = Budget::new(&crate::Limits::default()); + let data=b"level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t10\t95\tHello\n5\t1\t1\t1\t1\t2\tNaN\t20\t30\t10\t95\tbad\n"; + let out = parse_tsv(data, 0.5, 0.5, 65536, &b).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].bbox, [5.0, 10.0, 20.0, 15.0]); + assert!(out[0].ocr); + } + #[test] + fn output_caps_keep_the_limit_kind() { + let f = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(f.path(), b"oversized").unwrap(); + let err = read_bounded(f.path(), 2, LimitKind::MaxPdfStreamSize).unwrap_err(); + assert!(matches!(err,Error::ResourceLimit(ref e) if e.kind==LimitKind::MaxPdfStreamSize)); + } + #[cfg(unix)] + #[test] + fn expired_helper_is_killed_and_reaped() { + let d = tempfile::tempdir().unwrap(); + let b = Budget::new(&crate::Limits::default().timeout(Duration::from_millis(30))); + let err = run( + Command::new("sleep").arg("2"), + &d.path().join("unused"), + 100, + LimitKind::MaxPdfDecodedSize, + &b, + ) + .unwrap_err(); + assert!(matches!(err,Error::ResourceLimit(ref e) if e.kind==LimitKind::Timeout)); + } +} diff --git a/crates/markkit/src/pdf/ocr/rust.rs b/crates/markkit/src/pdf/ocr/rust.rs new file mode 100644 index 0000000..447f4f5 --- /dev/null +++ b/crates/markkit/src/pdf/ocr/rust.rs @@ -0,0 +1,131 @@ +//! Rust OCR with explicitly configured local ONNX models. No downloads, global +//! model cache, process execution or language fallback. Load once per document; +//! check deadlines around model operations and between bounded line batches. +//! Models are trusted deployment assets; inference allocations need host quotas. + +use super::{read_bounded, word_span}; +use crate::layout::{PdfTextSpan, PdfWord}; +use crate::limits::{Budget, LimitExceeded, LimitKind}; +use crate::{Error, MarkkitBuilder}; +use ocrs::{ImageSource, OcrEngine, OcrEngineParams, TextItem}; + +pub(super) struct Engine(OcrEngine); +fn error(e: impl std::fmt::Display) -> Error { + Error::PdfExtract(format!("ocrs: {e}")) +} +impl Engine { + pub(super) fn load(cfg: &MarkkitBuilder, clock: &Budget) -> Result { + let detection = cfg + .pdf_ocr_detection_model + .as_deref() + .ok_or_else(|| error("set pdf_ocr_detection_model to a local ONNX model"))?; + let recognition = cfg + .pdf_ocr_recognition_model + .as_deref() + .ok_or_else(|| error("set pdf_ocr_recognition_model to a local ONNX model"))?; + clock.check_time()?; + let bytes = read_bounded( + detection, + cfg.limits.max_pdf_decoded_size, + LimitKind::MaxPdfDecodedSize, + )?; + let remaining = cfg.limits.max_pdf_decoded_size.saturating_sub(bytes.len()); + let detection_model = rten::Model::load(bytes).map_err(error)?; + clock.check_time()?; + let bytes = read_bounded(recognition, remaining, LimitKind::MaxPdfDecodedSize)?; + let recognition_model = rten::Model::load(bytes).map_err(error)?; + clock.check_time()?; + let engine = OcrEngine::new(OcrEngineParams { + detection_model: Some(detection_model), + recognition_model: Some(recognition_model), + ..Default::default() + }) + .map_err(error)?; + clock.check_time()?; + Ok(Self(engine)) + } + + pub(super) fn recognize( + &self, + pixels: &[u8], + dims: (u32, u32), + page: (f64, f64), + mut storage: usize, + clock: &Budget, + ) -> Result, Error> { + clock.check_time()?; + let input = self + .0 + .prepare_input(ImageSource::from_bytes(pixels, dims).map_err(error)?) + .map_err(error)?; + clock.check_time()?; + let words = self.0.detect_words(&input).map_err(error)?; + clock.check_time()?; + clock.add_nodes(words.len())?; + let lines = self.0.find_text_lines(&input, &words); + clock.check_time()?; + let mut spans = Vec::new(); + let scale = (page.0 / f64::from(dims.0), page.1 / f64::from(dims.1)); + // Keep RTen batching while checking deadlines between groups. + for batch in lines.chunks(20) { + clock.check_time()?; + let texts = self.0.recognize_text(&input, batch).map_err(error)?; + clock.check_time()?; + for text in texts.into_iter().flatten() { + for word in text.words() { + // Check before constructing and cloning the output string. + let len = word + .chars() + .iter() + .map(|c| c.char.len_utf8()) + .sum::(); + storage = storage + .checked_sub( + std::mem::size_of::() + + std::mem::size_of::() + + len.saturating_mul(2), + ) + .ok_or_else(|| { + LimitExceeded::new( + LimitKind::MaxPdfDecodedSize, + "OCR word storage exceeds max_pdf_decoded_size", + ) + })?; + let rect = word.bounding_rect(); + let bbox = [ + f64::from(rect.left()), + f64::from(rect.top()), + f64::from(rect.right()), + f64::from(rect.bottom()), + ]; + if bbox[0] < 0.0 + || bbox[1] < 0.0 + || bbox[2] > f64::from(dims.0) + || bbox[3] > f64::from(dims.1) + || bbox[2] <= bbox[0] + || bbox[3] <= bbox[1] + { + continue; + } + let mut clean = String::with_capacity(len); + super::super::push_clean(&mut clean, &word.to_string()); + if clean.trim().is_empty() { + continue; + } + clock.add_nodes(2)?; + spans.push(word_span( + spans.len(), + clean, + [ + bbox[0] * scale.0, + bbox[1] * scale.1, + bbox[2] * scale.0, + bbox[3] * scale.1, + ], + )); + } + } + } + Ok(spans) + } +} diff --git a/crates/markkit/src/pdf/tags.rs b/crates/markkit/src/pdf/tags.rs new file mode 100644 index 0000000..e4ed167 --- /dev/null +++ b/crates/markkit/src/pdf/tags.rs @@ -0,0 +1,109 @@ +//! Bounded tagged-PDF role lookup. Structure-tree MCIDs override generic /Span +//! tags; custom roles follow RoleMap. Geometry still supplies missing structure. +use crate::limits::Budget; +use crate::{Error, MarkkitBuilder}; +use hayro_syntax::Pdf; +use hayro_syntax::object::{Dict, Name, Object, ObjectIdentifier}; +use std::collections::{HashMap, HashSet}; + +pub(super) type Roles = HashMap<(ObjectIdentifier, i32), String>; +pub(super) fn roles(pdf: &Pdf, cfg: &MarkkitBuilder, clock: &Budget) -> Result { + let mut out = HashMap::new(); + let mut root = None; + for obj in pdf.objects() { + clock.check_time()?; + clock.add_nodes(1)?; + if let Object::Dict(d) = obj + && super::font::name_of(d.get::>(b"Type")) == "Catalog" + { + root = d.get::>(b"StructTreeRoot"); + break; + } + } + let Some(root) = root else { + return Ok(out); + }; + let map = root.get::>(b"RoleMap"); + struct Walker<'a, 'b> { + out: &'b mut Roles, + seen: HashSet, + map: Option>, + clock: &'b Budget, + depth: usize, + } + impl Walker<'_, '_> { + fn walk( + &mut self, + obj: Object<'_>, + page: Option, + role: &str, + depth: usize, + ) -> Result<(), Error> { + self.clock.check_time()?; + self.clock.add_nodes(1)?; + if depth > self.depth { + return Err(Error::PdfExtract( + "PDF structure tree exceeds max_nesting_depth".into(), + )); + } + match obj { + Object::Number(n) => { + let n = n.as_f32(); + if n.is_finite() + && n >= 0.0 + && n.fract() == 0.0 + && let Some(page) = page + { + self.out.insert((page, n as i32), role.into()); + } + } + Object::Array(a) => { + for k in a.iter::>() { + self.walk(k, page, role, depth + 1)?; + } + } + Object::Dict(d) => { + if let Some(id) = d.obj_id() + && !self.seen.insert(id) + { + return Ok(()); + } + let page = d.get::>(b"Pg").and_then(|d| d.obj_id()).or(page); + let mut role = d + .get::>(b"S") + .map(|n| super::font::name_of(Some(n))) + .unwrap_or_else(|| role.into()); + if let Some(map) = &self.map + && let Some(mapped) = map.get::>(role.as_bytes()) + { + role = super::font::name_of(Some(mapped)); + } + if role.len() > 256 { + return Err(Error::PdfExtract( + "PDF structure role exceeds 256 bytes".into(), + )); + } + if let (Some(id), Some(mcid)) = (page, d.get::(b"MCID")) { + self.out.insert((id, mcid), role.clone()); + } + if let Some(k) = d.get::>(b"K") { + self.walk(k, page, &role, depth + 1)?; + } + } + _ => {} + } + Ok(()) + } + } + if let Some(k) = root.get::>(b"K") { + Walker { + out: &mut out, + seen: HashSet::new(), + map, + clock, + depth: cfg.limits.nesting_depth(), + } + .walk(k, None, "Span", 0)?; + } + Ok(out) +} diff --git a/crates/markkit/tests/pdf_layout.rs b/crates/markkit/tests/pdf_layout.rs new file mode 100644 index 0000000..a4b614c --- /dev/null +++ b/crates/markkit/tests/pdf_layout.rs @@ -0,0 +1,285 @@ +//! Positioned PDF text is opt-in, deterministic and independent of Markdown. +#![cfg(feature = "pdf")] +mod common; +use common::*; +use markkit::{DocumentHint, Markkit}; + +fn extract(bytes: &[u8]) -> markkit::Extraction { + Markkit::builder() + .pdf_layout(true) + .build() + .extract_document_sync(bytes, DocumentHint::default()) + .unwrap() +} + +fn page_with(attrs: &str, content: &str) -> Vec { + pdf_bytes(&[ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [4 0 R] /Count 1 /MediaBox [0 0 612 792] >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R {attrs} /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>").into_bytes(), + pdf_stream("", content.as_bytes()), + ]) +} + +fn near(actual: f64, expected: f64) { + assert!((actual - expected).abs() < 0.001, "{actual} != {expected}"); +} + +#[test] +fn layout_is_opt_in_and_survives_rendering() { + let bytes = pdf_pages(&["BT /F1 10 Tf 72 700 Td (Hello) Tj ET"]); + let plain = Markkit::builder() + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert!(plain.layout.is_none()); + assert!( + serde_json::to_value(&plain) + .unwrap() + .get("layout") + .is_none() + ); + let parsed = Markkit::builder() + .pdf_layout(true) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + let layout = parsed.layout.clone().unwrap(); + let e = parsed.render(); + assert_eq!(e.markdown, plain.markdown); + assert_eq!(e.layout.as_ref(), Some(&layout)); + assert_eq!(layout.version, 1); + let p = &layout.pages[0]; + assert_eq!( + (p.page, p.width, p.height, p.rotation), + (1, 612.0, 792.0, 0) + ); + let s = &p.spans[0]; + assert_eq!( + (s.id, s.text.as_str(), s.font.as_deref()), + (0, "Hello", Some("Helvetica")) + ); + assert_eq!(s.baseline, [[72.0, 92.0], [97.0, 92.0]]); + assert_eq!(s.bbox, [72.0, 84.0, 97.0, 94.0]); + near(s.size, 10.0); + near(s.angle, 0.0); + let json = serde_json::to_string(&layout).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + layout + ); +} + +#[test] +fn coordinates_account_for_crop_rotation_and_user_unit() { + for (rotation, width, height, start, end, angle) in [ + (0, 800.0, 1200.0, [100.0, 1000.0], [150.0, 1000.0], 0.0), + (90, 1200.0, 800.0, [200.0, 100.0], [200.0, 150.0], 90.0), + (180, 800.0, 1200.0, [700.0, 200.0], [650.0, 200.0], 180.0), + (270, 1200.0, 800.0, [1000.0, 700.0], [1000.0, 650.0], 270.0), + ] { + let bytes = page_with( + &format!("/CropBox [50 100 450 700] /Rotate {rotation} /UserUnit 2"), + "BT /F1 10 Tf 100 200 Td (Hello) Tj ET", + ); + let p = extract(&bytes).layout.unwrap().pages.remove(0); + assert_eq!((p.width, p.height, p.rotation), (width, height, rotation)); + assert_eq!(p.spans[0].baseline, [start, end]); + near(p.spans[0].size, 20.0); + near(p.spans[0].angle, angle); + } +} + +#[test] +fn rotated_and_reflected_runs_have_nonempty_bounds() { + let bytes = pdf_pages(&[ + "BT /F1 10 Tf 0 1 -1 0 100 200 Tm (Hello) Tj ET\nBT /F1 10 Tf -1 0 0 1 200 300 Tm (Hello) Tj ET", + ]); + let p = extract(&bytes).layout.unwrap().pages.remove(0); + assert_eq!(p.spans[0].baseline, [[100.0, 592.0], [100.0, 567.0]]); + near(p.spans[0].angle, 270.0); + assert_eq!(p.spans[1].baseline, [[200.0, 492.0], [175.0, 492.0]]); + for s in p.spans { + assert!(s.bbox[0] < s.bbox[2] && s.bbox[1] < s.bbox[3]); + } +} + +#[test] +fn inherited_page_boxes_and_negative_rotation_are_normalized() { + let bytes = pdf_bytes(&[ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [4 0 R] /Count 1 /MediaBox [10 20 610 820] /CropBox [20 40 500 700] /Rotate -90 >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>".to_vec(), + pdf_stream("", b"BT /F1 10 Tf 100 200 Td (Hello) Tj ET"), + ]); + let page = extract(&bytes).layout.unwrap().pages.remove(0); + assert_eq!( + (page.width, page.height, page.rotation), + (660.0, 480.0, 270) + ); + assert_eq!(page.spans[0].baseline, [[500.0, 400.0], [500.0, 375.0]]); +} + +#[test] +fn page_ids_empty_pages_and_page_cap() { + let bytes = pdf_pages(&[ + "BT /F1 10 Tf 72 700 Td (one) Tj (two) Tj ET", + "", + "BT /F1 10 Tf 72 700 Td (three) Tj ET", + ]); + let p = extract(&bytes).layout.unwrap().pages; + assert_eq!(p.len(), 3); + assert_eq!(p[0].spans.iter().map(|s| s.id).collect::>(), [0, 1]); + assert!(p[1].spans.is_empty()); + assert_eq!((p[2].page, p[2].spans[0].id), (3, 0)); + let capped = Markkit::builder() + .pdf_layout(true) + .max_pdf_pages(1) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert_eq!(capped.layout.unwrap().pages.len(), 1); + assert_eq!(capped.warnings[0].code, markkit::WarningCode::TruncatedPdf); + assert_eq!(extract(&bytes), extract(&bytes)); +} + +#[test] +fn form_matrix_and_tj_adjustments_preserve_individual_runs() { + let form = pdf_stream( + "/Type /XObject /Subtype /Form /BBox [0 0 600 800] /Matrix [2 0 0 2 10 20]", + b"BT /F1 10 Tf 10 100 Td [(Hi) -500 (there)] TJ ET", + ); + let bytes = pdf_pages_with(&["/X1 Do"], "/F1 3 0 R", &[form]); + let p = extract(&bytes).layout.unwrap().pages.remove(0); + assert_eq!(p.spans.len(), 2); + assert_eq!(p.spans[0].text, "Hi"); + assert_eq!(p.spans[1].text, "there"); + assert_eq!(p.spans[0].baseline, [[30.0, 572.0], [50.0, 572.0]]); + assert_eq!(p.spans[1].baseline[0], [60.0, 572.0]); + assert_eq!(extract(&bytes).markdown.trim(), "Hi there"); +} + +#[test] +fn layout_is_absent_for_non_pdf_inputs() { + let e = Markkit::builder() + .pdf_layout(true) + .build() + .extract_document_sync(b"plain text", DocumentHint::file_name("notes.txt")) + .unwrap(); + assert!(e.layout.is_none()); +} + +#[test] +fn security_layout_limits_apply_to_parse_only_callers() { + let bytes = pdf_pages(&["BT /F1 10 Tf 72 700 Td (Hello) Tj (world) Tj ET"]); + for (builder, kind) in [ + ( + Markkit::builder().max_nodes(2), + markkit::limits::LimitKind::MaxNodes, + ), + ( + Markkit::builder().max_output_size(100), + markkit::limits::LimitKind::MaxOutputSize, + ), + ( + Markkit::builder().max_pdf_decoded_size(300), + markkit::limits::LimitKind::MaxPdfDecodedSize, + ), + ( + Markkit::builder().timeout(std::time::Duration::ZERO), + markkit::limits::LimitKind::Timeout, + ), + ] { + let err = builder + .pdf_layout(true) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap_err(); + assert!( + matches!(err, markkit::Error::ResourceLimit(ref e) if e.kind == kind), + "{err:?}" + ); + } + // The compact JSON cap is exact, including escaping, page metadata and IDs. + let size = serde_json::to_vec(&extract(&bytes).layout.unwrap()) + .unwrap() + .len(); + assert!( + Markkit::builder() + .pdf_layout(true) + .max_output_size(size) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .is_ok() + ); + assert!( + Markkit::builder() + .pdf_layout(true) + .max_output_size(size - 1) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .is_err() + ); +} + +#[test] +fn security_invalid_page_dimensions_are_rejected() { + for attrs in [ + "/UserUnit -1", + "/UserUnit 0", + "/UserUnit 75001", + "/CropBox [0 0 0 0]", + "/CropBox [700 800 900 1000]", + ] { + let bytes = page_with(attrs, "BT /F1 10 Tf 72 700 Td (Hello) Tj ET"); + let err = Markkit::builder() + .pdf_layout(true) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap_err(); + assert!( + matches!(err, markkit::Error::PdfExtract(_)), + "{attrs}: {err:?}" + ); + } +} + +#[test] +fn security_extreme_transforms_never_publish_nonfinite_coordinates() { + let bytes = page_with( + "/UserUnit 75000", + "q 1e30 0 0 1e30 0 0 cm BT /F1 10 Tf 72 700 Td (Huge) Tj ET Q", + ); + // Malformed or overflowing operators may be skipped or fail, but a + // successful extraction must always serialize finite numeric geometry. + if let Ok(e) = Markkit::builder() + .pdf_layout(true) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + { + let value = serde_json::to_value(e.layout.unwrap()).unwrap(); + for p in value["pages"].as_array().unwrap() { + assert!(p["width"].is_number() && p["height"].is_number()); + for s in p["spans"].as_array().unwrap() { + assert!( + s["bbox"] + .as_array() + .unwrap() + .iter() + .all(serde_json::Value::is_number) + ); + } + } + } +} + +proptest::proptest! { + #![proptest_config(proptest::test_runner::Config::with_cases(64))] + #[test] + fn security_arbitrary_layout_streams_never_panic(content in proptest::collection::vec(proptest::prelude::any::(), 0..2048)) { + let bytes = page_with("", &String::from_utf8_lossy(&content)); + let _ = Markkit::builder().pdf_layout(true).max_nodes(1000).max_output_size(65536).build().extract_document_sync(&bytes, DocumentHint::default()); + } +} diff --git a/crates/markkit/tests/pdf_ocr.rs b/crates/markkit/tests/pdf_ocr.rs new file mode 100644 index 0000000..268bcdf --- /dev/null +++ b/crates/markkit/tests/pdf_ocr.rs @@ -0,0 +1,226 @@ +//! Native OCR integration; requires installed Tesseract and language data. +#![cfg(all(feature = "pdf-ocr", not(target_arch = "wasm32")))] +mod common; +use common::*; +use markkit::{DocumentHint, Markkit}; +fn scanned_pdf() -> Vec { + // Generate a scanned page with the Rust renderer; no PDF executable needed. + let source = pdf_pages(&["BT /F1 36 Tf 72 700 Td (MARKKIT OCR TEST) Tj ET"]); + let pdf = hayro_syntax::Pdf::new(source).unwrap(); + let raster = hayro::render( + &pdf.pages()[0], + &hayro::RenderCache::new(), + &Default::default(), + &Default::default(), + &hayro::PixmapSettings { + x_scale: 2.0, + y_scale: 2.0, + bg_color: hayro::vello_cpu::color::palette::css::WHITE, + }, + ); + let (width, height) = (1224, 1584); + let pixels = raster + .data_as_u8_slice() + .as_chunks::<4>() + .0 + .iter() + .map(|p| p[0]) + .collect::>(); + assert_eq!(pixels.len(), width * height); + pdf_bytes(&[ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /XObject << /Im 5 0 R >> >> /Contents 4 0 R >>".to_vec(), + pdf_stream("",b"q 612 0 0 792 0 0 cm /Im Do Q"), + pdf_stream(&format!("/Type /XObject /Subtype /Image /Width {width} /Height {height} /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"),&zlib(&pixels)), + ]) +} + +#[test] +#[ignore = "requires local tesseract and eng language data; run explicitly for OCR changes"] +fn image_only_pdf_is_recognized_with_positioned_words() { + let bytes = scanned_pdf(); + let plain = Markkit::builder() + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert!(plain.markdown.is_empty()); + let e = Markkit::builder() + .pdf_ocr(true) + .pdf_layout(true) + .max_processing_secs(180) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert!( + e.markdown.contains("MARKKIT") && e.markdown.contains("OCR") && e.markdown.contains("TEST"), + "{}", + e.markdown + ); + assert!( + e.layout.unwrap().pages[0] + .spans + .iter() + .all(|s| s.ocr && !s.words.is_empty()) + ); +} +#[test] +fn invalid_ocr_language_is_rejected_without_spawning_helpers() { + let bytes = pdf_pages(&[""]); + let err = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_language("eng;touch unsafe") + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap_err(); + assert!(matches!(err, markkit::Error::PdfExtract(_))); +} + +#[test] +fn raster_budget_is_checked_before_helper_execution() { + let error = Markkit::builder() + .pdf_ocr(true) + .max_pdf_stream_size(1024) + .build() + .extract_document_sync(&pdf_pages(&[""]), DocumentHint::default()) + .unwrap_err(); + match error { + markkit::Error::ResourceLimit(limit) => { + assert_eq!(limit.kind, markkit::LimitKind::MaxPdfStreamSize) + } + other => panic!("{other:?}"), + } +} + +#[test] +fn text_pages_skip_ocr_renderer_preflight() { + let bytes = pdf_bytes(&[ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>".to_vec(), + pdf_stream("", b"BT /F1 12 Tf 72 700 Td (Existing text) Tj ET"), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + pdf_stream("/Subtype /Image /Width 1 /Height 1 /Filter /JBIG2Decode", b"unused"), + ]); + let result = Markkit::builder() + .pdf_ocr(true) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert_eq!(result.markdown.trim(), "Existing text"); +} + +#[test] +fn unknown_ocr_backend_is_rejected() { + assert!("mystery".parse::().is_err()); + assert_eq!( + "ocrs".parse::().unwrap(), + markkit::PdfOcrBackend::Ocrs + ); +} + +#[cfg(not(feature = "pdf-ocr-ocrs"))] +#[test] +fn ocrs_requires_compiled_backend() { + let error = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs) + .build() + .extract_document_sync(&pdf_pages(&[""]), DocumentHint::default()) + .unwrap_err(); + assert!(error.to_string().contains("pdf-ocr-ocrs"), "{error}"); +} + +#[cfg(feature = "pdf-ocr-ocrs")] +#[test] +fn ocrs_requires_explicit_models_and_supported_language() { + let bytes = pdf_pages(&[""]); + let error = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap_err(); + assert!(error.to_string().contains("model"), "{error}"); + let error = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs) + .pdf_ocr_language("eng+deu") + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap_err(); + assert!(error.to_string().contains("eng"), "{error}"); +} + +#[cfg(feature = "pdf-ocr-ocrs")] +#[test] +#[ignore = "requires pinned local ONNX models; set MARKKIT_OCR_MODEL_DIR"] +fn ocrs_recognizes_scanned_text_with_source_word_geometry() { + let models = std::path::PathBuf::from( + std::env::var_os("MARKKIT_OCR_MODEL_DIR").expect("set MARKKIT_OCR_MODEL_DIR"), + ); + let result = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs) + .pdf_ocr_detection_model(models.join("text-detection.onnx")) + .pdf_ocr_recognition_model(models.join("text-recognition.onnx")) + .pdf_layout(true) + .max_processing_secs(180) + .build() + .extract_document_sync(&scanned_pdf(), DocumentHint::default()) + .unwrap(); + assert!( + result.markdown.contains("MARKKIT") + && result.markdown.contains("OCR") + && result.markdown.contains("TEST"), + "{}", + result.markdown + ); + let layout = result.layout.unwrap(); + let spans = &layout.pages[0].spans; + assert!(spans.len() >= 3); + for (id, span) in spans.iter().enumerate() { + assert_eq!(span.id, id); + assert!(span.ocr && !span.words.is_empty()); + // Authored text is at x=72,y=700 (PDF bottom origin), in 36pt type. + // Independent bounds enclose the known rasterized line after display transform. + assert!( + span.bbox[0] >= 65.0 && span.bbox[2] <= 500.0, + "{:?}", + span.bbox + ); + assert!( + span.bbox[1] >= 50.0 && span.bbox[3] <= 105.0, + "{:?}", + span.bbox + ); + } +} + +#[cfg(feature = "pdf-ocr-ocrs")] +#[test] +fn ocrs_model_size_and_invalid_data_fail_explicitly() { + let dir = tempfile::tempdir().unwrap(); + let model = dir.path().join("model.onnx"); + std::fs::write(&model, vec![0_u8; 1024 * 1024 + 1]).unwrap(); + let builder = Markkit::builder() + .pdf_ocr(true) + .pdf_ocr_backend(markkit::PdfOcrBackend::Ocrs) + .pdf_ocr_detection_model(&model) + .pdf_ocr_recognition_model(&model) + .max_pdf_decoded_size(1024 * 1024); + let error = builder + .clone() + .build() + .extract_document_sync(&pdf_pages(&[""]), DocumentHint::default()) + .unwrap_err(); + assert!( + matches!(error, markkit::Error::ResourceLimit(ref limit) if limit.kind == markkit::LimitKind::MaxPdfDecodedSize) + ); + std::fs::write(&model, b"not an ONNX model").unwrap(); + let error = builder + .build() + .extract_document_sync(&pdf_pages(&[""]), DocumentHint::default()) + .unwrap_err(); + assert!(matches!(error, markkit::Error::PdfExtract(_)), "{error}"); +} diff --git a/crates/markkit/tests/pdf_structure.rs b/crates/markkit/tests/pdf_structure.rs new file mode 100644 index 0000000..10f8034 --- /dev/null +++ b/crates/markkit/tests/pdf_structure.rs @@ -0,0 +1,259 @@ +//! Reading-order and table regressions, including ambiguous prose negatives. +#![cfg(feature = "pdf")] +mod common; +use common::*; +use markkit::ir::Node; +use markkit::{DocumentHint, Markkit}; +fn extract(content: &str) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync(&pdf_pages(&[content]), DocumentHint::default()) + .unwrap() +} +fn row(y: u32, left: &str, right: &str) -> String { + format!("BT /F1 10 Tf 72 {y} Td ({left}) Tj ET\nBT /F1 10 Tf 300 {y} Td ({right}) Tj ET\n") +} +#[test] +fn columns_are_read_top_to_bottom_before_moving_right() { + let c = row(700, "Left first", "Right first") + &row(686, "Left second", "Right second"); + assert_eq!( + extract(&c).markdown, + "Left first Left second\n\nRight first Right second\n" + ); +} +#[test] +fn shuffled_single_column_is_spatially_ordered() { + assert_eq!( + extract( + "BT /F1 10 Tf 72 686 Td (Second line) Tj ET\nBT /F1 10 Tf 72 700 Td (First line) Tj ET" + ) + .markdown, + "First line Second line\n" + ); +} +#[test] +fn ruled_table_uses_cells_not_paragraphs() { + let c = "72 660 400 60 re S\n72 680 m 472 680 l S\n72 700 m 472 700 l S\n272 660 m 272 720 l S\nBT /F1 10 Tf 80 705 Td (Item) Tj ET\nBT /F1 10 Tf 280 705 Td (Qty) Tj ET\nBT /F1 10 Tf 80 685 Td (Apples) Tj ET\nBT /F1 10 Tf 280 685 Td (3) Tj ET\nBT /F1 10 Tf 80 665 Td (Pears) Tj ET\nBT /F1 10 Tf 280 665 Td (2) Tj ET"; + let e = extract(c); + assert_eq!( + e.markdown, + "| Item | Qty |\n| --- | --- |\n| Apples | 3 |\n| Pears | 2 |\n" + ); + assert_eq!(e.stats.tables, Some(1)); +} +#[test] +fn borderless_table_aligns_rows_and_numeric_columns() { + let c = row(700, "Item", "Qty") + &row(684, "Apples", "3") + &row(668, "Pears", "2"); + assert_eq!( + extract(&c).markdown, + "| Item | Qty |\n| --- | --- |\n| Apples | 3 |\n| Pears | 2 |\n" + ); +} +#[test] +fn prose_columns_are_not_borderless_tables() { + let c = row( + 700, + "The first sentence begins here.", + "Another sentence starts here.", + ) + &row( + 686, + "It carries on in this column.", + "This is its next line of prose.", + ) + &row( + 672, + "And here it comes to an end.", + "The second column finishes here.", + ); + assert!(!extract(&c).markdown.contains('|')); +} +#[test] +fn merged_ruled_header_keeps_colspan_in_ir() { + let c = "72 660 400 60 re S\n72 680 m 472 680 l S\n72 700 m 472 700 l S\n272 660 m 272 700 l S\nBT /F1 10 Tf 80 705 Td (Inventory) Tj ET\nBT /F1 10 Tf 80 685 Td (Apples) Tj ET\nBT /F1 10 Tf 280 685 Td (3) Tj ET\nBT /F1 10 Tf 80 665 Td (Pears) Tj ET\nBT /F1 10 Tf 280 665 Td (2) Tj ET"; + let parsed = Markkit::builder() + .pdf_layout(true) + .build() + .parse_document_sync(&pdf_pages(&[c]), DocumentHint::default()) + .unwrap(); + assert!( + parsed.document.nodes.iter().any( + |n| matches!(n,Node::Table{headers,..} if headers.len()==1 && headers[0].colspan==2) + ) + ); + let block = &parsed.layout.unwrap().pages[0].blocks[0]; + assert_eq!(block.kind, "table"); + assert_eq!(block.cells[0].colspan, 2); +} +#[test] +fn tagged_actual_text_replaces_glyph_content_once() { + let c = "BT /F1 10 Tf 72 700 Td /Span << /ActualText (Accessible text) >> BDC (junk) Tj (junk2) Tj EMC ET"; + assert_eq!(extract(c).markdown, "Accessible text\n"); +} +#[test] +fn numbered_lists_preserve_start() { + assert_eq!( + extract("BT /F1 10 Tf 14 TL 72 700 Td (3. first) Tj T* (4. second) Tj ET").markdown, + "3. first\n4. second\n" + ); +} +#[test] +fn repeated_margin_headers_can_be_removed() { + let bytes = pdf_pages(&[ + "BT /F1 10 Tf 72 770 Td (Company report) Tj ET\nBT /F1 10 Tf 72 700 Td (Page one body) Tj ET", + "BT /F1 10 Tf 72 770 Td (Company report) Tj ET\nBT /F1 10 Tf 72 700 Td (Page two body) Tj ET", + ]); + let e = Markkit::builder() + .pdf_remove_repeated_headers(true) + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert!(!e.markdown.contains("Company report")); + assert!(e.markdown.contains("Page one body") && e.markdown.contains("Page two body")); +} +#[test] +fn layout_has_ordered_blocks_linked_to_semantic_nodes() { + let bytes = pdf_pages(&[ + &(row(700, "Left first", "Right first") + &row(686, "Left second", "Right second")) + ]); + let p = Markkit::builder() + .pdf_layout(true) + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + let page = &p.layout.as_ref().unwrap().pages[0]; + assert_eq!(page.blocks.len(), 2); + for (i, n) in p.document.nodes.iter().enumerate() { + assert_eq!(n.source().unwrap().layout_block, Some(i)); + assert_eq!(page.blocks[i].id, i); + assert!(!page.blocks[i].lines.is_empty()); + } +} + +#[test] +fn wrapped_borderless_cells_stay_in_their_row() { + let c = row(700, "Item", "Qty") + + &row(684, "Long description", "3") + + "BT /F1 10 Tf 72 672 Td (continued) Tj ET\n" + + &row(652, "Pears", "2"); + let e = extract(&c); + assert!( + e.markdown.contains("| Long description continued | 3 |"), + "{}", + e.markdown + ); + assert!(e.markdown.contains("| Pears | 2 |")); +} +#[test] +fn prose_after_a_table_remains_a_paragraph() { + let c = row(700, "Item", "Qty") + + &row(684, "Apples", "3") + + &row(668, "Pears", "2") + + "BT /F1 10 Tf 72 620 Td (A paragraph below the table.) Tj ET"; + assert!( + extract(&c) + .markdown + .ends_with("\n\nA paragraph below the table.\n") + ); +} +#[test] +fn bold_and_italic_font_runs_are_preserved() { + let bytes = pdf_pages_with( + &["BT /F2 10 Tf 72 700 Td (Bold words) Tj ET\nBT /F3 10 Tf 72 680 Td (Italic words) Tj ET"], + "/F2 6 0 R /F3 7 0 R", + &[ + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Oblique >>".to_vec(), + ], + ); + let e = Markkit::builder() + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert_eq!(e.markdown, "**Bold words**\n\n*Italic words*\n"); +} +#[test] +fn tagged_heading_level_takes_precedence_over_font_size() { + assert_eq!(extract("/H2 BMC BT /F1 10 Tf 72 700 Td (Section) Tj ET EMC\nBT /F1 10 Tf 72 670 Td (Body) Tj ET").markdown,"## Section\n\nBody\n"); +} +#[test] +fn security_table_grid_caps_apply_before_expansion() { + let c = "72 660 400 60 re S\n72 680 m 472 680 l S\n72 700 m 472 700 l S\n272 660 m 272 720 l S\nBT /F1 10 Tf 80 705 Td (Item) Tj ET\nBT /F1 10 Tf 280 685 Td (3) Tj ET"; + let err = Markkit::builder() + .max_spreadsheet_cells(2) + .build() + .parse_document_sync(&pdf_pages(&[c]), DocumentHint::default()) + .unwrap_err(); + assert!( + matches!(err,markkit::Error::ResourceLimit(ref e) if e.kind==markkit::limits::LimitKind::MaxSpreadsheetCells) + ); +} + +#[test] +fn narrow_gutter_and_offset_baselines_do_not_interleave_columns() { + let c = "BT /F1 10 Tf 72 700 Td (Long left line filling the space completely.) Tj ET\nBT /F1 10 Tf 302 703 Td (Right first) Tj ET\nBT /F1 10 Tf 72 686 Td (Another left line filling the space fully.) Tj ET\nBT /F1 10 Tf 302 689 Td (Right second) Tj ET"; + let e = extract(c); + assert!( + e.markdown.find("Another left").unwrap() < e.markdown.find("Right first").unwrap(), + "{}", + e.markdown + ); +} +#[test] +fn structure_tree_mcid_and_rolemap_supply_heading_role() { + let bytes = pdf_bytes(&[ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 6 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>" + .to_vec(), + pdf_stream( + "", + b"/Span << /MCID 0 >> BDC BT /F1 10 Tf 72 700 Td (Section) Tj ET EMC", + ), + b"<< /Type /StructTreeRoot /RoleMap << /SectionTitle /H2 >> /K [7 0 R] >>".to_vec(), + b"<< /Type /StructElem /S /SectionTitle /Pg 4 0 R /K 0 >>".to_vec(), + ]); + assert_eq!( + Markkit::builder() + .build() + .extract_document_sync(&bytes, DocumentHint::default()) + .unwrap() + .markdown, + "## Section\n" + ); +} + +#[test] +fn security_marked_content_depth_and_actual_text_storage_are_bounded() { + let deep = "/Span BMC\n".repeat(257) + "BT /F1 10 Tf 72 700 Td (X) Tj ET"; + assert!( + Markkit::builder() + .build() + .parse_document_sync(&pdf_pages(&[&deep]), DocumentHint::default()) + .is_err() + ); + let replacement = "A".repeat(800); + let c = + format!("/Span << /ActualText ({replacement}) >> BDC BT /F1 10 Tf 72 700 Td (X) Tj ET EMC"); + let err = Markkit::builder() + .max_pdf_decoded_size(1200) + .build() + .parse_document_sync(&pdf_pages(&[&c]), DocumentHint::default()) + .unwrap_err(); + assert!( + matches!(err,markkit::Error::ResourceLimit(ref e) if e.kind==markkit::limits::LimitKind::MaxPdfDecodedSize) + ); +} + +#[test] +fn security_many_fragments_on_one_baseline_stay_one_line() { + let content = (0..300) + .map(|i| format!("BT /F1 10 Tf {} 700 Td (X) Tj ET\n", 72 + i * 40)) + .collect::(); + let parsed = Markkit::builder() + .max_pdf_decoded_size(1_000_000) + .build() + .parse_document_sync(&pdf_pages(&[&content]), DocumentHint::default()) + .unwrap(); + assert_eq!(parsed.document.nodes.len(), 1); + assert_eq!(parsed.render().markdown.split_whitespace().count(), 300); +} diff --git a/docs/getting-started.md b/docs/getting-started.md index 7e81319..6586aa9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -50,6 +50,22 @@ let doc = markkit.extract_document_sync(&bytes, hint)?; println!("{}", doc.markdown); ``` +## PDF source layout + +Enable optional positioned text output with `Markkit::builder().pdf_layout(true)`. +The parse/extraction result has a `layout` sidecar: page dimensions and text runs +with approximate bounding boxes, baselines, font names, sizes and angles. Coordinates +are display points from the cropped/rotated page top-left (x right, y down). +Layout is optional and does not change Markdown; source runs retain drawing order. +Semantic blocks/lines/table cells are in reading order and link to IR provenance. +PDF extraction reconstructs columns and ruled/borderless tables. Native OCR is +opt-in with the `pdf-ocr` feature and installed Tesseract and language data. + +```bash +markkit extract --file report.pdf --json --layout +markkit extract --file scanned.pdf --ocr --ocr-language eng --timeout 300 +``` + ## IR and provenance `parse_document_sync` returns the IR (`markkit::ir::Document`) plus metadata instead of @@ -89,3 +105,14 @@ Node.js and WASM. - [Format support](../knowledge/formats/format-support.md): formats, detection, DOCX, spreadsheet and PDF mapping - [Threat model](../knowledge/security/threat-model.md): security boundaries and limits - [Examples](../examples/): runnable with `cargo run -p markkit --example ` + +### Optional Rust OCR + +Build the CLI with `--features pdf-ocr-ocrs`, run +`python3 scripts/prepare_ocr_models.py`, then select `--ocr --ocr-backend ocrs` +with `--ocr-detection-model target/ocr-models/text-detection.onnx` and +`--ocr-recognition-model target/ocr-models/text-recognition.onnx`. +The default engine remains Tesseract. Extraction is offline; model weights are +separate CC-BY-SA 4.0 assets and support Latin-alphabet text (`eng`). +See the [PDF OCR guide](../README.md#pdf-source-layout) for Rust configuration, +model attribution and resource-limit behavior. diff --git a/justfile b/justfile index 2804f07..6f78767 100644 --- a/justfile +++ b/justfile @@ -136,6 +136,10 @@ eval-list: eval *args: mira run --study-bin markkit-eval markkit_extract {{args}} +# PDF text/layout quality, including column and table reconstruction. +eval-pdf *args: + mira run --study-bin markkit-eval markkit_pdf {{args}} + # Run the LLM QA eval (markkit markdown vs raw HTML as model context). Needs # ANTHROPIC_API_KEY / OPENROUTER_API_KEY; unkeyed targets are skipped. Keys live # in Doppler: doppler run -- just eval-llm @@ -213,3 +217,11 @@ docs: # Build documentation (no open) docs-build: cargo doc --workspace --no-deps + +# Fetch and verify optional OCR model assets (separate CC-BY-SA weights). +ocr-models *args: + python3 scripts/prepare_ocr_models.py {{args}} + +# Repeated Tesseract vs ocrs comparisons on the existing pinned public scans. +eval-ocr *args: + target/public-benchmarks/venv/bin/python scripts/compare_pdf_ocr.py {{args}} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 25ec09a..ee5ca0a 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -37,7 +37,7 @@ tags: - **XLSB** (`spreadsheet`): Excel binary workbooks via markkit's own bounds-checked BIFF12 reader (calamine's XLSB reader panics on malformed records) - **Markdown** (`markdown`): Re-parsed with pulldown-cmark into the IR and rendered back (`text/markdown`, `text/x-markdown`, `.md`, `.markdown`, `.mdown`, `.mkd`, `.mkdn`) -- **PDF** (`pdf`): Text layer via hayro-syntax (pure Rust); no OCR, encrypted PDFs rejected +- **PDF** (`pdf`): Text/layout, reading order and tables via hayro-syntax (pure Rust); encrypted PDFs rejected. Native OCR is opt-in with `pdf-ocr` and installed helpers ### Detected, not supported - **DOC** (Word 97-2003 binary), **PPT** (PowerPoint 97-2003 binary): explicit `UnsupportedFormat` errors @@ -246,7 +246,7 @@ kept raw in the IR and escaped by the renderer (S17). ## PDF Mapping -Text layer only (`crates/markkit/src/pdf/`). Layout is reconstructed from glyph positions: +Text and spatial structure (`crates/markkit/src/pdf/`); optional native OCR. Base mappings: | PDF | Markdown | |---|---| @@ -270,9 +270,89 @@ stripped); otherwise WinAnsi. Type0 fonts without `ToUnicode` are skipped with OCR layers (mode 3) are extracted. Encrypted documents (an `/Encrypt` key anywhere) return `UnsupportedFormat`; a page set with no text emits `pdf_no_text`. -Known gaps (follow-ups): multi-column reading order relies on content-stream order, -no table detection, no repeated header/footer removal, bold/italic not mapped, numbered -lists stay paragraphs, rotated text is not re-oriented. +Spatial extraction (`pdf/analyze.rs`) uses glyph-derived words normalized to the +page's dominant orientation. Repeated gutters define XY cuts; spanning headings +and tables divide bands, with columns read top-to-bottom, left-to-right. Single +rows without column evidence remain lines. Lines form paragraphs, styled inline +content and numbered/bullet lists. Heading sizes rank levels; tagged heading roles +(including structure-tree MCIDs/RoleMap) take precedence. Marked-content ActualText +replaces enclosed glyph text once; recursion, role lengths and retained bytes are +bounded. Optional `pdf_remove_repeated_headers` removes text repeated in the outer +8% margins of at least two pages and 60% of the extracted page set. + +Tables: axis-aligned vector rulings form connected grids. Missing internal borders +infer rectangular merged cells (row/column spans); nonrectangular ambiguous merges +fall back to text. Borderless candidates need >=3 aligned logical rows and numeric +data or explicit bold/TH headers; wrapped first-column continuations stay in their +cell. Ordinary prose columns remain prose. The first logical row becomes the +header; bounds/row/column/span metadata are retained. IR uses the existing Table / +TableCell representation. `stats.tables` counts detected tables. + +`pdf_layout(true)` returns a version-1 `layout::PdfLayout` sidecar on parse and +extraction results; otherwise omitted. Pages contain source runs/words in drawing +order and semantic blocks/lines/cells in reading order. Every block has a page-local +ID, type, bounds, source span IDs and line geometry; table cells include row/column, +rowspan/colspan and bounds. IR Source.page + Source.layout_block link back to blocks +when layout is requested (independent of track_sources). Geometry remains source +data if IR transforms rewrite nodes. Coordinates are display points from the +intersected crop/media top-left, x right, y down after rotation/UserUnit. Font bounds +use 0.8 em ascent / 0.2 em descent, not outlines; OCR uses recognized word bounds. +Off-page text remains. Invalid/non-finite geometry fails explicitly. JSON remains +optional; `--layout` requires `--json`. Node/WASM recursively expose camelCase keys; +Python/Rust/CLI serialize snake_case keys. + +Native `pdf-ocr` adds opt-in `pdf_ocr(true)` (CLI --ocr, Node pdfOcr, Python +pdf_ocr). Only pages without text are recognized, so text layers are never doubled. +Hayro renders the displayed crop box in Rust at max 1800×1800 pixels onto white, +then exports bounded grayscale PGM. Installed Tesseract emits TSV words using one +OpenMP thread. Languages are validated installed codes, default eng; callers can +set pdf_ocr_language / CLI --ocr-language. Tesseract uses private temporary files, +direct argv, size/deadline polling and kill+reap on expiry. Hayro rendering is a +non-interruptible third-party step: deadlines are checked before and after it. +Hosts must provide CPU/memory isolation for hostile documents. OCR can need longer +deadlines, e.g. --timeout 300. WASM has no native process OCR. + +Optional `pdf-ocr-ocrs` adds `PdfOcrBackend::Ocrs`, selected by +pdf_ocr_backend / CLI --ocr-backend / Node pdfOcrBackend. Tesseract remains the +default; missing compiled support fails explicitly. Rust, Node and Python accept +pdf_ocr_detection_model / pdf_ocr_recognition_model (camelCase in Node): explicit +local ONNX paths. Both are required only when a textless page needs recognition. +Models load once per document; input file bytes together obey max_pdf_decoded_size. +The core never fetches models or falls back to a different engine. The default +Latin model currently requires language eng; alternate language codes fail. + +`prepare_ocr_models.py` verifies the pinned official example pair from +`scripts/ocr_models.json`, including size/SHA256, and records CC-BY-SA 4.0 model +attribution beside the files. These weights are not embedded in packages. RTen +and ocrs execute in Rust and emit character-derived word rectangles, converted +to the same OCR spans/IR/layout as Tesseract. Output storage/nodes obey the same +limits. Deadline checks surround model loading, preprocessing, detection, line +analysis and recognition batches (max 20 lines); individual inference operations +are non-interruptible. Trusted models and host CPU/memory quotas remain required. +RTEN_NUM_THREADS and RAYON_NUM_THREADS control process-level pools. Native bindings +expose the knobs; their optional Cargo feature must be enabled when building. +Markkit WASM still exposes text-layer PDF extraction rather than OCR. + +OCR rendering conservatively supports JPEG, header-bounded JPEG2000 and +generic-filter images without DecodeParms. Inline images, JBIG2 and CCITT are refused explicitly until their +internal allocations can be bounded. Per-image working storage is estimated at +16 bytes per source pixel against the stream cap; cumulative RGBA storage is +charged at four bytes per pixel against the decoded cap. Huge or indirect +dimensions fail closed. PGM has no DPI metadata, so Tesseract receives the displayed +page resolution explicitly. This is a working-storage estimate, not a hard +heap quota on Hayro. See [Native Dependencies](../security/native-dependencies.md) +for provenance and remaining C/C++ components. + +PDF source geometry/rules count against max_nodes; retained layout storage and +pairwise analysis work are separately bounded by max_pdf_decoded_size. Grids obey +max_sheet_rows/max_sheet_cols/max_spreadsheet_cells before expansion. Layout JSON is +counted without allocating a second JSON copy against max_output_size, including +parse-only callers. Markdown has its own cap at that same setting, excluding JSON +wrapper/pretty-print overhead. OCR raster/TSV bytes use PDF stream/decoded caps. + +Heuristic limits: irregular/nonrectangular tables, nested tables, vertical scripts, +rotated individual cells and incomplete tagging may fall back to text; OCR accuracy +depends on source resolution and installed languages. Images are not exported. ## Plain Text and Markdown diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index cdaa4c4..8b9a4d3 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -71,6 +71,16 @@ Via `MarkkitBuilder`: - Hidden content (`include_hidden_content`, default off): HTML hidden elements, DOCX hidden runs - URL fetch policy (`allow_private_networks`, `allow_host`, `max_redirects`) - Base URL for relative links, images and metadata URLs (`base_url`) +- PDF source layout (`pdf_layout`, off by default): `ParsedDocument::layout` and + `Extraction::layout` hold positioned source text separately from semantic IR. + Layout includes source runs/words and semantic blocks/lines/table cells; + extraction reconstructs columns and tables even when layout JSON is off. + Native OCR (Hayro plus default Tesseract or optional ocrs/RTen with local ONNX + model paths) and repeated margin + removal are explicit options. Rendering has cooperative deadlines; deployment + isolation provides hard process budgets. See [Native Dependencies](../security/native-dependencies.md). + Exhaustive Rust `MarkkitBuilder` literals must supply the new PDF fields; prefer + builder methods or struct update syntax with `Default` for extensibility. - Provenance (`track_sources`): per-block `Source` in the IR (off by default) - Main-content detection for HTML (`main_content`) diff --git a/knowledge/foundations/ir.md b/knowledge/foundations/ir.md index 81707a3..b1bfe12 100644 --- a/knowledge/foundations/ir.md +++ b/knowledge/foundations/ir.md @@ -85,18 +85,33 @@ All block variants except `ThematicBreak` also carry `source` (see Provenance). | `FootnoteReference` | `label: String` | Marker pointing at the `FootnoteDefinition` with the same label. | | `Math` | `tex: String` | Inline math, TeX source. Renders GitHub's `` $`tex`$ `` form: one line, trimmed, inside a code span so plain-GFM readers never parse it as Markdown or HTML. | +### PDF layout sidecar + +`ParsedDocument::layout` / `Extraction::layout` optionally hold `layout::PdfLayout` +when `pdf_layout(true)` is enabled. This is source geometry, not part of `Document` +or its schema: IR_VERSION remains 2. Layout has its own `version` (1), with pages +source spans/words in drawing order, and blocks/lines/table cells in reading order. +`(page, span.id)` identifies a source run deterministically. IR `Source.page` + +`Source.layout_block` link to the page-local semantic block ID. Source geometry +remains valid if transforms move or delete semantic nodes. +See [Format Support](../formats/format-support.md) for coordinates and limits. + ### Provenance (`Source`) Every block variant except `ThematicBreak` has `source: Option>` -(omitted from JSON when `None`). Parsers fill it only when -`MarkkitBuilder::track_sources(true)`; Markdown output is identical either way. +(omitted from JSON when `None`). Parsers fill it when +`MarkkitBuilder::track_sources(true)`; PDF layout output additionally supplies page +and layout-block references even when tracking is off. Markdown is identical either way. | Field | Set by | Meaning | |-------|--------|---------| | `path` | HTML | Absolute XPath in the parsed DOM, every step indexed: `/html[1]/body[1]/div[1]/p[2]`. Indexes count same-name element siblings (excluded elements included). Cached per parent, O(depth) per block. | | `index` | DOCX | 0-based index of the body-level `w:p`/`w:tbl` in document order, counting blank paragraphs and those inside `w:sdt`/`w:ins`. Lists take their first item's index. | | `sheet`, `range` | Spreadsheets | Sheet name on the heading and table; table gets the A1 bounding box of kept cells (`B2:C3`). | -| `page` | PPTX; paged formats (PDF, later) | 1-based page; PPTX: slide number on every block of the slide, notes included. | +| `page` | PPTX, PDF | 1-based page; PPTX: slide number on every block of the slide, notes included. | + +PDF layout additionally records `Source.layout_block`: the page-local block ID, +when layout output is enabled, even if track_sources is off. Blocks synthesized by normalization (inline runs wrapped in `Paragraph`) and blocks nested inside inline/table-cell content have no source. Accessors: diff --git a/knowledge/log.md b/knowledge/log.md index 9607d67..0de94f4 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,16 @@ ## 2026-10-28 +* **PDF ship review**: escaped dictionary keys/names and generic compression prefixes before image codecs cannot bypass stream/image preflight limits. Positive, negative and security regressions cover the name spelling and filter-chain boundaries. See [Threat Model](security/threat-model.md) T8a. + +* **Optional Rust OCR**: add ocrs/RTen behind pdf-ocr-ocrs, explicit local ONNX model paths in Rust/CLI/Node/Python, bounded model files/word storage and cooperative inference deadlines. Keep Tesseract default. Pin model sizes/hashes and model attribution separately; compare both engines on the same existing scans with quality, geometry, timings and sampled process-tree memory. See [Format Support](formats/format-support.md), [Evaluation Harness](operations/eval.md), [Native Dependencies](security/native-dependencies.md). + + +* **PDF rendering**: replace Poppler with Rust Hayro for optional OCR; retain Tesseract, document cooperative render deadlines and conservative image preflight. Record public maintainer/ownership review and remaining native code in [Native Dependencies](security/native-dependencies.md). + + +* **PDF extraction**: geometric reading order; ruled/borderless tables with merged/wrapped cells; source word positions, semantic blocks/lines/cells linked to IR; emphasis, numbered lists, tagged MCID/RoleMap headings and ActualText; optional repeated headers and native Hayro rendering and Tesseract OCR. Python gains structured documents; all bindings expose layout/tables. Twenty Mira scenes and the public corpus measure quality; parsing/analysis/grid/helper limits have positive/negative/security tests. See [Format Support](formats/format-support.md), [Evaluation Harness](operations/eval.md), [Threat Model](security/threat-model.md) T8b. + * **Footnote safety**: a blank line separates footnote markers from their indented bodies, preventing Markdown readers without footnotes from treating body text as a reference-link destination. Fixes the nightly renderer fuzz failure; regression checks both reader modes. See [Threat Model](security/threat-model.md) S17. * **Public benchmarks**: WCXB and AgentDocBench run through the Mira Python SDK with exact upstream scorers, pinned corpus blobs, release CLI timing and a canonical score regression gate. PRs use 178 cases; scheduled dev uses 2,754 after excluding 139 HTML duplicates shared with WCXB test. Test evaluation is manual/report-only for quality. See [Evaluation Harness](operations/eval.md) and [Performance Results](operations/performance-results.md). * **First keyed eval run**: `markkit_qa` with Claude Sonnet 5.5: markdown 23/23 at 464 input tokens vs raw HTML 23/23 at 945; `main_content` 22/23 at 377 (miss = known forum-question gap). OpenRouter Muse Spark returned HTTP 402 (key daily limit), so no numbers yet. `markkit_extract` re-run: main_content 3/10 pages (product_page fixed by the escaping change). Doppler keys live in `everruns-dev/dev`. See [Evaluation Harness](operations/eval.md). diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index 45bc289..4a82a8c 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -12,15 +12,76 @@ tags: ## Status -Two evals on the [mira](https://github.com/everruns/mira) eval framework, +Three evals on the [mira](https://github.com/everruns/mira) eval framework, modeled on everruns/bashkit's `bashkit-eval` study: - `markkit_extract`: offline, deterministic extraction quality. +- `markkit_pdf`: authored PDF geometry, reading order and table expectations. - `markkit_qa`: LLM answers questions from markkit output vs raw HTML, over a key-gated model matrix; see [LLM QA Eval](#llm-qa-eval-markkit_qa). LLM-as-judge is planned; see [Planned evals](#planned-evals). +## PDF extraction eval (`markkit_pdf`) + +`just eval-pdf` runs 20 authored scenes from `data/pdf-layout.json` through the +Rust subject (`src/pdf.rs`), without external data/API keys. Inputs and expected +text/coordinates are inspectable in the dataset and Mira files. Default/layout +modes check exact ideal text fidelity, expected geometry (0.001-point tolerance, +exact array lengths), block/cell structure where authored, and Markdown parity. +Requested geometry missing from successful output or extraction failures score +zero. Geometry is N/A only when capture is intentionally disabled. The full scene +set includes columns/narrow gutters, spanning headings, ruled/borderless/wrapped +cells, tags/ActualText, numbered lists and page/crop/rotation cases. + +Native OCR validation uses the optional `pdf_ocr` integration test (explicitly run +with --include-ignored, installed helpers required) plus the pinned scanned public +PDFs. No live pages, API keys or GPU are needed. Public WCXB/AgentDocBench scorer +contracts/baselines remain unchanged; compare default extraction and OCR-enabled +PDF scores separately, retaining image-only cases in their denominators. + +Recorded run: [PDF extraction results, 2026-10-28](../../crates/markkit-eval/results/pdf-layout-2026-10-28.json). + +Renderer follow-up: [Hayro results, 2026-10-29](../../crates/markkit-eval/results/hayro-2026-10-29.json). +178 public default cases have no regressions; all 40 authored PDF cases pass. +Three scanned documents run with only Tesseract on PATH (no Poppler): mean score +0.52337 → 0.56047. NASA Budget declines (0.3530 → 0.3253); NASA Notes and Oklahoma +improve. Retain the per-document scores rather than treating the mean as a +no-regression gate. Renderer deadline/codec limits and native dependencies are +recorded in [Native Dependencies](../security/native-dependencies.md). + +## OCR backend comparison + +`just ocr-models` prepares size/SHA256-pinned official example models with +CC-BY-SA 4.0 attribution; `just eval-ocr` runs `scripts/compare_pdf_ocr.py` in the +public scorer venv. It builds the release CLI with pdf-ocr-ocrs, verifies models +and the unchanged public manifest, then evaluates all three existing scanned PDFs +with both engines. Rendering, layout heuristics and scorer contracts are shared. + +Each engine has one untimed warm-up per document followed by three fresh-process +runs (order alternates). RTEN_NUM_THREADS, RAYON_NUM_THREADS and OMP_THREAD_LIMIT +are one; ocrs runs with an empty PATH. Timing includes startup/model loading and +rendering; scoring is outside the measured interval. Linux peak process-tree RSS +is sampled every 10ms (shared pages can be counted twice). JSON retains raw +measurements, source/binary/model/input provenance, document/order/table scores, +word counts and geometry checks. Valid geometry requires finite positive in-page +boxes and consistent IDs. Exact-word matched bounding-box IoU measures engine +agreement, not localization accuracy: public scans lack ground-truth word boxes. +The authored model-backed integration test checks independent known text bounds. + +Recorded comparison: [ocrs results, 2026-10-28](../../crates/markkit-eval/results/ocrs-2026-10-28.json). +Across the three unchanged scans, mean document scores are Tesseract 0.56047 and +ocrs 0.54917. Median document time is 4.28s / 12.78s; median sampled peak process-tree +RSS is 36.2 / 227.6 MiB (Tesseract / ocrs). The longer scan scores equally; the two +NASA scans score lower with ocrs. All word geometry validates, and authored known +word regions pass. Keep Tesseract as default; the optional Rust path removes +external OCR helpers with measured accuracy/CPU/memory tradeoffs. These are three +scans, not a general ranking of OCR engines. + +Run only with approved local models; weights are never fetched during extraction. +Public quality gates and corpus baselines remain unchanged. Neither backend is +removed from comparisons if execution/geometry/repeatability fails. + ## Architecture `markkit-eval` is a **mira study**: a binary that advertises its evals to the diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index bf0e7e5..68ecd33 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -26,6 +26,9 @@ tags: | `crates/markkit/tests/document_security.rs` | Zip bombs, grid amplification, XML entities, malformed containers | | `crates/markkit/tests/limits.rs` | Every `Limits` field enforced for HTML, DOCX, spreadsheets; small-stack depth ceiling | | `crates/markkit/tests/pdf_integration.rs` | PDF text layout, fonts/encodings, forms, page cap | +| `crates/markkit/tests/pdf_structure.rs` | Columns, narrow gutters, tagged roles/ActualText, ruled/borderless/wrapped/merged cells, prose negatives, grid caps | +| `crates/markkit/tests/pdf_ocr.rs` | Native OCR language validation and explicit helper-backed image-only integration | +| `crates/markkit/tests/pdf_layout.rs` | Optional geometry, crop/rotation/UserUnit, transformed runs, stable IDs, layout budgets and malformed inputs | | `crates/markkit/tests/pdf_security.rs` | PDF bombs, interpretation budget, encrypted/unverifiable input, malformed PDFs | | `crates/markkit/tests/common/` | In-memory DOCX/XLSX/XLSB/XLS/ODS/PDF fixture builders; `fixtures/xlsb/` holds two real XLSB files from calamine (MIT) | | `crates/markkit-cli/tests/cli_integration.rs` | CLI flags and output | @@ -77,6 +80,12 @@ regressions. It installs the optional scorer dependencies, runs all public-tool contract tests, and uploads canonical scores/timing/provenance even on failure. See [Evaluation Harness](eval.md) Public benchmarks for split and baseline policy. +Optional OCR: `MARKKIT_OCR_MODEL_DIR=target/ocr-models RTEN_NUM_THREADS=1 +RAYON_NUM_THREADS=1 cargo test --release -p markkit --features pdf-ocr-ocrs +--test pdf_ocr -- --include-ignored` runs model and helper-backed fixtures. +CI explicitly runs the ocrs recognition fixture after pinned model setup; ordinary +all-feature tests cover missing/invalid/oversized models and backend selection. + ## See also - [Threat Model](../security/threat-model.md), threat-to-test map diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 552ecb0..f4dd167 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -30,7 +30,7 @@ const doc = m.extractDocumentSync(buf, { fileName: 'report.docx' }); Every function takes an optional trailing `options` object (`MarkkitOptions`): the camelCase builder fields shared with the [WASM Package](wasm-package.md) -(`maxInputSize`, `maxOutputSize`, `maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, +(`pdfLayout`, `pdfRemoveRepeatedHeaders`, `pdfOcr`, `pdfOcrLanguage`, `pdfOcrBackend`, `pdfOcrDetectionModel`, `pdfOcrRecognitionModel`, `maxInputSize`, `maxOutputSize`, `maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet limits, `includeHiddenSheets`) plus the URL policy fields `timeout` (seconds, float; whole call), `maxProcessingSecs` (whole-second alias, `timeout` wins), `allowPrivateNetworks`, `allowedHosts`, @@ -49,7 +49,9 @@ Errors reject (or throw) with the Rust error's `Display` text as the reason. | Resource limits (`Limits`) | `limits=markkit.Limits(...)` on every function | flat option keys, incl. `timeout` | flat option keys, no `timeout` (no clock) | | URL | yes, defaults only | yes, with URL policy options | no (host fetches) | | Bytes | yes | yes | yes | -| `extract_document` (metadata) | no | yes | yes | +| Structured documents | `extract_document_sync` | yes | yes | +| PDF tables + optional source layout | `pdf_layout=True` | `pdfLayout: true` | `pdfLayout: true` | +| Native PDF OCR | `pdf_ocr=True` | `pdfOcr: true` | no native processes | | Agent tool | no | no | `MarkkitTool` | Tests: `crates/markkit-js/__test__/*.test.cjs` (`node --test`, run by `js.yml` @@ -69,6 +71,10 @@ Published to npm by `publish-js.yml` from a verified release tag; see [Release Process](../operations/release-process.md). CI: `js.yml` on changes to `crates/markkit-js/` or `crates/markkit/`. +Optional Cargo feature `pdf-ocr-ocrs` enables Rust OCR; default package builds +retain Tesseract. Local model paths, Latin-language scope and model licensing +follow [Format Support](../formats/format-support.md). + ## See also - [Python Package](python-package.md), the sibling binding diff --git a/knowledge/runtimes/python-package.md b/knowledge/runtimes/python-package.md index 87d8485..6c4daf5 100644 --- a/knowledge/runtimes/python-package.md +++ b/knowledge/runtimes/python-package.md @@ -67,6 +67,21 @@ Published to PyPI by `publish-python.yml` from a verified release tag; see CI: `python.yml` builds and imports the module on changes to `crates/markkit-python/` or `crates/markkit/`. +## Structured documents + +`extract_document_sync(data, *, file_name=None, content_type=None, pdf_layout=False, +pdf_remove_repeated_headers=False, pdf_ocr=False, pdf_ocr_language="eng", +pdf_ocr_backend="tesseract", pdf_ocr_detection_model=None, +pdf_ocr_recognition_model=None, limits=None)` +returns a Python dictionary with Markdown, metadata, statistics and optional layout. +PDF tables/reading order are built in. Native OCR requires installed Tesseract/languages; callers can increase Limits.timeout for longer scanned files. +`tests/test_documents.py` checks layout, tables and resource errors in python.yml. + +Source builds can enable Cargo feature `pdf-ocr-ocrs` and select `ocrs` with +explicit local ONNX paths; default wheels retain Tesseract. Model licensing, +Latin-only scope and cooperative inference deadlines follow +[Format Support](../formats/format-support.md). + ## See also - [Node.js Package](node-package.md), the sibling binding and parity gaps diff --git a/knowledge/runtimes/wasm-package.md b/knowledge/runtimes/wasm-package.md index 4ba6967..23ee3b7 100644 --- a/knowledge/runtimes/wasm-package.md +++ b/knowledge/runtimes/wasm-package.md @@ -20,7 +20,7 @@ tags: browsers (incl. iframes), edge workers, Node, Deno and Bun. - **Separate from the napi package** (`@everruns/markkit`), which stays the fast native option for Node on Linux x64. -- **Features**: `html`, `docx`, `spreadsheet`, `markdown`, `tool`. No `http_client`: reqwest +- **Features**: `html`, `pdf`, `docx`, `spreadsheet`, `markdown`, `tool`. No `http_client`: reqwest needs sockets, and the host fetching keeps network policy (SSRF) with the host. The tool therefore offers only `html` and `content_base64`. @@ -38,6 +38,7 @@ tool.toolDefinition(); await tool.execute(args); ``` Options are camelCase builder fields (`maxInputSize`, `maxOutputSize`, +`pdfLayout`, `pdfRemoveRepeatedHeaders`, `maxPdfPages`, `maxPdfStreamSize`, `maxPdfDecodedSize`, `maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet limits, `includeHiddenSheets`). Selectors add to the defaults, as in Python. @@ -65,14 +66,14 @@ bash crates/markkit-wasm/scripts/build.sh # -> crates/markkit-wasm/pkg ``` Uses the `wasm-release` profile (`opt-level = "z"`, `panic = "abort"`); runs -`wasm-opt -Oz` when binaryen is on PATH. Unoptimized output is about 1.4 MB. +`wasm-opt -Oz` when binaryen is on PATH. Output size depends on enabled parsers. With `panic = "abort"` a Rust panic traps the instance; parsers are expected to return errors, never panic (invariant S9). ## Tests `crates/markkit-wasm/__test__/*.test.mjs` (`node --test`) run against `pkg/`: -options and their validation, HTML, bytes, DOCX through the zip/XML stack +options and their validation, HTML, bytes, PDF text/layout/tables and limits, DOCX through the zip/XML stack (fixtures built by `__test__/zip.mjs`), archive limits, malformed input, version parity with `package.json`, and the tool (definition, locale, execution, `userFacing` errors). CI job `wasm` in `ci.yml` also checks the core crate for diff --git a/knowledge/security/index.md b/knowledge/security/index.md index ba83898..1b90a85 100644 --- a/knowledge/security/index.md +++ b/knowledge/security/index.md @@ -3,3 +3,5 @@ * [Security Model](security-model.md) - Threat categories and security invariants summary for untrusted document input. * [Fuzzing](fuzzing.md) - cargo-fuzz targets, the output-safety oracle they assert, seeds, the nightly workflow and how to triage a crash. * [Threat Model](threat-model.md) - Attack surface, threats T1-T9 with mitigations and test coverage, resource limits, and security invariants. + +* [Native Dependencies](native-dependencies.md) - Hayro provenance review and native code in core features, bindings and development tools. diff --git a/knowledge/security/native-dependencies.md b/knowledge/security/native-dependencies.md new file mode 100644 index 0000000..d28c232 --- /dev/null +++ b/knowledge/security/native-dependencies.md @@ -0,0 +1,78 @@ +--- +type: Architecture +title: Native Dependencies +description: Hayro provenance review and native code in core features, bindings and development tools. +tags: + - markkit + - dependencies + - security +--- + +# Native Dependencies + +## Hayro review (2026-10-29 UTC) + +Hayro 0.8.0 is a Rust PDF renderer, licensed MIT OR Apache-2.0. Public evidence: + +- [Repository](https://github.com/LaurenzV/hayro): individual account LaurenzV. +- Repository creation: 2025-04-22. The [earliest surviving default-branch commit](https://github.com/LaurenzV/hayro/commit/38efa5f7d08e68ebca2d1a209d4921c1deffeaf3) + is “Initial commit”, authored by Laurenz Stampfl that day. +- [Maintainer profile](https://github.com/LaurenzV): Laurenz Stampfl; public location + Vienna, Austria; company Canva at review time. +- [Crates.io ownership API](https://crates.io/api/v1/crates/hayro/owners): sole + owner LaurenzV. Version 0.8.0 was published by that account on 2026-10-04. +- Crates.io checksum: `4675ff5acd7ca41c0c34ccd2a3a154f5c86d8b5ec92bef0a114bfb194969e2af`. + +These records show no Russian organizational ownership. They cannot certify the +nationality, ancestry, residence or affiliations of every contributor, historical +contributor or transitive dependency author. No such conclusions are inferred +from names. This is a public ownership/location review, not a supply-chain audit. + +Use the registry release and committed lockfile, with embedded fonts/CMaps and +optional architecture-specific SIMD disabled. The renderer and its codec/font/ +color dependencies are implemented in Rust. It replaces the Poppler executable; +Tesseract remains necessary for text recognition. + +## Optional Rust OCR review (2026-10-28 Chicago) + +`ocrs` 0.13.1 and RTen 0.26 implement CPU inference in Rust. The sole registry +owner is [Robert Knight](https://github.com/robertknight), whose public profile +lists London, United Kingdom, and Shortest Path Software. Public ownership +records show no Russian organizational owner; this is not a statement about +every contributor's nationality or affiliations. + +The model pair recommended by the upstream example is pinned by SHA256 and +size in `scripts/ocr_models.json`. Its model card identifies CC-BY-SA 4.0; +weights are separate deployment assets, not embedded in the MIT/Apache library. +The setup script records license/attribution in NOTICE.json. The backend never +uses Python, ONNX Runtime C++, Tesseract or Leptonica; ONNX is a model file format +read by the Rust RTen runtime. Tesseract stays available/default when selected. + +## Remaining native components + +| Component | Where / reason | +|---|---| +| Tesseract (C++) and Leptonica (C) | Default Tesseract OCR backend; host-installed executable and image-processing libraries, plus language data. System codec dependencies vary by distribution. | +| AWS-LC (`aws-lc-sys`, C/assembly) | Optional `http_client`: reqwest → rustls → aws-lc-rs. CLI, Python and native Node packages enable HTTP. It is not part of text-only/default core or offline all-formats parsing. | +| CPython (C) | Python binding's host runtime through PyO3; not a core parser dependency. | +| Node/V8 (C++) | Native Node binding's host runtime through N-API; not a core parser dependency. | +| `alloca` C shim | Criterion development/benchmark dependency only; not production dependency edges. | +| OS APIs, libc and platform trust stores | Rust standard library/OS bindings and HTTP certificate verification; platform dependent. | + +`ring` appears in Cargo.lock/metadata as an optional dependency of TLS packages, +but is not enabled in the reviewed workspace's all-features graph (including +`--target all`). Cargo.lock presence alone does not imply linked native code. +`rayon-core`/`wasm-bindgen-shared` also have Cargo `links` metadata but are Rust; +that field alone does not prove C/C++ content. ZIP compression uses Rust zlib-rs/ +miniz_oxide rather than system zlib in the reviewed production graph. + +Reproduce with `cargo metadata --locked --all-features --format-version 1`, +`cargo tree --workspace --all-features -i aws-lc-sys`, +`cargo tree --workspace --all-features -i alloca`, and target-specific dependency +trees. Inspect build scripts and active dependency edges, not just names ending +in `-sys`. `ldd` on the installed Tesseract describes that host's libraries only. + +## See also + +- [Format Support](../formats/format-support.md): OCR support and renderer limits. +- [Threat Model](threat-model.md): untrusted-input budgets and host isolation. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 2abca6c..511e8fd 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -407,7 +407,16 @@ decompression or content amplification, crashing the parser, or hiding text. with the same bounded decoders, never hayro's `decoded()`. - Refused instead of trusted: an indirect or repeated `/Filter` (`Error::PdfExtract`), and encrypted files (`/Encrypt` key; the scan would only see ciphertext) → - `UnsupportedFormat`. Image filters end a chain; hayro is built without image decoders. + `UnsupportedFormat`. Image filters end a chain. OCR-enabled builds enable Hayro image decoders; + streams with image codecs must declare Image subtype and cannot declare structural + object/xref stream metadata. + +- Dictionary key/name matching honors PDF #xx escapes, including Filter, + Encrypt, Subtype and image dimensions. Compressed prefixes before an image + codec are still measured through bounded decoders; image termination cannot + bypass prefix caps. Regression tests: pdf::guard::tests::image_filter_prefixes_cannot_bypass_decoded_caps, + escaped_dictionary_keys_cannot_bypass_stream_guards and + escaped_image_names_and_dimensions_are_preflighted. #### T8b: Interpretation amplification - Every decoded content stream is charged to an interpretation budget equal to @@ -417,6 +426,50 @@ decompression or content amplification, crashing the parser, or hiding text. - `q` nesting capped at 256 saved states; non-finite matrices are ignored. - CMap `bfrange`s are never expanded (binary-searched ranges); `/W` arrays capped at 65 536 entries. - `max_pdf_pages` (10 000) truncates with a `truncated_pdf` warning. +- Source pages/runs/words, vector segments, structure-tree objects and semantic + regions count against `max_nodes`, including when layout JSON is disabled. Before + cloning each run, retained struct/text/font storage is charged to a separate + `max_pdf_decoded_size` budget. Compact layout JSON is counted through a bounded + writer (no JSON allocation) against `max_output_size`, even for parse-only + callers. The writer checks the deadline. Coordinates use f64 arithmetic; + invalid page dimensions/UserUnit and non-finite output geometry are rejected. + Pair comparisons have a separate decoded-size work budget and deadline checks; + table grids obey row/column/cell caps before allocation. Marked-content nesting + is capped at 256; structure-tree recursion obeys max_nesting_depth with cycle + detection; role names cap at 256 bytes and ActualText replacement is charged. +- Opt-in OCR uses Rust Hayro rendering with Tesseract via direct argv or optional + Rust ocrs/RTen inference using explicit local ONNX models. + Image allocation estimates and raster dimensions/bytes are checked first; inline + images, DecodeParms and JBIG2/CCITT OCR are explicitly unsupported pending + bounded decoder support. Raster dimensions cap at 1800 per side; f32 scale + finiteness and resulting dimensions are checked before Hayro allocation, + including extreme aspect ratios. Hayro rendering + is non-interruptible; check the deadline before and after, per S16. This is not + a hard memory/CPU quota: deployments processing hostile input require host + isolation. Tesseract has private temporary files, output caps and deadline + polling, with kill+reap on expiry. OCR requires explicit configuration. + Ocrs never downloads assets or invokes helpers. Its models are trusted host + configuration; combined model file bytes obey max_pdf_decoded_size before + parsing, but weights/activation heap allocations require host quotas. Check + deadlines around each non-interruptible inference phase and between recognition + batches of at most 20 lines. Detected boxes and retained OCR words count against + max_nodes; strings/clones are checked against retained storage before allocation. + Model-version/size/hash pins and attribution live outside library packages; + the preparation script refuses corrupt downloads without replacing existing assets. + Tests: pdf_ocr::ocrs_model_size_and_invalid_data_fail_explicitly, + ocrs_requires_explicit_models_and_supported_language; explicit pinned-model + ocrs_recognizes_scanned_text_with_source_word_geometry covers known word positions. + Tests: pdf_ocr::invalid_ocr_language_is_rejected_without_spawning_helpers, + pdf::ocr::tests::tsv_words_are_scaled_and_malformed_rows_skipped, and explicit + image_only_pdf_is_recognized_with_positioned_words integration. + Geometry/table tests: pdf_structure::security_table_grid_caps_apply_before_expansion, + security_marked_content_depth_and_actual_text_storage_are_bounded. OCR cap kind + and child cleanup: pdf::ocr::tests::output_caps_keep_the_limit_kind / + expired_helper_is_killed_and_reaped. + Tests: `pdf_layout::security_layout_limits_apply_to_parse_only_callers`, + `security_invalid_page_dimensions_are_rejected`, + `security_extreme_transforms_never_publish_nonfinite_coordinates`, + `security_arbitrary_layout_streams_never_panic`. #### T8c: Parser crashes on malformed input - hayro-syntax repairs broken xref tables and returns `None` on bad objects; markkit diff --git a/schema/ir.schema.json b/schema/ir.schema.json index 269a520..292d10c 100644 --- a/schema/ir.schema.json +++ b/schema/ir.schema.json @@ -674,6 +674,15 @@ "null" ] }, + "layout_block": { + "description": "PDF: page-local semantic layout block ID.", + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "page": { "description": "Paged formats: 1-based page number (PPTX: slide number).", "format": "uint32", diff --git a/scripts/compare_pdf_ocr.py b/scripts/compare_pdf_ocr.py new file mode 100644 index 0000000..68cee93 --- /dev/null +++ b/scripts/compare_pdf_ocr.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Compare optional OCR engines on the unchanged pinned public scanned PDFs. + +Sequential fresh release processes, one warm-up and median repeated timings. +Peak process-tree RSS is sampled on Linux; shared pages may be counted twice. +Geometry validity/agreement is reported, not ground-truth localization accuracy. +""" +import argparse +import difflib +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'scripts/public_benchmarks')) +from common import read_manifest, safe_path, sha256 + + +def tree_rss(pid): + """Sum sampled resident bytes for root and current descendants (Linux).""" + if sys.platform != 'linux': + return None + pending, seen, total = [pid], set(), 0 + while pending: + child = pending.pop() + if child in seen: + continue + seen.add(child) + try: + status = Path(f'/proc/{child}/status').read_text() + total += next((int(line.split()[1]) * 1024 for line in status.splitlines() if line.startswith('VmRSS:')), 0) + pending.extend(int(p) for p in Path(f'/proc/{child}/task/{child}/children').read_text().split()) + except (FileNotFoundError, ProcessLookupError): + pass + return total + + +def run(command, env, timeout): + with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: + start, peak = time.perf_counter(), None + process = subprocess.Popen(command, stdout=output, stderr=errors, env=env) + try: + while process.poll() is None: + rss = tree_rss(process.pid) + if rss is not None: + peak = max(peak or 0, rss) + if time.perf_counter() - start > timeout: + raise TimeoutError('OCR comparison exceeded external timeout') + time.sleep(0.01) + elapsed = time.perf_counter() - start + output.seek(0) + errors.seek(0) + if process.returncode: + raise RuntimeError(errors.read().decode(errors='replace')) + return json.load(output), dict(seconds=elapsed, peak_process_tree_rss_bytes=peak) + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def geometry(extraction): + pages = extraction['layout']['pages'] + word_count = 0 + for page in pages: + for index, span in enumerate(page['spans']): + assert span['id'] == index and span['ocr'] + for word in span['words']: + x0, y0, x1, y1 = word['bbox'] + assert all(math.isfinite(v) for v in word['bbox']) + assert 0 <= x0 < x1 <= page['width'] + 0.01 + assert 0 <= y0 < y1 <= page['height'] + 0.01 + assert word['text'].strip() + word_count += 1 + assert word_count + return dict(pages=len(pages), words=word_count, valid=True, tables=extraction['stats']['tables']) + + +def agreement(left, right): + def words(page): + return [(word['text'].casefold(), word['bbox']) for span in page['spans'] for word in span['words']] + matches, overlaps = 0, [] + for a, b in zip(left['layout']['pages'], right['layout']['pages'], strict=True): + a, b = words(a), words(b) + matcher = difflib.SequenceMatcher(None, [w[0] for w in a], [w[0] for w in b], autojunk=False) + for i, j, count in matcher.get_matching_blocks(): + for (unused, x), (unused2, y) in zip(a[i:i+count], b[j:j+count], strict=True): + intersection = max(0, min(x[2], y[2])-max(x[0], y[0])) * max(0, min(x[3], y[3])-max(x[1], y[1])) + union = (x[2]-x[0])*(x[3]-x[1])+(y[2]-y[0])*(y[3]-y[1])-intersection + overlaps.append(intersection / union if union else 0) + matches += 1 + return dict(exact_text_matches=matches, mean_matched_word_bbox_iou=statistics.mean(overlaps) if overlaps else None, + interpretation='Cross-engine agreement only; these scans have no ground-truth word boxes.') + + +def main(): + from study import values + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cache', type=Path, default=ROOT / 'target/public-benchmarks') + parser.add_argument('--models', type=Path, default=ROOT / 'target/ocr-models') + parser.add_argument('--out', type=Path, default=ROOT / 'target/ocr-comparison') + parser.add_argument('--runs', type=int, default=3) + parser.add_argument('--timeout', type=int, default=300) + args = parser.parse_args() + if args.runs < 1 or args.timeout < 1: + parser.error('--runs and --timeout must be positive') + args.out.mkdir(parents=True, exist_ok=True) + # A failed rerun must never leave an older successful summary looking current. + (args.out / 'summary.json').unlink(missing_ok=True) + subprocess.run(['cargo','build','--locked','--release','-p','markkit-cli','--features','pdf-ocr-ocrs'], cwd=ROOT, check=True) + metadata = json.loads(subprocess.check_output(['cargo','metadata','--format-version','1','--no-deps'], cwd=ROOT)) + binary = str(Path(metadata['target_directory']) / 'release/markkit') + pins = json.loads((ROOT / 'scripts/ocr_models.json').read_text()) + for model in pins['models']: + path = args.models / model['name'] + if path.stat().st_size != model['size'] or sha256(path.read_bytes()) != model['sha256']: + raise ValueError('model pin mismatch; run scripts/prepare_ocr_models.py') + manifest = read_manifest(args.cache / 'manifest-pr.json') + rows = [r for r in manifest['samples'] if r['suite']=='agentdocbench' and r['format']=='pdf' and r['ocr']] + assert len(rows) == 3, 'expected all three existing scan cases' + env = dict(os.environ, OMP_THREAD_LIMIT='1', RTEN_NUM_THREADS='1', RAYON_NUM_THREADS='1') + cases = [] + for row in rows: + path = safe_path(args.cache, row['file']) + input_sha = manifest['files'][row['file']] + if sha256(path.read_bytes()) != input_sha: + raise ValueError('corpus hash mismatch') + commands, documents, measurements = {}, {}, {'tesseract': [], 'ocrs': []} + for backend in measurements: + command = [binary,'extract','--file',str(path),'--ocr','--ocr-backend',backend,'--ocr-language','eng','--timeout',str(args.timeout),'--json','--layout'] + if backend == 'ocrs': + command += ['--ocr-detection-model',str((args.models/'text-detection.onnx').resolve()),'--ocr-recognition-model',str((args.models/'text-recognition.onnx').resolve())] + commands[backend] = command + documents[backend], _ = run(command, dict(env, PATH='') if backend == 'ocrs' else env, args.timeout+30) + for repeat in range(args.runs): + # Alternate order to reduce systematic cache/order bias. + for backend in (['tesseract','ocrs'] if repeat % 2 == 0 else ['ocrs','tesseract']): + document, timing = run(commands[backend], dict(env, PATH='') if backend == 'ocrs' else env, args.timeout+30) + assert document == documents[backend], 'OCR output changed across repeated fresh processes' + measurements[backend].append(timing) + result = {'id': row['id'], 'input_sha256': input_sha, 'engines': {}} + for backend, samples in measurements.items(): + document = documents[backend] + (args.out/f"{row['id']}-{backend}.json").write_text(json.dumps(document)+'\n') + peaks = [s['peak_process_tree_rss_bytes'] for s in samples if s['peak_process_tree_rss_bytes'] is not None] + result['engines'][backend] = dict(scores=values(row, document['markdown'], args.cache), geometry=geometry(document), + median_seconds=statistics.median(s['seconds'] for s in samples), + median_peak_process_tree_rss_bytes=statistics.median(peaks) if peaks else None, measurements=samples) + result['geometry_agreement'] = agreement(documents['tesseract'], documents['ocrs']) + cases.append(result) + print(json.dumps(result), flush=True) + summary = dict(scope='Same three pinned public scans, shared Hayro rendering and Markkit layout/Markdown; OCR backend is the variable.', + source_revision=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),source_dirty=True, + binary_sha256=sha256(Path(binary).read_bytes()),model_pins=pins,platform=platform.platform(), + versions=dict(ocrs='0.13.1',rten='0.26.0',tesseract=subprocess.check_output(['tesseract','--version'],text=True).splitlines()[:2]), + method=dict(profile='release',warmups=1,runs=args.runs,timeout_seconds=args.timeout,ocr_language='eng',raster_max_dimension=1800,ocrs_max_lines_per_batch=20,thread_environment={k:env[k] for k in ['OMP_THREAD_LIMIT','RTEN_NUM_THREADS','RAYON_NUM_THREADS']}, + timing='Fresh process end-to-end, including model loading and page rendering; scoring outside boundary.',memory='Peak sum of process-tree resident bytes, sampled every 10ms on Linux. Shared pages may be counted multiple times; inference buffers are included.'), + pure_rust_ocr_helper_check='All ocrs executions used an empty PATH.', + cases=cases,engines={backend:dict(mean_document_score=statistics.mean(c['engines'][backend]['scores']['document_score'] for c in cases), + median_document_seconds=statistics.median(c['engines'][backend]['median_seconds'] for c in cases)) for backend in measurements}) + (args.out/'summary.json').write_text(json.dumps(summary,indent=2)+'\n') + print(json.dumps(summary['engines'],indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/ocr_models.json b/scripts/ocr_models.json new file mode 100644 index 0000000..3f16129 --- /dev/null +++ b/scripts/ocr_models.json @@ -0,0 +1,23 @@ +{ + "source": "https://github.com/robertknight/ocrs/blob/main/ocrs/examples/download-models.sh", + "revision": "Official example model pair, content pinned by SHA256", + "license": "CC-BY-SA-4.0", + "license_url": "https://huggingface.co/robertknight/ocrs/blob/df0edd170279ab971b53e094c627255a87e1a503/README.md", + "attribution": "Ocrs models by Robert Knight; trained on HierText (CC-BY-SA 4.0).", + "models": [ + { + "name": "text-detection.onnx", + "url": "https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.onnx", + "sha256": "a917b23dbd9524b465df7e922641b3ff2981623df4ded5a0234004ef2fee7cfe", + "size": 2499479, + "last_modified": "Sun, 13 Sep 2026 08:22:54 GMT" + }, + { + "name": "text-recognition.onnx", + "url": "https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.onnx", + "sha256": "86c145c2edb96c8caed5b1ebb8f44d706408922211451c309c625157dd6061c5", + "size": 9713177, + "last_modified": "Sun, 13 Sep 2026 08:22:52 GMT" + } + ] +} diff --git a/scripts/prepare_ocr_models.py b/scripts/prepare_ocr_models.py new file mode 100644 index 0000000..29fd7ac --- /dev/null +++ b/scripts/prepare_ocr_models.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Fetch pinned optional OCR weights. Extraction itself never downloads models.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import tempfile +import urllib.request + +MANIFEST = Path(__file__).with_name('ocr_models.json') + + +def file_hash(path): + digest = hashlib.sha256() + with path.open('rb') as handle: + for chunk in iter(lambda: handle.read(65536), b''): + digest.update(chunk) + return digest.hexdigest() + + +def prepare(model, output): + target = output / model['name'] + if target.is_file() and target.stat().st_size == model['size'] and file_hash(target) == model['sha256']: + return target + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=output, prefix='.model-', delete=False) as destination: + temporary = Path(destination.name) + with urllib.request.urlopen(model['url'], timeout=60) as response: + remaining = model['size'] + while chunk := response.read(min(65536, remaining + 1)): + remaining -= len(chunk) + if remaining < 0: + raise ValueError(f"{model['name']}: exceeds pinned size") + destination.write(chunk) + if remaining or file_hash(temporary) != model['sha256']: + raise ValueError(f"{model['name']}: pinned size/SHA256 mismatch") + os.replace(temporary, target) + return target + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out-dir', type=Path, default=Path('target/ocr-models')) + args = parser.parse_args() + manifest = json.loads(MANIFEST.read_text()) + args.out_dir.mkdir(parents=True, exist_ok=True) + print(f"{manifest['attribution']} License: {manifest['license']}. {manifest['license_url']}") + for model in manifest['models']: + print(prepare(model, args.out_dir), flush=True) + (args.out_dir / 'NOTICE.json').write_text(json.dumps(manifest, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/tests/test_compare_pdf_ocr.py b/scripts/tests/test_compare_pdf_ocr.py new file mode 100644 index 0000000..7b913ee --- /dev/null +++ b/scripts/tests/test_compare_pdf_ocr.py @@ -0,0 +1,25 @@ +"""OCR comparison geometry checks distinguish validity from agreement.""" +import copy +from pathlib import Path +import sys +import unittest +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from compare_pdf_ocr import agreement, geometry + + +def document(box): + return {'layout': {'pages': [{'width': 100, 'height': 100, 'spans': [ + {'id': 0, 'ocr': True, 'words': [{'text': 'Hello', 'bbox': box}]}]}]}, 'stats': {'tables': 0}} + + +class OcrGeometry(unittest.TestCase): + def test_agreement_compares_positions_for_matching_text(self): + a = document([1, 2, 11, 12]) + self.assertEqual(agreement(a, copy.deepcopy(a))['mean_matched_word_bbox_iou'], 1) + self.assertEqual(agreement(a, document([20, 20, 30, 30]))['mean_matched_word_bbox_iou'], 0) + self.assertEqual(geometry(a)['words'], 1) + + def test_invalid_geometry_cannot_pass_via_text_quality(self): + for box in [[0, 0, float('nan'), 1], [-1, 0, 10, 10], [1, 1, 1, 2], [0, 0, 101, 10]]: + with self.subTest(box=box), self.assertRaises(AssertionError): + geometry(document(box)) diff --git a/scripts/tests/test_prepare_ocr_models.py b/scripts/tests/test_prepare_ocr_models.py new file mode 100644 index 0000000..a0a1037 --- /dev/null +++ b/scripts/tests/test_prepare_ocr_models.py @@ -0,0 +1,38 @@ +"""Pinned OCR model cache keeps verified assets and rejects corrupt downloads.""" +import hashlib +import io +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import prepare_ocr_models as models + + +class PrepareOcrModels(unittest.TestCase): + def test_pinned_download_and_offline_cache(self): + wanted = b'model weights' + pin = dict(name='model.onnx', size=len(wanted), sha256=hashlib.sha256(wanted).hexdigest(), url='https://example.com/pinned') + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + with patch.object(models.urllib.request, 'urlopen', return_value=io.BytesIO(wanted)): + path = models.prepare(pin, output) + self.assertEqual(path.read_bytes(), wanted) + with patch.object(models.urllib.request, 'urlopen', side_effect=AssertionError('network forbidden')): + self.assertEqual(models.prepare(pin, output), path) + + def test_corrupt_and_oversized_models_do_not_replace_existing_asset(self): + wanted = b'good' + pin = dict(name='model.onnx', size=4, sha256=hashlib.sha256(wanted).hexdigest(), url='https://example.com/pinned') + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + target = output / pin['name'] + target.write_bytes(b'previous') + for data in [b'evil', b'oversized', b'x']: + with self.subTest(data=data), patch.object(models.urllib.request, 'urlopen', return_value=io.BytesIO(data)): + with self.assertRaises(ValueError): + models.prepare(pin, output) + self.assertEqual(target.read_bytes(), b'previous') + self.assertEqual(list(output.iterdir()), [target]) From be1408762004e5787e01384ed6def57c3e03e3fc Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 29 Oct 2026 03:12:46 +0000 Subject: [PATCH 2/2] chore(knowledge): normalize update log spacing --- knowledge/log.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/knowledge/log.md b/knowledge/log.md index 0de94f4..a17ef43 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -6,10 +6,8 @@ * **Optional Rust OCR**: add ocrs/RTen behind pdf-ocr-ocrs, explicit local ONNX model paths in Rust/CLI/Node/Python, bounded model files/word storage and cooperative inference deadlines. Keep Tesseract default. Pin model sizes/hashes and model attribution separately; compare both engines on the same existing scans with quality, geometry, timings and sampled process-tree memory. See [Format Support](formats/format-support.md), [Evaluation Harness](operations/eval.md), [Native Dependencies](security/native-dependencies.md). - * **PDF rendering**: replace Poppler with Rust Hayro for optional OCR; retain Tesseract, document cooperative render deadlines and conservative image preflight. Record public maintainer/ownership review and remaining native code in [Native Dependencies](security/native-dependencies.md). - * **PDF extraction**: geometric reading order; ruled/borderless tables with merged/wrapped cells; source word positions, semantic blocks/lines/cells linked to IR; emphasis, numbered lists, tagged MCID/RoleMap headings and ActualText; optional repeated headers and native Hayro rendering and Tesseract OCR. Python gains structured documents; all bindings expose layout/tables. Twenty Mira scenes and the public corpus measure quality; parsing/analysis/grid/helper limits have positive/negative/security tests. See [Format Support](formats/format-support.md), [Evaluation Harness](operations/eval.md), [Threat Model](security/threat-model.md) T8b. * **Footnote safety**: a blank line separates footnote markers from their indented bodies, preventing Markdown readers without footnotes from treating body text as a reference-link destination. Fixes the nightly renderer fuzz failure; regression checks both reader modes. See [Threat Model](security/threat-model.md) S17.