From 2213ea7b7a8df9f21d15b6871ec239e38bedf89c Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 19:53:29 +0000 Subject: [PATCH 1/3] chore(deps): drop unused dependencies and apply major upgrades Remove dependencies nothing uses: async-trait, futures, regex, unicode-segmentation, encoding_rs, base64, mime, lol_html and tracing-subscriber from core and the workspace, plus unused deps of the CLI, bindings and the stub bench/eval crates. The stub bench deps were the only source of the instant/paste unmaintained advisories, so their deny.toml ignores go too. Upgrade scraper 0.29, html5ever 0.40, pyo3 0.30, criterion 0.8 and quick-xml 0.42, which unifies with calamine's copy; port the XML walkers to its str-based names and values. --- CHANGELOG.md | 16 +- Cargo.lock | 627 +++++----------------------- Cargo.toml | 28 +- NOTICE | 3 +- README.md | 2 +- crates/markkit-bench/Cargo.toml | 14 - crates/markkit-cli/Cargo.toml | 1 - crates/markkit-eval/Cargo.toml | 6 - crates/markkit-js/Cargo.toml | 3 - crates/markkit-python/Cargo.toml | 2 - crates/markkit/Cargo.toml | 20 +- crates/markkit/src/spreadsheet.rs | 22 +- crates/markkit/src/xml.rs | 32 +- deny.toml | 9 +- knowledge/log.md | 1 + knowledge/operations/maintenance.md | 4 + knowledge/security/threat-model.md | 2 +- 17 files changed, 146 insertions(+), 646 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8f4f4..0a93e34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] -### Fixed - -- Indented HTML (most real web pages) now keeps blank lines between blocks; previously - headings, paragraphs, lists and tables after the first ran together with a leading space - (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. - ### Changed - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. - CI runs every example; `just bench` now runs the criterion benchmarks. - README, SECURITY and binding docs corrected: Python module is `markkit`, URL fetching has no private-network blocking yet. +- Dependencies: drop unused `async-trait`, `futures`, `regex`, `unicode-segmentation`, `encoding_rs`, + `base64`, `mime`, `lol_html` (core) and `tracing-subscriber`, plus unused deps of the CLI, bindings + and stub crates. Upgrade scraper 0.29, html5ever 0.40, quick-xml 0.42 (now a single version shared + with calamine), pyo3 0.30, criterion 0.8. The `logging` feature is reserved and enables only `tracing`. + +### Fixed + +- Indented HTML (most real web pages) now keeps blank lines between blocks; previously + headings, paragraphs, lists and tables after the first ran together with a leading space + (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. ## [0.1.1] - 2026-10-22 diff --git a/Cargo.lock b/Cargo.lock index c851243..01303b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,10 +12,13 @@ dependencies = [ ] [[package]] -name = "allocator-api2" -version = "0.2.21" +name = "alloca" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] [[package]] name = "anes" @@ -79,26 +82,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "approx" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" -dependencies = [ - "num-traits", -] - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - [[package]] name = "atoi_simd" version = "0.18.1" @@ -144,12 +127,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64" version = "0.23.1" @@ -211,18 +188,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - -[[package]] -name = "bytemuck" -version = "1.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" - [[package]] name = "byteorder" version = "1.5.0" @@ -247,7 +212,7 @@ dependencies = [ "encoding_rs", "fast-float2", "log", - "quick-xml 0.42.0", + "quick-xml", "serde", "zip", ] @@ -301,7 +266,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -395,15 +360,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "colored" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "combine" version = "4.6.9" @@ -465,25 +421,24 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", "itertools", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -491,9 +446,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools", @@ -532,9 +487,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "cssparser" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59" dependencies = [ "cssparser-macros", "dtoa-short", @@ -545,12 +500,12 @@ dependencies = [ [[package]] name = "cssparser-macros" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" dependencies = [ "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -693,12 +648,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -834,7 +783,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "rand_core 0.10.1", + "rand_core", "wasm-bindgen", ] @@ -854,11 +803,6 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "heck" @@ -866,20 +810,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" - [[package]] name = "html5ever" -version = "0.39.0" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" +checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130" dependencies = [ "log", "markup5ever", + "memchr", ] [[package]] @@ -961,7 +900,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-channel", "futures-util", @@ -1093,32 +1032,12 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "ipnet" version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1127,9 +1046,9 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -1210,12 +1129,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" - [[package]] name = "libc" version = "0.2.191" @@ -1232,12 +1145,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1265,25 +1172,6 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" -[[package]] -name = "lol_html" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" -dependencies = [ - "bitflags", - "cfg-if", - "cssparser", - "encoding_rs", - "foldhash", - "hashbrown", - "memchr", - "mime", - "precomputed-hash", - "selectors 0.37.0", - "thiserror", -] - [[package]] name = "lru-slab" version = "0.1.3" @@ -1295,19 +1183,12 @@ name = "markkit" version = "0.1.1" dependencies = [ "anyhow", - "async-trait", - "base64 0.22.1", "calamine", "cfb", "criterion", - "encoding_rs", - "futures", "html5ever", - "lol_html", - "mime", "proptest", - "quick-xml 0.41.0", - "regex", + "quick-xml", "reqwest", "scraper", "serde", @@ -1316,8 +1197,6 @@ dependencies = [ "thiserror", "tokio", "tracing", - "tracing-subscriber", - "unicode-segmentation", "zip", ] @@ -1327,14 +1206,6 @@ version = "0.1.1" dependencies = [ "anyhow", "clap", - "colored", - "instant", - "markkit", - "serde", - "serde_json", - "statrs", - "tabled", - "tokio", ] [[package]] @@ -1344,7 +1215,6 @@ dependencies = [ "anyhow", "clap", "markkit", - "serde", "serde_json", "tempfile", "tokio", @@ -1355,13 +1225,7 @@ name = "markkit-eval" version = "0.1.1" dependencies = [ "anyhow", - "async-trait", "clap", - "markkit", - "regex", - "reqwest", - "serde", - "serde_json", "tokio", ] @@ -1373,9 +1237,6 @@ dependencies = [ "napi", "napi-build", "napi-derive", - "serde", - "serde_json", - "tokio", ] [[package]] @@ -1385,52 +1246,25 @@ dependencies = [ "markkit", "pyo3", "pyo3-async-runtimes", - "serde_json", - "tokio", ] [[package]] name = "markup5ever" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" +checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842" dependencies = [ "log", "tendril", "web_atoms", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matrixmultiply" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "mio" version = "1.2.4" @@ -1448,23 +1282,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" -[[package]] -name = "nalgebra" -version = "0.33.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d43ddcacf343185dfd6de2ee786d9e8b1c2301622afab66b6c73baf9882abfd" -dependencies = [ - "approx", - "matrixmultiply", - "num-complex", - "num-rational", - "num-traits", - "rand 0.8.8", - "rand_distr", - "simba", - "typenum", -] - [[package]] name = "napi" version = "3.15.0" @@ -1550,54 +1367,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1605,7 +1374,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -1633,14 +1401,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] -name = "papergrid" -version = "0.18.0" +name = "page_size" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0984e668274d34691bc2b262ef0d115de5fa9973bcdee7ae32213f93099153e" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ - "bytecount", - "fnv", - "unicode-width", + "libc", + "winapi", ] [[package]] @@ -1666,12 +1433,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1680,9 +1441,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "phf" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" dependencies = [ "phf_macros", "phf_shared", @@ -1691,9 +1452,9 @@ dependencies = [ [[package]] name = "phf_codegen" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" dependencies = [ "phf_generator", "phf_shared", @@ -1701,9 +1462,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" dependencies = [ "fastrand", "phf_shared", @@ -1711,9 +1472,9 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" dependencies = [ "phf_generator", "phf_shared", @@ -1724,9 +1485,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" dependencies = [ "siphasher", ] @@ -1786,15 +1547,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "precomputed-hash" version = "0.1.1" @@ -1831,7 +1583,7 @@ dependencies = [ "chacha20", "core_detect", "num-traits", - "rand 0.10.3", + "rand", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -1841,9 +1593,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" dependencies = [ "libc", "once_cell", @@ -1855,9 +1607,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" +checksum = "b877331c7817ccb6379e9d19a06993160141f1d0a386e55276128717df85e3fc" dependencies = [ "futures-channel", "futures-util", @@ -1869,18 +1621,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" dependencies = [ "libc", "pyo3-build-config", @@ -1888,26 +1640,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "pyo3-macros-backend" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1916,15 +1668,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.42.0" @@ -1965,7 +1708,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.3", + "rand", "rand_pcg", "ring", "rustc-hash", @@ -2007,17 +1750,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" -dependencies = [ - "libc", - "rand_chacha", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.10.3" @@ -2026,26 +1758,7 @@ checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", + "rand_core", ] [[package]] @@ -2054,23 +1767,13 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand 0.8.8", -] - [[package]] name = "rand_pcg" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -2079,15 +1782,9 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.10.1", + "rand_core", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "rayon" version = "1.12.0" @@ -2152,7 +1849,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-core", "http", @@ -2318,15 +2015,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "safe_arch" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" -dependencies = [ - "bytemuck", -] - [[package]] name = "same-file" version = "1.0.6" @@ -2353,17 +2041,16 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scraper" -version = "0.26.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f5297102b8b62b4454ee8561601b2d551b4913148feb4241ca9d1a04bf4526" +checksum = "6844b5f49a5579f12ba4f89d9f7eec6d71889e4aa0bc7181821040ce66f08fff" dependencies = [ "cssparser", "ego-tree", "getopts", "html5ever", "precomputed-hash", - "selectors 0.36.1", - "tendril", + "selectors", ] [[package]] @@ -2391,36 +2078,15 @@ dependencies = [ [[package]] name = "selectors" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d9c0c92a92d33f08817311cf3f2c29a3538a8240e94a6a3c622ce652d7e00c" -dependencies = [ - "bitflags", - "cssparser", - "derive_more", - "log", - "new_debug_unreachable", - "phf", - "phf_codegen", - "precomputed-hash", - "rustc-hash", - "servo_arc", - "smallvec", -] - -[[package]] -name = "selectors" -version = "0.37.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" +checksum = "8322aebea94a6cc288ff884e8a49bdbceef60206c400c6c46d22e62fc4fd91a6" dependencies = [ "bitflags", "cssparser", "derive_more", "log", "new_debug_unreachable", - "phf", - "phf_codegen", "precomputed-hash", "rustc-hash", "servo_arc", @@ -2478,41 +2144,19 @@ dependencies = [ [[package]] name = "servo_arc" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +checksum = "b4eceb519ec34d8290aaf6e46f435102f2bfa63f22b05a144fb16ab266d3b198" dependencies = [ "stable_deref_trait", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "simba" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95" -dependencies = [ - "approx", - "num-complex", - "num-traits", - "paste", - "wide", -] - [[package]] name = "simd-adler32" version = "0.3.10" @@ -2569,23 +2213,11 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "statrs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" -dependencies = [ - "approx", - "nalgebra", - "num-traits", - "rand 0.8.8", -] - [[package]] name = "string_cache" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" dependencies = [ "new_debug_unreachable", "parking_lot", @@ -2595,9 +2227,9 @@ dependencies = [ [[package]] name = "string_cache_codegen" -version = "0.6.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" dependencies = [ "phf_generator", "phf_shared", @@ -2659,29 +2291,6 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "tabled" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2596a104db1900b943f97d793a6c99addca918c4525c999f751a0f17d472eb" -dependencies = [ - "papergrid", - "tabled_derive", - "testing_table", -] - -[[package]] -name = "tabled_derive" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dca1937322a1e892b1a65f6a6736183bb0a29d3b4234d1d53bc436dff9beb76" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "target-lexicon" version = "0.13.5" @@ -2710,15 +2319,6 @@ dependencies = [ "new_debug_unreachable", ] -[[package]] -name = "testing_table" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f8daae29995a24f65619e19d8d31dea5b389f3d853d8bf297bbf607cd0014cc" -dependencies = [ - "unicode-width", -] - [[package]] name = "thiserror" version = "2.0.21" @@ -2739,15 +2339,6 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - [[package]] name = "tinystr" version = "0.8.5" @@ -2914,36 +2505,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", ] [[package]] @@ -2958,12 +2519,6 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unarray" version = "0.1.4" @@ -3028,12 +2583,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "wait-timeout" version = "0.2.1" @@ -3146,9 +2695,9 @@ dependencies = [ [[package]] name = "web_atoms" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3" +checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" dependencies = [ "phf", "phf_codegen", @@ -3166,15 +2715,21 @@ dependencies = [ ] [[package]] -name = "wide" -version = "0.7.33" +name = "winapi" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "bytemuck", - "safe_arch", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3184,6 +2739,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index fe4a37d..a67f998 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,6 @@ categories = ["text-processing", "command-line-utilities", "parser-implementatio [workspace.dependencies] # Async tokio = { version = "1", default-features = false } -async-trait = "0.1" -futures = "0.3" # Error handling thiserror = "2" @@ -42,35 +40,21 @@ clap = { version = "4", features = ["derive"] } reqwest = { version = "0.13", default-features = false, features = ["rustls", "json"] } # HTML parsing -scraper = "0.26" -html5ever = "0.39" -lol_html = "2" - -# Text processing -regex = "1" -unicode-segmentation = "1" +scraper = "0.29" +html5ever = "0.40" # Document formats calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS) zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip -quick-xml = "0.41" # DOCX/ODS XML parsing +quick-xml = "0.42" # DOCX/ODS XML parsing cfb = "0.15" # XLS (OLE2) container pre-scan -# Image / OCR (optional) -image = "0.25" - -# Encoding -encoding_rs = "0.8" -base64 = "0.22" -mime = "0.3" - # Logging tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Python bindings -pyo3 = { version = "0.29" } -pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } +pyo3 = { version = "0.30" } +pyo3-async-runtimes = { version = "0.30", features = ["tokio-runtime"] } # Node.js bindings napi = { version = "3", features = ["async", "serde-json"] } @@ -78,7 +62,7 @@ napi-derive = "3" # Testing tempfile = "3" -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.8", features = ["html_reports"] } proptest = "1" [profile.release] diff --git a/NOTICE b/NOTICE index b0d3111..1854781 100644 --- a/NOTICE +++ b/NOTICE @@ -28,8 +28,7 @@ Markkit relies on several permissively licensed crates: * scraper (MIT) – HTML parsing and CSS selector matching * html5ever (MIT/Apache-2.0) – HTML5 parsing -* lol_html (BSD-3-Clause) – Low-latency HTML rewriting -* pdf-extract (MIT) – PDF text extraction +* zip (MIT), quick-xml (MIT), calamine (MIT), cfb (MIT) – Office document containers and XML * reqwest (MIT/Apache-2.0) – HTTP client * tokio (MIT) – Async runtime * serde / serde_json (MIT/Apache-2.0) – Serialization diff --git a/README.md b/README.md index dc343aa..4987e1b 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ let markkit = Markkit::builder() | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` | | `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` | -| `logging` | No | Structured tracing output | +| `logging` | No | Reserved: pulls in `tracing`, emits no events yet | `http_client` is optional. Embedders that fetch content themselves can build without reqwest (no HTTP/TLS stack in the dependency tree): diff --git a/crates/markkit-bench/Cargo.toml b/crates/markkit-bench/Cargo.toml index f5d2f4a..45f7ed3 100644 --- a/crates/markkit-bench/Cargo.toml +++ b/crates/markkit-bench/Cargo.toml @@ -13,19 +13,5 @@ name = "markkit-bench" path = "src/main.rs" [dependencies] -markkit = { path = "../markkit", features = ["all_formats", "http_client"] } -tokio = { workspace = true, features = ["rt-multi-thread"] } -serde.workspace = true -serde_json.workspace = true clap.workspace = true anyhow.workspace = true - -# Timing -instant = "0.1" - -# Statistics -statrs = "0.18" - -# Terminal output -colored = "3" -tabled = "0.22" diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 21cf10d..69495f0 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -30,7 +30,6 @@ markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] } tokio = { workspace = true, features = ["macros", "net", "rt", "rt-multi-thread", "time", "fs"] } clap.workspace = true anyhow.workspace = true -serde.workspace = true serde_json.workspace = true [dev-dependencies] diff --git a/crates/markkit-eval/Cargo.toml b/crates/markkit-eval/Cargo.toml index 9ae387e..c8d5062 100644 --- a/crates/markkit-eval/Cargo.toml +++ b/crates/markkit-eval/Cargo.toml @@ -14,12 +14,6 @@ name = "markkit-eval" path = "src/main.rs" [dependencies] -markkit = { path = "../markkit", features = ["all_formats", "http_client"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -serde.workspace = true -serde_json.workspace = true clap.workspace = true anyhow.workspace = true -reqwest.workspace = true -regex.workspace = true -async-trait.workspace = true diff --git a/crates/markkit-js/Cargo.toml b/crates/markkit-js/Cargo.toml index fd26309..cf99468 100644 --- a/crates/markkit-js/Cargo.toml +++ b/crates/markkit-js/Cargo.toml @@ -17,9 +17,6 @@ doc = false markkit = { path = "../markkit", features = ["all_formats", "http_client"] } napi.workspace = true napi-derive.workspace = true -serde.workspace = true -serde_json.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [build-dependencies] napi-build = "2" diff --git a/crates/markkit-python/Cargo.toml b/crates/markkit-python/Cargo.toml index c0d6f49..1d7300e 100644 --- a/crates/markkit-python/Cargo.toml +++ b/crates/markkit-python/Cargo.toml @@ -18,5 +18,3 @@ doc = false markkit = { path = "../markkit", features = ["all_formats", "http_client"] } pyo3.workspace = true pyo3-async-runtimes.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -serde_json.workspace = true diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index e21e445..c5fe350 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -15,42 +15,28 @@ readme = "../../README.md" [features] default = ["html", "http_client"] -html = ["dep:scraper", "dep:html5ever", "dep:lol_html"] +html = ["dep:scraper", "dep:html5ever"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] http_client = ["dep:reqwest"] -logging = ["dep:tracing", "dep:tracing-subscriber"] +logging = ["dep:tracing"] all_formats = ["html", "pdf", "docx", "spreadsheet"] [dependencies] # Async tokio = { workspace = true, features = ["sync", "io-util"] } -async-trait.workspace = true -futures.workspace = true # Error handling thiserror.workspace = true -anyhow.workspace = true # Serialization serde.workspace = true serde_json.workspace = true -# Text processing -regex.workspace = true -unicode-segmentation.workspace = true - -# Encoding -encoding_rs.workspace = true -base64.workspace = true -mime.workspace = true - # HTML (optional) scraper = { workspace = true, optional = true } html5ever = { workspace = true, optional = true } -lol_html = { workspace = true, optional = true } - # Office documents (optional): DOCX + spreadsheets share the ZIP/XML stack zip = { workspace = true, optional = true } @@ -65,9 +51,9 @@ reqwest = { workspace = true, optional = true } # Logging (optional) tracing = { workspace = true, optional = true } -tracing-subscriber = { workspace = true, optional = true } [dev-dependencies] +anyhow.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tempfile.workspace = true proptest.workspace = true diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index c86b04b..2ca280c 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -550,15 +550,15 @@ fn scan_biff_sheet(sheet: &[u8]) -> Result { // ─── ODS pre-scan (row / column repeats) ──────────────────────────── -fn attr_u64(e: &BytesStart<'_>, local: &[u8]) -> Option { +fn attr_u64(e: &BytesStart<'_>, local: &str) -> Option { e.attributes().with_checks(false).flatten().find_map(|a| { (a.key.local_name().as_ref() == local) - .then(|| std::str::from_utf8(&a.value).ok()?.trim().parse().ok()) + .then(|| a.value.trim().parse().ok()) .flatten() }) } -fn has_attr(e: &BytesStart<'_>, local: &[u8]) -> bool { +fn has_attr(e: &BytesStart<'_>, local: &str) -> bool { e.attributes() .with_checks(false) .flatten() @@ -587,14 +587,14 @@ impl OdsTable { } fn start_row(&mut self, e: &BytesStart<'_>) { - let rep = attr_u64(e, b"number-rows-repeated").unwrap_or(1).max(1); + let rep = attr_u64(e, "number-rows-repeated").unwrap_or(1).max(1); self.row_repeat = rep.min(ODS_MAX_ROWS.saturating_sub(self.next_row)).max(1); self.col = 0; self.width = 0; } fn cell(&mut self, e: &BytesStart<'_>, non_empty: bool) { - let rep = attr_u64(e, b"number-columns-repeated").unwrap_or(1).max(1); + let rep = attr_u64(e, "number-columns-repeated").unwrap_or(1).max(1); self.col = self.col.saturating_add(rep).min(ODS_MAX_COLS); if non_empty { self.width = self.col; @@ -636,8 +636,8 @@ fn prescan_ods(content: &[u8], max_cells: u64) -> Result<(), Error> { Event::Start(ref e) | Event::Empty(ref e) => { let is_empty_el = matches!(ev, Event::Empty(_)); match e.local_name().as_ref() { - b"table" if !is_empty_el => table = Some(OdsTable::default()), - b"table-row" => { + "table" if !is_empty_el => table = Some(OdsTable::default()), + "table-row" => { if let Some(t) = table.as_mut() { t.start_row(e); if is_empty_el { @@ -645,9 +645,9 @@ fn prescan_ods(content: &[u8], max_cells: u64) -> Result<(), Error> { } } } - b"table-cell" | b"covered-table-cell" => { + "table-cell" | "covered-table-cell" => { if let Some(t) = table.as_mut() { - let non_empty = !is_empty_el || has_attr(e, b"value-type"); + let non_empty = !is_empty_el || has_attr(e, "value-type"); t.cell(e, non_empty); } } @@ -655,13 +655,13 @@ fn prescan_ods(content: &[u8], max_cells: u64) -> Result<(), Error> { } } Event::End(ref e) => match e.local_name().as_ref() { - b"table-row" => { + "table-row" => { if let Some(t) = table.as_mut() { t.end_row(); check(total.saturating_add(t.area()))?; } } - b"table" => { + "table" => { if let Some(t) = table.take() { total = total.saturating_add(t.area()); check(total)?; diff --git a/crates/markkit/src/xml.rs b/crates/markkit/src/xml.rs index f755cc4..a2881ba 100644 --- a/crates/markkit/src/xml.rs +++ b/crates/markkit/src/xml.rs @@ -7,7 +7,6 @@ //! billion-laughs / XXE payloads surface as unresolved references, which we drop. //! Tree depth is capped and the tree is built iteratively (no recursion on input). -use quick_xml::encoding::Decoder; use quick_xml::events::{BytesStart, Event}; use quick_xml::{Reader, XmlVersion}; @@ -92,21 +91,20 @@ impl Element { } } -fn local(name: &[u8]) -> String { - let local = match name.iter().rposition(|&b| b == b':') { - Some(i) => &name[i + 1..], - None => name, - }; - String::from_utf8_lossy(local).into_owned() +fn local(name: &str) -> String { + match name.rsplit_once(':') { + Some((_, local)) => local.to_string(), + None => name.to_string(), + } } -fn element_from(start: &BytesStart<'_>, decoder: Decoder) -> Element { +fn element_from(start: &BytesStart<'_>) -> Element { let mut attrs = Vec::new(); for attr in start.attributes().with_checks(false).flatten() { let key = local(attr.key.as_ref()); - let value = match attr.decoded_and_normalized_value(XmlVersion::Implicit1_0, decoder) { + let value = match attr.normalized_value(XmlVersion::Implicit1_0) { Ok(v) => v.into_owned(), - Err(_) => String::from_utf8_lossy(&attr.value).into_owned(), + Err(_) => attr.value.into_owned(), }; attrs.push((key, value)); } @@ -146,10 +144,10 @@ pub(crate) fn parse(bytes: &[u8]) -> Result { "XML nesting deeper than {MAX_XML_DEPTH}" ))); } - stack.push(element_from(&s, reader.decoder())); + stack.push(element_from(&s)); } Event::Empty(s) => { - let el = element_from(&s, reader.decoder()); + let el = element_from(&s); if let Some(top) = stack.last_mut() { top.children.push(XNode::Element(el)); } @@ -163,25 +161,21 @@ pub(crate) fn parse(bytes: &[u8]) -> Result { } } Event::Text(t) => { - let text = t - .xml_content(XmlVersion::Implicit1_0) - .map_err(|e| XmlError::Invalid(format!("bad text encoding: {e}")))?; + let text = t.xml_content(XmlVersion::Implicit1_0); if let Some(top) = stack.last_mut() { push_text(top, &text); } } Event::CData(t) => { - let text = String::from_utf8_lossy(&t); if let Some(top) = stack.last_mut() { - push_text(top, &text); + push_text(top, &t); } } Event::GeneralRef(r) => { let resolved: Option = match r.resolve_char_ref() { Ok(Some(c)) => Some(c.to_string()), Ok(None) => { - let name = String::from_utf8_lossy(&r); - quick_xml::escape::resolve_predefined_entity(&name).map(str::to_string) + quick_xml::escape::resolve_predefined_entity(&r).map(str::to_string) } Err(_) => None, }; diff --git a/deny.toml b/deny.toml index eed71ab..b4f7594 100644 --- a/deny.toml +++ b/deny.toml @@ -1,21 +1,14 @@ [advisories] -ignore = [ - { id = "RUSTSEC-2024-0436", reason = "paste is a transitive dep of statrs/nalgebra, no security impact" }, - { id = "RUSTSEC-2024-0384", reason = "instant is a transitive dep of markkit-bench, no security impact" }, -] +ignore = [] [licenses] allow = [ "MIT", "Apache-2.0", - "BSD-2-Clause", "BSD-3-Clause", "ISC", "Unicode-3.0", - "Unicode-DFS-2016", "Zlib", - "OpenSSL", - "BSL-1.0", "MPL-2.0", "CDLA-Permissive-2.0", "Apache-2.0 WITH LLVM-exception", diff --git a/knowledge/log.md b/knowledge/log.md index 8701f9f..1cb075e 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). * **OKF migration**: `specs/` became this OKF v0.2 bundle, grouped into `foundations/`, `formats/`, `security/`, `runtimes/`, `operations/`. Every concept gained frontmatter and a See also section; `just check-okf` and CI enforce conformance. See [Knowledge Maintenance Contract](knowledge-contract.md). * **Accuracy pass**: [Python Package](runtimes/python-package.md) now names the shipped `markkit` module; [Examples](operations/examples.md) lists the four real examples; [Evaluation Harness](operations/eval.md) is marked as a stub; [Threat Model](security/threat-model.md) T3 records that `http_client` is a default feature and SSRF mitigations are still open. diff --git a/knowledge/operations/maintenance.md b/knowledge/operations/maintenance.md index 9b21de8..e975a47 100644 --- a/knowledge/operations/maintenance.md +++ b/knowledge/operations/maintenance.md @@ -132,6 +132,10 @@ Findings shipped as separate PRs in the same pass: unused dependencies (core `async-trait`, `futures`, `regex`, `unicode-segmentation`, `encoding_rs`, `base64`, `mime`, `lol_html`; most stub-crate deps) and pending major upgrades; indented HTML losing paragraph breaks between blocks (smoke test of `examples/basic.rs`). +Dependency PR: 18 unused deps removed, quick-xml/scraper/html5ever/pyo3/criterion +majors applied, `deny.toml` ignores and unused license allowances dropped; +`cargo audit` reports nothing, `cargo deny` one duplicate (`syn` 1/2, proc-macro +ecosystem split). ## See also diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 5c0bb00..9d906ac 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -168,7 +168,7 @@ process, or smuggling content past a human reviewer. #### T6d: XML entity attacks (billion laughs, XXE) - quick-xml never expands DTD-declared or external entities; such references are dropped. Only the five predefined entities and character references are resolved. -- quick-xml >= 0.41 (via calamine 0.36) fixes RUSTSEC-2026-0194 (quadratic duplicate-attribute +- quick-xml 0.42 (one version shared with calamine 0.37) includes the RUSTSEC-2026-0194 fix (quadratic duplicate-attribute check); markkit's own XML walkers iterate attributes with `with_checks(false)` and never use `NsReader` (RUSTSEC-2026-0195). From 82c87a973f55cfd6a268051f3ed876d8d6b63e9f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 20:34:21 +0000 Subject: [PATCH 2/3] fix(security): block SSRF and bound extract_url fetches extract_url refuses non-public destinations by default (URL check, redirect policy, DNS resolver filter), allows only http/https, caps redirects, bounds the body by max_input_size while streaming and the fetch by max_processing_secs. New builder options allow_private_networks, allow_host, max_redirects and Error::UrlBlocked. Threat model T3 mitigated, invariant S12. --- AGENTS.md | 2 +- CHANGELOG.md | 11 +- README.md | 11 +- SECURITY.md | 2 +- crates/markkit/Cargo.toml | 4 +- crates/markkit/src/config.rs | 29 +++ crates/markkit/src/error.rs | 5 + crates/markkit/src/extract.rs | 27 +-- crates/markkit/src/fetch.rs | 336 ++++++++++++++++++++++++++ crates/markkit/src/lib.rs | 2 + crates/markkit/tests/url_security.rs | 262 ++++++++++++++++++++ knowledge/foundations/architecture.md | 1 + knowledge/log.md | 1 + knowledge/security/security-model.md | 5 +- knowledge/security/threat-model.md | 45 +++- 15 files changed, 707 insertions(+), 36 deletions(-) create mode 100644 crates/markkit/src/fetch.rs create mode 100644 crates/markkit/tests/url_security.rs diff --git a/AGENTS.md b/AGENTS.md index 97fa39f..9204780 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | formats/html-extraction | HTML element mapping, stripping, whitespace, depth limits | | formats/format-support | Formats, bytes-in API, detection order, DOCX + spreadsheet mapping | | security/security-model | One-page threat/invariant summary | -| security/threat-model | Threats T1-T6, mitigations, test map, resource limits, invariants S1-S11 | +| security/threat-model | Threats T1-T6, mitigations, test map, resource limits, invariants S1-S12 | | runtimes/python-package | PyO3 `markkit` module, maturin, PyPI | | runtimes/node-package | NAPI-RS `@everruns/markkit`, npm, binding parity table | | operations/testing | Test layout, categories, CI jobs | diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a93e34..6b78190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Security + +- `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: + loopback, private, link-local / cloud metadata and other reserved addresses, checked on the URL, + after DNS resolution and on every redirect. Only `http`/`https` are fetched. Redirects are capped + (`max_redirects`, default 5), the body is bounded by `max_input_size` while streaming, and the whole + fetch by `max_processing_secs`. New builder options `allow_private_networks(true)` and `allow_host(..)`; + refusals return the new `Error::UrlBlocked`. + ### Changed - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. - CI runs every example; `just bench` now runs the criterion benchmarks. -- README, SECURITY and binding docs corrected: Python module is `markkit`, URL fetching has no private-network blocking yet. +- README, SECURITY and binding docs corrected: Python module is `markkit`. - Dependencies: drop unused `async-trait`, `futures`, `regex`, `unicode-segmentation`, `encoding_rs`, `base64`, `mime`, `lol_html` (core) and `tracing-subscriber`, plus unused deps of the CLI, bindings and stub crates. Upgrade scraper 0.29, html5ever 0.40, quick-xml 0.42 (now a single version shared diff --git a/README.md b/README.md index 4987e1b..0d9e55a 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,8 @@ markkit = { version = "0.1", default-features = false, features = ["docx", "spre Office-document limits are configurable on the builder: `max_archive_entries`, `max_archive_entry_size`, `max_archive_total_size`, `max_spreadsheet_cells`, -`max_sheet_rows`, `max_sheet_cols`, `include_hidden_sheets`. +`max_sheet_rows`, `max_sheet_cols`, `include_hidden_sheets`. URL fetching: +`allow_private_networks`, `allow_host`, `max_redirects`. ## Security @@ -176,9 +177,11 @@ Markkit treats every input as untrusted. HTML scripts, styles, forms, iframes an SVG are stripped; office containers are decompressed under measured byte limits; spreadsheet grids are pre-scanned so sparse cells cannot force huge allocations; XML entities are never expanded; macros and formulas are never executed; hidden -sheets are skipped by default. URL fetching does not yet block private networks, -so embedders fetching attacker-chosen URLs should fetch themselves and call -`extract_document` (build with `default-features = false`). +sheets are skipped by default. `extract_url` accepts only http/https, refuses +loopback, private, link-local and cloud-metadata addresses (also after DNS +resolution and on every redirect), caps redirects, and bounds the body by +`max_input_size` and the fetch by `max_processing_secs`. Opt in to internal +hosts with `allow_private_networks(true)`; restrict hosts with `allow_host(..)`. See the [threat model](knowledge/security/threat-model.md) and the [security policy](SECURITY.md) for reporting vulnerabilities. diff --git a/SECURITY.md b/SECURITY.md index fb4d01c..93dc1d3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -32,7 +32,7 @@ Markkit processes untrusted input (HTML, DOCX, spreadsheets) and converts it to - **No code execution**: scripts, styles, forms, iframes and SVG are stripped; macros and formulas are never executed; XML entities are never expanded - **Resource limits**: input size, HTML/XML nesting depth, archive entry count and decompressed size, spreadsheet grid cells, rows and columns - **Explicit failure**: malformed input returns errors, never panics; unsupported formats fail explicitly -- **URL fetching**: enabled by the default `http_client` feature; private-network blocking and allowlists are not implemented yet, so fetch attacker-chosen URLs yourself and build with `default-features = false` +- **URL fetching**: enabled by the default `http_client` feature; http/https only, non-public addresses refused by default (after DNS and on every redirect), optional host allowlist, bounded redirects, body size and fetch time. Behind an environment HTTP proxy, the proxy resolves names, so restrict egress there too Full analysis: [threat model](knowledge/security/threat-model.md). diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index c5fe350..f6f5d96 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -19,7 +19,7 @@ html = ["dep:scraper", "dep:html5ever"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] -http_client = ["dep:reqwest"] +http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] all_formats = ["html", "pdf", "docx", "spreadsheet"] @@ -54,7 +54,7 @@ tracing = { workspace = true, optional = true } [dev-dependencies] anyhow.workspace = true -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "time"] } tempfile.workspace = true proptest.workspace = true criterion.workspace = true diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index 14b0f75..9f86747 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -32,6 +32,14 @@ pub struct MarkkitBuilder { pub max_spreadsheet_cells: u64, /// Render hidden / very hidden sheets. Default: false (skipped with a warning). pub include_hidden_sheets: bool, + /// Let `extract_url` reach loopback, private, link-local and other + /// non-public addresses. Default: false (SSRF protection). + pub allow_private_networks: bool, + /// Hosts `extract_url` may contact, matched exactly and case-insensitively + /// on every hop (empty = any public host). + pub allowed_hosts: Vec, + /// Maximum redirects `extract_url` follows. Default: 5. + pub max_redirects: usize, } impl Default for MarkkitBuilder { @@ -58,6 +66,9 @@ impl Default for MarkkitBuilder { max_sheet_cols: 256, max_spreadsheet_cells: 4_000_000, include_hidden_sheets: false, + allow_private_networks: false, + allowed_hosts: Vec::new(), + max_redirects: 5, } } } @@ -128,6 +139,24 @@ impl MarkkitBuilder { self } + /// Allow `extract_url` to reach non-public addresses (loopback, RFC 1918, + /// link-local, cloud metadata, ...). Only for trusted URLs. + pub fn allow_private_networks(mut self, yes: bool) -> Self { + self.allow_private_networks = yes; + self + } + + /// Restrict `extract_url` to this host (repeatable). Checked on every redirect hop. + pub fn allow_host(mut self, host: impl Into) -> Self { + self.allowed_hosts.push(host.into()); + self + } + + pub fn max_redirects(mut self, n: usize) -> Self { + self.max_redirects = n; + self + } + pub fn build(self) -> Markkit { Markkit::new(self) } diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index c291664..0474b18 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -16,6 +16,11 @@ pub enum Error { #[error("HTTP error: {0}")] Http(String), + /// `extract_url` refused a URL: disallowed scheme, host outside the + /// allowlist, or an address on a non-public network. + #[error("URL blocked: {0}")] + UrlBlocked(String), + #[error("Encoding error: {0}")] Encoding(String), diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 2d84a9d..06ce32b 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -235,25 +235,20 @@ impl Markkit { )) } - /// Extract markdown from a URL by fetching and converting. + /// Fetch a URL and extract markdown from the response. + /// + /// Treats the URL as untrusted (threat model T3): only `http`/`https`; + /// loopback, private, link-local and other non-public addresses are + /// refused with [`Error::UrlBlocked`] on every hop (including after DNS + /// resolution) unless [`MarkkitBuilder::allow_private_networks`] is set; + /// [`MarkkitBuilder::allow_host`] restricts hosts; redirects capped by + /// `max_redirects`; body capped by `max_input_size`; whole fetch capped + /// by `max_processing_secs` ([`Error::ResourceLimit`]). #[cfg(feature = "http_client")] pub async fn extract_url(&self, url: &str) -> crate::Result { - let response = reqwest::get(url) + let fetched = crate::fetch::fetch(url, &self.config).await?; + self.extract_bytes(&fetched.bytes, fetched.content_type.as_deref()) .await - .map_err(|e| Error::Http(e.to_string()))?; - - let content_type = response - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); - - let bytes = response - .bytes() - .await - .map_err(|e| Error::Http(e.to_string()))?; - - self.extract_bytes(&bytes, content_type.as_deref()).await } /// PDF extraction is not implemented yet; report it explicitly rather than diff --git a/crates/markkit/src/fetch.rs b/crates/markkit/src/fetch.rs new file mode 100644 index 0000000..bc4a127 --- /dev/null +++ b/crates/markkit/src/fetch.rs @@ -0,0 +1,336 @@ +//! Hardened HTTP fetch behind [`Markkit::extract_url`](crate::Markkit::extract_url) +//! (threat model T3). +//! +//! Decisions: +//! - Default-deny non-public destinations. Three layers: (1) URL check before +//! any I/O (scheme, allowlist, IP literal, `localhost` names), (2) the same +//! check on every redirect hop, (3) a DNS resolver that drops non-public +//! addresses, so a public name resolving to 127.0.0.1 / 169.254.169.254 +//! (DNS rebinding) never connects. Layer 3 pins the addresses reqwest +//! connects to, so there is no check-then-resolve gap. +//! - With an HTTP(S) proxy from the environment, the proxy resolves names; +//! layers 1-2 still apply, layer 3 cannot. Documented in the threat model. +//! - Body bounded by `max_input_size` while streaming (declared +//! `Content-Length` rejected up front); whole fetch bounded by +//! `max_processing_secs`. +//! - Policy and resolver run inside reqwest and can only return opaque +//! errors, so they record the block reason in a per-call slot that is +//! turned into [`Error::UrlBlocked`] afterwards. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use reqwest::Url; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::config::MarkkitBuilder; +use crate::error::Error; + +/// Fetched body plus its declared content type. +pub(crate) struct Fetched { + pub bytes: Vec, + pub content_type: Option, +} + +/// Per-call slot for the first block reason seen inside reqwest. +type Blocked = Arc>>; + +fn record(slot: &Blocked, reason: String) -> String { + let mut s = slot.lock().unwrap_or_else(|e| e.into_inner()); + s.get_or_insert(reason).clone() +} + +pub(crate) async fn fetch(url: &str, cfg: &MarkkitBuilder) -> crate::Result { + let parsed = Url::parse(url).map_err(|e| Error::UrlBlocked(format!("invalid URL: {e}")))?; + check_url(&parsed, cfg).map_err(Error::UrlBlocked)?; + + let blocked: Blocked = Arc::default(); + let client = client(cfg, &blocked)?; + let limit = cfg.max_input_size; + + let run = async { + let mut response = client.get(parsed).send().await?; + if let Some(len) = response.content_length() + && len > limit as u64 + { + return Ok(Err(Error::ResourceLimit(format!( + "response Content-Length {len} exceeds max_input_size {limit}" + )))); + } + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + if bytes.len() + chunk.len() > limit { + return Ok(Err(Error::ResourceLimit(format!( + "response body exceeds max_input_size {limit}" + )))); + } + bytes.extend_from_slice(&chunk); + } + Ok(Ok(Fetched { + bytes, + content_type, + })) + }; + + let result: Result, reqwest::Error> = run.await; + match result { + Ok(r) => r, + Err(e) => { + if let Some(reason) = blocked.lock().unwrap_or_else(|p| p.into_inner()).take() { + return Err(Error::UrlBlocked(reason)); + } + if e.is_timeout() { + return Err(Error::ResourceLimit(format!( + "fetch exceeded max_processing_secs {}", + cfg.max_processing_secs + ))); + } + Err(Error::Http(e.to_string())) + } + } +} + +fn client(cfg: &MarkkitBuilder, blocked: &Blocked) -> crate::Result { + let max_redirects = cfg.max_redirects; + let policy_cfg = cfg.clone(); + let policy_slot = blocked.clone(); + let policy = reqwest::redirect::Policy::custom(move |attempt| { + if attempt.previous().len() > max_redirects { + let reason = format!("too many redirects (max {max_redirects})"); + return attempt.error(record(&policy_slot, reason)); + } + match check_url(attempt.url(), &policy_cfg) { + Ok(()) => attempt.follow(), + Err(reason) => { + let reason = record(&policy_slot, format!("redirect: {reason}")); + attempt.error(reason) + } + } + }); + + let secs = cfg.max_processing_secs.max(1); + let mut builder = reqwest::Client::builder() + .redirect(policy) + .timeout(Duration::from_secs(secs)) + .connect_timeout(Duration::from_secs(secs.min(10))); + if !cfg.allow_private_networks { + builder = builder.dns_resolver(Arc::new(PublicOnlyResolver { + blocked: blocked.clone(), + })); + } + builder + .build() + .map_err(|e| Error::Http(format!("client setup: {e}"))) +} + +/// Check scheme, allowlist and (unless private networks are allowed) IP +/// literals / `localhost` names. Pure: no DNS, no I/O. +fn check_url(url: &Url, cfg: &MarkkitBuilder) -> Result<(), String> { + match url.scheme() { + "http" | "https" => {} + s => return Err(format!("scheme '{s}' not allowed (http/https only)")), + } + let host = url + .host_str() + .ok_or_else(|| "URL has no host".to_string())?; + let host = normalize_host(host); + + if !cfg.allowed_hosts.is_empty() && !cfg.allowed_hosts.iter().any(|h| normalize_host(h) == host) + { + return Err(format!("host '{host}' not in allowed hosts")); + } + + if cfg.allow_private_networks { + return Ok(()); + } + if host == "localhost" || host.ends_with(".localhost") { + return Err(format!("host '{host}' is a loopback name")); + } + if let Ok(ip) = host.parse::() + && is_forbidden_ip(ip) + { + return Err(format!("address {ip} is not public")); + } + Ok(()) +} + +/// Lowercase, strip IPv6 brackets and a trailing root dot. +fn normalize_host(host: &str) -> String { + let h = host.trim_start_matches('[').trim_end_matches(']'); + h.strip_suffix('.').unwrap_or(h).to_ascii_lowercase() +} + +/// True for any address that is not globally routable unicast. +pub(crate) fn is_forbidden_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => is_forbidden_v4(v4), + IpAddr::V6(v6) => is_forbidden_v6(v6), + } +} + +fn is_forbidden_v4(ip: Ipv4Addr) -> bool { + let [a, b, c, _] = ip.octets(); + ip.is_unspecified() + || ip.is_loopback() + || ip.is_private() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_multicast() + || a == 0 // 0.0.0.0/8 "this network" + || (a == 100 && (64..128).contains(&b)) // 100.64.0.0/10 CGNAT + || (a == 192 && b == 0 && c == 0) // 192.0.0.0/24 IETF protocol + || (a == 198 && (b == 18 || b == 19)) // 198.18.0.0/15 benchmarking + || a >= 240 // 240.0.0.0/4 reserved +} + +fn is_forbidden_v6(ip: Ipv6Addr) -> bool { + let seg = ip.segments(); + // Addresses that embed an IPv4 address are judged by it. + if let Some(v4) = ip.to_ipv4_mapped() { + return is_forbidden_v4(v4); + } + let embedded = |hi: u16, lo: u16| Ipv4Addr::from(((hi as u32) << 16) | lo as u32); + if seg[..6] == [0; 6] && !(ip.is_loopback() || ip.is_unspecified()) { + return is_forbidden_v4(embedded(seg[6], seg[7])); // ::a.b.c.d (deprecated compat) + } + if seg[..6] == [0x64, 0xff9b, 0, 0, 0, 0] { + return is_forbidden_v4(embedded(seg[6], seg[7])); // NAT64 64:ff9b::/96 + } + if seg[0] == 0x2002 { + return is_forbidden_v4(embedded(seg[1], seg[2])); // 6to4 2002::/16 + } + ip.is_loopback() + || ip.is_unspecified() + || ip.is_multicast() + || (seg[0] & 0xfe00) == 0xfc00 // fc00::/7 unique local + || (seg[0] & 0xffc0) == 0xfe80 // fe80::/10 link-local + || (seg[0] & 0xffc0) == 0xfec0 // fec0::/10 site-local (deprecated) + || seg[0] == 0x2001 && seg[1] == 0x0db8 // 2001:db8::/32 documentation +} + +/// Resolver that only hands reqwest public addresses. +struct PublicOnlyResolver { + blocked: Blocked, +} + +impl Resolve for PublicOnlyResolver { + fn resolve(&self, name: Name) -> Resolving { + let blocked = self.blocked.clone(); + let host = name.as_str().to_string(); + Box::pin(async move { + let addrs: Vec = + tokio::net::lookup_host((host.as_str(), 0)).await?.collect(); + let public: Vec = addrs + .iter() + .copied() + .filter(|a| !is_forbidden_ip(a.ip())) + .collect(); + if public.is_empty() { + let reason = if addrs.is_empty() { + format!("host '{host}' did not resolve") + } else { + format!("host '{host}' resolves only to non-public addresses") + }; + return Err(record(&blocked, reason).into()); + } + let addrs: Addrs = Box::new(public.into_iter()); + Ok(addrs) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn forbidden(s: &str) -> bool { + is_forbidden_ip(s.parse().unwrap()) + } + + #[test] + fn non_public_addresses_forbidden() { + for ip in [ + "127.0.0.1", + "10.1.2.3", + "172.31.255.255", + "192.168.0.1", + "169.254.169.254", + "100.64.0.1", + "0.1.2.3", + "192.0.0.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + "240.0.0.1", + "::1", + "::", + "::ffff:10.0.0.1", + "::127.0.0.1", + "64:ff9b::a9fe:a9fe", // NAT64 of 169.254.169.254 + "2002:7f00:1::", // 6to4 of 127.0.0.1 + "fd12::1", + "fe80::1", + "ff02::1", + "2001:db8::1", + ] { + assert!(forbidden(ip), "{ip} should be forbidden"); + } + } + + #[test] + fn public_addresses_allowed() { + for ip in [ + "93.184.216.34", + "8.8.8.8", + "100.128.0.1", + "172.32.0.1", + "2606:4700::1111", + "::ffff:8.8.8.8", + "64:ff9b::808:808", + "2002:808:808::", + ] { + assert!(!forbidden(ip), "{ip} should be allowed"); + } + } + + fn check(url: &str, cfg: &MarkkitBuilder) -> Result<(), String> { + check_url(&Url::parse(url).unwrap(), cfg) + } + + #[test] + fn check_url_default_policy() { + let cfg = MarkkitBuilder::default(); + assert!(check("https://example.com/a", &cfg).is_ok()); + assert!(check("http://8.8.8.8/", &cfg).is_ok()); + assert!(check("http://localhost/", &cfg).is_err()); + assert!(check("http://a.localhost./", &cfg).is_err()); + assert!(check("http://[::1]:8080/", &cfg).is_err()); + assert!(check("data:text/html,hi", &cfg).is_err()); + } + + #[test] + fn check_url_allowlist() { + let cfg = MarkkitBuilder::default().allow_host("Example.COM."); + assert!(check("https://example.com/", &cfg).is_ok()); + assert!(check("https://EXAMPLE.com./x", &cfg).is_ok()); + assert!(check("https://sub.example.com/", &cfg).is_err()); + assert!(check("https://example.com.evil.net/", &cfg).is_err()); + } + + #[test] + fn check_url_private_opt_in_keeps_scheme_and_allowlist() { + let cfg = MarkkitBuilder::default() + .allow_private_networks(true) + .allow_host("127.0.0.1"); + assert!(check("http://127.0.0.1:9/", &cfg).is_ok()); + assert!(check("http://10.0.0.1/", &cfg).is_err()); + assert!(check("file:///etc/passwd", &cfg).is_err()); + } +} diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 3252a3d..d6453e2 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -47,6 +47,8 @@ pub mod config; mod docx; pub mod error; pub mod extract; +#[cfg(feature = "http_client")] +mod fetch; pub mod format; #[cfg(feature = "html")] pub mod html; diff --git a/crates/markkit/tests/url_security.rs b/crates/markkit/tests/url_security.rs new file mode 100644 index 0000000..a7657d4 --- /dev/null +++ b/crates/markkit/tests/url_security.rs @@ -0,0 +1,262 @@ +//! Security tests for `extract_url` (threat model T3 SSRF, T2/T4 via remote bodies). +//! +//! A raw TCP server on loopback stands in for the remote host. Loopback is +//! itself a forbidden destination by default, which the first tests assert; +//! the rest opt in with `allow_private_networks(true)` to exercise limits. + +#![cfg(feature = "http_client")] + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use markkit::Markkit; +use markkit::error::Error; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +/// Serve `respond(path)` to every connection; returns base URL and a hit counter. +async fn serve(respond: F) -> (String, Arc) +where + F: Fn(&str) -> Vec + Send + Sync + 'static, +{ + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let respond = Arc::new(respond); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + counter.fetch_add(1, Ordering::SeqCst); + let respond = respond.clone(); + tokio::spawn(async move { + let mut buf = vec![0u8; 4096]; + let n = sock.read(&mut buf).await.unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..n]); + let path = req.split_whitespace().nth(1).unwrap_or("/").to_string(); + let _ = sock.write_all(&respond(&path)).await; + let _ = sock.shutdown().await; + }); + } + }); + (format!("http://{addr}"), hits) +} + +fn html(body: &str) -> Vec { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .into_bytes() +} + +fn redirect(to: &str) -> Vec { + format!( + "HTTP/1.1 302 Found\r\nLocation: {to}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + .into_bytes() +} + +fn private_ok() -> Markkit { + Markkit::builder().allow_private_networks(true).build() +} + +fn assert_blocked(r: markkit::Result) { + match r { + Err(Error::UrlBlocked(_)) => {} + other => panic!("expected UrlBlocked, got {other:?}"), + } +} + +// ─── Destination checks (default policy) ───────────────────────────── + +#[tokio::test] +async fn loopback_ip_blocked_by_default_without_connecting() { + let (base, hits) = serve(|_| html("

secret

")).await; + assert_blocked(Markkit::builder().build().extract_url(&base).await); + assert_eq!( + hits.load(Ordering::SeqCst), + 0, + "no request may reach the host" + ); +} + +#[tokio::test] +async fn localhost_name_blocked_by_default_without_connecting() { + let (base, hits) = serve(|_| html("

secret

")).await; + let url = base.replace("127.0.0.1", "localhost"); + assert_blocked(Markkit::builder().build().extract_url(&url).await); + assert_eq!(hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn forbidden_ip_literals_blocked() { + let m = Markkit::builder().build(); + for url in [ + "http://169.254.169.254/latest/meta-data/", + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://172.16.0.1/", + "http://100.64.0.1/", + "http://0.0.0.0/", + "http://[::1]/", + "http://[::ffff:127.0.0.1]/", + "http://[fd00::1]/", + "http://[fe80::1]/", + "http://2130706433/", // 127.0.0.1 as a decimal integer + "http://0x7f.0.0.1/", + ] { + assert_blocked(m.extract_url(url).await); + } +} + +#[tokio::test] +async fn non_http_schemes_blocked() { + let m = private_ok(); + for url in [ + "file:///etc/passwd", + "ftp://example.com/", + "gopher://127.0.0.1/", + ] { + assert_blocked(m.extract_url(url).await); + } +} + +#[tokio::test] +async fn invalid_url_is_error_not_panic() { + assert!( + Markkit::builder() + .build() + .extract_url("not a url") + .await + .is_err() + ); +} + +// ─── Opt-in private access ────────────────────────────────────────── + +#[tokio::test] +async fn private_networks_reachable_when_allowed() { + let (base, _) = serve(|_| html("

Hi

")).await; + assert_eq!(private_ok().extract_url(&base).await.unwrap(), "# Hi\n"); +} + +// ─── Host allowlist ───────────────────────────────────────────────── + +#[tokio::test] +async fn host_outside_allowlist_blocked() { + let (base, hits) = serve(|_| html("

x

")).await; + let m = Markkit::builder() + .allow_private_networks(true) + .allow_host("example.com") + .build(); + assert_blocked(m.extract_url(&base).await); + assert_eq!(hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn redirect_to_host_outside_allowlist_blocked() { + let (base, _) = serve(|path| { + if path == "/" { + redirect("http://localhost:9/") // different host than 127.0.0.1 + } else { + html("

x

") + } + }) + .await; + let m = Markkit::builder() + .allow_private_networks(true) + .allow_host("127.0.0.1") + .build(); + assert_blocked(m.extract_url(&base).await); +} + +#[tokio::test] +async fn allowlisted_host_matches_case_insensitively() { + let (base, _) = serve(|_| html("

ok

")).await; + let url = base.replace("127.0.0.1", "LOCALHOST"); + let m = Markkit::builder() + .allow_private_networks(true) + .allow_host("localhost") + .build(); + assert_eq!(m.extract_url(&url).await.unwrap(), "ok\n"); +} + +// ─── Redirects ────────────────────────────────────────────────────── + +#[tokio::test] +async fn redirect_loop_stops_at_limit() { + let (base, hits) = serve(|_| redirect("/again")).await; + let m = Markkit::builder() + .allow_private_networks(true) + .max_redirects(2) + .build(); + assert!(m.extract_url(&base).await.is_err()); + assert!(hits.load(Ordering::SeqCst) <= 3, "followed past the limit"); +} + +#[tokio::test] +async fn redirect_to_non_http_scheme_blocked() { + let (base, _) = serve(|_| redirect("file:///etc/passwd")).await; + assert!(private_ok().extract_url(&base).await.is_err()); +} + +// ─── Body size and time limits ────────────────────────────────────── + +#[tokio::test] +async fn declared_oversized_body_rejected() { + let (base, _) = serve(|_| { + b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 100000000\r\nConnection: close\r\n\r\n

".to_vec() + }) + .await; + let m = Markkit::builder() + .allow_private_networks(true) + .max_input_size(1024) + .build(); + match m.extract_url(&base).await { + Err(Error::ResourceLimit(_)) => {} + other => panic!("expected ResourceLimit, got {other:?}"), + } +} + +#[tokio::test] +async fn undeclared_oversized_body_rejected_while_streaming() { + let (base, _) = serve(|_| { + let mut r = + b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n".to_vec(); + r.extend(std::iter::repeat_n(b'a', 64 * 1024)); + r + }) + .await; + let m = Markkit::builder() + .allow_private_networks(true) + .max_input_size(1024) + .build(); + match m.extract_url(&base).await { + Err(Error::ResourceLimit(_)) => {} + other => panic!("expected ResourceLimit, got {other:?}"), + } +} + +#[tokio::test] +async fn stalled_server_times_out() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (_sock, _) = listener.accept().await.unwrap(); + tokio::time::sleep(Duration::from_secs(30)).await; // never answer + }); + let m = Markkit::builder() + .allow_private_networks(true) + .max_processing_secs(1) + .build(); + let start = Instant::now(); + assert!(m.extract_url(&format!("http://{addr}/")).await.is_err()); + assert!( + start.elapsed() < Duration::from_secs(10), + "timeout not enforced" + ); +} diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 1451347..9781979 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -53,6 +53,7 @@ Via `MarkkitBuilder`: - CSS selectors for include/exclude - Link and image preservation - Office-document limits (archive entries/sizes, sheet rows/cols, grid cells, hidden sheets) +- URL fetch policy (`allow_private_networks`, `allow_host`, `max_redirects`) ## See also diff --git a/knowledge/log.md b/knowledge/log.md index 1cb075e..1b2ff07 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). * **OKF migration**: `specs/` became this OKF v0.2 bundle, grouped into `foundations/`, `formats/`, `security/`, `runtimes/`, `operations/`. Every concept gained frontmatter and a See also section; `just check-okf` and CI enforce conformance. See [Knowledge Maintenance Contract](knowledge-contract.md). diff --git a/knowledge/security/security-model.md b/knowledge/security/security-model.md index ffdbc78..a5086a6 100644 --- a/knowledge/security/security-model.md +++ b/knowledge/security/security-model.md @@ -24,8 +24,9 @@ Markkit processes untrusted input from the internet. Key threats: - **Mitigation**: Nesting depth limits ### TM-SSRF: Server-side request forgery (URL fetching) -- **Status**: open, mitigations planned (see [Threat Model](threat-model.md) T3) -- **Planned**: URL allowlist, private IP range blocking, redirect limit, bounded response body +- **Mitigation**: Non-public addresses refused by default, after DNS resolution and on every redirect (opt in with `allow_private_networks`) +- **Mitigation**: http/https only, optional host allowlist, redirect limit, body bounded by `max_input_size`, fetch bounded by `max_processing_secs` +- **Residual**: an environment proxy resolves names itself; see [Threat Model](threat-model.md) T3 ### TM-DOC: Malicious office documents - **Mitigation**: Bounded ZIP decompression (entry count, per-entry and total size) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 9d906ac..49d78f1 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -95,20 +95,44 @@ tags: **Likelihood**: Medium -- `extract_url()` exists whenever `http_client` is enabled, and `http_client` is a **default** feature. -**Status**: Open. `extract_url()` calls `reqwest::get` directly: reqwest's default -redirect policy (10 hops) applies, nothing else does. +**Status**: Mitigated (`crates/markkit/src/fetch.rs`). **Mitigations**: -- URL allowlist support (planned) -- Private IP range blocking (planned) -- Response body bounded by `max_input_size` before download completes (planned; - today `max_input_size` is checked only after the full body is buffered, see T4) -- Request timeout from `max_processing_secs` (planned) +- Scheme allowlist: `http`/`https` only, initial URL and every redirect hop +- Non-public destinations refused by default (`Error::UrlBlocked`): loopback, + RFC 1918, link-local / cloud metadata (169.254.0.0/16), CGNAT, `0.0.0.0/8`, + reserved, multicast, documentation, benchmarking; IPv6 loopback, ULA, + link-local, site-local, multicast; IPv4 embedded in IPv6 (mapped, + compatible, NAT64, 6to4) judged by the IPv4. `localhost` / `*.localhost` + refused by name. Non-canonical IPv4 (`2130706433`, `0x7f.0.0.1`) is + normalized by the URL parser before the check +- DNS rebinding: a custom resolver drops non-public addresses, and reqwest + connects only to what it returns (no check-then-resolve gap) +- Opt-in `allow_private_networks(true)` for trusted URLs (CLI / bindings use + the default, so they block too) +- Host allowlist `allow_host(..)`: exact, case-insensitive, trailing dot + ignored, checked on every hop (no subdomain wildcard) +- Redirects capped by `max_redirects` (default 5) +- Response body bounded by `max_input_size`: declared `Content-Length` rejected + before reading, undeclared bodies stopped while streaming (`ResourceLimit`) +- Whole fetch (connect, headers, body) bounded by `max_processing_secs` - Opt-out: embedders that fetch content themselves build with `default-features = false` and never link reqwest (CI asserts this for `docx,spreadsheet`) -**Test Coverage**: None yet. +**Residual risk**: with an `HTTP(S)_PROXY` in the environment the proxy +resolves names, so the resolver filter does not apply to proxied requests; +scheme, allowlist, IP-literal and redirect checks still do. Restrict egress at +the proxy, or use `allow_host`. + +**Test Coverage**: +- `url_security::loopback_ip_blocked_by_default_without_connecting`, `localhost_name_blocked_by_default_without_connecting` (no TCP hit) +- `url_security::forbidden_ip_literals_blocked` (metadata, RFC 1918, CGNAT, IPv6, mapped, decimal / hex IPv4) +- `url_security::non_http_schemes_blocked`, `redirect_to_non_http_scheme_blocked`, `invalid_url_is_error_not_panic` +- `url_security::host_outside_allowlist_blocked`, `redirect_to_host_outside_allowlist_blocked`, `allowlisted_host_matches_case_insensitively` +- `url_security::redirect_loop_stops_at_limit`, `private_networks_reachable_when_allowed` +- `url_security::declared_oversized_body_rejected`, `undeclared_oversized_body_rejected_while_streaming`, `stalled_server_times_out` +- `fetch::tests::*` (address classification and URL check units) ### T4: Memory Exhaustion @@ -212,7 +236,9 @@ process, or smuggling content past a human reviewer. | Resource | Default | Configurable | Enforcement Point | |----------|---------|--------------|-------------------| | Input size | 50 MB | `max_input_size(usize)` | Before parsing | -| Processing time | 60 s | `max_processing_secs(u64)` | Config field (enforcement TBD) | +| Processing time | 60 s | `max_processing_secs(u64)` | Whole `extract_url` fetch (parsing itself TBD) | +| Redirects (`extract_url`) | 5 | `max_redirects(usize)` | Every redirect hop | +| Fetched body (`extract_url`) | = input size | `max_input_size(usize)` | `Content-Length` and while streaming | | Nesting depth | 200 | Hardcoded `MAX_DEPTH` | During HTML tree walk | | XML nesting depth (DOCX) | 256 | Hardcoded `MAX_XML_DEPTH` | While building the XML tree | | ZIP entries | 10 000 | `max_archive_entries(usize)` | On archive open | @@ -258,6 +284,7 @@ process, or smuggling content past a human reviewer. | S9 | Malformed office documents produce errors, never panics | `document_security::truncated_fixtures_never_panic`, `*_never_panic` proptests | | S10 | Spreadsheet parsing never allocates a grid beyond `max_spreadsheet_cells` | `document_security::xls_*_rejected`, `ods_repeated_rows_and_columns_rejected` | | S11 | Unsupported formats fail explicitly (no empty success) | `document_detection::pdf_is_explicitly_unsupported_not_empty` | +| S12 | Default `extract_url` never connects to a non-public address | `url_security::*_blocked*`, `fetch::tests::non_public_addresses_forbidden` | ## See also From 045d179c567a5629f88d0227edb7746fe89373a4 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 20:47:56 +0000 Subject: [PATCH 3/3] ci(coverage): run tarpaulin with the llvm engine The ptrace engine segfaults on the multi-threaded tokio socket tests in tests/url_security.rs. --- .github/workflows/coverage.yml | 4 +++- knowledge/operations/testing.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1fae950..12004fa 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -22,7 +22,9 @@ jobs: - name: Install cargo-tarpaulin run: cargo install cargo-tarpaulin - name: Generate coverage - run: cargo tarpaulin --workspace --out xml + # llvm engine: the default ptrace engine segfaults on multi-threaded + # tokio socket tests (tests/url_security.rs) + run: cargo tarpaulin --workspace --engine llvm --out xml - name: Upload coverage uses: codecov/codecov-action@v4 with: diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index e639d7f..49bdfba 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -52,7 +52,7 @@ each security test states exactly which bytes it feeds the parser. `cargo audit` + `cargo deny`, `cargo test --workspace` (default and all features), the `docx,spreadsheet` no-HTTP build with a reqwest leak check, and every example. `nightly.yml` repeats build + test on nightly Rust daily; `coverage.yml` uploads -tarpaulin coverage. All must pass before merge. +tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). All must pass before merge. ## See also