From 97cf3303cd31a68777a29eea12893eeb3d724161 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 23:19:09 +0800 Subject: [PATCH] feat(web-fetch): extract text from PDF documents --- Cargo.lock | 235 +++++++++++++++++++++++- core/Cargo.toml | 5 +- core/src/tools/builtin/web_fetch.rs | 51 ++++- core/src/tools/builtin/web_fetch/pdf.rs | 169 +++++++++++++++++ 4 files changed, 443 insertions(+), 17 deletions(-) create mode 100644 core/src/tools/builtin/web_fetch/pdf.rs diff --git a/Cargo.lock b/Cargo.lock index b53abff7..55d59594 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,6 +234,17 @@ dependencies = [ "pom", ] +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -1030,6 +1041,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1117,6 +1137,15 @@ dependencies = [ "either", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.62" @@ -1138,6 +1167,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "cff-parser" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5810ca1a2b5870df2aab1c03e11c40c361ba51d6e3e361e56310f1cb3b4e087" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1150,6 +1185,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chromiumoxide" version = "0.7.0" @@ -1232,6 +1278,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -1550,6 +1606,37 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1711,6 +1798,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "ego-tree" version = "0.10.0" @@ -2103,10 +2199,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -2713,6 +2812,16 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -2740,6 +2849,48 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" +dependencies = [ + "defmt", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "js-sys" version = "0.3.99" @@ -2882,21 +3033,32 @@ dependencies = [ [[package]] name = "lopdf" -version = "0.34.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" dependencies = [ + "aes", + "bitflags 2.11.1", + "cbc", "chrono", + "ecb", "encoding_rs", "flate2", + "getrandom 0.4.2", "indexmap 2.14.0", "itoa", + "jiff", "log", "md-5 0.10.6", - "nom 7.1.3", + "nom 8.0.0", + "rand 0.10.2", "rangemap", "rayon", + "sha2 0.10.9", + "stringprep", + "thiserror 2.0.18", "time", + "ttf-parser", "weezl", ] @@ -3403,13 +3565,15 @@ dependencies = [ [[package]] name = "pdf-extract" -version = "0.7.12" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73" dependencies = [ "adobe-cmap-parser", + "cff-parser", "encoding_rs", "euclid", + "log", "lopdf", "postscript", "type1-encoding-parser", @@ -3600,6 +3764,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postscript" version = "0.14.1" @@ -3785,6 +3964,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3823,6 +4013,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rangemap" version = "1.7.1" @@ -4641,6 +4837,17 @@ dependencies = [ "quote", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5184,6 +5391,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "tungstenite" version = "0.23.0" @@ -5217,6 +5430,12 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -5238,6 +5457,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" diff --git a/core/Cargo.toml b/core/Cargo.toml index 459a6910..4a459fdc 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -86,8 +86,7 @@ shell-words = "1.1" # HTML to text/markdown conversion html2text = "0.16" htmd = "0.5" -pdf-extract = "0.7" -lopdf = "0.34" +pdf-extract = "0.12" roxmltree = "0.20" zip = { version = "0.6", default-features = false, features = ["deflate"] } cfb = "0.14" @@ -147,6 +146,8 @@ s3 = [ serve = ["dep:cron"] [dev-dependencies] +# Build deterministic PDF fixtures for web_fetch extraction tests. +lopdf = "0.42" # Deterministically advance the exponential stream-retry backoff in async tests. tokio = { version = "1.35", features = ["test-util"] } # HTTP mocking for the RemoteGitBackend (and any future HTTP-backed workspace diff --git a/core/src/tools/builtin/web_fetch.rs b/core/src/tools/builtin/web_fetch.rs index 5a076ce6..40c6332e 100644 --- a/core/src/tools/builtin/web_fetch.rs +++ b/core/src/tools/builtin/web_fetch.rs @@ -8,6 +8,8 @@ use reqwest::{header::LOCATION, redirect::Policy, Url}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::time::Duration; +mod pdf; + /// Maximum response size (5MB) const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024; /// Maximum number of redirects followed by a single fetch. @@ -26,7 +28,7 @@ impl Tool for WebFetchTool { } fn description(&self) -> &str { - "Fetch content from a URL and convert to text or markdown. Supports HTML to Markdown conversion. 5MB download size limit and capped tool output. Configurable timeout (max 120 seconds)." + "Fetch content from a URL and convert to text or markdown. Supports HTML to Markdown conversion and text extraction from PDF documents. 5MB download size limit and capped tool output. Configurable timeout (max 120 seconds)." } fn parameters(&self) -> serde_json::Value { @@ -186,6 +188,8 @@ impl Tool for WebFetchTool { Ok( ToolOutput::success(range.content).with_metadata(serde_json::json!({ "source_anchors": source_anchors, + "document_kind": page.document_kind, + "content_type": page.content_type, "range": { "offset": offset, "requested_max_chars": requested_max_chars, @@ -266,6 +270,8 @@ fn parse_macos_proxy(text: &str) -> Option { struct FetchedPage { content: String, final_url: Url, + document_kind: &'static str, + content_type: String, } /// Fetch a URL while validating and pinning DNS results for every redirect hop. @@ -348,9 +354,22 @@ async fn fetch_url( bytes.extend_from_slice(&chunk); } - let body = String::from_utf8_lossy(&bytes).to_string(); - if content_type.contains("text/html") { - if let Some(location) = html_refresh_location(&body) { + let is_pdf = pdf::response_is_pdf(&content_type, &bytes); + let is_html = !is_pdf + && content_type + .split(';') + .next() + .is_some_and(|value| value.trim().eq_ignore_ascii_case("text/html")); + let document_kind = if is_pdf { + "pdf" + } else if is_html { + "html" + } else { + "text" + }; + let html_body = is_html.then(|| String::from_utf8_lossy(&bytes).into_owned()); + if let Some(body) = html_body.as_deref() { + if let Some(location) = html_refresh_location(body) { if redirect_count == MAX_REDIRECTS { return Err(format!( "Too many redirects while fetching URL (max: {})", @@ -361,21 +380,33 @@ async fn fetch_url( continue; } } - let body = if body_only && content_type.contains("text/html") { - extract_html_body(&body).unwrap_or(body) + let body = if is_pdf { + pdf::extract_text(bytes).await? } else { - body + let body = html_body.unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned()); + if body_only && is_html { + extract_html_body(&body).unwrap_or(body) + } else { + body + } }; let content = match format { "html" => body, - "text" if content_type.contains("text/html") => html_to_text(&body), - "markdown" if content_type.contains("text/html") => html_to_markdown(&body), - _ if content_type.contains("text/html") => html_to_markdown(&body), + "text" if is_html => html_to_text(&body), + "markdown" if is_html => html_to_markdown(&body), + _ if is_html => html_to_markdown(&body), _ => body, }; return Ok(FetchedPage { content, final_url: url, + document_kind, + content_type: content_type + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(), }); } diff --git a/core/src/tools/builtin/web_fetch/pdf.rs b/core/src/tools/builtin/web_fetch/pdf.rs new file mode 100644 index 00000000..d64eaa1e --- /dev/null +++ b/core/src/tools/builtin/web_fetch/pdf.rs @@ -0,0 +1,169 @@ +const PDF_CONTENT_TYPE: &str = "application/pdf"; +const PDF_MAGIC: &[u8] = b"%PDF-"; + +pub(super) fn response_is_pdf(content_type: &str, bytes: &[u8]) -> bool { + content_type + .split(';') + .next() + .is_some_and(|value| value.trim().eq_ignore_ascii_case(PDF_CONTENT_TYPE)) + || bytes.starts_with(PDF_MAGIC) +} + +/// Extract PDF text away from Tokio's async worker threads. +pub(super) async fn extract_text(bytes: Vec) -> Result { + let text = tokio::task::spawn_blocking(move || pdf_extract::extract_text_from_mem(&bytes)) + .await + .map_err(|error| format!("PDF text extraction worker failed: {error}"))? + .map_err(|error| format!("Could not parse or extract text from PDF: {error}"))?; + if text.trim().is_empty() { + return Err( + "PDF contains no extractable text; it may be image-only or scanned".to_string(), + ); + } + Ok(text) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::types::{Tool, ToolContext}; + use lopdf::{ + content::{Content, Operation}, + dictionary, Document, Object, Stream, + }; + + fn pdf_document(text: Option<&str>) -> Vec { + let mut document = Document::with_version("1.5"); + let pages_id = document.new_object_id(); + let font_id = document.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Courier", + }); + let resources_id = document.add_object(dictionary! { + "Font" => dictionary! { + "F1" => font_id, + }, + }); + let mut operations = Vec::new(); + if let Some(text) = text { + operations.extend([ + Operation::new("BT", vec![]), + Operation::new("Tf", vec!["F1".into(), 12.into()]), + Operation::new("Td", vec![72.into(), 720.into()]), + Operation::new("Tj", vec![Object::string_literal(text)]), + Operation::new("ET", vec![]), + ]); + } + let content = Content { operations }; + let content_id = document.add_object(Stream::new( + dictionary! {}, + content.encode().expect("test PDF content must encode"), + )); + let page_id = document.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + }); + document.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![page_id.into()], + "Count" => 1, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()], + }), + ); + let catalog_id = document.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + document.trailer.set("Root", catalog_id); + + let mut bytes = Vec::new(); + document + .save_to(&mut bytes) + .expect("test PDF must serialize"); + bytes + } + + #[test] + fn detects_pdf_from_content_type_or_magic() { + assert!(response_is_pdf( + "Application/PDF; charset=binary", + b"not a PDF payload" + )); + assert!(response_is_pdf("application/octet-stream", b"%PDF-1.7\n")); + assert!(!response_is_pdf("text/html", b"")); + assert!(!response_is_pdf("application/json", b"{\"pdf\":true}")); + } + + #[tokio::test(flavor = "current_thread")] + async fn extracts_text_from_pdf_without_external_io() { + let text = extract_text(pdf_document(Some("STORM research evidence"))) + .await + .expect("generated PDF text must extract"); + + assert!(text.contains("STORM research evidence"), "{text:?}"); + } + + #[tokio::test(flavor = "current_thread")] + async fn rejects_pdf_without_extractable_text() { + let error = extract_text(pdf_document(None)).await.unwrap_err(); + + assert!(error.contains("no extractable text"), "{error}"); + } + + #[tokio::test(flavor = "current_thread")] + async fn reports_malformed_pdf_as_parse_failure() { + let error = extract_text(b"%PDF-1.7\nmalformed".to_vec()) + .await + .unwrap_err(); + + assert!( + error.contains("Could not parse or extract text from PDF"), + "{error}" + ); + } + + #[tokio::test] + #[ignore = "requires external network"] + async fn extracts_real_storm_arxiv_pdf() { + let result = super::super::WebFetchTool + .execute( + &serde_json::json!({ + "url": "https://arxiv.org/pdf/2402.14207", + "format": "text", + "timeout": 30 + }), + &ToolContext::new(std::env::temp_dir()), + ) + .await + .expect("web_fetch must execute"); + assert!(result.success, "{}", result.content); + assert_eq!( + result + .metadata + .as_ref() + .map(|value| &value["document_kind"]), + Some(&serde_json::json!("pdf")) + ); + assert_eq!( + result.metadata.as_ref().map(|value| &value["content_type"]), + Some(&serde_json::json!("application/pdf")) + ); + assert!( + result.content.contains("STORM"), + "extracted text omitted the title" + ); + assert!( + result.content.contains("Wikipedia-like Articles"), + "extracted text omitted the paper subject" + ); + assert!( + !result.content.contains("Taylor Hawkins"), + "extracted PDF text included unrelated page content" + ); + } +}