diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ce0ae2..dd1a3b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,9 +84,9 @@ jobs: fi # Always-on formats (text, CSV, notebooks) must keep every limit with no format feature. cargo test -p markkit --no-default-features - cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt - if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt -e normal | grep -E 'reqwest|hyper'; then - echo "HTTP client leaked into the docx,spreadsheet,markdown,email,odt feature set" + cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub + if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub -e normal | grep -E 'reqwest|hyper'; then + echo "HTTP client leaked into the docx,spreadsheet,markdown,email,odt,epub feature set" exit 1 fi if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'tokio'; then diff --git a/CHANGELOG.md b/CHANGELOG.md index f8c30df..562590d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `markkit mcp`: Model Context Protocol server over stdio exposing the `markkit` tool (`tools/list`, `tools/call`). URL fetching keeps the SSRF guard; `--no-url` keeps it offline, the `extract` limit flags and `--max-output-chars` bound each call. +- EPUB input (`epub` feature, implies `html`, in `all_formats`; `.epub`, `application/epub+zip`, sniffed + from the `mimetype` entry): spine chapters through the HTML parser, OPF title, DRM-encrypted books + rejected. New `Format::Epub`, `Error::EpubExtract`, `ExtractionStats::chapters`, `WarningCode::MissingChapter`. - ODT input (`odt` feature, in `all_formats`; `.odt`, OpenDocument text content type, sniffed from the ODF `mimetype`): headings, Title style, bold/italic/strike, links, lists, tables, with the same bounded ZIP/XML reader as DOCX. New `Format::Odt`, `Error::OdtExtract`. diff --git a/README.md b/README.md index 7ed41f4..139ab0c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ package for browsers and edge runtimes. - **DOCX** – Headings, lists, tables, links, emphasis - **PPTX** – One section per slide: titles, bullets, tables, links, speaker notes - **ODT** – OpenDocument text: headings, lists, tables, links, emphasis +- **EPUB** – E-book chapters in reading order, title from the package metadata - **Spreadsheets** – XLSX/XLSM, XLSB, XLS and ODS workbooks to markdown tables, one section per sheet - **Email (EML)** – Subject, sender, recipients, date, body (plain preferred, HTML converted), attachment list - **Jupyter notebooks** – Markdown cells, code cells with language, text outputs and tracebacks @@ -39,6 +40,7 @@ package for browsers and edge runtimes. | DOCX | Supported | `docx` | | PPTX, PPTM, PPSX | Supported | `pptx` | | ODT | Supported | `odt` | +| EPUB | Supported | `epub` | | XLSX, XLSM, XLSB, XLS, ODS | Supported | `spreadsheet` | | Plain text | Supported | always on | | CSV, TSV | Supported | always on | @@ -208,7 +210,8 @@ let markkit = Markkit::builder() | `markdown` | No | Markdown normalization (pulldown-cmark) | | `email` | No | EML conversion (mail-parser) | | `odt` | No | ODT conversion (zip + quick-xml) | -| `all_formats` | No | `html`, `pdf`, `docx`, `pptx`, `spreadsheet`, `markdown`, `email`, `odt` | +| `epub` | No | EPUB conversion (implies `html`) | +| `all_formats` | No | `html`, `pdf`, `docx`, `pptx`, `spreadsheet`, `markdown`, `email`, `odt`, `epub` | | `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index d9fc613..9d07f64 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -28,6 +28,7 @@ spreadsheet = ["markkit/spreadsheet"] markdown = ["markkit/markdown"] email = ["markkit/email"] odt = ["markkit/odt"] +epub = ["markkit/epub"] [dependencies] markkit = { path = "../markkit", version = "0.1.1", features = ["http_client", "tool"] } diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 8ea3e62..5116dce 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -102,7 +102,7 @@ pub struct Warning { pub struct Extraction { pub markdown: String, pub title: Option, - /// Detected format (`html`, `docx`, `pptx`, `xlsx`, `xlsb`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`, `eml`, `odt`). + /// Detected format (`html`, `docx`, `pptx`, `xlsx`, `xlsb`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`, `eml`, `odt`, `epub`). pub format: String, pub warnings: Vec, pub stats: ExtractionStats, diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 9108e0b..2cd2f2f 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -23,6 +23,7 @@ spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] markdown = ["dep:pulldown-cmark"] email = ["dep:mail-parser"] odt = ["dep:zip", "dep:quick-xml"] +epub = ["html", "dep:zip", "dep:quick-xml"] # Opt-in: the default build has no network stack (no reqwest/hyper/TLS). http_client = ["dep:reqwest", "dep:tokio", "tokio/net"] logging = ["dep:tracing"] @@ -30,7 +31,7 @@ logging = ["dep:tracing"] tool = ["dep:tower", "dep:futures-core", "dep:base64", "dep:tokio", "tokio/sync"] # JSON Schema for the IR (`markkit::ir::json_schema()`) json-schema = ["dep:schemars"] -all_formats = ["html", "pdf", "docx", "pptx", "spreadsheet", "markdown", "email", "odt"] +all_formats = ["html", "pdf", "docx", "pptx", "spreadsheet", "markdown", "email", "odt", "epub"] [dependencies] # Async (optional): only URL fetching and the tool's output channel need tokio. diff --git a/crates/markkit/src/archive.rs b/crates/markkit/src/archive.rs index 58aac8f..cd78874 100644 --- a/crates/markkit/src/archive.rs +++ b/crates/markkit/src/archive.rs @@ -1,4 +1,4 @@ -//! Bounded ZIP access shared by the DOCX and spreadsheet parsers. +//! Bounded ZIP access shared by the DOCX, ODT, EPUB and spreadsheet parsers. //! //! Decision: declared sizes in ZIP headers are attacker-controlled, so limits are //! enforced on bytes actually produced by the decompressor (`Read::take`), never on @@ -66,7 +66,7 @@ impl<'a> ZipGuard<'a> { } /// Names of all entries. - #[cfg(feature = "spreadsheet")] + #[cfg(any(feature = "spreadsheet", feature = "epub"))] pub fn names(&self) -> impl Iterator { self.names.iter().map(String::as_str) } diff --git a/crates/markkit/src/epub.rs b/crates/markkit/src/epub.rs new file mode 100644 index 0000000..913b643 --- /dev/null +++ b/crates/markkit/src/epub.rs @@ -0,0 +1,278 @@ +//! EPUB (2 and 3) → IR. +//! +//! Decisions: +//! - Behind the `epub` feature (implies `html`): an EPUB is a ZIP of XHTML +//! chapters, so it reuses the bounded ZIP reader, the depth-capped XML tree +//! for the package files, and the HTML parser for chapters (same stripping +//! rules as web pages). +//! - Package: `META-INF/container.xml` → first rootfile (fallback: the only +//! `.opf` entry) → manifest + spine. Chapters render in spine order; +//! `linear="no"` items (notes, covers) are skipped, each item at most once. +//! Only XHTML/HTML manifest items are read. +//! - Title: OPF `dc:title`. Chapter boundaries become thematic breaks. +//! - DRM: if `META-INF/encryption.xml` encrypts a spine document, extraction +//! fails with `Error::EpubExtract` rather than emitting ciphertext. Font +//! obfuscation alone (also declared there) is fine. +//! - Relative hrefs are resolved against the OPF directory, percent-decoded, +//! and may not climb above the archive root. +//! - Limits: package XML and every chapter share the call's `Budget` (nodes, +//! timeout) and nesting depth; chapters go through `HtmlParser::with_budget`, +//! so hidden-content rules (S20) apply as for web pages. + +use std::collections::{HashMap, HashSet}; + +use crate::archive::{ArchiveError, ZipGuard}; +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::extract::{Warning, WarningCode}; +use crate::ir::{Document, Node}; +use crate::limits::{Budget, LimitExceeded}; +use crate::xml::{self, Element, XmlError}; + +/// Parsed EPUB. +pub(crate) struct EpubOutput { + pub doc: Document, + pub title: Option, + pub warnings: Vec, + pub chapters: usize, +} + +fn archive_err(e: ArchiveError) -> Error { + match e { + ArchiveError::Limit(m) => Error::ResourceLimit(m), + ArchiveError::Invalid(m) => Error::EpubExtract(m), + } +} + +fn xml_err(part: &str, e: XmlError) -> Error { + match e { + XmlError::Limit(m) => { + Error::ResourceLimit(LimitExceeded::new(m.kind, format!("{part}: {}", m.detail))) + } + XmlError::Invalid(m) => Error::EpubExtract(format!("{part}: {m}")), + } +} + +fn read_xml( + zip: &mut ZipGuard<'_>, + path: &str, + max_depth: usize, + budget: &Budget, +) -> Result, Error> { + match zip.read(path).map_err(archive_err)? { + Some(bytes) => xml::parse(&bytes, max_depth, budget) + .map(Some) + .map_err(|e| xml_err(path, e)), + None => Ok(None), + } +} + +/// Resolve `href` against directory `base` (no leading slash). `None` when +/// the path escapes the archive root. +fn resolve(base: &str, href: &str) -> Option { + let href = percent_decode(href.split(['#', '?']).next().unwrap_or("")); + let mut parts: Vec<&str> = if href.starts_with('/') { + Vec::new() + } else { + base.split('/').filter(|p| !p.is_empty()).collect() + }; + for seg in href.split('/') { + match seg { + "" | "." => {} + ".." => { + parts.pop()?; + } + s => parts.push(s), + } + } + let path = parts.join("/"); + (!path.is_empty()).then_some(path) +} + +fn percent_decode(s: &str) -> String { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' + && i + 2 < b.len() + && let (Some(h), Some(l)) = (hex(b[i + 1]), hex(b[i + 2])) + { + out.push(h * 16 + l); + i += 3; + continue; + } + out.push(b[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +fn hex(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +fn dir_of(path: &str) -> &str { + path.rsplit_once('/').map_or("", |(d, _)| d) +} + +pub(crate) fn parse( + bytes: &[u8], + config: &MarkkitBuilder, + budget: &Budget, +) -> Result { + let mut zip = ZipGuard::open(bytes, config).map_err(archive_err)?; + let depth = config.limits.nesting_depth(); + + let opf_path = read_xml(&mut zip, "META-INF/container.xml", depth, budget)? + .and_then(|c| { + c.find("rootfile") + .and_then(|r| r.attr("full-path")) + .and_then(|p| resolve("", p)) + }) + .or_else(|| { + let mut opfs = zip + .names() + .filter(|n| n.to_ascii_lowercase().ends_with(".opf")); + match (opfs.next(), opfs.next()) { + (Some(only), None) => Some(only.to_string()), + _ => None, + } + }) + .ok_or_else(|| Error::EpubExtract("no package document (container.xml rootfile)".into()))?; + let opf = read_xml(&mut zip, &opf_path, depth, budget)? + .ok_or_else(|| Error::EpubExtract(format!("package document '{opf_path}' missing")))?; + let package = opf + .find("package") + .ok_or_else(|| Error::EpubExtract("package document has no ".into()))?; + let base = dir_of(&opf_path); + + let title = package + .find("metadata") + .and_then(|m| m.child("title")) + .map(|t| t.text().split_whitespace().collect::>().join(" ")) + .filter(|t| !t.is_empty()); + + // id → (path, is_html) + let mut manifest: HashMap<&str, (String, bool)> = HashMap::new(); + if let Some(m) = package.child("manifest") { + for item in m.elements().filter(|e| e.name == "item") { + let (Some(id), Some(href)) = (item.attr("id"), item.attr("href")) else { + continue; + }; + let Some(path) = resolve(base, href) else { + continue; + }; + let media = item.attr("media-type").unwrap_or(""); + let lower = path.to_ascii_lowercase(); + let html = media.contains("html") + || lower.ends_with(".xhtml") + || lower.ends_with(".html") + || lower.ends_with(".htm"); + manifest.insert(id, (path, html)); + } + } + + let encrypted: HashSet = read_xml(&mut zip, "META-INF/encryption.xml", depth, budget)? + .map(|enc| { + let mut out = HashSet::new(); + let mut stack = vec![&enc]; + while let Some(e) = stack.pop() { + if e.name == "CipherReference" + && let Some(uri) = e.attr("URI").and_then(|u| resolve("", u)) + { + out.insert(uri); + } + stack.extend(e.elements()); + } + out + }) + .unwrap_or_default(); + + let spine: Vec<&str> = package + .child("spine") + .map(|s| { + s.elements() + .filter(|e| e.name == "itemref" && e.attr("linear") != Some("no")) + .filter_map(|e| e.attr("idref")) + .collect() + }) + .unwrap_or_default(); + + let html = crate::html::HtmlParser::with_budget(config, budget); + let mut warnings = Vec::new(); + let mut nodes: Vec = Vec::new(); + let mut seen = HashSet::new(); + let mut chapters = 0; + for idref in spine { + if budget.expired() { + return Err(budget.timeout_error()); + } + let Some((path, is_html)) = manifest.get(idref) else { + warnings.push(Warning::new( + WarningCode::MissingChapter, + format!("spine item '{idref}' not in manifest"), + )); + continue; + }; + if !is_html || !seen.insert(path.clone()) { + continue; + } + if encrypted.contains(path) { + return Err(Error::EpubExtract( + "EPUB content is encrypted (DRM); cannot extract".into(), + )); + } + let Some(bytes) = zip.read(path).map_err(archive_err)? else { + warnings.push(Warning::new( + WarningCode::MissingChapter, + format!("'{path}' not in archive"), + )); + continue; + }; + let text = String::from_utf8_lossy(&bytes); + let doc = html.parse(&text)?; + if doc.nodes.is_empty() { + continue; + } + if chapters > 0 { + nodes.push(Node::ThematicBreak); + } + chapters += 1; + nodes.extend(doc.nodes); + } + budget.check_time()?; + + Ok(EpubOutput { + doc: Document::new(nodes), + title, + warnings, + chapters, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_paths() { + assert_eq!( + resolve("OEBPS", "text/ch1.xhtml").as_deref(), + Some("OEBPS/text/ch1.xhtml") + ); + assert_eq!( + resolve("OEBPS/text", "../ch%201.xhtml#x").as_deref(), + Some("OEBPS/ch 1.xhtml") + ); + assert_eq!(resolve("", "/abs.xhtml").as_deref(), Some("abs.xhtml")); + assert_eq!(resolve("a", "../../etc/passwd"), None); + assert_eq!(resolve("", ""), None); + assert_eq!(percent_decode("%zz%4"), "%zz%4"); + } +} diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index 58303b8..ee1ed5c 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -22,6 +22,9 @@ pub enum Error { #[error("Notebook extraction error: {0}")] NotebookExtract(String), + #[error("EPUB extraction error: {0}")] + EpubExtract(String), + #[error("Email extraction error: {0}")] EmailExtract(String), diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 0a00622..61dea98 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -59,6 +59,8 @@ pub enum WarningCode { TruncatedTable, /// An email body part could not be fully decoded (bad charset or transfer encoding). EncodingProblem, + /// An EPUB spine item is missing from the manifest or the archive. + MissingChapter, /// Output was cut to a caller's size limit (agent tool `max_chars`). Truncated, /// Any other issue, e.g. from a custom [`crate::registry::FormatParser`]. @@ -82,6 +84,7 @@ impl WarningCode { WarningCode::MalformedCsv => "malformed_csv", WarningCode::TruncatedTable => "truncated_table", WarningCode::EncodingProblem => "encoding_problem", + WarningCode::MissingChapter => "missing_chapter", WarningCode::Truncated => "truncated", WarningCode::Other => "other", } @@ -165,6 +168,8 @@ pub struct ExtractionStats { pub cells: Option, /// Attachments listed (email). pub attachments: Option, + /// Chapters rendered (EPUB). + pub chapters: Option, } /// Result of [`Markkit::parse_document`]: the IR plus metadata, not rendered. @@ -322,7 +327,7 @@ impl Markkit { } /// Convert a document (HTML, PDF, DOCX, PPTX, ODT, XLSX/XLSB/XLS/ODS, plain text, Markdown, - /// CSV/TSV, Jupyter notebooks, email) to Markdown with metadata. + /// CSV/TSV, Jupyter notebooks, email, EPUB) to Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -403,6 +408,7 @@ impl Markkit { Format::Notebook => self.parse_notebook(bytes, budget), Format::Eml => self.parse_email(bytes, budget), Format::Odt => self.parse_odt(bytes, budget), + Format::Epub => self.parse_epub(bytes, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -479,6 +485,27 @@ impl Markkit { Ok(p) } + #[cfg(feature = "epub")] + fn parse_epub(&self, bytes: &[u8], budget: &Budget) -> crate::Result { + let out = guarded("EPUB", Error::EpubExtract, || { + crate::epub::parse(bytes, &self.config, budget) + })?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Epub); + p.title = out.title; + p.warnings = out.warnings; + p.stats.chapters = Some(out.chapters); + Ok(p) + } + + #[cfg(not(feature = "epub"))] + fn parse_epub(&self, _bytes: &[u8], _budget: &Budget) -> crate::Result { + Err(Error::UnsupportedFormat( + "EPUB support requires the 'epub' feature".to_string(), + )) + } + #[cfg(feature = "odt")] fn parse_odt(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let out = guarded("ODT", Error::OdtExtract, || { diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index 59642a9..eb46330 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -20,6 +20,8 @@ pub enum Format { Docx, /// Legacy Word 97-2003 binary (`.doc`). Detected but not supported. Doc, + /// EPUB e-book (`.epub`). + Epub, /// OpenDocument text (`.odt`, `.ott`). Odt, /// Office Open XML spreadsheet (`.xlsx`, `.xlsm`, `.xltx`). @@ -85,6 +87,7 @@ impl<'a> DocumentHint<'a> { const OLE2_MAGIC: &[u8] = b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"; const ODS_MIMETYPE: &[u8] = b"mimetypeapplication/vnd.oasis.opendocument.spreadsheet"; const ODT_MIMETYPE: &[u8] = b"mimetypeapplication/vnd.oasis.opendocument.text"; +const EPUB_MIMETYPE: &[u8] = b"mimetypeapplication/epub+zip"; impl Format { /// Short lowercase name (`"docx"`, `"xlsx"`, …). @@ -95,6 +98,7 @@ impl Format { Format::Docx => "docx", Format::Doc => "doc", Format::Odt => "odt", + Format::Epub => "epub", Format::Xlsx => "xlsx", Format::Xlsb => "xlsb", Format::Xls => "xls", @@ -129,6 +133,8 @@ impl Format { Format::Xlsx } else if ct.contains("ms-excel") { Format::Xls + } else if ct.contains("epub") { + Format::Epub } else if ct.contains("opendocument.text") { Format::Odt } else if ct.contains("opendocument.spreadsheet") { @@ -167,6 +173,7 @@ impl Format { "xls" | "xlt" => Format::Xls, "ods" => Format::Ods, "odt" | "ott" => Format::Odt, + "epub" => Format::Epub, "pptx" | "pptm" | "ppsx" | "ppsm" | "potx" | "potm" => Format::Pptx, "ppt" | "pps" | "pot" => Format::Ppt, "csv" => Format::Csv, @@ -243,6 +250,8 @@ fn sniff_zip(bytes: &[u8]) -> Format { Format::Ods } else if contains(head, ODT_MIMETYPE) { Format::Odt + } else if contains(head, EPUB_MIMETYPE) { + Format::Epub } else if contains(bytes, b"word/document") { Format::Docx } else if contains(bytes, b"xl/workbook.bin") { @@ -304,6 +313,7 @@ mod tests { Format::Notebook, Format::Eml, Format::Odt, + Format::Epub, Format::PlainText, Format::Markdown, Format::Unknown, diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index ec0e379..0733327 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -36,6 +36,7 @@ //! - **PPTX** (`pptx`): Convert PowerPoint presentations, one section per slide //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLSB, XLS and ODS workbooks to tables //! - **Plain text** (always on): `text/plain`, `.txt` +//! - **EPUB** (`epub`, implies `html`): e-books, chapters in reading order //! - **ODT** (`odt`): OpenDocument text documents //! - **Email** (`email`): `.eml` / `message/rfc822`, headers, body and attachment list //! - **Jupyter notebooks** (always on): `.ipynb`, cells and text outputs @@ -56,7 +57,8 @@ feature = "docx", feature = "pptx", feature = "spreadsheet", - feature = "odt" + feature = "odt", + feature = "epub" ))] mod archive; pub mod config; @@ -65,6 +67,8 @@ mod csv; mod docx; #[cfg(feature = "email")] mod eml; +#[cfg(feature = "epub")] +mod epub; pub mod error; pub mod extract; #[cfg(feature = "http_client")] @@ -98,7 +102,8 @@ pub mod transform; feature = "docx", feature = "pptx", feature = "spreadsheet", - feature = "odt" + feature = "odt", + feature = "epub" ))] #[cfg_attr(not(feature = "docx"), allow(dead_code))] // DOCX uses every helper mod xml; diff --git a/crates/markkit/tests/common/mod.rs b/crates/markkit/tests/common/mod.rs index d23061a..f9aae20 100644 --- a/crates/markkit/tests/common/mod.rs +++ b/crates/markkit/tests/common/mod.rs @@ -361,6 +361,50 @@ pub fn odt(body: &str) -> Vec { odt_with("", body, None, None) } +// ─── EPUB ─────────────────────────────────────────────────────────── + +pub const EPUB_MIME: &str = "application/epub+zip"; + +pub const EPUB_CONTAINER: &str = r#""#; + +/// OPF with `title`, manifest `(id, href, media-type)` and spine `(idref, linear)`. +pub fn epub_opf(title: &str, manifest: &[(&str, &str, &str)], spine: &[(&str, bool)]) -> String { + let items: String = manifest + .iter() + .map(|(id, href, mt)| format!(r#""#)) + .collect(); + let refs: String = spine + .iter() + .map(|(id, linear)| { + if *linear { + format!(r#""#) + } else { + format!(r#""#) + } + }) + .collect(); + format!( + r#"{}{items}{refs}"#, + xml_escape(title) + ) +} + +pub fn xhtml(body: &str) -> String { + format!( + r#"t{body}"# + ) +} + +/// EPUB from extra `(path, bytes)` entries plus mimetype and container. +pub fn epub(entries: &[(&str, &[u8])]) -> Vec { + let mut all: Vec<(&str, &[u8])> = vec![ + ("mimetype", EPUB_MIME.as_bytes()), + ("META-INF/container.xml", EPUB_CONTAINER.as_bytes()), + ]; + all.extend_from_slice(entries); + zip_bytes(&all) +} + // ─── XLS (BIFF8 in an OLE2 compound file) ─────────────────────────── fn record(out: &mut Vec, typ: u16, data: &[u8]) { diff --git a/crates/markkit/tests/epub_integration.rs b/crates/markkit/tests/epub_integration.rs new file mode 100644 index 0000000..e6f4095 --- /dev/null +++ b/crates/markkit/tests/epub_integration.rs @@ -0,0 +1,263 @@ +//! EPUB → Markdown integration and security tests (fixtures generated in-memory). + +#![cfg(feature = "epub")] + +mod common; + +use common::*; +use markkit::{DocumentHint, Error, Markkit, format::Format}; + +const XHTML: &str = "application/xhtml+xml"; + +fn try_extract(bytes: &[u8]) -> markkit::Result { + Markkit::builder() + .build() + .extract_document_sync(bytes, DocumentHint::file_name("book.epub")) +} + +fn book() -> Vec { + let opf = epub_opf( + "A Small\nBook", + &[ + ("cover", "cover.xhtml", XHTML), + ("c1", "text/ch%201.xhtml", XHTML), + ("c2", "text/ch2.xhtml", XHTML), + ("css", "style.css", "text/css"), + ("img", "img.png", "image/png"), + ], + &[ + ("cover", false), + ("c1", true), + ("c2", true), + ("c1", true), + ("css", true), + ], + ); + let ch1 = + xhtml("

Chapter 1

It was a dark night.

"); + let ch2 = xhtml("

Chapter 2

  • one
  • two
"); + let cover = xhtml("

COVER

"); + epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/text/ch 1.xhtml", ch1.as_bytes()), + ("OEBPS/text/ch2.xhtml", ch2.as_bytes()), + ("OEBPS/cover.xhtml", cover.as_bytes()), + ("OEBPS/style.css", b"p{}"), + ]) +} + +#[test] +fn chapters_in_spine_order() { + let e = try_extract(&book()).unwrap(); + assert_eq!(e.format, Format::Epub); + assert_eq!(e.title.as_deref(), Some("A Small Book")); + assert_eq!(e.stats.chapters, Some(2)); + assert_eq!( + e.markdown, + "# Chapter 1\n\nIt was a *dark* night.\n\n---\n\n# Chapter 2\n\n- one\n- two\n" + ); + assert!(e.warnings.is_empty(), "{:?}", e.warnings); +} + +#[test] +fn detection() { + assert_eq!(Format::from_extension("EPUB"), Format::Epub); + assert_eq!( + Format::from_content_type("application/epub+zip"), + Format::Epub + ); + assert_eq!(Format::sniff(&book()), Format::Epub); + let e = Markkit::builder() + .build() + .extract_document_sync(&book(), DocumentHint::default()) + .unwrap(); + assert_eq!(e.format, Format::Epub); +} + +#[test] +fn missing_container_falls_back_to_single_opf() { + let opf = epub_opf("T", &[("c", "c.xhtml", XHTML)], &[("c", true)]); + let ch = xhtml("

hi

"); + let bytes = zip_bytes(&[ + ("mimetype", EPUB_MIME.as_bytes()), + ("book/pkg.opf", opf.as_bytes()), + ("book/c.xhtml", ch.as_bytes()), + ]); + assert_eq!(try_extract(&bytes).unwrap().markdown, "hi\n"); +} + +#[test] +fn missing_chapter_warns() { + let opf = epub_opf( + "T", + &[("a", "a.xhtml", XHTML), ("gone", "gone.xhtml", XHTML)], + &[("a", true), ("gone", true), ("nope", true)], + ); + let a = xhtml("

a

"); + let e = try_extract(&epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/a.xhtml", a.as_bytes()), + ])) + .unwrap(); + assert_eq!(e.markdown, "a\n"); + assert_eq!(e.warnings.len(), 2, "{:?}", e.warnings); + assert!( + e.warnings + .iter() + .all(|w| w.code == markkit::WarningCode::MissingChapter) + ); +} + +// ─── Security (threat models T1, T6) ──────────────────────────────── + +#[test] +fn security_scripts_stripped() { + let e = try_extract(&book()).unwrap(); + assert!(!e.markdown.contains("evil") && !e.markdown.contains("ok

"); + let e = try_extract(&epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/ok.xhtml", ok.as_bytes()), + ])) + .unwrap(); + assert_eq!(e.markdown, "ok\n"); +} + +#[test] +fn security_drm_encrypted_is_error() { + let opf = epub_opf("T", &[("c", "c.xhtml", XHTML)], &[("c", true)]); + let enc = r#""#; + let bytes = epub(&[ + ("META-INF/encryption.xml", enc.as_bytes()), + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/c.xhtml", b"\x8f\x01ciphertext"), + ]); + assert!(matches!(try_extract(&bytes), Err(Error::EpubExtract(_)))); +} + +#[test] +fn security_repeated_spine_reads_once() { + // 5 000 itemrefs to one 100 KB chapter must not render it 5 000 times. + let opf = epub_opf("T", &[("c", "c.xhtml", XHTML)], &vec![("c", true); 5_000]); + let ch = xhtml(&format!("

{}

", "x".repeat(100_000))); + let e = try_extract(&epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/c.xhtml", ch.as_bytes()), + ])) + .unwrap(); + assert_eq!(e.stats.chapters, Some(1)); + assert!(e.markdown.len() < 110_000); +} + +#[test] +fn security_zip_bomb_chapter_bounded() { + let opf = epub_opf("T", &[("c", "c.xhtml", XHTML)], &[("c", true)]); + let big = vec![b' '; 2 * 1024 * 1024]; + let bytes = epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/c.xhtml", &big), + ]); + let err = Markkit::builder() + .max_archive_entry_size(1024 * 1024) + .build() + .extract_document_sync(&bytes, DocumentHint::file_name("a.epub")) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}"); +} + +#[test] +fn security_malformed_packages() { + for bytes in [ + b"not a zip".to_vec(), + epub(&[]), + epub(&[("OEBPS/content.opf", b"")]), + epub(&[("OEBPS/content.opf", b"")]), + ] { + match try_extract(&bytes) { + Err(Error::EpubExtract(_)) => {} + Ok(e) => assert_eq!(e.markdown, ""), + Err(e) => panic!("{e:?}"), + } + } +} + +#[test] +fn security_truncated_archive_never_panics() { + let full = book(); + for cut in (0..full.len()).step_by(5) { + let _ = try_extract(&full[..cut]); + } +} + +fn one_chapter(body: &str) -> Vec { + let opf = epub_opf("T", &[("c", "c.xhtml", XHTML)], &[("c", true)]); + let ch = xhtml(body); + epub(&[ + ("OEBPS/content.opf", opf.as_bytes()), + ("OEBPS/c.xhtml", ch.as_bytes()), + ]) +} + +#[test] +fn security_limits_apply() { + let bytes = one_chapter(&"

a

".repeat(500)); + let r = Markkit::builder() + .max_nodes(100) + .build() + .extract_document_sync(&bytes, DocumentHint::file_name("book.epub")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::MaxNodes)), + "{r:?}" + ); + let r = Markkit::builder() + .timeout(std::time::Duration::ZERO) + .build() + .extract_document_sync(&bytes, DocumentHint::file_name("book.epub")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::Timeout)), + "{r:?}" + ); + let r = Markkit::builder() + .max_output_size(100) + .build() + .extract_document_sync(&bytes, DocumentHint::file_name("book.epub")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::MaxOutputSize)), + "{r:?}" + ); +} + +#[test] +fn security_hidden_content_skipped_by_default() { + let bytes = one_chapter(r#"

shown

INJECT

"#); + let e = try_extract(&bytes).unwrap(); + assert_eq!(e.markdown, "shown\n"); + let e = Markkit::builder() + .include_hidden_content(true) + .build() + .extract_document_sync(&bytes, DocumentHint::file_name("book.epub")) + .unwrap(); + assert!(e.markdown.contains("INJECT"), "{}", e.markdown); +} + +#[test] +fn security_markup_in_text_is_escaped() { + let e = try_extract(&one_chapter( + "

<img src=x onerror=y> x

", + )) + .unwrap(); + assert!(e.markdown.contains("\\` section per non-empty worksheet, in workbook order, followed by a diff --git a/knowledge/log.md b/knowledge/log.md index 6e59613..f0d3d06 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -22,6 +22,7 @@ * **Limits in minimal builds**: with no format feature, `extract_document` skipped `max_output_size` and the final deadline check, so always-on text/CSV/notebook input ignored them (`csv_integration::security_timeout_applies` failed under `--no-default-features`). Bounded rendering is now unconditional, text decoding is followed by a deadline check, and CI tests `--no-default-features`. S14/S16 hold in every feature set. * **Email input**: `email` feature on mail-parser 0.11 (1.0 was two days old and a rewrite, so not adopted yet). Headers folded against line injection; attachments listed, never parsed; shares the call's `Limits` budget. T7/S18 extended. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). * **ODT input**: `odt` feature reusing the DOCX archive/XML guards; style inheritance for bold/italic/strike and outline levels, lists, tables with capped repeats; notes, annotations and tracked changes skipped; shares the call's `Limits` budget; hidden text (`text:display="none"` styles/sections, hidden fields) skipped per S20. T6 and T9 cover ODT. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). +* **EPUB input**: `epub` feature (implies `html`); spine chapters through the HTML parser, each once, traversal-safe href resolution, DRM-encrypted spine documents rejected; shares the call's `Limits` budget and hidden-content rules. T6 covers EPUB. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). ## 2026-10-26 diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index d5fe63a..68dd4a4 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -20,7 +20,7 @@ tags: | `crates/markkit/tests/hidden_content.rs` | Hidden HTML elements and DOCX `w:vanish` runs skipped by default (T9) | | `crates/markkit/tests/output_safety.rs` | Renderer escaping, URL scheme allowlist, fences; pulldown-cmark oracle + proptests | | `crates/markkit/tests/html_proptest.rs` | Property tests: no panic, no script/style, determinism, limits | -| `crates/markkit/tests/docx_integration.rs`, `odt_integration.rs`, `spreadsheet_integration.rs` | Office mapping (ODT security tests included) | +| `crates/markkit/tests/docx_integration.rs`, `odt_integration.rs`, `epub_integration.rs`, `spreadsheet_integration.rs` | Office and e-book mapping (ODT/EPUB security tests included) | | `crates/markkit/tests/text_integration.rs`, `csv_integration.rs`, `notebook_integration.rs`, `email_integration.rs` | Plain text, Markdown, CSV/TSV, notebook and email mapping, T7 security tests | | `crates/markkit/tests/document_detection.rs` | Format detection order and explicit unsupported errors | | `crates/markkit/tests/document_security.rs` | Zip bombs, grid amplification, XML entities, malformed containers | @@ -65,7 +65,7 @@ each security test states exactly which bytes it feeds the parser. `ci.yml` runs fmt, clippy, rustdoc (`-D warnings`), the knowledge bundle check, `cargo audit` + `cargo deny`, `cargo test --workspace` (default and all features), -`--no-default-features` (always-on formats keep every limit), the `docx,spreadsheet,markdown,email,odt` no-HTTP build with a reqwest leak check, and every example. +`--no-default-features` (always-on formats keep every limit), the `docx,spreadsheet,markdown,email,odt,epub` no-HTTP build with a reqwest leak check, and every example. `nightly.yml` repeats build + test on nightly Rust daily; `fuzz.yml` runs every cargo-fuzz target nightly and uploads crashes; `coverage.yml` uploads tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). All must pass before merge. diff --git a/knowledge/security/security-model.md b/knowledge/security/security-model.md index 3f0aba0..3f1601d 100644 --- a/knowledge/security/security-model.md +++ b/knowledge/security/security-model.md @@ -31,7 +31,7 @@ Markkit processes untrusted input from the internet. Key threats: - **Mitigation**: http/https only, optional host allowlist, redirect limit, body bounded by `max_input_size`, fetch bounded by `timeout` - **Residual**: an environment proxy resolves names itself; see [Threat Model](threat-model.md) T3 -### TM-DOC: Malicious office documents (DOCX, ODT, spreadsheets) +### TM-DOC: Malicious office documents and e-books (DOCX, ODT, EPUB, spreadsheets) - **Mitigation**: Bounded ZIP decompression (entry count, per-entry and total size) - **Mitigation**: Spreadsheet grid pre-scan / streaming (no dense-grid amplification) - **Mitigation**: No macro execution, no formula evaluation, hidden sheets skipped by default diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 15d187d..22990e9 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -236,7 +236,7 @@ the proxy, or use `allow_host`. - `html_security::very_long_text_node` (100KB text node) - `html_proptest::no_panic_on_arbitrary_input` -### T6: Malicious Office Documents (DOCX / ODT / PPTX / XLSX / XLSB / XLS / ODS) +### T6: Malicious Office Documents and E-books (DOCX / ODT / EPUB / PPTX / XLSX / XLSB / XLS / ODS) **Description**: Crafted containers aimed at exhausting memory/CPU, crashing the process, or smuggling content past a human reviewer. @@ -253,7 +253,8 @@ process, or smuggling content past a human reviewer. clamped to `max_archive_entry_size` and 16 MiB. - `max_archive_entries` (10 000), `max_archive_entry_size` (64 MiB), `max_archive_total_size` (256 MiB) → `Error::ResourceLimit`. -- DOCX and ODT read only the parts they need, each through the bounded reader. +- DOCX, ODT and EPUB read only the parts they need, each through the bounded reader; + EPUB reads each spine document once and ignores hrefs that escape the archive root. - Spreadsheets: every entry is decompressed into a sink first, because calamine reads entries unbounded and may follow arbitrary relationship targets. @@ -313,7 +314,9 @@ process, or smuggling content past a human reviewer. instructions are dropped. ODT tracked-change regions, notes and annotations are dropped. **Test Coverage** (`tests/document_security.rs`, plus `odt_integration::security_*`: zip bomb, -entities, 10 000-deep nesting, 4-billion repeat counts, style cycles, truncated archives): +entities, 10 000-deep nesting, 4-billion repeat counts, style cycles, truncated archives; +`epub_integration::security_*`: scripts, path traversal, DRM, repeated spine, zip bomb, +malformed packages, truncated archives): - `docx_zip_bomb_entry_rejected`, `docx_total_decompressed_size_capped`, `archive_entry_count_capped`, `xlsx_zip_bomb_sheet_rejected`, `xlsx_bomb_in_unreferenced_entry_rejected`, `max_input_size_honoured_for_office_documents` @@ -473,6 +476,7 @@ at an LLM that consumes the extracted Markdown. - ODT: spans, paragraphs and headings whose style chain sets `text:display="none"`, sections with `text:display="none"`, `text:hidden-text` / `text:hidden-paragraph`; same warning. +- EPUB: chapters use the HTML rules above. - Spreadsheets: hidden sheets already skipped (T6e). **Residual**: hiding through stylesheets or classes (`.sr-only`, off-screen @@ -562,7 +566,7 @@ shared renderer under `max_output_size` (`tests/registry.rs`). | S17 | Rendered markdown has no raw HTML, no non-allowlisted link/image scheme, and document text cannot create Markdown structure | `output_safety::*` (pulldown-cmark oracle + proptests) | | S18 | Text, Markdown, CSV/TSV, notebook and email output never contains C0 controls (except tab/newline); Markdown raw HTML (blocks, inline tags) is never copied | `text_integration::security_*`, `csv_integration::security_*`, `notebook_integration::security_*`, `email_integration::security_*` | | S19 | No PDF stream is decoded past `max_pdf_stream_size`, by markkit or hayro | `pdf_security::*bomb*`, `pdf_security::chained_filters_are_measured_after_every_stage` | -| S20 | Hidden content is not rendered by default (HTML, DOCX, ODT) | `hidden_content::*`, `odt_integration::security_hidden_content_skipped_by_default` | +| S20 | Hidden content is not rendered by default (HTML, DOCX, ODT, EPUB) | `hidden_content::*`, `odt_integration::security_hidden_content_skipped_by_default`, `epub_integration::security_hidden_content_skipped_by_default` | | S21 | HTML parse time is linear in input whatever the nesting; flattened deep subtrees never emit non-content (script/style/template) text | `html_security::deep_*_is_linear`, `script_and_style_bodies_do_not_leak_past_max_depth` | ## See also