diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19d666d..c8b31c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,9 +84,9 @@ jobs: fi # Always-on formats (text, CSV, notebooks) must keep every limit with no format feature. cargo test -p markkit --no-default-features - cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown - if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then - echo "HTTP client leaked into the docx,spreadsheet,markdown feature set" + cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown,email + if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown,email -e normal | grep -E 'reqwest|hyper'; then + echo "HTTP client leaked into the docx,spreadsheet,markdown,email feature set" exit 1 fi - name: Run examples diff --git a/CHANGELOG.md b/CHANGELOG.md index 90229c8..ed6ea4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `markkit mcp`: Model Context Protocol server over stdio exposing the `markkit` tool (`tools/list`, `tools/call`). URL fetching keeps the SSRF guard; `--no-url` keeps it offline, the `extract` limit flags and `--max-output-chars` bound each call. +- Email input (`email` feature, in `all_formats`; `.eml`, `message/rfc822`): subject as title, From / + To / Cc / Date, plain-text body (HTML alternative converted when there is no plain part) and an + attachment list. New `Format::Eml`, `Error::EmailExtract`, `ExtractionStats::attachments`, `WarningCode::EncodingProblem`. - Jupyter notebook input (`.ipynb`, `application/x-ipynb+json`, nbformat 4), always on: Markdown cells, code cells as fenced blocks tagged with the kernel language, text outputs and ANSI-free tracebacks; images/HTML outputs skipped. New `Format::Notebook`, `Error::NotebookExtract` and diff --git a/Cargo.lock b/Cargo.lock index b72c1e8..1021dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -827,6 +827,18 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashify" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "hayro-ccitt" version = "0.4.0" @@ -1244,6 +1256,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "mail-parser" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" +dependencies = [ + "hashify", +] + [[package]] name = "markkit" version = "0.1.1" @@ -1257,6 +1278,7 @@ dependencies = [ "futures-core", "hayro-syntax", "html5ever", + "mail-parser", "miniz_oxide", "proptest", "pulldown-cmark", diff --git a/Cargo.toml b/Cargo.toml index df8fff0..d8cfd0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ cfb = "0.15" # XLS (OLE2) container pre-scan hayro-syntax = { version = "0.8", default-features = false, features = ["std"] } # PDF object/content parsing (no image decoders) miniz_oxide = "0.9" # Bounded FlateDecode for PDF streams pulldown-cmark = { version = "0.13", default-features = false } # Markdown input; output-safety test oracle +mail-parser = { version = "0.11", default-features = false } # EML input (RFC 5322 / MIME) # Logging tracing = "0.1" diff --git a/README.md b/README.md index 180df6c..5d8a009 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ package for browsers and edge runtimes. - **DOCX** – Headings, lists, tables, links, emphasis - **PPTX** – One section per slide: titles, bullets, tables, links, speaker notes - **Spreadsheets** – XLSX/XLSM, XLSB, XLS and ODS workbooks to markdown tables, one section per sheet +- **Email (EML)** – Subject, sender, recipients, date, body (plain preferred, HTML converted), attachment list - **Jupyter notebooks** – Markdown cells, code cells with language, text outputs and tracebacks - **CSV / TSV** – One markdown table; `;` and `|` delimiters detected, RFC 4180 quoting - **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) @@ -40,6 +41,7 @@ package for browsers and edge runtimes. | Plain text | Supported | always on | | CSV, TSV | Supported | always on | | Jupyter notebooks (ipynb) | Supported | always on | +| Email (EML) | Supported | `email` | | Markdown | Supported | `markdown` | | PDF (text layer) | Supported | `pdf` | | DOC, PPT | Detected, rejected explicitly | – | @@ -202,7 +204,8 @@ let markkit = Markkit::builder() | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | PDF text-layer extraction (hayro-syntax, pure Rust) | | `markdown` | No | Markdown normalization (pulldown-cmark) | -| `all_formats` | No | `html`, `pdf`, `docx`, `pptx`, `spreadsheet`, `markdown` | +| `email` | No | EML conversion (mail-parser) | +| `all_formats` | No | `html`, `pdf`, `docx`, `pptx`, `spreadsheet`, `markdown`, `email` | | `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 825bc43..0b655a3 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -26,6 +26,7 @@ docx = ["markkit/docx"] pptx = ["markkit/pptx"] spreadsheet = ["markkit/spreadsheet"] markdown = ["markkit/markdown"] +email = ["markkit/email"] [dependencies] markkit = { path = "../markkit", version = "0.1.1", features = ["http_client", "tool"] } diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index dfa3692..5930f89 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -102,7 +102,7 @@ pub struct Warning { pub struct Extraction { pub markdown: String, pub title: Option, - /// Detected format (`html`, `docx`, `pptx`, `xlsx`, `xlsb`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`). + /// Detected format (`html`, `docx`, `pptx`, `xlsx`, `xlsb`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`, `eml`). pub format: String, pub warnings: Vec, pub stats: ExtractionStats, diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index df1e3cf..be276f4 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -21,6 +21,7 @@ docx = ["dep:zip", "dep:quick-xml"] pptx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] markdown = ["dep:pulldown-cmark"] +email = ["dep:mail-parser"] # Opt-in: the default build has no network stack (no reqwest/hyper/TLS). http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] @@ -28,7 +29,7 @@ logging = ["dep:tracing"] tool = ["dep:tower", "dep:futures-core", "dep:base64"] # JSON Schema for the IR (`markkit::ir::json_schema()`) json-schema = ["dep:schemars"] -all_formats = ["html", "pdf", "docx", "pptx", "spreadsheet", "markdown"] +all_formats = ["html", "pdf", "docx", "pptx", "spreadsheet", "markdown", "email"] [dependencies] # Async @@ -63,6 +64,9 @@ miniz_oxide = { workspace = true, optional = true } # Markdown (optional) pulldown-cmark = { workspace = true, optional = true } +# Email (optional) +mail-parser = { workspace = true, optional = true } + # HTTP client (optional) reqwest = { workspace = true, optional = true } diff --git a/crates/markkit/src/eml.rs b/crates/markkit/src/eml.rs new file mode 100644 index 0000000..0232276 --- /dev/null +++ b/crates/markkit/src/eml.rs @@ -0,0 +1,234 @@ +//! Email (`.eml`, RFC 5322 / MIME) input via mail-parser. +//! +//! Decisions: +//! - Behind the `email` feature (mail-parser 0.11, no default features: UTF-*, +//! Latin and other single-byte charsets, QP/base64, RFC 2047 words). +//! Selected by `message/rfc822` or `.eml`, never sniffed. +//! - Output: `# Subject`, a header paragraph (From, To, Cc, Date as RFC 3339), +//! then the body, then an `## Attachments` list (name, type, size). The body +//! follows RFC 8621 `textBody`: text/plain when present, otherwise the HTML +//! alternative. Plain parts go through the plain-text path; HTML parts +//! through the HTML parser (`html` feature) or mail-parser's HTML-to-text. +//! - Attachments, including nested `message/rfc822`, are listed, never parsed, +//! so extraction cost does not compound with nesting. +//! - Header values are text: control characters dropped and newlines folded. +//! - Shares the call's `Budget`: the deadline is checked after mail-parser +//! (not interruptible) and per attachment; body parts use the same budget. + +use std::borrow::Cow; + +use mail_parser::{Address, MessageParser, MimeHeaders, PartType}; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::extract::{Warning, WarningCode}; +use crate::ir::{Document, ListItem, Node}; +use crate::limits::Budget; + +/// Parsed email. +pub(crate) struct EmailOutput { + pub doc: Document, + pub title: Option, + pub warnings: Vec, + pub attachments: usize, +} + +/// Parse raw message bytes into the IR. +pub(crate) fn parse( + bytes: &[u8], + config: &MarkkitBuilder, + budget: &Budget, +) -> Result { + let msg = MessageParser::default() + .parse(bytes) + .filter(|m| !m.parts.is_empty()) + .ok_or_else(|| Error::EmailExtract("not an RFC 5322 message".into()))?; + budget.check_time()?; + budget.add_nodes(msg.parts.len())?; + let mut nodes = Vec::new(); + let mut warnings = Vec::new(); + + let title = msg.subject().map(header_text).filter(|s| !s.is_empty()); + if let Some(subject) = &title { + nodes.push(Node::heading(1, vec![Node::Text(subject.clone())])); + } + + let mut header = Vec::new(); + for (label, value) in [ + ("From", msg.from().map(addresses)), + ("To", msg.to().map(addresses)), + ("Cc", msg.cc().map(addresses)), + ("Date", msg.date().map(|d| d.to_rfc3339())), + ] { + let Some(value) = value.filter(|v| !v.is_empty()) else { + continue; + }; + if !header.is_empty() { + header.push(Node::LineBreak); + } + header.push(Node::Bold { + children: vec![Node::Text(format!("{label}:"))], + }); + header.push(Node::Text(format!(" {value}"))); + } + if !header.is_empty() { + nodes.push(Node::paragraph(header)); + } + + for (i, &id) in msg.text_body.iter().enumerate() { + let Some(part) = msg.part(id) else { continue }; + if part.is_encoding_problem { + warnings.push(Warning::new( + WarningCode::EncodingProblem, + "a body part could not be fully decoded", + )); + } + match &part.body { + PartType::Text(text) => { + let decoded = crate::text::decode(text.as_bytes()); + nodes.extend(crate::text::plain_to_ir(&decoded.text, budget)?.0.nodes); + } + PartType::Html(html) => nodes.extend(html_nodes(html, &msg, i, config, budget)?), + _ => {} + } + } + + let mut items = Vec::new(); + for part in msg.attachments() { + if budget.expired() { + return Err(budget.timeout_error()); + } + let name = match &part.body { + PartType::Message(m) => m.subject().map(|s| format!("{} (message)", header_text(s))), + _ => None, + } + .or_else(|| part.attachment_name().map(header_text)) + .filter(|n| !n.is_empty()) + .unwrap_or_else(|| "unnamed".to_string()); + let ctype = part + .content_type() + .map(|ct| match ct.subtype() { + Some(sub) => format!("{}/{}", ct.ctype(), sub), + None => ct.ctype().to_string(), + }) + .map(|t| header_text(&t)) + .unwrap_or_else(|| "application/octet-stream".into()); + items.push(ListItem::new(vec![Node::Text(format!( + "{name} ({ctype}, {})", + human_size(part.len()) + ))])); + } + let attachments = items.len(); + if !items.is_empty() { + nodes.push(Node::heading(2, vec![Node::Text("Attachments".into())])); + nodes.push(Node::List { + ordered: false, + start: None, + items, + source: None, + }); + } + + Ok(EmailOutput { + doc: Document::new(nodes), + title, + warnings, + attachments, + }) +} + +#[cfg(feature = "html")] +fn html_nodes( + html: &str, + _msg: &mail_parser::Message<'_>, + _pos: usize, + config: &MarkkitBuilder, + budget: &Budget, +) -> Result, Error> { + Ok(crate::html::HtmlParser::with_budget(config, budget) + .parse(html)? + .nodes) +} + +#[cfg(not(feature = "html"))] +fn html_nodes( + _html: &str, + msg: &mail_parser::Message<'_>, + pos: usize, + _config: &MarkkitBuilder, + budget: &Budget, +) -> Result, Error> { + let text = msg.body_text(pos).unwrap_or(Cow::Borrowed("")); + Ok( + crate::text::plain_to_ir(&crate::text::decode(text.as_bytes()).text, budget)? + .0 + .nodes, + ) +} + +/// Header text: controls dropped, line breaks folded, trimmed. +fn header_text(s: &str) -> String { + let t = crate::text::decode(s.as_bytes()).text; + t.split_whitespace().collect::>().join(" ") +} + +fn addresses(addr: &Address<'_>) -> String { + let one = |name: Option<&Cow<'_, str>>, address: Option<&Cow<'_, str>>| { + let name = name.map(|n| header_text(n)).filter(|n| !n.is_empty()); + let address = address.map(|a| header_text(a)).filter(|a| !a.is_empty()); + match (name, address) { + (Some(n), Some(a)) if n != a => format!("{n} <{a}>"), + (_, Some(a)) => a, + (Some(n), None) => n, + (None, None) => String::new(), + } + }; + let list: Vec = match addr { + Address::List(list) => list + .iter() + .map(|a| one(a.name.as_ref(), a.address.as_ref())) + .collect(), + Address::Group(groups) => groups + .iter() + .flat_map(|g| g.addresses.iter()) + .map(|a| one(a.name.as_ref(), a.address.as_ref())) + .collect(), + }; + list.into_iter() + .filter(|s| !s.is_empty()) + .collect::>() + .join(", ") +} + +fn human_size(bytes: usize) -> String { + const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"]; + let mut size = bytes as f64; + let mut unit = 0; + while size >= 1024.0 && unit < UNITS.len() - 1 { + size /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + format!("{size:.1} {}", UNITS[unit]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sizes() { + assert_eq!(human_size(0), "0 B"); + assert_eq!(human_size(1023), "1023 B"); + assert_eq!(human_size(1536), "1.5 KB"); + assert_eq!(human_size(5 * 1024 * 1024), "5.0 MB"); + } + + #[test] + fn header_folding() { + assert_eq!(header_text(" a\r\n\tb\u{7} c "), "a b c"); + } +} diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index 417fe73..e4efff8 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -20,6 +20,9 @@ pub enum Error { #[error("Notebook extraction error: {0}")] NotebookExtract(String), + #[error("Email extraction error: {0}")] + EmailExtract(String), + #[error("HTTP error: {0}")] Http(String), diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 20423d5..a3cfd4f 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -57,6 +57,8 @@ pub enum WarningCode { MalformedCsv, /// A CSV/TSV table exceeded the row/column caps and was cut. TruncatedTable, + /// An email body part could not be fully decoded (bad charset or transfer encoding). + EncodingProblem, /// Output was cut to a caller's size limit (agent tool `max_chars`). Truncated, /// Any other issue, e.g. from a custom [`crate::registry::FormatParser`]. @@ -79,6 +81,7 @@ impl WarningCode { WarningCode::InvalidUtf8 => "invalid_utf8", WarningCode::MalformedCsv => "malformed_csv", WarningCode::TruncatedTable => "truncated_table", + WarningCode::EncodingProblem => "encoding_problem", WarningCode::Truncated => "truncated", WarningCode::Other => "other", } @@ -160,6 +163,8 @@ pub struct ExtractionStats { pub tables: Option, /// Non-empty cells rendered (Jupyter notebooks). pub cells: Option, + /// Attachments listed (email). + pub attachments: Option, } /// Result of [`Markkit::parse_document`]: the IR plus metadata, not rendered. @@ -317,7 +322,7 @@ impl Markkit { } /// Convert a document (HTML, PDF, DOCX, PPTX, XLSX/XLSB/XLS/ODS, plain text, Markdown, - /// CSV/TSV, Jupyter notebooks) to Markdown with metadata. + /// CSV/TSV, Jupyter notebooks, email) to Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -396,6 +401,7 @@ impl Markkit { Format::Markdown => self.parse_markdown(bytes, budget), Format::Csv | Format::Tsv => self.parse_csv(bytes, format, budget), Format::Notebook => self.parse_notebook(bytes, budget), + Format::Eml => self.parse_email(bytes, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -472,6 +478,27 @@ impl Markkit { Ok(p) } + #[cfg(feature = "email")] + fn parse_email(&self, bytes: &[u8], budget: &Budget) -> crate::Result { + let out = guarded("email", Error::EmailExtract, || { + crate::eml::parse(bytes, &self.config, budget) + })?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Eml); + p.title = out.title; + p.warnings = out.warnings; + p.stats.attachments = Some(out.attachments); + Ok(p) + } + + #[cfg(not(feature = "email"))] + fn parse_email(&self, _bytes: &[u8], _budget: &Budget) -> crate::Result { + Err(Error::UnsupportedFormat( + "email support requires the 'email' feature".to_string(), + )) + } + fn parse_notebook(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let decoded = crate::text::decode(bytes); budget.check_time()?; diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index 7251296..666739a 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -36,6 +36,8 @@ pub enum Format { Csv, /// Tab separated values (`.tsv`, `.tab`). Tsv, + /// Email message (`.eml`, `message/rfc822`). + Eml, /// Jupyter notebook (`.ipynb`, nbformat 4). Serialized as `"ipynb"`. #[serde(rename = "ipynb")] Notebook, @@ -98,6 +100,7 @@ impl Format { Format::Csv => "csv", Format::Tsv => "tsv", Format::Notebook => "ipynb", + Format::Eml => "eml", Format::PlainText => "text", Format::Markdown => "markdown", Format::Unknown => "unknown", @@ -127,6 +130,8 @@ impl Format { } else if ct.contains("presentationml") || ct.contains("ms-powerpoint.") { // `ms-powerpoint.presentation.macroEnabled.12` & co. are OOXML. Format::Pptx + } else if ct.contains("message/rfc822") { + Format::Eml } else if ct.contains("x-ipynb") { Format::Notebook } else if ct.contains("ms-powerpoint") { @@ -160,6 +165,7 @@ impl Format { "csv" => Format::Csv, "tsv" | "tab" => Format::Tsv, "ipynb" => Format::Notebook, + "eml" => Format::Eml, "txt" | "text" => Format::PlainText, "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, @@ -286,6 +292,7 @@ mod tests { Format::Csv, Format::Tsv, Format::Notebook, + Format::Eml, Format::PlainText, Format::Markdown, Format::Unknown, diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 3f507a2..8d9ecca 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -36,6 +36,7 @@ //! - **PPTX** (`pptx`): Convert PowerPoint presentations, one section per slide //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLSB, XLS and ODS workbooks to tables //! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Email** (`email`): `.eml` / `message/rfc822`, headers, body and attachment list //! - **Jupyter notebooks** (always on): `.ipynb`, cells and text outputs //! - **CSV / TSV** (always on): `text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`, one table //! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR @@ -56,6 +57,8 @@ pub mod config; mod csv; #[cfg(feature = "docx")] mod docx; +#[cfg(feature = "email")] +mod eml; pub mod error; pub mod extract; #[cfg(feature = "http_client")] diff --git a/crates/markkit/tests/email_integration.rs b/crates/markkit/tests/email_integration.rs new file mode 100644 index 0000000..33425ce --- /dev/null +++ b/crates/markkit/tests/email_integration.rs @@ -0,0 +1,223 @@ +//! Email (.eml) → Markdown integration and security tests. +#![cfg(feature = "email")] + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; + +fn extract(raw: &str) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync(raw.as_bytes(), DocumentHint::file_name("mail.eml")) + .unwrap() +} + +const MULTIPART: &str = "From: \"Ada Lovelace\" \r\n\ +To: bob@example.com, =?UTF-8?Q?Ren=C3=A9?= \r\n\ +Cc: Team: carol@example.com;\r\n\ +Date: Sat, 20 Nov 2021 14:22:01 -0800\r\n\ +Subject: =?utf-8?q?Quarterly_report_=E2=98=95?=\r\n\ +MIME-Version: 1.0\r\n\ +Content-Type: multipart/mixed; boundary=\"outer\"\r\n\ +\r\n\ +--outer\r\n\ +Content-Type: multipart/alternative; boundary=\"alt\"\r\n\ +\r\n\ +--alt\r\n\ +Content-Type: text/plain; charset=utf-8\r\n\ +Content-Transfer-Encoding: quoted-printable\r\n\ +\r\n\ +Hi Bob,\r\n\ +\r\n\ +Numbers are up =E2=80=94 see attached.\r\n\ +--alt\r\n\ +Content-Type: text/html; charset=utf-8\r\n\ +\r\n\ +

HTML version

\r\n\ +--alt--\r\n\ +--outer\r\n\ +Content-Type: application/pdf; name=\"report.pdf\"\r\n\ +Content-Disposition: attachment; filename=\"report.pdf\"\r\n\ +Content-Transfer-Encoding: base64\r\n\ +\r\n\ +JVBERi0xLjQK\r\n\ +--outer--\r\n"; + +#[test] +fn multipart_prefers_plain_text_and_lists_attachments() { + let e = extract(MULTIPART); + assert_eq!(e.format, Format::Eml); + assert_eq!(e.title.as_deref(), Some("Quarterly report ☕")); + assert_eq!(e.stats.attachments, Some(1)); + assert_eq!( + e.markdown, + "# Quarterly report ☕\n\n\ +**From:** Ada Lovelace \\ \n\ +**To:** bob@example.com, René \\ \n\ +**Cc:** carol@example.com \n\ +**Date:** 2021-11-20T14:22:01-08:00\n\n\ +Hi Bob,\n\n\ +Numbers are up — see attached.\n\n\ +## Attachments\n\n\ +- report.pdf (application/pdf, 9 B)\n" + ); + assert!(!e.markdown.contains("HTML")); +} + +#[cfg(feature = "html")] +#[test] +fn html_only_body_goes_through_html_parser() { + let raw = "From: a@example.com\r\nSubject: News\r\nContent-Type: text/html\r\n\r\n\ +

Hello

See this.

\ +\r\n"; + let e = extract(raw); + assert!(e.markdown.contains("## Hello"), "{}", e.markdown); + assert!( + e.markdown.contains("[this](https://e.com)"), + "{}", + e.markdown + ); + for bad in [" {} + Err(e) => panic!("{bytes:?}: {e:?}"), + } + } +} + +#[test] +fn security_header_injection_folded() { + // Encoded words can smuggle newlines and control characters into headers. + let raw = "Subject: =?utf-8?q?Hi=0A#_Fake_heading=1B[2J?=\r\nFrom: =?utf-8?q?Eve=0D=0A**To:**_x?= \r\n\r\nbody\r\n"; + let e = extract(raw); + let first = e.markdown.lines().next().unwrap(); + assert_eq!(first, "# Hi # Fake heading\\[2J"); + assert!( + !e.markdown.lines().any(|l| l.starts_with("**To:**")), + "{}", + e.markdown + ); + assert!( + !e.markdown + .chars() + .any(|c| c.is_ascii_control() && c != '\n'), + "{:?}", + e.markdown + ); +} + +#[test] +fn security_deep_multipart_nesting() { + let depth = 5_000; + let mut raw = String::from("Subject: deep\r\n"); + for i in 0..depth { + raw.push_str(&format!( + "Content-Type: multipart/mixed; boundary=b{i}\r\n\r\n--b{i}\r\n" + )); + } + raw.push_str("Content-Type: text/plain\r\n\r\nleaf\r\n"); + let e = Markkit::builder() + .build() + .extract_document_sync(raw.as_bytes(), DocumentHint::file_name("a.eml")); + assert!(matches!(e, Ok(_) | Err(Error::EmailExtract(_)))); +} + +#[test] +fn security_many_attachments_bounded_by_input() { + let mut raw = + String::from("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n"); + for i in 0..20_000 { + raw.push_str(&format!( + "--b\r\nContent-Type: application/octet-stream; name=f{i}\r\nContent-Disposition: attachment\r\n\r\nx\r\n" + )); + } + raw.push_str("--b--\r\n"); + let e = extract(&raw); + // Output stays proportional to input (one short line per attachment). + assert!(e.markdown.len() < raw.len()); +} + +#[test] +fn security_input_size_limit() { + let m = Markkit::builder().max_input_size(8).build(); + let err = m + .extract_document_sync(MULTIPART.as_bytes(), DocumentHint::file_name("a.eml")) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_))); +} + +#[test] +fn security_limits_apply() { + let r = Markkit::builder() + .timeout(std::time::Duration::ZERO) + .build() + .extract_document_sync(MULTIPART.as_bytes(), DocumentHint::file_name("a.eml")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::Timeout)), + "{r:?}" + ); + let body = "

a

".repeat(500); + let raw = format!("Subject: x\r\nContent-Type: text/html\r\n\r\n{body}"); + let r = Markkit::builder() + .max_nodes(100) + .build() + .extract_document_sync(raw.as_bytes(), DocumentHint::file_name("a.eml")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::MaxNodes)), + "{r:?}" + ); + let raw = format!("Subject: x\r\n\r\n{}", "line\n\n".repeat(5000)); + let r = Markkit::builder() + .max_output_size(1000) + .build() + .extract_document_sync(raw.as_bytes(), DocumentHint::file_name("a.eml")); + assert!( + matches!(&r, Err(e) if e.limit_kind() == Some(markkit::LimitKind::MaxOutputSize)), + "{r:?}" + ); +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 824ef9b..b732dfe 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -1,7 +1,7 @@ --- type: Subsystem Design title: Format Support -description: Supported and planned formats, bytes-in API, format detection, DOCX, spreadsheet, PDF, plain text, Markdown, CSV/TSV and Jupyter notebook mapping. +description: Supported and planned formats, bytes-in API, format detection, DOCX, spreadsheet, PDF, plain text, Markdown, CSV/TSV, Jupyter notebook and email mapping. tags: - markkit - formats @@ -12,6 +12,7 @@ tags: - markdown - csv - notebook + - email --- # Format Support @@ -21,6 +22,7 @@ tags: ### Tier 1 (Core) - **HTML/XHTML**: Full support via html5ever + scraper (`html` feature, default) - **Plain text**: Always on, no dependencies (`text/plain`, `.txt`, `.text`) +- **Email** (`email`): `.eml` / `message/rfc822` via mail-parser 0.11 (no default features) - **Jupyter notebooks**: Always on, serde_json (`application/x-ipynb+json`, `.ipynb`; nbformat 4) - **CSV / TSV**: Always on, no dependencies (`text/csv`, `application/csv`, `.csv`; `text/tab-separated-values`, `.tsv`, `.tab`) @@ -85,7 +87,7 @@ only the Markdown. Parsing is CPU-bound; async callers should use the sync varia Warning codes (`WarningCode`, serialized `snake_case`): `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`, `hidden_slide_skipped`, `slide_error`, `truncated_pdf`, `pdf_unmapped_font`, `pdf_no_text`, `invalid_utf8`, `hidden_content_skipped`, `malformed_csv`, -`truncated_table`, `truncated` (agent tool only) and `other` (custom parsers). Callers match on +`truncated_table`, `encoding_problem`, `truncated` (agent tool only) and `other` (custom parsers). Callers match on `code`; `detail` is for humans and may change. `Markkit` is `Clone + Debug + Send + Sync` and holds only its configuration, so embedders can @@ -109,7 +111,7 @@ Priority order (`Format::detect`): - `%PDF-` within the first 1 KiB → PDF. - ` + +**From:** Name +**To:** a, b (groups flattened) +**Cc:** … +**Date:** RFC 3339 + + + +## Attachments + +- report.pdf (application/pdf, 1.2 MB) +``` + +- Body = RFC 8621 `textBody`: the text/plain alternative when present, else the HTML one. + Plain parts use the plain-text path; HTML parts the HTML parser (`html` feature) or + mail-parser's HTML-to-text conversion. +- Attachments (including nested `message/rfc822`, shown by subject) are listed, never + parsed. `stats.attachments` counts them. +- Header values: RFC 2047 decoded by mail-parser, then control characters dropped and all + whitespace (including encoded newlines) folded to single spaces, so headers cannot inject + lines. `encoding_problem` when a body part could not be fully decoded. +- Legacy multi-byte charsets (Shift-JIS, GB2312, …) need mail-parser's `full_encoding`, + which is not enabled; such parts decode lossily. + ## Resource Limits All formats enforce: diff --git a/knowledge/log.md b/knowledge/log.md index 24764a4..af7b3d2 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -19,6 +19,7 @@ * **HTML memory**: peak heap was about 25x input, mostly scraper's node arena grown by doubling plus four-slot inline node lists for one-node cells. The arena is now pre-sized from a clamped byte scan, inline lists are sized from the child count, and text nodes are no longer copied before whitespace collapsing. See [HTML Extraction](formats/html-extraction.md). * **Jupyter notebook input**: always on (serde_json); Markdown cells via the Markdown path, code cells fenced with the kernel language, text outputs kept, rich outputs skipped; shares the call's `Limits` budget. T7/S18 extended. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). * **Limits in minimal builds**: with no format feature, `extract_document` skipped `max_output_size` and the final deadline check, so always-on text/CSV/notebook input ignored them (`csv_integration::security_timeout_applies` failed under `--no-default-features`). Bounded rendering is now unconditional, text decoding is followed by a deadline check, and CI tests `--no-default-features`. S14/S16 hold in every feature set. +* **Email input**: `email` feature on mail-parser 0.11 (1.0 was two days old and a rewrite, so not adopted yet). Headers folded against line injection; attachments listed, never parsed; shares the call's `Limits` budget. T7/S18 extended. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). ## 2026-10-26 diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 1ffc0ee..39e5902 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -21,7 +21,7 @@ tags: | `crates/markkit/tests/output_safety.rs` | Renderer escaping, URL scheme allowlist, fences; pulldown-cmark oracle + proptests | | `crates/markkit/tests/html_proptest.rs` | Property tests: no panic, no script/style, determinism, limits | | `crates/markkit/tests/docx_integration.rs`, `spreadsheet_integration.rs` | Office mapping | -| `crates/markkit/tests/text_integration.rs`, `csv_integration.rs`, `notebook_integration.rs` | Plain text, Markdown, CSV/TSV and notebook mapping, T7 security tests | +| `crates/markkit/tests/text_integration.rs`, `csv_integration.rs`, `notebook_integration.rs`, `email_integration.rs` | Plain text, Markdown, CSV/TSV, notebook and email mapping, T7 security tests | | `crates/markkit/tests/document_detection.rs` | Format detection order and explicit unsupported errors | | `crates/markkit/tests/document_security.rs` | Zip bombs, grid amplification, XML entities, malformed containers | | `crates/markkit/tests/limits.rs` | Every `Limits` field enforced for HTML, DOCX, spreadsheets; small-stack depth ceiling | @@ -65,7 +65,7 @@ each security test states exactly which bytes it feeds the parser. `ci.yml` runs fmt, clippy, rustdoc (`-D warnings`), the knowledge bundle check, `cargo audit` + `cargo deny`, `cargo test --workspace` (default and all features), -`--no-default-features` (always-on formats keep every limit), the `docx,spreadsheet,markdown` no-HTTP build with a reqwest leak check, and every example. +`--no-default-features` (always-on formats keep every limit), the `docx,spreadsheet,markdown,email` no-HTTP build with a reqwest leak check, and every example. `nightly.yml` repeats build + test on nightly Rust daily; `fuzz.yml` runs every cargo-fuzz target nightly and uploads crashes; `coverage.yml` uploads tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). All must pass before merge. diff --git a/knowledge/security/security-model.md b/knowledge/security/security-model.md index 07a9019..f9e66f4 100644 --- a/knowledge/security/security-model.md +++ b/knowledge/security/security-model.md @@ -38,7 +38,7 @@ Markkit processes untrusted input from the internet. Key threats: - **Compatibility**: ZIP entry-name decoding errors propagate; Unicode and legacy CP437 part names preserve bounded case-insensitive lookup - See [Threat Model](threat-model.md) T6 -### TM-TXT: Text, Markdown, CSV and notebook input +### TM-TXT: Text, Markdown, CSV, notebook and email input - **Mitigation**: Selected only by content type / extension, total decoding, control characters dropped - **Mitigation**: CSV bounded by the sheet row/column caps while reading - **Mitigation**: Markdown re-parsed into the IR; raw HTML never copied (HTML blocks go through the HTML parser) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 7b16a26..d5b61fe 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -336,9 +336,9 @@ process, or smuggling content past a human reviewer. - `spreadsheet_integration::xlsx_hidden_sheet_skipped_by_default`, `spreadsheet_integration::xlsm_with_vba_project_extracts_values_only` -### T7: Text, Markdown, CSV and Notebook Input +### T7: Text, Markdown, CSV, Notebook and Email Input -**Description**: Plain text, Markdown, CSV/TSV or Jupyter notebooks carrying terminal escapes, NULs, invalid encodings, +**Description**: Plain text, Markdown, CSV/TSV, Jupyter notebooks or email carrying terminal escapes, NULs, invalid encodings, raw HTML (`