From 1cc32367e3adf84651b058c5b97bb9b52050ff46 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:35:58 +0000 Subject: [PATCH 1/3] feat(formats): add plain text and Markdown input Plain text (always on) goes through the IR: paragraphs on blank lines, hard breaks on single newlines. Markdown (new `markdown` feature, in all_formats) is re-parsed with pulldown-cmark into the IR and rendered back, so raw HTML never passes through. Both decode UTF-8/UTF-16 BOMs, drop control characters and warn on lossy decoding. Neither is sniffed from bytes. Threat model T7, invariant S13. --- .github/workflows/ci.yml | 6 +- CHANGELOG.md | 8 + Cargo.lock | 18 + Cargo.toml | 1 + README.md | 6 +- crates/markkit-cli/Cargo.toml | 1 + crates/markkit-cli/tests/cli_integration.rs | 31 ++ crates/markkit/Cargo.toml | 6 +- crates/markkit/src/extract.rs | 45 +- crates/markkit/src/format.rs | 8 +- crates/markkit/src/lib.rs | 5 + crates/markkit/src/markdown.rs | 505 ++++++++++++++++++++ crates/markkit/src/text.rs | 164 +++++++ crates/markkit/tests/text_integration.rs | 228 +++++++++ knowledge/formats/format-support.md | 37 +- knowledge/log.md | 1 + knowledge/operations/testing.md | 3 +- knowledge/security/security-model.md | 5 + knowledge/security/threat-model.md | 25 +- 19 files changed, 1084 insertions(+), 19 deletions(-) create mode 100644 crates/markkit/src/markdown.rs create mode 100644 crates/markkit/src/text.rs create mode 100644 crates/markkit/tests/text_integration.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 647799b..a70dd81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,9 +78,9 @@ jobs: run: cargo test --workspace --all-features - name: Run tests (office formats, no HTTP client) run: | - cargo test -p markkit --no-default-features --features docx,spreadsheet - if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'reqwest|hyper'; then - echo "HTTP client leaked into the docx,spreadsheet feature set" + cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown + if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then + echo "HTTP client leaked into the docx,spreadsheet,markdown feature set" exit 1 fi - name: Run examples diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..fd38d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- Plain text input (`text/plain`, `.txt`), always on: paragraphs and hard breaks through the IR, + UTF-8/UTF-16 BOM decoding, control characters dropped, `invalid_utf8` warning on lossy decode. +- Markdown input (`markdown` feature, in `all_formats`; `text/markdown`, `.md`): re-parsed with + pulldown-cmark and rendered back, so raw HTML never passes through. Front matter `title:` or the + first `# ` heading becomes `Extraction::title`. Available in the CLI, Python and Node bindings. + ### Security - `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: diff --git a/Cargo.lock b/Cargo.lock index 01303b3..edbaa19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,6 +1188,7 @@ dependencies = [ "criterion", "html5ever", "proptest", + "pulldown-cmark", "quick-xml", "reqwest", "scraper", @@ -1591,6 +1592,17 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + [[package]] name = "pyo3" version = "0.30.1" @@ -2525,6 +2537,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index a67f998..69bd085 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS) zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip quick-xml = "0.42" # DOCX/ODS XML parsing cfb = "0.15" # XLS (OLE2) container pre-scan +pulldown-cmark = { version = "0.13", default-features = false } # Markdown input # Logging tracing = "0.1" diff --git a/README.md b/README.md index 0d9e55a..1786881 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ a Rust library, a CLI, and native bindings for Python and Node.js. - **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default - **DOCX** – Headings, lists, tables, links, emphasis - **Spreadsheets** – XLSX/XLSM, XLS and ODS workbooks to markdown tables, one section per sheet +- **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) - **Format detection** – Content type, then file extension, then magic bytes - **URL fetching** – Fetch and extract in one call (`http_client` feature) - **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits @@ -31,6 +32,8 @@ a Rust library, a CLI, and native bindings for Python and Node.js. | HTML / XHTML | Supported | `html` (default) | | DOCX | Supported | `docx` | | XLSX, XLSM, XLS, ODS | Supported | `spreadsheet` | +| Plain text | Supported | always on | +| Markdown | Supported | `markdown` | | PDF | Planned | `pdf` | | DOC, PPTX, XLSB | Detected, rejected explicitly | – | @@ -154,7 +157,8 @@ let markkit = Markkit::builder() | `docx` | No | DOCX conversion (zip + quick-xml) | | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` | -| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` | +| `markdown` | No | Markdown normalization (pulldown-cmark) | +| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet`, `markdown` | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | `http_client` is optional. Embedders that fetch content themselves can build without diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 69495f0..b7bcc30 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -24,6 +24,7 @@ all_formats = ["markkit/all_formats"] pdf = ["markkit/pdf"] docx = ["markkit/docx"] spreadsheet = ["markkit/spreadsheet"] +markdown = ["markkit/markdown"] [dependencies] markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] } diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 2d50944..c3e23cc 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -103,6 +103,37 @@ fn extract_from_file() { assert!(stdout.contains("Paragraph"), "output: {stdout}"); } +#[test] +fn extract_text_and_markdown_files() { + let dir = tempfile::tempdir().unwrap(); + let cases = [ + ( + "notes.txt", + "line one\nline two\n", + "line one \nline two\n", + ), + ( + "doc.md", + "Title\n=====\n\n* a\n\n\n", + "# Title\n\n- a\n", + ), + ]; + for (name, input, expected) in cases { + let file_path = dir.path().join(name); + std::fs::write(&file_path, input).unwrap(); + let output = markkit_bin() + .args(["extract", "--file", file_path.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + assert!( + output.status.success(), + "{name} stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), expected, "{name}"); + } +} + #[test] fn extract_json_output() { let mut child = markkit_bin() diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index f6f5d96..082fad2 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -19,9 +19,10 @@ html = ["dep:scraper", "dep:html5ever"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] +markdown = ["dep:pulldown-cmark"] http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] -all_formats = ["html", "pdf", "docx", "spreadsheet"] +all_formats = ["html", "pdf", "docx", "spreadsheet", "markdown"] [dependencies] # Async @@ -46,6 +47,9 @@ quick-xml = { workspace = true, optional = true } calamine = { workspace = true, optional = true } cfb = { workspace = true, optional = true } +# Markdown (optional) +pulldown-cmark = { workspace = true, optional = true } + # HTTP client (optional) reqwest = { workspace = true, optional = true } diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 06ce32b..f264310 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -10,12 +10,14 @@ use crate::format::{DocumentHint, Format}; pub struct Extraction { /// Rendered Markdown. pub markdown: String, - /// Document title when known (DOCX core properties or `Title` style). + /// Document title when known (DOCX core properties or `Title` style, + /// Markdown front matter `title:` or first `# ` heading). pub title: Option, /// Format the document was parsed as. pub format: Format, /// Non-fatal issues, each prefixed with a stable code - /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`). + /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`, + /// `invalid_utf8:`). pub warnings: Vec, /// Format-specific counters. pub stats: ExtractionStats, @@ -31,9 +33,9 @@ pub struct ExtractionStats { pub sheets: Option, /// Data rows rendered across all sheets (header rows excluded). pub rows: Option, - /// Non-empty paragraphs, headings and list items (DOCX). + /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown). pub paragraphs: Option, - /// Tables (DOCX). + /// Tables (DOCX, Markdown). pub tables: Option, } @@ -142,7 +144,8 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, DOCX, XLSX/XLS/ODS) to Markdown with metadata. + /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to + /// Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -182,6 +185,8 @@ impl Markkit { Format::Pdf => self.extract_pdf_sync(bytes), Format::Docx => self.extract_docx(bytes), Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), + Format::PlainText => Ok(self.extract_plain_text(bytes)), + Format::Markdown => self.extract_markdown(bytes), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -195,6 +200,36 @@ impl Markkit { } } + fn extract_plain_text(&self, bytes: &[u8]) -> Extraction { + let decoded = crate::text::decode(bytes); + let (doc, paragraphs) = crate::text::plain_to_ir(&decoded.text); + let mut renderer = crate::render::MarkdownRenderer::new(); + let mut e = Extraction::new(renderer.render(&doc), Format::PlainText); + e.warnings = decoded.warnings; + e.stats.paragraphs = Some(paragraphs); + e + } + + #[cfg(feature = "markdown")] + fn extract_markdown(&self, bytes: &[u8]) -> crate::Result { + let decoded = crate::text::decode(bytes); + let out = crate::markdown::parse(&decoded.text, &self.config); + let mut renderer = crate::render::MarkdownRenderer::new(); + let mut e = Extraction::new(renderer.render(&out.doc), Format::Markdown); + e.title = out.title; + e.warnings = decoded.warnings; + e.stats.paragraphs = Some(out.paragraphs); + e.stats.tables = Some(out.tables); + Ok(e) + } + + #[cfg(not(feature = "markdown"))] + fn extract_markdown(&self, _bytes: &[u8]) -> crate::Result { + Err(Error::UnsupportedFormat( + "Markdown support requires the 'markdown' feature".to_string(), + )) + } + #[cfg(feature = "docx")] fn extract_docx(&self, bytes: &[u8]) -> crate::Result { let out = guarded("DOCX", Error::DocxExtract, || { diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index cf632a7..4c918c3 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -29,7 +29,9 @@ pub enum Format { /// Office Open XML presentation (`.pptx`). Detected but not supported. Pptx, Csv, + /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes. PlainText, + /// Markdown (`text/markdown`, `.md`), passed through normalized. Markdown, Unknown, } @@ -110,7 +112,7 @@ impl Format { Format::Csv } else if ct.contains("text/plain") { Format::PlainText - } else if ct.contains("text/markdown") { + } else if ct.contains("text/markdown") || ct.contains("text/x-markdown") { Format::Markdown } else { Format::Unknown @@ -129,8 +131,8 @@ impl Format { "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, "csv" | "tsv" => Format::Csv, - "txt" => Format::PlainText, - "md" | "markdown" => Format::Markdown, + "txt" | "text" => Format::PlainText, + "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, } } diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index d6453e2..bfca7ab 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -34,6 +34,8 @@ //! - **HTML** (`html`, default): Convert HTML to markdown with configurable rules //! - **DOCX** (`docx`): Convert Word documents //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables +//! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest //! @@ -53,9 +55,12 @@ pub mod format; #[cfg(feature = "html")] pub mod html; pub mod ir; +#[cfg(feature = "markdown")] +mod markdown; pub mod render; #[cfg(feature = "spreadsheet")] mod spreadsheet; +mod text; #[cfg(feature = "docx")] mod xml; diff --git a/crates/markkit/src/markdown.rs b/crates/markkit/src/markdown.rs new file mode 100644 index 0000000..cda2327 --- /dev/null +++ b/crates/markkit/src/markdown.rs @@ -0,0 +1,505 @@ +//! Markdown input: parse with pulldown-cmark into the IR, render back. +//! +//! Decisions: +//! - Markdown is re-parsed, not passed through, so every output goes through +//! the same IR and renderer (and the same output-safety rules). Raw HTML is +//! never copied: HTML blocks go through the HTML parser when the `html` +//! feature is on (scripts/styles stripped), otherwise they are dropped; +//! inline HTML tags are dropped and the text between them kept. +//! - Extensions: GFM tables, strikethrough, task lists, YAML front matter. +//! Front matter is not rendered; its `title:` becomes the title, otherwise +//! the first level-1 heading. +//! - Soft line breaks become spaces (paragraphs reflow); hard breaks stay. +//! - pulldown-cmark is iterative; the IR is built with an explicit stack and +//! capped at `MAX_DEPTH` open containers. Deeper content is flattened into +//! the deepest container so the recursive renderer cannot overflow. + +use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, MetadataBlockKind, Options, Parser, Tag}; + +use crate::config::MarkkitBuilder; +use crate::ir::{Document, ListItem, Node}; + +/// Max nested containers (block quotes, lists, emphasis, …) kept as structure. +const MAX_DEPTH: usize = 200; + +/// Parsed Markdown document. +pub(crate) struct ParsedMarkdown { + pub doc: Document, + pub title: Option, + pub paragraphs: usize, + pub tables: usize, +} + +enum Kind { + Root, + Para, + Heading(u8), + Quote, + List { + ordered: bool, + start: Option, + }, + Item, + Code { + language: Option, + }, + Table, + Head, + Row, + Cell, + Em, + Strong, + Strike, + Link { + url: String, + title: Option, + }, + Image { + src: String, + }, + Html, + Meta, + /// Beyond `MAX_DEPTH` or an unsupported container: children go to the parent. + Flat, +} + +struct Frame { + kind: Kind, + nodes: Vec, + items: Vec, + cells: Vec>, + rows: Vec>>, + /// Code, image alt, raw HTML or front matter text. + text: String, +} + +impl Frame { + fn new(kind: Kind) -> Self { + Self { + kind, + nodes: Vec::new(), + items: Vec::new(), + cells: Vec::new(), + rows: Vec::new(), + text: String::new(), + } + } +} + +/// What a finished frame hands to its parent. +enum Out { + Nodes(Vec), + Item(ListItem), + Cell(Vec), + Row(Vec>), + Head(Vec>), +} + +struct Builder<'c> { + #[cfg_attr(not(feature = "html"), allow(dead_code))] + config: &'c MarkkitBuilder, + stack: Vec, + meta_title: Option, + paragraphs: usize, + tables: usize, +} + +/// Parse Markdown text into the IR. +pub(crate) fn parse(text: &str, config: &MarkkitBuilder) -> ParsedMarkdown { + let opts = Options::ENABLE_TABLES + | Options::ENABLE_STRIKETHROUGH + | Options::ENABLE_TASKLISTS + | Options::ENABLE_YAML_STYLE_METADATA_BLOCKS; + let mut b = Builder { + config, + stack: vec![Frame::new(Kind::Root)], + meta_title: None, + paragraphs: 0, + tables: 0, + }; + for event in Parser::new_ext(text, opts) { + b.event(event); + } + while b.stack.len() > 1 { + b.end(); + } + let root = b.stack.pop().map(|f| f.nodes).unwrap_or_default(); + let title = b.meta_title.or_else(|| first_h1(&root)); + ParsedMarkdown { + doc: Document::new(root), + title, + paragraphs: b.paragraphs, + tables: b.tables, + } +} + +impl Builder<'_> { + fn top(&mut self) -> &mut Frame { + self.stack.last_mut().expect("root frame") + } + + fn event(&mut self, event: Event<'_>) { + match event { + Event::Start(tag) => self.start(tag), + Event::End(_) => self.end(), + Event::Text(t) => self.text(&t), + Event::Code(c) => self.push(Node::InlineCode(c.into_string())), + Event::Html(h) | Event::InlineHtml(h) => { + if matches!(self.top().kind, Kind::Html) { + self.top().text.push_str(&h); + } + } + Event::SoftBreak => self.text(" "), + Event::HardBreak => self.push(Node::LineBreak), + Event::Rule => self.push(Node::ThematicBreak), + Event::TaskListMarker(done) => self.text(if done { "[x] " } else { "[ ] " }), + Event::FootnoteReference(r) => self.text(&format!("[^{r}]")), + Event::InlineMath(m) | Event::DisplayMath(m) => { + self.push(Node::InlineCode(m.into_string())) + } + } + } + + fn start(&mut self, tag: Tag<'_>) { + let kind = if self.stack.len() > MAX_DEPTH { + Kind::Flat + } else { + match tag { + Tag::Paragraph => Kind::Para, + Tag::Heading { level, .. } => Kind::Heading(heading_level(level)), + Tag::BlockQuote(_) => Kind::Quote, + Tag::CodeBlock(kind) => Kind::Code { + language: match kind { + CodeBlockKind::Fenced(info) => info + .split_whitespace() + .next() + .map(|s| s.chars().filter(|c| *c != '`').collect::()) + .filter(|s| !s.is_empty()), + CodeBlockKind::Indented => None, + }, + }, + Tag::HtmlBlock => Kind::Html, + Tag::List(start) => Kind::List { + ordered: start.is_some(), + start, + }, + Tag::Item => Kind::Item, + Tag::Table(_) => Kind::Table, + Tag::TableHead => Kind::Head, + Tag::TableRow => Kind::Row, + Tag::TableCell => Kind::Cell, + Tag::Emphasis => Kind::Em, + Tag::Strong => Kind::Strong, + Tag::Strikethrough => Kind::Strike, + Tag::Link { + dest_url, title, .. + } => Kind::Link { + url: dest_url.into_string(), + title: Some(title.into_string()).filter(|t| !t.is_empty()), + }, + Tag::Image { dest_url, .. } => Kind::Image { + src: dest_url.into_string(), + }, + Tag::MetadataBlock(MetadataBlockKind::YamlStyle) => Kind::Meta, + _ => Kind::Flat, + } + }; + self.stack.push(Frame::new(kind)); + } + + fn end(&mut self) { + if self.stack.len() <= 1 { + return; + } + let f = self.stack.pop().expect("non-root frame"); + let out = match f.kind { + Kind::Root | Kind::Flat => Out::Nodes(f.nodes), + Kind::Para => { + self.paragraphs += 1; + Out::Nodes(vec![Node::Paragraph { children: f.nodes }]) + } + Kind::Heading(level) => { + self.paragraphs += 1; + Out::Nodes(vec![Node::Heading { + level, + children: f.nodes, + }]) + } + Kind::Quote => Out::Nodes(vec![Node::BlockQuote { + children: wrap_inline(f.nodes), + }]), + Kind::List { ordered, start } => Out::Nodes(vec![Node::List { + ordered, + start, + items: f.items, + }]), + Kind::Item => { + self.paragraphs += 1; + let has_block = f.nodes.iter().any(is_block); + let children = if has_block { + wrap_inline(f.nodes) + } else { + f.nodes + }; + Out::Item(ListItem::new(children)) + } + Kind::Code { language } => Out::Nodes(vec![Node::CodeBlock { + language, + code: f.text, + }]), + Kind::Table => { + self.tables += 1; + let mut nodes = vec![Node::Table { + headers: f.cells, + rows: f.rows, + }]; + nodes.extend(f.nodes); + Out::Nodes(nodes) + } + Kind::Head => Out::Head(f.cells), + Kind::Row => Out::Row(f.cells), + Kind::Cell => Out::Cell(f.nodes), + Kind::Em => Out::Nodes(vec![Node::Italic { children: f.nodes }]), + Kind::Strong => Out::Nodes(vec![Node::Bold { children: f.nodes }]), + Kind::Strike => Out::Nodes(vec![Node::Strikethrough { children: f.nodes }]), + Kind::Link { url, title } => Out::Nodes(vec![Node::Link { + url, + title, + children: f.nodes, + }]), + Kind::Image { src } => Out::Nodes(vec![Node::Image { src, alt: f.text }]), + Kind::Html => Out::Nodes(self.html_block(&f.text)), + Kind::Meta => { + if self.meta_title.is_none() { + self.meta_title = yaml_title(&f.text); + } + Out::Nodes(Vec::new()) + } + }; + self.accept(out); + } + + /// Hand a finished frame's output to the new top frame. + fn accept(&mut self, out: Out) { + let parent = self.top(); + match (&parent.kind, out) { + (Kind::List { .. }, Out::Item(item)) => parent.items.push(item), + (Kind::List { .. }, Out::Nodes(nodes)) if !nodes.is_empty() => { + parent.items.push(ListItem::new(nodes)) + } + (Kind::Head | Kind::Row, Out::Cell(cell)) => parent.cells.push(cell), + (Kind::Table, Out::Head(cells)) => parent.cells = cells, + (Kind::Table, Out::Row(cells)) => parent.rows.push(cells), + (_, Out::Nodes(nodes) | Out::Cell(nodes)) => self.push_all(nodes), + (_, Out::Item(item)) => self.push_all(item.children), + (_, Out::Row(cells) | Out::Head(cells)) => { + for cell in cells { + self.push_all(cell); + self.push(Node::Text(" ".into())); + } + } + } + } + + fn push_all(&mut self, nodes: Vec) { + for n in nodes { + self.push(n); + } + } + + fn push(&mut self, node: Node) { + let top = self.top(); + match top.kind { + // Alt text, code and front matter take plain text only. + Kind::Image { .. } | Kind::Code { .. } => plain_text(&node, &mut top.text), + Kind::Html | Kind::Meta => {} + Kind::List { .. } => top.items.push(ListItem::new(vec![node])), + Kind::Head | Kind::Row => top.cells.push(vec![node]), + _ => top.nodes.push(node), + } + } + + fn text(&mut self, t: &str) { + let top = self.top(); + match top.kind { + Kind::Image { .. } | Kind::Code { .. } | Kind::Meta => top.text.push_str(t), + Kind::Html => {} + _ => { + if let Some(Node::Text(prev)) = top.nodes.last_mut() { + prev.push_str(t); + } else { + self.push(Node::Text(t.to_string())); + } + } + } + } + + #[cfg(feature = "html")] + fn html_block(&self, html: &str) -> Vec { + crate::html::HtmlParser::new(self.config) + .parse(html) + .map(|d| d.nodes) + .unwrap_or_default() + } + + #[cfg(not(feature = "html"))] + fn html_block(&self, _html: &str) -> Vec { + Vec::new() + } +} + +fn heading_level(level: HeadingLevel) -> u8 { + match level { + HeadingLevel::H1 => 1, + HeadingLevel::H2 => 2, + HeadingLevel::H3 => 3, + HeadingLevel::H4 => 4, + HeadingLevel::H5 => 5, + HeadingLevel::H6 => 6, + } +} + +fn is_block(node: &Node) -> bool { + matches!( + node, + Node::Heading { .. } + | Node::Paragraph { .. } + | Node::List { .. } + | Node::BlockQuote { .. } + | Node::CodeBlock { .. } + | Node::Table { .. } + | Node::ThematicBreak + ) +} + +/// Group runs of inline nodes into paragraphs (for containers mixing both). +fn wrap_inline(nodes: Vec) -> Vec { + let mut out = Vec::with_capacity(nodes.len()); + let mut run = Vec::new(); + for n in nodes { + if is_block(&n) { + if !run.is_empty() { + out.push(Node::Paragraph { + children: std::mem::take(&mut run), + }); + } + out.push(n); + } else { + run.push(n); + } + } + if !run.is_empty() { + out.push(Node::Paragraph { children: run }); + } + out +} + +/// Append the plain text of a node (iteratively; nodes may be nested). +fn plain_text(node: &Node, out: &mut String) { + let mut stack = vec![node]; + while let Some(n) = stack.pop() { + match n { + Node::Text(t) | Node::InlineCode(t) => out.push_str(t), + Node::CodeBlock { code, .. } => out.push_str(code), + Node::Image { alt, .. } => out.push_str(alt), + Node::LineBreak => out.push(' '), + Node::Heading { children, .. } + | Node::Paragraph { children } + | Node::BlockQuote { children } + | Node::Bold { children } + | Node::Italic { children } + | Node::Strikethrough { children } + | Node::Link { children, .. } => stack.extend(children.iter().rev()), + Node::List { items, .. } => { + stack.extend(items.iter().rev().flat_map(|i| i.children.iter().rev())) + } + Node::Table { headers, rows } => { + stack.extend(headers.iter().chain(rows.iter().flatten()).flatten().rev()) + } + Node::ThematicBreak => {} + } + } +} + +fn first_h1(nodes: &[Node]) -> Option { + nodes.iter().find_map(|n| match n { + Node::Heading { level: 1, .. } => { + let mut t = String::new(); + plain_text(n, &mut t); + let t = t.trim(); + (!t.is_empty()).then(|| t.to_string()) + } + _ => None, + }) +} + +/// `title:` from YAML front matter (top-level key, scalar value only). +fn yaml_title(yaml: &str) -> Option { + yaml.lines().find_map(|line| { + let v = line.strip_prefix("title:")?.trim(); + let v = v + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| v.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) + .unwrap_or(v) + .trim(); + (!v.is_empty()).then(|| v.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn md(src: &str) -> String { + let p = parse(src, &MarkkitBuilder::default()); + crate::render::MarkdownRenderer::new().render(&p.doc) + } + + #[test] + fn roundtrip_common_blocks() { + let src = "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt *x*](i.png)\n"; + assert_eq!( + md(src), + "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt x](i.png)\n" + ); + } + + #[test] + fn table_and_tasks() { + assert_eq!( + md("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"), + "| a | b |\n| --- | --- |\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n" + ); + } + + #[test] + fn soft_breaks_reflow_hard_breaks_kept() { + assert_eq!(md("a\nb \nc\\\nd"), "a b \nc \nd\n"); + } + + #[test] + fn titles() { + let p = parse( + "---\ntitle: \"Doc\"\n---\n\n# H\n", + &MarkkitBuilder::default(), + ); + assert_eq!(p.title.as_deref(), Some("Doc")); + let p = parse("## sub\n\n# Real *one*\n", &MarkkitBuilder::default()); + assert_eq!(p.title.as_deref(), Some("Real one")); + assert_eq!(parse("text", &MarkkitBuilder::default()).title, None); + } + + #[test] + fn loose_item_with_nested_list() { + assert_eq!(md("- a **b**\n - c\n"), "- a **b**\n - c\n"); + } + + #[test] + fn deep_nesting_is_flattened() { + let src = format!("{}x", "> ".repeat(MAX_DEPTH * 5)); + let out = md(&src); + assert!(out.trim_end().ends_with('x')); + assert!(out.lines().next().unwrap().matches('>').count() <= MAX_DEPTH + 1); + } +} diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs new file mode 100644 index 0000000..19e7999 --- /dev/null +++ b/crates/markkit/src/text.rs @@ -0,0 +1,164 @@ +//! Plain text input, plus the text decoding shared with Markdown input. +//! +//! Decisions: +//! - Always compiled (no dependencies, no feature gate); selected only by a +//! content type (`text/plain`) or extension (`.txt`). Bytes are never +//! sniffed as text, so unknown binaries stay `UnsupportedFormat` instead +//! of rendering as mojibake. +//! - Decoding: UTF-8 (BOM stripped), UTF-16 LE/BE when a BOM is present; +//! anything else is decoded lossily with an `invalid_utf8` warning. +//! - C0 control characters other than tab/newline, and DEL, are dropped so +//! terminal escapes and NULs never reach the output; CRLF/CR become LF. +//! - Plain text goes through the IR: blank lines separate paragraphs, single +//! newlines become hard breaks, text stays `Node::Text` so renderer-level +//! escaping applies uniformly. + +use crate::ir::{Document, Node}; + +/// Decoded text plus non-fatal warnings. +pub(crate) struct Decoded { + pub text: String, + pub warnings: Vec, +} + +/// Decode bytes to normalized text (see module docs). +pub(crate) fn decode(bytes: &[u8]) -> Decoded { + let mut warnings = Vec::new(); + let raw = if let Some(rest) = bytes.strip_prefix(b"\xEF\xBB\xBF") { + utf8(rest, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFF\xFE") { + utf16(rest, u16::from_le_bytes, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFE\xFF") { + utf16(rest, u16::from_be_bytes, &mut warnings) + } else { + utf8(bytes, &mut warnings) + }; + Decoded { + text: normalize(&raw), + warnings, + } +} + +fn utf8(bytes: &[u8], warnings: &mut Vec) -> String { + match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => { + warnings.push("invalid_utf8: invalid UTF-8 sequences replaced with U+FFFD".into()); + String::from_utf8_lossy(bytes).into_owned() + } + } +} + +fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec) -> String { + let units = bytes.chunks_exact(2).map(|c| read([c[0], c[1]])); + let mut lossy = !bytes.len().is_multiple_of(2); + let s: String = char::decode_utf16(units) + .map(|r| { + r.unwrap_or_else(|_| { + lossy = true; + char::REPLACEMENT_CHARACTER + }) + }) + .collect(); + if lossy { + warnings.push("invalid_utf8: invalid UTF-16 sequences replaced with U+FFFD".into()); + } + s +} + +/// CRLF/CR → LF; drop C0 controls (except `\t`, `\n`) and DEL. +fn normalize(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\r' => { + if chars.peek() == Some(&'\n') { + chars.next(); + } + out.push('\n'); + } + '\t' | '\n' => out.push(c), + c if c.is_ascii_control() => {} + c => out.push(c), + } + } + out +} + +/// Plain text → IR. Returns the document and the paragraph count. +pub(crate) fn plain_to_ir(text: &str) -> (Document, usize) { + let mut nodes = Vec::new(); + let mut para: Vec = Vec::new(); + let flush = |para: &mut Vec, nodes: &mut Vec| { + if !para.is_empty() { + nodes.push(Node::Paragraph { + children: std::mem::take(para), + }); + } + }; + for line in text.split('\n') { + let line = line.trim_end(); + if line.is_empty() { + flush(&mut para, &mut nodes); + continue; + } + if !para.is_empty() { + para.push(Node::LineBreak); + } + para.push(Node::Text(line.to_string())); + } + flush(&mut para, &mut nodes); + let count = nodes.len(); + (Document::new(nodes), count) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_strips_bom_and_controls() { + let d = decode(b"\xEF\xBB\xBFa\r\nb\rc\x00\x1b[31m\x7f\td"); + assert_eq!(d.text, "a\nb\nc[31m\td"); + assert!(d.warnings.is_empty()); + } + + #[test] + fn decode_utf16() { + let mut le = b"\xFF\xFE".to_vec(); + le.extend("hé".encode_utf16().flat_map(u16::to_le_bytes)); + assert_eq!(decode(&le).text, "hé"); + let mut be = b"\xFE\xFF".to_vec(); + be.extend("hé".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(decode(&be).text, "hé"); + // Odd length and lone surrogate are lossy, never panic. + let d = decode(b"\xFF\xFE\x00\xD8a"); + assert!(d.text.contains('\u{FFFD}')); + assert_eq!(d.warnings.len(), 1); + } + + #[test] + fn decode_invalid_utf8_warns() { + let d = decode(b"ok \xFF\xFE!"); + assert_eq!(d.text, "ok \u{FFFD}\u{FFFD}!"); + assert!(d.warnings[0].starts_with("invalid_utf8:")); + } + + #[test] + fn plain_paragraphs_and_breaks() { + let (doc, n) = plain_to_ir("a\nb \n\n\n c\n"); + assert_eq!(n, 2); + assert_eq!( + doc.nodes[0], + Node::Paragraph { + children: vec![ + Node::Text("a".into()), + Node::LineBreak, + Node::Text("b".into()) + ] + } + ); + assert_eq!(plain_to_ir("\n \n").1, 0); + } +} diff --git a/crates/markkit/tests/text_integration.rs b/crates/markkit/tests/text_integration.rs new file mode 100644 index 0000000..d55f846 --- /dev/null +++ b/crates/markkit/tests/text_integration.rs @@ -0,0 +1,228 @@ +//! Plain text and Markdown → Markdown integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; + +fn extract(bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync(bytes, hint) + .unwrap() +} + +#[test] +fn plain_text_by_content_type_and_extension() { + let e = extract( + b"Hello\nworld\n\nSecond paragraph\n", + DocumentHint::content_type("text/plain; charset=utf-8"), + ); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "Hello \nworld\n\nSecond paragraph\n"); + assert_eq!(e.stats.paragraphs, Some(2)); + assert_eq!(e.title, None); + assert!(e.warnings.is_empty()); + + let e = extract(b"notes", DocumentHint::file_name("dir/NOTES.TXT")); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "notes\n"); +} + +#[test] +fn plain_text_crlf_and_blank_runs() { + let e = extract( + b"a\r\nb\r\n\r\n\r\n\r\nc \r\n", + DocumentHint::file_name("x.txt"), + ); + assert_eq!(e.markdown, "a \nb\n\nc\n"); +} + +#[test] +fn plain_text_empty() { + let e = extract(b"", DocumentHint::file_name("empty.txt")); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.paragraphs, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_normalized_with_title() { + let src = "\u{feff}---\ntitle: Guide\n---\n\n# Intro\n\nSome *text*\nwrapped.\n\n```rust\nfn main() {}\n```\n\n* item\n\n\n"; + let e = extract(src.as_bytes(), DocumentHint::file_name("guide.md")); + assert_eq!(e.format, Format::Markdown); + assert_eq!(e.title.as_deref(), Some("Guide")); + assert_eq!( + e.markdown, + "# Intro\n\nSome *text* wrapped.\n\n```rust\nfn main() {}\n```\n\n- item\n" + ); + assert_eq!(e.stats.tables, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_title_from_heading() { + for ct in ["text/markdown", "text/x-markdown; charset=utf-8"] { + let e = extract(b"# Title\n\nbody", DocumentHint::content_type(ct)); + assert_eq!(e.format, Format::Markdown, "{ct}"); + assert_eq!(e.title.as_deref(), Some("Title")); + assert_eq!(e.markdown, "# Title\n\nbody\n"); + } +} + +#[cfg(not(feature = "markdown"))] +#[test] +fn markdown_requires_feature() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# x", DocumentHint::file_name("a.md")) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[test] +fn text_is_never_sniffed_without_hint() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# just text", DocumentHint::default()) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[tokio::test] +async fn extract_bytes_text_plain() { + let m = Markkit::builder().build(); + let md = m.extract_bytes(b"x\ny", Some("text/plain")).await; + assert_eq!(md.unwrap(), "x \ny\n"); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +/// File names for every text-like format compiled in. +fn names() -> Vec<&'static str> { + let mut v = vec!["a.txt"]; + if cfg!(feature = "markdown") { + v.push("a.md"); + } + v +} + +fn has_control(s: &str) -> bool { + s.chars() + .any(|c| c.is_ascii_control() && c != '\n' && c != '\t') +} + +#[test] +fn security_control_chars_stripped() { + for name in names() { + let e = extract( + b"safe\x1b]0;pwned\x07\x00text\x7f\x08", + DocumentHint::file_name(name), + ); + assert!(!has_control(&e.markdown), "{name}: {:?}", e.markdown); + assert_eq!(e.markdown, "safe]0;pwnedtext\n"); + } +} + +#[test] +fn security_invalid_utf8_lossy_with_warning() { + let e = extract(b"caf\xE9 \xC3", DocumentHint::file_name("a.txt")); + assert_eq!(e.markdown, "caf\u{FFFD} \u{FFFD}\n"); + assert_eq!(e.warnings.len(), 1); + assert!(e.warnings[0].starts_with("invalid_utf8:")); +} + +#[test] +fn security_utf16_malformed_no_panic() { + for bytes in [ + &b"\xFF\xFE"[..], + b"\xFF\xFE\x41", + b"\xFE\xFF\xD8\x00\xD8\x00", + b"\xFF\xFE\x00\xDC", + ] { + let e = extract(bytes, DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= 16, "{bytes:?}"); + } +} + +#[test] +fn security_input_size_limit() { + let m = Markkit::builder().max_input_size(8).build(); + for name in ["a.txt", "a.md"] { + let err = m + .extract_document_sync(b"0123456789", DocumentHint::file_name(name)) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}"); + } +} + +#[test] +fn security_output_bounded_by_input() { + // Each line becomes a hard break (" \n" = +2 bytes per line): output + // stays linear in input size. + let input = "a\n".repeat(100_000); + let e = extract(input.as_bytes(), DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= input.len() * 2); + let blanks = "\n".repeat(1_000_000); + for name in names() { + let e = extract(blanks.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown, "", "{name}"); + } +} + +#[test] +fn security_long_line() { + let line = "x".repeat(2_000_000); + for name in names() { + let e = extract(line.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown.len(), line.len() + 1, "{name}"); + } +} + +#[test] +fn security_arbitrary_bytes_never_panic() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + for _ in 0..300 { + let len = (seed % 512) as usize; + let bytes: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Bias towards Markdown syntax so the parser is exercised. + b"#*_`[]()!<>|-~\n \\x\x00\x1b\xff"[(seed % 21) as usize] + }) + .collect(); + for name in names() { + let e = extract(&bytes, DocumentHint::file_name(name)); + assert!(!has_control(&e.markdown), "{name}: {bytes:?}"); + } + } +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_raw_html_never_passes_through() { + let src = b"\n\n
Hi there
\n\npara kept\n\n\n"; + let e = extract(src, DocumentHint::file_name("a.md")); + for bad in [ + " ".repeat(100_000)), + format!("{}x", "- ".repeat(50_000)), + format!("{}x{}", "*".repeat(50_000), "*".repeat(50_000)), + format!("{}x{}", "[".repeat(50_000), "](u)".repeat(50_000)), + ] { + let e = extract(src.as_bytes(), DocumentHint::file_name("a.md")); + assert!(e.markdown.contains('x')); + } +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 7e3a825..0dce0ff 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -1,13 +1,14 @@ --- type: Subsystem Design title: Format Support -description: Supported and planned formats, bytes-in API, format detection, DOCX and spreadsheet mapping. +description: Supported and planned formats, bytes-in API, format detection, DOCX, spreadsheet, plain text and Markdown mapping. tags: - markkit - formats - docx - spreadsheet - detection + - markdown --- # Format Support @@ -16,11 +17,12 @@ tags: ### Tier 1 (Core) - **HTML/XHTML**: Full support via html5ever + scraper (`html` feature, default) -- **Plain text**: Planned pass-through (currently `UnsupportedFormat` via the bytes API) +- **Plain text**: Always on, no dependencies (`text/plain`, `.txt`, `.text`) ### Tier 2 (Feature-gated) - **DOCX** (`docx`): Word documents via a direct `zip` + `quick-xml` reader (no docx-rs) - **XLSX/XLSM, XLS, ODS** (`spreadsheet`): Workbooks to Markdown tables via calamine +- **Markdown** (`markdown`): Re-parsed with pulldown-cmark into the IR and rendered back (`text/markdown`, `text/x-markdown`, `.md`, `.markdown`, `.mdown`, `.mkd`, `.mkdn`) - **PDF** (`pdf`): Planned. Returns `Error::UnsupportedFormat` (never an empty success) ### Detected, not supported @@ -30,7 +32,6 @@ tags: - **EPUB**: E-book extraction - **RTF**: Rich text format - **CSV/TSV**: Tabular data to markdown tables -- **Markdown**: Normalize/clean existing markdown ## Bytes-in API @@ -55,7 +56,7 @@ pub struct Extraction { // #[non_exhaustive] only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. -Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. +Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`, `invalid_utf8`. ## Format Detection @@ -71,6 +72,9 @@ Priority order (`Format::detect`): - `%PDF-` within the first 1 KiB → PDF. - ``, event handlers) or pathological nesting. + +**Impact**: Medium -- raw HTML passed through re-introduces T1; deep nesting can crash. + +**Mitigations**: +- Text and Markdown are never sniffed; a content type or extension must select them +- Decoding is total: invalid UTF-8/UTF-16 is replaced (U+FFFD) with an `invalid_utf8` warning +- C0 controls (except tab/newline) and DEL are dropped before parsing +- Markdown is re-parsed (pulldown-cmark) into the IR; raw HTML is never copied. HTML blocks + go through the HTML parser (T1 stripping) or are dropped; inline tags are dropped +- Explicit-stack IR builder, nesting capped at 200 (deeper content flattened) + +**Test Coverage** (`text_integration`): +- `security_control_chars_stripped`, `security_arbitrary_bytes_never_panic` +- `security_invalid_utf8_lossy_with_warning`, `security_utf16_malformed_no_panic` +- `security_markdown_raw_html_never_passes_through` +- `security_markdown_deep_nesting_no_stack_overflow`, `security_long_line` +- `security_input_size_limit`, `security_output_bounded_by_input` + ### T5: Information Disclosure **Description**: Error messages or output leak sensitive information. @@ -239,7 +261,7 @@ process, or smuggling content past a human reviewer. | Processing time | 60 s | `max_processing_secs(u64)` | Whole `extract_url` fetch (parsing itself TBD) | | Redirects (`extract_url`) | 5 | `max_redirects(usize)` | Every redirect hop | | Fetched body (`extract_url`) | = input size | `max_input_size(usize)` | `Content-Length` and while streaming | -| Nesting depth | 200 | Hardcoded `MAX_DEPTH` | During HTML tree walk | +| Nesting depth | 200 | Hardcoded `MAX_DEPTH` | During HTML tree walk and Markdown IR build | | XML nesting depth (DOCX) | 256 | Hardcoded `MAX_XML_DEPTH` | While building the XML tree | | ZIP entries | 10 000 | `max_archive_entries(usize)` | On archive open | | ZIP entry size (decompressed) | 64 MiB | `max_archive_entry_size(u64)` | While decompressing | @@ -285,6 +307,7 @@ process, or smuggling content past a human reviewer. | S10 | Spreadsheet parsing never allocates a grid beyond `max_spreadsheet_cells` | `document_security::xls_*_rejected`, `ods_repeated_rows_and_columns_rejected` | | S11 | Unsupported formats fail explicitly (no empty success) | `document_detection::pdf_is_explicitly_unsupported_not_empty` | | S12 | Default `extract_url` never connects to a non-public address | `url_security::*_blocked*`, `fetch::tests::non_public_addresses_forbidden` | +| S13 | Text and Markdown output never contains C0 controls (except tab/newline) or raw HTML from the input | `text_integration::security_*` | ## See also From 7e3a23392805788ed4f335a30fc434e0f9413388 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:37:16 +0000 Subject: [PATCH 2/3] fix(text): use as_chunks for UTF-16 decoding (clippy) --- crates/markkit/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs index 19e7999..0d33cb3 100644 --- a/crates/markkit/src/text.rs +++ b/crates/markkit/src/text.rs @@ -50,7 +50,7 @@ fn utf8(bytes: &[u8], warnings: &mut Vec) -> String { } fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec) -> String { - let units = bytes.chunks_exact(2).map(|c| read([c[0], c[1]])); + let units = bytes.as_chunks::<2>().0.iter().map(|c| read(*c)); let mut lossy = !bytes.len().is_multiple_of(2); let s: String = char::decode_utf16(units) .map(|r| { From ba3006272c6a28dcf926e3c60f6480e3c5462b9f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:51:54 +0000 Subject: [PATCH 3/3] chore(knowledge): scope S13 to markup copying, note text escaping residual --- knowledge/security/threat-model.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 1fb8377..9f075ed 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -233,6 +233,10 @@ raw HTML (`