diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9a9f5f..676561f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,9 +82,9 @@ jobs: echo "HTTP client leaked into the default feature set" exit 1 fi - cargo test -p markkit --no-default-features --features docx,spreadsheet - if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'reqwest|hyper'; then - echo "HTTP client leaked into the docx,spreadsheet feature set" + cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown + if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then + echo "HTTP client leaked into the docx,spreadsheet,markdown feature set" exit 1 fi - name: Run examples @@ -112,7 +112,7 @@ jobs: with: node-version: "24.22.1" - name: Check core for wasm32 (no warnings) - run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,tool --target wasm32-unknown-unknown + run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,markdown,tool --target wasm32-unknown-unknown - name: Install wasm-bindgen-cli (matches Cargo.lock) run: bash scripts/install-wasm-bindgen.sh - name: Build package diff --git a/CHANGELOG.md b/CHANGELOG.md index 40c03f4..4822d07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- Plain text input (`text/plain`, `.txt`), always on: paragraphs and hard breaks through the IR, + UTF-8/UTF-16 BOM decoding, control characters dropped, `invalid_utf8` warning on lossy decode. +- Markdown input (`markdown` feature, in `all_formats`; `text/markdown`, `.md`): re-parsed with + pulldown-cmark and rendered back, so raw HTML never passes through. Front matter `title:` or the + first `# ` heading becomes `Extraction::title`. Available in the CLI, Python, Node and WASM packages; `format` serializes as + `"text"` and `"markdown"`. - HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or ``) and the new `Extraction::metadata` (description, author, published date, language, canonical URL, site name, lead image) from `<meta>`, Open Graph, `<link rel=canonical>` and diff --git a/Cargo.toml b/Cargo.toml index e0df15c..17be211 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS) zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip quick-xml = "0.42" # DOCX/ODS XML parsing cfb = "0.15" # XLS (OLE2) container pre-scan +pulldown-cmark = { version = "0.13", default-features = false } # Markdown input; output-safety test oracle # Logging tracing = "0.1" @@ -73,7 +74,6 @@ napi-derive = "3" tempfile = "3" criterion = { version = "0.8", features = ["html_reports"] } proptest = "1" -pulldown-cmark = { version = "0.13", default-features = false } [profile.release] lto = "thin" diff --git a/README.md b/README.md index e8a273d..79909c7 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ package for browsers and edge runtimes. - **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default - **DOCX** – Headings, lists, tables, links, emphasis - **Spreadsheets** – XLSX/XLSM, XLS and ODS workbooks to markdown tables, one section per sheet +- **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) - **Format detection** – Content type, then file extension, then magic bytes - **URL fetching** – Fetch and extract in one call (`http_client` feature) - **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits @@ -32,6 +33,8 @@ package for browsers and edge runtimes. | HTML / XHTML | Supported | `html` (default) | | DOCX | Supported | `docx` | | XLSX, XLSM, XLS, ODS | Supported | `spreadsheet` | +| Plain text | Supported | always on | +| Markdown | Supported | `markdown` | | PDF | Planned | `pdf` | | DOC, PPTX, XLSB | Detected, rejected explicitly | – | @@ -181,7 +184,8 @@ let markkit = Markkit::builder() | `docx` | No | DOCX conversion (zip + quick-xml) | | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` | -| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` | +| `markdown` | No | Markdown normalization (pulldown-cmark) | +| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet`, `markdown` | | `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 69495f0..b7bcc30 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -24,6 +24,7 @@ all_formats = ["markkit/all_formats"] pdf = ["markkit/pdf"] docx = ["markkit/docx"] spreadsheet = ["markkit/spreadsheet"] +markdown = ["markkit/markdown"] [dependencies] markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] } diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 1b2e9c8..006b86f 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -103,6 +103,37 @@ fn extract_from_file() { assert!(stdout.contains("Paragraph"), "output: {stdout}"); } +#[test] +fn extract_text_and_markdown_files() { + let dir = tempfile::tempdir().unwrap(); + let cases = [ + ( + "notes.txt", + "line one\nline two\n", + "line one \nline two\n", + ), + ( + "doc.md", + "Title\n=====\n\n* a\n\n<script>x()</script>\n", + "# Title\n\n- a\n", + ), + ]; + for (name, input, expected) in cases { + let file_path = dir.path().join(name); + std::fs::write(&file_path, input).unwrap(); + let output = markkit_bin() + .args(["extract", "--file", file_path.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + assert!( + output.status.success(), + "{name} stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), expected, "{name}"); + } +} + #[test] fn extract_json_output() { let mut child = markkit_bin() diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 9f1f3bf..84483c9 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -90,7 +90,7 @@ pub struct Metadata { pub struct Extraction { pub markdown: String, pub title: Option<String>, - /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`). + /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`). pub format: String, pub warnings: Vec<String>, pub stats: ExtractionStats, diff --git a/crates/markkit-wasm/Cargo.toml b/crates/markkit-wasm/Cargo.toml index 7e00540..857ae55 100644 --- a/crates/markkit-wasm/Cargo.toml +++ b/crates/markkit-wasm/Cargo.toml @@ -28,6 +28,7 @@ markkit = { path = "../markkit", default-features = false, features = [ "html", "docx", "spreadsheet", + "markdown", "tool", ] } wasm-bindgen = "0.2" diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index 30c020a..dcc03bd 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -1,7 +1,8 @@ # @everruns/markkit-wasm -Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX and -spreadsheets (XLSX/XLSM, XLS, ODS) in the **browser and any JavaScript runtime**: +Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX, +spreadsheets (XLSX/XLSM, XLS, ODS), plain text and Markdown in the **browser +and any JavaScript runtime**: edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index 3c80df0..0458d6e 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -74,6 +74,17 @@ test("extractDocument exposes page metadata in camelCase", () => { assert.equal(out.metadata.site_name, undefined); }); +test("extractDocument reads plain text and Markdown", () => { + const mk = new Markkit(); + const txt = mk.extractDocument(new TextEncoder().encode("a\nb"), { fileName: "n.txt" }); + assert.equal(txt.format, "text"); + assert.equal(txt.markdown, "a \nb\n"); + const md = mk.extractDocument(new TextEncoder().encode("# T\n\n<script>x()</script>\n"), { fileName: "d.md" }); + assert.equal(md.format, "markdown"); + assert.equal(md.title, "T"); + assert.ok(!md.markdown.includes("script")); +}); + test("extractBytes and unsupported formats", () => { const mk = new Markkit(); assert.equal(mk.extractBytes(new TextEncoder().encode("<p>x</p>"), "text/html"), "x\n"); diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 2d179f5..34f30e1 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -19,6 +19,7 @@ html = ["dep:scraper", "dep:html5ever", "dep:ego-tree", "dep:url"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] +markdown = ["dep:pulldown-cmark"] # Opt-in: the default build has no network stack (no reqwest/hyper/TLS). http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] @@ -26,7 +27,7 @@ logging = ["dep:tracing"] tool = ["dep:tower", "dep:futures-core", "dep:base64"] # JSON Schema for the IR (`markkit::ir::json_schema()`) json-schema = ["dep:schemars"] -all_formats = ["html", "pdf", "docx", "spreadsheet"] +all_formats = ["html", "pdf", "docx", "spreadsheet", "markdown"] [dependencies] # Async @@ -54,6 +55,9 @@ quick-xml = { workspace = true, optional = true } calamine = { workspace = true, optional = true } cfb = { workspace = true, optional = true } +# Markdown (optional) +pulldown-cmark = { workspace = true, optional = true } + # HTTP client (optional) reqwest = { workspace = true, optional = true } diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index f657cf9..ae904ee 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -15,12 +15,14 @@ pub struct Extraction { pub markdown: String, /// Document title when known: HTML `og:title` / JSON-LD / `<title>`; DOCX /// core properties or `Title` style; XLSX `docProps/core.xml`, ODS - /// `meta.xml`, XLS `SummaryInformation`. + /// `meta.xml`, XLS `SummaryInformation`; + /// Markdown front matter `title:` or first `# ` heading. pub title: Option<String>, /// Format the document was parsed as. pub format: Format, /// Non-fatal issues, each prefixed with a stable code - /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`). + /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`, + /// `invalid_utf8:`). pub warnings: Vec<String>, /// Format-specific counters. pub stats: ExtractionStats, @@ -65,9 +67,9 @@ pub struct ExtractionStats { pub sheets: Option<usize>, /// Data rows rendered across all sheets (header rows excluded). pub rows: Option<usize>, - /// Non-empty paragraphs, headings and list items (DOCX). + /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown). pub paragraphs: Option<usize>, - /// Tables (DOCX). + /// Tables (DOCX, Markdown). pub tables: Option<usize>, } @@ -91,10 +93,6 @@ pub struct ParsedDocument { } impl ParsedDocument { - #[cfg_attr( - not(any(feature = "html", feature = "docx", feature = "spreadsheet")), - allow(dead_code) - )] fn new(document: Document, format: Format) -> Self { Self { document, @@ -226,7 +224,7 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, DOCX, XLSX/XLS/ODS) to Markdown with metadata. + /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -267,7 +265,7 @@ impl Markkit { Ok(parsed.render()) } - /// Parse a document (HTML, DOCX, XLSX/XLS/ODS) into the IR without + /// Parse a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) into the IR without /// rendering. Same detection and errors as [`Markkit::extract_document`]. pub async fn parse_document( &self, @@ -301,6 +299,8 @@ impl Markkit { Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => { self.parse_spreadsheet(bytes, budget) } + Format::PlainText => self.parse_plain_text(bytes, budget), + Format::Markdown => self.parse_markdown(bytes, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -333,6 +333,37 @@ impl Markkit { )) } + fn parse_plain_text(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> { + let decoded = crate::text::decode(bytes); + let (mut doc, paragraphs) = crate::text::plain_to_ir(&decoded.text, budget)?; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::PlainText); + p.warnings = decoded.warnings; + p.stats.paragraphs = Some(paragraphs); + Ok(p) + } + + #[cfg(feature = "markdown")] + fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> { + let decoded = crate::text::decode(bytes); + let out = crate::markdown::parse(&decoded.text, &self.config, budget)?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Markdown); + p.title = out.title; + p.warnings = decoded.warnings; + p.stats.paragraphs = Some(out.paragraphs); + p.stats.tables = Some(out.tables); + Ok(p) + } + + #[cfg(not(feature = "markdown"))] + fn parse_markdown(&self, _bytes: &[u8], _budget: &Budget) -> crate::Result<ParsedDocument> { + Err(Error::UnsupportedFormat( + "Markdown support requires the 'markdown' feature".to_string(), + )) + } + #[cfg(feature = "docx")] fn parse_docx(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> { let out = guarded("DOCX", Error::DocxExtract, || { diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index 509604c..fea7b95 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -31,7 +31,11 @@ pub enum Format { /// Office Open XML presentation (`.pptx`). Detected but not supported. Pptx, Csv, + /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes. + /// Serialized as `"text"`, same as `as_str`. + #[serde(rename = "text")] PlainText, + /// Markdown (`text/markdown`, `.md`), passed through normalized. Markdown, Unknown, } @@ -115,7 +119,7 @@ impl Format { Format::Csv } else if ct.contains("text/plain") { Format::PlainText - } else if ct.contains("text/markdown") { + } else if ct.contains("text/markdown") || ct.contains("text/x-markdown") { Format::Markdown } else { Format::Unknown @@ -135,8 +139,8 @@ impl Format { "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, "csv" | "tsv" => Format::Csv, - "txt" => Format::PlainText, - "md" | "markdown" => Format::Markdown, + "txt" | "text" => Format::PlainText, + "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, } } @@ -245,6 +249,28 @@ fn looks_like_html(bytes: &[u8]) -> bool { mod tests { use super::*; + #[test] + fn serde_name_matches_as_str() { + for f in [ + Format::Html, + Format::Pdf, + Format::Docx, + Format::Doc, + Format::Xlsx, + Format::Xls, + Format::Ods, + Format::Pptx, + Format::Csv, + Format::PlainText, + Format::Markdown, + Format::Unknown, + ] { + let json = serde_json::to_string(&f).unwrap(); + assert_eq!(json, format!("\"{}\"", f.as_str())); + assert_eq!(serde_json::from_str::<Format>(&json).unwrap(), f); + } + } + #[test] fn test_content_type_detection() { assert_eq!( diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 3a1f2bc..afb874d 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -34,6 +34,8 @@ //! - **HTML** (`html`, default): Convert HTML to markdown with configurable rules //! - **DOCX** (`docx`): Convert Word documents //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables +//! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, opt-in): Fetch and extract from URLs via reqwest //! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the @@ -58,9 +60,12 @@ pub mod format; pub mod html; pub mod ir; pub mod limits; +#[cfg(feature = "markdown")] +mod markdown; pub mod render; #[cfg(feature = "spreadsheet")] mod spreadsheet; +mod text; #[cfg(feature = "tool")] pub mod tool; pub mod transform; diff --git a/crates/markkit/src/markdown.rs b/crates/markkit/src/markdown.rs new file mode 100644 index 0000000..405d926 --- /dev/null +++ b/crates/markkit/src/markdown.rs @@ -0,0 +1,668 @@ +//! Markdown input: parse with pulldown-cmark into the IR, render back. +//! +//! Decisions: +//! - Markdown is re-parsed, not passed through, so every output goes through +//! the same IR and renderer (and the same output-safety rules). Raw HTML is +//! never copied: HTML blocks go through the HTML parser when the `html` +//! feature is on (scripts/styles stripped), otherwise they are dropped; +//! inline HTML tags are dropped and the text between them kept. +//! - Extensions map onto IR v2: GFM tables (with column alignment), +//! strikethrough, task lists (`ListItem::checked`), footnotes, definition +//! lists, YAML front matter. Front matter is not rendered; its `title:` +//! becomes the title, otherwise the first level-1 heading. +//! - Soft line breaks become spaces (paragraphs reflow); hard breaks stay. +//! - pulldown-cmark is iterative; the IR is built with an explicit stack and +//! capped at `limits.max_nesting_depth` open containers. Deeper content is +//! flattened into the deepest container so the recursive renderer cannot +//! overflow. +//! - Shares the call's `Budget`: every event ticks the deadline, every start +//! tag counts against `max_nodes`, and HTML blocks use the same budget. + +use pulldown_cmark::{ + Alignment as MdAlignment, CodeBlockKind, Event, HeadingLevel, MetadataBlockKind, Options, + Parser, Tag, +}; + +use crate::config::MarkkitBuilder; +use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell}; +use crate::limits::Budget; + +/// Parsed Markdown document. +pub(crate) struct ParsedMarkdown { + pub doc: Document, + pub title: Option<String>, + pub paragraphs: usize, + pub tables: usize, +} + +enum Kind { + Root, + Para, + Heading(u8), + Quote, + List { + ordered: bool, + start: Option<u64>, + }, + Item, + Code { + language: Option<String>, + }, + Table { + alignments: Vec<Alignment>, + }, + Head, + Row, + Cell, + Em, + Strong, + Strike, + Link { + url: String, + title: Option<String>, + }, + Image { + src: String, + }, + Footnote { + label: String, + }, + DefList, + DefTerm, + DefDesc, + Html, + Meta, + /// Beyond `max_depth` or an unsupported container: children go to the parent. + Flat, +} + +struct Frame { + kind: Kind, + nodes: Vec<Node>, + items: Vec<ListItem>, + cells: Vec<Vec<Node>>, + rows: Vec<Vec<Vec<Node>>>, + defs: Vec<DefinitionItem>, + /// Task list marker seen in this item. + checked: Option<bool>, + /// Code, image alt, raw HTML or front matter text. + text: String, +} + +impl Frame { + fn new(kind: Kind) -> Self { + Self { + kind, + nodes: Vec::new(), + items: Vec::new(), + cells: Vec::new(), + rows: Vec::new(), + defs: Vec::new(), + checked: None, + text: String::new(), + } + } +} + +/// What a finished frame hands to its parent. +enum Out { + Nodes(Vec<Node>), + Item(ListItem), + Cell(Vec<Node>), + Row(Vec<Vec<Node>>), + Head(Vec<Vec<Node>>), + Term(Vec<Node>), + Desc(Vec<Node>), +} + +struct Builder<'c> { + #[cfg_attr(not(feature = "html"), allow(dead_code))] + config: &'c MarkkitBuilder, + #[cfg_attr(not(feature = "html"), allow(dead_code))] + budget: &'c Budget, + /// Max open containers kept as structure (`limits.max_nesting_depth`). + max_depth: usize, + /// First error from a nested parser (HTML block); aborts the parse. + error: Option<crate::Error>, + stack: Vec<Frame>, + meta_title: Option<String>, + paragraphs: usize, + tables: usize, +} + +/// Parse Markdown text into the IR. +pub(crate) fn parse( + text: &str, + config: &MarkkitBuilder, + budget: &Budget, +) -> crate::Result<ParsedMarkdown> { + let opts = Options::ENABLE_TABLES + | Options::ENABLE_STRIKETHROUGH + | Options::ENABLE_TASKLISTS + | Options::ENABLE_FOOTNOTES + | Options::ENABLE_DEFINITION_LIST + | Options::ENABLE_YAML_STYLE_METADATA_BLOCKS; + let mut b = Builder { + config, + budget, + max_depth: config.limits.nesting_depth(), + error: None, + stack: vec![Frame::new(Kind::Root)], + meta_title: None, + paragraphs: 0, + tables: 0, + }; + for event in Parser::new_ext(text, opts) { + if budget.expired() { + return Err(budget.timeout_error()); + } + if matches!(event, Event::Start(_)) { + budget.add_nodes(1)?; + } + b.event(event); + if let Some(e) = b.error.take() { + return Err(e); + } + } + while b.stack.len() > 1 { + b.end(); + } + if let Some(e) = b.error.take() { + return Err(e); + } + let root = b.stack.pop().map(|f| f.nodes).unwrap_or_default(); + let title = b.meta_title.or_else(|| first_h1(&root)); + Ok(ParsedMarkdown { + doc: Document::new(root), + title, + paragraphs: b.paragraphs, + tables: b.tables, + }) +} + +impl Builder<'_> { + fn top(&mut self) -> &mut Frame { + self.stack.last_mut().expect("root frame") + } + + fn event(&mut self, event: Event<'_>) { + match event { + Event::Start(tag) => self.start(tag), + Event::End(_) => self.end(), + Event::Text(t) => self.text(&t), + Event::Code(c) => self.push(Node::InlineCode(c.into_string())), + Event::Html(h) | Event::InlineHtml(h) => { + if matches!(self.top().kind, Kind::Html) { + self.top().text.push_str(&h); + } + } + Event::SoftBreak => self.text(" "), + Event::HardBreak => self.push(Node::LineBreak), + Event::Rule => self.push(Node::ThematicBreak), + Event::TaskListMarker(done) => { + // Marker belongs to the innermost open item; text fallback when + // the item was flattened past `max_depth`. + match self + .stack + .iter_mut() + .rev() + .find(|f| matches!(f.kind, Kind::Item)) + { + Some(item) => item.checked = Some(done), + None => self.text(if done { "[x] " } else { "[ ] " }), + } + } + Event::FootnoteReference(r) => self.push(Node::FootnoteReference { + label: r.into_string(), + }), + Event::InlineMath(m) | Event::DisplayMath(m) => { + self.push(Node::InlineCode(m.into_string())) + } + } + } + + fn start(&mut self, tag: Tag<'_>) { + let kind = if self.stack.len() > self.max_depth { + Kind::Flat + } else { + match tag { + Tag::Paragraph => Kind::Para, + Tag::Heading { level, .. } => Kind::Heading(heading_level(level)), + Tag::BlockQuote(_) => Kind::Quote, + Tag::CodeBlock(kind) => Kind::Code { + language: match kind { + CodeBlockKind::Fenced(info) => info + .split_whitespace() + .next() + .map(|s| s.chars().filter(|c| *c != '`').collect::<String>()) + .filter(|s| !s.is_empty()), + CodeBlockKind::Indented => None, + }, + }, + Tag::HtmlBlock => Kind::Html, + Tag::List(start) => Kind::List { + ordered: start.is_some(), + start, + }, + Tag::Item => Kind::Item, + Tag::Table(aligns) => Kind::Table { + alignments: aligns + .into_iter() + .map(|a| match a { + MdAlignment::None => Alignment::None, + MdAlignment::Left => Alignment::Left, + MdAlignment::Center => Alignment::Center, + MdAlignment::Right => Alignment::Right, + }) + .collect(), + }, + Tag::TableHead => Kind::Head, + Tag::TableRow => Kind::Row, + Tag::TableCell => Kind::Cell, + Tag::Emphasis => Kind::Em, + Tag::Strong => Kind::Strong, + Tag::Strikethrough => Kind::Strike, + Tag::Link { + dest_url, title, .. + } => Kind::Link { + url: dest_url.into_string(), + title: Some(title.into_string()).filter(|t| !t.is_empty()), + }, + Tag::Image { dest_url, .. } => Kind::Image { + src: dest_url.into_string(), + }, + Tag::FootnoteDefinition(label) => Kind::Footnote { + label: label.into_string(), + }, + Tag::DefinitionList => Kind::DefList, + Tag::DefinitionListTitle => Kind::DefTerm, + Tag::DefinitionListDefinition => Kind::DefDesc, + Tag::MetadataBlock(MetadataBlockKind::YamlStyle) => Kind::Meta, + _ => Kind::Flat, + } + }; + self.stack.push(Frame::new(kind)); + } + + fn end(&mut self) { + if self.stack.len() <= 1 { + return; + } + let f = self.stack.pop().expect("non-root frame"); + let out = match f.kind { + Kind::Root | Kind::Flat => Out::Nodes(f.nodes), + Kind::Para => { + self.paragraphs += 1; + Out::Nodes(vec![Node::paragraph(f.nodes)]) + } + Kind::Heading(level) => { + self.paragraphs += 1; + Out::Nodes(vec![Node::heading(level, f.nodes)]) + } + Kind::Quote => Out::Nodes(vec![Node::BlockQuote { + children: wrap_inline(f.nodes), + source: None, + }]), + Kind::List { ordered, start } => Out::Nodes(vec![Node::List { + ordered, + start, + items: f.items, + source: None, + }]), + Kind::Item => { + self.paragraphs += 1; + let has_block = f.nodes.iter().any(Node::is_block); + let children = if has_block { + wrap_inline(f.nodes) + } else { + f.nodes + }; + Out::Item(match f.checked { + Some(done) => ListItem::task(children, done), + None => ListItem::new(children), + }) + } + Kind::Code { language } => Out::Nodes(vec![Node::CodeBlock { + language, + code: f.text, + source: None, + }]), + Kind::Table { alignments } => { + self.tables += 1; + let cells = |row: Vec<Vec<Node>>| row.into_iter().map(TableCell::new).collect(); + let mut nodes = vec![Node::Table { + alignments, + headers: cells(f.cells), + rows: f.rows.into_iter().map(cells).collect(), + source: None, + }]; + nodes.extend(f.nodes); + Out::Nodes(nodes) + } + Kind::Head => Out::Head(f.cells), + Kind::Row => Out::Row(f.cells), + Kind::Cell => Out::Cell(f.nodes), + Kind::Em => Out::Nodes(vec![Node::Italic { children: f.nodes }]), + Kind::Strong => Out::Nodes(vec![Node::Bold { children: f.nodes }]), + Kind::Strike => Out::Nodes(vec![Node::Strikethrough { children: f.nodes }]), + Kind::Link { url, title } => Out::Nodes(vec![Node::Link { + url, + title, + children: f.nodes, + }]), + Kind::Image { src } => Out::Nodes(vec![Node::Image { src, alt: f.text }]), + Kind::Footnote { label } => Out::Nodes(vec![Node::FootnoteDefinition { + label, + children: wrap_inline(f.nodes), + source: None, + }]), + Kind::DefList => { + let mut nodes = Vec::new(); + if !f.defs.is_empty() { + nodes.push(Node::DefinitionList { + items: f.defs, + source: None, + }); + } + nodes.extend(f.nodes); + Out::Nodes(nodes) + } + Kind::DefTerm => Out::Term(f.nodes), + Kind::DefDesc => Out::Desc(wrap_inline(f.nodes)), + Kind::Html => Out::Nodes(self.html_block(&f.text)), + Kind::Meta => { + if self.meta_title.is_none() { + self.meta_title = yaml_title(&f.text); + } + Out::Nodes(Vec::new()) + } + }; + self.accept(out); + } + + /// Hand a finished frame's output to the new top frame. + fn accept(&mut self, out: Out) { + let parent = self.top(); + match (&parent.kind, out) { + (Kind::List { .. }, Out::Item(item)) => parent.items.push(item), + (Kind::List { .. }, Out::Nodes(nodes)) if !nodes.is_empty() => { + parent.items.push(ListItem::new(nodes)) + } + (Kind::Head | Kind::Row, Out::Cell(cell)) => parent.cells.push(cell), + (Kind::Table { .. }, Out::Head(cells)) => parent.cells = cells, + (Kind::Table { .. }, Out::Row(cells)) => parent.rows.push(cells), + (Kind::DefList, Out::Term(term)) => parent.defs.push(DefinitionItem::new(term, vec![])), + (Kind::DefList, Out::Desc(desc)) => match parent.defs.last_mut() { + Some(item) => item.definitions.push(desc), + None => parent.defs.push(DefinitionItem::new(vec![], vec![desc])), + }, + (_, Out::Nodes(nodes) | Out::Cell(nodes) | Out::Term(nodes) | Out::Desc(nodes)) => { + self.push_all(nodes) + } + (_, Out::Item(item)) => self.push_all(item.children), + (_, Out::Row(cells) | Out::Head(cells)) => { + for cell in cells { + self.push_all(cell); + self.push(Node::Text(" ".into())); + } + } + } + } + + fn push_all(&mut self, nodes: Vec<Node>) { + for n in nodes { + self.push(n); + } + } + + fn push(&mut self, node: Node) { + let top = self.top(); + match top.kind { + // Alt text, code and front matter take plain text only. + Kind::Image { .. } | Kind::Code { .. } => plain_text(&node, &mut top.text), + Kind::Html | Kind::Meta => {} + Kind::List { .. } => top.items.push(ListItem::new(vec![node])), + Kind::Head | Kind::Row => top.cells.push(vec![node]), + _ => top.nodes.push(node), + } + } + + fn text(&mut self, t: &str) { + let top = self.top(); + match top.kind { + Kind::Image { .. } | Kind::Code { .. } | Kind::Meta => top.text.push_str(t), + Kind::Html => {} + _ => { + if let Some(Node::Text(prev)) = top.nodes.last_mut() { + prev.push_str(t); + } else { + self.push(Node::Text(t.to_string())); + } + } + } + } + + #[cfg(feature = "html")] + fn html_block(&mut self, html: &str) -> Vec<Node> { + match crate::html::HtmlParser::with_budget(self.config, self.budget).parse(html) { + Ok(d) => d.nodes, + Err(e) => { + self.error.get_or_insert(e); + Vec::new() + } + } + } + + #[cfg(not(feature = "html"))] + fn html_block(&mut self, _html: &str) -> Vec<Node> { + Vec::new() + } +} + +fn heading_level(level: HeadingLevel) -> u8 { + match level { + HeadingLevel::H1 => 1, + HeadingLevel::H2 => 2, + HeadingLevel::H3 => 3, + HeadingLevel::H4 => 4, + HeadingLevel::H5 => 5, + HeadingLevel::H6 => 6, + } +} + +/// Group runs of inline nodes into paragraphs (for containers mixing both). +fn wrap_inline(nodes: Vec<Node>) -> Vec<Node> { + let mut out = Vec::with_capacity(nodes.len()); + let mut run = Vec::new(); + for n in nodes { + if n.is_block() { + if !run.is_empty() { + out.push(Node::paragraph(std::mem::take(&mut run))); + } + out.push(n); + } else { + run.push(n); + } + } + if !run.is_empty() { + out.push(Node::paragraph(run)); + } + out +} + +/// Append the plain text of a node (iteratively; nodes may be nested). +fn plain_text(node: &Node, out: &mut String) { + let mut stack = vec![node]; + while let Some(n) = stack.pop() { + match n { + Node::Text(t) | Node::InlineCode(t) => out.push_str(t), + Node::CodeBlock { code, .. } => out.push_str(code), + Node::Image { alt, .. } => out.push_str(alt), + Node::LineBreak => out.push(' '), + Node::Heading { children, .. } + | Node::Paragraph { children, .. } + | Node::BlockQuote { children, .. } + | Node::FootnoteDefinition { children, .. } + | Node::Figure { children, .. } + | Node::Bold { children } + | Node::Italic { children } + | Node::Strikethrough { children } + | Node::Link { children, .. } => stack.extend(children.iter().rev()), + Node::List { items, .. } => { + stack.extend(items.iter().rev().flat_map(|i| i.children.iter().rev())) + } + Node::Table { headers, rows, .. } => stack.extend( + headers + .iter() + .chain(rows.iter().flatten()) + .flat_map(|c| c.children.iter()) + .rev(), + ), + Node::DefinitionList { items, .. } => stack.extend( + items + .iter() + .flat_map(|i| i.term.iter().chain(i.definitions.iter().flatten())) + .rev(), + ), + _ => {} + } + } +} + +fn first_h1(nodes: &[Node]) -> Option<String> { + nodes.iter().find_map(|n| match n { + Node::Heading { level: 1, .. } => { + let mut t = String::new(); + plain_text(n, &mut t); + let t = t.trim(); + (!t.is_empty()).then(|| t.to_string()) + } + _ => None, + }) +} + +/// `title:` from YAML front matter (top-level key, scalar value only). +fn yaml_title(yaml: &str) -> Option<String> { + yaml.lines().find_map(|line| { + let v = line.strip_prefix("title:")?.trim(); + let v = v + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| v.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) + .unwrap_or(v) + .trim(); + (!v.is_empty()).then(|| v.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parsed(src: &str) -> ParsedMarkdown { + let cfg = MarkkitBuilder::default(); + parse(src, &cfg, &Budget::new(&cfg.limits)).unwrap() + } + + fn md(src: &str) -> String { + let p = parsed(src); + crate::render::MarkdownRenderer::new().render(&p.doc) + } + + #[test] + fn roundtrip_common_blocks() { + let src = "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt *x*](i.png)\n"; + assert_eq!( + md(src), + "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt x](i.png)\n" + ); + } + + #[test] + fn table_and_tasks() { + assert_eq!( + md("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"), + "| a | b |\n| --- | --- |\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n" + ); + } + + #[test] + fn ir_v2_extensions() { + let p = parsed( + "| l | c | r |\n|:--|:-:|--:|\n| 1 | 2 | 3 |\n\n- [x] done\n\nNote[^n].\n\n[^n]: The note.\n\nTerm\n: Definition\n", + ); + let Node::Table { alignments, .. } = &p.doc.nodes[0] else { + panic!("{:?}", p.doc.nodes[0]) + }; + assert_eq!( + alignments, + &[Alignment::Left, Alignment::Center, Alignment::Right] + ); + let Node::List { items, .. } = &p.doc.nodes[1] else { + panic!("{:?}", p.doc.nodes[1]) + }; + assert_eq!(items[0].checked, Some(true)); + assert!( + p.doc + .nodes + .iter() + .any(|n| matches!(n, Node::FootnoteDefinition { label, .. } if label == "n")) + ); + assert!( + p.doc + .nodes + .iter() + .any(|n| matches!(n, Node::DefinitionList { .. })) + ); + let out = crate::render::MarkdownRenderer::new().render(&p.doc); + assert!(out.contains("| :--- | :---: | ---: |"), "{out}"); + assert!(out.contains("- [x] done"), "{out}"); + assert!(out.contains("Note[^n]."), "{out}"); + assert!(out.contains("[^n]: The note."), "{out}"); + } + + #[test] + fn soft_breaks_reflow_hard_breaks_kept() { + assert_eq!(md("a\nb \nc\\\nd"), "a b \nc \nd\n"); + } + + #[test] + fn titles() { + let p = parsed("---\ntitle: \"Doc\"\n---\n\n# H\n"); + assert_eq!(p.title.as_deref(), Some("Doc")); + let p = parsed("## sub\n\n# Real *one*\n"); + assert_eq!(p.title.as_deref(), Some("Real one")); + assert_eq!(parsed("text").title, None); + } + + #[test] + fn loose_item_with_nested_list() { + assert_eq!(md("- a **b**\n - c\n"), "- a **b**\n - c\n"); + } + + #[test] + fn deep_nesting_is_flattened() { + let depth = crate::Limits::default().nesting_depth(); + let src = format!("{}x", "> ".repeat(depth * 5)); + let out = md(&src); + assert!(out.trim_end().ends_with('x')); + assert!(out.lines().next().unwrap().matches('>').count() <= depth + 1); + } + + #[test] + fn nesting_depth_follows_limits() { + let cfg = MarkkitBuilder::default().limits(crate::Limits::default().max_nesting_depth(3)); + let out = parse(&"> ".repeat(10), &cfg, &Budget::new(&cfg.limits)).unwrap(); + let rendered = crate::render::MarkdownRenderer::new().render(&out.doc); + assert!(rendered.matches('>').count() <= 4, "{rendered}"); + } + + #[test] + fn max_nodes_is_enforced() { + let cfg = MarkkitBuilder::default().limits(crate::Limits::default().max_nodes(10)); + let err = parse(&"a\n\n".repeat(50), &cfg, &Budget::new(&cfg.limits)) + .err() + .expect("node limit"); + assert!(matches!(err, crate::Error::ResourceLimit(_)), "{err}"); + } +} diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs new file mode 100644 index 0000000..1cd6f1d --- /dev/null +++ b/crates/markkit/src/text.rs @@ -0,0 +1,177 @@ +//! Plain text input, plus the text decoding shared with Markdown input. +//! +//! Decisions: +//! - Always compiled (no dependencies, no feature gate); selected only by a +//! content type (`text/plain`) or extension (`.txt`). Bytes are never +//! sniffed as text, so unknown binaries stay `UnsupportedFormat` instead +//! of rendering as mojibake. +//! - Decoding: UTF-8 (BOM stripped), UTF-16 LE/BE when a BOM is present; +//! anything else is decoded lossily with an `invalid_utf8` warning. +//! - C0 control characters other than tab/newline, and DEL, are dropped so +//! terminal escapes and NULs never reach the output; CRLF/CR become LF. +//! - Plain text goes through the IR: blank lines separate paragraphs, single +//! newlines become hard breaks, text stays `Node::Text` so renderer-level +//! escaping applies uniformly. + +use crate::ir::{Document, Node}; +use crate::limits::Budget; + +/// Decoded text plus non-fatal warnings. +pub(crate) struct Decoded { + pub text: String, + pub warnings: Vec<String>, +} + +/// Decode bytes to normalized text (see module docs). +pub(crate) fn decode(bytes: &[u8]) -> Decoded { + let mut warnings = Vec::new(); + let raw = if let Some(rest) = bytes.strip_prefix(b"\xEF\xBB\xBF") { + utf8(rest, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFF\xFE") { + utf16(rest, u16::from_le_bytes, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFE\xFF") { + utf16(rest, u16::from_be_bytes, &mut warnings) + } else { + utf8(bytes, &mut warnings) + }; + Decoded { + text: normalize(&raw), + warnings, + } +} + +fn utf8(bytes: &[u8], warnings: &mut Vec<String>) -> String { + match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => { + warnings.push("invalid_utf8: invalid UTF-8 sequences replaced with U+FFFD".into()); + String::from_utf8_lossy(bytes).into_owned() + } + } +} + +fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec<String>) -> String { + let units = bytes.as_chunks::<2>().0.iter().map(|c| read(*c)); + let mut lossy = !bytes.len().is_multiple_of(2); + let s: String = char::decode_utf16(units) + .map(|r| { + r.unwrap_or_else(|_| { + lossy = true; + char::REPLACEMENT_CHARACTER + }) + }) + .collect(); + if lossy { + warnings.push("invalid_utf8: invalid UTF-16 sequences replaced with U+FFFD".into()); + } + s +} + +/// CRLF/CR → LF; drop C0 controls (except `\t`, `\n`) and DEL. +fn normalize(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\r' => { + if chars.peek() == Some(&'\n') { + chars.next(); + } + out.push('\n'); + } + '\t' | '\n' => out.push(c), + c if c.is_ascii_control() => {} + c => out.push(c), + } + } + out +} + +/// Plain text → IR. Returns the document and the paragraph count. +/// Every line ticks the call's deadline and counts against `max_nodes`. +pub(crate) fn plain_to_ir(text: &str, budget: &Budget) -> crate::Result<(Document, usize)> { + let mut nodes = Vec::new(); + let mut para: Vec<Node> = Vec::new(); + let flush = |para: &mut Vec<Node>, nodes: &mut Vec<Node>| { + if !para.is_empty() { + nodes.push(Node::paragraph(std::mem::take(para))); + } + }; + for line in text.split('\n') { + if budget.expired() { + return Err(budget.timeout_error()); + } + budget.add_nodes(1)?; + let line = line.trim_end(); + if line.is_empty() { + flush(&mut para, &mut nodes); + continue; + } + if !para.is_empty() { + para.push(Node::LineBreak); + } + para.push(Node::Text(line.to_string())); + } + flush(&mut para, &mut nodes); + let count = nodes.len(); + Ok((Document::new(nodes), count)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ir(text: &str) -> (Document, usize) { + plain_to_ir(text, &Budget::new(&crate::Limits::default())).unwrap() + } + + #[test] + fn max_nodes_is_enforced() { + let budget = Budget::new(&crate::Limits::default().max_nodes(5)); + let err = plain_to_ir(&"a\n".repeat(10), &budget).unwrap_err(); + assert!(matches!(err, crate::Error::ResourceLimit(_)), "{err}"); + } + + #[test] + fn decode_strips_bom_and_controls() { + let d = decode(b"\xEF\xBB\xBFa\r\nb\rc\x00\x1b[31m\x7f\td"); + assert_eq!(d.text, "a\nb\nc[31m\td"); + assert!(d.warnings.is_empty()); + } + + #[test] + fn decode_utf16() { + let mut le = b"\xFF\xFE".to_vec(); + le.extend("hé".encode_utf16().flat_map(u16::to_le_bytes)); + assert_eq!(decode(&le).text, "hé"); + let mut be = b"\xFE\xFF".to_vec(); + be.extend("hé".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(decode(&be).text, "hé"); + // Odd length and lone surrogate are lossy, never panic. + let d = decode(b"\xFF\xFE\x00\xD8a"); + assert!(d.text.contains('\u{FFFD}')); + assert_eq!(d.warnings.len(), 1); + } + + #[test] + fn decode_invalid_utf8_warns() { + let d = decode(b"ok \xFF\xFE!"); + assert_eq!(d.text, "ok \u{FFFD}\u{FFFD}!"); + assert!(d.warnings[0].starts_with("invalid_utf8:")); + } + + #[test] + fn plain_paragraphs_and_breaks() { + let (doc, n) = ir("a\nb \n\n\n c\n"); + assert_eq!(n, 2); + assert_eq!( + doc.nodes[0], + Node::paragraph(vec![ + Node::Text("a".into()), + Node::LineBreak, + Node::Text("b".into()) + ]) + ); + assert_eq!(ir("\n \n").1, 0); + } +} diff --git a/crates/markkit/tests/text_integration.rs b/crates/markkit/tests/text_integration.rs new file mode 100644 index 0000000..56b9f7b --- /dev/null +++ b/crates/markkit/tests/text_integration.rs @@ -0,0 +1,271 @@ +//! Plain text and Markdown → Markdown integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; + +fn extract(bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync(bytes, hint) + .unwrap() +} + +#[test] +fn plain_text_by_content_type_and_extension() { + let e = extract( + b"Hello\nworld\n\nSecond paragraph\n", + DocumentHint::content_type("text/plain; charset=utf-8"), + ); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "Hello \nworld\n\nSecond paragraph\n"); + assert_eq!(e.stats.paragraphs, Some(2)); + assert_eq!(e.title, None); + assert!(e.warnings.is_empty()); + + let e = extract(b"notes", DocumentHint::file_name("dir/NOTES.TXT")); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "notes\n"); +} + +#[test] +fn plain_text_crlf_and_blank_runs() { + let e = extract( + b"a\r\nb\r\n\r\n\r\n\r\nc \r\n", + DocumentHint::file_name("x.txt"), + ); + assert_eq!(e.markdown, "a \nb\n\nc\n"); +} + +#[test] +fn plain_text_empty() { + let e = extract(b"", DocumentHint::file_name("empty.txt")); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.paragraphs, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_normalized_with_title() { + let src = "\u{feff}---\ntitle: Guide\n---\n\n# Intro\n\nSome *text*\nwrapped.\n\n```rust\nfn main() {}\n```\n\n* item\n\n\n"; + let e = extract(src.as_bytes(), DocumentHint::file_name("guide.md")); + assert_eq!(e.format, Format::Markdown); + assert_eq!(e.title.as_deref(), Some("Guide")); + assert_eq!( + e.markdown, + "# Intro\n\nSome *text* wrapped.\n\n```rust\nfn main() {}\n```\n\n- item\n" + ); + assert_eq!(e.stats.tables, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_title_from_heading() { + for ct in ["text/markdown", "text/x-markdown; charset=utf-8"] { + let e = extract(b"# Title\n\nbody", DocumentHint::content_type(ct)); + assert_eq!(e.format, Format::Markdown, "{ct}"); + assert_eq!(e.title.as_deref(), Some("Title")); + assert_eq!(e.markdown, "# Title\n\nbody\n"); + } +} + +#[cfg(not(feature = "markdown"))] +#[test] +fn markdown_requires_feature() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# x", DocumentHint::file_name("a.md")) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[test] +fn text_is_never_sniffed_without_hint() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# just text", DocumentHint::default()) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[tokio::test] +async fn extract_bytes_text_plain() { + let m = Markkit::builder().build(); + let md = m.extract_bytes(b"x\ny", Some("text/plain")).await; + assert_eq!(md.unwrap(), "x \ny\n"); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +/// File names for every text-like format compiled in. +fn names() -> Vec<&'static str> { + let mut v = vec!["a.txt"]; + if cfg!(feature = "markdown") { + v.push("a.md"); + } + v +} + +fn has_control(s: &str) -> bool { + s.chars() + .any(|c| c.is_ascii_control() && c != '\n' && c != '\t') +} + +#[test] +fn security_control_chars_stripped() { + for name in names() { + let e = extract( + b"safe\x1b]0;pwned\x07\x00text\x7f\x08", + DocumentHint::file_name(name), + ); + assert!(!has_control(&e.markdown), "{name}: {:?}", e.markdown); + assert_eq!(e.markdown, "safe\\]0;pwnedtext\n"); + } +} + +#[test] +fn security_invalid_utf8_lossy_with_warning() { + let e = extract(b"caf\xE9 \xC3", DocumentHint::file_name("a.txt")); + assert_eq!(e.markdown, "caf\u{FFFD} \u{FFFD}\n"); + assert_eq!(e.warnings.len(), 1); + assert!(e.warnings[0].starts_with("invalid_utf8:")); +} + +#[test] +fn security_utf16_malformed_no_panic() { + for bytes in [ + &b"\xFF\xFE"[..], + b"\xFF\xFE\x41", + b"\xFE\xFF\xD8\x00\xD8\x00", + b"\xFF\xFE\x00\xDC", + ] { + let e = extract(bytes, DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= 16, "{bytes:?}"); + } +} + +#[test] +fn security_input_size_limit() { + let m = Markkit::builder().max_input_size(8).build(); + for name in ["a.txt", "a.md"] { + let err = m + .extract_document_sync(b"0123456789", DocumentHint::file_name(name)) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}"); + } +} + +#[test] +fn security_output_bounded_by_input() { + // Each line becomes a hard break (" \n" = +2 bytes per line): output + // stays linear in input size. + let input = "a\n".repeat(100_000); + let e = extract(input.as_bytes(), DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= input.len() * 2); + let blanks = "\n".repeat(1_000_000); + for name in names() { + let e = extract(blanks.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown, "", "{name}"); + } +} + +#[test] +fn security_long_line() { + let line = "x".repeat(2_000_000); + for name in names() { + let e = extract(line.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown.len(), line.len() + 1, "{name}"); + } +} + +#[test] +fn security_arbitrary_bytes_never_panic() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + for _ in 0..300 { + let len = (seed % 512) as usize; + let bytes: Vec<u8> = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Bias towards Markdown syntax so the parser is exercised. + b"#*_`[]()!<>|-~\n \\x\x00\x1b\xff"[(seed % 21) as usize] + }) + .collect(); + for name in names() { + let e = extract(&bytes, DocumentHint::file_name(name)); + assert!(!has_control(&e.markdown), "{name}: {bytes:?}"); + } + } +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_raw_html_never_passes_through() { + let src = b"<script>alert(1)</script>\n\n<div onclick=\"x()\">Hi <b>there</b></div>\n\npara <img src=x onerror=alert(1)> <span>kept</span>\n\n<style>p{}</style>\n"; + let e = extract(src, DocumentHint::file_name("a.md")); + for bad in [ + "<script", "alert(1)", "onclick", "onerror", "<style", "<img", "<span", "<div", + ] { + assert!(!e.markdown.contains(bad), "{bad}: {:?}", e.markdown); + } + assert!(e.markdown.contains("kept"), "{:?}", e.markdown); + #[cfg(feature = "html")] + assert!(e.markdown.contains("Hi **there**"), "{:?}", e.markdown); +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_deep_nesting_no_stack_overflow() { + for src in [ + format!("{}x", "> ".repeat(100_000)), + format!("{}x", "- ".repeat(50_000)), + format!("{}x{}", "*".repeat(50_000), "*".repeat(50_000)), + format!("{}x{}", "[".repeat(50_000), "](u)".repeat(50_000)), + ] { + let e = extract(src.as_bytes(), DocumentHint::file_name("a.md")); + assert!(e.markdown.contains('x')); + } +} + +#[test] +fn security_text_and_markdown_share_limits() { + use std::time::Duration; + let m = Markkit::builder().timeout(Duration::ZERO).build(); + for name in ["a.txt", "a.md"] { + let r = m.extract_document_sync(b"x", DocumentHint::file_name(name)); + if cfg!(not(feature = "markdown")) && name == "a.md" { + continue; + } + assert!( + matches!(&r, Err(Error::ResourceLimit(msg)) if msg.contains("timeout")), + "{name}: {r:?}" + ); + } + let m = Markkit::builder().max_nodes(100).build(); + let big = "line\n".repeat(1000); + for name in ["a.txt", "a.md"] { + if cfg!(not(feature = "markdown")) && name == "a.md" { + continue; + } + let r = m.extract_document_sync(big.as_bytes(), DocumentHint::file_name(name)); + let r2 = m.extract_document_sync( + "p\n\n".repeat(1000).as_bytes(), + DocumentHint::file_name(name), + ); + assert!( + matches!(&r, Err(Error::ResourceLimit(_))) + || matches!(&r2, Err(Error::ResourceLimit(_))), + "{name}: {r:?}" + ); + } +} + +#[test] +fn security_plain_text_markup_is_escaped() { + let e = extract( + b"# not heading\n<img src=x onerror=alert(1)>", + DocumentHint::file_name("a.txt"), + ); + assert!(e.markdown.contains("\\<img"), "{:?}", e.markdown); + assert!(e.markdown.starts_with("\\#"), "{:?}", e.markdown); +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 0ae6deb..436d7ba 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -1,13 +1,14 @@ --- type: Subsystem Design title: Format Support -description: Supported and planned formats, bytes-in API, format detection, DOCX and spreadsheet mapping. +description: Supported and planned formats, bytes-in API, format detection, DOCX, spreadsheet, plain text and Markdown mapping. tags: - markkit - formats - docx - spreadsheet - detection + - markdown --- # Format Support @@ -16,11 +17,12 @@ tags: ### Tier 1 (Core) - **HTML/XHTML**: Full support via html5ever + scraper (`html` feature, default) -- **Plain text**: Planned pass-through (currently `UnsupportedFormat` via the bytes API) +- **Plain text**: Always on, no dependencies (`text/plain`, `.txt`, `.text`) ### Tier 2 (Feature-gated) - **DOCX** (`docx`): Word documents via a direct `zip` + `quick-xml` reader (no docx-rs) - **XLSX/XLSM, XLS, ODS** (`spreadsheet`): Workbooks to Markdown tables via calamine +- **Markdown** (`markdown`): Re-parsed with pulldown-cmark into the IR and rendered back (`text/markdown`, `text/x-markdown`, `.md`, `.markdown`, `.mdown`, `.mkd`, `.mkdn`) - **PDF** (`pdf`): Planned. Returns `Error::UnsupportedFormat` (never an empty success) ### Detected, not supported @@ -30,7 +32,6 @@ tags: - **EPUB**: E-book extraction - **RTF**: Rich text format - **CSV/TSV**: Tabular data to markdown tables -- **Markdown**: Normalize/clean existing markdown ## Bytes-in API @@ -72,7 +73,7 @@ malformed property part yields `None` without failing the extraction. only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. -Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. +Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`, `invalid_utf8`. ## Format Detection @@ -90,6 +91,9 @@ Priority order (`Format::detect`): - `%PDF-` within the first 1 KiB → PDF. - `<!doctype html`, `<html`, `<head`, `<body` (after BOM/whitespace) → HTML. +Plain text and Markdown are never sniffed: they need a content type or extension, so +unlabelled bytes stay `UnsupportedFormat` rather than rendering binary as text. + Spreadsheet parsing re-sniffs the container (XLSX vs XLS vs ODS), so a workbook mislabelled as another Excel type still parses. @@ -129,6 +133,32 @@ numbering through the main part's relationships. - Only cached cell values are read: formulas are not evaluated, VBA/XLM macros are never executed or rendered. +## Plain Text and Markdown + +Shared decoding: UTF-8 (BOM stripped); UTF-16 LE/BE only with a BOM; otherwise lossy UTF-8 +with an `invalid_utf8` warning. CRLF/CR become LF; C0 controls other than tab/newline, and +DEL, are dropped (no terminal escapes or NULs in output). + +Plain text goes through the IR: blank lines separate paragraphs (`stats.paragraphs`), +single newlines become hard breaks (` \n`), trailing whitespace is trimmed, text stays +`Node::Text`. No title. + +Markdown is parsed, not passed through: + +| Markdown | Output | +|---|---| +| CommonMark blocks and inlines | Same construct via the IR (`*` bullets become `-`, setext headings become ATX) | +| GFM tables (with `:---:` alignment), `~~strike~~`, task lists `[x]` | IR v2 `Table.alignments`, `Strikethrough`, `ListItem::checked` | +| Footnotes `[^n]`, definition lists (`Term` / `: def`) | IR v2 `FootnoteReference` / `FootnoteDefinition`, `DefinitionList` | +| Soft line break | Space (paragraphs reflow); hard breaks kept | +| YAML front matter | Not rendered; `title:` becomes `Extraction::title` (else first `# ` heading) | +| HTML block | Through the HTML parser when `html` is on (scripts, styles, handlers stripped), else dropped | +| Inline HTML tag | Dropped; text between tags kept | +| Math, wikilinks, other extensions | Not enabled (literal text) | + +Containers nested deeper than `max_nesting_depth` (256) are flattened into the deepest kept container +(explicit-stack builder, so neither the builder nor the recursive renderer can overflow). + ## Resource Limits All formats enforce: diff --git a/knowledge/log.md b/knowledge/log.md index a728d37..d6177cb 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -3,6 +3,7 @@ ## 2026-10-27 * **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. +* **Plain text and Markdown input**: text is always on and goes through the IR; Markdown (`markdown` feature) is re-parsed with pulldown-cmark rather than passed through, so raw HTML never reaches output. Neither is sniffed from bytes. Shares the call's `Limits` budget (timeout, nodes, nesting depth). New threat T7 and invariant S18. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md). ## 2026-10-26 diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 2be08e1..b83ad64 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -20,6 +20,7 @@ tags: | `crates/markkit/tests/output_safety.rs` | Renderer escaping, URL scheme allowlist, fences; pulldown-cmark oracle + proptests | | `crates/markkit/tests/html_proptest.rs` | Property tests: no panic, no script/style, determinism, limits | | `crates/markkit/tests/docx_integration.rs`, `spreadsheet_integration.rs` | Office mapping | +| `crates/markkit/tests/text_integration.rs` | Plain text and Markdown mapping, T7 security tests | | `crates/markkit/tests/document_detection.rs` | Format detection order and explicit unsupported errors | | `crates/markkit/tests/document_security.rs` | Zip bombs, grid amplification, XML entities, malformed containers | | `crates/markkit/tests/limits.rs` | Every `Limits` field enforced for HTML, DOCX, spreadsheets; small-stack depth ceiling | @@ -55,7 +56,7 @@ each security test states exactly which bytes it feeds the parser. `ci.yml` runs fmt, clippy, rustdoc (`-D warnings`), the knowledge bundle check, `cargo audit` + `cargo deny`, `cargo test --workspace` (default and all features), -the `docx,spreadsheet` no-HTTP build with a reqwest leak check, and every example. +the `docx,spreadsheet,markdown` no-HTTP build with a reqwest leak check, and every example. `nightly.yml` repeats build + test on nightly Rust daily; `coverage.yml` uploads tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). All must pass before merge. diff --git a/knowledge/runtimes/wasm-package.md b/knowledge/runtimes/wasm-package.md index 3887068..e1b65e9 100644 --- a/knowledge/runtimes/wasm-package.md +++ b/knowledge/runtimes/wasm-package.md @@ -20,7 +20,7 @@ tags: browsers (incl. iframes), edge workers, Node, Deno and Bun. - **Separate from the napi package** (`@everruns/markkit`), which stays the fast native option for Node on Linux x64. -- **Features**: `html`, `docx`, `spreadsheet`, `tool`. No `http_client`: reqwest +- **Features**: `html`, `docx`, `spreadsheet`, `markdown`, `tool`. No `http_client`: reqwest needs sockets, and the host fetching keeps network policy (SSRF) with the host. The tool therefore offers only `html` and `content_base64`. diff --git a/knowledge/security/security-model.md b/knowledge/security/security-model.md index fecb3b1..f2998c8 100644 --- a/knowledge/security/security-model.md +++ b/knowledge/security/security-model.md @@ -38,6 +38,11 @@ Markkit processes untrusted input from the internet. Key threats: - **Compatibility**: ZIP entry-name decoding errors propagate; Unicode and legacy CP437 part names preserve bounded case-insensitive lookup - See [Threat Model](threat-model.md) T6 +### TM-TXT: Text and Markdown input +- **Mitigation**: Selected only by content type / extension, total decoding, control characters dropped +- **Mitigation**: Markdown re-parsed into the IR; raw HTML never copied (HTML blocks go through the HTML parser) +- See [Threat Model](threat-model.md) T7 + ### TM-MEM: Memory exhaustion - **Mitigation**: Streaming processing where possible - **Mitigation**: Memory budgets for large documents diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 18a7367..469def6 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -296,6 +296,34 @@ process, or smuggling content past a human reviewer. - `spreadsheet_integration::xlsx_hidden_sheet_skipped_by_default`, `spreadsheet_integration::xlsm_with_vba_project_extracts_values_only` +### T7: Text and Markdown Input + +**Description**: Plain text or Markdown carrying terminal escapes, NULs, invalid encodings, +raw HTML (`<script>`, event handlers) or pathological nesting. + +**Impact**: Medium -- raw HTML passed through re-introduces T1; deep nesting can crash. + +**Mitigations**: +- Text and Markdown are never sniffed; a content type or extension must select them +- Decoding is total: invalid UTF-8/UTF-16 is replaced (U+FFFD) with an `invalid_utf8` warning +- C0 controls (except tab/newline) and DEL are dropped before parsing +- Markdown is re-parsed (pulldown-cmark) into the IR; raw HTML is never copied. HTML blocks + go through the HTML parser (T1 stripping) or are dropped; inline tags are dropped +- Explicit-stack IR builder, nesting capped at `max_nesting_depth` (deeper content flattened); + the call's `timeout` and `max_nodes` budget covers decoding, parsing and HTML blocks + +**Residual**: none specific to text. Literal text that looks like markup (a `<script>` line +in a `.txt`) is emitted as `Node::Text` and escaped by the renderer (S17), like every format. + +**Test Coverage** (`text_integration`): +- `security_control_chars_stripped`, `security_arbitrary_bytes_never_panic` +- `security_invalid_utf8_lossy_with_warning`, `security_utf16_malformed_no_panic` +- `security_markdown_raw_html_never_passes_through` +- `security_markdown_deep_nesting_no_stack_overflow`, `security_long_line` +- `security_input_size_limit`, `security_output_bounded_by_input` +- `security_text_and_markdown_share_limits` (timeout, `max_nodes`) +- `security_plain_text_markup_is_escaped` + ### T5: Information Disclosure **Description**: Error messages or output leak sensitive information. @@ -317,7 +345,7 @@ process, or smuggling content past a human reviewer. | Output size | 128 MiB | `max_output_size(usize)` | Renderer, per block / list item / table row | | Processing time | 60 s | `timeout(Duration)`, `max_processing_secs(u64)` | One budget per call: fetch, parse, render (cooperative) | | Parsed nodes | 5 000 000 | `max_nodes(usize)` | After html5ever (DOM count); while building DOCX XML trees | -| Nesting depth | 256 (hard max 512) | `max_nesting_depth(usize)` | HTML walk (flatten); DOCX XML build (reject) | +| Nesting depth | 256 (hard max 512) | `max_nesting_depth(usize)` | HTML walk and Markdown IR build (flatten); DOCX XML build (reject) | | Redirects (`extract_url`) | 5 | `max_redirects(usize)` | Every redirect hop | | Fetched body (`extract_url`) | = input size | `max_input_size(usize)` | `Content-Length` and while streaming | | Table span grid | 1 000 000 slots | Hardcoded `MAX_TABLE_GRID_CELLS` | Renderer table layout (spans dropped past it) | @@ -372,6 +400,7 @@ process, or smuggling content past a human reviewer. | S15 | No configuration of `max_nesting_depth` can overflow the stack | `limits::*hard_ceiling_survives_small_stack` | | S16 | Extraction stops with `ResourceLimit` once `timeout` passes (checked at least after each non-interruptible third-party step) | `limits::*zero_timeout_rejects`, `limits::tests::*` | | S17 | Rendered markdown has no raw HTML, no non-allowlisted link/image scheme, and document text cannot create Markdown structure | `output_safety::*` (pulldown-cmark oracle + proptests) | +| S18 | Text and Markdown output never contains C0 controls (except tab/newline); Markdown raw HTML (blocks, inline tags) is never copied | `text_integration::security_*` | ## See also