diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f9a9f5f..676561f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,9 +82,9 @@ jobs:
echo "HTTP client leaked into the default feature set"
exit 1
fi
- cargo test -p markkit --no-default-features --features docx,spreadsheet
- if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'reqwest|hyper'; then
- echo "HTTP client leaked into the docx,spreadsheet feature set"
+ cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown
+ if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then
+ echo "HTTP client leaked into the docx,spreadsheet,markdown feature set"
exit 1
fi
- name: Run examples
@@ -112,7 +112,7 @@ jobs:
with:
node-version: "24.22.1"
- name: Check core for wasm32 (no warnings)
- run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,tool --target wasm32-unknown-unknown
+ run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,markdown,tool --target wasm32-unknown-unknown
- name: Install wasm-bindgen-cli (matches Cargo.lock)
run: bash scripts/install-wasm-bindgen.sh
- name: Build package
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40c03f4..4822d07 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
### Added
+- Plain text input (`text/plain`, `.txt`), always on: paragraphs and hard breaks through the IR,
+ UTF-8/UTF-16 BOM decoding, control characters dropped, `invalid_utf8` warning on lossy decode.
+- Markdown input (`markdown` feature, in `all_formats`; `text/markdown`, `.md`): re-parsed with
+ pulldown-cmark and rendered back, so raw HTML never passes through. Front matter `title:` or the
+ first `# ` heading becomes `Extraction::title`. Available in the CLI, Python, Node and WASM packages; `format` serializes as
+ `"text"` and `"markdown"`.
- HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or
`
`) and the new `Extraction::metadata` (description, author, published date, language,
canonical URL, site name, lead image) from ``, Open Graph, `` and
diff --git a/Cargo.toml b/Cargo.toml
index e0df15c..17be211 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -52,6 +52,7 @@ calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS)
zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip
quick-xml = "0.42" # DOCX/ODS XML parsing
cfb = "0.15" # XLS (OLE2) container pre-scan
+pulldown-cmark = { version = "0.13", default-features = false } # Markdown input; output-safety test oracle
# Logging
tracing = "0.1"
@@ -73,7 +74,6 @@ napi-derive = "3"
tempfile = "3"
criterion = { version = "0.8", features = ["html_reports"] }
proptest = "1"
-pulldown-cmark = { version = "0.13", default-features = false }
[profile.release]
lto = "thin"
diff --git a/README.md b/README.md
index e8a273d..79909c7 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,7 @@ package for browsers and edge runtimes.
- **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default
- **DOCX** – Headings, lists, tables, links, emphasis
- **Spreadsheets** – XLSX/XLSM, XLS and ODS workbooks to markdown tables, one section per sheet
+- **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through)
- **Format detection** – Content type, then file extension, then magic bytes
- **URL fetching** – Fetch and extract in one call (`http_client` feature)
- **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits
@@ -32,6 +33,8 @@ package for browsers and edge runtimes.
| HTML / XHTML | Supported | `html` (default) |
| DOCX | Supported | `docx` |
| XLSX, XLSM, XLS, ODS | Supported | `spreadsheet` |
+| Plain text | Supported | always on |
+| Markdown | Supported | `markdown` |
| PDF | Planned | `pdf` |
| DOC, PPTX, XLSB | Detected, rejected explicitly | – |
@@ -181,7 +184,8 @@ let markkit = Markkit::builder()
| `docx` | No | DOCX conversion (zip + quick-xml) |
| `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) |
| `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` |
-| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` |
+| `markdown` | No | Markdown normalization (pulldown-cmark) |
+| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet`, `markdown` |
| `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) |
| `logging` | No | Reserved: pulls in `tracing`, emits no events yet |
diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml
index 69495f0..b7bcc30 100644
--- a/crates/markkit-cli/Cargo.toml
+++ b/crates/markkit-cli/Cargo.toml
@@ -24,6 +24,7 @@ all_formats = ["markkit/all_formats"]
pdf = ["markkit/pdf"]
docx = ["markkit/docx"]
spreadsheet = ["markkit/spreadsheet"]
+markdown = ["markkit/markdown"]
[dependencies]
markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] }
diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs
index 1b2e9c8..006b86f 100644
--- a/crates/markkit-cli/tests/cli_integration.rs
+++ b/crates/markkit-cli/tests/cli_integration.rs
@@ -103,6 +103,37 @@ fn extract_from_file() {
assert!(stdout.contains("Paragraph"), "output: {stdout}");
}
+#[test]
+fn extract_text_and_markdown_files() {
+ let dir = tempfile::tempdir().unwrap();
+ let cases = [
+ (
+ "notes.txt",
+ "line one\nline two\n",
+ "line one \nline two\n",
+ ),
+ (
+ "doc.md",
+ "Title\n=====\n\n* a\n\n\n",
+ "# Title\n\n- a\n",
+ ),
+ ];
+ for (name, input, expected) in cases {
+ let file_path = dir.path().join(name);
+ std::fs::write(&file_path, input).unwrap();
+ let output = markkit_bin()
+ .args(["extract", "--file", file_path.to_str().unwrap()])
+ .output()
+ .expect("failed to run markkit");
+ assert!(
+ output.status.success(),
+ "{name} stderr: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert_eq!(String::from_utf8_lossy(&output.stdout), expected, "{name}");
+ }
+}
+
#[test]
fn extract_json_output() {
let mut child = markkit_bin()
diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs
index 9f1f3bf..84483c9 100644
--- a/crates/markkit-js/src/lib.rs
+++ b/crates/markkit-js/src/lib.rs
@@ -90,7 +90,7 @@ pub struct Metadata {
pub struct Extraction {
pub markdown: String,
pub title: Option,
- /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`).
+ /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`).
pub format: String,
pub warnings: Vec,
pub stats: ExtractionStats,
diff --git a/crates/markkit-wasm/Cargo.toml b/crates/markkit-wasm/Cargo.toml
index 7e00540..857ae55 100644
--- a/crates/markkit-wasm/Cargo.toml
+++ b/crates/markkit-wasm/Cargo.toml
@@ -28,6 +28,7 @@ markkit = { path = "../markkit", default-features = false, features = [
"html",
"docx",
"spreadsheet",
+ "markdown",
"tool",
] }
wasm-bindgen = "0.2"
diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md
index 30c020a..dcc03bd 100644
--- a/crates/markkit-wasm/README.md
+++ b/crates/markkit-wasm/README.md
@@ -1,7 +1,8 @@
# @everruns/markkit-wasm
-Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX and
-spreadsheets (XLSX/XLSM, XLS, ODS) in the **browser and any JavaScript runtime**:
+Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX,
+spreadsheets (XLSX/XLSM, XLS, ODS), plain text and Markdown in the **browser
+and any JavaScript runtime**:
edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun.
Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP`
diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs
index 3c80df0..0458d6e 100644
--- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs
+++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs
@@ -74,6 +74,17 @@ test("extractDocument exposes page metadata in camelCase", () => {
assert.equal(out.metadata.site_name, undefined);
});
+test("extractDocument reads plain text and Markdown", () => {
+ const mk = new Markkit();
+ const txt = mk.extractDocument(new TextEncoder().encode("a\nb"), { fileName: "n.txt" });
+ assert.equal(txt.format, "text");
+ assert.equal(txt.markdown, "a \nb\n");
+ const md = mk.extractDocument(new TextEncoder().encode("# T\n\n\n"), { fileName: "d.md" });
+ assert.equal(md.format, "markdown");
+ assert.equal(md.title, "T");
+ assert.ok(!md.markdown.includes("script"));
+});
+
test("extractBytes and unsupported formats", () => {
const mk = new Markkit();
assert.equal(mk.extractBytes(new TextEncoder().encode("
x
"), "text/html"), "x\n");
diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml
index 2d179f5..34f30e1 100644
--- a/crates/markkit/Cargo.toml
+++ b/crates/markkit/Cargo.toml
@@ -19,6 +19,7 @@ html = ["dep:scraper", "dep:html5ever", "dep:ego-tree", "dep:url"]
pdf = []
docx = ["dep:zip", "dep:quick-xml"]
spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"]
+markdown = ["dep:pulldown-cmark"]
# Opt-in: the default build has no network stack (no reqwest/hyper/TLS).
http_client = ["dep:reqwest", "tokio/net"]
logging = ["dep:tracing"]
@@ -26,7 +27,7 @@ logging = ["dep:tracing"]
tool = ["dep:tower", "dep:futures-core", "dep:base64"]
# JSON Schema for the IR (`markkit::ir::json_schema()`)
json-schema = ["dep:schemars"]
-all_formats = ["html", "pdf", "docx", "spreadsheet"]
+all_formats = ["html", "pdf", "docx", "spreadsheet", "markdown"]
[dependencies]
# Async
@@ -54,6 +55,9 @@ quick-xml = { workspace = true, optional = true }
calamine = { workspace = true, optional = true }
cfb = { workspace = true, optional = true }
+# Markdown (optional)
+pulldown-cmark = { workspace = true, optional = true }
+
# HTTP client (optional)
reqwest = { workspace = true, optional = true }
diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs
index f657cf9..ae904ee 100644
--- a/crates/markkit/src/extract.rs
+++ b/crates/markkit/src/extract.rs
@@ -15,12 +15,14 @@ pub struct Extraction {
pub markdown: String,
/// Document title when known: HTML `og:title` / JSON-LD / ``; DOCX
/// core properties or `Title` style; XLSX `docProps/core.xml`, ODS
- /// `meta.xml`, XLS `SummaryInformation`.
+ /// `meta.xml`, XLS `SummaryInformation`;
+ /// Markdown front matter `title:` or first `# ` heading.
pub title: Option,
/// Format the document was parsed as.
pub format: Format,
/// Non-fatal issues, each prefixed with a stable code
- /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`).
+ /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`,
+ /// `invalid_utf8:`).
pub warnings: Vec,
/// Format-specific counters.
pub stats: ExtractionStats,
@@ -65,9 +67,9 @@ pub struct ExtractionStats {
pub sheets: Option,
/// Data rows rendered across all sheets (header rows excluded).
pub rows: Option,
- /// Non-empty paragraphs, headings and list items (DOCX).
+ /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown).
pub paragraphs: Option,
- /// Tables (DOCX).
+ /// Tables (DOCX, Markdown).
pub tables: Option,
}
@@ -91,10 +93,6 @@ pub struct ParsedDocument {
}
impl ParsedDocument {
- #[cfg_attr(
- not(any(feature = "html", feature = "docx", feature = "spreadsheet")),
- allow(dead_code)
- )]
fn new(document: Document, format: Format) -> Self {
Self {
document,
@@ -226,7 +224,7 @@ impl Markkit {
.map(|e| e.markdown)
}
- /// Convert a document (HTML, DOCX, XLSX/XLS/ODS) to Markdown with metadata.
+ /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to Markdown with metadata.
///
/// The format is detected from `hint.content_type`, then the extension of
/// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline;
@@ -267,7 +265,7 @@ impl Markkit {
Ok(parsed.render())
}
- /// Parse a document (HTML, DOCX, XLSX/XLS/ODS) into the IR without
+ /// Parse a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) into the IR without
/// rendering. Same detection and errors as [`Markkit::extract_document`].
pub async fn parse_document(
&self,
@@ -301,6 +299,8 @@ impl Markkit {
Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => {
self.parse_spreadsheet(bytes, budget)
}
+ Format::PlainText => self.parse_plain_text(bytes, budget),
+ Format::Markdown => self.parse_markdown(bytes, budget),
Format::Doc => Err(Error::UnsupportedFormat(
"legacy Word 97-2003 (.doc) documents are not supported".into(),
)),
@@ -333,6 +333,37 @@ impl Markkit {
))
}
+ fn parse_plain_text(&self, bytes: &[u8], budget: &Budget) -> crate::Result {
+ let decoded = crate::text::decode(bytes);
+ let (mut doc, paragraphs) = crate::text::plain_to_ir(&decoded.text, budget)?;
+ self.config.transforms.apply(&mut doc)?;
+ let mut p = ParsedDocument::new(doc, Format::PlainText);
+ p.warnings = decoded.warnings;
+ p.stats.paragraphs = Some(paragraphs);
+ Ok(p)
+ }
+
+ #[cfg(feature = "markdown")]
+ fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result {
+ let decoded = crate::text::decode(bytes);
+ let out = crate::markdown::parse(&decoded.text, &self.config, budget)?;
+ let mut doc = out.doc;
+ self.config.transforms.apply(&mut doc)?;
+ let mut p = ParsedDocument::new(doc, Format::Markdown);
+ p.title = out.title;
+ p.warnings = decoded.warnings;
+ p.stats.paragraphs = Some(out.paragraphs);
+ p.stats.tables = Some(out.tables);
+ Ok(p)
+ }
+
+ #[cfg(not(feature = "markdown"))]
+ fn parse_markdown(&self, _bytes: &[u8], _budget: &Budget) -> crate::Result {
+ Err(Error::UnsupportedFormat(
+ "Markdown support requires the 'markdown' feature".to_string(),
+ ))
+ }
+
#[cfg(feature = "docx")]
fn parse_docx(&self, bytes: &[u8], budget: &Budget) -> crate::Result {
let out = guarded("DOCX", Error::DocxExtract, || {
diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs
index 509604c..fea7b95 100644
--- a/crates/markkit/src/format.rs
+++ b/crates/markkit/src/format.rs
@@ -31,7 +31,11 @@ pub enum Format {
/// Office Open XML presentation (`.pptx`). Detected but not supported.
Pptx,
Csv,
+ /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes.
+ /// Serialized as `"text"`, same as `as_str`.
+ #[serde(rename = "text")]
PlainText,
+ /// Markdown (`text/markdown`, `.md`), passed through normalized.
Markdown,
Unknown,
}
@@ -115,7 +119,7 @@ impl Format {
Format::Csv
} else if ct.contains("text/plain") {
Format::PlainText
- } else if ct.contains("text/markdown") {
+ } else if ct.contains("text/markdown") || ct.contains("text/x-markdown") {
Format::Markdown
} else {
Format::Unknown
@@ -135,8 +139,8 @@ impl Format {
"ods" => Format::Ods,
"pptx" | "pptm" | "ppt" => Format::Pptx,
"csv" | "tsv" => Format::Csv,
- "txt" => Format::PlainText,
- "md" | "markdown" => Format::Markdown,
+ "txt" | "text" => Format::PlainText,
+ "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown,
_ => Format::Unknown,
}
}
@@ -245,6 +249,28 @@ fn looks_like_html(bytes: &[u8]) -> bool {
mod tests {
use super::*;
+ #[test]
+ fn serde_name_matches_as_str() {
+ for f in [
+ Format::Html,
+ Format::Pdf,
+ Format::Docx,
+ Format::Doc,
+ Format::Xlsx,
+ Format::Xls,
+ Format::Ods,
+ Format::Pptx,
+ Format::Csv,
+ Format::PlainText,
+ Format::Markdown,
+ Format::Unknown,
+ ] {
+ let json = serde_json::to_string(&f).unwrap();
+ assert_eq!(json, format!("\"{}\"", f.as_str()));
+ assert_eq!(serde_json::from_str::(&json).unwrap(), f);
+ }
+ }
+
#[test]
fn test_content_type_detection() {
assert_eq!(
diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs
index 3a1f2bc..afb874d 100644
--- a/crates/markkit/src/lib.rs
+++ b/crates/markkit/src/lib.rs
@@ -34,6 +34,8 @@
//! - **HTML** (`html`, default): Convert HTML to markdown with configurable rules
//! - **DOCX** (`docx`): Convert Word documents
//! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables
+//! - **Plain text** (always on): `text/plain`, `.txt`
+//! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR
//! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`]
//! - **URL fetching** (`http_client`, opt-in): Fetch and extract from URLs via reqwest
//! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the
@@ -58,9 +60,12 @@ pub mod format;
pub mod html;
pub mod ir;
pub mod limits;
+#[cfg(feature = "markdown")]
+mod markdown;
pub mod render;
#[cfg(feature = "spreadsheet")]
mod spreadsheet;
+mod text;
#[cfg(feature = "tool")]
pub mod tool;
pub mod transform;
diff --git a/crates/markkit/src/markdown.rs b/crates/markkit/src/markdown.rs
new file mode 100644
index 0000000..405d926
--- /dev/null
+++ b/crates/markkit/src/markdown.rs
@@ -0,0 +1,668 @@
+//! Markdown input: parse with pulldown-cmark into the IR, render back.
+//!
+//! Decisions:
+//! - Markdown is re-parsed, not passed through, so every output goes through
+//! the same IR and renderer (and the same output-safety rules). Raw HTML is
+//! never copied: HTML blocks go through the HTML parser when the `html`
+//! feature is on (scripts/styles stripped), otherwise they are dropped;
+//! inline HTML tags are dropped and the text between them kept.
+//! - Extensions map onto IR v2: GFM tables (with column alignment),
+//! strikethrough, task lists (`ListItem::checked`), footnotes, definition
+//! lists, YAML front matter. Front matter is not rendered; its `title:`
+//! becomes the title, otherwise the first level-1 heading.
+//! - Soft line breaks become spaces (paragraphs reflow); hard breaks stay.
+//! - pulldown-cmark is iterative; the IR is built with an explicit stack and
+//! capped at `limits.max_nesting_depth` open containers. Deeper content is
+//! flattened into the deepest container so the recursive renderer cannot
+//! overflow.
+//! - Shares the call's `Budget`: every event ticks the deadline, every start
+//! tag counts against `max_nodes`, and HTML blocks use the same budget.
+
+use pulldown_cmark::{
+ Alignment as MdAlignment, CodeBlockKind, Event, HeadingLevel, MetadataBlockKind, Options,
+ Parser, Tag,
+};
+
+use crate::config::MarkkitBuilder;
+use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell};
+use crate::limits::Budget;
+
+/// Parsed Markdown document.
+pub(crate) struct ParsedMarkdown {
+ pub doc: Document,
+ pub title: Option,
+ pub paragraphs: usize,
+ pub tables: usize,
+}
+
+enum Kind {
+ Root,
+ Para,
+ Heading(u8),
+ Quote,
+ List {
+ ordered: bool,
+ start: Option,
+ },
+ Item,
+ Code {
+ language: Option,
+ },
+ Table {
+ alignments: Vec,
+ },
+ Head,
+ Row,
+ Cell,
+ Em,
+ Strong,
+ Strike,
+ Link {
+ url: String,
+ title: Option,
+ },
+ Image {
+ src: String,
+ },
+ Footnote {
+ label: String,
+ },
+ DefList,
+ DefTerm,
+ DefDesc,
+ Html,
+ Meta,
+ /// Beyond `max_depth` or an unsupported container: children go to the parent.
+ Flat,
+}
+
+struct Frame {
+ kind: Kind,
+ nodes: Vec,
+ items: Vec,
+ cells: Vec>,
+ rows: Vec>>,
+ defs: Vec,
+ /// Task list marker seen in this item.
+ checked: Option,
+ /// Code, image alt, raw HTML or front matter text.
+ text: String,
+}
+
+impl Frame {
+ fn new(kind: Kind) -> Self {
+ Self {
+ kind,
+ nodes: Vec::new(),
+ items: Vec::new(),
+ cells: Vec::new(),
+ rows: Vec::new(),
+ defs: Vec::new(),
+ checked: None,
+ text: String::new(),
+ }
+ }
+}
+
+/// What a finished frame hands to its parent.
+enum Out {
+ Nodes(Vec),
+ Item(ListItem),
+ Cell(Vec),
+ Row(Vec>),
+ Head(Vec>),
+ Term(Vec),
+ Desc(Vec),
+}
+
+struct Builder<'c> {
+ #[cfg_attr(not(feature = "html"), allow(dead_code))]
+ config: &'c MarkkitBuilder,
+ #[cfg_attr(not(feature = "html"), allow(dead_code))]
+ budget: &'c Budget,
+ /// Max open containers kept as structure (`limits.max_nesting_depth`).
+ max_depth: usize,
+ /// First error from a nested parser (HTML block); aborts the parse.
+ error: Option,
+ stack: Vec,
+ meta_title: Option,
+ paragraphs: usize,
+ tables: usize,
+}
+
+/// Parse Markdown text into the IR.
+pub(crate) fn parse(
+ text: &str,
+ config: &MarkkitBuilder,
+ budget: &Budget,
+) -> crate::Result {
+ let opts = Options::ENABLE_TABLES
+ | Options::ENABLE_STRIKETHROUGH
+ | Options::ENABLE_TASKLISTS
+ | Options::ENABLE_FOOTNOTES
+ | Options::ENABLE_DEFINITION_LIST
+ | Options::ENABLE_YAML_STYLE_METADATA_BLOCKS;
+ let mut b = Builder {
+ config,
+ budget,
+ max_depth: config.limits.nesting_depth(),
+ error: None,
+ stack: vec![Frame::new(Kind::Root)],
+ meta_title: None,
+ paragraphs: 0,
+ tables: 0,
+ };
+ for event in Parser::new_ext(text, opts) {
+ if budget.expired() {
+ return Err(budget.timeout_error());
+ }
+ if matches!(event, Event::Start(_)) {
+ budget.add_nodes(1)?;
+ }
+ b.event(event);
+ if let Some(e) = b.error.take() {
+ return Err(e);
+ }
+ }
+ while b.stack.len() > 1 {
+ b.end();
+ }
+ if let Some(e) = b.error.take() {
+ return Err(e);
+ }
+ let root = b.stack.pop().map(|f| f.nodes).unwrap_or_default();
+ let title = b.meta_title.or_else(|| first_h1(&root));
+ Ok(ParsedMarkdown {
+ doc: Document::new(root),
+ title,
+ paragraphs: b.paragraphs,
+ tables: b.tables,
+ })
+}
+
+impl Builder<'_> {
+ fn top(&mut self) -> &mut Frame {
+ self.stack.last_mut().expect("root frame")
+ }
+
+ fn event(&mut self, event: Event<'_>) {
+ match event {
+ Event::Start(tag) => self.start(tag),
+ Event::End(_) => self.end(),
+ Event::Text(t) => self.text(&t),
+ Event::Code(c) => self.push(Node::InlineCode(c.into_string())),
+ Event::Html(h) | Event::InlineHtml(h) => {
+ if matches!(self.top().kind, Kind::Html) {
+ self.top().text.push_str(&h);
+ }
+ }
+ Event::SoftBreak => self.text(" "),
+ Event::HardBreak => self.push(Node::LineBreak),
+ Event::Rule => self.push(Node::ThematicBreak),
+ Event::TaskListMarker(done) => {
+ // Marker belongs to the innermost open item; text fallback when
+ // the item was flattened past `max_depth`.
+ match self
+ .stack
+ .iter_mut()
+ .rev()
+ .find(|f| matches!(f.kind, Kind::Item))
+ {
+ Some(item) => item.checked = Some(done),
+ None => self.text(if done { "[x] " } else { "[ ] " }),
+ }
+ }
+ Event::FootnoteReference(r) => self.push(Node::FootnoteReference {
+ label: r.into_string(),
+ }),
+ Event::InlineMath(m) | Event::DisplayMath(m) => {
+ self.push(Node::InlineCode(m.into_string()))
+ }
+ }
+ }
+
+ fn start(&mut self, tag: Tag<'_>) {
+ let kind = if self.stack.len() > self.max_depth {
+ Kind::Flat
+ } else {
+ match tag {
+ Tag::Paragraph => Kind::Para,
+ Tag::Heading { level, .. } => Kind::Heading(heading_level(level)),
+ Tag::BlockQuote(_) => Kind::Quote,
+ Tag::CodeBlock(kind) => Kind::Code {
+ language: match kind {
+ CodeBlockKind::Fenced(info) => info
+ .split_whitespace()
+ .next()
+ .map(|s| s.chars().filter(|c| *c != '`').collect::())
+ .filter(|s| !s.is_empty()),
+ CodeBlockKind::Indented => None,
+ },
+ },
+ Tag::HtmlBlock => Kind::Html,
+ Tag::List(start) => Kind::List {
+ ordered: start.is_some(),
+ start,
+ },
+ Tag::Item => Kind::Item,
+ Tag::Table(aligns) => Kind::Table {
+ alignments: aligns
+ .into_iter()
+ .map(|a| match a {
+ MdAlignment::None => Alignment::None,
+ MdAlignment::Left => Alignment::Left,
+ MdAlignment::Center => Alignment::Center,
+ MdAlignment::Right => Alignment::Right,
+ })
+ .collect(),
+ },
+ Tag::TableHead => Kind::Head,
+ Tag::TableRow => Kind::Row,
+ Tag::TableCell => Kind::Cell,
+ Tag::Emphasis => Kind::Em,
+ Tag::Strong => Kind::Strong,
+ Tag::Strikethrough => Kind::Strike,
+ Tag::Link {
+ dest_url, title, ..
+ } => Kind::Link {
+ url: dest_url.into_string(),
+ title: Some(title.into_string()).filter(|t| !t.is_empty()),
+ },
+ Tag::Image { dest_url, .. } => Kind::Image {
+ src: dest_url.into_string(),
+ },
+ Tag::FootnoteDefinition(label) => Kind::Footnote {
+ label: label.into_string(),
+ },
+ Tag::DefinitionList => Kind::DefList,
+ Tag::DefinitionListTitle => Kind::DefTerm,
+ Tag::DefinitionListDefinition => Kind::DefDesc,
+ Tag::MetadataBlock(MetadataBlockKind::YamlStyle) => Kind::Meta,
+ _ => Kind::Flat,
+ }
+ };
+ self.stack.push(Frame::new(kind));
+ }
+
+ fn end(&mut self) {
+ if self.stack.len() <= 1 {
+ return;
+ }
+ let f = self.stack.pop().expect("non-root frame");
+ let out = match f.kind {
+ Kind::Root | Kind::Flat => Out::Nodes(f.nodes),
+ Kind::Para => {
+ self.paragraphs += 1;
+ Out::Nodes(vec![Node::paragraph(f.nodes)])
+ }
+ Kind::Heading(level) => {
+ self.paragraphs += 1;
+ Out::Nodes(vec![Node::heading(level, f.nodes)])
+ }
+ Kind::Quote => Out::Nodes(vec![Node::BlockQuote {
+ children: wrap_inline(f.nodes),
+ source: None,
+ }]),
+ Kind::List { ordered, start } => Out::Nodes(vec![Node::List {
+ ordered,
+ start,
+ items: f.items,
+ source: None,
+ }]),
+ Kind::Item => {
+ self.paragraphs += 1;
+ let has_block = f.nodes.iter().any(Node::is_block);
+ let children = if has_block {
+ wrap_inline(f.nodes)
+ } else {
+ f.nodes
+ };
+ Out::Item(match f.checked {
+ Some(done) => ListItem::task(children, done),
+ None => ListItem::new(children),
+ })
+ }
+ Kind::Code { language } => Out::Nodes(vec![Node::CodeBlock {
+ language,
+ code: f.text,
+ source: None,
+ }]),
+ Kind::Table { alignments } => {
+ self.tables += 1;
+ let cells = |row: Vec>| row.into_iter().map(TableCell::new).collect();
+ let mut nodes = vec![Node::Table {
+ alignments,
+ headers: cells(f.cells),
+ rows: f.rows.into_iter().map(cells).collect(),
+ source: None,
+ }];
+ nodes.extend(f.nodes);
+ Out::Nodes(nodes)
+ }
+ Kind::Head => Out::Head(f.cells),
+ Kind::Row => Out::Row(f.cells),
+ Kind::Cell => Out::Cell(f.nodes),
+ Kind::Em => Out::Nodes(vec![Node::Italic { children: f.nodes }]),
+ Kind::Strong => Out::Nodes(vec![Node::Bold { children: f.nodes }]),
+ Kind::Strike => Out::Nodes(vec![Node::Strikethrough { children: f.nodes }]),
+ Kind::Link { url, title } => Out::Nodes(vec![Node::Link {
+ url,
+ title,
+ children: f.nodes,
+ }]),
+ Kind::Image { src } => Out::Nodes(vec![Node::Image { src, alt: f.text }]),
+ Kind::Footnote { label } => Out::Nodes(vec![Node::FootnoteDefinition {
+ label,
+ children: wrap_inline(f.nodes),
+ source: None,
+ }]),
+ Kind::DefList => {
+ let mut nodes = Vec::new();
+ if !f.defs.is_empty() {
+ nodes.push(Node::DefinitionList {
+ items: f.defs,
+ source: None,
+ });
+ }
+ nodes.extend(f.nodes);
+ Out::Nodes(nodes)
+ }
+ Kind::DefTerm => Out::Term(f.nodes),
+ Kind::DefDesc => Out::Desc(wrap_inline(f.nodes)),
+ Kind::Html => Out::Nodes(self.html_block(&f.text)),
+ Kind::Meta => {
+ if self.meta_title.is_none() {
+ self.meta_title = yaml_title(&f.text);
+ }
+ Out::Nodes(Vec::new())
+ }
+ };
+ self.accept(out);
+ }
+
+ /// Hand a finished frame's output to the new top frame.
+ fn accept(&mut self, out: Out) {
+ let parent = self.top();
+ match (&parent.kind, out) {
+ (Kind::List { .. }, Out::Item(item)) => parent.items.push(item),
+ (Kind::List { .. }, Out::Nodes(nodes)) if !nodes.is_empty() => {
+ parent.items.push(ListItem::new(nodes))
+ }
+ (Kind::Head | Kind::Row, Out::Cell(cell)) => parent.cells.push(cell),
+ (Kind::Table { .. }, Out::Head(cells)) => parent.cells = cells,
+ (Kind::Table { .. }, Out::Row(cells)) => parent.rows.push(cells),
+ (Kind::DefList, Out::Term(term)) => parent.defs.push(DefinitionItem::new(term, vec![])),
+ (Kind::DefList, Out::Desc(desc)) => match parent.defs.last_mut() {
+ Some(item) => item.definitions.push(desc),
+ None => parent.defs.push(DefinitionItem::new(vec![], vec![desc])),
+ },
+ (_, Out::Nodes(nodes) | Out::Cell(nodes) | Out::Term(nodes) | Out::Desc(nodes)) => {
+ self.push_all(nodes)
+ }
+ (_, Out::Item(item)) => self.push_all(item.children),
+ (_, Out::Row(cells) | Out::Head(cells)) => {
+ for cell in cells {
+ self.push_all(cell);
+ self.push(Node::Text(" ".into()));
+ }
+ }
+ }
+ }
+
+ fn push_all(&mut self, nodes: Vec) {
+ for n in nodes {
+ self.push(n);
+ }
+ }
+
+ fn push(&mut self, node: Node) {
+ let top = self.top();
+ match top.kind {
+ // Alt text, code and front matter take plain text only.
+ Kind::Image { .. } | Kind::Code { .. } => plain_text(&node, &mut top.text),
+ Kind::Html | Kind::Meta => {}
+ Kind::List { .. } => top.items.push(ListItem::new(vec![node])),
+ Kind::Head | Kind::Row => top.cells.push(vec![node]),
+ _ => top.nodes.push(node),
+ }
+ }
+
+ fn text(&mut self, t: &str) {
+ let top = self.top();
+ match top.kind {
+ Kind::Image { .. } | Kind::Code { .. } | Kind::Meta => top.text.push_str(t),
+ Kind::Html => {}
+ _ => {
+ if let Some(Node::Text(prev)) = top.nodes.last_mut() {
+ prev.push_str(t);
+ } else {
+ self.push(Node::Text(t.to_string()));
+ }
+ }
+ }
+ }
+
+ #[cfg(feature = "html")]
+ fn html_block(&mut self, html: &str) -> Vec {
+ match crate::html::HtmlParser::with_budget(self.config, self.budget).parse(html) {
+ Ok(d) => d.nodes,
+ Err(e) => {
+ self.error.get_or_insert(e);
+ Vec::new()
+ }
+ }
+ }
+
+ #[cfg(not(feature = "html"))]
+ fn html_block(&mut self, _html: &str) -> Vec {
+ Vec::new()
+ }
+}
+
+fn heading_level(level: HeadingLevel) -> u8 {
+ match level {
+ HeadingLevel::H1 => 1,
+ HeadingLevel::H2 => 2,
+ HeadingLevel::H3 => 3,
+ HeadingLevel::H4 => 4,
+ HeadingLevel::H5 => 5,
+ HeadingLevel::H6 => 6,
+ }
+}
+
+/// Group runs of inline nodes into paragraphs (for containers mixing both).
+fn wrap_inline(nodes: Vec) -> Vec {
+ let mut out = Vec::with_capacity(nodes.len());
+ let mut run = Vec::new();
+ for n in nodes {
+ if n.is_block() {
+ if !run.is_empty() {
+ out.push(Node::paragraph(std::mem::take(&mut run)));
+ }
+ out.push(n);
+ } else {
+ run.push(n);
+ }
+ }
+ if !run.is_empty() {
+ out.push(Node::paragraph(run));
+ }
+ out
+}
+
+/// Append the plain text of a node (iteratively; nodes may be nested).
+fn plain_text(node: &Node, out: &mut String) {
+ let mut stack = vec![node];
+ while let Some(n) = stack.pop() {
+ match n {
+ Node::Text(t) | Node::InlineCode(t) => out.push_str(t),
+ Node::CodeBlock { code, .. } => out.push_str(code),
+ Node::Image { alt, .. } => out.push_str(alt),
+ Node::LineBreak => out.push(' '),
+ Node::Heading { children, .. }
+ | Node::Paragraph { children, .. }
+ | Node::BlockQuote { children, .. }
+ | Node::FootnoteDefinition { children, .. }
+ | Node::Figure { children, .. }
+ | Node::Bold { children }
+ | Node::Italic { children }
+ | Node::Strikethrough { children }
+ | Node::Link { children, .. } => stack.extend(children.iter().rev()),
+ Node::List { items, .. } => {
+ stack.extend(items.iter().rev().flat_map(|i| i.children.iter().rev()))
+ }
+ Node::Table { headers, rows, .. } => stack.extend(
+ headers
+ .iter()
+ .chain(rows.iter().flatten())
+ .flat_map(|c| c.children.iter())
+ .rev(),
+ ),
+ Node::DefinitionList { items, .. } => stack.extend(
+ items
+ .iter()
+ .flat_map(|i| i.term.iter().chain(i.definitions.iter().flatten()))
+ .rev(),
+ ),
+ _ => {}
+ }
+ }
+}
+
+fn first_h1(nodes: &[Node]) -> Option {
+ nodes.iter().find_map(|n| match n {
+ Node::Heading { level: 1, .. } => {
+ let mut t = String::new();
+ plain_text(n, &mut t);
+ let t = t.trim();
+ (!t.is_empty()).then(|| t.to_string())
+ }
+ _ => None,
+ })
+}
+
+/// `title:` from YAML front matter (top-level key, scalar value only).
+fn yaml_title(yaml: &str) -> Option {
+ yaml.lines().find_map(|line| {
+ let v = line.strip_prefix("title:")?.trim();
+ let v = v
+ .strip_prefix('"')
+ .and_then(|v| v.strip_suffix('"'))
+ .or_else(|| v.strip_prefix('\'').and_then(|v| v.strip_suffix('\'')))
+ .unwrap_or(v)
+ .trim();
+ (!v.is_empty()).then(|| v.to_string())
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn parsed(src: &str) -> ParsedMarkdown {
+ let cfg = MarkkitBuilder::default();
+ parse(src, &cfg, &Budget::new(&cfg.limits)).unwrap()
+ }
+
+ fn md(src: &str) -> String {
+ let p = parsed(src);
+ crate::render::MarkdownRenderer::new().render(&p.doc)
+ }
+
+ #[test]
+ fn roundtrip_common_blocks() {
+ let src = "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") \n";
+ assert_eq!(
+ md(src),
+ "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") \n"
+ );
+ }
+
+ #[test]
+ fn table_and_tasks() {
+ assert_eq!(
+ md("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"),
+ "| a | b |\n| --- | --- |\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"
+ );
+ }
+
+ #[test]
+ fn ir_v2_extensions() {
+ let p = parsed(
+ "| l | c | r |\n|:--|:-:|--:|\n| 1 | 2 | 3 |\n\n- [x] done\n\nNote[^n].\n\n[^n]: The note.\n\nTerm\n: Definition\n",
+ );
+ let Node::Table { alignments, .. } = &p.doc.nodes[0] else {
+ panic!("{:?}", p.doc.nodes[0])
+ };
+ assert_eq!(
+ alignments,
+ &[Alignment::Left, Alignment::Center, Alignment::Right]
+ );
+ let Node::List { items, .. } = &p.doc.nodes[1] else {
+ panic!("{:?}", p.doc.nodes[1])
+ };
+ assert_eq!(items[0].checked, Some(true));
+ assert!(
+ p.doc
+ .nodes
+ .iter()
+ .any(|n| matches!(n, Node::FootnoteDefinition { label, .. } if label == "n"))
+ );
+ assert!(
+ p.doc
+ .nodes
+ .iter()
+ .any(|n| matches!(n, Node::DefinitionList { .. }))
+ );
+ let out = crate::render::MarkdownRenderer::new().render(&p.doc);
+ assert!(out.contains("| :--- | :---: | ---: |"), "{out}");
+ assert!(out.contains("- [x] done"), "{out}");
+ assert!(out.contains("Note[^n]."), "{out}");
+ assert!(out.contains("[^n]: The note."), "{out}");
+ }
+
+ #[test]
+ fn soft_breaks_reflow_hard_breaks_kept() {
+ assert_eq!(md("a\nb \nc\\\nd"), "a b \nc \nd\n");
+ }
+
+ #[test]
+ fn titles() {
+ let p = parsed("---\ntitle: \"Doc\"\n---\n\n# H\n");
+ assert_eq!(p.title.as_deref(), Some("Doc"));
+ let p = parsed("## sub\n\n# Real *one*\n");
+ assert_eq!(p.title.as_deref(), Some("Real one"));
+ assert_eq!(parsed("text").title, None);
+ }
+
+ #[test]
+ fn loose_item_with_nested_list() {
+ assert_eq!(md("- a **b**\n - c\n"), "- a **b**\n - c\n");
+ }
+
+ #[test]
+ fn deep_nesting_is_flattened() {
+ let depth = crate::Limits::default().nesting_depth();
+ let src = format!("{}x", "> ".repeat(depth * 5));
+ let out = md(&src);
+ assert!(out.trim_end().ends_with('x'));
+ assert!(out.lines().next().unwrap().matches('>').count() <= depth + 1);
+ }
+
+ #[test]
+ fn nesting_depth_follows_limits() {
+ let cfg = MarkkitBuilder::default().limits(crate::Limits::default().max_nesting_depth(3));
+ let out = parse(&"> ".repeat(10), &cfg, &Budget::new(&cfg.limits)).unwrap();
+ let rendered = crate::render::MarkdownRenderer::new().render(&out.doc);
+ assert!(rendered.matches('>').count() <= 4, "{rendered}");
+ }
+
+ #[test]
+ fn max_nodes_is_enforced() {
+ let cfg = MarkkitBuilder::default().limits(crate::Limits::default().max_nodes(10));
+ let err = parse(&"a\n\n".repeat(50), &cfg, &Budget::new(&cfg.limits))
+ .err()
+ .expect("node limit");
+ assert!(matches!(err, crate::Error::ResourceLimit(_)), "{err}");
+ }
+}
diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs
new file mode 100644
index 0000000..1cd6f1d
--- /dev/null
+++ b/crates/markkit/src/text.rs
@@ -0,0 +1,177 @@
+//! Plain text input, plus the text decoding shared with Markdown input.
+//!
+//! Decisions:
+//! - Always compiled (no dependencies, no feature gate); selected only by a
+//! content type (`text/plain`) or extension (`.txt`). Bytes are never
+//! sniffed as text, so unknown binaries stay `UnsupportedFormat` instead
+//! of rendering as mojibake.
+//! - Decoding: UTF-8 (BOM stripped), UTF-16 LE/BE when a BOM is present;
+//! anything else is decoded lossily with an `invalid_utf8` warning.
+//! - C0 control characters other than tab/newline, and DEL, are dropped so
+//! terminal escapes and NULs never reach the output; CRLF/CR become LF.
+//! - Plain text goes through the IR: blank lines separate paragraphs, single
+//! newlines become hard breaks, text stays `Node::Text` so renderer-level
+//! escaping applies uniformly.
+
+use crate::ir::{Document, Node};
+use crate::limits::Budget;
+
+/// Decoded text plus non-fatal warnings.
+pub(crate) struct Decoded {
+ pub text: String,
+ pub warnings: Vec,
+}
+
+/// Decode bytes to normalized text (see module docs).
+pub(crate) fn decode(bytes: &[u8]) -> Decoded {
+ let mut warnings = Vec::new();
+ let raw = if let Some(rest) = bytes.strip_prefix(b"\xEF\xBB\xBF") {
+ utf8(rest, &mut warnings)
+ } else if let Some(rest) = bytes.strip_prefix(b"\xFF\xFE") {
+ utf16(rest, u16::from_le_bytes, &mut warnings)
+ } else if let Some(rest) = bytes.strip_prefix(b"\xFE\xFF") {
+ utf16(rest, u16::from_be_bytes, &mut warnings)
+ } else {
+ utf8(bytes, &mut warnings)
+ };
+ Decoded {
+ text: normalize(&raw),
+ warnings,
+ }
+}
+
+fn utf8(bytes: &[u8], warnings: &mut Vec) -> String {
+ match std::str::from_utf8(bytes) {
+ Ok(s) => s.to_string(),
+ Err(_) => {
+ warnings.push("invalid_utf8: invalid UTF-8 sequences replaced with U+FFFD".into());
+ String::from_utf8_lossy(bytes).into_owned()
+ }
+ }
+}
+
+fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec) -> String {
+ let units = bytes.as_chunks::<2>().0.iter().map(|c| read(*c));
+ let mut lossy = !bytes.len().is_multiple_of(2);
+ let s: String = char::decode_utf16(units)
+ .map(|r| {
+ r.unwrap_or_else(|_| {
+ lossy = true;
+ char::REPLACEMENT_CHARACTER
+ })
+ })
+ .collect();
+ if lossy {
+ warnings.push("invalid_utf8: invalid UTF-16 sequences replaced with U+FFFD".into());
+ }
+ s
+}
+
+/// CRLF/CR → LF; drop C0 controls (except `\t`, `\n`) and DEL.
+fn normalize(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut chars = s.chars().peekable();
+ while let Some(c) = chars.next() {
+ match c {
+ '\r' => {
+ if chars.peek() == Some(&'\n') {
+ chars.next();
+ }
+ out.push('\n');
+ }
+ '\t' | '\n' => out.push(c),
+ c if c.is_ascii_control() => {}
+ c => out.push(c),
+ }
+ }
+ out
+}
+
+/// Plain text → IR. Returns the document and the paragraph count.
+/// Every line ticks the call's deadline and counts against `max_nodes`.
+pub(crate) fn plain_to_ir(text: &str, budget: &Budget) -> crate::Result<(Document, usize)> {
+ let mut nodes = Vec::new();
+ let mut para: Vec = Vec::new();
+ let flush = |para: &mut Vec, nodes: &mut Vec| {
+ if !para.is_empty() {
+ nodes.push(Node::paragraph(std::mem::take(para)));
+ }
+ };
+ for line in text.split('\n') {
+ if budget.expired() {
+ return Err(budget.timeout_error());
+ }
+ budget.add_nodes(1)?;
+ let line = line.trim_end();
+ if line.is_empty() {
+ flush(&mut para, &mut nodes);
+ continue;
+ }
+ if !para.is_empty() {
+ para.push(Node::LineBreak);
+ }
+ para.push(Node::Text(line.to_string()));
+ }
+ flush(&mut para, &mut nodes);
+ let count = nodes.len();
+ Ok((Document::new(nodes), count))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ir(text: &str) -> (Document, usize) {
+ plain_to_ir(text, &Budget::new(&crate::Limits::default())).unwrap()
+ }
+
+ #[test]
+ fn max_nodes_is_enforced() {
+ let budget = Budget::new(&crate::Limits::default().max_nodes(5));
+ let err = plain_to_ir(&"a\n".repeat(10), &budget).unwrap_err();
+ assert!(matches!(err, crate::Error::ResourceLimit(_)), "{err}");
+ }
+
+ #[test]
+ fn decode_strips_bom_and_controls() {
+ let d = decode(b"\xEF\xBB\xBFa\r\nb\rc\x00\x1b[31m\x7f\td");
+ assert_eq!(d.text, "a\nb\nc[31m\td");
+ assert!(d.warnings.is_empty());
+ }
+
+ #[test]
+ fn decode_utf16() {
+ let mut le = b"\xFF\xFE".to_vec();
+ le.extend("hé".encode_utf16().flat_map(u16::to_le_bytes));
+ assert_eq!(decode(&le).text, "hé");
+ let mut be = b"\xFE\xFF".to_vec();
+ be.extend("hé".encode_utf16().flat_map(u16::to_be_bytes));
+ assert_eq!(decode(&be).text, "hé");
+ // Odd length and lone surrogate are lossy, never panic.
+ let d = decode(b"\xFF\xFE\x00\xD8a");
+ assert!(d.text.contains('\u{FFFD}'));
+ assert_eq!(d.warnings.len(), 1);
+ }
+
+ #[test]
+ fn decode_invalid_utf8_warns() {
+ let d = decode(b"ok \xFF\xFE!");
+ assert_eq!(d.text, "ok \u{FFFD}\u{FFFD}!");
+ assert!(d.warnings[0].starts_with("invalid_utf8:"));
+ }
+
+ #[test]
+ fn plain_paragraphs_and_breaks() {
+ let (doc, n) = ir("a\nb \n\n\n c\n");
+ assert_eq!(n, 2);
+ assert_eq!(
+ doc.nodes[0],
+ Node::paragraph(vec![
+ Node::Text("a".into()),
+ Node::LineBreak,
+ Node::Text("b".into())
+ ])
+ );
+ assert_eq!(ir("\n \n").1, 0);
+ }
+}
diff --git a/crates/markkit/tests/text_integration.rs b/crates/markkit/tests/text_integration.rs
new file mode 100644
index 0000000..56b9f7b
--- /dev/null
+++ b/crates/markkit/tests/text_integration.rs
@@ -0,0 +1,271 @@
+//! Plain text and Markdown → Markdown integration and security tests.
+
+use markkit::format::Format;
+use markkit::{DocumentHint, Error, Markkit};
+
+fn extract(bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction {
+ Markkit::builder()
+ .build()
+ .extract_document_sync(bytes, hint)
+ .unwrap()
+}
+
+#[test]
+fn plain_text_by_content_type_and_extension() {
+ let e = extract(
+ b"Hello\nworld\n\nSecond paragraph\n",
+ DocumentHint::content_type("text/plain; charset=utf-8"),
+ );
+ assert_eq!(e.format, Format::PlainText);
+ assert_eq!(e.markdown, "Hello \nworld\n\nSecond paragraph\n");
+ assert_eq!(e.stats.paragraphs, Some(2));
+ assert_eq!(e.title, None);
+ assert!(e.warnings.is_empty());
+
+ let e = extract(b"notes", DocumentHint::file_name("dir/NOTES.TXT"));
+ assert_eq!(e.format, Format::PlainText);
+ assert_eq!(e.markdown, "notes\n");
+}
+
+#[test]
+fn plain_text_crlf_and_blank_runs() {
+ let e = extract(
+ b"a\r\nb\r\n\r\n\r\n\r\nc \r\n",
+ DocumentHint::file_name("x.txt"),
+ );
+ assert_eq!(e.markdown, "a \nb\n\nc\n");
+}
+
+#[test]
+fn plain_text_empty() {
+ let e = extract(b"", DocumentHint::file_name("empty.txt"));
+ assert_eq!(e.markdown, "");
+ assert_eq!(e.stats.paragraphs, Some(0));
+}
+
+#[cfg(feature = "markdown")]
+#[test]
+fn markdown_normalized_with_title() {
+ let src = "\u{feff}---\ntitle: Guide\n---\n\n# Intro\n\nSome *text*\nwrapped.\n\n```rust\nfn main() {}\n```\n\n* item\n\n\n";
+ let e = extract(src.as_bytes(), DocumentHint::file_name("guide.md"));
+ assert_eq!(e.format, Format::Markdown);
+ assert_eq!(e.title.as_deref(), Some("Guide"));
+ assert_eq!(
+ e.markdown,
+ "# Intro\n\nSome *text* wrapped.\n\n```rust\nfn main() {}\n```\n\n- item\n"
+ );
+ assert_eq!(e.stats.tables, Some(0));
+}
+
+#[cfg(feature = "markdown")]
+#[test]
+fn markdown_title_from_heading() {
+ for ct in ["text/markdown", "text/x-markdown; charset=utf-8"] {
+ let e = extract(b"# Title\n\nbody", DocumentHint::content_type(ct));
+ assert_eq!(e.format, Format::Markdown, "{ct}");
+ assert_eq!(e.title.as_deref(), Some("Title"));
+ assert_eq!(e.markdown, "# Title\n\nbody\n");
+ }
+}
+
+#[cfg(not(feature = "markdown"))]
+#[test]
+fn markdown_requires_feature() {
+ let m = Markkit::builder().build();
+ let err = m
+ .extract_document_sync(b"# x", DocumentHint::file_name("a.md"))
+ .unwrap_err();
+ assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}");
+}
+
+#[test]
+fn text_is_never_sniffed_without_hint() {
+ let m = Markkit::builder().build();
+ let err = m
+ .extract_document_sync(b"# just text", DocumentHint::default())
+ .unwrap_err();
+ assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}");
+}
+
+#[tokio::test]
+async fn extract_bytes_text_plain() {
+ let m = Markkit::builder().build();
+ let md = m.extract_bytes(b"x\ny", Some("text/plain")).await;
+ assert_eq!(md.unwrap(), "x \ny\n");
+}
+
+// ─── Security (threat model T7) ─────────────────────────────────────
+
+/// File names for every text-like format compiled in.
+fn names() -> Vec<&'static str> {
+ let mut v = vec!["a.txt"];
+ if cfg!(feature = "markdown") {
+ v.push("a.md");
+ }
+ v
+}
+
+fn has_control(s: &str) -> bool {
+ s.chars()
+ .any(|c| c.is_ascii_control() && c != '\n' && c != '\t')
+}
+
+#[test]
+fn security_control_chars_stripped() {
+ for name in names() {
+ let e = extract(
+ b"safe\x1b]0;pwned\x07\x00text\x7f\x08",
+ DocumentHint::file_name(name),
+ );
+ assert!(!has_control(&e.markdown), "{name}: {:?}", e.markdown);
+ assert_eq!(e.markdown, "safe\\]0;pwnedtext\n");
+ }
+}
+
+#[test]
+fn security_invalid_utf8_lossy_with_warning() {
+ let e = extract(b"caf\xE9 \xC3", DocumentHint::file_name("a.txt"));
+ assert_eq!(e.markdown, "caf\u{FFFD} \u{FFFD}\n");
+ assert_eq!(e.warnings.len(), 1);
+ assert!(e.warnings[0].starts_with("invalid_utf8:"));
+}
+
+#[test]
+fn security_utf16_malformed_no_panic() {
+ for bytes in [
+ &b"\xFF\xFE"[..],
+ b"\xFF\xFE\x41",
+ b"\xFE\xFF\xD8\x00\xD8\x00",
+ b"\xFF\xFE\x00\xDC",
+ ] {
+ let e = extract(bytes, DocumentHint::file_name("a.txt"));
+ assert!(e.markdown.len() <= 16, "{bytes:?}");
+ }
+}
+
+#[test]
+fn security_input_size_limit() {
+ let m = Markkit::builder().max_input_size(8).build();
+ for name in ["a.txt", "a.md"] {
+ let err = m
+ .extract_document_sync(b"0123456789", DocumentHint::file_name(name))
+ .unwrap_err();
+ assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}");
+ }
+}
+
+#[test]
+fn security_output_bounded_by_input() {
+ // Each line becomes a hard break (" \n" = +2 bytes per line): output
+ // stays linear in input size.
+ let input = "a\n".repeat(100_000);
+ let e = extract(input.as_bytes(), DocumentHint::file_name("a.txt"));
+ assert!(e.markdown.len() <= input.len() * 2);
+ let blanks = "\n".repeat(1_000_000);
+ for name in names() {
+ let e = extract(blanks.as_bytes(), DocumentHint::file_name(name));
+ assert_eq!(e.markdown, "", "{name}");
+ }
+}
+
+#[test]
+fn security_long_line() {
+ let line = "x".repeat(2_000_000);
+ for name in names() {
+ let e = extract(line.as_bytes(), DocumentHint::file_name(name));
+ assert_eq!(e.markdown.len(), line.len() + 1, "{name}");
+ }
+}
+
+#[test]
+fn security_arbitrary_bytes_never_panic() {
+ let mut seed = 0x2545_F491_4F6C_DD1Du64;
+ for _ in 0..300 {
+ let len = (seed % 512) as usize;
+ let bytes: Vec = (0..len)
+ .map(|_| {
+ seed ^= seed << 13;
+ seed ^= seed >> 7;
+ seed ^= seed << 17;
+ // Bias towards Markdown syntax so the parser is exercised.
+ b"#*_`[]()!<>|-~\n \\x\x00\x1b\xff"[(seed % 21) as usize]
+ })
+ .collect();
+ for name in names() {
+ let e = extract(&bytes, DocumentHint::file_name(name));
+ assert!(!has_control(&e.markdown), "{name}: {bytes:?}");
+ }
+ }
+}
+
+#[cfg(feature = "markdown")]
+#[test]
+fn security_markdown_raw_html_never_passes_through() {
+ let src = b"\n\n
Hi there
\n\npara kept\n\n\n";
+ let e = extract(src, DocumentHint::file_name("a.md"));
+ for bad in [
+ "