feat(formats): add plain text and Markdown input

.github/workflows/ci.yml

@@ -82,9 +82,9 @@ jobs:
             echo "HTTP client leaked into the default feature set"
             exit 1
           fi
-          cargo test -p markkit --no-default-features --features docx,spreadsheet
-          if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'reqwest|hyper'; then
-            echo "HTTP client leaked into the docx,spreadsheet feature set"
+          cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown
+          if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then
+            echo "HTTP client leaked into the docx,spreadsheet,markdown feature set"
             exit 1
           fi
       - name: Run examples
@@ -112,7 +112,7 @@ jobs:
         with:
           node-version: "24.22.1"
       - name: Check core for wasm32 (no warnings)
-        run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,tool --target wasm32-unknown-unknown
+        run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,markdown,tool --target wasm32-unknown-unknown
       - name: Install wasm-bindgen-cli (matches Cargo.lock)
         run: bash scripts/install-wasm-bindgen.sh
       - name: Build package

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
+- Plain text input (`text/plain`, `.txt`), always on: paragraphs and hard breaks through the IR,
+  UTF-8/UTF-16 BOM decoding, control characters dropped, `invalid_utf8` warning on lossy decode.
+- Markdown input (`markdown` feature, in `all_formats`; `text/markdown`, `.md`): re-parsed with
+  pulldown-cmark and rendered back, so raw HTML never passes through. Front matter `title:` or the
+  first `# ` heading becomes `Extraction::title`. Available in the CLI, Python, Node and WASM packages; `format` serializes as
+  `"text"` and `"markdown"`.
 - HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or
   `<title>`) and the new `Extraction::metadata` (description, author, published date, language,
   canonical URL, site name, lead image) from `<meta>`, Open Graph, `<link rel=canonical>` and

Cargo.toml

@@ -52,6 +52,7 @@ calamine = "0.37"       # Excel/spreadsheet (XLSX/XLS/ODS)
 zip = { version = "9", default-features = false, features = ["deflate"] }  # OOXML/ODF containers; matches calamine's zip
 quick-xml = "0.42"      # DOCX/ODS XML parsing
 cfb = "0.15"            # XLS (OLE2) container pre-scan
+pulldown-cmark = { version = "0.13", default-features = false }  # Markdown input; output-safety test oracle
 
 # Logging
 tracing = "0.1"
@@ -73,7 +74,6 @@ napi-derive = "3"
 tempfile = "3"
 criterion = { version = "0.8", features = ["html_reports"] }
 proptest = "1"
-pulldown-cmark = { version = "0.13", default-features = false }
 
 [profile.release]
 lto = "thin"

README.md

@@ -20,6 +20,7 @@ package for browsers and edge runtimes.
 - **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default
 - **DOCX** – Headings, lists, tables, links, emphasis
 - **Spreadsheets** – XLSX/XLSM, XLS and ODS workbooks to markdown tables, one section per sheet
+- **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through)
 - **Format detection** – Content type, then file extension, then magic bytes
 - **URL fetching** – Fetch and extract in one call (`http_client` feature)
 - **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits
@@ -32,6 +33,8 @@ package for browsers and edge runtimes.
 | HTML / XHTML | Supported | `html` (default) |
 | DOCX | Supported | `docx` |
 | XLSX, XLSM, XLS, ODS | Supported | `spreadsheet` |
+| Plain text | Supported | always on |
+| Markdown | Supported | `markdown` |
 | PDF | Planned | `pdf` |
 | DOC, PPTX, XLSB | Detected, rejected explicitly | – |
 
@@ -181,7 +184,8 @@ let markkit = Markkit::builder()
 | `docx` | No | DOCX conversion (zip + quick-xml) |
 | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) |
 | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` |
-| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` |
+| `markdown` | No | Markdown normalization (pulldown-cmark) |
+| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet`, `markdown` |
 | `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) |
 | `logging` | No | Reserved: pulls in `tracing`, emits no events yet |
 

crates/markkit-cli/Cargo.toml

@@ -24,6 +24,7 @@ all_formats = ["markkit/all_formats"]
 pdf = ["markkit/pdf"]
 docx = ["markkit/docx"]
 spreadsheet = ["markkit/spreadsheet"]
+markdown = ["markkit/markdown"]
 
 [dependencies]
 markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] }

crates/markkit-cli/tests/cli_integration.rs

@@ -103,6 +103,37 @@ fn extract_from_file() {
     assert!(stdout.contains("Paragraph"), "output: {stdout}");
 }
 
+#[test]
+fn extract_text_and_markdown_files() {
+    let dir = tempfile::tempdir().unwrap();
+    let cases = [
+        (
+            "notes.txt",
+            "line one\nline two\n",
+            "line one  \nline two\n",
+        ),
+        (
+            "doc.md",
+            "Title\n=====\n\n* a\n\n<script>x()</script>\n",
+            "# Title\n\n- a\n",
+        ),
+    ];
+    for (name, input, expected) in cases {
+        let file_path = dir.path().join(name);
+        std::fs::write(&file_path, input).unwrap();
+        let output = markkit_bin()
+            .args(["extract", "--file", file_path.to_str().unwrap()])
+            .output()
+            .expect("failed to run markkit");
+        assert!(
+            output.status.success(),
+            "{name} stderr: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
+        assert_eq!(String::from_utf8_lossy(&output.stdout), expected, "{name}");
+    }
+}
+
 #[test]
 fn extract_json_output() {
     let mut child = markkit_bin()

crates/markkit-js/src/lib.rs

@@ -90,7 +90,7 @@ pub struct Metadata {
 pub struct Extraction {
     pub markdown: String,
     pub title: Option<String>,
-    /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`).
+    /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`).
     pub format: String,
     pub warnings: Vec<String>,
     pub stats: ExtractionStats,

crates/markkit-wasm/Cargo.toml

@@ -28,6 +28,7 @@ markkit = { path = "../markkit", default-features = false, features = [
     "html",
     "docx",
     "spreadsheet",
+    "markdown",
     "tool",
 ] }
 wasm-bindgen = "0.2"

crates/markkit-wasm/README.md

@@ -1,7 +1,8 @@
 # @everruns/markkit-wasm
 
-Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX and
-spreadsheets (XLSX/XLSM, XLS, ODS) in the **browser and any JavaScript runtime**:
+Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX,
+spreadsheets (XLSX/XLSM, XLS, ODS), plain text and Markdown in the **browser
+and any JavaScript runtime**:
 edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun.
 
 Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP`

crates/markkit-wasm/__test__/markkit-wasm.test.mjs

@@ -74,6 +74,17 @@ test("extractDocument exposes page metadata in camelCase", () => {
   assert.equal(out.metadata.site_name, undefined);
 });
 
+test("extractDocument reads plain text and Markdown", () => {
+  const mk = new Markkit();
+  const txt = mk.extractDocument(new TextEncoder().encode("a\nb"), { fileName: "n.txt" });
+  assert.equal(txt.format, "text");
+  assert.equal(txt.markdown, "a  \nb\n");
+  const md = mk.extractDocument(new TextEncoder().encode("# T\n\n<script>x()</script>\n"), { fileName: "d.md" });
+  assert.equal(md.format, "markdown");
+  assert.equal(md.title, "T");
+  assert.ok(!md.markdown.includes("script"));
+});
+
 test("extractBytes and unsupported formats", () => {
   const mk = new Markkit();
   assert.equal(mk.extractBytes(new TextEncoder().encode("<p>x</p>"), "text/html"), "x\n");

crates/markkit/Cargo.toml

@@ -19,14 +19,15 @@ html = ["dep:scraper", "dep:html5ever", "dep:ego-tree", "dep:url"]
 pdf = []
 docx = ["dep:zip", "dep:quick-xml"]
 spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"]
+markdown = ["dep:pulldown-cmark"]
 # Opt-in: the default build has no network stack (no reqwest/hyper/TLS).
 http_client = ["dep:reqwest", "tokio/net"]
 logging = ["dep:tracing"]
 # LLM tool contract (`markkit::tool`): Everruns toolkit-library shape.
 tool = ["dep:tower", "dep:futures-core", "dep:base64"]
 # JSON Schema for the IR (`markkit::ir::json_schema()`)
 json-schema = ["dep:schemars"]
-all_formats = ["html", "pdf", "docx", "spreadsheet"]
+all_formats = ["html", "pdf", "docx", "spreadsheet", "markdown"]
 
 [dependencies]
 # Async
@@ -54,6 +55,9 @@ quick-xml = { workspace = true, optional = true }
 calamine = { workspace = true, optional = true }
 cfb = { workspace = true, optional = true }
 
+# Markdown (optional)
+pulldown-cmark = { workspace = true, optional = true }
+
 # HTTP client (optional)
 reqwest = { workspace = true, optional = true }
 

crates/markkit/src/extract.rs

@@ -15,12 +15,14 @@ pub struct Extraction {
     pub markdown: String,
     /// Document title when known: HTML `og:title` / JSON-LD / `<title>`; DOCX
     /// core properties or `Title` style; XLSX `docProps/core.xml`, ODS
-    /// `meta.xml`, XLS `SummaryInformation`.
+    /// `meta.xml`, XLS `SummaryInformation`;
+    /// Markdown front matter `title:` or first `# ` heading.
     pub title: Option<String>,
     /// Format the document was parsed as.
     pub format: Format,
     /// Non-fatal issues, each prefixed with a stable code
-    /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`).
+    /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`,
+    /// `invalid_utf8:`).
     pub warnings: Vec<String>,
     /// Format-specific counters.
     pub stats: ExtractionStats,
@@ -65,9 +67,9 @@ pub struct ExtractionStats {
     pub sheets: Option<usize>,
     /// Data rows rendered across all sheets (header rows excluded).
     pub rows: Option<usize>,
-    /// Non-empty paragraphs, headings and list items (DOCX).
+    /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown).
     pub paragraphs: Option<usize>,
-    /// Tables (DOCX).
+    /// Tables (DOCX, Markdown).
     pub tables: Option<usize>,
 }
 
@@ -91,10 +93,6 @@ pub struct ParsedDocument {
 }
 
 impl ParsedDocument {
-    #[cfg_attr(
-        not(any(feature = "html", feature = "docx", feature = "spreadsheet")),
-        allow(dead_code)
-    )]
     fn new(document: Document, format: Format) -> Self {
         Self {
             document,
@@ -226,7 +224,7 @@ impl Markkit {
         .map(|e| e.markdown)
     }
 
-    /// Convert a document (HTML, DOCX, XLSX/XLS/ODS) to Markdown with metadata.
+    /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to Markdown with metadata.
     ///
     /// The format is detected from `hint.content_type`, then the extension of
     /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline;
@@ -267,7 +265,7 @@ impl Markkit {
         Ok(parsed.render())
     }
 
-    /// Parse a document (HTML, DOCX, XLSX/XLS/ODS) into the IR without
+    /// Parse a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) into the IR without
     /// rendering. Same detection and errors as [`Markkit::extract_document`].
     pub async fn parse_document(
         &self,
@@ -301,6 +299,8 @@ impl Markkit {
             Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => {
                 self.parse_spreadsheet(bytes, budget)
             }
+            Format::PlainText => self.parse_plain_text(bytes, budget),
+            Format::Markdown => self.parse_markdown(bytes, budget),
             Format::Doc => Err(Error::UnsupportedFormat(
                 "legacy Word 97-2003 (.doc) documents are not supported".into(),
             )),
@@ -333,6 +333,37 @@ impl Markkit {
         ))
     }
 
+    fn parse_plain_text(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> {
+        let decoded = crate::text::decode(bytes);
+        let (mut doc, paragraphs) = crate::text::plain_to_ir(&decoded.text, budget)?;
+        self.config.transforms.apply(&mut doc)?;
+        let mut p = ParsedDocument::new(doc, Format::PlainText);
+        p.warnings = decoded.warnings;
+        p.stats.paragraphs = Some(paragraphs);
+        Ok(p)
+    }
+
+    #[cfg(feature = "markdown")]
+    fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> {
+        let decoded = crate::text::decode(bytes);
+        let out = crate::markdown::parse(&decoded.text, &self.config, budget)?;
+        let mut doc = out.doc;
+        self.config.transforms.apply(&mut doc)?;
+        let mut p = ParsedDocument::new(doc, Format::Markdown);
+        p.title = out.title;
+        p.warnings = decoded.warnings;
+        p.stats.paragraphs = Some(out.paragraphs);
+        p.stats.tables = Some(out.tables);
+        Ok(p)
+    }
+
+    #[cfg(not(feature = "markdown"))]
+    fn parse_markdown(&self, _bytes: &[u8], _budget: &Budget) -> crate::Result<ParsedDocument> {
+        Err(Error::UnsupportedFormat(
+            "Markdown support requires the 'markdown' feature".to_string(),
+        ))
+    }
+
     #[cfg(feature = "docx")]
     fn parse_docx(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> {
         let out = guarded("DOCX", Error::DocxExtract, || {

crates/markkit/src/format.rs

@@ -31,7 +31,11 @@ pub enum Format {
     /// Office Open XML presentation (`.pptx`). Detected but not supported.
     Pptx,
     Csv,
+    /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes.
+    /// Serialized as `"text"`, same as `as_str`.
+    #[serde(rename = "text")]
     PlainText,
+    /// Markdown (`text/markdown`, `.md`), passed through normalized.
     Markdown,
     Unknown,
 }
@@ -115,7 +119,7 @@ impl Format {
             Format::Csv
         } else if ct.contains("text/plain") {
             Format::PlainText
-        } else if ct.contains("text/markdown") {
+        } else if ct.contains("text/markdown") || ct.contains("text/x-markdown") {
             Format::Markdown
         } else {
             Format::Unknown
@@ -135,8 +139,8 @@ impl Format {
             "ods" => Format::Ods,
             "pptx" | "pptm" | "ppt" => Format::Pptx,
             "csv" | "tsv" => Format::Csv,
-            "txt" => Format::PlainText,
-            "md" | "markdown" => Format::Markdown,
+            "txt" | "text" => Format::PlainText,
+            "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown,
             _ => Format::Unknown,
         }
     }
@@ -245,6 +249,28 @@ fn looks_like_html(bytes: &[u8]) -> bool {
 mod tests {
     use super::*;
 
+    #[test]
+    fn serde_name_matches_as_str() {
+        for f in [
+            Format::Html,
+            Format::Pdf,
+            Format::Docx,
+            Format::Doc,
+            Format::Xlsx,
+            Format::Xls,
+            Format::Ods,
+            Format::Pptx,
+            Format::Csv,
+            Format::PlainText,
+            Format::Markdown,
+            Format::Unknown,
+        ] {
+            let json = serde_json::to_string(&f).unwrap();
+            assert_eq!(json, format!("\"{}\"", f.as_str()));
+            assert_eq!(serde_json::from_str::<Format>(&json).unwrap(), f);
+        }
+    }
+
     #[test]
     fn test_content_type_detection() {
         assert_eq!(

crates/markkit/src/lib.rs

@@ -34,6 +34,8 @@
 //! - **HTML** (`html`, default): Convert HTML to markdown with configurable rules
 //! - **DOCX** (`docx`): Convert Word documents
 //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables
+//! - **Plain text** (always on): `text/plain`, `.txt`
+//! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR
 //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`]
 //! - **URL fetching** (`http_client`, opt-in): Fetch and extract from URLs via reqwest
 //! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the
@@ -58,9 +60,12 @@ pub mod format;
 pub mod html;
 pub mod ir;
 pub mod limits;
+#[cfg(feature = "markdown")]
+mod markdown;
 pub mod render;
 #[cfg(feature = "spreadsheet")]
 mod spreadsheet;
+mod text;
 #[cfg(feature = "tool")]
 pub mod tool;
 pub mod transform;