diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 676561f..19d666d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,8 @@ jobs: echo "HTTP client leaked into the default feature set" exit 1 fi + # Always-on formats (text, CSV, notebooks) must keep every limit with no format feature. + cargo test -p markkit --no-default-features cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then echo "HTTP client leaked into the docx,spreadsheet,markdown feature set" diff --git a/CHANGELOG.md b/CHANGELOG.md index c4cdf1a..f4a3194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `markkit mcp`: Model Context Protocol server over stdio exposing the `markkit` tool (`tools/list`, `tools/call`). URL fetching keeps the SSRF guard; `--no-url` keeps it offline, the `extract` limit flags and `--max-output-chars` bound each call. +- Jupyter notebook input (`.ipynb`, `application/x-ipynb+json`, nbformat 4), always on: Markdown + cells, code cells as fenced blocks tagged with the kernel language, text outputs and ANSI-free + tracebacks; images/HTML outputs skipped. New `Format::Notebook`, `Error::NotebookExtract` and + `ExtractionStats::cells`. - CSV and TSV input (`text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`), always on: one GFM table, RFC 4180 quoting, `;`/`|`/tab delimiter detection, bounded by `max_sheet_rows` / `max_sheet_cols` (`truncated_table` warning). New `Format::Tsv`. diff --git a/README.md b/README.md index 1f58e93..f4ec409 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ package for browsers and edge runtimes. - **DOCX** – Headings, lists, tables, links, emphasis - **PPTX** – One section per slide: titles, bullets, tables, links, speaker notes - **Spreadsheets** – XLSX/XLSM, XLSB, XLS and ODS workbooks to markdown tables, one section per sheet +- **Jupyter notebooks** – Markdown cells, code cells with language, text outputs and tracebacks - **CSV / TSV** – One markdown table; `;` and `|` delimiters detected, RFC 4180 quoting - **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) - **Format detection** – Content type, then file extension, then magic bytes @@ -38,6 +39,7 @@ package for browsers and edge runtimes. | XLSX, XLSM, XLSB, XLS, ODS | Supported | `spreadsheet` | | Plain text | Supported | always on | | CSV, TSV | Supported | always on | +| Jupyter notebooks (ipynb) | Supported | always on | | Markdown | Supported | `markdown` | | PDF (text layer) | Supported | `pdf` | | DOC, PPT | Detected, rejected explicitly | – | diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 7fe7fbc..355ff29 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -93,7 +93,7 @@ pub struct Metadata { pub struct Extraction { pub markdown: String, pub title: Option, - /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`). + /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`). pub format: String, pub warnings: Vec, pub stats: ExtractionStats, diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index 9459c4d..05dc404 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -1,8 +1,8 @@ # @everruns/markkit-wasm Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX, -spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, plain text and Markdown in the -**browser and any JavaScript runtime**: +spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, Jupyter notebooks, plain text and +Markdown in the **browser and any JavaScript runtime**: edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index 85800ad..ccb3f2e 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -94,6 +94,14 @@ test("extractDocument reads CSV and TSV", () => { assert.equal(tsv.format, "tsv"); }); +test("extractDocument reads Jupyter notebooks", () => { + const nb = { nbformat: 4, metadata: {}, cells: [{ cell_type: "markdown", source: "# NB" }, { cell_type: "code", source: "1+1", outputs: [] }] }; + const out = new Markkit().extractDocument(new TextEncoder().encode(JSON.stringify(nb)), { fileName: "a.ipynb" }); + assert.equal(out.format, "ipynb"); + assert.equal(out.title, "NB"); + assert.ok(out.markdown.includes("1+1")); +}); + test("extractBytes and unsupported formats", () => { const mk = new Markkit(); assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index e1eb794..681e3b2 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -17,6 +17,9 @@ pub enum Error { #[error("Spreadsheet extraction error: {0}")] SpreadsheetExtract(String), + #[error("Notebook extraction error: {0}")] + NotebookExtract(String), + #[error("HTTP error: {0}")] Http(String), diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index bd0444d..55e158b 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -72,6 +72,8 @@ pub struct ExtractionStats { pub paragraphs: Option, /// Tables (DOCX, Markdown, CSV/TSV). pub tables: Option, + /// Non-empty cells rendered (Jupyter notebooks). + pub cells: Option, } /// Result of [`Markkit::parse_document`]: the IR plus metadata, not rendered. @@ -114,14 +116,6 @@ impl ParsedDocument { } /// [`ParsedDocument::render`] with `max_output_size` and the deadline enforced. - #[cfg(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf", - feature = "markdown" - ))] fn render_bounded(self, max_output: usize, budget: &Budget) -> crate::Result { let markdown = crate::render::MarkdownRenderer::new().render_bounded( &self.document, @@ -233,8 +227,8 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, PDF, DOCX, PPTX, XLSX/XLSB/XLS/ODS, plain text, Markdown, CSV/TSV) - /// to Markdown with metadata. + /// Convert a document (HTML, PDF, DOCX, PPTX, XLSX/XLSB/XLS/ODS, plain text, Markdown, + /// CSV/TSV, Jupyter notebooks) to Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -267,26 +261,7 @@ impl Markkit { budget: &Budget, ) -> crate::Result { let parsed = self.parse_document_with(bytes, hint, budget)?; - #[cfg(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf", - feature = "markdown" - ))] - { - parsed.render_bounded(self.config.limits.max_output_size, budget) - } - #[cfg(not(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf", - feature = "markdown" - )))] - Ok(parsed.render()) + parsed.render_bounded(self.config.limits.max_output_size, budget) } /// Parse a document (HTML, PDF, DOCX, PPTX, XLSX/XLSB/XLS/ODS, plain text, Markdown) into the IR without @@ -331,6 +306,7 @@ impl Markkit { Format::PlainText => self.parse_plain_text(bytes, budget), Format::Markdown => self.parse_markdown(bytes, budget), Format::Csv | Format::Tsv => self.parse_csv(bytes, format, budget), + Format::Notebook => self.parse_notebook(bytes, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -379,6 +355,7 @@ impl Markkit { fn parse_plain_text(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let decoded = crate::text::decode(bytes); + budget.check_time()?; let (mut doc, paragraphs) = crate::text::plain_to_ir(&decoded.text, budget)?; self.config.transforms.apply(&mut doc)?; let mut p = ParsedDocument::new(doc, Format::PlainText); @@ -394,6 +371,7 @@ impl Markkit { budget: &Budget, ) -> crate::Result { let decoded = crate::text::decode(bytes); + budget.check_time()?; let out = crate::csv::parse(&decoded.text, format == Format::Tsv, &self.config, budget)?; let mut doc = out.doc; self.config.transforms.apply(&mut doc)?; @@ -405,9 +383,23 @@ impl Markkit { Ok(p) } + fn parse_notebook(&self, bytes: &[u8], budget: &Budget) -> crate::Result { + let decoded = crate::text::decode(bytes); + budget.check_time()?; + let out = crate::notebook::parse(&decoded.text, &self.config, budget)?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Notebook); + p.title = out.title; + p.warnings = decoded.warnings; + p.stats.cells = Some(out.cells); + Ok(p) + } + #[cfg(feature = "markdown")] fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let decoded = crate::text::decode(bytes); + budget.check_time()?; let out = crate::markdown::parse(&decoded.text, &self.config, budget)?; let mut doc = out.doc; self.config.transforms.apply(&mut doc)?; diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index 71d0c0f..7251296 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -36,6 +36,9 @@ pub enum Format { Csv, /// Tab separated values (`.tsv`, `.tab`). Tsv, + /// Jupyter notebook (`.ipynb`, nbformat 4). Serialized as `"ipynb"`. + #[serde(rename = "ipynb")] + Notebook, /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes (nor CSV/TSV/Markdown). /// Serialized as `"text"`, same as `as_str`. #[serde(rename = "text")] @@ -94,6 +97,7 @@ impl Format { Format::Ppt => "ppt", Format::Csv => "csv", Format::Tsv => "tsv", + Format::Notebook => "ipynb", Format::PlainText => "text", Format::Markdown => "markdown", Format::Unknown => "unknown", @@ -123,6 +127,8 @@ impl Format { } else if ct.contains("presentationml") || ct.contains("ms-powerpoint.") { // `ms-powerpoint.presentation.macroEnabled.12` & co. are OOXML. Format::Pptx + } else if ct.contains("x-ipynb") { + Format::Notebook } else if ct.contains("ms-powerpoint") { Format::Ppt } else if ct.contains("tab-separated-values") { @@ -153,6 +159,7 @@ impl Format { "ppt" | "pps" | "pot" => Format::Ppt, "csv" => Format::Csv, "tsv" | "tab" => Format::Tsv, + "ipynb" => Format::Notebook, "txt" | "text" => Format::PlainText, "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, @@ -277,6 +284,8 @@ mod tests { Format::Ods, Format::Pptx, Format::Csv, + Format::Tsv, + Format::Notebook, Format::PlainText, Format::Markdown, Format::Unknown, diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 26772bf..d2d95b8 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -36,6 +36,7 @@ //! - **PPTX** (`pptx`): Convert PowerPoint presentations, one section per slide //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLSB, XLS and ODS workbooks to tables //! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Jupyter notebooks** (always on): `.ipynb`, cells and text outputs //! - **CSV / TSV** (always on): `text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`, one table //! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Text layer of PDF documents (no OCR); encrypted PDFs are @@ -66,6 +67,7 @@ pub mod ir; pub mod limits; #[cfg(feature = "markdown")] mod markdown; +mod notebook; #[cfg(any(feature = "docx", feature = "pptx"))] mod ooxml; #[cfg(feature = "pdf")] diff --git a/crates/markkit/src/limits.rs b/crates/markkit/src/limits.rs index 7dcf027..9f656f6 100644 --- a/crates/markkit/src/limits.rs +++ b/crates/markkit/src/limits.rs @@ -198,54 +198,21 @@ impl Limits { } } -// Budget is only driven by the feature-gated parsers. /// How many [`Budget::expired`] calls share one clock read. -#[cfg_attr( - not(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf" - )), - allow(dead_code) -)] const CLOCK_EVERY: u32 = 256; /// Per-call deadline and node counter, shared by reference through parsers and /// the renderer (interior mutability so `&self` walkers can charge it). #[derive(Debug)] -#[cfg_attr( - not(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf" - )), - allow(dead_code) -)] pub(crate) struct Budget { deadline: Option, timeout: Duration, ticks: Cell, expired: Cell, - #[cfg_attr(not(any(feature = "html", feature = "docx")), allow(dead_code))] nodes: Cell, - #[cfg_attr(not(any(feature = "html", feature = "docx")), allow(dead_code))] max_nodes: usize, } -#[cfg_attr( - not(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf" - )), - allow(dead_code) -)] impl Budget { /// Budget starting now. pub fn new(limits: &Limits) -> Self { @@ -322,7 +289,6 @@ impl Budget { } /// Charge `n` parsed nodes against `max_nodes`. - #[cfg_attr(not(any(feature = "html", feature = "docx")), allow(dead_code))] pub fn add_nodes(&self, n: usize) -> crate::Result<()> { let total = self.nodes.get().saturating_add(n); self.nodes.set(total); diff --git a/crates/markkit/src/notebook.rs b/crates/markkit/src/notebook.rs new file mode 100644 index 0000000..e1ec480 --- /dev/null +++ b/crates/markkit/src/notebook.rs @@ -0,0 +1,272 @@ +//! Jupyter notebook (`.ipynb`, nbformat 4) input. +//! +//! Decisions: +//! - Always compiled (serde_json is already a core dependency). Selected by +//! extension or `application/x-ipynb+json`, never sniffed. +//! - Markdown cells go through the Markdown parser when the `markdown` feature +//! is on (same IR, raw HTML never copied), otherwise they render as plain +//! text paragraphs. Code cells become fenced blocks tagged with the kernel +//! language; raw cells become untagged blocks. +//! - Outputs are kept because they often carry the result a reader needs: +//! `stream` and `text/plain` as ```` ```text ```` blocks, `text/markdown` +//! through the Markdown path, errors as `ename: evalue` plus traceback. +//! Images, HTML and widget outputs are skipped. ANSI escape sequences (common +//! in tracebacks) are stripped before the usual control-character pass. +//! - nbformat < 4 and non-notebook JSON are `Error::NotebookExtract`. serde_json +//! caps nesting at 128, so hostile JSON fails instead of overflowing. +//! - Shares the call's `Budget`: each cell ticks the deadline and counts +//! against `max_nodes`; Markdown cells use the same budget. + +use serde_json::Value; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::ir::{Document, Node}; +use crate::limits::Budget; + +/// Parsed notebook. +pub(crate) struct NotebookOutput { + pub doc: Document, + pub title: Option, + /// Cells rendered (empty cells excluded). + pub cells: usize, +} + +/// Parse decoded notebook JSON into the IR. +pub(crate) fn parse( + json: &str, + config: &MarkkitBuilder, + budget: &Budget, +) -> Result { + let nb: Value = serde_json::from_str(json) + .map_err(|e| Error::NotebookExtract(format!("invalid JSON: {e}")))?; + let major = nb.get("nbformat").and_then(Value::as_u64).unwrap_or(0); + if major < 4 { + return Err(Error::NotebookExtract(format!( + "unsupported nbformat {major} (only 4.x is supported)" + ))); + } + let cells = nb + .get("cells") + .and_then(Value::as_array) + .ok_or_else(|| Error::NotebookExtract("missing 'cells' array".into()))?; + let meta = nb.get("metadata"); + let language = meta + .and_then(|m| m.pointer("/kernelspec/language")) + .or_else(|| meta.and_then(|m| m.pointer("/language_info/name"))) + .and_then(Value::as_str) + .map(|l| { + l.chars() + .filter(|c| c.is_ascii_alphanumeric() || "+-_#.".contains(*c)) + .collect::() + }) + .filter(|l| !l.is_empty()); + + let mut nodes = Vec::new(); + let mut count = 0; + for cell in cells { + if budget.expired() { + return Err(budget.timeout_error()); + } + budget.add_nodes(1)?; + let source = multiline(cell.get("source")); + let before = nodes.len(); + match cell.get("cell_type").and_then(Value::as_str) { + Some("markdown") => nodes.extend(markdown_nodes(&clean(&source), config, budget)?), + Some("code") => { + let code = clean(&source); + if !code.trim().is_empty() { + nodes.push(code_block(language.clone(), code)); + } + if let Some(outputs) = cell.get("outputs").and_then(Value::as_array) { + for out in outputs { + nodes.extend(output_nodes(out, config, budget)?); + } + } + } + Some("raw") => { + let raw = clean(&source); + if !raw.trim().is_empty() { + nodes.push(code_block(None, raw)); + } + } + _ => {} + } + if nodes.len() > before { + count += 1; + } + } + + let title = meta + .and_then(|m| m.get("title")) + .and_then(Value::as_str) + .map(|t| clean(t).trim().to_string()) + .filter(|t| !t.is_empty()) + .or_else(|| first_h1(&nodes)); + Ok(NotebookOutput { + doc: Document::new(nodes), + title, + cells: count, + }) +} + +/// nbformat stores text either as a string or a list of lines. +fn multiline(v: Option<&Value>) -> String { + match v { + Some(Value::String(s)) => s.clone(), + Some(Value::Array(parts)) => parts.iter().filter_map(Value::as_str).collect(), + _ => String::new(), + } +} + +/// Strip ANSI escape sequences, then apply the shared text normalization. +fn clean(s: &str) -> String { + crate::text::decode(strip_ansi(s).as_bytes()).text +} + +fn strip_ansi(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c != '\u{1b}' { + out.push(c); + continue; + } + match chars.peek() { + // CSI: ESC [ params… final byte in @..~ + Some('[') => { + chars.next(); + for c in chars.by_ref() { + if ('@'..='~').contains(&c) { + break; + } + } + } + // OSC: ESC ] … BEL or ESC \ + Some(']') => { + chars.next(); + while let Some(c) = chars.next() { + if c == '\u{7}' { + break; + } + if c == '\u{1b}' && chars.peek() == Some(&'\\') { + chars.next(); + break; + } + } + } + // Charset designation: ESC ( B and friends. + Some('(' | ')' | '*' | '+') => { + chars.next(); + chars.next(); + } + // Two-character escape. + Some(_) => { + chars.next(); + } + None => {} + } + } + out +} + +fn code_block(language: Option, code: String) -> Node { + Node::CodeBlock { + language, + code: code.trim_end_matches('\n').to_string(), + source: None, + } +} + +fn text_block(text: String) -> Option { + (!text.trim().is_empty()).then(|| code_block(Some("text".into()), text)) +} + +#[cfg(feature = "markdown")] +fn markdown_nodes(src: &str, config: &MarkkitBuilder, budget: &Budget) -> Result, Error> { + Ok(crate::markdown::parse(src, config, budget)?.doc.nodes) +} + +#[cfg(not(feature = "markdown"))] +fn markdown_nodes( + src: &str, + _config: &MarkkitBuilder, + budget: &Budget, +) -> Result, Error> { + Ok(crate::text::plain_to_ir(src, budget)?.0.nodes) +} + +fn output_nodes(out: &Value, config: &MarkkitBuilder, budget: &Budget) -> Result, Error> { + Ok(match out.get("output_type").and_then(Value::as_str) { + Some("stream") => text_block(clean(&multiline(out.get("text")))) + .into_iter() + .collect(), + Some("execute_result" | "display_data") => { + let data = out.get("data"); + if let Some(md) = data.and_then(|d| d.get("text/markdown")) { + markdown_nodes(&clean(&multiline(Some(md))), config, budget)? + } else if let Some(txt) = data.and_then(|d| d.get("text/plain")) { + text_block(clean(&multiline(Some(txt)))) + .into_iter() + .collect() + } else { + Vec::new() + } + } + Some("error") => { + let field = |k| out.get(k).and_then(Value::as_str).unwrap_or(""); + let mut text = format!("{}: {}", field("ename"), field("evalue")); + if let Some(tb) = out.get("traceback").and_then(Value::as_array) { + for line in tb.iter().filter_map(Value::as_str) { + text.push('\n'); + text.push_str(line); + } + } + text_block(clean(&text)).into_iter().collect() + } + _ => Vec::new(), + }) +} + +fn first_h1(nodes: &[Node]) -> Option { + nodes.iter().find_map(|n| match n { + Node::Heading { + level: 1, children, .. + } => { + let t: String = children + .iter() + .filter_map(|c| match c { + Node::Text(t) | Node::InlineCode(t) => Some(t.as_str()), + _ => None, + }) + .collect(); + let t = t.trim(); + (!t.is_empty()).then(|| t.to_string()) + } + _ => None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_ansi_sequences() { + assert_eq!( + strip_ansi("\u{1b}[0;31mValueError\u{1b}[0m: x\u{1b}]0;title\u{7}!\u{1b}(B"), + "ValueError: x!" + ); + assert_eq!(strip_ansi("trailing \u{1b}"), "trailing "); + assert_eq!(strip_ansi("ok\u{1b}[12;"), "ok"); + } + + #[test] + fn multiline_forms() { + let v: Value = serde_json::json!(["a\n", "b"]); + assert_eq!(multiline(Some(&v)), "a\nb"); + assert_eq!(multiline(Some(&Value::String("x".into()))), "x"); + assert_eq!(multiline(Some(&Value::Null)), ""); + assert_eq!(multiline(None), ""); + } +} diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs index 1847558..44632e4 100644 --- a/crates/markkit/src/render/markdown.rs +++ b/crates/markkit/src/render/markdown.rs @@ -77,14 +77,6 @@ impl MarkdownRenderer { } /// Render with `max_output` bytes and the budget's deadline enforced. - #[cfg(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf", - feature = "markdown" - ))] pub(crate) fn render_bounded( &mut self, doc: &Document, @@ -726,14 +718,6 @@ impl Default for MarkdownRenderer { } } -#[cfg(any( - feature = "html", - feature = "docx", - feature = "pptx", - feature = "spreadsheet", - feature = "pdf", - feature = "markdown" -))] fn output_error(max: usize) -> crate::Error { crate::Error::ResourceLimit(format!("output exceeds max_output_size {max}")) } diff --git a/crates/markkit/tests/notebook_integration.rs b/crates/markkit/tests/notebook_integration.rs new file mode 100644 index 0000000..e07f233 --- /dev/null +++ b/crates/markkit/tests/notebook_integration.rs @@ -0,0 +1,209 @@ +//! Jupyter notebook (.ipynb) → Markdown integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; +use serde_json::json; + +fn extract(nb: &serde_json::Value) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync( + nb.to_string().as_bytes(), + DocumentHint::file_name("nb.ipynb"), + ) + .unwrap() +} + +fn err(bytes: &[u8]) -> Error { + Markkit::builder() + .build() + .extract_document_sync(bytes, DocumentHint::file_name("nb.ipynb")) + .unwrap_err() +} + +fn notebook(cells: serde_json::Value) -> serde_json::Value { + json!({ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {"kernelspec": {"name": "python3", "language": "python"}}, + "cells": cells + }) +} + +#[test] +fn cells_and_outputs() { + let nb = notebook(json!([ + {"cell_type": "markdown", "source": ["# Analysis\n", "\n", "Load the data."]}, + {"cell_type": "code", "source": "import pandas as pd\ndf = pd.read_csv('x.csv')\ndf.shape", + "outputs": [ + {"output_type": "stream", "name": "stdout", "text": ["loading\n"]}, + {"output_type": "execute_result", "data": {"text/plain": ["(3, 2)"], "image/png": "iVBOR"}}, + {"output_type": "display_data", "data": {"image/png": "iVBOR"}} + ]}, + {"cell_type": "code", "source": [], "outputs": []}, + {"cell_type": "raw", "source": "raw text"} + ])); + let e = extract(&nb); + assert_eq!(e.format, Format::Notebook); + if cfg!(feature = "markdown") { + assert_eq!(e.title.as_deref(), Some("Analysis")); + assert!(e.markdown.starts_with("# Analysis\n\nLoad the data.\n")); + } else { + // Without the parser, Markdown cells are plain text paragraphs. + assert_eq!(e.title, None); + } + assert_eq!(e.stats.cells, Some(3)); + assert!(e.markdown.ends_with( + "```python\nimport pandas as pd\ndf = pd.read_csv('x.csv')\ndf.shape\n```\n\n```text\nloading\n```\n\n```text\n(3, 2)\n```\n\n```\nraw text\n```\n" + ), "{}", e.markdown); + assert!(!e.markdown.contains("iVBOR")); +} + +#[test] +fn error_output_strips_ansi() { + let nb = notebook(json!([ + {"cell_type": "code", "source": "1/0", "outputs": [{ + "output_type": "error", "ename": "ZeroDivisionError", "evalue": "division by zero", + "traceback": ["\u{1b}[0;31m---------\u{1b}[0m", "\u{1b}[0;31mZeroDivisionError\u{1b}[0m: division by zero"] + }]} + ])); + let e = extract(&nb); + assert!( + e.markdown.contains("```text\nZeroDivisionError: division by zero\n---------\nZeroDivisionError: division by zero\n```"), + "{}", + e.markdown + ); + assert!(!e.markdown.contains('\u{1b}') && !e.markdown.contains("[0;31m")); +} + +#[test] +fn metadata_title_and_language_info() { + let nb = json!({ + "nbformat": 4, "nbformat_minor": 2, + "metadata": {"title": "My NB", "language_info": {"name": "julia"}}, + "cells": [{"cell_type": "code", "source": "1 + 1", "outputs": []}] + }); + let e = extract(&nb); + assert_eq!(e.title.as_deref(), Some("My NB")); + assert_eq!(e.markdown, "```julia\n1 + 1\n```\n"); +} + +#[test] +fn detection() { + assert_eq!(Format::from_extension("IPYNB"), Format::Notebook); + assert_eq!( + Format::from_content_type("application/x-ipynb+json"), + Format::Notebook + ); + // Never sniffed: plain JSON bytes without a hint stay unsupported. + let e = Markkit::builder() + .build() + .extract_document_sync(br#"{"nbformat":4,"cells":[]}"#, DocumentHint::default()) + .unwrap_err(); + assert!(matches!(e, Error::UnsupportedFormat(_))); +} + +#[test] +fn empty_notebook() { + let e = extract(¬ebook(json!([]))); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.cells, Some(0)); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +#[test] +fn security_malformed_json_is_error() { + for bytes in [ + &b""[..], + b"not json", + b"{\"nbformat\": 4", + b"[1,2,3]", + b"{\"nbformat\": 4, \"cells\": 7}", + b"{\"nbformat\": 3, \"worksheets\": []}", + b"\xff\xfe\x00", + ] { + assert!(matches!(err(bytes), Error::NotebookExtract(_)), "{bytes:?}"); + } +} + +#[test] +fn security_deep_json_nesting_is_error_not_overflow() { + let deep = format!( + "{{\"nbformat\":4,\"cells\":[],\"metadata\":{}{}}}", + "[".repeat(100_000), + "]".repeat(100_000) + ); + assert!(matches!(err(deep.as_bytes()), Error::NotebookExtract(_))); +} + +#[test] +fn security_wrong_types_ignored() { + let nb = json!({ + "nbformat": 4, + "metadata": {"kernelspec": {"language": "py`thon\n```"}, "title": 5}, + "cells": [ + {"cell_type": 1, "source": "x"}, + {"cell_type": "code", "source": {"a": 1}, "outputs": "nope"}, + {"cell_type": "code", "source": ["ok", 3, null], "outputs": [7, {"output_type": "stream", "text": 9}]}, + "not a cell" + ] + }); + let e = extract(&nb); + assert_eq!(e.markdown, "```python\nok\n```\n"); + assert_eq!(e.title, None); +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_cells_and_outputs_drop_raw_html() { + let nb = notebook(json!([ + {"cell_type": "markdown", "source": "hi \n\n"}, + {"cell_type": "code", "source": "x", "outputs": [ + {"output_type": "display_data", "data": {"text/html": "", "text/markdown": "**ok**