From acc13b5aeaa1c9465669aa15fc26c7bbc39acf6e Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:52:17 +0000 Subject: [PATCH] feat: IR transform passes between parse and render - transform::Transform trait, TransformChain on MarkkitBuilder (.transform(..)), applied by parse_html/parse_document/extract_* - built-ins NormalizeHeadings, DropImages; transform::from_fn - Document::visit/visit_mut, Node::for_each_child_list[_ref] - Error is #[non_exhaustive], new Error::Transform - examples/transforms.rs --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 + crates/markkit/Cargo.toml | 5 + crates/markkit/src/config.rs | 9 + crates/markkit/src/error.rs | 5 + crates/markkit/src/extract.rs | 18 +- crates/markkit/src/ir.rs | 107 +++++++++++ crates/markkit/src/lib.rs | 1 + crates/markkit/src/transform.rs | 219 +++++++++++++++++++++++ crates/markkit/tests/docx_integration.rs | 14 ++ crates/markkit/tests/html_integration.rs | 71 ++++++++ examples/transforms.rs | 34 ++++ justfile | 2 +- knowledge/foundations/architecture.md | 13 ++ knowledge/log.md | 2 + knowledge/operations/examples.md | 7 + 16 files changed, 505 insertions(+), 8 deletions(-) create mode 100644 crates/markkit/src/transform.rs create mode 100644 examples/transforms.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4e9f4d..a814768 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: fi - name: Run examples run: | - for ex in basic custom_selectors configuration ir_access; do + for ex in basic custom_selectors configuration ir_access transforms; do cargo run -q -p markkit --example "$ex" done cargo run -q -p markkit --features tool --example agent_tool diff --git a/CHANGELOG.md b/CHANGELOG.md index 8682337..cff0862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). no longer dropped. - `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON. - Math: `$tex$` and `$$` blocks from MathML (`application/x-tex` annotation), KaTeX and MathJax output. +- IR transforms: `MarkkitBuilder::transform(..)` runs `Transform` passes after parsing and before + rendering, for every format. Built-ins `transform::NormalizeHeadings` and `transform::DropImages`; + `transform::from_fn` for closures; `Document::visit` / `visit_mut` to walk the tree. - JSON Schema for the IR at `schema/ir.schema.json`; `json-schema` feature adds `markkit::ir::json_schema()`. - Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations. @@ -50,6 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **Breaking:** `Error` is `#[non_exhaustive]` (new `Error::Transform`); matches need a `_` arm. - **Breaking (IR API):** `Node`, `Document`, `ListItem` and the new IR structs are `#[non_exhaustive]`; matches need a `_` arm and structs are built with constructors. `Node::Table` cells are now `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. Block variants gain a diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 6c1af79..b40ede5 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -98,6 +98,11 @@ name = "agent_tool" path = "../../examples/agent_tool.rs" required-features = ["html", "tool"] +[[example]] +name = "transforms" +path = "../../examples/transforms.rs" +required-features = ["html"] + [[bench]] name = "extraction" harness = false diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index 542d7d7..87d1bca 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -44,6 +44,8 @@ pub struct MarkkitBuilder { /// HTML element XPath, DOCX block index, spreadsheet sheet and range. /// Default: false (no cost, no `source` in IR JSON). pub track_sources: bool, + /// IR passes run after parsing, before rendering (see [`crate::transform`]). + pub transforms: crate::transform::TransformChain, } impl Default for MarkkitBuilder { @@ -74,6 +76,7 @@ impl Default for MarkkitBuilder { allowed_hosts: Vec::new(), max_redirects: 5, track_sources: false, + transforms: Default::default(), } } } @@ -105,6 +108,12 @@ impl MarkkitBuilder { self } + /// Append an IR transform pass; passes run in registration order. + pub fn transform(mut self, t: impl crate::transform::Transform + 'static) -> Self { + self.transforms.push(t); + self + } + pub fn include_selector(mut self, selector: impl Into) -> Self { self.include_selectors.push(selector.into()); self diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index 0474b18..9e3f80b 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -1,5 +1,6 @@ /// Errors that can occur during markdown extraction. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum Error { #[error("HTML parsing error: {0}")] HtmlParse(String), @@ -30,6 +31,10 @@ pub enum Error { #[error("Resource limit exceeded: {0}")] ResourceLimit(String), + /// A [`crate::transform::Transform`] failed. + #[error("Transform error: {0}")] + Transform(String), + #[error("I/O error: {0}")] Io(#[from] std::io::Error), } diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 1d3df75..73db1b9 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -167,8 +167,7 @@ impl Markkit { #[cfg(feature = "html")] { - let parser = crate::html::HtmlParser::new(&self.config); - let doc = parser.parse(html)?; + let doc = self.parse_html(html)?; let mut renderer = crate::render::MarkdownRenderer::new(); Ok(renderer.render(&doc)) } @@ -189,7 +188,9 @@ impl Markkit { ))); } let parser = crate::html::HtmlParser::new(&self.config); - parser.parse(html) + let mut doc = parser.parse(html)?; + self.config.transforms.apply(&mut doc)?; + Ok(doc) } /// Extract markdown from raw bytes, auto-detecting the format. @@ -283,7 +284,8 @@ impl Markkit { fn parse_html_doc(&self, bytes: &[u8]) -> crate::Result { let text = String::from_utf8_lossy(bytes); let parser = crate::html::HtmlParser::new(&self.config); - let (doc, page) = parser.parse_page(&text)?; + let (mut doc, page) = parser.parse_page(&text)?; + self.config.transforms.apply(&mut doc)?; let mut p = ParsedDocument::new(doc, Format::Html); p.title = page.title; p.metadata = page.metadata; @@ -302,7 +304,9 @@ impl Markkit { let out = guarded("DOCX", Error::DocxExtract, || { crate::docx::parse(bytes, &self.config) })?; - let mut p = ParsedDocument::new(out.doc, Format::Docx); + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Docx); p.title = out.title; p.stats.paragraphs = Some(out.paragraphs); p.stats.tables = Some(out.tables); @@ -321,7 +325,9 @@ impl Markkit { let out = guarded("spreadsheet", Error::SpreadsheetExtract, || { crate::spreadsheet::parse(bytes, &self.config) })?; - let mut p = ParsedDocument::new(out.doc, out.format); + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, out.format); p.warnings = out.warnings; p.stats.sheets = Some(out.sheets); p.stats.rows = Some(out.rows); diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs index 1682481..356e9ce 100644 --- a/crates/markkit/src/ir.rs +++ b/crates/markkit/src/ir.rs @@ -50,6 +50,29 @@ impl Document { pub fn is_empty(&self) -> bool { self.nodes.is_empty() } + + /// Visit every node, pre-order (parents before children). + pub fn visit(&self, f: &mut dyn FnMut(&Node)) { + fn walk(nodes: &[Node], f: &mut dyn FnMut(&Node)) { + for n in nodes { + f(n); + n.for_each_child_list_ref(&mut |list| walk(list, f)); + } + } + walk(&self.nodes, f); + } + + /// Visit every node mutably, pre-order. Children are visited after `f` + /// ran on their parent, so `f` may rewrite a subtree before it is walked. + pub fn visit_mut(&mut self, f: &mut dyn FnMut(&mut Node)) { + fn walk(nodes: &mut [Node], f: &mut dyn FnMut(&mut Node)) { + for n in nodes { + f(n); + n.for_each_child_list(&mut |list| walk(list, f)); + } + } + walk(&mut self.nodes, f); + } } /// A block-level or inline node in the document tree. @@ -196,6 +219,90 @@ impl Node { } } + /// Call `f` on each child list of this node (block or inline children, + /// list-item children, table cells, definition terms and bodies, + /// figure captions). Leaves have none. + pub fn for_each_child_list(&mut self, f: &mut dyn FnMut(&mut Vec)) { + match self { + Node::Heading { children, .. } + | Node::Paragraph { children, .. } + | Node::BlockQuote { children, .. } + | Node::FootnoteDefinition { children, .. } + | Node::Bold { children } + | Node::Italic { children } + | Node::Strikethrough { children } + | Node::Link { children, .. } => f(children), + Node::Figure { + children, caption, .. + } => { + f(children); + if let Some(c) = caption { + f(c); + } + } + Node::List { items, .. } => { + for item in items { + f(&mut item.children); + } + } + Node::Table { headers, rows, .. } => { + for cell in headers.iter_mut().chain(rows.iter_mut().flatten()) { + f(&mut cell.children); + } + } + Node::DefinitionList { items, .. } => { + for item in items { + f(&mut item.term); + for d in &mut item.definitions { + f(d); + } + } + } + _ => {} + } + } + + /// Read-only [`Node::for_each_child_list`]. + pub fn for_each_child_list_ref(&self, f: &mut dyn FnMut(&Vec)) { + match self { + Node::Heading { children, .. } + | Node::Paragraph { children, .. } + | Node::BlockQuote { children, .. } + | Node::FootnoteDefinition { children, .. } + | Node::Bold { children } + | Node::Italic { children } + | Node::Strikethrough { children } + | Node::Link { children, .. } => f(children), + Node::Figure { + children, caption, .. + } => { + f(children); + if let Some(c) = caption { + f(c); + } + } + Node::List { items, .. } => { + for item in items { + f(&item.children); + } + } + Node::Table { headers, rows, .. } => { + for cell in headers.iter().chain(rows.iter().flatten()) { + f(&cell.children); + } + } + Node::DefinitionList { items, .. } => { + for item in items { + f(&item.term); + for d in &item.definitions { + f(d); + } + } + } + _ => {} + } + } + /// Where this block came from, when the parser recorded it. pub fn source(&self) -> Option<&Source> { match self { diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 29b4dc4..7f9788b 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -61,6 +61,7 @@ pub mod render; mod spreadsheet; #[cfg(feature = "tool")] pub mod tool; +pub mod transform; #[cfg(feature = "docx")] mod xml; diff --git a/crates/markkit/src/transform.rs b/crates/markkit/src/transform.rs new file mode 100644 index 0000000..fbe9404 --- /dev/null +++ b/crates/markkit/src/transform.rs @@ -0,0 +1,219 @@ +//! IR transform passes, run between parsing and rendering. +//! +//! Decisions: +//! - A transform is `Fn(&mut Document) -> Result<()>` behind the +//! [`Transform`] trait; passes run in registration order on every parse +//! (`parse_*` and `extract_*`), after the parser and before the renderer. +//! - Transforms are trusted embedder code: they run without limits, and an +//! error aborts the extraction (returned as is). +//! - Passes are `Arc`-shared so [`crate::MarkkitBuilder`] stays `Clone`. +//! - Built-ins stay small and format-neutral; format-specific cleanup belongs +//! in parsers or element rules. + +use std::fmt; +use std::sync::Arc; + +use crate::ir::{Document, Node}; + +/// A pass over the IR. +pub trait Transform: Send + Sync { + /// Short name for debugging and error messages. + fn name(&self) -> &str; + + /// Rewrite the document in place. + fn apply(&self, doc: &mut Document) -> crate::Result<()>; +} + +/// Ordered list of transforms held by [`crate::MarkkitBuilder`]. +#[derive(Clone, Default)] +pub struct TransformChain(Vec>); + +impl TransformChain { + /// Append a pass. + pub fn push(&mut self, t: impl Transform + 'static) { + self.0.push(Arc::new(t)); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + /// Names of the passes, in order. + pub fn names(&self) -> impl Iterator { + self.0.iter().map(|t| t.name()) + } + + /// Run every pass in order; stops at the first error. + pub fn apply(&self, doc: &mut Document) -> crate::Result<()> { + for t in &self.0 { + t.apply(doc)?; + } + Ok(()) + } +} + +impl fmt::Debug for TransformChain { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.names()).finish() + } +} + +/// Transform from a closure; see [`from_fn`]. +pub struct FnTransform { + name: String, + f: F, +} + +/// Build a transform from a closure. +/// +/// ``` +/// use markkit::{Markkit, ir::Node, transform}; +/// +/// let m = Markkit::builder() +/// .transform(transform::from_fn("drop-rules", |doc| { +/// doc.nodes.retain(|n| !matches!(n, Node::ThematicBreak)); +/// Ok(()) +/// })) +/// .build(); +/// # #[cfg(feature = "html")] +/// assert_eq!(m.extract_html_sync("

a

b

").unwrap(), "a\n\nb\n"); +/// ``` +pub fn from_fn(name: impl Into, f: F) -> FnTransform +where + F: Fn(&mut Document) -> crate::Result<()> + Send + Sync, +{ + FnTransform { + name: name.into(), + f, + } +} + +impl Transform for FnTransform +where + F: Fn(&mut Document) -> crate::Result<()> + Send + Sync, +{ + fn name(&self) -> &str { + &self.name + } + + fn apply(&self, doc: &mut Document) -> crate::Result<()> { + (self.f)(doc) + } +} + +/// Shift heading levels so the shallowest heading becomes `#`, keeping +/// relative depth (an `h3`/`h4` page becomes `#`/`##`). +#[derive(Debug, Clone, Copy, Default)] +pub struct NormalizeHeadings; + +impl Transform for NormalizeHeadings { + fn name(&self) -> &str { + "normalize-headings" + } + + fn apply(&self, doc: &mut Document) -> crate::Result<()> { + let mut min = u8::MAX; + doc.visit(&mut |n| { + if let Node::Heading { level, .. } = n { + min = min.min(*level); + } + }); + if min > 1 && min != u8::MAX { + let shift = min - 1; + doc.visit_mut(&mut |n| { + if let Node::Heading { level, .. } = n { + *level = level.saturating_sub(shift).clamp(1, 6); + } + }); + } + Ok(()) + } +} + +/// Remove images (and figures left empty by it). +#[derive(Debug, Clone, Copy, Default)] +pub struct DropImages; + +impl Transform for DropImages { + fn name(&self) -> &str { + "drop-images" + } + + fn apply(&self, doc: &mut Document) -> crate::Result<()> { + fn prune(nodes: &mut Vec) { + nodes.retain(|n| !matches!(n, Node::Image { .. })); + for n in nodes.iter_mut() { + n.for_each_child_list(&mut |list| prune(list)); + } + nodes.retain(|n| match n { + Node::Paragraph { children, .. } | Node::Figure { children, .. } => { + !children.is_empty() + } + _ => true, + }); + } + prune(&mut doc.nodes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chain_runs_in_order_and_stops_on_error() { + let mut chain = TransformChain::default(); + chain.push(from_fn("a", |d| { + d.nodes.push(Node::Text("a".into())); + Ok(()) + })); + chain.push(from_fn("fail", |_| { + Err(crate::Error::Transform("nope".into())) + })); + chain.push(from_fn("c", |d| { + d.nodes.push(Node::Text("c".into())); + Ok(()) + })); + assert_eq!(format!("{chain:?}"), r#"["a", "fail", "c"]"#); + let mut doc = Document::new(vec![]); + assert!(chain.apply(&mut doc).is_err()); + assert_eq!(doc.nodes, vec![Node::Text("a".into())]); + } + + #[test] + fn normalize_headings_shifts() { + let mut doc = Document::new(vec![Node::heading(3, vec![]), Node::heading(5, vec![])]); + NormalizeHeadings.apply(&mut doc).unwrap(); + let levels: Vec = doc + .nodes + .iter() + .filter_map(|n| match n { + Node::Heading { level, .. } => Some(*level), + _ => None, + }) + .collect(); + assert_eq!(levels, vec![1, 3]); + } + + #[test] + fn drop_images_prunes_empty_paragraphs() { + let img = || Node::Image { + src: "a.png".into(), + alt: String::new(), + }; + let mut doc = Document::new(vec![ + Node::paragraph(vec![img()]), + Node::paragraph(vec![Node::Text("t".into()), img()]), + ]); + DropImages.apply(&mut doc).unwrap(); + assert_eq!( + doc.nodes, + vec![Node::paragraph(vec![Node::Text("t".into())])] + ); + } +} diff --git a/crates/markkit/tests/docx_integration.rs b/crates/markkit/tests/docx_integration.rs index fd2275f..f9bb16a 100644 --- a/crates/markkit/tests/docx_integration.rs +++ b/crates/markkit/tests/docx_integration.rs @@ -380,3 +380,17 @@ fn sources_record_body_block_index() { assert_eq!(idx, vec![Some(0), Some(2), Some(4)]); assert_eq!(p.render().markdown, md(&body)); } + +#[test] +fn transforms_apply_to_docx() { + let m = Markkit::builder() + .transform(markkit::transform::NormalizeHeadings) + .build(); + let out = m + .extract_document_sync( + &docx(&styled_para("Heading3", "Deep")), + DocumentHint::content_type(DOCX_CT), + ) + .unwrap(); + assert_eq!(out.markdown, "# Deep\n"); +} diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index 12918dc..9f446af 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -954,3 +954,74 @@ fn math_dollars_cannot_break_out() { r#"

a$ **b** $c

"#; assert_eq!(convert(html), "$a\\$ **b** \\$c$\n"); } + +// ─── Transforms ────────────────────────────────────────────────────── + +#[test] +fn transforms_run_in_order_for_extract_and_parse() { + use markkit::transform::{self, NormalizeHeadings}; + let m = Markkit::builder() + .transform(NormalizeHeadings) + .transform(transform::from_fn("uppercase", |doc| { + doc.visit_mut(&mut |n| { + if let markkit::ir::Node::Text(t) = n { + *t = t.to_uppercase(); + } + }); + Ok(()) + })) + .build(); + assert_eq!( + m.extract_html_sync("

a

b

").unwrap(), + "# A\n\n## B\n" + ); + let doc = m.parse_html("

x

").unwrap(); + assert!(matches!( + &doc.nodes[0], + markkit::ir::Node::Heading { level: 1, .. } + )); + assert_eq!( + format!("{:?}", m.config().transforms), + r#"["normalize-headings", "uppercase"]"# + ); +} + +#[test] +fn transform_error_aborts_extraction() { + let m = Markkit::builder() + .transform(markkit::transform::from_fn("fail", |_| { + Err(markkit::Error::Transform("boom".into())) + })) + .build(); + let err = m.extract_html_sync("

a

").unwrap_err(); + assert!( + matches!(err, markkit::Error::Transform(ref m) if m == "boom"), + "{err}" + ); +} + +#[test] +fn drop_images_transform() { + let m = Markkit::builder() + .transform(markkit::transform::DropImages) + .build(); + assert_eq!( + m.extract_html_sync(r#"

t

"#) + .unwrap(), + "t\n" + ); +} + +#[test] +fn transforms_apply_to_extract_document_html() { + let m = Markkit::builder() + .transform(markkit::transform::NormalizeHeadings) + .build(); + let e = m + .extract_document_sync( + b"

x

", + markkit::DocumentHint::content_type("text/html"), + ) + .unwrap(); + assert_eq!(e.markdown, "# x\n"); +} diff --git a/examples/transforms.rs b/examples/transforms.rs new file mode 100644 index 0000000..4d8284a --- /dev/null +++ b/examples/transforms.rs @@ -0,0 +1,34 @@ +// Transforms: rewrite the IR between parsing and rendering. + +use markkit::Markkit; +use markkit::ir::Node; +use markkit::transform::{self, NormalizeHeadings}; + +fn main() { + let html = r#" +

Release notes

+

See the changes.

+

Fixes

+
  • Tables keep row headers
+ "#; + + let markkit = Markkit::builder() + // Built-in: shift headings so the shallowest is `#`. + .transform(NormalizeHeadings) + // Custom: replace links with their text. + .transform(transform::from_fn("unlink", |doc| { + doc.visit_mut(&mut |node| { + if let Node::Link { children, .. } = node { + *node = Node::Italic { + children: std::mem::take(children), + }; + } + }); + Ok(()) + })) + .build(); + + println!("Passes: {:?}\n", markkit.config().transforms); + let md = markkit.extract_html_sync(html).expect("extract failed"); + println!("{md}"); +} diff --git a/justfile b/justfile index 26f31ca..49a3bbb 100644 --- a/justfile +++ b/justfile @@ -62,7 +62,7 @@ schema: examples: #!/usr/bin/env bash set -euo pipefail - for ex in basic custom_selectors configuration ir_access; do + for ex in basic custom_selectors configuration ir_access transforms; do echo "── example: $ex" cargo run -q -p markkit --example "$ex" done diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 5acb6b3..8649b79 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -30,10 +30,23 @@ Input (bytes/URL/file) → Format Detection (MIME, magic bytes, extension) → Parser (HTML, PDF, DOCX, etc.) → Normalized AST (intermediate representation) + → Transforms (embedder IR passes, in registration order) → Markdown Converter (AST → markdown string) → Output (String / streaming) ``` +### Transforms + +`MarkkitBuilder::transform(t)` appends a `transform::Transform` +(`name()`, `apply(&mut Document) -> Result<()>`) to `config.transforms`, a +`TransformChain` of `Arc`s so the builder stays `Clone`. Every parse path runs +the chain: `parse_html`, `parse_document`, and so every `extract_*`. Passes +are trusted embedder code; the first error aborts and is returned as is +(`Error::Transform` exists for passes to use). Built-ins: +`NormalizeHeadings` (shallowest heading becomes `#`), `DropImages`; +`transform::from_fn(name, closure)` wraps a closure. `Document::visit` / +`visit_mut` and `Node::for_each_child_list[_ref]` walk the tree. + ## Crate Structure | Crate | Purpose | diff --git a/knowledge/log.md b/knowledge/log.md index 7048a43..3c026e0 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-26 +* **Transforms**: `transform::Transform` passes between parse and render (`MarkkitBuilder::transform`), built-ins `NormalizeHeadings` / `DropImages`, `from_fn`, IR walkers. `Error` is now `#[non_exhaustive]` with `Error::Transform`. See [Markkit Architecture](foundations/architecture.md), [Examples](operations/examples.md). + * **Math + IR JSON Schema**: `Math` / `MathBlock` nodes (TeX), mapped from MathML annotations and KaTeX / MathJax wrappers; `schema/ir.schema.json` generated by `schemars` (`json-schema` feature) and checked in tests. See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md). * **Node options parity**: [Node.js Package](runtimes/node-package.md) functions take a trailing options object (all builder fields plus URL policy) and gain `extractDocument`/`extractDocumentSync`; the parity table now covers Python, Node and WASM. * **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. diff --git a/knowledge/operations/examples.md b/knowledge/operations/examples.md index 6363268..a4b557c 100644 --- a/knowledge/operations/examples.md +++ b/knowledge/operations/examples.md @@ -18,6 +18,7 @@ examples/ configuration.rs # How builder options change output ir_access.rs # Parse to IR, inspect nodes, render separately agent_tool.rs # LLM tool definition, system prompt, execution + transforms.rs # Built-in and closure IR transform passes ``` Examples live at the workspace root and are registered as `[[example]]` targets @@ -55,6 +56,12 @@ Run one with `cargo run -p markkit --example `, all with `just examples`. - Prints the OpenAI tool definition and system prompt, then executes model-style arguments and prints `ToolOutput::result` (see [Tool Contract](../integrations/tool-contract.md)) +### transforms.rs + +- `transform(NormalizeHeadings)` shifts `h3`/`h4` to `#`/`##` +- `transform::from_fn` + `Document::visit_mut` rewrites links in place +- Prints the pass names (`config().transforms`) and the result + ## Requirements - No network or file access; inline HTML samples only