From 9b8cbe03ce1ae24959a46908a3f2532de8151b33 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:49:30 +0000 Subject: [PATCH 1/3] feat(ir): math nodes and IR JSON Schema - Node::Math / MathBlock (TeX), rendered as $tex$ / $$ blocks with dollar escaping - HTML: MathML x-tex annotation, KaTeX and MathJax wrappers - schema/ir.schema.json generated via schemars (json-schema feature), drift-checked in tests; just schema regenerates --- CHANGELOG.md | 2 + Cargo.lock | 63 ++ Cargo.toml | 1 + crates/markkit/Cargo.toml | 3 + crates/markkit/src/html/parser.rs | 59 ++ crates/markkit/src/ir.rs | 69 ++- crates/markkit/src/lib.rs | 1 + crates/markkit/src/render/markdown.rs | 63 +- crates/markkit/tests/html_integration.rs | 34 + justfile | 4 + knowledge/formats/html-extraction.md | 2 + knowledge/foundations/ir.md | 13 +- knowledge/log.md | 1 + schema/ir.schema.json | 759 +++++++++++++++++++++++ 14 files changed, 1069 insertions(+), 5 deletions(-) create mode 100644 schema/ir.schema.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 3164e20..8682337 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `
`/`
` and table spans/alignment. Row-header `` cells in table bodies are no longer dropped. - `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON. +- Math: `$tex$` and `$$` blocks from MathML (`application/x-tex` annotation), KaTeX and MathJax output. +- JSON Schema for the IR at `schema/ir.schema.json`; `json-schema` feature adds `markkit::ir::json_schema()`. - Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations. - `Markkit::parse_document` / `parse_document_sync` return the IR plus metadata (`ParsedDocument`) diff --git a/Cargo.lock b/Cargo.lock index 1383fef..7a91720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,6 +573,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ego-tree" version = "0.11.1" @@ -1193,6 +1199,7 @@ dependencies = [ "proptest", "quick-xml", "reqwest", + "schemars", "scraper", "serde", "serde_json", @@ -1831,6 +1838,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "regex" version = "1.13.1" @@ -2050,6 +2077,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.9", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2157,6 +2209,17 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "serde_json" version = "1.0.154" diff --git a/Cargo.toml b/Cargo.toml index aae37d7..5fa6158 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ reqwest = { version = "0.13", default-features = false, features = ["rustls", "j # HTML parsing scraper = "0.29" ego-tree = "0.11" +schemars = "1" html5ever = "0.40" # Document formats diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 90c26e8..6c1af79 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -23,6 +23,8 @@ http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] # LLM tool contract (`markkit::tool`): Everruns toolkit-library shape. tool = ["dep:tower", "dep:futures-core", "dep:base64"] +# JSON Schema for the IR (`markkit::ir::json_schema()`) +json-schema = ["dep:schemars"] all_formats = ["html", "pdf", "docx", "spreadsheet"] [dependencies] @@ -35,6 +37,7 @@ thiserror.workspace = true # Serialization serde.workspace = true serde_json.workspace = true +schemars = { workspace = true, optional = true } # HTML (optional) scraper = { workspace = true, optional = true } diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 759e819..19cbb05 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -12,6 +12,10 @@ //! - Provenance (`track_sources`): blocks get the absolute XPath of the //! element they came from. Sibling indexes are computed once per parent and //! cached, so cost is O(depth) per block, not O(siblings). +//! - Math: MathML `` (also inside KaTeX `.katex` / `.katex-display` +//! and MathJax ``) maps to `Math` / `MathBlock` using the +//! `application/x-tex` annotation; without one, the text content is kept as +//! plain text. KaTeX's visual HTML duplicate is dropped. //! - `
` becomes `Figure` only when it has a non-empty `
`; //! otherwise it stays a transparent container as before. @@ -193,6 +197,11 @@ impl<'a> HtmlParser<'a> { out.push(fig); continue; } + if let Some(mut m) = self.math_wrapper(el) { + self.mark_source(&mut m, el); + out.push(m); + continue; + } // Transparent containers: recurse directly into their children if is_transparent_container(name) { self.convert_children(el, exclude, out, depth + 1); @@ -223,6 +232,10 @@ impl<'a> HtmlParser<'a> { return Some(Node::Text(trimmed.to_string())); } + if let Some(m) = self.math_wrapper(el) { + return Some(m); + } + let tag = el.value().name(); match tag { @@ -424,6 +437,45 @@ impl<'a> HtmlParser<'a> { } } + /// MathML or a KaTeX / MathJax wrapper around it. + fn math_wrapper(&self, el: scraper::ElementRef<'_>) -> Option { + let v = el.value(); + let (display, math) = match v.name() { + "math" => (v.attr("display") == Some("block"), el), + "mjx-container" => (v.attr("display") == Some("true"), find_math(el)?), + _ if has_class(&el, "katex-display") => (true, find_math(el)?), + _ if has_class(&el, "katex") => (false, find_math(el)?), + _ => return None, + }; + let display = display || math.value().attr("display") == Some("block"); + let annotation = math + .descendants() + .filter_map(scraper::ElementRef::wrap) + .find(|e| { + e.value().name() == "annotation" + && e.value() + .attr("encoding") + .is_some_and(|enc| enc.eq_ignore_ascii_case("application/x-tex")) + }); + match annotation { + Some(a) => { + let tex = a.text().collect::().trim().to_string(); + if tex.is_empty() { + None + } else if display { + Some(Node::MathBlock { tex, source: None }) + } else { + Some(Node::Math { tex }) + } + } + None => { + let text = collapse_whitespace(&math.text().collect::()); + let text = text.trim(); + (!text.is_empty()).then(|| Node::Text(text.to_string())) + } + } + } + /// Generic container (`div`, `span`, ...) in inline/element context. fn convert_container( &self, @@ -873,6 +925,13 @@ fn task_checkbox(li: scraper::ElementRef<'_>) -> Option { first_checkbox(li, false) } +/// First `` element inside a wrapper. +fn find_math(el: scraper::ElementRef<'_>) -> Option> { + el.descendants() + .filter_map(scraper::ElementRef::wrap) + .find(|e| e.value().name() == "math") +} + fn has_class(el: &scraper::ElementRef<'_>, class: &str) -> bool { el.value() .attr("class") diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs index 1636d5d..1682481 100644 --- a/crates/markkit/src/ir.rs +++ b/crates/markkit/src/ir.rs @@ -19,11 +19,13 @@ use serde::{Deserialize, Serialize}; /// Current IR schema version written by this crate. /// /// v1: initial IR (implicit, no version field). v2: footnotes, definition -/// lists, task lists, figures, table cells with alignment and spans. +/// lists, task lists, figures, math, table cells with alignment and spans, +/// per-block provenance. pub const IR_VERSION: u32 = 2; /// A parsed document. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub struct Document { /// IR schema version ([`IR_VERSION`] for documents built by this crate; @@ -52,6 +54,7 @@ impl Document { /// A block-level or inline node in the document tree. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub enum Node { /// Heading with level 1–6. @@ -114,6 +117,12 @@ pub enum Node { #[serde(default, skip_serializing_if = "Option::is_none")] source: Option>, }, + /// Display math (TeX source). + MathBlock { + tex: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Footnote body, referenced by [`Node::FootnoteReference`] with the same label. FootnoteDefinition { label: String, @@ -143,6 +152,8 @@ pub enum Node { LineBreak, /// Footnote marker pointing at a [`Node::FootnoteDefinition`]. FootnoteReference { label: String }, + /// Inline math (TeX source). + Math { tex: String }, } impl Node { @@ -160,6 +171,7 @@ impl Node { | Node::DefinitionList { .. } | Node::Figure { .. } | Node::FootnoteDefinition { .. } + | Node::MathBlock { .. } ) } @@ -178,7 +190,8 @@ impl Node { | Node::Table { source, .. } | Node::DefinitionList { source, .. } | Node::Figure { source, .. } - | Node::FootnoteDefinition { source, .. } => Some(source), + | Node::FootnoteDefinition { source, .. } + | Node::MathBlock { source, .. } => Some(source), _ => None, } } @@ -194,7 +207,8 @@ impl Node { | Node::Table { source, .. } | Node::DefinitionList { source, .. } | Node::Figure { source, .. } - | Node::FootnoteDefinition { source, .. } => source.as_deref(), + | Node::FootnoteDefinition { source, .. } + | Node::MathBlock { source, .. } => source.as_deref(), _ => None, } } @@ -236,6 +250,7 @@ impl Node { /// Only the fields meaningful for the input format are set. Recorded when /// [`crate::MarkkitBuilder::track_sources`] is enabled. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub struct Source { /// HTML: absolute XPath of the element, e.g. `/html/body/article[1]/p[3]` @@ -291,6 +306,7 @@ impl Source { /// A single item in a list. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub struct ListItem { pub children: Vec, @@ -318,6 +334,7 @@ impl ListItem { /// Horizontal alignment of a table column. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub enum Alignment { #[default] @@ -329,6 +346,7 @@ pub enum Alignment { /// A table cell: inline content plus its span (both ≥ 1). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub struct TableCell { pub children: Vec, @@ -371,6 +389,7 @@ impl From> for TableCell { /// One term of a [`Node::DefinitionList`] with its definitions. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[non_exhaustive] pub struct DefinitionItem { /// Inline content of the term. @@ -385,6 +404,27 @@ impl DefinitionItem { } } +/// JSON Schema (draft 2020-12) of the serialized IR ([`Document`]). +/// +/// The same schema is committed as `schema/ir.schema.json`; a test keeps +/// them in sync (regenerate with `just schema`). +#[cfg(feature = "json-schema")] +pub fn json_schema() -> serde_json::Value { + let mut schema = + serde_json::to_value(schemars::schema_for!(Document)).expect("schema serializes"); + if let Some(obj) = schema.as_object_mut() { + obj.insert( + "$id".into(), + "https://github.com/everruns/markkit/schema/ir.schema.json".into(), + ); + obj.insert( + "description".into(), + format!("markkit IR document, ir_version {IR_VERSION}").into(), + ); + } + schema +} + #[cfg(test)] mod tests { use super::*; @@ -444,6 +484,14 @@ mod tests { caption: Some(vec![Node::Text("cap".into())]), source: None, }, + Node::MathBlock { + tex: "e^{i\\pi}+1=0".into(), + source: None, + }, + Node::Paragraph { + children: vec![Node::Math { tex: "x^2".into() }], + source: None, + }, Node::FootnoteDefinition { label: "1".into(), children: vec![Node::paragraph(vec![Node::Text("note".into())])], @@ -488,6 +536,21 @@ mod tests { assert!(json.contains(r#""source":{"index":3}"#), "{json}"); } + #[cfg(feature = "json-schema")] + #[test] + fn committed_schema_is_current() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../schema/ir.schema.json"); + let generated = serde_json::to_string_pretty(&json_schema()).unwrap() + "\n"; + if std::env::var_os("MARKKIT_UPDATE_SCHEMA").is_some() { + std::fs::write(path, &generated).unwrap(); + } + let committed = std::fs::read_to_string(path).unwrap_or_default(); + assert!( + committed == generated, + "schema/ir.schema.json is stale; run `just schema`" + ); + } + #[test] fn zero_span_clamped() { let c = TableCell::new(vec![]).with_span(0, 0); diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 4bb43a5..4c703b2 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -38,6 +38,7 @@ //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest //! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the //! Everruns toolkit-library contract (schemas, OpenAI definition, `tower::Service`) +//! - **JSON Schema** (`json-schema`): [`ir::json_schema`] for the serialized IR //! //! Embedders that do their own fetching can drop reqwest entirely: //! `markkit = { version = "0.1", default-features = false, features = ["docx", "spreadsheet"] }`. diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs index 9592f80..dca3459 100644 --- a/crates/markkit/src/render/markdown.rs +++ b/crates/markkit/src/render/markdown.rs @@ -138,6 +138,18 @@ impl MarkdownRenderer { } } + Node::MathBlock { tex, .. } => { + let tex = tex.trim().replace("$$", "$ $"); + if !tex.is_empty() { + self.buf.push_str("$$\n"); + for line in tex.lines().filter(|l| !l.trim().is_empty()) { + self.buf.push_str(line); + self.buf.push('\n'); + } + self.buf.push_str("$$\n"); + } + } + Node::FootnoteDefinition { label, children, .. } => { @@ -157,7 +169,8 @@ impl MarkdownRenderer { | Node::Link { .. } | Node::Image { .. } | Node::LineBreak - | Node::FootnoteReference { .. } => { + | Node::FootnoteReference { .. } + | Node::Math { .. } => { self.render_inline_node(node); } } @@ -318,6 +331,14 @@ impl MarkdownRenderer { Node::LineBreak => { self.buf.push_str(" \n"); } + Node::Math { tex } => { + let tex = inline_math(tex); + if !tex.is_empty() { + self.buf.push('$'); + self.buf.push_str(&tex); + self.buf.push('$'); + } + } Node::FootnoteReference { label } => { self.buf.push_str("[^"); self.buf.push_str(&footnote_label(label)); @@ -458,6 +479,20 @@ fn push_hanging(buf: &mut String, inner: &str, first: &str, rest: &str) { } } +/// Inline TeX for `$...$`: one line, `$` escaped, no edge spaces (which +/// would stop most parsers from treating it as math). +fn inline_math(tex: &str) -> String { + let mut out = String::with_capacity(tex.len()); + for c in tex.trim().chars() { + match c { + '\r' | '\n' => out.push(' '), + '$' if !out.ends_with('\\') => out.push_str("\\$"), + c => out.push(c), + } + } + out +} + /// Footnote labels are emitted inside `[^...]`: keep `[A-Za-z0-9_.-]`, map /// everything else to `-` so a label cannot close the bracket or break the line. fn footnote_label(label: &str) -> String { @@ -562,6 +597,32 @@ mod tests { assert_eq!(normalize_output("a \nb\n"), "a \nb\n"); } + #[test] + fn test_inline_math() { + assert_eq!(inline_math(" a$b\nc "), "a\\$b c"); + assert_eq!(inline_math("\\$"), "\\$"); + } + + #[test] + fn math_rendering() { + let doc = Document::new(vec![ + Node::paragraph(vec![ + Node::Text("Euler ".into()), + Node::Math { + tex: "e^{i\\pi}".into(), + }, + ]), + Node::MathBlock { + tex: "a\n\nb $$ c".into(), + source: None, + }, + ]); + assert_eq!( + MarkdownRenderer::new().render(&doc), + "Euler $e^{i\\pi}$\n\n$$\na\nb $ $ c\n$$\n" + ); + } + #[test] fn test_footnote_label() { assert_eq!(footnote_label("1"), "1"); diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index a77f18f..12918dc 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -920,3 +920,37 @@ fn sources_do_not_change_markdown() { let b = convert_with(html, |b| b.track_sources(true)); assert_eq!(a, b); } + +// ─── Math ──────────────────────────────────────────────────────────── + +#[test] +fn mathml_with_tex_annotation() { + let html = r#"

Area π\pi r^2 here.

"#; + assert_eq!(convert(html), "Area $\\pi r^2$ here.\n"); +} + +#[test] +fn katex_inline_and_display() { + let html = r#"

Inline xx^2.

+
E = mc^2E=mc2
"#; + assert_eq!(convert(html), "Inline $x^2$.\n\n$$\nE = mc^2\n$$\n"); +} + +#[test] +fn mathjax_container() { + let html = r#"visual\int f"#; + assert_eq!(convert(html), "$$\n\\int f\n$$\n"); +} + +#[test] +fn mathml_without_tex_keeps_text() { + let html = "

See a+b.

"; + assert_eq!(convert(html), "See a+b.\n"); +} + +#[test] +fn math_dollars_cannot_break_out() { + let html = + r#"

a$ **b** $c

"#; + assert_eq!(convert(html), "$a\\$ **b** \\$c$\n"); +} diff --git a/justfile b/justfile index 5aa093d..26f31ca 100644 --- a/justfile +++ b/justfile @@ -54,6 +54,10 @@ test-one name: test-scripts: python3 -m unittest discover -s scripts/tests -p 'test_*.py' +# Regenerate schema/ir.schema.json from the IR types +schema: + MARKKIT_UPDATE_SCHEMA=1 cargo test -p markkit --features json-schema --lib ir::tests::committed_schema_is_current + # Run every example (CI runs this too) examples: #!/usr/bin/env bash diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index b175633..cef2a4a 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -39,6 +39,7 @@ HTML from the web: malformed markup, complex layouts, embedded scripts, etc. | `
  • ` | `- [x] item` | | `
    ` + `
    ` | content + `*caption*` | | footnote refs / sections | `[^1]` / `[^1]: body` | +| ``, KaTeX, MathJax | `$tex$` / `$$` block | | `/` | `*text*` | | `/` | `**text**` | | `/` | `~~text~~` | @@ -74,6 +75,7 @@ HTML from the web: malformed markup, complex layouts, embedded scripts, etc. | ``, ``, `` | `Strikethrough { children }` | Empty -> elided. | | `` | `Link { url, title, children }` | When `preserve_links=false`, unwrapped to children. Empty -> elided. Missing href -> empty string. | | `` | `Image { src, alt }` | When `preserve_images=false`, elided entirely. Missing attrs -> empty strings. | +| `` | `Math { tex }` / `MathBlock { tex }` | TeX from ``; `display="block"` -> block. Same for KaTeX `.katex` (inline) / `.katex-display` (block) and MathJax `` wrappers, whose visual HTML is dropped. No TeX annotation -> collapsed text content. | | footnote `` | `FootnoteReference { label }` | `` with `data-footnote-ref`, `role=doc-noteref`, or class `footnote-ref`. Label = link text without `[]` (else target id). Back-links (`data-footnote-backref`, `role=doc-backlink`, `.footnote-back[ref]`) dropped. | ### Transparent Container Elements diff --git a/knowledge/foundations/ir.md b/knowledge/foundations/ir.md index c18398a..58b71ab 100644 --- a/knowledge/foundations/ir.md +++ b/knowledge/foundations/ir.md @@ -37,7 +37,8 @@ Methods: `new(Vec)` (sets `ir_version = IR_VERSION`), `is_empty() -> bool` - `markkit::ir::IR_VERSION` is the serialized schema version. v1 was the initial, unversioned IR. v2 adds footnotes, definition lists, task lists, - figures and table cells with alignment and spans. Bump it on any breaking + figures, math, table cells with alignment and spans, and per-block + provenance. Bump it on any breaking change to serialized form. - `Node`, `ListItem`, `TableCell`, `DefinitionItem`, `Alignment` and `Document` are `#[non_exhaustive]`: new kinds/fields are additive; external @@ -67,6 +68,7 @@ All block variants except `ThematicBreak` also carry `source` (see Provenance). | `DefinitionList` | `items: Vec` | Terms with one or more definitions. | | `Figure` | `children: Vec`, `caption: Option>` | Block content with inline caption. | | `FootnoteDefinition` | `label: String`, `children: Vec` | Footnote body (blocks). | +| `MathBlock` | `tex: String` | Display math, TeX source. Renders `$$` fenced; `$$` inside is split. | #### Inline Variants @@ -81,6 +83,7 @@ All block variants except `ThematicBreak` also carry `source` (see Provenance). | `Image` | `src: String`, `alt: String` | Image reference. Leaf node (no children). | | `LineBreak` | (unit) | Hard line break. Renders as two trailing spaces + newline. | | `FootnoteReference` | `label: String` | Marker pointing at the `FootnoteDefinition` with the same label. | +| `Math` | `tex: String` | Inline math, TeX source. Renders `$tex$`: one line, trimmed, unescaped `$` escaped. | ### Provenance (`Source`) @@ -194,6 +197,14 @@ and default-valued fields (`alignments`, `colspan`/`rowspan` = 1, `checked`, } ``` +### JSON Schema + +`schema/ir.schema.json` (draft 2020-12) describes the serialized `Document`. +It is generated from the Rust types with `schemars` behind the `json-schema` +feature (`markkit::ir::json_schema()`); `ir::tests::committed_schema_is_current` +fails when the committed file drifts. Regenerate with `just schema`. Python and +Node consumers of IR JSON validate against this file. + Roundtrip guarantee: `serialize -> deserialize` produces `==` document. Tested in `ir::tests::ir_roundtrip_json` and `html_integration::ir_serialization_roundtrip`. diff --git a/knowledge/log.md b/knowledge/log.md index 5e83aa4..4fb7068 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Math + IR JSON Schema**: `Math` / `MathBlock` nodes (TeX), mapped from MathML annotations and KaTeX / MathJax wrappers; `schema/ir.schema.json` generated by `schemars` (`json-schema` feature) and checked in tests. See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md). * **Node options parity**: [Node.js Package](runtimes/node-package.md) functions take a trailing options object (all builder fields plus URL policy) and gain `extractDocument`/`extractDocumentSync`; the parity table now covers Python, Node and WASM. * **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. diff --git a/schema/ir.schema.json b/schema/ir.schema.json new file mode 100644 index 0000000..84045ee --- /dev/null +++ b/schema/ir.schema.json @@ -0,0 +1,759 @@ +{ + "$defs": { + "Alignment": { + "description": "Horizontal alignment of a table column.", + "enum": [ + "None", + "Left", + "Center", + "Right" + ], + "type": "string" + }, + "DefinitionItem": { + "description": "One term of a [`Node::DefinitionList`] with its definitions.", + "properties": { + "definitions": { + "description": "Each definition is a list of block (or inline) nodes.", + "items": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "type": "array" + }, + "term": { + "description": "Inline content of the term.", + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "term", + "definitions" + ], + "type": "object" + }, + "ListItem": { + "description": "A single item in a list.", + "properties": { + "checked": { + "description": "Task-list state: `Some(true)` checked, `Some(false)` unchecked, `None` plain item.", + "type": [ + "boolean", + "null" + ] + }, + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "children" + ], + "type": "object" + }, + "Node": { + "description": "A block-level or inline node in the document tree.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Heading with level 1–6.", + "properties": { + "Heading": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "level": { + "format": "uint8", + "maximum": 255, + "minimum": 0, + "type": "integer" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "level", + "children" + ], + "type": "object" + } + }, + "required": [ + "Heading" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Paragraph containing inline content.", + "properties": { + "Paragraph": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "Paragraph" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Ordered or unordered list.", + "properties": { + "List": { + "properties": { + "items": { + "items": { + "$ref": "#/$defs/ListItem" + }, + "type": "array" + }, + "ordered": { + "type": "boolean" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + }, + "start": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "ordered", + "items" + ], + "type": "object" + } + }, + "required": [ + "List" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Block quotation.", + "properties": { + "BlockQuote": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "BlockQuote" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Fenced code block.", + "properties": { + "CodeBlock": { + "properties": { + "code": { + "type": "string" + }, + "language": { + "type": [ + "string", + "null" + ] + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "code" + ], + "type": "object" + } + }, + "required": [ + "CodeBlock" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Table with optional header row.", + "properties": { + "Table": { + "properties": { + "alignments": { + "description": "Per-column alignment; missing entries mean [`Alignment::None`].", + "items": { + "$ref": "#/$defs/Alignment" + }, + "type": "array" + }, + "headers": { + "items": { + "$ref": "#/$defs/TableCell" + }, + "type": "array" + }, + "rows": { + "items": { + "items": { + "$ref": "#/$defs/TableCell" + }, + "type": "array" + }, + "type": "array" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "headers", + "rows" + ], + "type": "object" + } + }, + "required": [ + "Table" + ], + "type": "object" + }, + { + "const": "ThematicBreak", + "description": "Horizontal rule / thematic break.", + "type": "string" + }, + { + "additionalProperties": false, + "description": "Definition list (`
    `): terms with one or more definitions.", + "properties": { + "DefinitionList": { + "properties": { + "items": { + "items": { + "$ref": "#/$defs/DefinitionItem" + }, + "type": "array" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "items" + ], + "type": "object" + } + }, + "required": [ + "DefinitionList" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Figure: block content with an optional inline caption.", + "properties": { + "Figure": { + "properties": { + "caption": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": [ + "array", + "null" + ] + }, + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "Figure" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Display math (TeX source).", + "properties": { + "MathBlock": { + "properties": { + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + }, + "tex": { + "type": "string" + } + }, + "required": [ + "tex" + ], + "type": "object" + } + }, + "required": [ + "MathBlock" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Footnote body, referenced by [`Node::FootnoteReference`] with the same label.", + "properties": { + "FootnoteDefinition": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "label": { + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/Source" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "label", + "children" + ], + "type": "object" + } + }, + "required": [ + "FootnoteDefinition" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Plain text.", + "properties": { + "Text": { + "type": "string" + } + }, + "required": [ + "Text" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Bold / strong emphasis.", + "properties": { + "Bold": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "Bold" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Italic / emphasis.", + "properties": { + "Italic": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "Italic" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Strikethrough.", + "properties": { + "Strikethrough": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "required": [ + "Strikethrough" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Inline code.", + "properties": { + "InlineCode": { + "type": "string" + } + }, + "required": [ + "InlineCode" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Hyperlink.", + "properties": { + "Link": { + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": "string" + } + }, + "required": [ + "url", + "children" + ], + "type": "object" + } + }, + "required": [ + "Link" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Image.", + "properties": { + "Image": { + "properties": { + "alt": { + "type": "string" + }, + "src": { + "type": "string" + } + }, + "required": [ + "src", + "alt" + ], + "type": "object" + } + }, + "required": [ + "Image" + ], + "type": "object" + }, + { + "const": "LineBreak", + "description": "Hard line break.", + "type": "string" + }, + { + "additionalProperties": false, + "description": "Footnote marker pointing at a [`Node::FootnoteDefinition`].", + "properties": { + "FootnoteReference": { + "properties": { + "label": { + "type": "string" + } + }, + "required": [ + "label" + ], + "type": "object" + } + }, + "required": [ + "FootnoteReference" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Inline math (TeX source).", + "properties": { + "Math": { + "properties": { + "tex": { + "type": "string" + } + }, + "required": [ + "tex" + ], + "type": "object" + } + }, + "required": [ + "Math" + ], + "type": "object" + } + ] + }, + "Source": { + "description": "Provenance of a block: where in the input it came from.\n\nOnly the fields meaningful for the input format are set. Recorded when\n[`crate::MarkkitBuilder::track_sources`] is enabled.", + "properties": { + "index": { + "description": "DOCX: 0-based index of the body-level paragraph or table.", + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "page": { + "description": "Paged formats: 1-based page number.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "path": { + "description": "HTML: absolute XPath of the element, e.g. `/html/body/article[1]/p[3]`\n(indexes count same-name element siblings, 1-based, in the parsed DOM).", + "type": [ + "string", + "null" + ] + }, + "range": { + "description": "Spreadsheets: A1-style cell range, e.g. `A1:D20`.", + "type": [ + "string", + "null" + ] + }, + "sheet": { + "description": "Spreadsheets: sheet name.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "TableCell": { + "description": "A table cell: inline content plus its span (both ≥ 1).", + "properties": { + "children": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + }, + "colspan": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "rowspan": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "children" + ], + "type": "object" + } + }, + "$id": "https://github.com/everruns/markkit/schema/ir.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "markkit IR document, ir_version 2", + "properties": { + "ir_version": { + "default": 1, + "description": "IR schema version ([`IR_VERSION`] for documents built by this crate;\n`1` when missing from deserialized JSON).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "nodes": { + "items": { + "$ref": "#/$defs/Node" + }, + "type": "array" + } + }, + "required": [ + "nodes" + ], + "title": "Document", + "type": "object" +} From 848888e59b8bb994a356136290ae53021dcf15af Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 00:02:33 +0000 Subject: [PATCH 2/3] docs: fix json_schema intra-doc link when feature is off --- crates/markkit/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 4c703b2..29b4dc4 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -38,7 +38,7 @@ //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest //! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the //! Everruns toolkit-library contract (schemas, OpenAI definition, `tower::Service`) -//! - **JSON Schema** (`json-schema`): [`ir::json_schema`] for the serialized IR +//! - **JSON Schema** (`json-schema`): `ir::json_schema()` for the serialized IR //! //! Embedders that do their own fetching can drop reqwest entirely: //! `markkit = { version = "0.1", default-features = false, features = ["docx", "spreadsheet"] }`. From f38631a3235b52f039d7bc0514cc66f055341422 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 00:27:51 +0000 Subject: [PATCH 3/3] chore(knowledge): tidy log after rebase --- knowledge/log.md | 1 - 1 file changed, 1 deletion(-) diff --git a/knowledge/log.md b/knowledge/log.md index 4fb7068..7048a43 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,7 +4,6 @@ * **Math + IR JSON Schema**: `Math` / `MathBlock` nodes (TeX), mapped from MathML annotations and KaTeX / MathJax wrappers; `schema/ir.schema.json` generated by `schemars` (`json-schema` feature) and checked in tests. See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md). * **Node options parity**: [Node.js Package](runtimes/node-package.md) functions take a trailing options object (all builder fields plus URL policy) and gain `extractDocument`/`extractDocumentSync`; the parity table now covers Python, Node and WASM. - * **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. * **Ship contract**: "ship" now ends with the merge commit's `main` CI green, not at merge; the `ship` skill gains existing-PR, auto-merge, and post-merge verification steps aligned with bashkit. See [Maintenance](operations/maintenance.md). * **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md).