diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e175..b68f4fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `
`/`
` and table spans/alignment. Row-header `` cells in table bodies are no longer dropped. - `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON. +- Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block + (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations. +- `Markkit::parse_document` / `parse_document_sync` return the IR plus metadata (`ParsedDocument`) + for any supported format; `ParsedDocument::render()` produces the `Extraction`. - `tool` feature: `markkit::tool::MarkkitTool`, an LLM agent tool following the Everruns toolkit-library contract (builder, OpenAI tool definition, input/output schemas, `tower::Service`, single-use `ToolExecution`, `en-US`/`uk-UA` text). One call converts a `url`, `html` or base64 @@ -38,7 +42,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **Breaking (IR API):** `Node`, `Document`, `ListItem` and the new IR structs are `#[non_exhaustive]`; matches need a `_` arm and structs are built with constructors. `Node::Table` cells are now - `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. DOCX `gridSpan` is + `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. Block variants gain a + `source` field (use `..` in patterns, `Node::paragraph`/`Node::heading` to construct). DOCX `gridSpan` is carried as `colspan`. - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. diff --git a/Cargo.lock b/Cargo.lock index b3aa161..2c348ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1187,6 +1187,7 @@ dependencies = [ "calamine", "cfb", "criterion", + "ego-tree", "futures-core", "html5ever", "proptest", diff --git a/Cargo.toml b/Cargo.toml index ab34c1a..de8f51b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ reqwest = { version = "0.13", default-features = false, features = ["rustls", "j # HTML parsing scraper = "0.29" +ego-tree = "0.11" html5ever = "0.40" # Document formats diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 424777b..90c26e8 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -15,7 +15,7 @@ readme = "../../README.md" [features] default = ["html", "http_client"] -html = ["dep:scraper", "dep:html5ever"] +html = ["dep:scraper", "dep:html5ever", "dep:ego-tree"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] @@ -38,6 +38,7 @@ serde_json.workspace = true # HTML (optional) scraper = { workspace = true, optional = true } +ego-tree = { workspace = true, optional = true } html5ever = { workspace = true, optional = true } # Office documents (optional): DOCX + spreadsheets share the ZIP/XML stack diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index 9f86747..542d7d7 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -40,6 +40,10 @@ pub struct MarkkitBuilder { pub allowed_hosts: Vec, /// Maximum redirects `extract_url` follows. Default: 5. pub max_redirects: usize, + /// Record per-block provenance ([`crate::ir::Source`]) in the IR: + /// HTML element XPath, DOCX block index, spreadsheet sheet and range. + /// Default: false (no cost, no `source` in IR JSON). + pub track_sources: bool, } impl Default for MarkkitBuilder { @@ -69,6 +73,7 @@ impl Default for MarkkitBuilder { allow_private_networks: false, allowed_hosts: Vec::new(), max_redirects: 5, + track_sources: false, } } } @@ -94,6 +99,12 @@ impl MarkkitBuilder { self } + /// Record per-block provenance in the IR (see [`MarkkitBuilder::track_sources`] field). + pub fn track_sources(mut self, yes: bool) -> Self { + self.track_sources = yes; + self + } + pub fn include_selector(mut self, selector: impl Into) -> Self { self.include_selectors.push(selector.into()); self diff --git a/crates/markkit/src/docx.rs b/crates/markkit/src/docx.rs index 98d98ac..4f030d6 100644 --- a/crates/markkit/src/docx.rs +++ b/crates/markkit/src/docx.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::archive::{ArchiveError, ZipGuard}; use crate::config::MarkkitBuilder; use crate::error::Error; -use crate::ir::{Document, ListItem, Node, TableCell}; +use crate::ir::{Document, ListItem, Node, Source, TableCell}; use crate::render::escape_table_cell; use crate::xml::{self, Element, XmlError}; @@ -589,7 +589,7 @@ fn plain_text(nodes: &[Node], out: &mut String) { | Node::Strikethrough { children } | Node::Link { children, .. } | Node::Heading { children, .. } - | Node::Paragraph { children } => plain_text(children, out), + | Node::Paragraph { children, .. } => plain_text(children, out), _ => {} } } @@ -602,6 +602,7 @@ struct ListEntry { ordered: bool, start: Option, content: Vec, + index: usize, } #[derive(Default)] @@ -611,6 +612,11 @@ struct Body { title_text: Option, paragraphs: usize, tables: usize, + /// Record `Source::index` on blocks (`track_sources`). + track: bool, + /// Next body block index (every `w:p` / `w:tbl`, in document order, + /// including those inside `w:sdt` / `w:ins` wrappers and blank ones). + next_index: usize, } impl Body { @@ -622,9 +628,18 @@ impl Body { let base = entries.iter().map(|e| e.level).min().unwrap_or(0); let mut pos = 0; while pos < entries.len() { - self.blocks.push(build_list(&entries, &mut pos, base)); + let index = entries[pos].index; + let list = build_list(&entries, &mut pos, base); + self.push(list, index); } } + + fn push(&mut self, mut node: Node, index: usize) { + if self.track { + node.set_source(Source::index(index)); + } + self.blocks.push(node); + } } /// Build a (nested) list starting at `entries[*pos]` for `level`. @@ -652,6 +667,7 @@ fn build_list(entries: &[ListEntry], pos: &mut usize, level: u32) -> Node { } items.push(ListItem::new(vec![Node::Paragraph { children: e.content.clone(), + source: None, }])); *pos += 1; } @@ -659,6 +675,7 @@ fn build_list(entries: &[ListEntry], pos: &mut usize, level: u32) -> Node { ordered, start, items, + source: None, } } @@ -666,12 +683,18 @@ impl Ctx<'_> { fn blocks(&self, container: &Element, body: &mut Body) { for child in container.elements() { match child.name.as_str() { - "p" => self.paragraph(child, body), + "p" => { + let index = body.next_index; + body.next_index += 1; + self.paragraph(child, body, index); + } "tbl" => { + let index = body.next_index; + body.next_index += 1; body.flush_list(); if let Some(t) = self.table(child) { body.tables += 1; - body.blocks.push(t); + body.push(t, index); } } "sdt" => { @@ -685,7 +708,7 @@ impl Ctx<'_> { } } - fn paragraph(&self, p: &Element, body: &mut Body) { + fn paragraph(&self, p: &Element, body: &mut Body, index: usize) { let ppr = p.child("pPr"); let style = ppr.and_then(|x| x.child_val("pStyle")); let content = self.inline(p); @@ -709,10 +732,14 @@ impl Ctx<'_> { body.title_text = Some(t.trim().to_string()); } body.flush_list(); - body.blocks.push(Node::Heading { - level, - children: content, - }); + body.push( + Node::Heading { + level, + children: content, + source: None, + }, + index, + ); return; } @@ -749,10 +776,17 @@ impl Ctx<'_> { ordered, start, content, + index, }), None => { body.flush_list(); - body.blocks.push(Node::Paragraph { children: content }); + body.push( + Node::Paragraph { + children: content, + source: None, + }, + index, + ); } } } @@ -828,6 +862,7 @@ impl Ctx<'_> { alignments: Vec::new(), headers, rows, + source: None, }) } } @@ -907,7 +942,10 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result, } -impl Extraction { - fn new(markdown: String, format: Format) -> Self { +/// Result of [`Markkit::parse_document`]: the IR plus metadata, not rendered. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[non_exhaustive] +pub struct ParsedDocument { + /// Canonical IR (blocks carry [`crate::ir::Source`] when + /// [`MarkkitBuilder::track_sources`] is enabled). + pub document: Document, + /// Document title when known. + pub title: Option, + /// Format the document was parsed as. + pub format: Format, + /// Non-fatal issues (see [`Extraction::warnings`]). + pub warnings: Vec, + /// Format-specific counters. + pub stats: ExtractionStats, + /// Document metadata (see [`Extraction::metadata`]). + pub metadata: Metadata, +} + +impl ParsedDocument { + #[cfg_attr( + not(any(feature = "html", feature = "docx", feature = "spreadsheet")), + allow(dead_code) + )] + fn new(document: Document, format: Format) -> Self { Self { - markdown, + document, title: None, format, warnings: Vec::new(), @@ -78,6 +102,19 @@ impl Extraction { metadata: Metadata::default(), } } + + /// Render to Markdown, keeping the metadata. + pub fn render(self) -> Extraction { + let mut renderer = crate::render::MarkdownRenderer::new(); + Extraction { + markdown: renderer.render(&self.document), + title: self.title, + format: self.format, + warnings: self.warnings, + stats: self.stats, + metadata: self.metadata, + } + } } /// Run a parser, converting a panic into an error (defence in depth for @@ -196,6 +233,26 @@ impl Markkit { bytes: &[u8], hint: DocumentHint<'_>, ) -> crate::Result { + self.parse_document_sync(bytes, hint) + .map(ParsedDocument::render) + } + + /// Parse a document (HTML, DOCX, XLSX/XLS/ODS) into the IR without + /// rendering. Same detection and errors as [`Markkit::extract_document`]. + pub async fn parse_document( + &self, + bytes: &[u8], + hint: DocumentHint<'_>, + ) -> crate::Result { + self.parse_document_sync(bytes, hint) + } + + /// Synchronous [`Markkit::parse_document`]. + pub fn parse_document_sync( + &self, + bytes: &[u8], + hint: DocumentHint<'_>, + ) -> crate::Result { if bytes.len() > self.config.max_input_size { return Err(Error::ResourceLimit(format!( "Input size {} exceeds limit {}", @@ -205,10 +262,10 @@ impl Markkit { } match Format::detect(bytes, hint) { - Format::Html => self.extract_html_document(&String::from_utf8_lossy(bytes)), - Format::Pdf => self.extract_pdf_sync(bytes), - Format::Docx => self.extract_docx(bytes), - Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), + Format::Html => self.parse_html_doc(bytes), + Format::Pdf => self.parse_pdf(bytes), + Format::Docx => self.parse_docx(bytes), + Format::Xlsx | Format::Xls | Format::Ods => self.parse_spreadsheet(bytes), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -223,57 +280,56 @@ impl Markkit { } #[cfg(feature = "html")] - fn extract_html_document(&self, html: &str) -> crate::Result { + fn parse_html_doc(&self, bytes: &[u8]) -> crate::Result { + let text = String::from_utf8_lossy(bytes); let parser = crate::html::HtmlParser::new(&self.config); - let (doc, page) = parser.parse_page(html)?; - let mut renderer = crate::render::MarkdownRenderer::new(); - let mut e = Extraction::new(renderer.render(&doc), Format::Html); - e.title = page.title; - e.metadata = page.metadata; - Ok(e) + let (doc, page) = parser.parse_page(&text)?; + let mut p = ParsedDocument::new(doc, Format::Html); + p.title = page.title; + p.metadata = page.metadata; + Ok(p) } #[cfg(not(feature = "html"))] - fn extract_html_document(&self, html: &str) -> crate::Result { - self.extract_html_sync(html) - .map(|md| Extraction::new(md, Format::Html)) + fn parse_html_doc(&self, _bytes: &[u8]) -> crate::Result { + Err(Error::UnsupportedFormat( + "HTML support requires the 'html' feature".to_string(), + )) } #[cfg(feature = "docx")] - fn extract_docx(&self, bytes: &[u8]) -> crate::Result { + fn parse_docx(&self, bytes: &[u8]) -> crate::Result { let out = guarded("DOCX", Error::DocxExtract, || { crate::docx::parse(bytes, &self.config) })?; - let mut renderer = crate::render::MarkdownRenderer::new(); - let mut e = Extraction::new(renderer.render(&out.doc), Format::Docx); - e.title = out.title; - e.stats.paragraphs = Some(out.paragraphs); - e.stats.tables = Some(out.tables); - Ok(e) + let mut p = ParsedDocument::new(out.doc, Format::Docx); + p.title = out.title; + p.stats.paragraphs = Some(out.paragraphs); + p.stats.tables = Some(out.tables); + Ok(p) } #[cfg(not(feature = "docx"))] - fn extract_docx(&self, _bytes: &[u8]) -> crate::Result { + fn parse_docx(&self, _bytes: &[u8]) -> crate::Result { Err(Error::UnsupportedFormat( "DOCX support requires the 'docx' feature".to_string(), )) } #[cfg(feature = "spreadsheet")] - fn extract_spreadsheet(&self, bytes: &[u8]) -> crate::Result { + fn parse_spreadsheet(&self, bytes: &[u8]) -> crate::Result { let out = guarded("spreadsheet", Error::SpreadsheetExtract, || { crate::spreadsheet::parse(bytes, &self.config) })?; - let mut renderer = crate::render::MarkdownRenderer::new(); - let mut e = Extraction::new(renderer.render(&out.doc), out.format); - e.warnings = out.warnings; - e.stats.sheets = Some(out.sheets); - e.stats.rows = Some(out.rows); - Ok(e) + let mut p = ParsedDocument::new(out.doc, out.format); + p.warnings = out.warnings; + p.stats.sheets = Some(out.sheets); + p.stats.rows = Some(out.rows); + Ok(p) } #[cfg(not(feature = "spreadsheet"))] - fn extract_spreadsheet(&self, _bytes: &[u8]) -> crate::Result { + fn parse_spreadsheet(&self, _bytes: &[u8]) -> crate::Result { Err(Error::UnsupportedFormat( "spreadsheet support requires the 'spreadsheet' feature".to_string(), )) @@ -310,7 +366,7 @@ impl Markkit { /// PDF extraction is not implemented yet; report it explicitly rather than /// returning an empty (misleading) success. - fn extract_pdf_sync(&self, _bytes: &[u8]) -> crate::Result { + fn parse_pdf(&self, _bytes: &[u8]) -> crate::Result { #[cfg(feature = "pdf")] let msg = "PDF extraction is not implemented yet"; #[cfg(not(feature = "pdf"))] diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 657dfe9..759e819 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -9,17 +9,21 @@ //! - Table `colspan`/`rowspan` are clamped to the HTML limits (1000 / 65534; //! `rowspan=0` means "to the end", stored as 65534); the renderer lays them //! out under a grid budget. +//! - Provenance (`track_sources`): blocks get the absolute XPath of the +//! element they came from. Sibling indexes are computed once per parent and +//! cached, so cost is O(depth) per block, not O(siblings). //! - `
` becomes `Figure` only when it has a non-empty `
`; //! otherwise it stays a transparent container as before. use std::cell::RefCell; use std::collections::HashMap; +use ego_tree::{NodeId, NodeRef}; use scraper::{Html, Node as HtmlNode, Selector}; use crate::config::MarkkitBuilder; use crate::error::Error; -use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell}; +use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, TableCell}; use super::metadata::PageInfo; @@ -34,6 +38,8 @@ pub struct HtmlParser<'a> { config: &'a MarkkitBuilder, /// Footnote target id (without `#`) -> label, filled by references. footnote_labels: RefCell>, + /// Cached 1-based index of an element among same-name element siblings. + sibling_index: RefCell>, } impl<'a> HtmlParser<'a> { @@ -41,6 +47,7 @@ impl<'a> HtmlParser<'a> { Self { config, footnote_labels: RefCell::new(HashMap::new()), + sibling_index: RefCell::new(HashMap::new()), } } @@ -88,6 +95,51 @@ impl<'a> HtmlParser<'a> { Ok(Document::new(normalize_blocks(nodes))) } + /// Attach the element's XPath to a block that has no source yet. + fn mark_source(&self, node: &mut Node, el: scraper::ElementRef<'_>) { + if self.config.track_sources && node.is_block() && node.source().is_none() { + node.set_source(Source::path(self.xpath(*el))); + } + } + + /// Absolute XPath (`/html[1]/body[1]/p[3]`) of an element. + fn xpath(&self, node: NodeRef<'_, HtmlNode>) -> String { + let mut segs: Vec = Vec::new(); + let mut cur = Some(node); + while let Some(n) = cur { + if let Some(e) = n.value().as_element() { + segs.push(format!("{}[{}]", e.name(), self.index_among_siblings(n))); + } + cur = n.parent(); + } + let mut out = String::new(); + for s in segs.iter().rev() { + out.push('/'); + out.push_str(s); + } + out + } + + fn index_among_siblings(&self, node: NodeRef<'_, HtmlNode>) -> usize { + if let Some(&i) = self.sibling_index.borrow().get(&node.id()) { + return i; + } + let Some(parent) = node.parent() else { + return 1; + }; + // Index every element child of this parent in one pass. + let mut counts: HashMap<&str, usize> = HashMap::new(); + let mut cache = self.sibling_index.borrow_mut(); + for child in parent.children() { + if let Some(e) = child.value().as_element() { + let n = counts.entry(e.name()).or_insert(0); + *n += 1; + cache.insert(child.id(), *n); + } + } + cache.get(&node.id()).copied().unwrap_or(1) + } + fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool { exclude.iter().any(|sel| sel.matches(el)) } @@ -135,15 +187,18 @@ impl<'a> HtmlParser<'a> { continue; } if name == "figure" - && let Some(fig) = self.convert_figure(el, exclude, depth + 1) + && let Some(mut fig) = self.convert_figure(el, exclude, depth + 1) { + self.mark_source(&mut fig, el); out.push(fig); continue; } // Transparent containers: recurse directly into their children if is_transparent_container(name) { self.convert_children(el, exclude, out, depth + 1); - } else if let Some(node) = self.convert_element(el, exclude, depth + 1) { + } else if let Some(mut node) = self.convert_element(el, exclude, depth + 1) + { + self.mark_source(&mut node, el); out.push(node); } } @@ -177,7 +232,11 @@ impl<'a> HtmlParser<'a> { if children.is_empty() { return None; } - Some(Node::Heading { level, children }) + Some(Node::Heading { + level, + children, + source: None, + }) } "p" => { @@ -185,7 +244,10 @@ impl<'a> HtmlParser<'a> { if children.is_empty() { return None; } - Some(Node::Paragraph { children }) + Some(Node::Paragraph { + children, + source: None, + }) } "figure" => self @@ -207,6 +269,7 @@ impl<'a> HtmlParser<'a> { ordered: false, start: None, items, + source: None, }) } @@ -220,6 +283,7 @@ impl<'a> HtmlParser<'a> { ordered: true, start, items, + source: None, }) } @@ -230,7 +294,10 @@ impl<'a> HtmlParser<'a> { if children.is_empty() { return None; } - Some(Node::BlockQuote { children }) + Some(Node::BlockQuote { + children, + source: None, + }) } "pre" => { @@ -241,12 +308,17 @@ impl<'a> HtmlParser<'a> { .attr("class") .and_then(extract_language_from_class); let code = code_el.text().collect::(); - Some(Node::CodeBlock { language, code }) + Some(Node::CodeBlock { + language, + code, + source: None, + }) } else { let code = el.text().collect::(); Some(Node::CodeBlock { language: None, code, + source: None, }) } } @@ -298,7 +370,10 @@ impl<'a> HtmlParser<'a> { } else if children.len() == 1 { Some(children.into_iter().next().unwrap()) } else { - Some(Node::Paragraph { children }) + Some(Node::Paragraph { + children, + source: None, + }) }; } let url = el.value().attr("href").unwrap_or("").to_string(); @@ -340,7 +415,10 @@ impl<'a> HtmlParser<'a> { } else if children.len() == 1 { Some(children.into_iter().next().unwrap()) } else { - Some(Node::Paragraph { children }) + Some(Node::Paragraph { + children, + source: None, + }) } } } @@ -360,7 +438,10 @@ impl<'a> HtmlParser<'a> { } else if nodes.is_empty() { None } else if nodes.iter().all(Node::is_inline) { - Some(Node::Paragraph { children: nodes }) + Some(Node::Paragraph { + children: nodes, + source: None, + }) } else { None } @@ -387,11 +468,15 @@ impl<'a> HtmlParser<'a> { let mut children = normalize_blocks(nodes); if !children.is_empty() && children.iter().all(Node::is_inline) { trim_edge_text(&mut children); - children = vec![Node::Paragraph { children }]; + children = vec![Node::Paragraph { + children, + source: None, + }]; } Some(Node::Figure { children, caption: Some(caption), + source: None, }) } @@ -409,7 +494,10 @@ impl<'a> HtmlParser<'a> { if items.is_empty() { return None; } - Some(Node::DefinitionList { items }) + Some(Node::DefinitionList { + items, + source: None, + }) } fn collect_definitions( @@ -496,11 +584,20 @@ impl<'a> HtmlParser<'a> { if children.iter().all(Node::is_inline) { trim_edge_text(&mut children); if !children.is_empty() { - children = vec![Node::Paragraph { children }]; + children = vec![Node::Paragraph { + children, + source: None, + }]; } } if !children.is_empty() { - out.push(Node::FootnoteDefinition { label, children }); + let mut def = Node::FootnoteDefinition { + label, + children, + source: None, + }; + self.mark_source(&mut def, li); + out.push(def); } } } @@ -582,7 +679,10 @@ impl<'a> HtmlParser<'a> { if checked.is_some() { if children.iter().all(Node::is_inline) { trim_edge_text(&mut children); - } else if let Some(Node::Paragraph { children: first }) = children.first_mut() { + } else if let Some(Node::Paragraph { + children: first, .. + }) = children.first_mut() + { trim_edge_text(first); } } @@ -666,6 +766,7 @@ impl<'a> HtmlParser<'a> { alignments, headers, rows, + source: None, }) } @@ -877,7 +978,10 @@ fn flush_inline_run(run: &mut Vec, out: &mut Vec) { let mut children = std::mem::take(run); trim_edge_text(&mut children); if !children.is_empty() { - out.push(Node::Paragraph { children }); + out.push(Node::Paragraph { + children, + source: None, + }); } } diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs index fa6fce1..1636d5d 100644 --- a/crates/markkit/src/ir.rs +++ b/crates/markkit/src/ir.rs @@ -55,21 +55,38 @@ impl Document { #[non_exhaustive] pub enum Node { /// Heading with level 1–6. - Heading { level: u8, children: Vec }, + Heading { + level: u8, + children: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Paragraph containing inline content. - Paragraph { children: Vec }, + Paragraph { + children: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Ordered or unordered list. List { ordered: bool, start: Option, items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, }, /// Block quotation. - BlockQuote { children: Vec }, + BlockQuote { + children: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Fenced code block. CodeBlock { language: Option, code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, }, /// Table with optional header row. Table { @@ -78,19 +95,32 @@ pub enum Node { alignments: Vec, headers: Vec, rows: Vec>, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, }, /// Horizontal rule / thematic break. ThematicBreak, /// Definition list (`
`): terms with one or more definitions. - DefinitionList { items: Vec }, + DefinitionList { + items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Figure: block content with an optional inline caption. Figure { children: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] caption: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, }, /// Footnote body, referenced by [`Node::FootnoteReference`] with the same label. - FootnoteDefinition { label: String, children: Vec }, + FootnoteDefinition { + label: String, + children: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + source: Option>, + }, /// Plain text. Text(String), /// Bold / strong emphasis. @@ -137,6 +167,126 @@ impl Node { pub fn is_inline(&self) -> bool { !self.is_block() } + + fn source_slot(&mut self) -> Option<&mut Option>> { + match self { + Node::Heading { source, .. } + | Node::Paragraph { source, .. } + | Node::List { source, .. } + | Node::BlockQuote { source, .. } + | Node::CodeBlock { source, .. } + | Node::Table { source, .. } + | Node::DefinitionList { source, .. } + | Node::Figure { source, .. } + | Node::FootnoteDefinition { source, .. } => Some(source), + _ => None, + } + } + + /// Where this block came from, when the parser recorded it. + pub fn source(&self) -> Option<&Source> { + match self { + Node::Heading { source, .. } + | Node::Paragraph { source, .. } + | Node::List { source, .. } + | Node::BlockQuote { source, .. } + | Node::CodeBlock { source, .. } + | Node::Table { source, .. } + | Node::DefinitionList { source, .. } + | Node::Figure { source, .. } + | Node::FootnoteDefinition { source, .. } => source.as_deref(), + _ => None, + } + } + + /// Attach provenance to a block; ignored for inline nodes and + /// `ThematicBreak`. Returns `self` for chaining. + pub fn with_source(mut self, src: Source) -> Self { + self.set_source(src); + self + } + + /// In-place [`Node::with_source`]. + pub fn set_source(&mut self, src: Source) { + if let Some(slot) = self.source_slot() { + *slot = Some(Box::new(src)); + } + } + + /// Paragraph with inline children and no provenance. + pub fn paragraph(children: Vec) -> Self { + Node::Paragraph { + children, + source: None, + } + } + + /// Heading (level clamped to 1-6) with no provenance. + pub fn heading(level: u8, children: Vec) -> Self { + Node::Heading { + level: level.clamp(1, 6), + children, + source: None, + } + } +} + +/// Provenance of a block: where in the input it came from. +/// +/// Only the fields meaningful for the input format are set. Recorded when +/// [`crate::MarkkitBuilder::track_sources`] is enabled. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct Source { + /// HTML: absolute XPath of the element, e.g. `/html/body/article[1]/p[3]` + /// (indexes count same-name element siblings, 1-based, in the parsed DOM). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + /// DOCX: 0-based index of the body-level paragraph or table. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub index: Option, + /// Spreadsheets: sheet name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sheet: Option, + /// Spreadsheets: A1-style cell range, e.g. `A1:D20`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range: Option, + /// Paged formats: 1-based page number. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub page: Option, +} + +impl Source { + /// HTML element path. + pub fn path(path: impl Into) -> Self { + Self { + path: Some(path.into()), + ..Self::default() + } + } + + /// Block index (DOCX body order). + pub fn index(index: usize) -> Self { + Self { + index: Some(index), + ..Self::default() + } + } + + /// Spreadsheet sheet, optionally with a cell range. + pub fn sheet(name: impl Into, range: Option) -> Self { + Self { + sheet: Some(name.into()), + range, + ..Self::default() + } + } + + /// Set the page number. + pub fn with_page(mut self, page: u32) -> Self { + self.page = Some(page); + self + } } /// A single item in a list. @@ -251,6 +401,7 @@ mod tests { Node::Heading { level: 1, children: vec![Node::Text("Hello".into())], + source: None, }, Node::Paragraph { children: vec![ @@ -261,6 +412,7 @@ mod tests { Node::Text(" text".into()), Node::FootnoteReference { label: "1".into() }, ], + source: None, }, Node::List { ordered: false, @@ -269,17 +421,20 @@ mod tests { ListItem::task(vec![Node::Text("done".into())], true), ListItem::new(vec![Node::Text("plain".into())]), ], + source: None, }, Node::Table { alignments: vec![Alignment::Left, Alignment::Right], headers: vec![TableCell::new(vec![Node::Text("a".into())]).with_span(2, 1)], rows: vec![vec![TableCell::new(vec![]), TableCell::new(vec![])]], + source: None, }, Node::DefinitionList { items: vec![DefinitionItem::new( vec![Node::Text("term".into())], vec![vec![Node::Text("def".into())]], )], + source: None, }, Node::Figure { children: vec![Node::Image { @@ -287,12 +442,12 @@ mod tests { alt: "A".into(), }], caption: Some(vec![Node::Text("cap".into())]), + source: None, }, Node::FootnoteDefinition { label: "1".into(), - children: vec![Node::Paragraph { - children: vec![Node::Text("note".into())], - }], + children: vec![Node::paragraph(vec![Node::Text("note".into())])], + source: Some(Box::new(Source::path("/html/body/section[1]/ol[1]/li[1]"))), }, ]); let json = serde_json::to_string(&doc).unwrap(); @@ -307,6 +462,7 @@ mod tests { alignments: vec![], headers: vec![TableCell::new(vec![Node::Text("h".into())])], rows: vec![], + source: None, }]); let json = serde_json::to_string(&doc).unwrap(); assert!(json.starts_with(r#"{"ir_version":2,"#), "{json}"); @@ -320,6 +476,18 @@ mod tests { assert_eq!(doc.ir_version, 1); } + #[test] + fn source_on_blocks_only() { + let p = Node::paragraph(vec![]).with_source(Source::index(3)); + assert_eq!(p.source().and_then(|s| s.index), Some(3)); + let t = Node::Text("x".into()).with_source(Source::index(3)); + assert!(t.source().is_none()); + let json = serde_json::to_string(&Node::paragraph(vec![])).unwrap(); + assert!(!json.contains("source"), "{json}"); + let json = serde_json::to_string(&p).unwrap(); + assert!(json.contains(r#""source":{"index":3}"#), "{json}"); + } + #[test] fn zero_span_clamped() { let c = TableCell::new(vec![]).with_span(0, 0); diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index e68e111..4bb43a5 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -65,7 +65,7 @@ mod xml; pub use config::MarkkitBuilder; pub use error::Error; -pub use extract::{Extraction, ExtractionStats, Markkit, Metadata}; +pub use extract::{Extraction, ExtractionStats, Markkit, Metadata, ParsedDocument}; pub use format::{DocumentHint, Format}; pub use ir::Document; pub use render::MarkdownRenderer; diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs index 9d68347..9592f80 100644 --- a/crates/markkit/src/render/markdown.rs +++ b/crates/markkit/src/render/markdown.rs @@ -50,7 +50,9 @@ impl MarkdownRenderer { fn render_node(&mut self, node: &Node, depth: usize) { match node { - Node::Heading { level, children } => { + Node::Heading { + level, children, .. + } => { let prefix = "#".repeat(*level as usize); self.buf.push_str(&prefix); self.buf.push(' '); @@ -60,7 +62,7 @@ impl MarkdownRenderer { self.buf.push('\n'); } - Node::Paragraph { children } => { + Node::Paragraph { children, .. } => { let start = self.buf.len(); self.render_inline(children); trim_trailing_inline(&mut self.buf, start); @@ -71,11 +73,12 @@ impl MarkdownRenderer { ordered, start, items, + .. } => { self.render_list(*ordered, *start, items, depth); } - Node::BlockQuote { children } => { + Node::BlockQuote { children, .. } => { let inner = { let mut sub = MarkdownRenderer::new(); sub.render_nodes(children, depth); @@ -92,7 +95,7 @@ impl MarkdownRenderer { } } - Node::CodeBlock { language, code } => { + Node::CodeBlock { language, code, .. } => { self.buf.push_str("```"); if let Some(lang) = language { self.buf.push_str(lang); @@ -109,6 +112,7 @@ impl MarkdownRenderer { alignments, headers, rows, + .. } => { self.render_table(alignments, headers, rows); } @@ -117,9 +121,11 @@ impl MarkdownRenderer { self.buf.push_str("---\n"); } - Node::DefinitionList { items } => self.render_definition_list(items, depth), + Node::DefinitionList { items, .. } => self.render_definition_list(items, depth), - Node::Figure { children, caption } => { + Node::Figure { + children, caption, .. + } => { self.render_nodes(children, depth); if let Some(caption) = caption { self.buf.push('\n'); @@ -132,7 +138,9 @@ impl MarkdownRenderer { } } - Node::FootnoteDefinition { label, children } => { + Node::FootnoteDefinition { + label, children, .. + } => { let inner = render_blocks(children, depth); self.buf.push_str("[^"); self.buf.push_str(&footnote_label(label)); @@ -200,7 +208,7 @@ impl MarkdownRenderer { if first { first = false; match child { - Node::Paragraph { children } => { + Node::Paragraph { children, .. } => { self.render_inline(children); self.buf.push('\n'); } @@ -208,6 +216,7 @@ impl MarkdownRenderer { ordered: sub_ord, start: sub_start, items: sub_items, + .. } => { self.buf.push('\n'); self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); @@ -226,6 +235,7 @@ impl MarkdownRenderer { ordered: sub_ord, start: sub_start, items: sub_items, + .. } => { self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); } diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index 2b04212..5b1f806 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -28,7 +28,7 @@ use crate::archive::{ArchiveError, ZipGuard}; use crate::config::MarkkitBuilder; use crate::error::Error; use crate::format::Format; -use crate::ir::{Document, Node, TableCell}; +use crate::ir::{Document, Node, Source, TableCell}; use crate::render::escape_table_cell; pub(crate) struct SheetOutput { @@ -218,12 +218,28 @@ fn fmt_ref(d: &DataRef<'_>) -> String { // ─── Grid collection (bounded) ────────────────────────────────────── struct SheetTable { + /// A1 range of the rendered block (bounding box of kept rows/columns). + range: String, header: Vec, rows: Vec>, truncated_rows: bool, truncated_cols: bool, } +/// Column letters for a 0-based column index (`0` -> `A`, `26` -> `AA`). +fn col_name(mut col: u32) -> String { + let mut out = Vec::new(); + loop { + out.push(b'A' + (col % 26) as u8); + if col < 26 { + break; + } + col = col / 26 - 1; + } + out.reverse(); + String::from_utf8(out).expect("ASCII") +} + /// Sparse, capped cell store: keeps the first `max_rows + 1` non-empty rows /// (header + data) and at most `max_cols` cells per row. struct Grid { @@ -288,6 +304,15 @@ impl Grid { } else { span }; + let first_row = *self.rows.keys().next()?; + let last_row = *self.rows.keys().next_back()?; + let range = format!( + "{}{}:{}{}", + col_name(min_col), + first_row as u64 + 1, + col_name(min_col + (width as u32 - 1)), + last_row as u64 + 1 + ); let mut rows = self.rows.into_values().map(|cells| { let mut row = vec![String::new(); width]; for (c, v) in cells { @@ -300,6 +325,7 @@ impl Grid { }); let header = rows.next()?; Some(SheetTable { + range, header, rows: rows.collect(), truncated_rows: self.truncated_rows, @@ -355,6 +381,7 @@ impl<'c> Collector<'c> { fn push_sheet(&mut self, name: &str, grid: Grid) { let (max_rows, max_cols) = (grid.max_rows, grid.max_cols); let Some(SheetTable { + range, header, rows, truncated_rows: trunc_rows, @@ -382,9 +409,12 @@ impl<'c> Collector<'c> { vec![Node::Text(s)] }) }; + let track = self.config.track_sources; + let src = |range: Option| track.then(|| Box::new(Source::sheet(name, range))); self.nodes.push(Node::Heading { level: 2, children: vec![Node::Text(name.replace(['\r', '\n'], " "))], + source: src(None), }); self.nodes.push(Node::Table { alignments: Vec::new(), @@ -393,6 +423,7 @@ impl<'c> Collector<'c> { .into_iter() .map(|r| r.into_iter().map(cell).collect()) .collect(), + source: src(Some(range)), }); } diff --git a/crates/markkit/tests/docx_integration.rs b/crates/markkit/tests/docx_integration.rs index 11f7c48..fd2275f 100644 --- a/crates/markkit/tests/docx_integration.rs +++ b/crates/markkit/tests/docx_integration.rs @@ -353,3 +353,30 @@ fn list_styles_resolve_numbering_and_switch_list_type() { .markdown; assert_eq!(out, "- one\n - nested\n- two\n\n1. step a\n2. step b\n"); } + +// ─── Provenance ────────────────────────────────────────────────────── + +#[test] +fn sources_record_body_block_index() { + let body = format!( + "{}{}{}{}{}", + styled_para("Heading1", "Title"), + para(""), + list_para(1, 0, "one"), + list_para(1, 0, "two"), + para("After"), + ); + let m = Markkit::builder().track_sources(true).build(); + let p = m + .parse_document_sync(&docx(&body), DocumentHint::content_type(DOCX_CT)) + .unwrap(); + let idx: Vec> = p + .document + .nodes + .iter() + .map(|n| n.source().and_then(|s| s.index)) + .collect(); + // Blank paragraph 1 is skipped but still counted. + assert_eq!(idx, vec![Some(0), Some(2), Some(4)]); + assert_eq!(p.render().markdown, md(&body)); +} diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index e2402e3..a77f18f 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -869,3 +869,54 @@ fn ir_v2_json_has_version() { assert_eq!(json["ir_version"], markkit::ir::IR_VERSION); assert_eq!(json["nodes"][0]["List"]["items"][0]["checked"], true); } + +// ─── Provenance ────────────────────────────────────────────────────── + +#[test] +fn sources_are_xpaths() { + let m = Markkit::builder().track_sources(true).build(); + let doc = m + .parse_html( + "

T

a

b

\ +

  • x

    q

", + ) + .unwrap(); + let paths: Vec<&str> = doc + .nodes + .iter() + .map(|n| n.source().and_then(|s| s.path.as_deref()).unwrap()) + .collect(); + assert_eq!( + paths, + vec![ + "/html[1]/body[1]/h1[1]", + "/html[1]/body[1]/div[1]/p[1]", + "/html[1]/body[1]/div[1]/p[2]", + "/html[1]/body[1]/ul[1]", + ] + ); + let markkit::ir::Node::List { items, .. } = &doc.nodes[3] else { + panic!("list expected"); + }; + let quote = items[0].children[1].source().unwrap(); + assert_eq!( + quote.path.as_deref(), + Some("/html[1]/body[1]/ul[1]/li[1]/blockquote[1]") + ); +} + +#[test] +fn sources_off_by_default_and_not_in_json() { + let m = Markkit::builder().build(); + let doc = m.parse_html("

a

").unwrap(); + assert!(doc.nodes[0].source().is_none()); + assert!(!serde_json::to_string(&doc).unwrap().contains("source")); +} + +#[test] +fn sources_do_not_change_markdown() { + let html = "

A

b c

x
1
"; + let a = convert(html); + let b = convert_with(html, |b| b.track_sources(true)); + assert_eq!(a, b); +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index c51b2e5..c229a6e 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -233,3 +233,15 @@ fn deep_definition_lists_bounded() { let html = "
".repeat(500) + "x" + &"
".repeat(500); let _ = convert(&html); } + +#[test] +fn source_tracking_many_siblings_is_linear() { + // 200k siblings: per-block XPath must not rescan siblings (O(n^2)). + let html = "

x

".repeat(200_000); + let m = Markkit::builder().track_sources(true).build(); + let start = std::time::Instant::now(); + let doc = m.parse_html(&html).unwrap(); + assert!(start.elapsed().as_secs() < 20); + let last = doc.nodes.last().unwrap().source().unwrap(); + assert_eq!(last.path.as_deref(), Some("/html[1]/body[1]/p[200000]")); +} diff --git a/crates/markkit/tests/spreadsheet_integration.rs b/crates/markkit/tests/spreadsheet_integration.rs index d976897..3c54e02 100644 --- a/crates/markkit/tests/spreadsheet_integration.rs +++ b/crates/markkit/tests/spreadsheet_integration.rs @@ -275,3 +275,33 @@ fn xls_by_content_type() { .unwrap(); assert!(e.markdown.contains("| x |"), "{}", e.markdown); } + +// ─── Provenance ────────────────────────────────────────────────────── + +#[test] +fn xlsx_sources_record_sheet_and_range() { + let bytes = xlsx(&[Sheet::new( + "Data", + vec![ + vec![Empty, Empty], + vec![Empty, S("h1"), S("h2")], + vec![Empty, S("a"), S("b")], + ], + )]); + let m = Markkit::builder().track_sources(true).build(); + let p = m + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + let nodes = &p.document.nodes; + let heading = nodes[0].source().unwrap(); + assert_eq!(heading.sheet.as_deref(), Some("Data")); + assert_eq!(heading.range, None); + let table = nodes[1].source().unwrap(); + assert_eq!(table.range.as_deref(), Some("B2:C3")); + // Off by default. + let p = Markkit::builder() + .build() + .parse_document_sync(&bytes, DocumentHint::default()) + .unwrap(); + assert!(p.document.nodes.iter().all(|n| n.source().is_none())); +} diff --git a/docs/getting-started.md b/docs/getting-started.md index 35dbc5a..ab06b78 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -50,6 +50,22 @@ let doc = markkit.extract_document_sync(&bytes, hint)?; println!("{}", doc.markdown); ``` +## IR and provenance + +`parse_document_sync` returns the IR (`markkit::ir::Document`) plus metadata instead of +Markdown. With `track_sources(true)` every block carries a `Source`: the element XPath +for HTML, the body block index for DOCX, the sheet and A1 range for spreadsheets. Use it +to cite where a chunk came from. + +```rust +let markkit = Markkit::builder().track_sources(true).build(); +let parsed = markkit.parse_document_sync(&bytes, hint)?; +for block in &parsed.document.nodes { + println!("{:?}", block.source()); +} +let markdown = parsed.render().markdown; +``` + ## Configuration ```rust diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 9781979..5acb6b3 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -54,6 +54,14 @@ Via `MarkkitBuilder`: - Link and image preservation - Office-document limits (archive entries/sizes, sheet rows/cols, grid cells, hidden sheets) - URL fetch policy (`allow_private_networks`, `allow_host`, `max_redirects`) +- Provenance (`track_sources`): per-block `Source` in the IR (off by default) + +## Public entry points + +- `extract_*` return Markdown (`extract_document` adds metadata as `Extraction`). +- `parse_html` and `parse_document` return the IR (`ParsedDocument` = `Document` + + title, format, warnings, stats); `ParsedDocument::render()` gives the + `Extraction`. `extract_document` is `parse_document` + `render`. ## See also diff --git a/knowledge/foundations/ir.md b/knowledge/foundations/ir.md index 433a98f..c18398a 100644 --- a/knowledge/foundations/ir.md +++ b/knowledge/foundations/ir.md @@ -53,10 +53,12 @@ Tagged enum. Variants are block-level or inline. #### Block Variants +All block variants except `ThematicBreak` also carry `source` (see Provenance). + | Variant | Fields | Semantics | |---------|--------|-----------| -| `Heading` | `level: u8`, `children: Vec` | ATX heading. Level 1-6. Children are inline nodes. | -| `Paragraph` | `children: Vec` | Block of inline content. Separated by blank lines in output. | +| `Heading` | `level: u8`, `children: Vec`, `source` | ATX heading. Level 1-6. Children are inline nodes. | +| `Paragraph` | `children: Vec`, `source` | Block of inline content. Separated by blank lines in output. | | `List` | `ordered: bool`, `start: Option`, `items: Vec` | Ordered or unordered list. `start` only meaningful when `ordered=true`. | | `BlockQuote` | `children: Vec` | Nested block content. Children can be any node type. | | `CodeBlock` | `language: Option`, `code: String` | Fenced code block. Language extracted from `language-*` or `lang-*` CSS class. | @@ -80,6 +82,24 @@ Tagged enum. Variants are block-level or inline. | `LineBreak` | (unit) | Hard line break. Renders as two trailing spaces + newline. | | `FootnoteReference` | `label: String` | Marker pointing at the `FootnoteDefinition` with the same label. | +### Provenance (`Source`) + +Every block variant except `ThematicBreak` has `source: Option>` +(omitted from JSON when `None`). Parsers fill it only when +`MarkkitBuilder::track_sources(true)`; Markdown output is identical either way. + +| Field | Set by | Meaning | +|-------|--------|---------| +| `path` | HTML | Absolute XPath in the parsed DOM, every step indexed: `/html[1]/body[1]/div[1]/p[2]`. Indexes count same-name element siblings (excluded elements included). Cached per parent, O(depth) per block. | +| `index` | DOCX | 0-based index of the body-level `w:p`/`w:tbl` in document order, counting blank paragraphs and those inside `w:sdt`/`w:ins`. Lists take their first item's index. | +| `sheet`, `range` | Spreadsheets | Sheet name on the heading and table; table gets the A1 bounding box of kept cells (`B2:C3`). | +| `page` | Paged formats (PDF, later) | 1-based page. | + +Blocks synthesized by normalization (inline runs wrapped in `Paragraph`) and +blocks nested inside inline/table-cell content have no source. Accessors: +`Node::source()`, `set_source()`, `with_source()`; constructors +`Source::path/index/sheet`, `with_page`. + ### Helper structs | Type | Fields | Notes | @@ -96,6 +116,7 @@ Tagged enum. Variants are block-level or inline. Input format -> `Document`. Each parser (HTML, future PDF/DOCX) produces the same IR. - HTML: `HtmlParser::parse(&str) -> Result` +- Any format: `Markkit::parse_document_sync(bytes, hint) -> Result` - Input size checked before parsing (configurable via `max_input_size`) ### 2. Normalize diff --git a/knowledge/log.md b/knowledge/log.md index 38c2dc7..32731a1 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md). * **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from ``, Open Graph, `` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. * **IR v2**: footnotes, definition lists, task lists, figures, table alignment and spans; `Document.ir_version` (`IR_VERSION = 2`), `#[non_exhaustive]` IR types, `Node::is_block`/`is_inline`. HTML maps GitHub/Pandoc footnotes, `
`, checkbox items, captioned figures, `colspan`/`rowspan`/alignment and row-header ``. New threat T2e and invariant S13 (span grid budget, footnote label sanitizing). See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md), [Threat Model](security/threat-model.md). * **Agent tool**: new [Tool Contract](integrations/tool-contract.md) concept in a new `integrations/` domain. `MarkkitTool` (opt-in `tool` feature) follows the Everruns toolkit-library contract; tool arguments are a new untrusted surface in the [Threat Model](security/threat-model.md).