feat(ir): per-block provenance and parse_document

CHANGELOG.md

@@ -19,6 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   `<figure>`/`<figcaption>` and table spans/alignment. Row-header `<th>` cells in table bodies are
   no longer dropped.
 - `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON.
+- Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block
+  (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations.
+- `Markkit::parse_document` / `parse_document_sync` return the IR plus metadata (`ParsedDocument`)
+  for any supported format; `ParsedDocument::render()` produces the `Extraction`.
 - `tool` feature: `markkit::tool::MarkkitTool`, an LLM agent tool following the Everruns
   toolkit-library contract (builder, OpenAI tool definition, input/output schemas, `tower::Service`,
   single-use `ToolExecution`, `en-US`/`uk-UA` text). One call converts a `url`, `html` or base64
@@ -38,7 +42,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 - **Breaking (IR API):** `Node`, `Document`, `ListItem` and the new IR structs are `#[non_exhaustive]`;
   matches need a `_` arm and structs are built with constructors. `Node::Table` cells are now
-  `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. DOCX `gridSpan` is
+  `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. Block variants gain a
+  `source` field (use `..` in patterns, `Node::paragraph`/`Node::heading` to construct). DOCX `gridSpan` is
   carried as `colspan`.
 - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI.
 - CI workflows declare read-only default permissions and do not persist checkout credentials.

Cargo.toml

@@ -41,6 +41,7 @@ reqwest = { version = "0.13", default-features = false, features = ["rustls", "j
 
 # HTML parsing
 scraper = "0.29"
+ego-tree = "0.11"
 html5ever = "0.40"
 
 # Document formats

crates/markkit/Cargo.toml

@@ -15,7 +15,7 @@ readme = "../../README.md"
 
 [features]
 default = ["html", "http_client"]
-html = ["dep:scraper", "dep:html5ever"]
+html = ["dep:scraper", "dep:html5ever", "dep:ego-tree"]
 pdf = []
 docx = ["dep:zip", "dep:quick-xml"]
 spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"]
@@ -38,6 +38,7 @@ serde_json.workspace = true
 
 # HTML (optional)
 scraper = { workspace = true, optional = true }
+ego-tree = { workspace = true, optional = true }
 html5ever = { workspace = true, optional = true }
 
 # Office documents (optional): DOCX + spreadsheets share the ZIP/XML stack

crates/markkit/src/config.rs

@@ -40,6 +40,10 @@ pub struct MarkkitBuilder {
     pub allowed_hosts: Vec<String>,
     /// Maximum redirects `extract_url` follows. Default: 5.
     pub max_redirects: usize,
+    /// Record per-block provenance ([`crate::ir::Source`]) in the IR:
+    /// HTML element XPath, DOCX block index, spreadsheet sheet and range.
+    /// Default: false (no cost, no `source` in IR JSON).
+    pub track_sources: bool,
 }
 
 impl Default for MarkkitBuilder {
@@ -69,6 +73,7 @@ impl Default for MarkkitBuilder {
             allow_private_networks: false,
             allowed_hosts: Vec::new(),
             max_redirects: 5,
+            track_sources: false,
         }
     }
 }
@@ -94,6 +99,12 @@ impl MarkkitBuilder {
         self
     }
 
+    /// Record per-block provenance in the IR (see [`MarkkitBuilder::track_sources`] field).
+    pub fn track_sources(mut self, yes: bool) -> Self {
+        self.track_sources = yes;
+        self
+    }
+
     pub fn include_selector(mut self, selector: impl Into<String>) -> Self {
         self.include_selectors.push(selector.into());
         self

crates/markkit/src/docx.rs

@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::archive::{ArchiveError, ZipGuard};
 use crate::config::MarkkitBuilder;
 use crate::error::Error;
-use crate::ir::{Document, ListItem, Node, TableCell};
+use crate::ir::{Document, ListItem, Node, Source, TableCell};
 use crate::render::escape_table_cell;
 use crate::xml::{self, Element, XmlError};
 
@@ -589,7 +589,7 @@ fn plain_text(nodes: &[Node], out: &mut String) {
             | Node::Strikethrough { children }
             | Node::Link { children, .. }
             | Node::Heading { children, .. }
-            | Node::Paragraph { children } => plain_text(children, out),
+            | Node::Paragraph { children, .. } => plain_text(children, out),
             _ => {}
         }
     }
@@ -602,6 +602,7 @@ struct ListEntry {
     ordered: bool,
     start: Option<u64>,
     content: Vec<Node>,
+    index: usize,
 }
 
 #[derive(Default)]
@@ -611,6 +612,11 @@ struct Body {
     title_text: Option<String>,
     paragraphs: usize,
     tables: usize,
+    /// Record `Source::index` on blocks (`track_sources`).
+    track: bool,
+    /// Next body block index (every `w:p` / `w:tbl`, in document order,
+    /// including those inside `w:sdt` / `w:ins` wrappers and blank ones).
+    next_index: usize,
 }
 
 impl Body {
@@ -622,9 +628,18 @@ impl Body {
         let base = entries.iter().map(|e| e.level).min().unwrap_or(0);
         let mut pos = 0;
         while pos < entries.len() {
-            self.blocks.push(build_list(&entries, &mut pos, base));
+            let index = entries[pos].index;
+            let list = build_list(&entries, &mut pos, base);
+            self.push(list, index);
         }
     }
+
+    fn push(&mut self, mut node: Node, index: usize) {
+        if self.track {
+            node.set_source(Source::index(index));
+        }
+        self.blocks.push(node);
+    }
 }
 
 /// Build a (nested) list starting at `entries[*pos]` for `level`.
@@ -652,26 +667,34 @@ fn build_list(entries: &[ListEntry], pos: &mut usize, level: u32) -> Node {
         }
         items.push(ListItem::new(vec![Node::Paragraph {
             children: e.content.clone(),
+            source: None,
         }]));
         *pos += 1;
     }
     Node::List {
         ordered,
         start,
         items,
+        source: None,
     }
 }
 
 impl Ctx<'_> {
     fn blocks(&self, container: &Element, body: &mut Body) {
         for child in container.elements() {
             match child.name.as_str() {
-                "p" => self.paragraph(child, body),
+                "p" => {
+                    let index = body.next_index;
+                    body.next_index += 1;
+                    self.paragraph(child, body, index);
+                }
                 "tbl" => {
+                    let index = body.next_index;
+                    body.next_index += 1;
                     body.flush_list();
                     if let Some(t) = self.table(child) {
                         body.tables += 1;
-                        body.blocks.push(t);
+                        body.push(t, index);
                     }
                 }
                 "sdt" => {
@@ -685,7 +708,7 @@ impl Ctx<'_> {
         }
     }
 
-    fn paragraph(&self, p: &Element, body: &mut Body) {
+    fn paragraph(&self, p: &Element, body: &mut Body, index: usize) {
         let ppr = p.child("pPr");
         let style = ppr.and_then(|x| x.child_val("pStyle"));
         let content = self.inline(p);
@@ -709,10 +732,14 @@ impl Ctx<'_> {
                 body.title_text = Some(t.trim().to_string());
             }
             body.flush_list();
-            body.blocks.push(Node::Heading {
-                level,
-                children: content,
-            });
+            body.push(
+                Node::Heading {
+                    level,
+                    children: content,
+                    source: None,
+                },
+                index,
+            );
             return;
         }
 
@@ -749,10 +776,17 @@ impl Ctx<'_> {
                 ordered,
                 start,
                 content,
+                index,
             }),
             None => {
                 body.flush_list();
-                body.blocks.push(Node::Paragraph { children: content });
+                body.push(
+                    Node::Paragraph {
+                        children: content,
+                        source: None,
+                    },
+                    index,
+                );
             }
         }
     }
@@ -828,6 +862,7 @@ impl Ctx<'_> {
             alignments: Vec::new(),
             headers,
             rows,
+            source: None,
         })
     }
 }
@@ -907,7 +942,10 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<DocxOutput,
         styles: &styles,
         numbering: &numbering,
     };
-    let mut body = Body::default();
+    let mut body = Body {
+        track: config.track_sources,
+        ..Body::default()
+    };
     ctx.blocks(body_el, &mut body);
     body.flush_list();