From 70c658e5bc37a06e0362febd8496f14382366602 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:36:50 +0000 Subject: [PATCH] feat(ir): IR v2 with footnotes, definition lists, task lists, figures, table spans - Node: FootnoteReference/FootnoteDefinition, DefinitionList, Figure; ListItem.checked; Table cells as TableCell (colspan/rowspan) + alignments - Document.ir_version (IR_VERSION = 2); IR types #[non_exhaustive]; Node::is_block/is_inline - HTML: GitHub/Pandoc footnotes,
, checkbox items, captioned figures, table spans/alignment, row-header kept - Renderer: GFM + footnote/deflist extensions; span grid layout bounded by MAX_TABLE_GRID_CELLS (T2e, S13) --- CHANGELOG.md | 13 + crates/markkit/src/docx.rs | 19 +- crates/markkit/src/html/parser.rs | 464 ++++++++++++++++++++--- crates/markkit/src/ir.rs | 220 ++++++++++- crates/markkit/src/render/markdown.rs | 271 ++++++++++--- crates/markkit/src/spreadsheet.rs | 7 +- crates/markkit/tests/html_integration.rs | 130 +++++++ crates/markkit/tests/html_security.rs | 29 ++ knowledge/formats/html-extraction.md | 16 +- knowledge/foundations/ir.md | 60 ++- knowledge/log.md | 2 + knowledge/security/security-model.md | 1 + knowledge/security/threat-model.md | 10 + 13 files changed, 1102 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..6eb370e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- IR v2: footnotes (`[^1]`), definition lists (`Term` / `: definition`), task lists (`- [x]`), + figures with captions, table column alignment (`:---:`) and `colspan`/`rowspan` (laid out on a + grid; GFM has no spans). HTML maps GitHub and Pandoc footnote markup, `
`, checkbox list items, + `
`/`
` and table spans/alignment. Row-header `` cells in table bodies are + no longer dropped. +- `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON. + ### Security - `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: @@ -18,6 +27,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **Breaking (IR API):** `Node`, `Document`, `ListItem` and the new IR structs are `#[non_exhaustive]`; + matches need a `_` arm and structs are built with constructors. `Node::Table` cells are now + `TableCell` (`children`, `colspan`, `rowspan`) with an `alignments` field. DOCX `gridSpan` is + carried as `colspan`. - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. - CI runs every example; `just bench` now runs the criterion benchmarks. diff --git a/crates/markkit/src/docx.rs b/crates/markkit/src/docx.rs index cf2795f..98d98ac 100644 --- a/crates/markkit/src/docx.rs +++ b/crates/markkit/src/docx.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use crate::archive::{ArchiveError, ZipGuard}; use crate::config::MarkkitBuilder; use crate::error::Error; -use crate::ir::{Document, ListItem, Node}; +use crate::ir::{Document, ListItem, Node, TableCell}; use crate::render::escape_table_cell; use crate::xml::{self, Element, XmlError}; @@ -803,7 +803,7 @@ impl Ctx<'_> { } fn table(&self, tbl: &Element) -> Option { - let mut rows: Vec>> = Vec::new(); + let mut rows: Vec> = Vec::new(); for tr in tbl.elements().filter(|e| e.name == "tr") { let mut tcs = Vec::new(); self.row_cells(tr, &mut tcs); @@ -811,23 +811,24 @@ impl Ctx<'_> { for tc in tcs { let mut content = Vec::new(); self.cell_content(tc, &mut content); - row.push(table_cell(content)); - let span: usize = tc + let span: u32 = tc .child("tcPr") .and_then(|p| p.child_val("gridSpan")) .and_then(|v| v.parse().ok()) .unwrap_or(1); - for _ in 1..span.min(64) { - row.push(Vec::new()); - } + row.push(TableCell::new(table_cell(content)).with_span(span.min(64), 1)); } rows.push(row); } - if rows.iter().all(|r| r.iter().all(|c| is_blank(c))) { + if rows.iter().all(|r| r.iter().all(|c| is_blank(&c.children))) { return None; } let headers = rows.remove(0); - Some(Node::Table { headers, rows }) + Some(Node::Table { + alignments: Vec::new(), + headers, + rows, + }) } } diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 6cfebd5..49c686f 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -1,22 +1,45 @@ //! HTML → IR parser using scraper/html5ever. +//! +//! Decisions: +//! - Footnotes follow the markup GitHub (`data-footnote-ref`, +//! `section[data-footnotes]`) and Pandoc (`role="doc-noteref"`, +//! `.footnote-ref`, `role="doc-endnotes"`, `.footnotes`) emit. References +//! record `target id -> label`; definitions look their `
  • ` up there +//! and fall back to their 1-based position. Back-links are dropped. +//! - Table `colspan`/`rowspan` are clamped to the HTML limits (1000 / 65534; +//! `rowspan=0` means "to the end", stored as 65534); the renderer lays them +//! out under a grid budget. +//! - `
    ` becomes `Figure` only when it has a non-empty `
    `; +//! otherwise it stays a transparent container as before. + +use std::cell::RefCell; +use std::collections::HashMap; use scraper::{Html, Node as HtmlNode, Selector}; use crate::config::MarkkitBuilder; use crate::error::Error; -use crate::ir::{Document, ListItem, Node}; +use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell}; /// Maximum recursion depth for nested elements. const MAX_DEPTH: usize = 200; +/// HTML-spec clamps for table spans. +const MAX_COLSPAN: u32 = 1000; +const MAX_ROWSPAN: u32 = 65534; /// Parses HTML into the canonical IR. pub struct HtmlParser<'a> { config: &'a MarkkitBuilder, + /// Footnote target id (without `#`) -> label, filled by references. + footnote_labels: RefCell>, } impl<'a> HtmlParser<'a> { pub fn new(config: &'a MarkkitBuilder) -> Self { - Self { config } + Self { + config, + footnote_labels: RefCell::new(HashMap::new()), + } } /// Parse an HTML string into a [`Document`]. @@ -91,8 +114,23 @@ impl<'a> HtmlParser<'a> { if self.is_excluded(&el, exclude) { continue; } + let name = el.value().name(); + // Captions are consumed by the enclosing figure + if name == "figcaption" && parent.value().name() == "figure" { + continue; + } + if is_footnote_section(&el) { + self.convert_footnotes(el, exclude, out, depth + 1); + continue; + } + if name == "figure" + && let Some(fig) = self.convert_figure(el, exclude, depth + 1) + { + out.push(fig); + continue; + } // Transparent containers: recurse directly into their children - if is_transparent_container(el.value().name()) { + if is_transparent_container(name) { self.convert_children(el, exclude, out, depth + 1); } else if let Some(node) = self.convert_element(el, exclude, depth + 1) { out.push(node); @@ -139,20 +177,16 @@ impl<'a> HtmlParser<'a> { Some(Node::Paragraph { children }) } - "div" | "section" | "article" | "main" | "span" | "figure" | "figcaption" => { - let mut nodes = Vec::new(); - self.convert_children(el, exclude, &mut nodes, depth); - if nodes.len() == 1 { - Some(nodes.remove(0)) - } else if nodes.is_empty() { - None - } else if nodes.iter().all(is_inline) { - Some(Node::Paragraph { children: nodes }) - } else { - None - } + "figure" => self + .convert_figure(el, exclude, depth) + .or_else(|| self.convert_container(el, exclude, depth)), + + "div" | "section" | "article" | "main" | "span" | "figcaption" => { + self.convert_container(el, exclude, depth) } + "dl" => self.convert_definition_list(el, exclude, depth), + "ul" => { let items = self.collect_list_items(el, exclude, depth); if items.is_empty() { @@ -238,6 +272,14 @@ impl<'a> HtmlParser<'a> { } "a" => { + if is_footnote_backref(&el) { + return None; + } + if is_footnote_ref(&el) + && let Some(node) = self.footnote_reference(el) + { + return Some(node); + } if !self.config.preserve_links { let children = self.collect_inline(el, exclude, depth); return if children.is_empty() { @@ -293,6 +335,181 @@ impl<'a> HtmlParser<'a> { } } + /// Generic container (`div`, `span`, ...) in inline/element context. + fn convert_container( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + let mut nodes = Vec::new(); + self.convert_children(el, exclude, &mut nodes, depth); + if nodes.len() == 1 { + Some(nodes.remove(0)) + } else if nodes.is_empty() { + None + } else if nodes.iter().all(Node::is_inline) { + Some(Node::Paragraph { children: nodes }) + } else { + None + } + } + + /// `
    ` with a non-empty `
    `; `None` otherwise. + fn convert_figure( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + let figcaption = el + .children() + .filter_map(scraper::ElementRef::wrap) + .find(|c| c.value().name() == "figcaption" && !self.is_excluded(c, exclude))?; + let mut caption = self.collect_inline(figcaption, exclude, depth + 1); + trim_edge_text(&mut caption); + if caption.is_empty() { + return None; + } + let mut nodes = Vec::new(); + self.convert_children(el, exclude, &mut nodes, depth); + let mut children = normalize_blocks(nodes); + if !children.is_empty() && children.iter().all(Node::is_inline) { + trim_edge_text(&mut children); + children = vec![Node::Paragraph { children }]; + } + Some(Node::Figure { + children, + caption: Some(caption), + }) + } + + /// `
    `: `
    ` starts a term, each `
    ` adds a definition. `
    ` + /// wrappers (allowed by HTML) are walked through. + fn convert_definition_list( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + let mut items: Vec = Vec::new(); + self.collect_definitions(el, exclude, depth, &mut items); + items.retain(|i| !i.term.is_empty() || !i.definitions.is_empty()); + if items.is_empty() { + return None; + } + Some(Node::DefinitionList { items }) + } + + fn collect_definitions( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + items: &mut Vec, + ) { + if depth >= MAX_DEPTH { + return; + } + for child in el.children().filter_map(scraper::ElementRef::wrap) { + if self.is_excluded(&child, exclude) { + continue; + } + match child.value().name() { + "dt" => { + let mut term = self.collect_inline(child, exclude, depth + 1); + trim_edge_text(&mut term); + if term.is_empty() { + continue; + } + // Consecutive
    s share the following definitions. + match items.last_mut() { + Some(last) if last.definitions.is_empty() && !last.term.is_empty() => { + last.term.push(Node::LineBreak); + last.term.extend(term); + } + _ => items.push(DefinitionItem::new(term, Vec::new())), + } + } + "dd" => { + let mut nodes = Vec::new(); + self.convert_children(child, exclude, &mut nodes, depth + 1); + let mut def = normalize_blocks(nodes); + if def.iter().all(Node::is_inline) { + trim_edge_text(&mut def); + } + if def.is_empty() { + continue; + } + match items.last_mut() { + Some(last) => last.definitions.push(def), + None => items.push(DefinitionItem::new(Vec::new(), vec![def])), + } + } + "div" => self.collect_definitions(child, exclude, depth + 1, items), + _ => {} + } + } + } + + /// Footnote section: one `FootnoteDefinition` per `
  • ` of its list. + fn convert_footnotes( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + out: &mut Vec, + depth: usize, + ) { + let ol_sel = Selector::parse("ol").unwrap(); + let Some(list) = (if el.value().name() == "ol" { + Some(el) + } else { + el.select(&ol_sel).next() + }) else { + return; + }; + let mut n = 0usize; + for li in list.children().filter_map(scraper::ElementRef::wrap) { + if li.value().name() != "li" || self.is_excluded(&li, exclude) { + continue; + } + n += 1; + let label = li + .value() + .attr("id") + .and_then(|id| self.footnote_labels.borrow().get(id).cloned()) + .unwrap_or_else(|| n.to_string()); + let mut nodes = Vec::new(); + self.convert_children(li, exclude, &mut nodes, depth + 1); + let mut children = normalize_blocks(nodes); + if children.iter().all(Node::is_inline) { + trim_edge_text(&mut children); + if !children.is_empty() { + children = vec![Node::Paragraph { children }]; + } + } + if !children.is_empty() { + out.push(Node::FootnoteDefinition { label, children }); + } + } + } + + /// `` that is a footnote reference: record target and build the node. + fn footnote_reference(&self, el: scraper::ElementRef<'_>) -> Option { + let target = el.value().attr("href")?.strip_prefix('#')?; + let text: String = el.text().collect(); + let text = text + .trim() + .trim_start_matches('[') + .trim_end_matches(']') + .trim(); + let label = if text.is_empty() { target } else { text }.to_string(); + self.footnote_labels + .borrow_mut() + .insert(target.to_string(), label.clone()); + Some(Node::FootnoteReference { label }) + } + fn collect_inline( &self, el: scraper::ElementRef<'_>, @@ -349,9 +566,20 @@ impl<'a> HtmlParser<'a> { { let mut children = Vec::new(); self.convert_children(el, exclude, &mut children, depth + 1); - let children = normalize_blocks(children); + let mut children = normalize_blocks(children); + let checked = task_checkbox(el); + if checked.is_some() { + if children.iter().all(Node::is_inline) { + trim_edge_text(&mut children); + } else if let Some(Node::Paragraph { children: first }) = children.first_mut() { + trim_edge_text(first); + } + } if !children.is_empty() { - items.push(ListItem::new(children)); + items.push(match checked { + Some(c) => ListItem::task(children, c), + None => ListItem::new(children), + }); } } } @@ -367,18 +595,20 @@ impl<'a> HtmlParser<'a> { let thead_sel = Selector::parse("thead").unwrap(); let tbody_sel = Selector::parse("tbody").unwrap(); let tr_sel = Selector::parse("tr").unwrap(); - let th_sel = Selector::parse("th").unwrap(); - let td_sel = Selector::parse("td").unwrap(); - let mut headers: Vec> = Vec::new(); - let mut rows: Vec>> = Vec::new(); + let mut headers: Vec = Vec::new(); + let mut alignments: Vec = Vec::new(); + let mut rows: Vec> = Vec::new(); if let Some(thead) = table_el.select(&thead_sel).next() && let Some(tr) = thead.select(&tr_sel).next() { - for th in tr.select(&th_sel) { - headers.push(self.collect_inline(th, exclude, depth + 1)); - } + let ths = row_cells(tr, &["th", "td"]); + alignments = column_alignments(&ths); + headers = ths + .into_iter() + .map(|th| self.table_cell(th, exclude, depth)) + .collect(); } let body_rows: Vec> = @@ -389,19 +619,24 @@ impl<'a> HtmlParser<'a> { }; for tr in body_rows { - if headers.is_empty() { - let ths: Vec<_> = tr.select(&th_sel).collect(); - if !ths.is_empty() { - for th in ths { - headers.push(self.collect_inline(th, exclude, depth + 1)); - } - continue; - } + let row = row_cells(tr, &["td", "th"]); + // A leading all- row is the header. + if headers.is_empty() + && rows.is_empty() + && !row.is_empty() + && row.iter().all(|c| c.value().name() == "th") + { + alignments = column_alignments(&row); + headers = row + .into_iter() + .map(|th| self.table_cell(th, exclude, depth)) + .collect(); + continue; } - let cells: Vec> = tr - .select(&td_sel) - .map(|td| self.collect_inline(td, exclude, depth + 1)) + let cells: Vec = row + .into_iter() + .map(|td| self.table_cell(td, exclude, depth)) .collect(); if !cells.is_empty() { @@ -412,11 +647,149 @@ impl<'a> HtmlParser<'a> { if headers.is_empty() && rows.is_empty() { return None; } + if alignments.iter().all(|a| *a == Alignment::None) { + alignments.clear(); + } + + Some(Node::Table { + alignments, + headers, + rows, + }) + } - Some(Node::Table { headers, rows }) + fn table_cell( + &self, + cell: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> TableCell { + let colspan = span_attr(cell, "colspan").map_or(1, |n| n.clamp(1, MAX_COLSPAN)); + let rowspan = span_attr(cell, "rowspan").map_or(1, |n| { + if n == 0 { + MAX_ROWSPAN + } else { + n.min(MAX_ROWSPAN) + } + }); + TableCell::new(self.collect_inline(cell, exclude, depth + 1)).with_span(colspan, rowspan) } } +/// Direct `th`/`td` children of a row (nested tables' cells excluded). +fn row_cells<'a>(tr: scraper::ElementRef<'a>, names: &[&str]) -> Vec> { + tr.children() + .filter_map(scraper::ElementRef::wrap) + .filter(|c| names.contains(&c.value().name())) + .collect() +} + +fn span_attr(cell: scraper::ElementRef<'_>, name: &str) -> Option { + let v = cell.value().attr(name)?.trim(); + // Very long digit strings saturate instead of failing to parse. + let digits: String = v + .chars() + .take_while(char::is_ascii_digit) + .take(10) + .collect(); + if digits.is_empty() { + return None; + } + Some( + digits + .parse::() + .map_or(u32::MAX, |n| n.min(u32::MAX as u64) as u32), + ) +} + +/// Per-grid-column alignment from a header row (`align` attr or `text-align` style). +fn column_alignments(cells: &[scraper::ElementRef<'_>]) -> Vec { + let mut out = Vec::new(); + for cell in cells { + let a = cell_alignment(cell); + let span = span_attr(*cell, "colspan").map_or(1, |n| n.clamp(1, MAX_COLSPAN)); + for _ in 0..span { + out.push(a); + } + } + out +} + +fn cell_alignment(cell: &scraper::ElementRef<'_>) -> Alignment { + let from = |v: &str| match v.trim().to_ascii_lowercase().as_str() { + "left" | "start" => Alignment::Left, + "center" => Alignment::Center, + "right" | "end" => Alignment::Right, + _ => Alignment::None, + }; + if let Some(style) = cell.value().attr("style") { + for decl in style.split(';') { + if let Some((k, v)) = decl.split_once(':') + && k.trim().eq_ignore_ascii_case("text-align") + { + return from(v); + } + } + } + cell.value().attr("align").map_or(Alignment::None, from) +} + +/// Task-list checkbox leading a `
  • ` (directly or inside its first `

    `/`

    `): terms with one or more definitions. + DefinitionList { items: Vec }, + /// Figure: block content with an optional inline caption. + Figure { + children: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + caption: Option>, + }, + /// Footnote body, referenced by [`Node::FootnoteReference`] with the same label. + FootnoteDefinition { label: String, children: Vec }, /// Plain text. Text(String), /// Bold / strong emphasis. @@ -68,17 +111,127 @@ pub enum Node { Image { src: String, alt: String }, /// Hard line break. LineBreak, + /// Footnote marker pointing at a [`Node::FootnoteDefinition`]. + FootnoteReference { label: String }, +} + +impl Node { + /// True for block-level nodes (headings, paragraphs, lists, tables, ...). + pub fn is_block(&self) -> bool { + matches!( + self, + Node::Heading { .. } + | Node::Paragraph { .. } + | Node::List { .. } + | Node::BlockQuote { .. } + | Node::CodeBlock { .. } + | Node::Table { .. } + | Node::ThematicBreak + | Node::DefinitionList { .. } + | Node::Figure { .. } + | Node::FootnoteDefinition { .. } + ) + } + + /// True for inline nodes (text, emphasis, links, ...). + pub fn is_inline(&self) -> bool { + !self.is_block() + } } /// A single item in a list. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] pub struct ListItem { pub children: Vec, + /// Task-list state: `Some(true)` checked, `Some(false)` unchecked, `None` plain item. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checked: Option, } impl ListItem { pub fn new(children: Vec) -> Self { - Self { children } + Self { + children, + checked: None, + } + } + + /// Task-list item with a checkbox. + pub fn task(children: Vec, checked: bool) -> Self { + Self { + children, + checked: Some(checked), + } + } +} + +/// Horizontal alignment of a table column. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub enum Alignment { + #[default] + None, + Left, + Center, + Right, +} + +/// A table cell: inline content plus its span (both ≥ 1). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct TableCell { + pub children: Vec, + #[serde(default = "one", skip_serializing_if = "is_one")] + pub colspan: u32, + #[serde(default = "one", skip_serializing_if = "is_one")] + pub rowspan: u32, +} + +fn one() -> u32 { + 1 +} + +fn is_one(n: &u32) -> bool { + *n == 1 +} + +impl TableCell { + pub fn new(children: Vec) -> Self { + Self { + children, + colspan: 1, + rowspan: 1, + } + } + + /// Set the span; zero is treated as one. + pub fn with_span(mut self, colspan: u32, rowspan: u32) -> Self { + self.colspan = colspan.max(1); + self.rowspan = rowspan.max(1); + self + } +} + +impl From> for TableCell { + fn from(children: Vec) -> Self { + Self::new(children) + } +} + +/// One term of a [`Node::DefinitionList`] with its definitions. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct DefinitionItem { + /// Inline content of the term. + pub term: Vec, + /// Each definition is a list of block (or inline) nodes. + pub definitions: Vec>, +} + +impl DefinitionItem { + pub fn new(term: Vec, definitions: Vec>) -> Self { + Self { term, definitions } } } @@ -106,11 +259,70 @@ mod tests { children: vec![Node::Text("bold".into())], }, Node::Text(" text".into()), + Node::FootnoteReference { label: "1".into() }, ], }, + Node::List { + ordered: false, + start: None, + items: vec![ + ListItem::task(vec![Node::Text("done".into())], true), + ListItem::new(vec![Node::Text("plain".into())]), + ], + }, + Node::Table { + alignments: vec![Alignment::Left, Alignment::Right], + headers: vec![TableCell::new(vec![Node::Text("a".into())]).with_span(2, 1)], + rows: vec![vec![TableCell::new(vec![]), TableCell::new(vec![])]], + }, + Node::DefinitionList { + items: vec![DefinitionItem::new( + vec![Node::Text("term".into())], + vec![vec![Node::Text("def".into())]], + )], + }, + Node::Figure { + children: vec![Node::Image { + src: "a.png".into(), + alt: "A".into(), + }], + caption: Some(vec![Node::Text("cap".into())]), + }, + Node::FootnoteDefinition { + label: "1".into(), + children: vec![Node::Paragraph { + children: vec![Node::Text("note".into())], + }], + }, ]); let json = serde_json::to_string(&doc).unwrap(); let parsed: Document = serde_json::from_str(&json).unwrap(); assert_eq!(doc, parsed); + assert_eq!(parsed.ir_version, IR_VERSION); + } + + #[test] + fn json_omits_defaults() { + let doc = Document::new(vec![Node::Table { + alignments: vec![], + headers: vec![TableCell::new(vec![Node::Text("h".into())])], + rows: vec![], + }]); + let json = serde_json::to_string(&doc).unwrap(); + assert!(json.starts_with(r#"{"ir_version":2,"#), "{json}"); + assert!(!json.contains("colspan"), "{json}"); + assert!(!json.contains("alignments"), "{json}"); + } + + #[test] + fn missing_version_reads_as_v1() { + let doc: Document = serde_json::from_str(r#"{"nodes":[{"Text":"x"}]}"#).unwrap(); + assert_eq!(doc.ir_version, 1); + } + + #[test] + fn zero_span_clamped() { + let c = TableCell::new(vec![]).with_span(0, 0); + assert_eq!((c.colspan, c.rowspan), (1, 1)); } } diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs index eec02a1..9d68347 100644 --- a/crates/markkit/src/render/markdown.rs +++ b/crates/markkit/src/render/markdown.rs @@ -1,6 +1,22 @@ //! IR → Markdown renderer. - -use crate::ir::{Document, ListItem, Node}; +//! +//! Decisions: +//! - GFM is the target. Constructs GFM lacks use the de facto extensions most +//! Markdown tools read: footnotes `[^label]` / `[^label]: body`, definition +//! lists `Term` + `: definition` (PHP Markdown Extra / Pandoc), task items +//! `- [x]`. Figures render their content plus an italic caption paragraph. +//! - Table spans are laid out on a grid (GFM has no spans): a spanned cell's +//! content goes in its top-left slot, covered slots stay empty. Layout is +//! bounded by `MAX_TABLE_GRID_CELLS`; a table whose spans would exceed it is +//! laid out ignoring spans (threat model T2, invariant S13). + +use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell}; + +/// Grid budget for span layout (rows × columns). +const MAX_TABLE_GRID_CELLS: usize = 1_000_000; +/// Span clamps applied at layout time (HTML limits; IR may come from anywhere). +const MAX_COLSPAN: usize = 1000; +const MAX_ROWSPAN: usize = 65534; /// Renders a [`Document`] IR into Markdown text. pub struct MarkdownRenderer { @@ -25,7 +41,7 @@ impl MarkdownRenderer { /// Render a slice of nodes. fn render_nodes(&mut self, nodes: &[Node], depth: usize) { for (i, node) in nodes.iter().enumerate() { - if i > 0 && is_block(node) && is_block(&nodes[i - 1]) { + if i > 0 && node.is_block() && nodes[i - 1].is_block() { self.buf.push('\n'); } self.render_node(node, depth); @@ -89,14 +105,41 @@ impl MarkdownRenderer { self.buf.push_str("```\n"); } - Node::Table { headers, rows } => { - self.render_table(headers, rows); + Node::Table { + alignments, + headers, + rows, + } => { + self.render_table(alignments, headers, rows); } Node::ThematicBreak => { self.buf.push_str("---\n"); } + Node::DefinitionList { items } => self.render_definition_list(items, depth), + + Node::Figure { children, caption } => { + self.render_nodes(children, depth); + if let Some(caption) = caption { + self.buf.push('\n'); + let start = self.buf.len(); + self.render_inline_node(&Node::Italic { + children: caption.clone(), + }); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + } + } + + Node::FootnoteDefinition { label, children } => { + let inner = render_blocks(children, depth); + self.buf.push_str("[^"); + self.buf.push_str(&footnote_label(label)); + self.buf.push_str("]:"); + push_hanging(&mut self.buf, &inner, " ", " "); + } + // Inline nodes rendered at block level get wrapped in a line Node::Text(_) | Node::Bold { .. } @@ -105,12 +148,31 @@ impl MarkdownRenderer { | Node::InlineCode(_) | Node::Link { .. } | Node::Image { .. } - | Node::LineBreak => { + | Node::LineBreak + | Node::FootnoteReference { .. } => { self.render_inline_node(node); } } } + /// Render a definition list: `Term` line, then `: definition` per definition. + fn render_definition_list(&mut self, items: &[DefinitionItem], depth: usize) { + for (i, item) in items.iter().enumerate() { + if i > 0 { + self.buf.push('\n'); + } + let start = self.buf.len(); + self.render_inline(&item.term); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + for def in &item.definitions { + let inner = render_blocks(def, depth); + self.buf.push(':'); + push_hanging(&mut self.buf, &inner, " ", " "); + } + } + } + /// Render a list. fn render_list(&mut self, ordered: bool, start: Option, items: &[ListItem], depth: usize) { let indent = " ".repeat(depth); @@ -123,9 +185,14 @@ impl MarkdownRenderer { format!("{}- ", indent) }; self.buf.push_str(&marker); + match item.checked { + Some(true) => self.buf.push_str("[x] "), + Some(false) => self.buf.push_str("[ ] "), + None => {} + } // Render item children - let has_block = item.children.iter().any(|n| !is_inline_node(n)); + let has_block = item.children.iter().any(Node::is_block); if has_block { // Multi-line item: render sub-blocks indented let mut first = true; @@ -241,53 +308,162 @@ impl MarkdownRenderer { Node::LineBreak => { self.buf.push_str(" \n"); } + Node::FootnoteReference { label } => { + self.buf.push_str("[^"); + self.buf.push_str(&footnote_label(label)); + self.buf.push(']'); + } // Block nodes shouldn't appear here but handle gracefully other => self.render_node(other, 0), } } - /// Render a GFM table. - fn render_table(&mut self, headers: &[Vec], rows: &[Vec>]) { - // Determine column count - let col_count = headers - .len() - .max(rows.iter().map(|r| r.len()).max().unwrap_or(0)); - + /// Render a GFM table, laying spans out on a grid. + fn render_table( + &mut self, + alignments: &[Alignment], + headers: &[TableCell], + rows: &[Vec], + ) { + let all_rows: Vec<&[TableCell]> = std::iter::once(headers) + .chain(rows.iter().map(Vec::as_slice)) + .collect(); + let grid = layout_table(&all_rows, true).unwrap_or_else(|| { + layout_table(&all_rows, false).expect("span-free layout is within budget") + }); + let col_count = grid.iter().map(Vec::len).max().unwrap_or(0); if col_count == 0 { return; } - // Render header row - self.buf.push('|'); - for i in 0..col_count { - self.buf.push(' '); - if let Some(cells) = headers.get(i) { - self.render_inline(cells); - } - self.buf.push_str(" |"); - } - self.buf.push('\n'); - - // Separator - self.buf.push('|'); - for _ in 0..col_count { - self.buf.push_str(" --- |"); - } - self.buf.push('\n'); - - // Data rows - for row in rows { + for (r, row) in grid.iter().enumerate() { self.buf.push('|'); for i in 0..col_count { self.buf.push(' '); - if let Some(cells) = row.get(i) { + if let Some(Some(cells)) = row.get(i) { self.render_inline(cells); } self.buf.push_str(" |"); } self.buf.push('\n'); + + if r == 0 { + self.buf.push('|'); + for i in 0..col_count { + self.buf + .push_str(match alignments.get(i).copied().unwrap_or_default() { + Alignment::Left => " :--- |", + Alignment::Center => " :---: |", + Alignment::Right => " ---: |", + _ => " --- |", + }); + } + self.buf.push('\n'); + } + } + } +} + +/// Grid slot: `Some(content)` at a cell's top-left, `None` where covered or empty. +type GridRow<'a> = Vec>; + +/// Place cells on a grid honouring `colspan`/`rowspan` when `spans` is set. +/// +/// Returns `None` when the grid would exceed `MAX_TABLE_GRID_CELLS`; the +/// caller then retries with `spans = false`, whose size is bounded by the +/// input (one slot per cell, rows padded at render time). +fn layout_table<'a>(rows: &[&'a [TableCell]], spans: bool) -> Option>> { + let mut grid: Vec> = Vec::with_capacity(rows.len()); + // Remaining rows each column stays covered by a rowspan from above. + let mut pending: Vec = Vec::new(); + let mut width = 0usize; + for row in rows { + let busy: Vec = pending.iter().map(|&p| p > 0).collect(); + for p in pending.iter_mut() { + *p = p.saturating_sub(1); + } + let mut out: GridRow<'a> = Vec::new(); + let mut col = 0usize; + for cell in row.iter() { + while busy.get(col).copied().unwrap_or(false) { + col += 1; + } + let (cs, rs) = if spans { + ( + (cell.colspan as usize).clamp(1, MAX_COLSPAN), + (cell.rowspan as usize).clamp(1, MAX_ROWSPAN), + ) + } else { + (1, 1) + }; + let end = col + cs; + if spans && end.saturating_mul(grid.len() + 1) > MAX_TABLE_GRID_CELLS { + return None; + } + if out.len() < end { + out.resize(end, None); + } + out[col] = Some(cell.children.as_slice()); + if rs > 1 { + if pending.len() < end { + pending.resize(end, 0); + } + for p in &mut pending[col..end] { + *p = rs - 1; + } + } + col = end; } + width = width.max(out.len()).max(pending.len()); + if spans && width.saturating_mul(grid.len() + 1) > MAX_TABLE_GRID_CELLS { + return None; + } + grid.push(out); } + Some(grid) +} + +/// Render block children to normalized Markdown in a fresh renderer. +fn render_blocks(nodes: &[Node], depth: usize) -> String { + let mut sub = MarkdownRenderer::new(); + sub.render_nodes(nodes, depth); + normalize_output(&sub.buf) +} + +/// Append `inner` with `first` before its first line and `rest` before the +/// other non-blank lines (hanging indent for footnotes / definitions). +fn push_hanging(buf: &mut String, inner: &str, first: &str, rest: &str) { + let mut lines = inner.lines(); + if let Some(line) = lines.next() { + buf.push_str(first); + buf.push_str(line); + } + buf.push('\n'); + for line in lines { + if !line.is_empty() { + buf.push_str(rest); + buf.push_str(line); + } + buf.push('\n'); + } +} + +/// Footnote labels are emitted inside `[^...]`: keep `[A-Za-z0-9_.-]`, map +/// everything else to `-` so a label cannot close the bracket or break the line. +fn footnote_label(label: &str) -> String { + let out: String = label + .trim() + .chars() + .take(64) + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') { + c + } else { + '-' + } + }) + .collect(); + if out.is_empty() { "note".into() } else { out } } impl Default for MarkdownRenderer { @@ -324,24 +500,6 @@ pub(crate) fn escape_table_cell(s: &str) -> String { out } -/// Check if a node is block-level. -fn is_block(node: &Node) -> bool { - matches!( - node, - Node::Heading { .. } - | Node::Paragraph { .. } - | Node::List { .. } - | Node::BlockQuote { .. } - | Node::CodeBlock { .. } - | Node::Table { .. } - | Node::ThematicBreak - ) -} - -fn is_inline_node(node: &Node) -> bool { - !is_block(node) -} - /// Normalize output: collapse multiple blank lines, preserve hard breaks (two trailing spaces). fn normalize_output(s: &str) -> String { let mut result = String::with_capacity(s.len()); @@ -394,6 +552,13 @@ mod tests { assert_eq!(normalize_output("a \nb\n"), "a \nb\n"); } + #[test] + fn test_footnote_label() { + assert_eq!(footnote_label("1"), "1"); + assert_eq!(footnote_label("a]b\n[c"), "a-b--c"); + assert_eq!(footnote_label(" "), "note"); + } + #[cfg(any(feature = "docx", feature = "spreadsheet"))] #[test] fn test_escape_table_cell() { diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index 2ca280c..2b04212 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -28,7 +28,7 @@ use crate::archive::{ArchiveError, ZipGuard}; use crate::config::MarkkitBuilder; use crate::error::Error; use crate::format::Format; -use crate::ir::{Document, Node}; +use crate::ir::{Document, Node, TableCell}; use crate::render::escape_table_cell; pub(crate) struct SheetOutput { @@ -376,17 +376,18 @@ impl<'c> Collector<'c> { self.sheets += 1; self.rows += rows.len(); let cell = |s: String| { - if s.is_empty() { + TableCell::new(if s.is_empty() { vec![] } else { vec![Node::Text(s)] - } + }) }; self.nodes.push(Node::Heading { level: 2, children: vec![Node::Text(name.replace(['\r', '\n'], " "))], }); self.nodes.push(Node::Table { + alignments: Vec::new(), headers: header.into_iter().map(cell).collect(), rows: rows .into_iter() diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index afd18f6..e2402e3 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -739,3 +739,133 @@ fn loose_text_between_blocks_becomes_paragraph() { fn inline_only_document_unchanged() { assert_eq!(convert("Hello world"), "Hello **world**\n"); } + +// ─── IR v2: tasks, definition lists, figures, footnotes, tables ───── + +#[test] +fn task_list_github_markup() { + let html = r#"
      +
    • Done
      • +
    • Todo
      • +
    • Plain
      • +
    "#; + assert_eq!(convert(html), "- [x] Done\n- [ ] Todo\n- Plain\n"); +} + +#[test] +fn task_list_checkbox_inside_paragraph() { + let html = r#"

    1. Ship it

    "#; + assert_eq!(convert(html), "1. [x] Ship it\n"); +} + +#[test] +fn checkbox_after_text_is_not_task() { + let html = r#"
    • Agree
    "#; + assert_eq!(convert(html), "- Agree\n"); +} + +#[test] +fn non_checkbox_input_is_not_task() { + let html = r#"
    • Name
    "#; + assert!(!convert(html).contains('[')); +} + +#[test] +fn definition_list() { + let html = "
    Rust
    A language.
    Also a fungus.
    \ +
    Go

    Another.

    Second para.

    "; + assert_eq!( + convert(html), + "Rust\n: A language.\n: Also a fungus.\n\nGo\n: Another.\n\n Second para.\n" + ); +} + +#[test] +fn definition_list_div_wrappers_and_shared_terms() { + let html = "
    A
    B
    Both
    "; + assert_eq!(convert(html), "A \nB\n: Both\n"); +} + +#[test] +fn empty_definition_list_elided() { + assert_eq!(convert("

    x

    "), "x\n"); +} + +#[test] +fn figure_with_caption() { + let html = + r#"
    Cat
    A cat.
    "#; + assert_eq!(convert(html), "![Cat](cat.png)\n\n*A **cat**.*\n"); +} + +#[test] +fn figure_without_caption_is_transparent() { + let html = r#"
    Cat
    "#; + assert_eq!(convert(html), "![Cat](cat.png)\n"); +} + +#[test] +fn footnotes_github_markup() { + let html = r##"

    Claim1.

    +

    Footnotes

    +

    1. Source. ↩

    +
    "##; + assert_eq!(convert(html), "Claim[^1].\n\n[^1]: Source.\n"); +} + +#[test] +fn footnotes_pandoc_markup() { + let html = r##"

    Texta

    +
    +

    1. First.

      Second.↩︎

    +
    "##; + assert_eq!(convert(html), "Text[^a]\n\n[^a]: First.\n\n Second.\n"); +} + +#[test] +fn footnote_label_cannot_break_out() { + let html = r##"

    x]](javascript:alert(1))

    "##; + let md = convert(html); + assert!(!md.contains("]("), "{md}"); + assert!(md.starts_with("x[^"), "{md}"); +} + +#[test] +fn table_alignment() { + let html = r#" +
    LCRN
    1234
    "#; + assert_eq!( + convert(html), + "| L | C | R | N |\n| :--- | :---: | ---: | --- |\n| 1 | 2 | 3 | 4 |\n" + ); +} + +#[test] +fn table_colspan_and_rowspan() { + let html = r#" + + + +
    ABC
    x12
    34
    "#; + assert_eq!( + convert(html), + "| AB | | C |\n| --- | --- | --- |\n| x | 1 | 2 |\n| | 3 | 4 |\n" + ); +} + +#[test] +fn table_row_header_cells_kept() { + let html = "
    KV
    a1
    "; + assert_eq!(convert(html), "| K | V |\n| --- | --- |\n| a | 1 |\n"); +} + +#[test] +fn ir_v2_json_has_version() { + let m = Markkit::builder().build(); + let doc = m + .parse_html("
    • x
    ") + .unwrap(); + let json = serde_json::to_value(&doc).unwrap(); + assert_eq!(json["ir_version"], markkit::ir::IR_VERSION); + assert_eq!(json["nodes"][0]["List"]["items"][0]["checked"], true); +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index d09ba18..c51b2e5 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -204,3 +204,32 @@ fn depth_limit_exceeded() { md ); } + +// ─── Table spans (T2: span amplification) ──────────────────────────── + +#[test] +fn hostile_colspan_rowspan_bounded() { + // Huge spans on many rows must not blow up the layout grid. + let mut html = String::from(""); + for _ in 0..5_000 { + html.push_str(r#""#); + } + html.push_str("
    ab
    "); + let start = std::time::Instant::now(); + let md = convert(&html); + assert!(start.elapsed().as_secs() < 10); + // Output stays proportional to input (spans dropped past the grid budget). + assert!(md.len() < html.len() * 4, "output {} bytes", md.len()); +} + +#[test] +fn non_numeric_spans_ignored() { + let md = convert(r#"
    ab
    "#); + assert!(md.contains("| a | b |"), "{md}"); +} + +#[test] +fn deep_definition_lists_bounded() { + let html = "
    ".repeat(500) + "x" + &"
    ".repeat(500); + let _ = convert(&html); +} diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index c1bcc1a..9588f57 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -34,7 +34,11 @@ HTML from the web: malformed markup, complex layouts, embedded scripts, etc. | `