From bba8a300f83c9f476d9c8a988e27bb31791bef78 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:58:17 +0000 Subject: [PATCH 1/4] feat(html): declarative element rules - rules::HtmlRule { selector, action } with ElementAction (skip, unwrap, heading, paragraph, block_quote+title, code_block, inline_code, bold, italic, text); serde JSON shape - MarkkitBuilder::html_rule / html_rules; tried before built-in mapping, invalid selectors fail with Error::HtmlParse - CLI: markkit extract --rules rules.json --- CHANGELOG.md | 3 + README.md | 4 + crates/markkit-cli/src/main.rs | 12 ++ crates/markkit-cli/tests/cli_integration.rs | 46 ++++++ crates/markkit/src/config.rs | 15 ++ crates/markkit/src/html/parser.rs | 156 +++++++++++++++++++- crates/markkit/src/lib.rs | 1 + crates/markkit/src/rules.rs | 97 ++++++++++++ crates/markkit/tests/html_integration.rs | 71 +++++++++ crates/markkit/tests/html_security.rs | 10 ++ knowledge/formats/html-extraction.md | 25 ++++ knowledge/log.md | 1 + 12 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 crates/markkit/src/rules.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fabe68..924b255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - IR transforms: `MarkkitBuilder::transform(..)` runs `Transform` passes after parsing and before rendering, for every format. Built-ins `transform::NormalizeHeadings` and `transform::DropImages`; `transform::from_fn` for closures; `Document::visit` / `visit_mut` to walk the tree. +- Declarative HTML element rules: `MarkkitBuilder::html_rule(".note", ElementAction::BlockQuote { .. })` + maps elements by CSS selector to skip / unwrap / heading / paragraph / block quote (with title) / + code / emphasis / text, before the built-in mapping. CLI: `markkit extract --rules rules.json`. - JSON Schema for the IR at `schema/ir.schema.json`; `json-schema` feature adds `markkit::ir::json_schema()`. - Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations. diff --git a/README.md b/README.md index 94dffb6..cb347e7 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,10 @@ markkit extract --file page.html --base-url https://example.com/blog/ # From stdin cat page.html | markkit extract - +# Map site-specific markup with element rules (JSON file) +# [{"selector": ".note", "action": "block_quote", "title": "Note"}, {"selector": ".ad", "action": "skip"}] +markkit extract --file page.html --rules rules.json + # MCP server (stdio) exposing the `markkit` tool to agents markkit mcp # add --no-url to keep it offline ``` diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 753cf5d..7f74cbc 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -42,6 +42,11 @@ enum Commands { #[arg(long = "exclude", short = 'e')] exclude_selectors: Vec, + /// JSON file with HTML element rules, e.g. + /// `[{"selector": ".note", "action": "block_quote", "title": "Note"}]` + #[arg(long)] + rules: Option, + /// Output as JSON with metadata #[arg(long)] json: bool, @@ -193,6 +198,7 @@ async fn main() -> Result<()> { include_selectors, exclude_selectors, limits, + rules, json: output_json, base_url, main_content, @@ -218,6 +224,12 @@ async fn main() -> Result<()> { builder = builder.base_url(base); } + if let Some(path) = &rules { + let text = std::fs::read_to_string(path)?; + builder.html_rules = serde_json::from_str(&text) + .map_err(|e| anyhow::anyhow!("invalid rules file {}: {e}", path.display()))?; + } + let markkit = builder.build(); let source_label = source.clone().unwrap_or_else(|| "".to_string()); diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 006b86f..4692455 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -251,3 +251,49 @@ fn limit_flags_enforced() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("max_output_size"), "stderr: {stderr}"); } + +#[test] +fn extract_with_rules_file() { + let dir = tempfile::tempdir().unwrap(); + let page = dir.path().join("page.html"); + std::fs::write( + &page, + r#"
Careful
Buy
"#, + ) + .unwrap(); + let rules = dir.path().join("rules.json"); + std::fs::write( + &rules, + r#"[{"selector": ".note", "action": "block_quote", "title": "Note"}, + {"selector": ".ad", "action": "skip"}]"#, + ) + .unwrap(); + + let output = markkit_bin() + .args(["extract", "--file", page.to_str().unwrap()]) + .args(["--rules", rules.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim_end(), + "> **Note**\n>\n> Careful" + ); +} + +#[test] +fn extract_with_bad_rules_file_fails() { + let dir = tempfile::tempdir().unwrap(); + let rules = dir.path().join("rules.json"); + std::fs::write(&rules, r#"[{"selector": "p", "action": "exec"}]"#).unwrap(); + let output = markkit_bin() + .args(["extract", "--stdin", "--rules", rules.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("invalid rules file")); +} diff --git a/crates/markkit/src/config.rs b/crates/markkit/src/config.rs index f6c66d3..85f428c 100644 --- a/crates/markkit/src/config.rs +++ b/crates/markkit/src/config.rs @@ -41,6 +41,9 @@ pub struct MarkkitBuilder { /// Readability-style. Include selectors win; falls back to the full body when /// nothing scores. Default: false. pub main_content: bool, + /// Declarative HTML element rules, tried in order before the built-in + /// mapping (see [`crate::rules`]). + pub html_rules: Vec, } impl Default for MarkkitBuilder { @@ -66,6 +69,7 @@ impl Default for MarkkitBuilder { track_sources: false, transforms: Default::default(), main_content: false, + html_rules: Vec::new(), } } } @@ -132,6 +136,17 @@ impl MarkkitBuilder { self } + /// Add an HTML element rule: elements matching `selector` become `action`. + pub fn html_rule( + mut self, + selector: impl Into, + action: crate::rules::ElementAction, + ) -> Self { + self.html_rules + .push(crate::rules::HtmlRule::new(selector, action)); + self + } + pub fn include_selector(mut self, selector: impl Into) -> Self { self.include_selectors.push(selector.into()); self diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 68f5b46..4abf193 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -16,6 +16,9 @@ //! and MathJax ``) maps to `Math` / `MathBlock` using the //! `application/x-tex` annotation; without one, the text content is kept as //! plain text. KaTeX's visual HTML duplicate is dropped. +//! - Element rules (`MarkkitBuilder::html_rule`, see `crate::rules`) are +//! compiled once per parser and tried before the built-in mapping in both +//! block and inline context; invalid selectors fail the parse. //! - `
` becomes `Figure` only when it has a non-empty `
`; //! otherwise it stays a transparent container as before. @@ -29,12 +32,20 @@ use crate::config::MarkkitBuilder; use crate::error::Error; use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, TableCell}; use crate::limits::Budget; +use crate::rules::ElementAction; use url::Url; use super::metadata::PageInfo; use super::{readability, urls}; +/// Outcome of a matching element rule. +enum RuleHit { + Skip, + Unwrap, + Node(Option), +} + /// HTML-spec clamps for table spans. const MAX_COLSPAN: u32 = 1000; const MAX_ROWSPAN: u32 = 65534; @@ -60,11 +71,32 @@ pub struct HtmlParser<'a> { sibling_index: RefCell>, /// Main-content mode: also drop "unlikely" subtrees (sidebars, share bars, ...). drop_unlikely: Cell, + /// Compiled element rules, in priority order. + rules: Vec<(Selector, ElementAction)>, + /// First invalid rule selector, reported by `parse`. + rule_error: Option, +} + +/// Compile rule selectors; keep the first invalid one for `parse` to report. +fn compile_rules(config: &MarkkitBuilder) -> (Vec<(Selector, ElementAction)>, Option) { + let mut rules = Vec::with_capacity(config.html_rules.len()); + let mut rule_error = None; + for r in &config.html_rules { + match Selector::parse(&r.selector) { + Ok(sel) => rules.push((sel, r.action.clone())), + Err(e) => { + rule_error + .get_or_insert_with(|| format!("invalid rule selector {:?}: {e}", r.selector)); + } + } + } + (rules, rule_error) } impl<'a> HtmlParser<'a> { /// Parser with a fresh per-call budget (deadline starts now). pub fn new(config: &'a MarkkitBuilder) -> Self { + let (rules, rule_error) = compile_rules(config); Self { config, budget: BudgetRef::Owned(Budget::new(&config.limits)), @@ -73,10 +105,13 @@ impl<'a> HtmlParser<'a> { base: urls::parse_base(config.base_url.as_deref()), sibling_index: RefCell::new(HashMap::new()), drop_unlikely: Cell::new(false), + rules, + rule_error, } } pub(crate) fn with_budget(config: &'a MarkkitBuilder, budget: &'a Budget) -> Self { + let (rules, rule_error) = compile_rules(config); Self { config, budget: BudgetRef::Borrowed(budget), @@ -85,6 +120,8 @@ impl<'a> HtmlParser<'a> { base: urls::parse_base(config.base_url.as_deref()), sibling_index: RefCell::new(HashMap::new()), drop_unlikely: Cell::new(false), + rules, + rule_error, } } @@ -111,6 +148,9 @@ impl<'a> HtmlParser<'a> { } fn parse_dom(&self, dom: &Html) -> Result { + if let Some(e) = &self.rule_error { + return Err(Error::HtmlParse(e.clone())); + } self.for_document(dom).convert_dom(dom) } @@ -125,6 +165,8 @@ impl<'a> HtmlParser<'a> { sibling_index: RefCell::new(HashMap::new()), base: urls::document_base(dom, self.base.clone()), drop_unlikely: Cell::new(false), + rules: self.rules.clone(), + rule_error: None, } } @@ -267,6 +309,21 @@ impl<'a> HtmlParser<'a> { if self.is_excluded(&el, exclude) { continue; } + match self.apply_rule(el, exclude, depth + 1) { + Some(RuleHit::Skip) => continue, + Some(RuleHit::Unwrap) => { + self.convert_children(el, exclude, out, depth + 1); + continue; + } + Some(RuleHit::Node(node)) => { + if let Some(mut node) = node { + self.mark_source(&mut node, el); + out.push(node); + } + continue; + } + None => {} + } let name = el.value().name(); // Captions are consumed by the enclosing figure if name == "figcaption" && parent.value().name() == "figure" { @@ -321,6 +378,13 @@ impl<'a> HtmlParser<'a> { return Some(Node::Text(trimmed.to_string())); } + match self.apply_rule(el, exclude, depth) { + Some(RuleHit::Skip) => return None, + Some(RuleHit::Unwrap) => return self.convert_container(el, exclude, depth), + Some(RuleHit::Node(node)) => return node, + None => {} + } + if let Some(m) = self.math_wrapper(el) { return Some(m); } @@ -526,6 +590,83 @@ impl<'a> HtmlParser<'a> { } } + /// First matching element rule, applied. `None` when no rule matches (or + /// at the depth limit, where the built-in flattening takes over). + fn apply_rule( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + if self.rules.is_empty() || depth >= self.max_depth { + return None; + } + let action = self + .rules + .iter() + .find(|(sel, _)| sel.matches(&el)) + .map(|(_, a)| a)?; + let inline = |wrap: fn(Vec) -> Node| { + let mut children = self.collect_inline(el, exclude, depth); + trim_edge_text(&mut children); + (!children.is_empty()).then(|| wrap(children)) + }; + let text = || { + collapse_whitespace(&el.text().collect::()) + .trim() + .to_string() + }; + Some(match action { + ElementAction::Skip => RuleHit::Skip, + ElementAction::Unwrap => RuleHit::Unwrap, + ElementAction::Heading { level } => { + let mut children = self.collect_inline(el, exclude, depth); + trim_edge_text(&mut children); + RuleHit::Node((!children.is_empty()).then(|| Node::heading(*level, children))) + } + ElementAction::Paragraph => RuleHit::Node(inline(Node::paragraph)), + ElementAction::Bold => RuleHit::Node(inline(|children| Node::Bold { children })), + ElementAction::Italic => RuleHit::Node(inline(|children| Node::Italic { children })), + ElementAction::BlockQuote { title } => { + let mut nodes = Vec::new(); + self.convert_children(el, exclude, &mut nodes, depth); + let mut children = normalize_blocks(nodes); + if !children.is_empty() && children.iter().all(Node::is_inline) { + trim_edge_text(&mut children); + children = vec![Node::paragraph(children)]; + } + if let Some(t) = title.as_deref().map(str::trim).filter(|t| !t.is_empty()) { + children.insert( + 0, + Node::paragraph(vec![Node::Bold { + children: vec![Node::Text(t.to_string())], + }]), + ); + } + RuleHit::Node((!children.is_empty()).then_some(Node::BlockQuote { + children, + source: None, + })) + } + ElementAction::CodeBlock { language } => { + let code: String = el.text().collect(); + RuleHit::Node((!code.trim().is_empty()).then(|| Node::CodeBlock { + language: language.clone(), + code, + source: None, + })) + } + ElementAction::InlineCode => { + let t = text(); + RuleHit::Node((!t.is_empty()).then_some(Node::InlineCode(t))) + } + ElementAction::Text => { + let t = text(); + RuleHit::Node((!t.is_empty()).then_some(Node::Text(t))) + } + }) + } + /// MathML or a KaTeX / MathJax wrapper around it. fn math_wrapper(&self, el: scraper::ElementRef<'_>) -> Option { let v = el.value(); @@ -794,8 +935,19 @@ impl<'a> HtmlParser<'a> { if self.is_excluded(&child_el, exclude) { continue; } - if let Some(node) = self.convert_element(child_el, exclude, depth + 1) { - nodes.push(node); + match self.apply_rule(child_el, exclude, depth + 1) { + Some(RuleHit::Skip) => {} + Some(RuleHit::Unwrap) => { + nodes.extend(self.collect_inline(child_el, exclude, depth + 1)); + } + Some(RuleHit::Node(node)) => nodes.extend(node), + None => { + if let Some(node) = + self.convert_element(child_el, exclude, depth + 1) + { + nodes.push(node); + } + } } } } diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 696d56a..c44c321 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -66,6 +66,7 @@ mod markdown; #[cfg(feature = "pdf")] mod pdf; pub mod render; +pub mod rules; #[cfg(feature = "spreadsheet")] mod spreadsheet; mod text; diff --git a/crates/markkit/src/rules.rs b/crates/markkit/src/rules.rs new file mode 100644 index 0000000..0ba3c1c --- /dev/null +++ b/crates/markkit/src/rules.rs @@ -0,0 +1,97 @@ +//! Declarative HTML element rules: map elements matching a CSS selector to +//! an IR construct without writing a parser. +//! +//! Decisions: +//! - Rules run before the built-in element mapping; the first matching rule +//! wins. `exclude_selectors` still run first (excluded elements are gone). +//! - Rules are data (serde, `snake_case` `action` tag) so the CLI and +//! bindings can load them from JSON: +//! `{"selector": ".note", "action": "block_quote", "title": "Note"}`. +//! - An invalid selector is an error at parse time ([`crate::Error::HtmlParse`]), +//! not silently ignored: a typo in a rule should not quietly change output. +//! - Actions only build existing IR nodes, so every IR invariant and the +//! depth limit still hold; rule text (`title`, `language`) is plain text. + +use serde::{Deserialize, Serialize}; + +/// What to do with an element that matches a rule. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case")] +#[non_exhaustive] +pub enum ElementAction { + /// Drop the element and its content. + Skip, + /// Ignore the element itself; convert its children in place. + Unwrap, + /// Heading of `level` (clamped to 1-6) from the inline content. + Heading { level: u8 }, + /// Paragraph from the inline content. + Paragraph, + /// Block quote from the block content, with an optional bold title line + /// (callouts: `.note`, `.warning`, `[role=note]`). + BlockQuote { + #[serde(default, skip_serializing_if = "Option::is_none")] + title: Option, + }, + /// Fenced code block from the raw text content. + CodeBlock { + #[serde(default, skip_serializing_if = "Option::is_none")] + language: Option, + }, + /// Inline code from the text content. + InlineCode, + /// Bold inline content. + Bold, + /// Italic inline content. + Italic, + /// Plain text content (markup inside dropped). + Text, +} + +/// A selector plus the action for matching elements. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HtmlRule { + /// CSS selector (same syntax as `include_selector`). + pub selector: String, + #[serde(flatten)] + pub action: ElementAction, +} + +impl HtmlRule { + pub fn new(selector: impl Into, action: ElementAction) -> Self { + Self { + selector: selector.into(), + action, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_shape() { + let rules: Vec = serde_json::from_str( + r#"[{"selector": ".note", "action": "block_quote", "title": "Note"}, + {"selector": ".ad", "action": "skip"}, + {"selector": ".h", "action": "heading", "level": 2}]"#, + ) + .unwrap(); + assert_eq!( + rules[0].action, + ElementAction::BlockQuote { + title: Some("Note".into()) + } + ); + assert_eq!(rules[1].action, ElementAction::Skip); + assert_eq!(rules[2].action, ElementAction::Heading { level: 2 }); + let back = serde_json::to_string(&rules[1]).unwrap(); + assert_eq!(back, r#"{"selector":".ad","action":"skip"}"#); + } + + #[test] + fn unknown_action_rejected() { + assert!(serde_json::from_str::(r#"{"selector":"p","action":"eval"}"#).is_err()); + } +} diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index 483f9d0..f313e51 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -1029,3 +1029,74 @@ fn transforms_apply_to_extract_document_html() { .unwrap(); assert_eq!(e.markdown, "# x\n"); } + +// ─── Element rules ─────────────────────────────────────────────────── + +#[test] +fn element_rules_map_selectors() { + use markkit::rules::ElementAction as A; + let html = r#"

Read this.

+
Buy now
+ Ctrl +
Big title
+
ls -la +pwd
+

Inner

+

Press Enter, wow.

"#; + let md = convert_with(html, |b| { + b.html_rule( + ".note", + A::BlockQuote { + title: Some("Note".into()), + }, + ) + .html_rule(".ad", A::Skip) + .html_rule(".title", A::Heading { level: 2 }) + .html_rule( + ".shell", + A::CodeBlock { + language: Some("sh".into()), + }, + ) + .html_rule(".wrap", A::Unwrap) + .html_rule(".key", A::InlineCode) + .html_rule(".hl", A::Bold) + }); + assert_eq!( + md, + "> **Note**\n>\n> Read this.\n\nCtrl\n\n## Big title\n\n```sh\nls -la\npwd\n```\n\nInner\n\nPress `Enter`, **wow**.\n" + ); +} + +#[test] +fn first_matching_rule_wins_and_overrides_builtin() { + use markkit::rules::ElementAction as A; + let md = convert_with("

Title

x

", |b| { + b.html_rule("h1", A::Paragraph).html_rule("h1", A::Skip) + }); + assert_eq!(md, "Title\n\nx\n"); +} + +#[test] +fn invalid_rule_selector_is_an_error() { + let m = Markkit::builder() + .html_rule("p[", markkit::rules::ElementAction::Skip) + .build(); + let err = m.extract_html_sync("

x

").unwrap_err(); + assert!(matches!(err, markkit::Error::HtmlParse(_)), "{err}"); +} + +#[test] +fn rules_load_from_json() { + let rules: Vec = serde_json::from_str( + r#"[{"selector": ".warn", "action": "block_quote", "title": "Warning"}]"#, + ) + .unwrap(); + let mut b = Markkit::builder(); + b.html_rules = rules; + let md = b + .build() + .extract_html_sync(r#""#) + .unwrap(); + assert_eq!(md, "> **Warning**\n>\n> Hot\n"); +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index a156c3d..8931da9 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -242,3 +242,13 @@ fn source_tracking_many_siblings_is_linear() { let last = doc.nodes.last().unwrap().source().unwrap(); assert_eq!(last.path.as_deref(), Some("/html[1]/body[1]/p[200000]")); } + +#[test] +fn element_rules_respect_depth_limit() { + // Unwrap on every level must not recurse past MAX_DEPTH. + let html = "
".repeat(5_000) + "deep" + &"
".repeat(5_000); + let m = Markkit::builder() + .html_rule("div", markkit::rules::ElementAction::Unwrap) + .build(); + assert!(m.extract_html_sync(&html).unwrap().contains("deep")); +} diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index c0157a9..64f6621 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -177,6 +177,31 @@ providing defense-in-depth. Headings above the article container (page `

`) are usually outside it; the title is in `Extraction::title`. Fixture: `news_page.html` -> `news_page.main.md`. +## Element Rules + +`MarkkitBuilder::html_rule(selector, ElementAction)` (or `html_rules` from +JSON; CLI `--rules file.json`) maps matching elements without code. Rules are +compiled once per parse; an invalid selector fails with `Error::HtmlParse`. +They run after `exclude_selectors` and before every built-in mapping +(including figure, footnote and math handling), in block and inline context; +the first matching rule wins. + +| `action` | Result | +|----------|--------| +| `skip` | Element and content dropped | +| `unwrap` | Element ignored, children converted in place | +| `heading` (`level`) | `Heading` (level clamped 1-6) from inline content | +| `paragraph` | `Paragraph` from inline content | +| `block_quote` (`title?`) | `BlockQuote` of block content, optional `**title**` first line (callouts) | +| `code_block` (`language?`) | `CodeBlock` from raw text | +| `inline_code` | `InlineCode` from collapsed text | +| `bold`, `italic` | Inline emphasis | +| `text` | Collapsed text only | + +Empty results are elided like built-ins. At `MAX_DEPTH` rules are ignored and +the subtree is flattened as usual. Rule files are embedder configuration, +trusted like code; untrusted HTML cannot add rules. + ## Page Metadata `extract_document` on HTML fills `Extraction::title` and `Extraction::metadata` diff --git a/knowledge/log.md b/knowledge/log.md index 42c26dc..3f80aa1 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -9,6 +9,7 @@ ## 2026-10-26 +* **Element rules**: declarative `HtmlRule { selector, action }` (`MarkkitBuilder::html_rule`, CLI `--rules`) tried before built-in HTML mapping; invalid selectors error. See [HTML Extraction](formats/html-extraction.md). * **Output safety**: the renderer escaped nothing, so document text could emit raw HTML, links, headings and fence breakouts, and `javascript:` links passed through. Escaping and a URL scheme allowlist now live in `render/escape.rs`; DOCX and spreadsheet parsers stop pre-escaping table pipes, so the IR holds raw text. New invariant S17 with a pulldown-cmark oracle. A first fuzzing pass against that oracle found eight more container breakouts (list indentation by marker width, blocks in inline context, adjacent code spans, nested links, ...), all fixed. Math now renders as code (`` $`tex`$ ``, ` ```math `) because TeX from page markup reached plain-GFM output unescaped. See [Threat Model](security/threat-model.md) T1. * **Main content**: opt-in Readability-style detection (`main_content`, CLI `--main-content`) keeps the article body and drops sidebars, comments, share bars; falls back to the full body when nothing scores. Algorithm in [HTML Extraction](formats/html-extraction.md); bounds in [Threat Model](security/threat-model.md) T2g. * **Relative URLs**: links, images and metadata URLs resolve against `base_url` (or the final `extract_url` response URL) plus ``. Only scheme-less references change; page bases are limited to `http(s)`. New `extract_url_document`. See [HTML Extraction](formats/html-extraction.md) and [Threat Model](security/threat-model.md) T1. From c83a1326334586a94c22e99e9facf9364dd58899 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 00:48:09 +0000 Subject: [PATCH 2/4] test(html): element rules cannot resurrect stripped elements --- crates/markkit/tests/html_security.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index 8931da9..e22194b 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -252,3 +252,18 @@ fn element_rules_respect_depth_limit() { .build(); assert!(m.extract_html_sync(&html).unwrap().contains("deep")); } + +#[test] +fn element_rules_cannot_resurrect_stripped_elements() { + // Stripping runs before rules: a rule matching "; + let m = Markkit::builder() + .html_rule("script", markkit::rules::ElementAction::Paragraph) + .html_rule( + "style", + markkit::rules::ElementAction::CodeBlock { language: None }, + ) + .build(); + let md = m.extract_html_sync(html).unwrap(); + assert_eq!(md, "ok\n"); +} From 32fb460aeb74d2086ceea25fc4557e18fb26795f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 01:20:08 +0000 Subject: [PATCH 3/4] chore(agents): breaking changes approved pre-1.0 --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 713eefe..937d95f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn - Small, incremental PR-sized changes - Write failing test before fixing bug - All input is untrusted: parsing changes need security tests +- Breaking changes approved (pre-1.0): change public APIs, IR, CLI and bindings freely when it improves the design; no deprecation shims. Mark `**Breaking:**` in CHANGELOG and update knowledge ### Knowledge From c5dd25b1c633ec033882931e44fc35700fcdba63 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 01:44:46 +0000 Subject: [PATCH 4/4] chore(knowledge): rules follow limits nesting depth --- crates/markkit/tests/html_security.rs | 2 +- knowledge/formats/html-extraction.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index e22194b..c2b50fc 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -245,7 +245,7 @@ fn source_tracking_many_siblings_is_linear() { #[test] fn element_rules_respect_depth_limit() { - // Unwrap on every level must not recurse past MAX_DEPTH. + // Unwrap on every level must not recurse past the nesting limit. let html = "
".repeat(5_000) + "deep" + &"
".repeat(5_000); let m = Markkit::builder() .html_rule("div", markkit::rules::ElementAction::Unwrap) diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index 64f6621..8e60802 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -198,7 +198,7 @@ the first matching rule wins. | `bold`, `italic` | Inline emphasis | | `text` | Collapsed text only | -Empty results are elided like built-ins. At `MAX_DEPTH` rules are ignored and +Empty results are elided like built-ins. At the nesting limit (`limits.max_nesting_depth`) rules are ignored and the subtree is flattened as usual. Rule files are embedder configuration, trusted like code; untrusted HTML cannot add rules.