feat(html): declarative element rules

AGENTS.md

@@ -16,6 +16,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn
 - Small, incremental PR-sized changes
 - Write failing test before fixing bug
 - All input is untrusted: parsing changes need security tests
+- Breaking changes approved (pre-1.0): change public APIs, IR, CLI and bindings freely when it improves the design; no deprecation shims. Mark `**Breaking:**` in CHANGELOG and update knowledge
 
 ### Knowledge
 

CHANGELOG.md

@@ -44,6 +44,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - IR transforms: `MarkkitBuilder::transform(..)` runs `Transform` passes after parsing and before
   rendering, for every format. Built-ins `transform::NormalizeHeadings` and `transform::DropImages`;
   `transform::from_fn` for closures; `Document::visit` / `visit_mut` to walk the tree.
+- Declarative HTML element rules: `MarkkitBuilder::html_rule(".note", ElementAction::BlockQuote { .. })`
+  maps elements by CSS selector to skip / unwrap / heading / paragraph / block quote (with title) /
+  code / emphasis / text, before the built-in mapping. CLI: `markkit extract --rules rules.json`.
 - JSON Schema for the IR at `schema/ir.schema.json`; `json-schema` feature adds `markkit::ir::json_schema()`.
 - Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block
   (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations.

README.md

@@ -92,6 +92,10 @@ markkit extract --file page.html --base-url https://example.com/blog/
 # From stdin
 cat page.html | markkit extract -
 
+# Map site-specific markup with element rules (JSON file)
+# [{"selector": ".note", "action": "block_quote", "title": "Note"}, {"selector": ".ad", "action": "skip"}]
+markkit extract --file page.html --rules rules.json
+
 # MCP server (stdio) exposing the `markkit` tool to agents
 markkit mcp            # add --no-url to keep it offline
 ```

crates/markkit-cli/src/main.rs

@@ -42,6 +42,11 @@ enum Commands {
         #[arg(long = "exclude", short = 'e')]
         exclude_selectors: Vec<String>,
 
+        /// JSON file with HTML element rules, e.g.
+        /// `[{"selector": ".note", "action": "block_quote", "title": "Note"}]`
+        #[arg(long)]
+        rules: Option<std::path::PathBuf>,
+
         /// Output as JSON with metadata
         #[arg(long)]
         json: bool,
@@ -193,6 +198,7 @@ async fn main() -> Result<()> {
             include_selectors,
             exclude_selectors,
             limits,
+            rules,
             json: output_json,
             base_url,
             main_content,
@@ -218,6 +224,12 @@ async fn main() -> Result<()> {
                 builder = builder.base_url(base);
             }
 
+            if let Some(path) = &rules {
+                let text = std::fs::read_to_string(path)?;
+                builder.html_rules = serde_json::from_str(&text)
+                    .map_err(|e| anyhow::anyhow!("invalid rules file {}: {e}", path.display()))?;
+            }
+
             let markkit = builder.build();
 
             let source_label = source.clone().unwrap_or_else(|| "<stdin>".to_string());

crates/markkit-cli/tests/cli_integration.rs

@@ -251,3 +251,49 @@ fn limit_flags_enforced() {
     let stderr = String::from_utf8_lossy(&output.stderr);
     assert!(stderr.contains("max_output_size"), "stderr: {stderr}");
 }
+
+#[test]
+fn extract_with_rules_file() {
+    let dir = tempfile::tempdir().unwrap();
+    let page = dir.path().join("page.html");
+    std::fs::write(
+        &page,
+        r#"<div class="note">Careful</div><div class="ad">Buy</div>"#,
+    )
+    .unwrap();
+    let rules = dir.path().join("rules.json");
+    std::fs::write(
+        &rules,
+        r#"[{"selector": ".note", "action": "block_quote", "title": "Note"},
+            {"selector": ".ad", "action": "skip"}]"#,
+    )
+    .unwrap();
+
+    let output = markkit_bin()
+        .args(["extract", "--file", page.to_str().unwrap()])
+        .args(["--rules", rules.to_str().unwrap()])
+        .output()
+        .expect("failed to run markkit");
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    assert_eq!(
+        String::from_utf8_lossy(&output.stdout).trim_end(),
+        "> **Note**\n>\n> Careful"
+    );
+}
+
+#[test]
+fn extract_with_bad_rules_file_fails() {
+    let dir = tempfile::tempdir().unwrap();
+    let rules = dir.path().join("rules.json");
+    std::fs::write(&rules, r#"[{"selector": "p", "action": "exec"}]"#).unwrap();
+    let output = markkit_bin()
+        .args(["extract", "--stdin", "--rules", rules.to_str().unwrap()])
+        .output()
+        .expect("failed to run markkit");
+    assert!(!output.status.success());
+    assert!(String::from_utf8_lossy(&output.stderr).contains("invalid rules file"));
+}

crates/markkit/src/config.rs

@@ -41,6 +41,9 @@ pub struct MarkkitBuilder {
     /// Readability-style. Include selectors win; falls back to the full body when
     /// nothing scores. Default: false.
     pub main_content: bool,
+    /// Declarative HTML element rules, tried in order before the built-in
+    /// mapping (see [`crate::rules`]).
+    pub html_rules: Vec<crate::rules::HtmlRule>,
 }
 
 impl Default for MarkkitBuilder {
@@ -66,6 +69,7 @@ impl Default for MarkkitBuilder {
             track_sources: false,
             transforms: Default::default(),
             main_content: false,
+            html_rules: Vec::new(),
         }
     }
 }
@@ -132,6 +136,17 @@ impl MarkkitBuilder {
         self
     }
 
+    /// Add an HTML element rule: elements matching `selector` become `action`.
+    pub fn html_rule(
+        mut self,
+        selector: impl Into<String>,
+        action: crate::rules::ElementAction,
+    ) -> Self {
+        self.html_rules
+            .push(crate::rules::HtmlRule::new(selector, action));
+        self
+    }
+
     pub fn include_selector(mut self, selector: impl Into<String>) -> Self {
         self.include_selectors.push(selector.into());
         self

crates/markkit/src/html/parser.rs

@@ -16,6 +16,9 @@
 //!   and MathJax `<mjx-container>`) maps to `Math` / `MathBlock` using the
 //!   `application/x-tex` annotation; without one, the text content is kept as
 //!   plain text. KaTeX's visual HTML duplicate is dropped.
+//! - Element rules (`MarkkitBuilder::html_rule`, see `crate::rules`) are
+//!   compiled once per parser and tried before the built-in mapping in both
+//!   block and inline context; invalid selectors fail the parse.
 //! - `<figure>` becomes `Figure` only when it has a non-empty `<figcaption>`;
 //!   otherwise it stays a transparent container as before.
 
@@ -29,12 +32,20 @@ use crate::config::MarkkitBuilder;
 use crate::error::Error;
 use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, TableCell};
 use crate::limits::Budget;
+use crate::rules::ElementAction;
 
 use url::Url;
 
 use super::metadata::PageInfo;
 use super::{readability, urls};
 
+/// Outcome of a matching element rule.
+enum RuleHit {
+    Skip,
+    Unwrap,
+    Node(Option<Node>),
+}
+
 /// HTML-spec clamps for table spans.
 const MAX_COLSPAN: u32 = 1000;
 const MAX_ROWSPAN: u32 = 65534;
@@ -60,11 +71,32 @@ pub struct HtmlParser<'a> {
     sibling_index: RefCell<HashMap<NodeId, usize>>,
     /// Main-content mode: also drop "unlikely" subtrees (sidebars, share bars, ...).
     drop_unlikely: Cell<bool>,
+    /// Compiled element rules, in priority order.
+    rules: Vec<(Selector, ElementAction)>,
+    /// First invalid rule selector, reported by `parse`.
+    rule_error: Option<String>,
+}
+
+/// Compile rule selectors; keep the first invalid one for `parse` to report.
+fn compile_rules(config: &MarkkitBuilder) -> (Vec<(Selector, ElementAction)>, Option<String>) {
+    let mut rules = Vec::with_capacity(config.html_rules.len());
+    let mut rule_error = None;
+    for r in &config.html_rules {
+        match Selector::parse(&r.selector) {
+            Ok(sel) => rules.push((sel, r.action.clone())),
+            Err(e) => {
+                rule_error
+                    .get_or_insert_with(|| format!("invalid rule selector {:?}: {e}", r.selector));
+            }
+        }
+    }
+    (rules, rule_error)
 }
 
 impl<'a> HtmlParser<'a> {
     /// Parser with a fresh per-call budget (deadline starts now).
     pub fn new(config: &'a MarkkitBuilder) -> Self {
+        let (rules, rule_error) = compile_rules(config);
         Self {
             config,
             budget: BudgetRef::Owned(Budget::new(&config.limits)),
@@ -73,10 +105,13 @@ impl<'a> HtmlParser<'a> {
             base: urls::parse_base(config.base_url.as_deref()),
             sibling_index: RefCell::new(HashMap::new()),
             drop_unlikely: Cell::new(false),
+            rules,
+            rule_error,
         }
     }
 
     pub(crate) fn with_budget(config: &'a MarkkitBuilder, budget: &'a Budget) -> Self {
+        let (rules, rule_error) = compile_rules(config);
         Self {
             config,
             budget: BudgetRef::Borrowed(budget),
@@ -85,6 +120,8 @@ impl<'a> HtmlParser<'a> {
             base: urls::parse_base(config.base_url.as_deref()),
             sibling_index: RefCell::new(HashMap::new()),
             drop_unlikely: Cell::new(false),
+            rules,
+            rule_error,
         }
     }
 
@@ -111,6 +148,9 @@ impl<'a> HtmlParser<'a> {
     }
 
     fn parse_dom(&self, dom: &Html) -> Result<Document, Error> {
+        if let Some(e) = &self.rule_error {
+            return Err(Error::HtmlParse(e.clone()));
+        }
         self.for_document(dom).convert_dom(dom)
     }
 
@@ -125,6 +165,8 @@ impl<'a> HtmlParser<'a> {
             sibling_index: RefCell::new(HashMap::new()),
             base: urls::document_base(dom, self.base.clone()),
             drop_unlikely: Cell::new(false),
+            rules: self.rules.clone(),
+            rule_error: None,
         }
     }
 
@@ -267,6 +309,21 @@ impl<'a> HtmlParser<'a> {
                         if self.is_excluded(&el, exclude) {
                             continue;
                         }
+                        match self.apply_rule(el, exclude, depth + 1) {
+                            Some(RuleHit::Skip) => continue,
+                            Some(RuleHit::Unwrap) => {
+                                self.convert_children(el, exclude, out, depth + 1);
+                                continue;
+                            }
+                            Some(RuleHit::Node(node)) => {
+                                if let Some(mut node) = node {
+                                    self.mark_source(&mut node, el);
+                                    out.push(node);
+                                }
+                                continue;
+                            }
+                            None => {}
+                        }
                         let name = el.value().name();
                         // Captions are consumed by the enclosing figure
                         if name == "figcaption" && parent.value().name() == "figure" {
@@ -321,6 +378,13 @@ impl<'a> HtmlParser<'a> {
             return Some(Node::Text(trimmed.to_string()));
         }
 
+        match self.apply_rule(el, exclude, depth) {
+            Some(RuleHit::Skip) => return None,
+            Some(RuleHit::Unwrap) => return self.convert_container(el, exclude, depth),
+            Some(RuleHit::Node(node)) => return node,
+            None => {}
+        }
+
         if let Some(m) = self.math_wrapper(el) {
             return Some(m);
         }
@@ -526,6 +590,83 @@ impl<'a> HtmlParser<'a> {
         }
     }
 
+    /// First matching element rule, applied. `None` when no rule matches (or
+    /// at the depth limit, where the built-in flattening takes over).
+    fn apply_rule(
+        &self,
+        el: scraper::ElementRef<'_>,
+        exclude: &[Selector],
+        depth: usize,
+    ) -> Option<RuleHit> {
+        if self.rules.is_empty() || depth >= self.max_depth {
+            return None;
+        }
+        let action = self
+            .rules
+            .iter()
+            .find(|(sel, _)| sel.matches(&el))
+            .map(|(_, a)| a)?;
+        let inline = |wrap: fn(Vec<Node>) -> Node| {
+            let mut children = self.collect_inline(el, exclude, depth);
+            trim_edge_text(&mut children);
+            (!children.is_empty()).then(|| wrap(children))
+        };
+        let text = || {
+            collapse_whitespace(&el.text().collect::<String>())
+                .trim()
+                .to_string()
+        };
+        Some(match action {
+            ElementAction::Skip => RuleHit::Skip,
+            ElementAction::Unwrap => RuleHit::Unwrap,
+            ElementAction::Heading { level } => {
+                let mut children = self.collect_inline(el, exclude, depth);
+                trim_edge_text(&mut children);
+                RuleHit::Node((!children.is_empty()).then(|| Node::heading(*level, children)))
+            }
+            ElementAction::Paragraph => RuleHit::Node(inline(Node::paragraph)),
+            ElementAction::Bold => RuleHit::Node(inline(|children| Node::Bold { children })),
+            ElementAction::Italic => RuleHit::Node(inline(|children| Node::Italic { children })),
+            ElementAction::BlockQuote { title } => {
+                let mut nodes = Vec::new();
+                self.convert_children(el, exclude, &mut nodes, depth);
+                let mut children = normalize_blocks(nodes);
+                if !children.is_empty() && children.iter().all(Node::is_inline) {
+                    trim_edge_text(&mut children);
+                    children = vec![Node::paragraph(children)];
+                }
+                if let Some(t) = title.as_deref().map(str::trim).filter(|t| !t.is_empty()) {
+                    children.insert(
+                        0,
+                        Node::paragraph(vec![Node::Bold {
+                            children: vec![Node::Text(t.to_string())],
+                        }]),
+                    );
+                }
+                RuleHit::Node((!children.is_empty()).then_some(Node::BlockQuote {
+                    children,
+                    source: None,
+                }))
+            }
+            ElementAction::CodeBlock { language } => {
+                let code: String = el.text().collect();
+                RuleHit::Node((!code.trim().is_empty()).then(|| Node::CodeBlock {
+                    language: language.clone(),
+                    code,
+                    source: None,
+                }))
+            }
+            ElementAction::InlineCode => {
+                let t = text();
+                RuleHit::Node((!t.is_empty()).then_some(Node::InlineCode(t)))
+            }
+            ElementAction::Text => {
+                let t = text();
+                RuleHit::Node((!t.is_empty()).then_some(Node::Text(t)))
+            }
+        })
+    }
+
     /// MathML or a KaTeX / MathJax wrapper around it.
     fn math_wrapper(&self, el: scraper::ElementRef<'_>) -> Option<Node> {
         let v = el.value();
@@ -794,8 +935,19 @@ impl<'a> HtmlParser<'a> {
                         if self.is_excluded(&child_el, exclude) {
                             continue;
                         }
-                        if let Some(node) = self.convert_element(child_el, exclude, depth + 1) {
-                            nodes.push(node);
+                        match self.apply_rule(child_el, exclude, depth + 1) {
+                            Some(RuleHit::Skip) => {}
+                            Some(RuleHit::Unwrap) => {
+                                nodes.extend(self.collect_inline(child_el, exclude, depth + 1));
+                            }
+                            Some(RuleHit::Node(node)) => nodes.extend(node),
+                            None => {
+                                if let Some(node) =
+                                    self.convert_element(child_el, exclude, depth + 1)
+                                {
+                                    nodes.push(node);
+                                }
+                            }
                         }
                     }
                 }