feat(html): Readability-style main-content detection

CHANGELOG.md

@@ -44,6 +44,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
   `extractDocument` / `extractDocumentSync` return Markdown with title, format, warnings, stats and
   page metadata.
 - `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats.
+- Main-content detection: `MarkkitBuilder::main_content(true)` (CLI `--main-content`) keeps only
+  the article body of a web page, Readability-style, dropping navigation, sidebars, comments, share
+  bars and related-link blocks. Off by default; include selectors take precedence; falls back to the
+  full page when no article-like block is found.
 
 ### Security
 

README.md

@@ -80,6 +80,9 @@ markkit extract https://example.com --include "article" --exclude "nav,footer"
 # JSON output with format, title and page metadata
 markkit extract https://example.com --json
 
+# Keep only the article body (drops sidebars, comments, share bars)
+markkit extract https://example.com/post --main-content
+
 # Resolve relative links in a saved page
 markkit extract --file page.html --base-url https://example.com/blog/
 

crates/markkit-cli/src/main.rs

@@ -50,6 +50,10 @@ enum Commands {
         /// (URLs use the final response URL)
         #[arg(long)]
         base_url: Option<String>,
+
+        /// Keep only the detected main content (article body) of HTML pages
+        #[arg(long)]
+        main_content: bool,
     },
 }
 
@@ -67,14 +71,17 @@ async fn main() -> Result<()> {
             max_size,
             json: output_json,
             base_url,
+            main_content,
         } => {
             let read_stdin = stdin || source.as_deref() == Some("-");
 
             if !read_stdin && source.is_none() {
                 anyhow::bail!("source is required unless --stdin or \"-\" is used");
             }
 
-            let mut builder = Markkit::builder().max_input_size(max_size);
+            let mut builder = Markkit::builder()
+                .max_input_size(max_size)
+                .main_content(main_content);
 
             for sel in &include_selectors {
                 builder = builder.include_selector(sel);

crates/markkit/src/config.rs

@@ -50,6 +50,10 @@ pub struct MarkkitBuilder {
     pub track_sources: bool,
     /// IR passes run after parsing, before rendering (see [`crate::transform`]).
     pub transforms: crate::transform::TransformChain,
+    /// Keep only the detected main content (article body) of HTML pages,
+    /// Readability-style. Include selectors win; falls back to the full body when
+    /// nothing scores. Default: false.
+    pub main_content: bool,
 }
 
 impl Default for MarkkitBuilder {
@@ -82,6 +86,7 @@ impl Default for MarkkitBuilder {
             base_url: None,
             track_sources: false,
             transforms: Default::default(),
+            main_content: false,
         }
     }
 }
@@ -189,6 +194,13 @@ impl MarkkitBuilder {
         self
     }
 
+    /// Extract only the detected main content of HTML pages (drops sidebars,
+    /// comments, share bars and other boilerplate).
+    pub fn main_content(mut self, yes: bool) -> Self {
+        self.main_content = yes;
+        self
+    }
+
     pub fn build(self) -> Markkit {
         Markkit::new(self)
     }

crates/markkit/src/html/mod.rs

@@ -4,6 +4,7 @@
 
 mod metadata;
 mod parser;
+mod readability;
 mod urls;
 
 pub use parser::HtmlParser;

crates/markkit/src/html/parser.rs

@@ -19,7 +19,7 @@
 //! - `<figure>` becomes `Figure` only when it has a non-empty `<figcaption>`;
 //!   otherwise it stays a transparent container as before.
 
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
 use std::collections::HashMap;
 
 use ego_tree::{NodeId, NodeRef};
@@ -32,7 +32,7 @@ use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, Tab
 use url::Url;
 
 use super::metadata::PageInfo;
-use super::urls;
+use super::{readability, urls};
 
 /// Maximum recursion depth for nested elements.
 const MAX_DEPTH: usize = 200;
@@ -49,6 +49,8 @@ pub struct HtmlParser<'a> {
     base: Option<Url>,
     /// Cached 1-based index of an element among same-name element siblings.
     sibling_index: RefCell<HashMap<NodeId, usize>>,
+    /// Main-content mode: also drop "unlikely" subtrees (sidebars, share bars, ...).
+    drop_unlikely: Cell<bool>,
 }
 
 impl<'a> HtmlParser<'a> {
@@ -58,6 +60,7 @@ impl<'a> HtmlParser<'a> {
             footnote_labels: RefCell::new(HashMap::new()),
             base: urls::parse_base(config.base_url.as_deref()),
             sibling_index: RefCell::new(HashMap::new()),
+            drop_unlikely: Cell::new(false),
         }
     }
 
@@ -87,27 +90,13 @@ impl<'a> HtmlParser<'a> {
             footnote_labels: RefCell::new(HashMap::new()),
             sibling_index: RefCell::new(HashMap::new()),
             base: urls::document_base(dom, self.base.clone()),
+            drop_unlikely: Cell::new(false),
         }
     }
 
     fn convert_dom(&self, dom: &Html) -> Result<Document, Error> {
         let root = dom.root_element();
 
-        let content_nodes: Vec<scraper::ElementRef<'_>> =
-            if !self.config.include_selectors.is_empty() {
-                let sels: Vec<Selector> = self
-                    .config
-                    .include_selectors
-                    .iter()
-                    .filter_map(|s| Selector::parse(s).ok())
-                    .collect();
-                sels.iter().flat_map(|sel| dom.select(sel)).collect()
-            } else {
-                let body_sel = Selector::parse("body").unwrap();
-                let body: Vec<_> = dom.select(&body_sel).collect();
-                if body.is_empty() { vec![root] } else { body }
-            };
-
         let exclude_selectors: Vec<Selector> = self
             .config
             .exclude_selectors
@@ -116,10 +105,40 @@ impl<'a> HtmlParser<'a> {
             .collect();
 
         let mut nodes = Vec::new();
-        for content_root in content_nodes {
-            self.convert_children(content_root, &exclude_selectors, &mut nodes, 0);
+        if !self.config.include_selectors.is_empty() {
+            let sels: Vec<Selector> = self
+                .config
+                .include_selectors
+                .iter()
+                .filter_map(|s| Selector::parse(s).ok())
+                .collect();
+            for content_root in sels.iter().flat_map(|sel| dom.select(sel)) {
+                self.convert_children(content_root, &exclude_selectors, &mut nodes, 0);
+            }
+            return Ok(Document::new(normalize_blocks(nodes)));
+        }
+
+        let body_sel = Selector::parse("body").unwrap();
+        let body = dom.select(&body_sel).next().unwrap_or(root);
+
+        if self.config.main_content
+            && let Some(roots) = readability::main_content(body, &exclude_selectors)
+        {
+            self.drop_unlikely.set(true);
+            for r in roots {
+                if r.whole && !is_transparent_container(r.el.value().name()) {
+                    if let Some(mut node) = self.convert_element(r.el, &exclude_selectors, 1) {
+                        self.mark_source(&mut node, r.el);
+                        nodes.push(node);
+                    }
+                } else {
+                    self.convert_children(r.el, &exclude_selectors, &mut nodes, 0);
+                }
+            }
+            return Ok(Document::new(normalize_blocks(nodes)));
         }
 
+        self.convert_children(body, &exclude_selectors, &mut nodes, 0);
         Ok(Document::new(normalize_blocks(nodes)))
     }
 
@@ -170,6 +189,7 @@ impl<'a> HtmlParser<'a> {
 
     fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool {
         exclude.iter().any(|sel| sel.matches(el))
+            || (self.drop_unlikely.get() && readability::is_unlikely(*el))
     }
 
     fn convert_children(