feat(html): resolve relative URLs against a base URL

CHANGELOG.md

@@ -12,13 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or
   `<title>`) and the new `Extraction::metadata` (description, author, published date, language,
   canonical URL, site name, lead image) from `<meta>`, Open Graph, `<link rel=canonical>` and
-  JSON-LD. `markkit extract --json` includes `format`, `title` and `metadata` for files and stdin.
+  JSON-LD. `markkit extract --json` includes `format`, `title` and `metadata`.
 - IR v2: footnotes (`[^1]`), definition lists (`Term` / `:   definition`), task lists (`- [x]`),
   figures with captions, table column alignment (`:---:`) and `colspan`/`rowspan` (laid out on a
   grid; GFM has no spans). HTML maps GitHub and Pandoc footnote markup, `<dl>`, checkbox list items,
   `<figure>`/`<figcaption>` and table spans/alignment. Row-header `<th>` cells in table bodies are
   no longer dropped.
 - `Document.ir_version` (`markkit::ir::IR_VERSION = 2`) is serialized with the IR JSON.
+- Relative URL resolution: links, images and metadata URLs resolve against the new
+  `MarkkitBuilder::base_url` (CLI `--base-url`) and the page's `<base href>`; `extract_url` uses
+  the final URL after redirects. Absolute and fragment-only URLs are left unchanged.
 - Math: `$tex$` and `$$` blocks from MathML (`application/x-tex` annotation), KaTeX and MathJax output.
 - IR transforms: `MarkkitBuilder::transform(..)` runs `Transform` passes after parsing and before
   rendering, for every format. Built-ins `transform::NormalizeHeadings` and `transform::DropImages`;

Cargo.toml

@@ -41,6 +41,7 @@ clap = { version = "4", features = ["derive"] }
 reqwest = { version = "0.13", default-features = false, features = ["rustls", "json"] }
 
 # HTML parsing
+url = "2"
 scraper = "0.29"
 ego-tree = "0.11"
 schemars = "1"

README.md

@@ -77,8 +77,11 @@ markkit extract --file document.html
 # With CSS selector filtering
 markkit extract https://example.com --include "article" --exclude "nav,footer"
 
-# JSON output; files and stdin also carry format, title and page metadata
-markkit extract --file page.html --json
+# JSON output with format, title and page metadata
+markkit extract https://example.com --json
+
+# Resolve relative links in a saved page
+markkit extract --file page.html --base-url https://example.com/blog/
 
 # From stdin
 cat page.html | markkit extract -

crates/markkit-cli/src/main.rs

@@ -45,6 +45,11 @@ enum Commands {
         /// Output as JSON with metadata
         #[arg(long)]
         json: bool,
+
+        /// Resolve relative links and images in files / stdin against this URL
+        /// (URLs use the final response URL)
+        #[arg(long)]
+        base_url: Option<String>,
     },
 }
 
@@ -61,6 +66,7 @@ async fn main() -> Result<()> {
             exclude_selectors,
             max_size,
             json: output_json,
+            base_url,
         } => {
             let read_stdin = stdin || source.as_deref() == Some("-");
 
@@ -77,43 +83,41 @@ async fn main() -> Result<()> {
                 builder = builder.exclude_selector(sel);
             }
 
+            if let Some(base) = base_url {
+                builder = builder.base_url(base);
+            }
+
             let markkit = builder.build();
 
             let source_label = source.clone().unwrap_or_else(|| "<stdin>".to_string());
 
             let extraction = if read_stdin {
                 let mut buf = Vec::new();
                 std::io::stdin().read_to_end(&mut buf)?;
-                Some(markkit.extract_document_sync(
-                    &buf,
-                    markkit::DocumentHint::content_type("text/html"),
-                )?)
+                markkit
+                    .extract_document_sync(&buf, markkit::DocumentHint::content_type("text/html"))?
             } else if file {
                 let src = source.as_deref().unwrap();
                 let bytes = tokio::fs::read(src).await?;
                 // Format detected from the extension, then magic bytes.
-                Some(markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))?)
+                markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))?
             } else {
-                None
-            };
-            let markdown = match &extraction {
-                Some(e) => e.markdown.clone(),
-                None => markkit.extract_url(source.as_deref().unwrap()).await?,
+                markkit
+                    .extract_url_document(source.as_deref().unwrap())
+                    .await?
             };
 
             if output_json {
-                let mut output = serde_json::json!({
+                let output = serde_json::json!({
                     "source": source_label,
-                    "markdown": markdown,
+                    "markdown": extraction.markdown,
+                    "format": extraction.format,
+                    "title": extraction.title,
+                    "metadata": extraction.metadata,
                 });
-                if let Some(e) = &extraction {
-                    output["format"] = serde_json::to_value(e.format)?;
-                    output["title"] = serde_json::to_value(&e.title)?;
-                    output["metadata"] = serde_json::to_value(&e.metadata)?;
-                }
                 println!("{}", serde_json::to_string_pretty(&output)?);
             } else {
-                print!("{markdown}");
+                print!("{}", extraction.markdown);
             }
         }
     }

crates/markkit/Cargo.toml

@@ -15,7 +15,7 @@ readme = "../../README.md"
 
 [features]
 default = ["html", "http_client"]
-html = ["dep:scraper", "dep:html5ever", "dep:ego-tree"]
+html = ["dep:scraper", "dep:html5ever", "dep:ego-tree", "dep:url"]
 pdf = []
 docx = ["dep:zip", "dep:quick-xml"]
 spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"]
@@ -43,6 +43,7 @@ schemars = { workspace = true, optional = true }
 scraper = { workspace = true, optional = true }
 ego-tree = { workspace = true, optional = true }
 html5ever = { workspace = true, optional = true }
+url = { workspace = true, optional = true }
 
 # Office documents (optional): DOCX + spreadsheets share the ZIP/XML stack
 zip = { workspace = true, optional = true }

crates/markkit/src/config.rs

@@ -40,6 +40,10 @@ pub struct MarkkitBuilder {
     pub allowed_hosts: Vec<String>,
     /// Maximum redirects `extract_url` follows. Default: 5.
     pub max_redirects: usize,
+    /// Absolute URL that relative HTML links, images and metadata URLs resolve
+    /// against (a document `<base href>` applies on top). `extract_url` uses the
+    /// final response URL instead. Default: none (relative URLs kept as is).
+    pub base_url: Option<String>,
     /// Record per-block provenance ([`crate::ir::Source`]) in the IR:
     /// HTML element XPath, DOCX block index, spreadsheet sheet and range.
     /// Default: false (no cost, no `source` in IR JSON).
@@ -75,6 +79,7 @@ impl Default for MarkkitBuilder {
             allow_private_networks: false,
             allowed_hosts: Vec::new(),
             max_redirects: 5,
+            base_url: None,
             track_sources: false,
             transforms: Default::default(),
         }
@@ -177,6 +182,13 @@ impl MarkkitBuilder {
         self
     }
 
+    /// Resolve relative HTML URLs against this absolute URL. Ignored when it
+    /// does not parse or cannot be a base (`data:`, `javascript:`, ...).
+    pub fn base_url(mut self, url: impl Into<String>) -> Self {
+        self.base_url = Some(url.into());
+        self
+    }
+
     pub fn build(self) -> Markkit {
         Markkit::new(self)
     }

crates/markkit/src/extract.rs

@@ -350,6 +350,8 @@ impl Markkit {
     /// [`MarkkitBuilder::allow_host`] restricts hosts; redirects capped by
     /// `max_redirects`; body capped by `max_input_size`; whole fetch capped
     /// by `max_processing_secs` ([`Error::ResourceLimit`]).
+    ///
+    /// Relative links and images resolve against the final URL after redirects.
     #[cfg(feature = "http_client")]
     pub async fn extract_url(&self, url: &str) -> crate::Result<String> {
         self.extract_url_document(url).await.map(|e| e.markdown)
@@ -358,10 +360,13 @@ impl Markkit {
     /// [`Markkit::extract_url`] with metadata (format, title, warnings, stats).
     ///
     /// The response `Content-Type` is the format hint; same URL policy and limits.
+    /// Relative HTML URLs resolve against the final URL after redirects.
     #[cfg(feature = "http_client")]
     pub async fn extract_url_document(&self, url: &str) -> crate::Result<Extraction> {
         let fetched = crate::fetch::fetch(url, &self.config).await?;
-        self.extract_document_sync(
+        let mut config = self.config.clone();
+        config.base_url = Some(fetched.url);
+        Markkit::new(config).extract_document_sync(
             &fetched.bytes,
             DocumentHint {
                 content_type: fetched.content_type.as_deref(),

crates/markkit/src/fetch.rs

@@ -31,6 +31,8 @@ use crate::error::Error;
 pub(crate) struct Fetched {
     pub bytes: Vec<u8>,
     pub content_type: Option<String>,
+    /// Final URL after redirects.
+    pub url: String,
 }
 
 /// Per-call slot for the first block reason seen inside reqwest.
@@ -63,6 +65,7 @@ pub(crate) async fn fetch(url: &str, cfg: &MarkkitBuilder) -> crate::Result<Fetc
             .get(reqwest::header::CONTENT_TYPE)
             .and_then(|v| v.to_str().ok())
             .map(str::to_string);
+        let final_url = response.url().to_string();
         let mut bytes = Vec::new();
         while let Some(chunk) = response.chunk().await? {
             if bytes.len() + chunk.len() > limit {
@@ -75,6 +78,7 @@ pub(crate) async fn fetch(url: &str, cfg: &MarkkitBuilder) -> crate::Result<Fetc
         Ok(Ok(Fetched {
             bytes,
             content_type,
+            url: final_url,
         }))
     };
 

crates/markkit/src/html/metadata.rs

@@ -12,6 +12,7 @@
 
 use scraper::{ElementRef, Html, Selector};
 use serde_json::Value;
+use url::Url;
 
 use crate::extract::Metadata;
 
@@ -32,7 +33,7 @@ pub(crate) struct PageInfo {
 }
 
 /// Read title and metadata from a parsed document.
-pub(crate) fn extract(dom: &Html) -> PageInfo {
+pub(crate) fn extract(dom: &Html, base: Option<&Url>) -> PageInfo {
     let meta = MetaTags::collect(dom);
     let ld = JsonLd::collect(dom);
 
@@ -86,6 +87,11 @@ pub(crate) fn extract(dom: &Html) -> PageInfo {
             .or_else(|| ld.image()),
     };
 
+    let mut metadata = metadata;
+    let resolve = |u: Option<String>| u.map(|u| super::urls::resolve(base, &u));
+    metadata.canonical_url = resolve(metadata.canonical_url);
+    metadata.image = resolve(metadata.image);
+
     PageInfo { title, metadata }
 }
 
@@ -355,7 +361,7 @@ mod tests {
     use super::*;
 
     fn info(html: &str) -> PageInfo {
-        extract(&Html::parse_document(html))
+        extract(&Html::parse_document(html), None)
     }
 
     #[test]

crates/markkit/src/html/mod.rs

@@ -4,5 +4,6 @@
 
 mod metadata;
 mod parser;
+mod urls;
 
 pub use parser::HtmlParser;

crates/markkit/src/html/parser.rs

@@ -29,7 +29,10 @@ use crate::config::MarkkitBuilder;
 use crate::error::Error;
 use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, Source, TableCell};
 
+use url::Url;
+
 use super::metadata::PageInfo;
+use super::urls;
 
 /// Maximum recursion depth for nested elements.
 const MAX_DEPTH: usize = 200;
@@ -42,6 +45,8 @@ pub struct HtmlParser<'a> {
     config: &'a MarkkitBuilder,
     /// Footnote target id (without `#`) -> label, filled by references.
     footnote_labels: RefCell<HashMap<String, String>>,
+    /// Base for relative URLs (caller base, then the document `<base href>`).
+    base: Option<Url>,
     /// Cached 1-based index of an element among same-name element siblings.
     sibling_index: RefCell<HashMap<NodeId, usize>>,
 }
@@ -51,6 +56,7 @@ impl<'a> HtmlParser<'a> {
         Self {
             config,
             footnote_labels: RefCell::new(HashMap::new()),
+            base: urls::parse_base(config.base_url.as_deref()),
             sibling_index: RefCell::new(HashMap::new()),
         }
     }
@@ -63,10 +69,28 @@ impl<'a> HtmlParser<'a> {
     /// Parse an HTML string into a [`Document`] plus page title and metadata.
     pub(crate) fn parse_page(&self, html: &str) -> Result<(Document, PageInfo), Error> {
         let dom = Html::parse_document(html);
-        Ok((self.parse_dom(&dom)?, super::metadata::extract(&dom)))
+        let this = self.for_document(&dom);
+        Ok((
+            this.convert_dom(&dom)?,
+            super::metadata::extract(&dom, this.base.as_ref()),
+        ))
     }
 
     fn parse_dom(&self, dom: &Html) -> Result<Document, Error> {
+        self.for_document(dom).convert_dom(dom)
+    }
+
+    /// Parser whose base includes the document's `<base href>`.
+    fn for_document(&self, dom: &Html) -> Self {
+        Self {
+            config: self.config,
+            footnote_labels: RefCell::new(HashMap::new()),
+            sibling_index: RefCell::new(HashMap::new()),
+            base: urls::document_base(dom, self.base.clone()),
+        }
+    }
+
+    fn convert_dom(&self, dom: &Html) -> Result<Document, Error> {
         let root = dom.root_element();
 
         let content_nodes: Vec<scraper::ElementRef<'_>> =
@@ -389,7 +413,7 @@ impl<'a> HtmlParser<'a> {
                         })
                     };
                 }
-                let url = el.value().attr("href").unwrap_or("").to_string();
+                let url = urls::resolve(self.base.as_ref(), el.value().attr("href").unwrap_or(""));
                 let title = el.value().attr("title").map(|s| s.to_string());
                 let children = self.collect_inline(el, exclude, depth);
                 if children.is_empty() {
@@ -406,7 +430,7 @@ impl<'a> HtmlParser<'a> {
                 if !self.config.preserve_images {
                     return None;
                 }
-                let src = el.value().attr("src").unwrap_or("").to_string();
+                let src = urls::resolve(self.base.as_ref(), el.value().attr("src").unwrap_or(""));
                 let alt = el.value().attr("alt").unwrap_or("").to_string();
                 Some(Node::Image { src, alt })
             }