feat(html): extract page metadata from meta, Open Graph and JSON-LD

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
+- HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or
+  `<title>`) and the new `Extraction::metadata` (description, author, published date, language,
+  canonical URL, site name, lead image) from `<meta>`, Open Graph, `<link rel=canonical>` and
+  JSON-LD. `markkit extract --json` includes `format`, `title` and `metadata` for files and stdin.
 - IR v2: footnotes (`[^1]`), definition lists (`Term` / `:   definition`), task lists (`- [x]`),
   figures with captions, table column alignment (`:---:`) and `colspan`/`rowspan` (laid out on a
   grid; GFM has no spans). HTML maps GitHub and Pandoc footnote markup, `<dl>`, checkbox list items,

README.md

@@ -59,9 +59,10 @@ println!("{:?} {:?} {:?}", doc.format, doc.title, doc.warnings);
 println!("{}", doc.markdown);
 ```
 
-`Extraction` carries `markdown`, `title`, `format`, `warnings` (e.g. `truncated_sheet: …`)
-and `stats` (sheets/rows/paragraphs/tables). `extract_bytes` remains as a Markdown-only
-shortcut.
+`Extraction` carries `markdown`, `title`, `format`, `warnings` (e.g. `truncated_sheet: …`),
+`stats` (sheets/rows/paragraphs/tables) and, for HTML, `metadata` (description, author,
+published date, language, canonical URL, site name, lead image from `<meta>`, Open Graph
+and JSON-LD). `extract_bytes` remains as a Markdown-only shortcut.
 
 ### CLI
 
@@ -75,8 +76,8 @@ markkit extract --file document.html
 # With CSS selector filtering
 markkit extract https://example.com --include "article" --exclude "nav,footer"
 
-# JSON output with metadata
-markkit extract https://example.com --json
+# JSON output; files and stdin also carry format, title and page metadata
+markkit extract --file page.html --json
 
 # From stdin
 cat page.html | markkit extract -

crates/markkit-cli/src/main.rs

@@ -81,27 +81,36 @@ async fn main() -> Result<()> {
 
             let source_label = source.clone().unwrap_or_else(|| "<stdin>".to_string());
 
-            let markdown = if read_stdin {
-                let mut buf = String::new();
-                std::io::stdin().read_to_string(&mut buf)?;
-                markkit.extract_html_sync(&buf)?
+            let extraction = if read_stdin {
+                let mut buf = Vec::new();
+                std::io::stdin().read_to_end(&mut buf)?;
+                Some(markkit.extract_document_sync(
+                    &buf,
+                    markkit::DocumentHint::content_type("text/html"),
+                )?)
             } else if file {
                 let src = source.as_deref().unwrap();
                 let bytes = tokio::fs::read(src).await?;
                 // Format detected from the extension, then magic bytes.
-                markkit
-                    .extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))?
-                    .markdown
+                Some(markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))?)
             } else {
-                let src = source.as_deref().unwrap();
-                markkit.extract_url(src).await?
+                None
+            };
+            let markdown = match &extraction {
+                Some(e) => e.markdown.clone(),
+                None => markkit.extract_url(source.as_deref().unwrap()).await?,
             };
 
             if output_json {
-                let output = serde_json::json!({
+                let mut output = serde_json::json!({
                     "source": source_label,
                     "markdown": markdown,
                 });
+                if let Some(e) = &extraction {
+                    output["format"] = serde_json::to_value(e.format)?;
+                    output["title"] = serde_json::to_value(&e.title)?;
+                    output["metadata"] = serde_json::to_value(&e.metadata)?;
+                }
                 println!("{}", serde_json::to_string_pretty(&output)?);
             } else {
                 print!("{markdown}");

crates/markkit/src/extract.rs

@@ -10,7 +10,8 @@ use crate::format::{DocumentHint, Format};
 pub struct Extraction {
     /// Rendered Markdown.
     pub markdown: String,
-    /// Document title when known (DOCX core properties or `Title` style).
+    /// Document title when known (HTML `og:title` / JSON-LD / `<title>`;
+    /// DOCX core properties or `Title` style).
     pub title: Option<String>,
     /// Format the document was parsed as.
     pub format: Format,
@@ -19,6 +20,35 @@ pub struct Extraction {
     pub warnings: Vec<String>,
     /// Format-specific counters.
     pub stats: ExtractionStats,
+    /// Document metadata (HTML `<meta>`, Open Graph, JSON-LD); empty when unknown.
+    pub metadata: Metadata,
+}
+
+/// Document-level metadata. Every field is `None` when absent.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
+#[non_exhaustive]
+pub struct Metadata {
+    /// Summary (`og:description`, `<meta name=description>`, JSON-LD `description`).
+    pub description: Option<String>,
+    /// Author name(s), comma-separated (`<meta name=author>`, JSON-LD `author`).
+    pub author: Option<String>,
+    /// Publication date as written in the source, typically ISO 8601.
+    pub published: Option<String>,
+    /// Content language (`<html lang>`, `Content-Language`, `og:locale`).
+    pub language: Option<String>,
+    /// Canonical URL (`<link rel=canonical>`, `og:url`, JSON-LD `url`).
+    pub canonical_url: Option<String>,
+    /// Site / publisher name (`og:site_name`, JSON-LD `publisher.name`).
+    pub site_name: Option<String>,
+    /// Lead image URL (`og:image`, `twitter:image`, JSON-LD `image`).
+    pub image: Option<String>,
+}
+
+impl Metadata {
+    /// True when no field is set.
+    pub fn is_empty(&self) -> bool {
+        *self == Self::default()
+    }
 }
 
 /// Counters describing the extracted content; `None` when not applicable.
@@ -45,6 +75,7 @@ impl Extraction {
             format,
             warnings: Vec::new(),
             stats: ExtractionStats::default(),
+            metadata: Metadata::default(),
         }
     }
 }
@@ -174,11 +205,7 @@ impl Markkit {
         }
 
         match Format::detect(bytes, hint) {
-            Format::Html => {
-                let text = String::from_utf8_lossy(bytes);
-                let md = self.extract_html_sync(&text)?;
-                Ok(Extraction::new(md, Format::Html))
-            }
+            Format::Html => self.extract_html_document(&String::from_utf8_lossy(bytes)),
             Format::Pdf => self.extract_pdf_sync(bytes),
             Format::Docx => self.extract_docx(bytes),
             Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes),
@@ -195,6 +222,23 @@ impl Markkit {
         }
     }
 
+    #[cfg(feature = "html")]
+    fn extract_html_document(&self, html: &str) -> crate::Result<Extraction> {
+        let parser = crate::html::HtmlParser::new(&self.config);
+        let (doc, page) = parser.parse_page(html)?;
+        let mut renderer = crate::render::MarkdownRenderer::new();
+        let mut e = Extraction::new(renderer.render(&doc), Format::Html);
+        e.title = page.title;
+        e.metadata = page.metadata;
+        Ok(e)
+    }
+
+    #[cfg(not(feature = "html"))]
+    fn extract_html_document(&self, html: &str) -> crate::Result<Extraction> {
+        self.extract_html_sync(html)
+            .map(|md| Extraction::new(md, Format::Html))
+    }
+
     #[cfg(feature = "docx")]
     fn extract_docx(&self, bytes: &[u8]) -> crate::Result<Extraction> {
         let out = guarded("DOCX", Error::DocxExtract, || {