diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eb370e..666e41e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- HTML page metadata: `extract_document` fills `Extraction::title` (Open Graph, JSON-LD or + ``) and the new `Extraction::metadata` (description, author, published date, language, + canonical URL, site name, lead image) from `<meta>`, Open Graph, `<link rel=canonical>` and + JSON-LD. `markkit extract --json` includes `format`, `title` and `metadata` for files and stdin. - IR v2: footnotes (`[^1]`), definition lists (`Term` / `: definition`), task lists (`- [x]`), figures with captions, table column alignment (`:---:`) and `colspan`/`rowspan` (laid out on a grid; GFM has no spans). HTML maps GitHub and Pandoc footnote markup, `<dl>`, checkbox list items, diff --git a/README.md b/README.md index 0d9e55a..34c744d 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,10 @@ println!("{:?} {:?} {:?}", doc.format, doc.title, doc.warnings); println!("{}", doc.markdown); ``` -`Extraction` carries `markdown`, `title`, `format`, `warnings` (e.g. `truncated_sheet: …`) -and `stats` (sheets/rows/paragraphs/tables). `extract_bytes` remains as a Markdown-only -shortcut. +`Extraction` carries `markdown`, `title`, `format`, `warnings` (e.g. `truncated_sheet: …`), +`stats` (sheets/rows/paragraphs/tables) and, for HTML, `metadata` (description, author, +published date, language, canonical URL, site name, lead image from `<meta>`, Open Graph +and JSON-LD). `extract_bytes` remains as a Markdown-only shortcut. ### CLI @@ -75,8 +76,8 @@ markkit extract --file document.html # With CSS selector filtering markkit extract https://example.com --include "article" --exclude "nav,footer" -# JSON output with metadata -markkit extract https://example.com --json +# JSON output; files and stdin also carry format, title and page metadata +markkit extract --file page.html --json # From stdin cat page.html | markkit extract - diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 4f2742d..206b454 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -81,27 +81,36 @@ async fn main() -> Result<()> { let source_label = source.clone().unwrap_or_else(|| "<stdin>".to_string()); - let markdown = if read_stdin { - let mut buf = String::new(); - std::io::stdin().read_to_string(&mut buf)?; - markkit.extract_html_sync(&buf)? + let extraction = if read_stdin { + let mut buf = Vec::new(); + std::io::stdin().read_to_end(&mut buf)?; + Some(markkit.extract_document_sync( + &buf, + markkit::DocumentHint::content_type("text/html"), + )?) } else if file { let src = source.as_deref().unwrap(); let bytes = tokio::fs::read(src).await?; // Format detected from the extension, then magic bytes. - markkit - .extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))? - .markdown + Some(markkit.extract_document_sync(&bytes, markkit::DocumentHint::file_name(src))?) } else { - let src = source.as_deref().unwrap(); - markkit.extract_url(src).await? + None + }; + let markdown = match &extraction { + Some(e) => e.markdown.clone(), + None => markkit.extract_url(source.as_deref().unwrap()).await?, }; if output_json { - let output = serde_json::json!({ + let mut output = serde_json::json!({ "source": source_label, "markdown": markdown, }); + if let Some(e) = &extraction { + output["format"] = serde_json::to_value(e.format)?; + output["title"] = serde_json::to_value(&e.title)?; + output["metadata"] = serde_json::to_value(&e.metadata)?; + } println!("{}", serde_json::to_string_pretty(&output)?); } else { print!("{markdown}"); diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 06ce32b..5f6fe03 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -10,7 +10,8 @@ use crate::format::{DocumentHint, Format}; pub struct Extraction { /// Rendered Markdown. pub markdown: String, - /// Document title when known (DOCX core properties or `Title` style). + /// Document title when known (HTML `og:title` / JSON-LD / `<title>`; + /// DOCX core properties or `Title` style). pub title: Option<String>, /// Format the document was parsed as. pub format: Format, @@ -19,6 +20,35 @@ pub struct Extraction { pub warnings: Vec<String>, /// Format-specific counters. pub stats: ExtractionStats, + /// Document metadata (HTML `<meta>`, Open Graph, JSON-LD); empty when unknown. + pub metadata: Metadata, +} + +/// Document-level metadata. Every field is `None` when absent. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +#[non_exhaustive] +pub struct Metadata { + /// Summary (`og:description`, `<meta name=description>`, JSON-LD `description`). + pub description: Option<String>, + /// Author name(s), comma-separated (`<meta name=author>`, JSON-LD `author`). + pub author: Option<String>, + /// Publication date as written in the source, typically ISO 8601. + pub published: Option<String>, + /// Content language (`<html lang>`, `Content-Language`, `og:locale`). + pub language: Option<String>, + /// Canonical URL (`<link rel=canonical>`, `og:url`, JSON-LD `url`). + pub canonical_url: Option<String>, + /// Site / publisher name (`og:site_name`, JSON-LD `publisher.name`). + pub site_name: Option<String>, + /// Lead image URL (`og:image`, `twitter:image`, JSON-LD `image`). + pub image: Option<String>, +} + +impl Metadata { + /// True when no field is set. + pub fn is_empty(&self) -> bool { + *self == Self::default() + } } /// Counters describing the extracted content; `None` when not applicable. @@ -45,6 +75,7 @@ impl Extraction { format, warnings: Vec::new(), stats: ExtractionStats::default(), + metadata: Metadata::default(), } } } @@ -174,11 +205,7 @@ impl Markkit { } match Format::detect(bytes, hint) { - Format::Html => { - let text = String::from_utf8_lossy(bytes); - let md = self.extract_html_sync(&text)?; - Ok(Extraction::new(md, Format::Html)) - } + Format::Html => self.extract_html_document(&String::from_utf8_lossy(bytes)), Format::Pdf => self.extract_pdf_sync(bytes), Format::Docx => self.extract_docx(bytes), Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), @@ -195,6 +222,23 @@ impl Markkit { } } + #[cfg(feature = "html")] + fn extract_html_document(&self, html: &str) -> crate::Result<Extraction> { + let parser = crate::html::HtmlParser::new(&self.config); + let (doc, page) = parser.parse_page(html)?; + let mut renderer = crate::render::MarkdownRenderer::new(); + let mut e = Extraction::new(renderer.render(&doc), Format::Html); + e.title = page.title; + e.metadata = page.metadata; + Ok(e) + } + + #[cfg(not(feature = "html"))] + fn extract_html_document(&self, html: &str) -> crate::Result<Extraction> { + self.extract_html_sync(html) + .map(|md| Extraction::new(md, Format::Html)) + } + #[cfg(feature = "docx")] fn extract_docx(&self, bytes: &[u8]) -> crate::Result<Extraction> { let out = guarded("DOCX", Error::DocxExtract, || { diff --git a/crates/markkit/src/html/metadata.rs b/crates/markkit/src/html/metadata.rs new file mode 100644 index 0000000..22296fd --- /dev/null +++ b/crates/markkit/src/html/metadata.rs @@ -0,0 +1,416 @@ +//! Page metadata from `<head>`: `<title>`, `<meta>`, Open Graph, `<link rel=canonical>`, JSON-LD. +//! +//! Decisions: +//! - Precedence per field is fixed (documented in `knowledge/formats/html-extraction.md`): +//! explicit social/structured tags beat generic ones (`og:title` > JSON-LD > `<title>`). +//! - All values are untrusted text: whitespace collapsed, control chars dropped, length +//! capped at [`MAX_VALUE_CHARS`]. URL fields keep only `http(s)` or scheme-less (relative) +//! values, so `javascript:` / `data:` never surface as a canonical URL or image. +//! - JSON-LD is bounded: at most [`MAX_JSONLD_SCRIPTS`] scripts of [`MAX_JSONLD_BYTES`] each, +//! at most [`MAX_JSONLD_NODES`] candidate nodes; serde_json caps nesting at 128. Invalid +//! JSON is ignored, never an error. + +use scraper::{ElementRef, Html, Selector}; +use serde_json::Value; + +use crate::extract::Metadata; + +/// Longest metadata value kept, in chars; longer values are truncated. +const MAX_VALUE_CHARS: usize = 2048; +/// JSON-LD scripts inspected per page. +const MAX_JSONLD_SCRIPTS: usize = 16; +/// Larger JSON-LD scripts are skipped unparsed. +const MAX_JSONLD_BYTES: usize = 1024 * 1024; +/// JSON-LD objects (top level, arrays, `@graph`) considered across all scripts. +const MAX_JSONLD_NODES: usize = 256; + +/// Title plus [`Metadata`] read from one parsed page. +#[derive(Debug, Default)] +pub(crate) struct PageInfo { + pub title: Option<String>, + pub metadata: Metadata, +} + +/// Read title and metadata from a parsed document. +pub(crate) fn extract(dom: &Html) -> PageInfo { + let meta = MetaTags::collect(dom); + let ld = JsonLd::collect(dom); + + let title = meta + .get(&["og:title", "twitter:title"]) + .or_else(|| ld.text("headline")) + .or_else(|| first_text(dom, "title")) + .or_else(|| meta.get(&["dc.title", "dcterms.title"])) + .or_else(|| ld.text("name")); + + let metadata = Metadata { + description: meta + .get(&["og:description", "description", "twitter:description"]) + .or_else(|| ld.text("description")) + .or_else(|| meta.get(&["dc.description", "dcterms.description"])), + author: meta + .get(&["author", "dc.creator", "dcterms.creator"]) + .or_else(|| ld.author()) + // `article:author` is usually a profile URL; keep it only when it is a name. + .or_else(|| meta.get(&["article:author"]).filter(|a| !looks_like_url(a))), + published: meta + .get(&["article:published_time"]) + .or_else(|| ld.text("datePublished")) + .or_else(|| { + meta.get(&[ + "date", + "dc.date", + "dcterms.created", + "dc.date.issued", + "pubdate", + ]) + }), + language: html_lang(dom) + .or_else(|| meta.http_equiv("content-language")) + .or_else(|| meta.get(&["og:locale"])) + .or_else(|| ld.text("inLanguage")), + canonical_url: canonical_link(dom) + .or_else(|| meta.get(&["og:url"]).and_then(url_value)) + .or_else(|| ld.text("url").and_then(url_value)), + site_name: meta + .get(&["og:site_name", "application-name"]) + .or_else(|| ld.publisher()), + image: meta + .get(&[ + "og:image", + "og:image:url", + "og:image:secure_url", + "twitter:image", + ]) + .and_then(url_value) + .or_else(|| ld.image()), + }; + + PageInfo { title, metadata } +} + +/// `<meta>` values keyed by lowercased `property` / `name`, first occurrence wins. +struct MetaTags { + named: Vec<(String, String)>, + http_equiv: Vec<(String, String)>, +} + +impl MetaTags { + fn collect(dom: &Html) -> Self { + let sel = Selector::parse("meta").unwrap(); + let mut named = Vec::new(); + let mut http_equiv = Vec::new(); + for el in dom.select(&sel) { + let attrs = el.value(); + let Some(content) = attrs.attr("content").and_then(clean) else { + continue; + }; + for key in [ + attrs.attr("property"), + attrs.attr("name"), + attrs.attr("itemprop"), + ] + .into_iter() + .flatten() + { + named.push((key.trim().to_ascii_lowercase(), content.clone())); + } + if let Some(key) = attrs.attr("http-equiv") { + http_equiv.push((key.trim().to_ascii_lowercase(), content)); + } + } + Self { named, http_equiv } + } + + /// First value of the first key (in priority order) that is present. + fn get(&self, keys: &[&str]) -> Option<String> { + keys.iter().find_map(|k| { + self.named + .iter() + .find(|(name, _)| name == k) + .map(|(_, v)| v.clone()) + }) + } + + fn http_equiv(&self, key: &str) -> Option<String> { + self.http_equiv + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.clone()) + } +} + +/// Candidate JSON-LD objects, in document order, article-like types first. +struct JsonLd { + nodes: Vec<Value>, +} + +impl JsonLd { + fn collect(dom: &Html) -> Self { + let sel = Selector::parse("script").unwrap(); + let mut nodes = Vec::new(); + let scripts = dom + .select(&sel) + .filter(|el| { + el.value() + .attr("type") + .is_some_and(|t| t.trim().eq_ignore_ascii_case("application/ld+json")) + }) + .take(MAX_JSONLD_SCRIPTS); + for script in scripts { + let raw: String = script.text().collect(); + if raw.len() > MAX_JSONLD_BYTES { + continue; + } + if let Ok(value) = serde_json::from_str::<Value>(&raw) { + flatten_nodes(value, &mut nodes); + } + } + // Stable: article-like nodes first, otherwise document order. + nodes.sort_by_key(|n| !is_article(n)); + Self { nodes } + } + + fn text(&self, key: &str) -> Option<String> { + self.nodes.iter().find_map(|n| n.get(key).and_then(text_of)) + } + + fn author(&self) -> Option<String> { + self.nodes + .iter() + .find_map(|n| n.get("author").and_then(names_of)) + } + + fn publisher(&self) -> Option<String> { + self.nodes + .iter() + .find_map(|n| n.get("publisher").and_then(names_of)) + } + + fn image(&self) -> Option<String> { + self.nodes.iter().find_map(|n| { + let img = n.get("image")?; + let img = match img { + Value::Array(items) => items.first()?, + other => other, + }; + match img { + Value::String(s) => clean(s), + Value::Object(o) => o.get("url").and_then(text_of), + _ => None, + } + .and_then(url_value) + }) + } +} + +/// Push objects from a JSON-LD value: the value itself, array members and `@graph` members. +fn flatten_nodes(value: Value, out: &mut Vec<Value>) { + let mut stack = vec![value]; + while let Some(v) = stack.pop() { + if out.len() >= MAX_JSONLD_NODES { + return; + } + match v { + Value::Array(items) => stack.extend(items.into_iter().rev()), + Value::Object(mut obj) => { + let graph = obj.remove("@graph"); + out.push(Value::Object(obj)); + if let Some(g) = graph { + stack.push(g); + } + } + _ => {} + } + } +} + +fn is_article(node: &Value) -> bool { + const ARTICLE_TYPES: &[&str] = &[ + "Article", + "NewsArticle", + "BlogPosting", + "Report", + "ScholarlyArticle", + "TechArticle", + "SocialMediaPosting", + "LiveBlogPosting", + "OpinionNewsArticle", + "AnalysisNewsArticle", + "ReportageNewsArticle", + ]; + let is = |t: &Value| t.as_str().is_some_and(|s| ARTICLE_TYPES.contains(&s)); + match node.get("@type") { + Some(Value::Array(types)) => types.iter().any(is), + Some(t) => is(t), + None => false, + } +} + +/// A scalar JSON-LD value as cleaned text (strings and numbers only). +fn text_of(v: &Value) -> Option<String> { + match v { + Value::String(s) => clean(s), + Value::Number(n) => clean(&n.to_string()), + Value::Array(items) => items.iter().find_map(text_of), + _ => None, + } +} + +/// Names from a JSON-LD person/organization value: `"x"`, `{name}`, or an array of those. +fn names_of(v: &Value) -> Option<String> { + let one = |v: &Value| match v { + Value::String(s) if !looks_like_url(s) => clean(s), + Value::Object(o) => o.get("name").and_then(text_of), + _ => None, + }; + let names: Vec<String> = match v { + Value::Array(items) => items.iter().take(16).filter_map(one).collect(), + other => one(other).into_iter().collect(), + }; + if names.is_empty() { + None + } else { + clean(&names.join(", ")) + } +} + +fn first_text(dom: &Html, selector: &str) -> Option<String> { + let sel = Selector::parse(selector).unwrap(); + dom.select(&sel) + .next() + .and_then(|el| clean(&el.text().collect::<String>())) +} + +fn html_lang(dom: &Html) -> Option<String> { + let root: ElementRef<'_> = dom.root_element(); + root.value() + .attr("lang") + .or_else(|| root.value().attr("xml:lang")) + .and_then(clean) +} + +fn canonical_link(dom: &Html) -> Option<String> { + let sel = Selector::parse("link[rel][href]").unwrap(); + dom.select(&sel) + .find(|el| { + el.value().attr("rel").is_some_and(|r| { + r.split_ascii_whitespace() + .any(|t| t.eq_ignore_ascii_case("canonical")) + }) + }) + .and_then(|el| el.value().attr("href")) + .and_then(clean) + .and_then(url_value) +} + +/// Keep a URL only when it is `http(s)` or has no scheme (relative). +fn url_value(url: String) -> Option<String> { + match url.split_once(':') { + Some((scheme, _)) + if !scheme.contains(['/', '?', '#']) + && !scheme.eq_ignore_ascii_case("http") + && !scheme.eq_ignore_ascii_case("https") => + { + None + } + _ => Some(url), + } +} + +fn looks_like_url(s: &str) -> bool { + let s = s.trim_start(); + s.starts_with("http://") || s.starts_with("https://") || s.starts_with("//") +} + +/// Collapse whitespace, drop control chars, cap length; `None` when nothing is left. +fn clean(s: &str) -> Option<String> { + let mut out = String::new(); + let mut chars = 0; + let mut pending_space = false; + for ch in s.chars() { + if ch.is_whitespace() { + pending_space = !out.is_empty(); + continue; + } + if ch.is_control() { + continue; + } + if chars >= MAX_VALUE_CHARS { + break; + } + if pending_space { + out.push(' '); + chars += 1; + pending_space = false; + } + out.push(ch); + chars += 1; + } + (!out.is_empty()).then_some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(html: &str) -> PageInfo { + extract(&Html::parse_document(html)) + } + + #[test] + fn clean_collapses_and_caps() { + assert_eq!(clean(" a \n\t b "), Some("a b".into())); + assert_eq!(clean(" \u{0}\u{7} "), None); + assert_eq!( + clean(&"x".repeat(5000)).unwrap().chars().count(), + MAX_VALUE_CHARS + ); + } + + #[test] + fn url_value_filters_schemes() { + assert_eq!( + url_value("https://a.b/c".into()), + Some("https://a.b/c".into()) + ); + assert_eq!(url_value("/rel/path".into()), Some("/rel/path".into())); + assert_eq!(url_value("rel?q=a:b".into()), Some("rel?q=a:b".into())); + assert_eq!(url_value("javascript:alert(1)".into()), None); + assert_eq!(url_value("JaVaScRiPt:alert(1)".into()), None); + assert_eq!(url_value("data:text/html,x".into()), None); + } + + #[test] + fn title_precedence() { + let i = info("<title>T"); + assert_eq!(i.title.as_deref(), Some("OG")); + let i = info(" Plain \n title "); + assert_eq!(i.title.as_deref(), Some("Plain title")); + assert!(info("

no head

").title.is_none()); + } + + #[test] + fn jsonld_graph_and_author_forms() { + let i = info( + r#""#, + ); + assert_eq!(i.title.as_deref(), Some("H")); + assert_eq!(i.metadata.author.as_deref(), Some("A, B")); + assert_eq!(i.metadata.published.as_deref(), Some("2024-01-02")); + assert_eq!(i.metadata.site_name.as_deref(), Some("Pub")); + assert_eq!(i.metadata.image.as_deref(), Some("https://i/1.png")); + } + + #[test] + fn invalid_jsonld_ignored() { + let i = info(r#"T"#); + assert_eq!(i.title.as_deref(), Some("T")); + } +} diff --git a/crates/markkit/src/html/mod.rs b/crates/markkit/src/html/mod.rs index 4809ce9..85b6099 100644 --- a/crates/markkit/src/html/mod.rs +++ b/crates/markkit/src/html/mod.rs @@ -2,6 +2,7 @@ //! //! Converts HTML input into the canonical IR. +mod metadata; mod parser; pub use parser::HtmlParser; diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 49c686f..657dfe9 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -21,6 +21,8 @@ use crate::config::MarkkitBuilder; use crate::error::Error; use crate::ir::{Alignment, DefinitionItem, Document, ListItem, Node, TableCell}; +use super::metadata::PageInfo; + /// Maximum recursion depth for nested elements. const MAX_DEPTH: usize = 200; /// HTML-spec clamps for table spans. @@ -44,7 +46,16 @@ impl<'a> HtmlParser<'a> { /// Parse an HTML string into a [`Document`]. pub fn parse(&self, html: &str) -> Result { + self.parse_dom(&Html::parse_document(html)) + } + + /// Parse an HTML string into a [`Document`] plus page title and metadata. + pub(crate) fn parse_page(&self, html: &str) -> Result<(Document, PageInfo), Error> { let dom = Html::parse_document(html); + Ok((self.parse_dom(&dom)?, super::metadata::extract(&dom))) + } + + fn parse_dom(&self, dom: &Html) -> Result { let root = dom.root_element(); let content_nodes: Vec> = diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index d6453e2..2faead0 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -61,7 +61,7 @@ mod xml; pub use config::MarkkitBuilder; pub use error::Error; -pub use extract::{Extraction, ExtractionStats, Markkit}; +pub use extract::{Extraction, ExtractionStats, Markkit, Metadata}; pub use format::{DocumentHint, Format}; pub use ir::Document; pub use render::MarkdownRenderer; diff --git a/crates/markkit/tests/html_metadata.rs b/crates/markkit/tests/html_metadata.rs new file mode 100644 index 0000000..9c4ea5e --- /dev/null +++ b/crates/markkit/tests/html_metadata.rs @@ -0,0 +1,174 @@ +//! Page metadata (`Extraction::title` / `Extraction::metadata`) from HTML. + +#![cfg(feature = "html")] + +use markkit::{DocumentHint, Extraction, Markkit}; + +fn extract(html: &str) -> Extraction { + Markkit::builder() + .build() + .extract_document_sync(html.as_bytes(), DocumentHint::content_type("text/html")) + .unwrap() +} + +const ARTICLE: &str = r#" + + + Fallback title | Site + + + + + + + + + +

Headline

Body text.

+"#; + +#[test] +fn meta_and_open_graph_fields() { + let e = extract(ARTICLE); + assert_eq!(e.title.as_deref(), Some("OG title")); + let m = &e.metadata; + assert_eq!(m.description.as_deref(), Some("OG description")); + assert_eq!(m.author.as_deref(), Some("Jane Doe")); + assert_eq!(m.published.as_deref(), Some("2026-10-01T09:00:00Z")); + assert_eq!(m.language.as_deref(), Some("en-GB")); + assert_eq!(m.canonical_url.as_deref(), Some("https://example.com/a/1")); + assert_eq!(m.site_name.as_deref(), Some("Example News")); + assert_eq!(m.image.as_deref(), Some("https://example.com/lead.jpg")); + assert_eq!(e.markdown, "# Headline\n\nBody text.\n"); +} + +#[test] +fn json_ld_fills_missing_fields() { + let e = extract( + r#" +

x

"#, + ); + assert_eq!(e.title.as_deref(), Some("LD headline")); + let m = &e.metadata; + assert_eq!(m.description.as_deref(), Some("LD description")); + assert_eq!(m.author.as_deref(), Some("Ivan")); + assert_eq!(m.published.as_deref(), Some("2025-05-05")); + assert_eq!(m.language.as_deref(), Some("uk")); + assert_eq!(m.site_name.as_deref(), Some("LD Pub")); + assert_eq!(m.canonical_url.as_deref(), Some("https://ld.example/post")); + assert_eq!(m.image.as_deref(), Some("https://ld.example/i.png")); + assert!( + !e.markdown.contains("LD headline"), + "JSON-LD must not leak into markdown" + ); +} + +#[test] +fn title_tag_used_without_social_tags() { + let e = + extract(" Just a title

x

"); + assert_eq!(e.title.as_deref(), Some("Just a title")); + assert!(e.metadata.is_empty()); +} + +#[test] +fn bare_fragment_has_no_metadata() { + let e = extract("

hello

"); + assert_eq!(e.title, None); + assert!(e.metadata.is_empty()); +} + +#[test] +fn author_profile_url_is_not_a_name() { + let e = extract( + r#""#, + ); + assert_eq!(e.metadata.author, None); +} + +#[test] +fn metadata_serializes_in_json() { + let json = serde_json::to_value(extract(ARTICLE)).unwrap(); + assert_eq!(json["title"], "OG title"); + assert_eq!(json["metadata"]["site_name"], "Example News"); +} + +// ─── Untrusted input ──────────────────────────────────────────────── + +#[test] +fn dangerous_url_schemes_dropped() { + let e = extract( + r#" + + + + + "#, + ); + assert_eq!(e.metadata.canonical_url, None); + assert_eq!(e.metadata.image, None); +} + +#[test] +fn relative_canonical_kept_as_is() { + let e = extract(r#""#); + assert_eq!(e.metadata.canonical_url.as_deref(), Some("/a/1")); +} + +#[test] +fn values_are_single_line_and_capped() { + let long = "y".repeat(10_000); + let e = extract(&format!( + "a\n\n\tb\u{7}c" + )); + assert_eq!(e.title.as_deref(), Some("a bc")); + assert_eq!(e.metadata.description.unwrap().chars().count(), 2048); +} + +#[test] +fn malformed_json_ld_ignored() { + let e = extract( + r#"T + + "#, + ); + // Numbers are accepted as text; the broken script is skipped. + assert_eq!(e.title.as_deref(), Some("7")); +} + +#[test] +fn deeply_nested_json_ld_does_not_overflow() { + let deep = format!("{}{}", "[".repeat(100_000), "]".repeat(100_000)); + let e = extract(&format!( + "T" + )); + assert_eq!(e.title.as_deref(), Some("T")); +} + +#[test] +fn oversized_json_ld_skipped() { + let pad = " ".repeat(1024 * 1024); + let e = extract(&format!( + r#""# + )); + assert_eq!(e.title, None); +} + +#[test] +fn many_json_ld_scripts_bounded() { + let scripts: String = (0..5_000) + .map(|i| format!(r#""#)) + .collect(); + let e = extract(&format!( + r#"{scripts}"# + )); + // Only the first 16 scripts are read, so the late headline is never reached. + assert_eq!(e.title, None); +} diff --git a/crates/markkit/tests/html_proptest.rs b/crates/markkit/tests/html_proptest.rs index 1453afa..5dafc6c 100644 --- a/crates/markkit/tests/html_proptest.rs +++ b/crates/markkit/tests/html_proptest.rs @@ -39,6 +39,31 @@ proptest! { let _ = m.extract_html_sync(&s); } + /// Page metadata from arbitrary head markup never panics and stays single-line. + #[test] + fn metadata_values_single_line( + key in prop::sample::select(vec!["og:title", "description", "author", "og:url", "og:image"]), + value in "\\PC{0,300}", + ld in "\\PC{0,300}", + ) { + let html = format!( + "{value}\ + ", + value.replace('"', """) + ); + let e = Markkit::builder() + .build() + .extract_document_sync(html.as_bytes(), markkit::DocumentHint::content_type("text/html")) + .unwrap(); + let m = &e.metadata; + for v in [&e.title, &m.description, &m.author, &m.canonical_url, &m.image].into_iter().flatten() { + prop_assert!(!v.contains(['\n', '\r']) && v.chars().count() <= 2048, "{:?}", v); + } + for v in [&m.canonical_url, &m.image].into_iter().flatten() { + prop_assert!(!v.to_ascii_lowercase().starts_with("javascript:"), "{:?}", v); + } + } + /// Extraction is deterministic: same input always gives same output. #[test] fn deterministic_output(s in "\\PC{0,200}") { diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 7e3a825..f1f915b 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -44,10 +44,11 @@ impl Markkit { pub struct Extraction { // #[non_exhaustive] pub markdown: String, - pub title: Option, // DOCX core.xml dc:title, else first Title-style paragraph + pub title: Option, // HTML og:title / JSON-LD / ; DOCX core.xml dc:title, else Title style pub format: Format, pub warnings: Vec<String>, // "<code>: <message>" pub stats: ExtractionStats, // pages / sheets / rows / paragraphs / tables (Option each) + pub metadata: Metadata, // HTML page metadata, see HTML Extraction; empty otherwise } ``` diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index 9588f57..b175633 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -159,6 +159,32 @@ providing defense-in-depth. - **Full page** (implemented): no include selector converts the whole body minus excludes - **Readability** (planned): article-content detection heuristic, not implemented +## Page Metadata + +`extract_document` on HTML fills `Extraction::title` and `Extraction::metadata` +(`html/metadata.rs`) from the same parsed DOM; `extract_html*` return Markdown only. +Metadata never changes the Markdown. First present source wins: + +| Field | Sources, in priority order | +|-------|----------------------------| +| `title` | `og:title`, `twitter:title`, JSON-LD `headline`, `<title>`, `dc.title`, JSON-LD `name` | +| `description` | `og:description`, `<meta name=description>`, `twitter:description`, JSON-LD `description`, `dc.description` | +| `author` | `<meta name=author>`, `dc.creator`, JSON-LD `author` (string, `{name}`, array joined `, `), `article:author` unless a URL | +| `published` | `article:published_time`, JSON-LD `datePublished`, `date` / `dc.date` / `dcterms.created` / `pubdate` (raw string) | +| `language` | `<html lang>`, `http-equiv=content-language`, `og:locale`, JSON-LD `inLanguage` | +| `canonical_url` | `<link rel=canonical>`, `og:url`, JSON-LD `url` | +| `site_name` | `og:site_name`, `application-name`, JSON-LD `publisher` name | +| `image` | `og:image` (`:url`, `:secure_url`), `twitter:image`, JSON-LD `image` (string, `{url}`, array) | + +`<meta>` keys come from `property`, `name` or `itemprop`, matched case-insensitively. +JSON-LD nodes are the top-level value, array members and `@graph` members; article-like +`@type`s (`Article`, `NewsArticle`, `BlogPosting`, ...) are consulted first. + +Untrusted-input rules: values are whitespace-collapsed, control characters dropped, +capped at 2048 chars. URL fields keep only `http(s)` or scheme-less (relative, not yet +resolved) values. JSON-LD: first 16 scripts, each at most 1 MiB, at most 256 nodes; +serde_json's 128-level recursion limit bounds nesting; invalid JSON is ignored. + ## Edge Cases - Nested lists: supported via recursive `collect_list_items` with depth tracking diff --git a/knowledge/log.md b/knowledge/log.md index ea6e8bb..90da485 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from `<meta>`, Open Graph, `<link rel=canonical>` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. * **IR v2**: footnotes, definition lists, task lists, figures, table alignment and spans; `Document.ir_version` (`IR_VERSION = 2`), `#[non_exhaustive]` IR types, `Node::is_block`/`is_inline`. HTML maps GitHub/Pandoc footnotes, `<dl>`, checkbox items, captioned figures, `colspan`/`rowspan`/alignment and row-header `<th>`. New threat T2e and invariant S13 (span grid budget, footnote label sanitizing). See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md), [Threat Model](security/threat-model.md). * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index d97296e..322faa3 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -37,6 +37,8 @@ tags: - Event handler attributes (onclick, onload, etc.) are not preserved -- only `href`, `src`, `alt`, `title`, `class`, `start` attributes are read - Output is markdown text with no HTML pass-through - Form elements (`<form>`, `<input>`, `<button>`, `<select>`, `<textarea>`) stripped +- Page metadata (`Extraction::metadata`): URL fields drop non-`http(s)` schemes + (`javascript:`, `data:`); values single-line and capped at 2048 chars **Test Coverage**: - `html_security::xss_script_tag` @@ -48,6 +50,7 @@ tags: - `html_security::xss_iframe` - `html_proptest::no_script_tags_in_output` - `html_proptest::no_style_tags_in_output` +- `html_metadata::dangerous_url_schemes_dropped`, `values_are_single_line_and_capped`, `html_proptest::metadata_values_single_line` ### T2: Denial of Service (Resource Exhaustion) @@ -84,6 +87,12 @@ tags: - No custom entity processing - Input size limit bounds total expansion +#### T2f: JSON-LD Amplification +- First 16 `application/ld+json` scripts only, each at most 1 MiB, at most 256 nodes +- serde_json recursion limit (128) bounds nesting; parse errors are ignored +- Tests: `html_metadata::deeply_nested_json_ld_does_not_overflow`, + `oversized_json_ld_skipped`, `many_json_ld_scripts_bounded`, `malformed_json_ld_ignored` + **Test Coverage**: - `html_security::deeply_nested_elements` - `html_security::very_long_text_node`