feat: pluggable FormatParser registry

CHANGELOG.md

@@ -53,6 +53,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - Declarative HTML element rules: `MarkkitBuilder::html_rule(".note", ElementAction::BlockQuote { .. })`
   maps elements by CSS selector to skip / unwrap / heading / paragraph / block quote (with title) /
   code / emphasis / text, before the built-in mapping. CLI: `markkit extract --rules rules.json`.
+- Pluggable parsers: `MarkkitBuilder::parser(..)` registers a `registry::FormatParser` that handles
+  detected formats (e.g. PDF or CSV) or claims inputs by content, before the built-ins. Its IR goes
+  through transforms and the shared renderer; panics become the new `Error::Parser`.
 - JSON Schema for the IR at `schema/ir.schema.json`; `json-schema` feature adds `markkit::ir::json_schema()`.
 - Per-block provenance: `MarkkitBuilder::track_sources(true)` records a `Source` on each IR block
   (HTML element XPath, DOCX body block index, spreadsheet sheet and A1 range) for citations.

crates/markkit/src/config.rs

@@ -48,6 +48,8 @@ pub struct MarkkitBuilder {
     /// Declarative HTML element rules, tried in order before the built-in
     /// mapping (see [`crate::rules`]).
     pub html_rules: Vec<crate::rules::HtmlRule>,
+    /// Custom format parsers, consulted before the built-ins (see [`crate::registry`]).
+    pub parsers: crate::registry::ParserRegistry,
 }
 
 impl Default for MarkkitBuilder {
@@ -75,6 +77,7 @@ impl Default for MarkkitBuilder {
             transforms: Default::default(),
             main_content: false,
             html_rules: Vec::new(),
+            parsers: Default::default(),
         }
     }
 }
@@ -152,6 +155,13 @@ impl MarkkitBuilder {
         self
     }
 
+    /// Register a custom format parser; it runs instead of the built-in one
+    /// for the formats it handles or the inputs it claims.
+    pub fn parser(mut self, p: impl crate::registry::FormatParser + 'static) -> Self {
+        self.parsers.push(p);
+        self
+    }
+
     pub fn include_selector(mut self, selector: impl Into<String>) -> Self {
         self.include_selectors.push(selector.into());
         self

crates/markkit/src/error.rs

@@ -31,6 +31,10 @@ pub enum Error {
     #[error("Resource limit exceeded: {0}")]
     ResourceLimit(String),
 
+    /// A custom [`crate::registry::FormatParser`] failed or panicked.
+    #[error("Parser error: {0}")]
+    Parser(String),
+
     /// A [`crate::transform::Transform`] failed.
     #[error("Transform error: {0}")]
     Transform(String),

crates/markkit/src/extract.rs

@@ -94,7 +94,9 @@ pub struct ParsedDocument {
 }
 
 impl ParsedDocument {
-    fn new(document: Document, format: Format) -> Self {
+    /// IR plus format with empty metadata (for custom
+    /// [`crate::registry::FormatParser`]s).
+    pub fn new(document: Document, format: Format) -> Self {
         Self {
             document,
             title: None,
@@ -143,7 +145,6 @@ impl ParsedDocument {
 /// Run a parser, converting a panic into an error (defence in depth for
 /// third-party parsers; only effective when built with `panic = "unwind"`, which
 /// markkit's own release profile uses).
-#[cfg(any(feature = "docx", feature = "spreadsheet", feature = "pdf"))]
 fn guarded<T>(
     what: &str,
     make_err: fn(String) -> Error,
@@ -313,6 +314,10 @@ impl Markkit {
         self.config.limits.check_input(bytes.len())?;
 
         let format = Format::detect(bytes, hint);
+        if let Some(p) = self.config.parsers.select(bytes, hint, format) {
+            return self.parse_custom(p, bytes, budget);
+        }
+
         match format {
             Format::Html => self.parse_html_doc(bytes, budget),
             Format::Pdf => self.parse_pdf(bytes, budget),
@@ -336,6 +341,19 @@ impl Markkit {
         }
     }
 
+    fn parse_custom(
+        &self,
+        p: &dyn crate::registry::FormatParser,
+        bytes: &[u8],
+        budget: &Budget,
+    ) -> crate::Result<ParsedDocument> {
+        let mut parsed = guarded(p.name(), Error::Parser, || p.parse(bytes, &self.config))?;
+        // Custom parsers are not interruptible: check the deadline afterwards.
+        budget.check_time()?;
+        self.config.transforms.apply(&mut parsed.document)?;
+        Ok(parsed)
+    }
+
     #[cfg(feature = "html")]
     fn parse_html_doc(&self, bytes: &[u8], budget: &Budget) -> crate::Result<ParsedDocument> {
         let text = String::from_utf8_lossy(bytes);

crates/markkit/src/lib.rs

@@ -67,6 +67,7 @@ pub mod limits;
 mod markdown;
 #[cfg(feature = "pdf")]
 mod pdf;
+pub mod registry;
 pub mod render;
 pub mod rules;
 #[cfg(feature = "spreadsheet")]

crates/markkit/src/registry.rs

@@ -0,0 +1,86 @@
+//! Pluggable format parsers.
+//!
+//! Decisions:
+//! - Embedders register [`FormatParser`]s on the builder
+//!   ([`crate::MarkkitBuilder::parser`]). For each document, registered
+//!   parsers are consulted before the built-ins, in registration order:
+//!   first one whose [`FormatParser::claims`] returns true, else the first
+//!   one listing the detected [`Format`] in [`FormatParser::formats`].
+//!   This lets an embedder add a format (PDF, EPUB, ...) or replace a
+//!   built-in without forking.
+//! - A custom parser gets the same guarantees as built-ins: input is
+//!   size-checked first, a panic becomes [`crate::Error::Parser`], transforms
+//!   run on its output, and the shared renderer produces the Markdown. Its own
+//!   resource limits are its responsibility (it sees the [`MarkkitBuilder`]).
+//! - Parsers are `Arc`-shared so the builder stays `Clone`.
+
+use std::fmt;
+use std::sync::Arc;
+
+use crate::config::MarkkitBuilder;
+use crate::extract::ParsedDocument;
+use crate::format::{DocumentHint, Format};
+
+/// A parser from bytes to the IR.
+pub trait FormatParser: Send + Sync {
+    /// Short name for debugging and error messages.
+    fn name(&self) -> &str;
+
+    /// Detected formats this parser handles (see [`Format::detect`]).
+    fn formats(&self) -> &[Format] {
+        &[]
+    }
+
+    /// Claim input before format detection (for formats [`Format`] has no
+    /// variant for). Inputs are untrusted; keep this cheap.
+    fn claims(&self, _bytes: &[u8], _hint: DocumentHint<'_>) -> bool {
+        false
+    }
+
+    /// Parse `bytes` (already within `config.max_input_size`) into the IR.
+    fn parse(&self, bytes: &[u8], config: &MarkkitBuilder) -> crate::Result<ParsedDocument>;
+}
+
+/// Ordered list of custom parsers held by [`MarkkitBuilder`].
+#[derive(Clone, Default)]
+pub struct ParserRegistry(Vec<Arc<dyn FormatParser>>);
+
+impl ParserRegistry {
+    /// Register a parser (lower priority than those registered earlier).
+    pub fn push(&mut self, p: impl FormatParser + 'static) {
+        self.0.push(Arc::new(p));
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    /// Names of the parsers, in priority order.
+    pub fn names(&self) -> impl Iterator<Item = &str> {
+        self.0.iter().map(|p| p.name())
+    }
+
+    /// Parser that claims the input, else the first one for `detected`.
+    pub(crate) fn select(
+        &self,
+        bytes: &[u8],
+        hint: DocumentHint<'_>,
+        detected: Format,
+    ) -> Option<&dyn FormatParser> {
+        self.0
+            .iter()
+            .find(|p| p.claims(bytes, hint))
+            .or_else(|| self.0.iter().find(|p| p.formats().contains(&detected)))
+            .map(|p| p.as_ref())
+    }
+}
+
+impl fmt::Debug for ParserRegistry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.names()).finish()
+    }
+}

crates/markkit/tests/registry.rs

@@ -0,0 +1,201 @@
+//! Custom FormatParser registry.
+
+use markkit::ir::{Document, Node, TableCell};
+use markkit::registry::FormatParser;
+use markkit::{DocumentHint, Error, Format, Markkit, MarkkitBuilder, ParsedDocument};
+
+/// Naive CSV: comma-split lines, first line is the header (replaces the built-in).
+struct Csv;
+
+impl FormatParser for Csv {
+    fn name(&self) -> &str {
+        "csv"
+    }
+    fn formats(&self) -> &[Format] {
+        &[Format::Csv]
+    }
+    fn parse(&self, bytes: &[u8], _: &MarkkitBuilder) -> markkit::Result<ParsedDocument> {
+        let text = String::from_utf8_lossy(bytes);
+        let mut rows = text.lines().map(|l| {
+            l.split(',')
+                .map(|c| TableCell::new(vec![Node::Text(c.trim().into())]))
+                .collect::<Vec<_>>()
+        });
+        let headers = rows.next().unwrap_or_default();
+        let table = Node::Table {
+            alignments: vec![],
+            headers,
+            rows: rows.collect(),
+            source: None,
+        };
+        Ok(ParsedDocument::new(Document::new(vec![table]), Format::Csv))
+    }
+}
+
+/// Claims inputs starting with a magic prefix.
+struct Magic;
+
+/// Replaces the built-in HTML parser.
+struct PlainHtml;
+
+impl FormatParser for PlainHtml {
+    fn name(&self) -> &str {
+        "plain-html"
+    }
+    fn formats(&self) -> &[Format] {
+        &[Format::Html]
+    }
+    fn parse(&self, bytes: &[u8], _: &MarkkitBuilder) -> markkit::Result<ParsedDocument> {
+        let text = String::from_utf8_lossy(bytes).into_owned();
+        Ok(ParsedDocument::new(
+            Document::new(vec![Node::CodeBlock {
+                language: Some("html".into()),
+                code: text,
+                source: None,
+            }]),
+            Format::Html,
+        ))
+    }
+}
+
+impl FormatParser for Magic {
+    fn name(&self) -> &str {
+        "magic"
+    }
+    fn claims(&self, bytes: &[u8], _: DocumentHint<'_>) -> bool {
+        bytes.starts_with(b"MAGIC1\n")
+    }
+    fn parse(&self, bytes: &[u8], _: &MarkkitBuilder) -> markkit::Result<ParsedDocument> {
+        let body = String::from_utf8_lossy(&bytes[7..]).into_owned();
+        Ok(ParsedDocument::new(
+            Document::new(vec![Node::heading(3, vec![Node::Text(body)])]),
+            Format::Unknown,
+        ))
+    }
+}
+
+struct Panics;
+
+impl FormatParser for Panics {
+    fn name(&self) -> &str {
+        "panics"
+    }
+    fn formats(&self) -> &[Format] {
+        &[Format::Markdown]
+    }
+    fn parse(&self, _: &[u8], _: &MarkkitBuilder) -> markkit::Result<ParsedDocument> {
+        panic!("boom")
+    }
+}
+
+#[test]
+fn custom_parser_for_detected_format() {
+    let m = Markkit::builder().parser(Csv).build();
+    let e = m
+        .extract_document_sync(b"a,b\n1,2\n", DocumentHint::file_name("data.csv"))
+        .unwrap();
+    assert_eq!(e.format, Format::Csv);
+    assert_eq!(e.markdown, "| a | b |\n| --- | --- |\n| 1 | 2 |\n");
+}
+
+#[test]
+fn unregistered_format_still_unsupported() {
+    // CSV parser registered; PPTX has no built-in and no custom parser.
+    let m = Markkit::builder().parser(Csv).build();
+    let err = m
+        .extract_document_sync(b"PK\x03\x04", DocumentHint::file_name("deck.pptx"))
+        .unwrap_err();
+    assert!(matches!(err, Error::UnsupportedFormat(_)), "{err}");
+}
+
+#[test]
+fn claims_run_before_detection_and_transforms_apply() {
+    let m = Markkit::builder()
+        .parser(Csv)
+        .parser(Magic)
+        .transform(markkit::transform::NormalizeHeadings)
+        .build();
+    // A .csv name, but Magic claims the bytes first.
+    let e = m
+        .extract_document_sync(b"MAGIC1\nhello", DocumentHint::file_name("x.csv"))
+        .unwrap();
+    assert_eq!(e.markdown, "# hello\n");
+    assert_eq!(format!("{:?}", m.config().parsers), r#"["csv", "magic"]"#);
+}
+
+#[cfg(feature = "html")]
+#[test]
+fn builtin_runs_when_nothing_claims() {
+    let m = Markkit::builder().parser(Magic).build();
+    // Without a claim the built-in HTML parser still runs.
+    let e = m
+        .extract_document_sync(b"<p>x</p>", DocumentHint::content_type("text/html"))
+        .unwrap();
+    assert_eq!(e.format, Format::Html);
+}
+
+#[test]
+fn panicking_parser_is_an_error() {
+    let m = Markkit::builder().parser(Panics).build();
+    let err = m
+        .extract_document_sync(b"# x", DocumentHint::file_name("a.md"))
+        .unwrap_err();
+    assert!(
+        matches!(err, Error::Parser(ref s) if s.contains("panics")),
+        "{err}"
+    );
+}
+
+#[test]
+fn size_limit_checked_before_custom_parser() {
+    let m = Markkit::builder().max_input_size(4).parser(Csv).build();
+    let err = m
+        .extract_document_sync(b"a,b,c,d", DocumentHint::file_name("x.csv"))
+        .unwrap_err();
+    assert!(matches!(err, Error::ResourceLimit(_)), "{err}");
+}
+
+#[test]
+fn custom_parser_replaces_builtin() {
+    let m = Markkit::builder().parser(PlainHtml).build();
+    let e = m
+        .extract_document_sync(b"<p>x</p>", DocumentHint::content_type("text/html"))
+        .unwrap();
+    assert_eq!(e.markdown, "```html\n<p>x</p>\n```\n");
+}
+
+/// Sleeps past any small deadline.
+struct Slow;
+
+impl FormatParser for Slow {
+    fn name(&self) -> &str {
+        "slow"
+    }
+    fn formats(&self) -> &[Format] {
+        &[Format::Csv]
+    }
+    fn parse(&self, _: &[u8], _: &MarkkitBuilder) -> markkit::Result<ParsedDocument> {
+        std::thread::sleep(std::time::Duration::from_millis(50));
+        Ok(ParsedDocument::new(Document::new(vec![]), Format::Csv))
+    }
+}
+
+#[test]
+fn deadline_checked_after_custom_parser() {
+    let limits = markkit::Limits::default().timeout(std::time::Duration::from_millis(1));
+    let m = Markkit::builder().limits(limits).parser(Slow).build();
+    let err = m
+        .extract_document_sync(b"a", DocumentHint::file_name("x.csv"))
+        .unwrap_err();
+    assert!(matches!(err, Error::ResourceLimit(_)), "{err}");
+}
+
+#[test]
+fn output_limit_applies_to_custom_parser() {
+    let limits = markkit::Limits::default().max_output_size(8);
+    let m = Markkit::builder().limits(limits).parser(Csv).build();
+    let err = m
+        .extract_document_sync(b"aaaa,bbbb\ncccc,dddd", DocumentHint::file_name("x.csv"))
+        .unwrap_err();
+    assert!(matches!(err, Error::ResourceLimit(_)), "{err}");
+}