From 30d3f30235bd9504b7f4f8a65bdfce21aa253a69 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:35:37 +0000 Subject: [PATCH 01/11] feat(bench): implement markkit-bench harness with stored results Measures per-case latency, ops/s, throughput and heap allocations (count, bytes, peak) plus process peak RSS over a generated HTML/DOCX/XLSX/ODS corpus. just bench saves JSON + Markdown reports to crates/markkit-bench/results/, mirroring bashkit-bench. Criterion moves to just bench-criterion. --- AGENTS.md | 6 +- CHANGELOG.md | 8 +- CONTRIBUTING.md | 2 +- Cargo.lock | 5 + README.md | 2 +- crates/markkit-bench/Cargo.toml | 14 +- crates/markkit-bench/README.md | 28 ++ crates/markkit-bench/src/alloc.rs | 106 +++++ crates/markkit-bench/src/cases.rs | 411 ++++++++++++++++++ crates/markkit-bench/src/main.rs | 214 +++++++++- crates/markkit-bench/src/report.rs | 444 ++++++++++++++++++++ justfile | 22 +- knowledge/foundations/architecture.md | 2 +- knowledge/log.md | 1 + knowledge/operations/index.md | 1 + knowledge/operations/performance-results.md | 79 ++++ knowledge/operations/testing.md | 8 +- 17 files changed, 1332 insertions(+), 21 deletions(-) create mode 100644 crates/markkit-bench/README.md create mode 100644 crates/markkit-bench/src/alloc.rs create mode 100644 crates/markkit-bench/src/cases.rs create mode 100644 crates/markkit-bench/src/report.rs create mode 100644 knowledge/operations/performance-results.md diff --git a/AGENTS.md b/AGENTS.md index 9204780..c240825 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | operations/examples | `examples/` contents and requirements | | operations/maintenance | Maintenance invocation contract + checklist + workflow credential audit | | operations/release-process | Agent-driven release, crates.io + PyPI + npm | +| operations/performance-results | Benchmark harness, metrics, committed results | | operations/eval | LLM-as-judge eval design (harness is a stub) | ### Documentation @@ -67,7 +68,8 @@ just lint # clippy -D warnings just check # fmt + clippy + test + knowledge + scripts + examples just check-okf # knowledge/ OKF conformance just examples # Run every example -just bench # Criterion benches (crates/markkit/benches) +just bench # Benchmark harness, saves results to crates/markkit-bench/results +just bench-criterion # Criterion micro-benches (crates/markkit/benches) just pre-pr # Pre-PR checks ``` @@ -77,7 +79,7 @@ just pre-pr # Pre-PR checks |-------|---------| | `markkit` | Core extraction library (HTML, DOCX, spreadsheets) | | `markkit-cli` | `markkit` CLI | -| `markkit-bench` | Benchmark harness (stub; real benches are criterion in `markkit`) | +| `markkit-bench` | Benchmark harness: ops/s, throughput, allocations; results committed | | `markkit-eval` | LLM evaluation harness (stub) | | `markkit-python` | Python bindings (PyO3), module `markkit` | | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..c096d98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/). fetch by `max_processing_secs`. New builder options `allow_private_networks(true)` and `allow_host(..)`; refusals return the new `Error::UrlBlocked`. +### Added + +- `markkit-bench` harness (`just bench`): per-case latency, ops/s, throughput, heap allocations and + peak heap, plus process peak RSS, over a generated HTML/DOCX/XLSX/ODS corpus. Runs are saved as + JSON + Markdown in `crates/markkit-bench/results/` and committed. + ### Changed - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. - CI workflows declare read-only default permissions and do not persist checkout credentials. -- CI runs every example; `just bench` now runs the criterion benchmarks. +- CI runs every example; criterion benchmarks run with `just bench-criterion`. - README, SECURITY and binding docs corrected: Python module is `markkit`. - Dependencies: drop unused `async-trait`, `futures`, `regex`, `unicode-segmentation`, `encoding_rs`, `base64`, `mime`, `lol_html` (core) and `tracing-subscriber`, plus unused deps of the CLI, bindings diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c1f58f..49c9cfb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ We use [Conventional Commits](https://www.conventionalcommits.org/): - Write failing tests before fixing bugs - Cover positive and negative scenarios - Security tests for input parsing changes -- Run `just bench` for performance-sensitive changes +- Run `just bench` for performance-sensitive changes and commit the saved results - Run `just examples` when the public API changes ## Knowledge diff --git a/Cargo.lock b/Cargo.lock index 01303b3..a0f7354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1206,6 +1206,11 @@ version = "0.1.1" dependencies = [ "anyhow", "clap", + "markkit", + "serde", + "serde_json", + "tempfile", + "zip", ] [[package]] diff --git a/README.md b/README.md index 0d9e55a..505b07e 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ just test # Run all tests just lint # Run clippy just check # fmt, clippy, tests, knowledge, examples just pre-pr # Full pre-PR validation -just bench # Criterion benchmarks +just bench # Benchmarks, results saved to crates/markkit-bench/results ``` Coding agents: start with [AGENTS.md](AGENTS.md). diff --git a/crates/markkit-bench/Cargo.toml b/crates/markkit-bench/Cargo.toml index 45f7ed3..b220c0f 100644 --- a/crates/markkit-bench/Cargo.toml +++ b/crates/markkit-bench/Cargo.toml @@ -1,4 +1,7 @@ -# Benchmark tool for measuring markkit extraction performance +# Benchmark harness for markkit extraction. +# Measures latency, ops/sec, throughput and heap allocations per case and saves +# JSON + Markdown reports to results/ (committed, like bashkit-bench). +# Usage: cargo run -p markkit-bench --release -- [options] [package] name = "markkit-bench" @@ -7,11 +10,20 @@ edition.workspace = true license.workspace = true authors.workspace = true description = "Benchmark tool for markkit markdown extraction" +publish = false [[bin]] name = "markkit-bench" path = "src/main.rs" [dependencies] +# No http_client: the harness is offline and must not link reqwest/hyper. +markkit = { path = "../markkit", default-features = false, features = ["html", "docx", "spreadsheet"] } +zip = { workspace = true } +serde.workspace = true +serde_json.workspace = true clap.workspace = true anyhow.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/markkit-bench/README.md b/crates/markkit-bench/README.md new file mode 100644 index 0000000..2f499b6 --- /dev/null +++ b/crates/markkit-bench/README.md @@ -0,0 +1,28 @@ +# markkit-bench + +End-to-end benchmark harness for markkit extraction. Runs in-process over a +generated corpus (HTML, DOCX, XLSX, ODS, format detection) and reports, per case: + +- latency: mean, median, stddev, min, max +- ops/sec and input throughput (MiB/s) +- heap allocations per operation: count, bytes requested, peak live heap +- process peak RSS (Linux) + +## Usage + +```bash +just bench # run + save to results/ +just bench-accurate # 100 iterations, 10 warmup, save +just bench-quick --category html # explore, nothing saved +just bench-list # list cases +``` + +Or directly: `cargo run -p markkit-bench --release -- --help`. + +## Results + +Saved runs live in [`results/`](results/) as +`bench--.{json,md}` and are committed. Compare runs from the +same moniker only. Commit a fresh run with performance-relevant PRs. + +Design and metric definitions: `knowledge/operations/performance-results.md`. diff --git a/crates/markkit-bench/src/alloc.rs b/crates/markkit-bench/src/alloc.rs new file mode 100644 index 0000000..9c5203f --- /dev/null +++ b/crates/markkit-bench/src/alloc.rs @@ -0,0 +1,106 @@ +//! Counting global allocator. +//! +//! Wraps the system allocator and tracks allocation count, bytes requested and +//! live/peak heap bytes with relaxed atomics. Overhead is a few atomic adds per +//! call, so timings stay comparable; allocation stats are still collected on a +//! separate pass (see `runner`) so they never skew the timed iterations. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +pub struct CountingAlloc; + +static ALLOCS: AtomicU64 = AtomicU64::new(0); +static BYTES: AtomicU64 = AtomicU64::new(0); +static LIVE: AtomicU64 = AtomicU64::new(0); +static PEAK: AtomicU64 = AtomicU64::new(0); + +fn on_alloc(size: usize) { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(size as u64, Relaxed); + let live = LIVE.fetch_add(size as u64, Relaxed) + size as u64; + PEAK.fetch_max(live, Relaxed); +} + +fn on_dealloc(size: usize) { + LIVE.fetch_sub(size as u64, Relaxed); +} + +// SAFETY: forwards every call to `System` unchanged; only bookkeeping is added. +unsafe impl GlobalAlloc for CountingAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + on_alloc(layout.size()); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + on_alloc(layout.size()); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + on_dealloc(layout.size()); + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let p = unsafe { System.realloc(ptr, layout, new_size) }; + if !p.is_null() { + // A realloc counts as one allocation of the new size. + on_dealloc(layout.size()); + on_alloc(new_size); + } + p + } +} + +/// Heap activity between [`Snapshot::start`] and [`Snapshot::finish`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AllocStats { + pub allocations: u64, + pub allocated_bytes: u64, + /// Peak live heap above the level at `start`. + pub peak_bytes: u64, +} + +pub struct Snapshot { + allocs: u64, + bytes: u64, + live: u64, +} + +impl Snapshot { + /// Start a measurement window. Resets the peak to the current live level, + /// so windows must not overlap (the harness is single-threaded). + pub fn start() -> Self { + let live = LIVE.load(Relaxed); + PEAK.store(live, Relaxed); + Self { + allocs: ALLOCS.load(Relaxed), + bytes: BYTES.load(Relaxed), + live, + } + } + + pub fn finish(self) -> AllocStats { + AllocStats { + allocations: ALLOCS.load(Relaxed) - self.allocs, + allocated_bytes: BYTES.load(Relaxed) - self.bytes, + peak_bytes: PEAK.load(Relaxed).saturating_sub(self.live), + } + } +} + +/// Process peak resident set size in bytes (Linux `VmHWM`), if available. +pub fn peak_rss_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|l| l.starts_with("VmHWM:"))?; + let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb * 1024) +} diff --git a/crates/markkit-bench/src/cases.rs b/crates/markkit-bench/src/cases.rs new file mode 100644 index 0000000..38a1d40 --- /dev/null +++ b/crates/markkit-bench/src/cases.rs @@ -0,0 +1,411 @@ +//! Benchmark corpus. +//! +//! Inputs are generated deterministically in code (no network, no committed +//! binaries) so every run on every machine measures the same bytes. Sizes stay +//! under the default builder limits: the harness measures the happy path, the +//! security limits are covered by tests. + +use std::io::{Cursor, Write}; + +use markkit::{DocumentHint, Format, Markkit}; +use zip::write::{SimpleFileOptions, ZipWriter}; + +/// What a case feeds to markkit. +pub enum Input { + /// `Markkit::extract_html_sync`. + Html(String), + /// `Markkit::extract_document_sync` with a file-name hint. + Document { + bytes: Vec, + file_name: &'static str, + }, + /// `Format::detect` on raw bytes with no hint (magic-byte sniffing). + Detect(Vec), +} + +pub struct Case { + pub name: &'static str, + pub category: &'static str, + pub description: &'static str, + pub input: Input, +} + +impl Case { + pub fn input_bytes(&self) -> usize { + match &self.input { + Input::Html(s) => s.len(), + Input::Document { bytes, .. } | Input::Detect(bytes) => bytes.len(), + } + } + + /// Run one operation. Returns output size in bytes. + pub fn run(&self, mk: &Markkit) -> Result { + match &self.input { + Input::Html(html) => mk + .extract_html_sync(html) + .map(|md| md.len()) + .map_err(|e| e.to_string()), + Input::Document { bytes, file_name } => mk + .extract_document_sync( + bytes, + DocumentHint { + content_type: None, + file_name: Some(file_name), + }, + ) + .map(|e| e.markdown.len()) + .map_err(|e| e.to_string()), + Input::Detect(bytes) => match Format::detect(bytes, DocumentHint::default()) { + Format::Unknown => Err("format not detected".into()), + _ => Ok(0), + }, + } + } +} + +pub const CATEGORIES: &[&str] = &["html", "docx", "spreadsheet", "detect"]; + +pub fn all() -> Vec { + let docx_large = docx_report(400); + vec![ + Case { + name: "html_fixture_article", + category: "html", + description: "Golden fixture full_article.html", + input: Input::Html( + include_str!("../../markkit/tests/fixtures/html/full_article.html").into(), + ), + }, + Case { + name: "html_blog_post", + category: "html", + description: "Article with nav/footer/scripts, 20 sections", + input: Input::Html(html_page(20, true)), + }, + Case { + name: "html_docs_page", + category: "html", + description: "Long docs page, 400 sections, no chrome", + input: Input::Html(html_page(400, false)), + }, + Case { + name: "html_tables", + category: "html", + description: "50 tables x 40 rows x 6 cols", + input: Input::Html(html_tables(50, 40, 6)), + }, + Case { + name: "html_boilerplate", + category: "html", + description: "Mostly script/style/nav to strip, little content", + input: Input::Html(html_boilerplate(400)), + }, + Case { + name: "html_deep_nesting", + category: "html", + description: "Divs nested past the depth limit (1000 levels)", + input: Input::Html(format!( + "{}

deep

{}", + "
".repeat(1000), + "
".repeat(1000) + )), + }, + Case { + name: "docx_memo", + category: "docx", + description: "Short memo: heading, 10 paragraphs, 1 list", + input: Input::Document { + bytes: docx_report(1), + file_name: "memo.docx", + }, + }, + Case { + name: "docx_report", + category: "docx", + description: "Report: 400 sections with lists and tables", + input: Input::Document { + bytes: docx_large.clone(), + file_name: "report.docx", + }, + }, + Case { + name: "xlsx_small", + category: "spreadsheet", + description: "1 sheet, 50 rows x 5 cols", + input: Input::Document { + bytes: xlsx(1, 50, 5), + file_name: "small.xlsx", + }, + }, + Case { + name: "xlsx_large", + category: "spreadsheet", + description: "3 sheets, 5000 rows x 10 cols each (150k cells)", + input: Input::Document { + bytes: xlsx(3, 5000, 10), + file_name: "large.xlsx", + }, + }, + Case { + name: "ods_medium", + category: "spreadsheet", + description: "1 sheet, 2000 rows x 8 cols", + input: Input::Document { + bytes: ods(2000, 8), + file_name: "medium.ods", + }, + }, + Case { + name: "detect_docx_magic", + category: "detect", + description: "Sniff DOCX from bytes without a hint", + input: Input::Detect(docx_large), + }, + ] +} + +// ─── HTML ─────────────────────────────────────────────────────────── + +const LOREM: &str = "Markkit turns documents into clean markdown for language models. \ +It keeps structure, drops noise, and links to \ +reference docs stay intact."; + +fn html_page(sections: usize, chrome: bool) -> String { + let mut s = String::from("Bench page"); + if chrome { + s.push_str(""); + s.push_str(""); + } + s.push_str(""); + if chrome { + s.push_str(""); + } + s.push_str("

Benchmark document

"); + for i in 0..sections { + s.push_str(&format!("

Section {i}

")); + for _ in 0..3 { + s.push_str(&format!("

{LOREM}

")); + } + s.push_str( + "
  • First point
  • Second inline()
  • Third
", + ); + if i % 4 == 0 { + s.push_str("
fn main() {\n    println!(\"hi\");\n}
"); + } + if i % 5 == 0 { + s.push_str("

A quoted remark.

"); + s.push_str("\"Chart\""); + } + } + s.push_str("
"); + if chrome { + s.push_str("

Copyright

"); + } + s.push_str(""); + s +} + +fn html_tables(tables: usize, rows: usize, cols: usize) -> String { + let mut s = String::from(""); + for t in 0..tables { + s.push_str(&format!("

Table {t}

")); + for c in 0..cols { + s.push_str(&format!("")); + } + s.push_str(""); + for r in 0..rows { + s.push_str(""); + for c in 0..cols { + s.push_str(&format!("")); + } + s.push_str(""); + } + s.push_str("
Col {c}
r{r}c{c} value
"); + } + s.push_str(""); + s +} + +fn html_boilerplate(blocks: usize) -> String { + let mut s = String::from(""); + for i in 0..blocks { + s.push_str(&format!( + "" + )); + s.push_str(&format!( + "" + )); + } + s.push_str("

Title

"); + s.push_str(&format!("

{LOREM}

")); + s.push_str("
Footer
"); + s +} + +// ─── Office ───────────────────────────────────────────────────────── + +fn zip_bytes(entries: &[(&str, &[u8])]) -> Vec { + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + for (name, data) in entries { + let opts = if *name == "mimetype" { + SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored) + } else { + SimpleFileOptions::default() + }; + w.start_file(*name, opts).expect("zip entry"); + w.write_all(data).expect("zip write"); + } + w.finish().expect("zip finish").into_inner() +} + +const W_NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships""#; + +const NUMBERING: &str = r#""#; + +fn docx_report(sections: usize) -> Vec { + let mut body = String::new(); + for i in 0..sections { + body.push_str(&format!( + "Section {i}" + )); + for _ in 0..10 { + body.push_str( + "Plain text paragraph with \ + bold\ + and more words to read.", + ); + } + for item in ["Alpha", "Beta", "Gamma"] { + body.push_str(&format!( + "{item}" + )); + } + if sections > 1 && i % 10 == 0 { + body.push_str(""); + for r in 0..10 { + body.push_str(""); + for c in 0..4 { + body.push_str(&format!( + "r{r}c{c}" + )); + } + body.push_str(""); + } + body.push_str(""); + } + } + let document = format!( + r#"{body}"# + ); + let numbering = + format!(r#"{NUMBERING}"#); + zip_bytes(&[ + ( + "[Content_Types].xml", + br#""#, + ), + ( + "_rels/.rels", + br#""#, + ), + ("word/document.xml", document.as_bytes()), + ("word/numbering.xml", numbering.as_bytes()), + ]) +} + +fn col_name(mut c: usize) -> String { + let mut s = Vec::new(); + loop { + s.push(b'A' + (c % 26) as u8); + if c < 26 { + break; + } + c = c / 26 - 1; + } + s.reverse(); + String::from_utf8(s).expect("ascii") +} + +fn xlsx(sheets: usize, rows: usize, cols: usize) -> Vec { + let mut ct = String::from( + r#""#, + ); + let mut wb_sheets = String::new(); + let mut wb_rels = String::new(); + let mut parts: Vec<(String, Vec)> = Vec::new(); + for n in 1..=sheets { + ct.push_str(&format!(r#""#)); + wb_sheets.push_str(&format!( + r#""# + )); + wb_rels.push_str(&format!(r#""#)); + let mut data = String::new(); + for r in 0..rows { + data.push_str(&format!(r#""#, r + 1)); + for c in 0..cols { + let at = format!("{}{}", col_name(c), r + 1); + if r == 0 || c % 2 == 0 { + data.push_str(&format!( + r#"item {r}-{c}"# + )); + } else { + data.push_str(&format!(r#"{}.25"#, r * c)); + } + } + data.push_str(""); + } + let xml = format!( + r#"{data}"# + ); + parts.push((format!("xl/worksheets/sheet{n}.xml"), xml.into_bytes())); + } + ct.push_str(""); + let workbook = format!( + r#"{wb_sheets}"# + ); + let rels = format!( + r#"{wb_rels}"# + ); + let root_rels = br#""#; + let mut entries: Vec<(&str, &[u8])> = vec![ + ("[Content_Types].xml", ct.as_bytes()), + ("_rels/.rels", root_rels), + ("xl/workbook.xml", workbook.as_bytes()), + ("xl/_rels/workbook.xml.rels", rels.as_bytes()), + ]; + entries.extend(parts.iter().map(|(n, d)| (n.as_str(), d.as_slice()))); + zip_bytes(&entries) +} + +fn ods(rows: usize, cols: usize) -> Vec { + let mut table = String::from(r#""#); + for r in 0..rows { + table.push_str(""); + for c in 0..cols { + if r == 0 || c % 2 == 0 { + table.push_str(&format!( + r#"item {r}-{c}"# + )); + } else { + let v = r * c; + table.push_str(&format!( + r#"{v}"# + )); + } + } + table.push_str(""); + } + table.push_str(""); + let content = format!( + r#"{table}"# + ); + let manifest = br#""#; + zip_bytes(&[ + ( + "mimetype", + b"application/vnd.oasis.opendocument.spreadsheet", + ), + ("content.xml", content.as_bytes()), + ("META-INF/manifest.xml", manifest), + ]) +} diff --git a/crates/markkit-bench/src/main.rs b/crates/markkit-bench/src/main.rs index 91f4396..ff9c7c0 100644 --- a/crates/markkit-bench/src/main.rs +++ b/crates/markkit-bench/src/main.rs @@ -1,21 +1,221 @@ -use anyhow::Result; +//! markkit-bench: extraction benchmark harness. +//! +//! Decisions: +//! - Mirrors everruns/bashkit's `bashkit-bench`: in-process runs, warmup then +//! timed iterations, JSON + Markdown reports saved as +//! `results/bench--.{json,md}` and committed, so history +//! lives in git next to the code that produced it. +//! - Adds what bashkit lacks: heap allocations per op (count, bytes, peak) from +//! a counting global allocator, measured on a separate untimed pass, plus +//! process peak RSS. +//! - Criterion (`crates/markkit/benches`) stays for statistically rigorous +//! micro-benchmarks; this harness is the end-to-end, comparable-over-time view. +//! - Corpus is generated in code (`cases.rs`), offline, deterministic. + +mod alloc; +mod cases; +mod report; + +use std::path::PathBuf; +use std::time::Instant; + +use anyhow::{Result, bail}; use clap::Parser; +use crate::cases::Case; +use crate::report::{CaseResult, Report, SystemInfo}; + +#[global_allocator] +static GLOBAL: alloc::CountingAlloc = alloc::CountingAlloc; + #[derive(Parser)] #[command(name = "markkit-bench", about = "Benchmark markkit extraction")] struct Cli { - /// Number of iterations per benchmark - #[arg(long, default_value = "100")] + /// Timed iterations per case + #[arg(long, default_value_t = 20)] iterations: usize, - /// Category to benchmark (html, pdf, docx, all) + /// Untimed warmup iterations per case + #[arg(long, default_value_t = 3)] + warmup: usize, + + /// Category to run (html, docx, spreadsheet, detect, all) #[arg(long, default_value = "all")] category: String, + + /// Only run cases whose name contains this substring + #[arg(long)] + filter: Option, + + /// List cases and exit + #[arg(long)] + list: bool, + + /// Save JSON + Markdown reports + #[arg(long)] + save: bool, + + /// Directory for saved reports + #[arg(long, default_value = concat!(env!("CARGO_MANIFEST_DIR"), "/results"))] + output_dir: PathBuf, + + /// Machine label used in file names (default: --) + #[arg(long)] + moniker: Option, } fn main() -> Result<()> { - let _cli = Cli::parse(); - // TODO: Implement benchmark harness - println!("markkit-bench: benchmark harness not yet implemented"); + let cli = Cli::parse(); + if cli.iterations == 0 { + bail!("--iterations must be at least 1"); + } + if cli.category != "all" && !cases::CATEGORIES.contains(&cli.category.as_str()) { + bail!( + "unknown category '{}', expected one of: all, {}", + cli.category, + cases::CATEGORIES.join(", ") + ); + } + + let selected = select(cases::all(), &cli.category, cli.filter.as_deref()); + if cli.list { + for c in &selected { + println!( + "{:<22} {:<12} {:>10} B {}", + c.name, + c.category, + c.input_bytes(), + c.description + ); + } + return Ok(()); + } + if selected.is_empty() { + bail!("no cases match the given category/filter"); + } + + let mk = markkit::Markkit::builder().build(); + let mut results = Vec::with_capacity(selected.len()); + for case in &selected { + let r = run_case(&mk, case, cli.warmup, cli.iterations); + eprintln!("{}", report::progress_line(&r)); + results.push(r); + } + + let system = SystemInfo::collect(cli.moniker.as_deref()); + let report = Report::new(system, cli.iterations, cli.warmup, results); + println!("{}", report.to_markdown()); + + if cli.save { + let (json, md) = report.save(&cli.output_dir)?; + eprintln!("saved {}", json.display()); + eprintln!("saved {}", md.display()); + } + if report.results.iter().any(|r| r.errors > 0) { + bail!("one or more cases failed"); + } Ok(()) } + +fn select(all: Vec, category: &str, filter: Option<&str>) -> Vec { + all.into_iter() + .filter(|c| category == "all" || c.category == category) + .filter(|c| filter.is_none_or(|f| c.name.contains(f))) + .collect() +} + +/// Warmup, timed iterations, then one untimed pass under the allocation counter. +fn run_case(mk: &markkit::Markkit, case: &Case, warmup: usize, iterations: usize) -> CaseResult { + let mut errors = Vec::new(); + for _ in 0..warmup { + if let Err(e) = case.run(mk) { + errors.push(e); + } + } + + let mut times_ns = Vec::with_capacity(iterations); + let mut output_bytes = 0; + for _ in 0..iterations { + let start = Instant::now(); + let out = case.run(mk); + times_ns.push(start.elapsed().as_nanos() as u64); + match out { + Ok(n) => output_bytes = n, + Err(e) => errors.push(e), + } + } + + let snap = alloc::Snapshot::start(); + let out = case.run(mk); + let allocs = snap.finish(); + if let Err(e) = out { + errors.push(e); + } + + errors.dedup(); + CaseResult::new(case, times_ns, output_bytes, allocs, errors) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_case_runs_without_errors() { + let mk = markkit::Markkit::builder().build(); + for case in cases::all() { + let r = run_case(&mk, &case, 0, 1); + assert_eq!(r.errors, 0, "{}: {:?}", case.name, r.error_messages); + if case.category != "detect" { + assert!(r.output_bytes > 0, "{} produced no output", case.name); + } + assert!(r.allocations > 0 || case.category == "detect"); + } + } + + #[test] + fn case_names_are_unique_and_categories_known() { + let all = cases::all(); + let mut names: Vec<_> = all.iter().map(|c| c.name).collect(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), all.len()); + assert!(all.iter().all(|c| cases::CATEGORIES.contains(&c.category))); + } + + #[test] + fn select_filters_by_category_and_name() { + let html = select(cases::all(), "html", None); + assert!(!html.is_empty()); + assert!(html.iter().all(|c| c.category == "html")); + let one = select(cases::all(), "all", Some("xlsx_small")); + assert_eq!(one.len(), 1); + assert!(select(cases::all(), "docx", Some("xlsx")).is_empty()); + } + + #[test] + fn failing_case_is_reported_not_panicking() { + let case = Case { + name: "bad", + category: "detect", + description: "", + input: cases::Input::Detect(b"\x00\x01".to_vec()), + }; + let mk = markkit::Markkit::builder().build(); + let r = run_case(&mk, &case, 1, 2); + assert!(r.errors > 0); + assert_eq!(r.error_messages, vec!["format not detected".to_string()]); + } + + #[test] + fn alloc_counter_sees_heap_activity() { + let snap = alloc::Snapshot::start(); + let v: Vec = std::hint::black_box(vec![0u8; 4096]); + let stats = snap.finish(); + drop(v); + assert!(stats.allocations >= 1); + // Counters are process-wide and tests run in parallel, so only the + // monotonic counters are asserted. + assert!(stats.allocated_bytes >= 4096); + } +} diff --git a/crates/markkit-bench/src/report.rs b/crates/markkit-bench/src/report.rs new file mode 100644 index 0000000..820fdfd --- /dev/null +++ b/crates/markkit-bench/src/report.rs @@ -0,0 +1,444 @@ +//! Result statistics, system info, and JSON / Markdown reports. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde::Serialize; + +use crate::alloc::{self, AllocStats}; +use crate::cases::{CATEGORIES, Case}; + +#[derive(Debug, Serialize)] +pub struct CaseResult { + pub name: String, + pub category: String, + pub description: String, + pub iterations: usize, + pub input_bytes: usize, + pub output_bytes: usize, + pub times_ns: Vec, + pub mean_ns: f64, + pub median_ns: f64, + pub stddev_ns: f64, + pub min_ns: u64, + pub max_ns: u64, + /// Operations per second, from the mean. + pub ops_per_sec: f64, + /// Input MiB processed per second, from the mean. + pub throughput_mib_s: f64, + /// Heap allocations in one operation. + pub allocations: u64, + /// Heap bytes requested in one operation. + pub allocated_bytes: u64, + /// Peak live heap bytes during one operation. + pub peak_heap_bytes: u64, + pub errors: usize, + pub error_messages: Vec, +} + +impl CaseResult { + pub fn new( + case: &Case, + times_ns: Vec, + output_bytes: usize, + allocs: AllocStats, + error_messages: Vec, + ) -> Self { + let s = Stats::of(×_ns); + let input_bytes = case.input_bytes(); + let secs = s.mean / 1e9; + Self { + name: case.name.into(), + category: case.category.into(), + description: case.description.into(), + iterations: times_ns.len(), + input_bytes, + output_bytes, + mean_ns: s.mean, + median_ns: s.median, + stddev_ns: s.stddev, + min_ns: s.min, + max_ns: s.max, + ops_per_sec: if secs > 0.0 { 1.0 / secs } else { 0.0 }, + throughput_mib_s: if secs > 0.0 { + input_bytes as f64 / MIB / secs + } else { + 0.0 + }, + allocations: allocs.allocations, + allocated_bytes: allocs.allocated_bytes, + peak_heap_bytes: allocs.peak_bytes, + errors: error_messages.len(), + error_messages, + times_ns, + } + } +} + +const MIB: f64 = 1024.0 * 1024.0; + +#[derive(Debug, PartialEq)] +pub struct Stats { + pub mean: f64, + pub median: f64, + pub stddev: f64, + pub min: u64, + pub max: u64, +} + +impl Stats { + pub fn of(samples: &[u64]) -> Self { + if samples.is_empty() { + return Self { + mean: 0.0, + median: 0.0, + stddev: 0.0, + min: 0, + max: 0, + }; + } + let n = samples.len() as f64; + let mean = samples.iter().map(|&x| x as f64).sum::() / n; + let var = samples + .iter() + .map(|&x| (x as f64 - mean).powi(2)) + .sum::() + / n; + let mut sorted = samples.to_vec(); + sorted.sort_unstable(); + let mid = sorted.len() / 2; + let median = if sorted.len().is_multiple_of(2) { + (sorted[mid - 1] + sorted[mid]) as f64 / 2.0 + } else { + sorted[mid] as f64 + }; + Self { + mean, + median, + stddev: var.sqrt(), + min: sorted[0], + max: sorted[sorted.len() - 1], + } + } +} + +#[derive(Debug, Serialize)] +pub struct SystemInfo { + pub moniker: String, + pub hostname: String, + pub os: String, + pub arch: String, + pub cpus: usize, + pub markkit_version: String, + pub git_commit: Option, + pub profile: String, +} + +impl SystemInfo { + pub fn collect(moniker: Option<&str>) -> Self { + let hostname = hostname(); + let os = std::env::consts::OS.to_string(); + let arch = std::env::consts::ARCH.to_string(); + let moniker = sanitize_moniker( + &moniker + .map(str::to_string) + .unwrap_or_else(|| format!("{hostname}-{os}-{arch}")), + ); + Self { + moniker, + hostname, + os, + arch, + cpus: std::thread::available_parallelism().map_or(0, |n| n.get()), + markkit_version: env!("CARGO_PKG_VERSION").into(), + git_commit: git_commit(), + profile: if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + .into(), + } + } +} + +fn hostname() -> String { + std::env::var("HOSTNAME") + .ok() + .or_else(|| std::fs::read_to_string("/etc/hostname").ok()) + .or_else(|| { + std::process::Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + }) + .map(|h| h.trim().to_string()) + .filter(|h| !h.is_empty()) + .unwrap_or_else(|| "unknown".into()) +} + +fn git_commit() -> Option { + let out = std::process::Command::new("git") + .args(["rev-parse", "--short=12", "HEAD"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .ok()?; + let sha = String::from_utf8(out.stdout).ok()?.trim().to_string(); + (out.status.success() && sha.chars().all(|c| c.is_ascii_hexdigit()) && !sha.is_empty()) + .then_some(sha) +} + +/// The moniker becomes part of a file name: keep `[A-Za-z0-9._-]`, map the rest +/// to `-`, never start with `.`, cap the length. Hostnames and `--moniker` are +/// untrusted, so `../x` or `a/b` must not escape the results directory. +pub fn sanitize_moniker(raw: &str) -> String { + let mut s: String = raw + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { + c + } else { + '-' + } + }) + .take(64) + .collect(); + while s.starts_with('.') { + s.replace_range(..1, "-"); + } + if s.is_empty() { "unknown".into() } else { s } +} + +#[derive(Debug, Serialize)] +pub struct Report { + pub timestamp: u64, + pub system: SystemInfo, + pub iterations: usize, + pub warmup: usize, + /// Process peak RSS after all cases (Linux only). + pub peak_rss_bytes: Option, + pub results: Vec, +} + +impl Report { + pub fn new( + system: SystemInfo, + iterations: usize, + warmup: usize, + results: Vec, + ) -> Self { + Self { + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()), + system, + iterations, + warmup, + peak_rss_bytes: alloc::peak_rss_bytes(), + results, + } + } + + pub fn base_name(&self) -> String { + format!("bench-{}-{}", self.system.moniker, self.timestamp) + } + + /// Write `/.json` and `.md`. + pub fn save(&self, dir: &Path) -> Result<(PathBuf, PathBuf)> { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + // Not `with_extension`: monikers may contain dots (`mac.local`). + let base = self.base_name(); + let json = dir.join(format!("{base}.json")); + let md = dir.join(format!("{base}.md")); + std::fs::write(&json, serde_json::to_string_pretty(self)? + "\n") + .with_context(|| format!("write {}", json.display()))?; + std::fs::write(&md, self.to_markdown()) + .with_context(|| format!("write {}", md.display()))?; + Ok((json, md)) + } + + pub fn to_markdown(&self) -> String { + let sys = &self.system; + let mut md = String::from("# Markkit Benchmark Report\n\n## System Information\n\n"); + md.push_str(&format!("- **Moniker**: `{}`\n", sys.moniker)); + md.push_str(&format!("- **OS / Arch**: {} / {}\n", sys.os, sys.arch)); + md.push_str(&format!("- **CPUs**: {}\n", sys.cpus)); + md.push_str(&format!("- **markkit**: {}", sys.markkit_version)); + if let Some(sha) = &sys.git_commit { + md.push_str(&format!(" (`{sha}`)")); + } + md.push('\n'); + md.push_str(&format!("- **Profile**: {}\n", sys.profile)); + md.push_str(&format!("- **Timestamp**: {}\n", self.timestamp)); + md.push_str(&format!( + "- **Iterations**: {} (warmup {})\n", + self.iterations, self.warmup + )); + if let Some(rss) = self.peak_rss_bytes { + md.push_str(&format!("- **Peak RSS**: {}\n", fmt_bytes(rss))); + } + + let errors: usize = self.results.iter().map(|r| r.errors).sum(); + let total_ms: f64 = self.results.iter().map(|r| r.mean_ns).sum::() / 1e6; + md.push_str(&format!( + "\n## Summary\n\n{} cases, {:.3} ms total mean time per pass, {} errors.\n\n", + self.results.len(), + total_ms, + errors + )); + md.push_str( + "Allocations, allocated and peak heap are per single operation. \ + Throughput is input bytes over mean time.\n", + ); + + for cat in CATEGORIES { + let rows: Vec<_> = self.results.iter().filter(|r| r.category == *cat).collect(); + if rows.is_empty() { + continue; + } + md.push_str(&format!("\n### {}\n\n", title(cat))); + md.push_str( + "| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors |\n", + ); + md.push_str( + "|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:|\n", + ); + for r in rows { + md.push_str(&format!( + "| {} | {} | {} | {} | {} | ±{} | {:.0} | {:.1} | {} | {} | {} | {} |\n", + r.name, + fmt_bytes(r.input_bytes as u64), + fmt_bytes(r.output_bytes as u64), + fmt_ns(r.mean_ns), + fmt_ns(r.median_ns), + fmt_ns(r.stddev_ns), + r.ops_per_sec, + r.throughput_mib_s, + r.allocations, + fmt_bytes(r.allocated_bytes), + fmt_bytes(r.peak_heap_bytes), + if r.errors == 0 { + "-".to_string() + } else { + r.errors.to_string() + }, + )); + } + } + md + } +} + +pub fn progress_line(r: &CaseResult) -> String { + format!( + "{:<22} {:>10} {:>10.0} ops/s {:>8} allocs {:>10} peak{}", + r.name, + fmt_ns(r.mean_ns), + r.ops_per_sec, + r.allocations, + fmt_bytes(r.peak_heap_bytes), + if r.errors > 0 { " ERROR" } else { "" } + ) +} + +fn title(cat: &str) -> String { + match cat { + "html" => "HTML".into(), + "docx" => "DOCX".into(), + other => { + let mut c = other.chars(); + c.next() + .map(|f| f.to_uppercase().chain(c).collect()) + .unwrap_or_default() + } + } +} + +pub fn fmt_ns(ns: f64) -> String { + if ns >= 1e9 { + format!("{:.2} s", ns / 1e9) + } else if ns >= 1e6 { + format!("{:.2} ms", ns / 1e6) + } else if ns >= 1e3 { + format!("{:.2} µs", ns / 1e3) + } else { + format!("{ns:.0} ns") + } +} + +pub fn fmt_bytes(b: u64) -> String { + let b = b as f64; + if b >= MIB { + format!("{:.2} MiB", b / MIB) + } else if b >= 1024.0 { + format!("{:.1} KiB", b / 1024.0) + } else { + format!("{b:.0} B") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stats_basic() { + let s = Stats::of(&[10, 20, 30, 40]); + assert_eq!(s.mean, 25.0); + assert_eq!(s.median, 25.0); + assert_eq!(s.min, 10); + assert_eq!(s.max, 40); + assert!((s.stddev - 125f64.sqrt()).abs() < 1e-9); + assert_eq!(Stats::of(&[7]).median, 7.0); + assert_eq!(Stats::of(&[]).mean, 0.0); + } + + #[test] + fn moniker_cannot_escape_results_dir() { + assert_eq!(sanitize_moniker("host-linux-x86_64"), "host-linux-x86_64"); + assert_eq!(sanitize_moniker("../../etc/passwd"), "-.-..-etc-passwd"); + assert_eq!(sanitize_moniker("a/b\\c d\n"), "a-b-c-d-"); + assert_eq!(sanitize_moniker(".hidden"), "-hidden"); + assert_eq!(sanitize_moniker(""), "unknown"); + assert_eq!(sanitize_moniker(&"x".repeat(500)).len(), 64); + for bad in ["..", "../x", "/abs", "a\0b"] { + let m = sanitize_moniker(bad); + assert!(!m.contains('/') && !m.contains('\\') && !m.starts_with('.')); + } + } + + #[test] + fn formatting() { + assert_eq!(fmt_ns(500.0), "500 ns"); + assert_eq!(fmt_ns(1_500.0), "1.50 µs"); + assert_eq!(fmt_ns(2_500_000.0), "2.50 ms"); + assert_eq!(fmt_bytes(512), "512 B"); + assert_eq!(fmt_bytes(2048), "2.0 KiB"); + assert_eq!(fmt_bytes(3 * 1024 * 1024), "3.00 MiB"); + assert_eq!(title("spreadsheet"), "Spreadsheet"); + } + + #[test] + fn save_writes_json_and_markdown() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("results"); + let report = Report::new(SystemInfo::collect(Some("../evil")), 1, 0, Vec::new()); + let (json, md) = report.save(&dir).unwrap(); + assert_eq!(json.parent(), Some(dir.as_path())); + assert!( + json.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("bench--.-evil-") + ); + let parsed: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&json).unwrap()).unwrap(); + assert_eq!(parsed["iterations"], 1); + assert!( + std::fs::read_to_string(&md) + .unwrap() + .starts_with("# Markkit Benchmark Report") + ); + } +} diff --git a/justfile b/justfile index 85d67fd..ee63dd1 100644 --- a/justfile +++ b/justfile @@ -79,14 +79,26 @@ extract-file path: # ─── Benchmarks ────────────────────────────────────────────── -# Run criterion benchmarks for the core crate -bench *args: - cargo bench -p markkit --bench extraction {{args}} +# Run the benchmark harness and save JSON/Markdown results to crates/markkit-bench/results/ +bench: + cargo run -p markkit-bench --release -- --save + +# Benchmark with more iterations for accuracy and save results +bench-accurate: + cargo run -p markkit-bench --release -- --iterations 100 --warmup 10 --save -# Run the markkit-bench harness (stub, not yet implemented) -bench-harness *args: +# Exploratory run, nothing saved (e.g. just bench-quick --category html) +bench-quick *args: cargo run -p markkit-bench --release -- {{args}} +# List benchmark cases +bench-list: + cargo run -p markkit-bench --release -- --list + +# Run criterion micro-benchmarks for the core crate +bench-criterion *args: + cargo bench -p markkit --bench extraction {{args}} + # ─── Eval ──────────────────────────────────────────────────── # Run the LLM evaluation harness (stub, not yet implemented) diff --git a/knowledge/foundations/architecture.md b/knowledge/foundations/architecture.md index 9781979..90f1a77 100644 --- a/knowledge/foundations/architecture.md +++ b/knowledge/foundations/architecture.md @@ -40,7 +40,7 @@ Input (bytes/URL/file) |-------|---------| | `markkit` | Core library – parsing, conversion, API | | `markkit-cli` | Command-line tool | -| `markkit-bench` | Performance benchmarks | +| `markkit-bench` | Benchmark harness, results committed in `results/` | | `markkit-eval` | LLM-based quality evaluation | | `markkit-python` | Python bindings (PyO3) | | `markkit-js` | Node.js bindings (NAPI-RS) | diff --git a/knowledge/log.md b/knowledge/log.md index 1b2ff07..e739db0 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Benchmarks**: `markkit-bench` is no longer a stub. It measures latency, ops/s, throughput and per-op heap allocations (count, bytes, peak) plus process peak RSS over a generated corpus, and `just bench` saves JSON + Markdown runs to `crates/markkit-bench/results/`, like bashkit. Criterion moves to `just bench-criterion`. See [Performance Results](operations/performance-results.md). * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). diff --git a/knowledge/operations/index.md b/knowledge/operations/index.md index 0747777..7d52a00 100644 --- a/knowledge/operations/index.md +++ b/knowledge/operations/index.md @@ -5,4 +5,5 @@ * [Examples](examples.md) - Runnable Rust examples in examples/, what each demonstrates, and their requirements. * [Maintenance](maintenance.md) - Maintenance invocation contract and the dependency, security, test, docs, and CI checklist. * [Release Process](release-process.md) - Versioning, agent-driven release flow, and publication to crates.io, PyPI, and npm. +* [Performance Results](performance-results.md) - Benchmark harnesses, measured metrics, committed result locations, naming, and run contract. * [Evaluation Harness](eval.md) - LLM-as-judge extraction quality evaluation with its methodology, metrics, and providers. diff --git a/knowledge/operations/performance-results.md b/knowledge/operations/performance-results.md new file mode 100644 index 0000000..f08f8bf --- /dev/null +++ b/knowledge/operations/performance-results.md @@ -0,0 +1,79 @@ +--- +type: Playbook +title: Performance Results +description: Benchmark harnesses, measured metrics, committed result locations, naming, and run contract. +tags: + - markkit + - benchmarks + - performance +--- + +# Performance Results + +## Status +Implemented + +Benchmark runs are historical artifacts committed to the repo, following +everruns/bashkit's `bashkit-bench` layout. Reviewers compare a PR's run with the +latest committed run from the same moniker. + +## Harnesses + +| Harness | Purpose | Results | +|---------|---------|---------| +| `markkit-bench` (`crates/markkit-bench`) | End-to-end extraction per case: latency, ops/s, throughput, heap allocations | `crates/markkit-bench/results/` | +| Criterion (`crates/markkit/benches/extraction.rs`) | Statistically rigorous micro-benchmarks | `target/criterion/` (not committed) | + +## Metrics (per case) + +| Field | Meaning | +|-------|---------| +| `mean_ns`, `median_ns`, `stddev_ns`, `min_ns`, `max_ns`, `times_ns` | Wall time of timed iterations (after warmup) | +| `ops_per_sec` | `1 / mean` | +| `throughput_mib_s` | Input bytes / mean. Office inputs are compressed, so MiB/s is lower than for HTML | +| `allocations`, `allocated_bytes` | Heap allocations and bytes requested in one operation (realloc counts as one) | +| `peak_heap_bytes` | Peak live heap above the pre-call level in one operation | +| `input_bytes`, `output_bytes` | Input size, Markdown size | +| `errors`, `error_messages` | Failed operations; a run with errors exits non-zero | + +Report-level: system info (moniker, OS, arch, CPUs, markkit version, git +commit, build profile), iterations, warmup, `peak_rss_bytes` (Linux `VmHWM`). + +Allocation stats come from a counting global allocator in the harness binary and +are measured on one extra untimed pass, so counting overhead never enters timings. + +## Corpus + +Generated in `crates/markkit-bench/src/cases.rs`: deterministic, offline, no +committed binaries, within default builder limits. Categories: `html`, `docx`, +`spreadsheet`, `detect`. Adding or changing a case changes the numbers; note it +in the PR and in `log.md`, since older runs stop being comparable for that case. + +## Result Files + +`results/bench--.json` plus matching `.md`. Moniker defaults +to `--`, overridable with `--moniker`, sanitized to +`[A-Za-z0-9._-]` with no leading dot (no path traversal from hostnames). + +## Run Commands + +| Recipe | Saves | Use | +|--------|-------|-----| +| `just bench` | yes | Default run (20 iterations, 3 warmup) | +| `just bench-accurate` | yes | 100 iterations, 10 warmup | +| `just bench-quick [--category C] [--filter S]` | no | Exploration | +| `just bench-list` | no | List cases | +| `just bench-criterion` | no | Criterion micro-benches | + +## Contract + +- Release builds only (`--release`); debug numbers are meaningless. +- Commit saved runs for performance-relevant PRs and before releases. +- Compare runs from the same moniker only; CPU, OS and container differ. +- `cargo test -p markkit-bench` runs every case once, so a broken case fails CI. + +## See also + +- [Testing Strategy](testing.md), where benchmarks sit among test categories +- [Architecture](../foundations/architecture.md), crate map +- [Threat Model](../security/threat-model.md), limits the corpus stays within diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 49bdfba..c59bcb1 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -23,7 +23,8 @@ tags: | `crates/markkit/tests/document_security.rs` | Zip bombs, grid amplification, XML entities, malformed containers | | `crates/markkit/tests/common/` | In-memory DOCX/XLSX/XLS/ODS fixture builders | | `crates/markkit-cli/tests/cli_integration.rs` | CLI flags and output | -| `crates/markkit/benches/extraction.rs` | Criterion throughput benchmarks | +| `crates/markkit/benches/extraction.rs` | Criterion micro-benchmarks | +| `crates/markkit-bench/` | Benchmark harness; `cargo test` runs every case once | Office fixtures are generated in code (`tests/common`), not committed binaries, so each security test states exactly which bytes it feeds the parser. @@ -37,7 +38,9 @@ each security test states exactly which bytes it feeds the parser. indentation between blocks never changes output - **Security**: one or more tests per mitigated threat; the [Threat Model](../security/threat-model.md) maps threats to test names -- **Benchmarks**: `just bench` (criterion); `markkit-bench` harness is a stub +- **Benchmarks**: `just bench` runs `markkit-bench` (ops/s, throughput, allocations) + and saves results; `just bench-criterion` for micro-benchmarks. See + [Performance Results](performance-results.md) ## Rules @@ -59,3 +62,4 @@ tarpaulin coverage (`--engine llvm`; ptrace segfaults on threaded socket tests). - [Threat Model](../security/threat-model.md), threat-to-test map - [Examples](examples.md), runnable examples CI executes - [Maintenance](maintenance.md), periodic test-health review +- [Performance Results](performance-results.md), benchmark harness and stored results From 7ce3a302f7dbbd12afa5ed209a9156761239623f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:36:06 +0000 Subject: [PATCH 02/11] chore(bench): add baseline results for vm-linux-x86_64 --- crates/markkit-bench/README.md | 14 + .../bench-vm-linux-x86_64-1793057755.json | 522 ++++++++++++++++++ .../bench-vm-linux-x86_64-1793057755.md | 50 ++ 3 files changed, 586 insertions(+) create mode 100644 crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json create mode 100644 crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md diff --git a/crates/markkit-bench/README.md b/crates/markkit-bench/README.md index 2f499b6..b08dc3d 100644 --- a/crates/markkit-bench/README.md +++ b/crates/markkit-bench/README.md @@ -26,3 +26,17 @@ Saved runs live in [`results/`](results/) as same moniker only. Commit a fresh run with performance-relevant PRs. Design and metric definitions: `knowledge/operations/performance-results.md`. + +## Latest baseline (vm, linux x86_64, 4 CPUs, 2026-10-26) + +20 iterations, 3 warmup, release. Full report: +[`bench-vm-linux-x86_64-1793057755.md`](results/bench-vm-linux-x86_64-1793057755.md). + +| Case | Input | Mean | Ops/s | Allocs/op | Peak heap | +|------|------:|-----:|------:|----------:|----------:| +| html_fixture_article | 881 B | 45.6 µs | 21,948 | 204 | 27.3 KiB | +| html_docs_page | 299.6 KiB | 12.0 ms | 83 | 42,778 | 7.23 MiB | +| html_tables | 256.6 KiB | 13.5 ms | 74 | 44,149 | 8.23 MiB | +| docx_report | 9.1 KiB (262 KiB out) | 36.6 ms | 27 | 265,311 | 18.26 MiB | +| xlsx_large (150k cells) | 765.8 KiB | 180.7 ms | 6 | 722,794 | 22.65 MiB | +| ods_medium | 69.8 KiB | 26.1 ms | 38 | 86,513 | 2.76 MiB | diff --git a/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json new file mode 100644 index 0000000..cc4b206 --- /dev/null +++ b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json @@ -0,0 +1,522 @@ +{ + "timestamp": 1793057755, + "system": { + "moniker": "vm-linux-x86_64", + "hostname": "vm", + "os": "linux", + "arch": "x86_64", + "cpus": 4, + "markkit_version": "0.1.1", + "git_commit": "30d3f30235bd", + "profile": "release" + }, + "iterations": 20, + "warmup": 3, + "peak_rss_bytes": 35422208, + "results": [ + { + "name": "html_fixture_article", + "category": "html", + "description": "Golden fixture full_article.html", + "iterations": 20, + "input_bytes": 881, + "output_bytes": 315, + "times_ns": [ + 46733, + 44687, + 44434, + 50542, + 43708, + 42874, + 42635, + 73044, + 50159, + 44050, + 42911, + 42677, + 42727, + 45199, + 42665, + 42636, + 42317, + 42167, + 42411, + 42668 + ], + "mean_ns": 45562.2, + "median_ns": 42892.5, + "stddev_ns": 6734.065292228758, + "min_ns": 42167, + "max_ns": 73044, + "ops_per_sec": 21948.018313426484, + "throughput_mib_s": 18.440441259506922, + "allocations": 204, + "allocated_bytes": 61987, + "peak_heap_bytes": 27923, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_blog_post", + "category": "html", + "description": "Article with nav/footer/scripts, 20 sections", + "iterations": 20, + "input_bytes": 15685, + "output_bytes": 12030, + "times_ns": [ + 471014, + 499678, + 457744, + 476267, + 453427, + 2102312, + 520266, + 602805, + 845149, + 624104, + 464402, + 478485, + 452050, + 472440, + 452836, + 445556, + 592397, + 968555, + 479343, + 479621 + ], + "mean_ns": 616922.55, + "median_ns": 478914.0, + "stddev_ns": 366316.38893796096, + "min_ns": 445556, + "max_ns": 2102312, + "ops_per_sec": 1620.9490153990964, + "throughput_mib_s": 24.24677401212199, + "allocations": 2187, + "allocated_bytes": 586222, + "peak_heap_bytes": 296465, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_docs_page", + "category": "html", + "description": "Long docs page, 400 sections, no chrome", + "iterations": 20, + "input_bytes": 306808, + "output_bytes": 240691, + "times_ns": [ + 10878629, + 11251405, + 11231725, + 12042285, + 18611872, + 14126986, + 11613613, + 11991803, + 11317119, + 11302788, + 11245938, + 11788844, + 11796289, + 11230212, + 11715010, + 12663333, + 11327479, + 10993213, + 12139453, + 11398843 + ], + "mean_ns": 12033341.95, + "median_ns": 11506228.0, + "stddev_ns": 1663514.676858683, + "min_ns": 10878629, + "max_ns": 18611872, + "ops_per_sec": 83.10243356792499, + "throughput_mib_s": 24.315349042995386, + "allocations": 42778, + "allocated_bytes": 14996208, + "peak_heap_bytes": 7584432, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_tables", + "category": "html", + "description": "50 tables x 40 rows x 6 cols", + "iterations": 20, + "input_bytes": 262766, + "output_bytes": 174139, + "times_ns": [ + 13492822, + 13464994, + 13115520, + 13693410, + 13342196, + 12935762, + 13327798, + 13798139, + 13455504, + 13093854, + 13302022, + 13482016, + 13152919, + 13910396, + 13886257, + 13563418, + 13159095, + 13936925, + 13502738, + 13485762 + ], + "mean_ns": 13455077.35, + "median_ns": 13473505.0, + "stddev_ns": 279628.7583479702, + "min_ns": 12935762, + "max_ns": 13936925, + "ops_per_sec": 74.32138619403776, + "throughput_mib_s": 18.624432911551022, + "allocations": 44149, + "allocated_bytes": 14791309, + "peak_heap_bytes": 8632186, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_boilerplate", + "category": "html", + "description": "Mostly script/style/nav to strip, little content", + "iterations": 20, + "input_bytes": 46312, + "output_bytes": 191, + "times_ns": [ + 598722, + 560273, + 578157, + 556227, + 591066, + 579802, + 639725, + 581736, + 552900, + 921400, + 601038, + 555379, + 641597, + 550428, + 582971, + 578193, + 553641, + 653719, + 557770, + 637998 + ], + "mean_ns": 603637.1, + "median_ns": 580769.0, + "stddev_ns": 79634.07890400942, + "min_ns": 550428, + "max_ns": 921400, + "ops_per_sec": 1656.6244851418178, + "throughput_mib_s": 73.1674129065398, + "allocations": 75, + "allocated_bytes": 580870, + "peak_heap_bytes": 311894, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_deep_nesting", + "category": "html", + "description": "Divs nested past the depth limit (1000 levels)", + "iterations": 20, + "input_bytes": 11011, + "output_bytes": 5, + "times_ns": [ + 3491665, + 3336652, + 3902614, + 3392118, + 3374372, + 3389074, + 4379516, + 4486779, + 3442020, + 3499564, + 3666329, + 3344379, + 3314038, + 3442249, + 3276835, + 3508168, + 3631702, + 3808258, + 3312032, + 3661831 + ], + "mean_ns": 3583009.75, + "median_ns": 3466957.0, + "stddev_ns": 328250.3253257908, + "min_ns": 3276835, + "max_ns": 4486779, + "ops_per_sec": 279.09497036674264, + "throughput_mib_s": 2.9307505786020314, + "allocations": 42, + "allocated_bytes": 294880, + "peak_heap_bytes": 150656, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_memo", + "category": "docx", + "description": "Short memo: heading, 10 paragraphs, 1 list", + "iterations": 20, + "input_bytes": 1429, + "output_bytes": 636, + "times_ns": [ + 92051, + 86132, + 81992, + 81873, + 79215, + 80207, + 80616, + 79227, + 143084, + 86787, + 84573, + 79341, + 91342, + 82325, + 77731, + 78863, + 78899, + 78812, + 78354, + 107373 + ], + "mean_ns": 86439.85, + "median_ns": 81244.5, + "stddev_ns": 14670.735599399915, + "min_ns": 77731, + "max_ns": 143084, + "ops_per_sec": 11568.738261345894, + "throughput_mib_s": 15.765883422339709, + "allocations": 733, + "allocated_bytes": 254408, + "peak_heap_bytes": 95495, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_report", + "category": "docx", + "description": "Report: 400 sections with lists and tables", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 268569, + "times_ns": [ + 38563104, + 39946914, + 40400973, + 42929655, + 35497961, + 34450808, + 33388991, + 35404730, + 34945976, + 35248734, + 35595644, + 36811658, + 35682956, + 36232068, + 35607630, + 35858003, + 35826557, + 35378520, + 36192197, + 38237376 + ], + "mean_ns": 36610022.75, + "median_ns": 35754756.5, + "stddev_ns": 2238333.897012639, + "min_ns": 33388991, + "max_ns": 42929655, + "ops_per_sec": 27.31492429897493, + "throughput_mib_s": 0.24223468881241597, + "allocations": 265311, + "allocated_bytes": 35096917, + "peak_heap_bytes": 19147932, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_small", + "category": "spreadsheet", + "description": "1 sheet, 50 rows x 5 cols", + "iterations": 20, + "input_bytes": 2865, + "output_bytes": 2723, + "times_ns": [ + 293470, + 413862, + 289198, + 275693, + 312184, + 287404, + 271367, + 465388, + 286593, + 268022, + 308133, + 278957, + 274380, + 271218, + 294057, + 276520, + 266732, + 296531, + 273841, + 272162 + ], + "mean_ns": 298785.6, + "median_ns": 282775.0, + "stddev_ns": 49257.545061036086, + "min_ns": 266732, + "max_ns": 465388, + "ops_per_sec": 3346.881509684537, + "throughput_mib_s": 9.144607091184804, + "allocations": 1516, + "allocated_bytes": 664245, + "peak_heap_bytes": 132240, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_large", + "category": "spreadsheet", + "description": "3 sheets, 5000 rows x 10 cols each (150k cells)", + "iterations": 20, + "input_bytes": 784216, + "output_bytes": 1844036, + "times_ns": [ + 217208114, + 165399426, + 168449136, + 205360024, + 160141083, + 165733912, + 157310629, + 216229269, + 210931692, + 168108368, + 172151502, + 170154227, + 172094068, + 163912317, + 185913467, + 182266079, + 217077277, + 160083119, + 199078258, + 157254780 + ], + "mean_ns": 180742837.35, + "median_ns": 171124147.5, + "stddev_ns": 21282944.71812606, + "min_ns": 157254780, + "max_ns": 217208114, + "ops_per_sec": 5.532722705152333, + "throughput_mib_s": 4.137849492019407, + "allocations": 722794, + "allocated_bytes": 39916771, + "peak_heap_bytes": 23753817, + "errors": 0, + "error_messages": [] + }, + { + "name": "ods_medium", + "category": "spreadsheet", + "description": "1 sheet, 2000 rows x 8 cols", + "iterations": 20, + "input_bytes": 71489, + "output_bytes": 166354, + "times_ns": [ + 21955061, + 21794390, + 24472027, + 29725985, + 23660532, + 30579957, + 40923801, + 40971475, + 40221158, + 24937719, + 22723999, + 21739432, + 22033237, + 22418817, + 21926907, + 21891831, + 22304871, + 22315878, + 22482345, + 22571227 + ], + "mean_ns": 26082532.45, + "median_ns": 22526786.0, + "stddev_ns": 6591975.686144144, + "min_ns": 21739432, + "max_ns": 40971475, + "ops_per_sec": 38.33983536364775, + "throughput_mib_s": 2.6139035132520814, + "allocations": 86513, + "allocated_bytes": 11707308, + "peak_heap_bytes": 2891155, + "errors": 0, + "error_messages": [] + }, + { + "name": "detect_docx_magic", + "category": "detect", + "description": "Sniff DOCX from bytes without a hint", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 0, + "times_ns": [ + 2629, + 2157, + 2150, + 2153, + 2160, + 2157, + 2159, + 2156, + 2161, + 2156, + 2156, + 2161, + 2158, + 2159, + 2164, + 2160, + 2164, + 2156, + 2158, + 2156 + ], + "mean_ns": 2181.5, + "median_ns": 2158.0, + "stddev_ns": 102.71489668008239, + "min_ns": 2150, + "max_ns": 2629, + "ops_per_sec": 458400.1833600733, + "throughput_mib_s": 4065.1925135281767, + "allocations": 0, + "allocated_bytes": 0, + "peak_heap_bytes": 0, + "errors": 0, + "error_messages": [] + } + ] +} diff --git a/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md new file mode 100644 index 0000000..dca1365 --- /dev/null +++ b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md @@ -0,0 +1,50 @@ +# Markkit Benchmark Report + +## System Information + +- **Moniker**: `vm-linux-x86_64` +- **OS / Arch**: linux / x86_64 +- **CPUs**: 4 +- **markkit**: 0.1.1 (`30d3f30235bd`) +- **Profile**: release +- **Timestamp**: 1793057755 +- **Iterations**: 20 (warmup 3) +- **Peak RSS**: 33.78 MiB + +## Summary + +12 cases, 274.160 ms total mean time per pass, 0 errors. + +Allocations, allocated and peak heap are per single operation. Throughput is input bytes over mean time. + +### HTML + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| html_fixture_article | 881 B | 315 B | 45.56 µs | 42.89 µs | ±6.73 µs | 21948 | 18.4 | 204 | 60.5 KiB | 27.3 KiB | - | +| html_blog_post | 15.3 KiB | 11.7 KiB | 616.92 µs | 478.91 µs | ±366.32 µs | 1621 | 24.2 | 2187 | 572.5 KiB | 289.5 KiB | - | +| html_docs_page | 299.6 KiB | 235.0 KiB | 12.03 ms | 11.51 ms | ±1.66 ms | 83 | 24.3 | 42778 | 14.30 MiB | 7.23 MiB | - | +| html_tables | 256.6 KiB | 170.1 KiB | 13.46 ms | 13.47 ms | ±279.63 µs | 74 | 18.6 | 44149 | 14.11 MiB | 8.23 MiB | - | +| html_boilerplate | 45.2 KiB | 191 B | 603.64 µs | 580.77 µs | ±79.63 µs | 1657 | 73.2 | 75 | 567.3 KiB | 304.6 KiB | - | +| html_deep_nesting | 10.8 KiB | 5 B | 3.58 ms | 3.47 ms | ±328.25 µs | 279 | 2.9 | 42 | 288.0 KiB | 147.1 KiB | - | + +### DOCX + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| docx_memo | 1.4 KiB | 636 B | 86.44 µs | 81.24 µs | ±14.67 µs | 11569 | 15.8 | 733 | 248.4 KiB | 93.3 KiB | - | +| docx_report | 9.1 KiB | 262.3 KiB | 36.61 ms | 35.75 ms | ±2.24 ms | 27 | 0.2 | 265311 | 33.47 MiB | 18.26 MiB | - | + +### Spreadsheet + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| xlsx_small | 2.8 KiB | 2.7 KiB | 298.79 µs | 282.77 µs | ±49.26 µs | 3347 | 9.1 | 1516 | 648.7 KiB | 129.1 KiB | - | +| xlsx_large | 765.8 KiB | 1.76 MiB | 180.74 ms | 171.12 ms | ±21.28 ms | 6 | 4.1 | 722794 | 38.07 MiB | 22.65 MiB | - | +| ods_medium | 69.8 KiB | 162.5 KiB | 26.08 ms | 22.53 ms | ±6.59 ms | 38 | 2.6 | 86513 | 11.16 MiB | 2.76 MiB | - | + +### Detect + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| detect_docx_magic | 9.1 KiB | 0 B | 2.18 µs | 2.16 µs | ±103 ns | 458400 | 4065.2 | 0 | 0 B | 0 B | - | From b04c1d70547e6d9f61b0cf0517b61bbfb8403135 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:37:12 +0000 Subject: [PATCH 03/11] fix(bench): avoid rustdoc HTML-tag lint in --moniker help --- crates/markkit-bench/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/markkit-bench/src/main.rs b/crates/markkit-bench/src/main.rs index ff9c7c0..afa64f7 100644 --- a/crates/markkit-bench/src/main.rs +++ b/crates/markkit-bench/src/main.rs @@ -59,7 +59,7 @@ struct Cli { #[arg(long, default_value = concat!(env!("CARGO_MANIFEST_DIR"), "/results"))] output_dir: PathBuf, - /// Machine label used in file names (default: --) + /// Machine label used in file names (default: HOSTNAME-OS-ARCH) #[arg(long)] moniker: Option, } From 4edb6163e53bba81e1d846bbbb675b68762951b5 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:53:09 +0000 Subject: [PATCH 04/11] docs(bench): add performance summary to README; rerun baseline on latest main --- README.md | 35 ++ crates/markkit-bench/README.md | 26 +- .../bench-vm-linux-x86_64-1793058767.json | 522 ++++++++++++++++++ .../bench-vm-linux-x86_64-1793058767.md | 50 ++ 4 files changed, 623 insertions(+), 10 deletions(-) create mode 100644 crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json create mode 100644 crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md diff --git a/README.md b/README.md index 7754c02..4fa119d 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,41 @@ Office-document limits are configurable on the builder: `max_archive_entries`, `max_sheet_rows`, `max_sheet_cols`, `include_hidden_sheets`. URL fetching: `allow_private_networks`, `allow_host`, `max_redirects`. +## Performance + +Measured with `just bench` (in-process, release build, 20 iterations after 3 warmup) on a +4-CPU Linux x86_64 VM, markkit 0.1.1. Allocations and peak heap are per single extraction. +Full report: [`bench-vm-linux-x86_64-1793058767.md`](https://github.com/everruns/markkit/blob/main/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md); history in +[`crates/markkit-bench/results/`](https://github.com/everruns/markkit/blob/main/crates/markkit-bench/results/). + +| Case | Input | Mean | Ops/s | Throughput | Allocs/op | Peak heap | +|------|------:|-----:|------:|-----------:|----------:|----------:| +| HTML article (fixture) | 881 B | 49 µs | 20,365 | 17 MiB/s | 213 | 28 KiB | +| HTML blog post with nav/scripts | 15 KiB | 641 µs | 1,561 | 23 MiB/s | 2,427 | 304 KiB | +| HTML long docs page | 300 KiB | 13.0 ms | 77 | 22 MiB/s | 47,578 | 7.5 MiB | +| HTML 50 tables | 257 KiB | 15.0 ms | 67 | 17 MiB/s | 50,349 | 8.6 MiB | +| HTML mostly boilerplate | 45 KiB | 706 µs | 1,417 | 63 MiB/s | 81 | 305 KiB | +| HTML 1000 nested divs | 11 KiB | 3.6 ms | 282 | 3 MiB/s | 442 | 147 KiB | +| DOCX memo | 1.4 KiB | 87 µs | 11,508 | 16 MiB/s | 733 | 93 KiB | +| DOCX 400-section report | 9 KiB (262 KiB out) | 35.7 ms | 28 | 0.2 MiB/s | 265,791 | 18.5 MiB | +| XLSX 50×5 | 2.8 KiB | 291 µs | 3,435 | 9 MiB/s | 1,668 | 129 KiB | +| XLSX 3 sheets, 150k cells | 766 KiB | 172 ms | 6 | 4 MiB/s | 782,800 | 24.9 MiB | +| ODS 2000×8 | 70 KiB | 23.2 ms | 43 | 3 MiB/s | 92,515 | 3.0 MiB | +| Format sniffing (DOCX) | 9 KiB | 0.9 µs | 1,141,227 | n/a | 0 | 0 B | + +Takeaways: + +- **HTML** extracts at 17–25 MiB/s; peak heap is ~25x the input size. +- **Boilerplate-heavy pages are cheap**: scripts, styles and nav are dropped at ~60 MiB/s. +- **Deep nesting is the slow outlier**: 1000 nested `
`s run at 3 MiB/s with few + allocations, so the cost is CPU in tree building, not memory. +- **Office formats allocate most**: a 150k-cell XLSX takes 172 ms and ~5 allocations per cell; + a 9 KiB compressed DOCX peaks at 18.5 MiB of heap (~2000x its compressed size). +- **Format detection** is effectively free (under 1 µs, no allocations). + +Timings on a shared VM are noisy (stddev up to ~70% on the smallest cases); allocation counts +are deterministic and the better regression signal. Compare runs from the same machine only. + ## Security Markkit treats every input as untrusted. HTML scripts, styles, forms, iframes and diff --git a/crates/markkit-bench/README.md b/crates/markkit-bench/README.md index b08dc3d..2429931 100644 --- a/crates/markkit-bench/README.md +++ b/crates/markkit-bench/README.md @@ -30,13 +30,19 @@ Design and metric definitions: `knowledge/operations/performance-results.md`. ## Latest baseline (vm, linux x86_64, 4 CPUs, 2026-10-26) 20 iterations, 3 warmup, release. Full report: -[`bench-vm-linux-x86_64-1793057755.md`](results/bench-vm-linux-x86_64-1793057755.md). - -| Case | Input | Mean | Ops/s | Allocs/op | Peak heap | -|------|------:|-----:|------:|----------:|----------:| -| html_fixture_article | 881 B | 45.6 µs | 21,948 | 204 | 27.3 KiB | -| html_docs_page | 299.6 KiB | 12.0 ms | 83 | 42,778 | 7.23 MiB | -| html_tables | 256.6 KiB | 13.5 ms | 74 | 44,149 | 8.23 MiB | -| docx_report | 9.1 KiB (262 KiB out) | 36.6 ms | 27 | 265,311 | 18.26 MiB | -| xlsx_large (150k cells) | 765.8 KiB | 180.7 ms | 6 | 722,794 | 22.65 MiB | -| ods_medium | 69.8 KiB | 26.1 ms | 38 | 86,513 | 2.76 MiB | +[`bench-vm-linux-x86_64-1793058767.md`](results/bench-vm-linux-x86_64-1793058767.md). + +| Case | Input | Mean | Ops/s | Throughput | Allocs/op | Peak heap | +|------|------:|-----:|------:|-----------:|----------:|----------:| +| HTML article (fixture) | 881 B | 49 µs | 20,365 | 17 MiB/s | 213 | 28 KiB | +| HTML blog post with nav/scripts | 15 KiB | 641 µs | 1,561 | 23 MiB/s | 2,427 | 304 KiB | +| HTML long docs page | 300 KiB | 13.0 ms | 77 | 22 MiB/s | 47,578 | 7.5 MiB | +| HTML 50 tables | 257 KiB | 15.0 ms | 67 | 17 MiB/s | 50,349 | 8.6 MiB | +| HTML mostly boilerplate | 45 KiB | 706 µs | 1,417 | 63 MiB/s | 81 | 305 KiB | +| HTML 1000 nested divs | 11 KiB | 3.6 ms | 282 | 3 MiB/s | 442 | 147 KiB | +| DOCX memo | 1.4 KiB | 87 µs | 11,508 | 16 MiB/s | 733 | 93 KiB | +| DOCX 400-section report | 9 KiB (262 KiB out) | 35.7 ms | 28 | 0.2 MiB/s | 265,791 | 18.5 MiB | +| XLSX 50×5 | 2.8 KiB | 291 µs | 3,435 | 9 MiB/s | 1,668 | 129 KiB | +| XLSX 3 sheets, 150k cells | 766 KiB | 172 ms | 6 | 4 MiB/s | 782,800 | 24.9 MiB | +| ODS 2000×8 | 70 KiB | 23.2 ms | 43 | 3 MiB/s | 92,515 | 3.0 MiB | +| Format sniffing (DOCX) | 9 KiB | 0.9 µs | 1,141,227 | n/a | 0 | 0 B | diff --git a/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json new file mode 100644 index 0000000..f18462a --- /dev/null +++ b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json @@ -0,0 +1,522 @@ +{ + "timestamp": 1793058767, + "system": { + "moniker": "vm-linux-x86_64", + "hostname": "vm", + "os": "linux", + "arch": "x86_64", + "cpus": 4, + "markkit_version": "0.1.1", + "git_commit": "cde4ebc60a6f", + "profile": "release" + }, + "iterations": 20, + "warmup": 3, + "peak_rss_bytes": 39542784, + "results": [ + { + "name": "html_fixture_article", + "category": "html", + "description": "Golden fixture full_article.html", + "iterations": 20, + "input_bytes": 881, + "output_bytes": 315, + "times_ns": [ + 47559, + 45862, + 45418, + 44935, + 45143, + 44563, + 43485, + 44051, + 120534, + 54600, + 45142, + 44871, + 44728, + 43913, + 43979, + 43020, + 49364, + 43744, + 43508, + 43648 + ], + "mean_ns": 49103.35, + "median_ns": 44799.5, + "stddev_ns": 16587.04984099041, + "min_ns": 43020, + "max_ns": 120534, + "ops_per_sec": 20365.20929834726, + "throughput_mib_s": 17.11058558639902, + "allocations": 213, + "allocated_bytes": 63921, + "peak_heap_bytes": 28907, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_blog_post", + "category": "html", + "description": "Article with nav/footer/scripts, 20 sections", + "iterations": 20, + "input_bytes": 15685, + "output_bytes": 12030, + "times_ns": [ + 606767, + 536341, + 2681532, + 550974, + 507343, + 490556, + 645800, + 499916, + 528093, + 495001, + 506726, + 492886, + 570201, + 494751, + 597818, + 475796, + 507213, + 564130, + 584385, + 475880 + ], + "mean_ns": 640605.45, + "median_ns": 517718.0, + "stddev_ns": 470547.03534689004, + "min_ns": 475796, + "max_ns": 2681532, + "ops_per_sec": 1561.0232476167664, + "throughput_mib_s": 23.35038150679491, + "allocations": 2427, + "allocated_bytes": 611398, + "peak_heap_bytes": 311521, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_docs_page", + "category": "html", + "description": "Long docs page, 400 sections, no chrome", + "iterations": 20, + "input_bytes": 306808, + "output_bytes": 240691, + "times_ns": [ + 11042442, + 13777475, + 12865523, + 11023504, + 10914108, + 10774392, + 13755664, + 14849510, + 11272597, + 11800104, + 16349788, + 19583388, + 19145305, + 12456048, + 11657200, + 11864906, + 11181273, + 13662463, + 11603532, + 11227946 + ], + "mean_ns": 13040358.4, + "median_ns": 11832505.0, + "stddev_ns": 2555380.7601565253, + "min_ns": 10774392, + "max_ns": 19583388, + "ops_per_sec": 76.68500890282279, + "throughput_mib_s": 22.437643252808815, + "allocations": 47578, + "allocated_bytes": 15524152, + "peak_heap_bytes": 7897080, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_tables", + "category": "html", + "description": "50 tables x 40 rows x 6 cols", + "iterations": 20, + "input_bytes": 262766, + "output_bytes": 174139, + "times_ns": [ + 14216673, + 13696047, + 13835662, + 13298999, + 13410126, + 13960863, + 14410063, + 14615209, + 14311158, + 14760484, + 16270500, + 17706639, + 16716248, + 15859556, + 15733504, + 16240112, + 15148803, + 14411163, + 15800456, + 16002096 + ], + "mean_ns": 15020218.05, + "median_ns": 14687846.5, + "stddev_ns": 1181193.987371104, + "min_ns": 13298999, + "max_ns": 17706639, + "ops_per_sec": 66.57692962053903, + "throughput_mib_s": 16.683724869413908, + "allocations": 50349, + "allocated_bytes": 16051725, + "peak_heap_bytes": 9029610, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_boilerplate", + "category": "html", + "description": "Mostly script/style/nav to strip, little content", + "iterations": 20, + "input_bytes": 46312, + "output_bytes": 191, + "times_ns": [ + 564847, + 625417, + 562169, + 589018, + 586922, + 556429, + 586505, + 558075, + 661947, + 588924, + 555093, + 837752, + 614948, + 558862, + 696483, + 563339, + 2210966, + 590241, + 694208, + 915287 + ], + "mean_ns": 705871.6, + "median_ns": 588971.0, + "stddev_ns": 357905.447485282, + "min_ns": 555093, + "max_ns": 2210966, + "ops_per_sec": 1416.6882475509713, + "throughput_mib_s": 62.57025348718698, + "allocations": 81, + "allocated_bytes": 581386, + "peak_heap_bytes": 312174, + "errors": 0, + "error_messages": [] + }, + { + "name": "html_deep_nesting", + "category": "html", + "description": "Divs nested past the depth limit (1000 levels)", + "iterations": 20, + "input_bytes": 11011, + "output_bytes": 5, + "times_ns": [ + 3485895, + 3418052, + 3473491, + 3496727, + 3466432, + 3478178, + 3475688, + 3515289, + 3410251, + 3440638, + 3539990, + 3476987, + 3552173, + 3414363, + 4616971, + 3532312, + 3463396, + 3487846, + 3763640, + 3459102 + ], + "mean_ns": 3548371.05, + "median_ns": 3477582.5, + "stddev_ns": 255769.26395512716, + "min_ns": 3410251, + "max_ns": 4616971, + "ops_per_sec": 281.81945628262304, + "throughput_mib_s": 2.9593601542739507, + "allocations": 442, + "allocated_bytes": 305712, + "peak_heap_bytes": 150656, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_memo", + "category": "docx", + "description": "Short memo: heading, 10 paragraphs, 1 list", + "iterations": 20, + "input_bytes": 1429, + "output_bytes": 636, + "times_ns": [ + 96403, + 82064, + 86685, + 82502, + 155812, + 88777, + 79984, + 81868, + 89166, + 79356, + 79125, + 78513, + 77750, + 77359, + 82958, + 78068, + 100142, + 85054, + 78451, + 77886 + ], + "mean_ns": 86896.15, + "median_ns": 81966.0, + "stddev_ns": 16957.525059026153, + "min_ns": 77359, + "max_ns": 155812, + "ops_per_sec": 11507.989709555603, + "throughput_mib_s": 15.683095259623485, + "allocations": 733, + "allocated_bytes": 255960, + "peak_heap_bytes": 95495, + "errors": 0, + "error_messages": [] + }, + { + "name": "docx_report", + "category": "docx", + "description": "Report: 400 sections with lists and tables", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 268569, + "times_ns": [ + 40163390, + 35543631, + 39913980, + 44253492, + 34460997, + 34429776, + 34119970, + 34544883, + 35869003, + 33690352, + 35809890, + 35476536, + 35059409, + 34981853, + 33447102, + 34098923, + 35564423, + 33558630, + 34639253, + 34410093 + ], + "mean_ns": 35701779.3, + "median_ns": 34810553.0, + "stddev_ns": 2626144.678405478, + "min_ns": 33447102, + "max_ns": 44253492, + "ops_per_sec": 28.009808463523832, + "throughput_mib_s": 0.248397072698887, + "allocations": 265791, + "allocated_bytes": 35992757, + "peak_heap_bytes": 19402268, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_small", + "category": "spreadsheet", + "description": "1 sheet, 50 rows x 5 cols", + "iterations": 20, + "input_bytes": 2865, + "output_bytes": 2723, + "times_ns": [ + 298704, + 284093, + 301211, + 293310, + 279144, + 278960, + 304703, + 288245, + 277666, + 337048, + 292357, + 279091, + 276705, + 306350, + 300087, + 276399, + 300309, + 287437, + 283053, + 277079 + ], + "mean_ns": 291097.55, + "median_ns": 287841.0, + "stddev_ns": 14545.223478774742, + "min_ns": 276399, + "max_ns": 337048, + "ops_per_sec": 3435.2745325407236, + "throughput_mib_s": 9.386121307114767, + "allocations": 1668, + "allocated_bytes": 685885, + "peak_heap_bytes": 132240, + "errors": 0, + "error_messages": [] + }, + { + "name": "xlsx_large", + "category": "spreadsheet", + "description": "3 sheets, 5000 rows x 10 cols each (150k cells)", + "iterations": 20, + "input_bytes": 784216, + "output_bytes": 1844036, + "times_ns": [ + 161909562, + 162003795, + 198601320, + 184202128, + 164043235, + 164880025, + 173530491, + 164780148, + 172603706, + 167683350, + 173782292, + 211840216, + 165060727, + 164221020, + 166417853, + 167527631, + 167108176, + 169548303, + 171410572, + 172671069 + ], + "mean_ns": 172191280.95, + "median_ns": 167605490.5, + "stddev_ns": 12300383.266388725, + "min_ns": 161909562, + "max_ns": 211840216, + "ops_per_sec": 5.807494981644133, + "throughput_mib_s": 4.343348011517558, + "allocations": 782800, + "allocated_bytes": 53236891, + "peak_heap_bytes": 26153905, + "errors": 0, + "error_messages": [] + }, + { + "name": "ods_medium", + "category": "spreadsheet", + "description": "1 sheet, 2000 rows x 8 cols", + "iterations": 20, + "input_bytes": 71489, + "output_bytes": 166354, + "times_ns": [ + 23547802, + 23255792, + 22610967, + 22967538, + 22978193, + 22868932, + 23106346, + 23501550, + 23282356, + 22911416, + 23477332, + 22736954, + 22796369, + 23220167, + 23100117, + 22814699, + 23946706, + 23340409, + 22915737, + 23618416 + ], + "mean_ns": 23149889.9, + "median_ns": 23103231.5, + "stddev_ns": 336873.3301304661, + "min_ns": 22610967, + "max_ns": 23946706, + "ops_per_sec": 43.19674971758722, + "throughput_mib_s": 2.945034447250931, + "allocations": 92515, + "allocated_bytes": 12811348, + "peak_heap_bytes": 3147387, + "errors": 0, + "error_messages": [] + }, + { + "name": "detect_docx_magic", + "category": "detect", + "description": "Sniff DOCX from bytes without a hint", + "iterations": 20, + "input_bytes": 9299, + "output_bytes": 0, + "times_ns": [ + 1301, + 848, + 834, + 880, + 847, + 843, + 879, + 884, + 850, + 845, + 821, + 843, + 857, + 852, + 859, + 868, + 820, + 857, + 871, + 866 + ], + "mean_ns": 876.25, + "median_ns": 854.5, + "stddev_ns": 98.95800877139757, + "min_ns": 820, + "max_ns": 1301, + "ops_per_sec": 1141226.8188302426, + "throughput_mib_s": 10120.647609999109, + "allocations": 0, + "allocated_bytes": 0, + "peak_heap_bytes": 0, + "errors": 0, + "error_messages": [] + } + ] +} diff --git a/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md new file mode 100644 index 0000000..46a109c --- /dev/null +++ b/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md @@ -0,0 +1,50 @@ +# Markkit Benchmark Report + +## System Information + +- **Moniker**: `vm-linux-x86_64` +- **OS / Arch**: linux / x86_64 +- **CPUs**: 4 +- **markkit**: 0.1.1 (`cde4ebc60a6f`) +- **Profile**: release +- **Timestamp**: 1793058767 +- **Iterations**: 20 (warmup 3) +- **Peak RSS**: 37.71 MiB + +## Summary + +12 cases, 264.426 ms total mean time per pass, 0 errors. + +Allocations, allocated and peak heap are per single operation. Throughput is input bytes over mean time. + +### HTML + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| html_fixture_article | 881 B | 315 B | 49.10 µs | 44.80 µs | ±16.59 µs | 20365 | 17.1 | 213 | 62.4 KiB | 28.2 KiB | - | +| html_blog_post | 15.3 KiB | 11.7 KiB | 640.61 µs | 517.72 µs | ±470.55 µs | 1561 | 23.4 | 2427 | 597.1 KiB | 304.2 KiB | - | +| html_docs_page | 299.6 KiB | 235.0 KiB | 13.04 ms | 11.83 ms | ±2.56 ms | 77 | 22.4 | 47578 | 14.80 MiB | 7.53 MiB | - | +| html_tables | 256.6 KiB | 170.1 KiB | 15.02 ms | 14.69 ms | ±1.18 ms | 67 | 16.7 | 50349 | 15.31 MiB | 8.61 MiB | - | +| html_boilerplate | 45.2 KiB | 191 B | 705.87 µs | 588.97 µs | ±357.91 µs | 1417 | 62.6 | 81 | 567.8 KiB | 304.9 KiB | - | +| html_deep_nesting | 10.8 KiB | 5 B | 3.55 ms | 3.48 ms | ±255.77 µs | 282 | 3.0 | 442 | 298.5 KiB | 147.1 KiB | - | + +### DOCX + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| docx_memo | 1.4 KiB | 636 B | 86.90 µs | 81.97 µs | ±16.96 µs | 11508 | 15.7 | 733 | 250.0 KiB | 93.3 KiB | - | +| docx_report | 9.1 KiB | 262.3 KiB | 35.70 ms | 34.81 ms | ±2.63 ms | 28 | 0.2 | 265791 | 34.33 MiB | 18.50 MiB | - | + +### Spreadsheet + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| xlsx_small | 2.8 KiB | 2.7 KiB | 291.10 µs | 287.84 µs | ±14.55 µs | 3435 | 9.4 | 1668 | 669.8 KiB | 129.1 KiB | - | +| xlsx_large | 765.8 KiB | 1.76 MiB | 172.19 ms | 167.61 ms | ±12.30 ms | 6 | 4.3 | 782800 | 50.77 MiB | 24.94 MiB | - | +| ods_medium | 69.8 KiB | 162.5 KiB | 23.15 ms | 23.10 ms | ±336.87 µs | 43 | 2.9 | 92515 | 12.22 MiB | 3.00 MiB | - | + +### Detect + +| Benchmark | Input | Output | Mean | Median | StdDev | Ops/s | MiB/s | Allocs | Allocated | Peak heap | Errors | +|-----------|------:|-------:|-----:|-------:|-------:|------:|------:|-------:|----------:|----------:|-------:| +| detect_docx_magic | 9.1 KiB | 0 B | 876 ns | 854 ns | ±99 ns | 1141227 | 10120.6 | 0 | 0 B | 0 B | - | From 6a28b7fb39e56a6b2725880ecac907580b60f040 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 00:09:54 +0000 Subject: [PATCH 05/11] chore(knowledge): restore page-metadata log entry lost in merge --- knowledge/log.md | 1 + 1 file changed, 1 insertion(+) diff --git a/knowledge/log.md b/knowledge/log.md index fcf5344..0a7d1a0 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,6 +4,7 @@ * **Benchmarks**: `markkit-bench` is no longer a stub. It measures latency, ops/s, throughput and per-op heap allocations (count, bytes, peak) plus process peak RSS over a generated corpus, and `just bench` saves JSON + Markdown runs to `crates/markkit-bench/results/`, like bashkit. Criterion moves to `just bench-criterion`. See [Performance Results](operations/performance-results.md). * **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md).* **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from ``, Open Graph, `` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. +* **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from ``, Open Graph, `` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. * **IR v2**: footnotes, definition lists, task lists, figures, table alignment and spans; `Document.ir_version` (`IR_VERSION = 2`), `#[non_exhaustive]` IR types, `Node::is_block`/`is_inline`. HTML maps GitHub/Pandoc footnotes, `
`, checkbox items, captioned figures, `colspan`/`rowspan`/alignment and row-header ``. New threat T2e and invariant S13 (span grid budget, footnote label sanitizing). See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md), [Threat Model](security/threat-model.md). * **Agent tool**: new [Tool Contract](integrations/tool-contract.md) concept in a new `integrations/` domain. `MarkkitTool` (opt-in `tool` feature) follows the Everruns toolkit-library contract; tool arguments are a new untrusted surface in the [Threat Model](security/threat-model.md). * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). From 4a00f9c2e203e9186975d2f05f53b89b95961fa6 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Tue, 27 Oct 2026 01:40:19 +0000 Subject: [PATCH 06/11] fix(html): make deeply nested input linear and stop nested-table row duplication html5ever's tree builder is quadratic in nesting depth: 687 KiB of nested divs took 12.6 s and 292 KiB of unclosed nested tables ran past the 60 s timeout. A pre-scan (html::nesting::cap_nesting) now drops tags nested past max_nesting_depth + 8 before html5ever runs, keeping text. Tables took rows via descendant queries, copying nested-table rows into every enclosing table (4x output per level). Rows now come from direct structure only. Text flattened at max_nesting_depth leaked script/style/template bodies; flat_text skips non-content elements. New invariant S17. --- CHANGELOG.md | 8 + crates/markkit/src/html/mod.rs | 1 + crates/markkit/src/html/nesting.rs | 389 ++++++++++++++++++++++++++ crates/markkit/src/html/parser.rs | 100 +++++-- crates/markkit/tests/html_security.rs | 77 +++++ knowledge/formats/html-extraction.md | 22 +- knowledge/log.md | 1 + knowledge/security/threat-model.md | 10 + 8 files changed, 581 insertions(+), 27 deletions(-) create mode 100644 crates/markkit/src/html/nesting.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf3d53..51786a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- HTML: deeply nested input no longer costs quadratic time. 687 KiB of nested `
`s took + 12.6 s and 292 KiB of unclosed nested tables ran past the 60 s timeout; tags nested past + `max_nesting_depth` are now dropped (text kept) before html5ever parses, so both take + milliseconds. +- HTML: rows of nested tables are no longer copied into every enclosing table (output grew 4x + per nesting level). +- HTML: text flattened past `max_nesting_depth` no longer includes `" + ); + assert!(matches!(cap_nesting(&html, 5), Cow::Borrowed(_))); + } + + #[test] + fn optional_end_tags_and_voids_do_not_count() { + let html = "

a

  • b
    c
  • ` closes the unclosed spans inside it; depth must not drift. + let html = "
    x
    ".repeat(1000); + assert!(matches!(cap_nesting(&html, 10), Cow::Borrowed(_))); + } + + #[test] + fn raw_text_end_tags_survive_while_dropping() { + // Dropping `` would turn the rest of the page into script. + let html = format!( + "{}

    after

    {}", + "
    ".repeat(50), + "
    ".repeat(50) + ); + let capped = cap_nesting(&html, 10); + assert!(capped.contains("")); + assert!(capped.contains("after")); + } + + #[test] + fn uncounted_end_tags_do_not_consume_overflow() { + let html = format!( + "{}

    a

  • b
  • {}", + "
    ".repeat(20), + "
    ".repeat(20) + ); + let capped = cap_nesting(&html, 10); + // Past the cap the p/li tags go too, their text stays, and their end + // tags do not close dropped divs. + assert!(capped.contains("ab")); + assert_eq!(capped.matches("
    ").count(), 10); + assert_eq!(capped.matches("
    ").count(), 10); + } + + #[test] + fn non_content_past_the_cap_is_dropped_whole() { + let html = format!( + "{}
    f

    after

    ", + "
    ".repeat(20) + ); + let capped = cap_nesting(&html, 10); + assert!(!capped.contains("tpl") && !capped.contains("in<") && !capped.contains("f")); + assert!(capped.contains("after")); + let objects = "".repeat(100_000) + "x"; + assert!(!cap_nesting(&objects, 10).contains('x')); + } + + #[test] + fn malformed_tails_do_not_panic() { + for s in [ + "<", "", "", "<1>", "<\u{e9}>", + ] { + let _ = cap_nesting(s, 1); + let _ = cap_nesting(&format!("
    {s}"), 0); + } + } +} diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 68f5b46..d1ee896 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -33,8 +33,14 @@ use crate::limits::Budget; use url::Url; use super::metadata::PageInfo; +use super::nesting::{self, NON_CONTENT}; use super::{readability, urls}; +/// Extra levels kept above `max_nesting_depth` before raw tags are dropped: +/// the walker flattens there anyway, the slack covers `html`/`body` and +/// implied elements the pre-scan does not count. +const NESTING_SLACK: usize = 8; + /// HTML-spec clamps for table spans. const MAX_COLSPAN: u32 = 1000; const MAX_ROWSPAN: u32 = 65534; @@ -97,12 +103,17 @@ impl<'a> HtmlParser<'a> { /// Parse an HTML string into a [`Document`]. pub fn parse(&self, html: &str) -> Result { - self.parse_dom(&Html::parse_document(html)) + self.parse_dom(&self.parse_html5ever(html)) + } + + /// html5ever parse of `html` with nesting capped first (see `nesting`). + fn parse_html5ever(&self, html: &str) -> Html { + Html::parse_document(&nesting::cap_nesting(html, self.max_depth + NESTING_SLACK)) } /// Parse an HTML string into a [`Document`] plus page title and metadata. pub(crate) fn parse_page(&self, html: &str) -> Result<(Document, PageInfo), Error> { - let dom = Html::parse_document(html); + let dom = self.parse_html5ever(html); let this = self.for_document(&dom); Ok(( this.convert_dom(&dom)?, @@ -243,7 +254,7 @@ impl<'a> HtmlParser<'a> { } if depth >= self.max_depth { // At max depth, just extract text - let text: String = parent.text().collect(); + let text = flat_text(parent); let trimmed = text.trim(); if !trimmed.is_empty() { out.push(Node::Text(trimmed.to_string())); @@ -313,7 +324,7 @@ impl<'a> HtmlParser<'a> { return None; } if depth >= self.max_depth { - let text: String = el.text().collect(); + let text = flat_text(el); let trimmed = text.trim(); if trimmed.is_empty() { return None; @@ -505,9 +516,7 @@ impl<'a> HtmlParser<'a> { "hr" => Some(Node::ThematicBreak), // Skip non-content elements - "script" | "style" | "noscript" | "iframe" | "object" | "embed" | "form" | "input" - | "button" | "select" | "textarea" | "svg" | "canvas" | "video" | "audio" - | "source" | "track" | "map" | "area" => None, + name if NON_CONTENT.contains(&name) => None, // Unknown element: try to extract inline content _ => { @@ -746,7 +755,7 @@ impl<'a> HtmlParser<'a> { /// `` that is a footnote reference: record target and build the node. fn footnote_reference(&self, el: scraper::ElementRef<'_>) -> Option { let target = el.value().attr("href")?.strip_prefix('#')?; - let text: String = el.text().collect(); + let text = flat_text(el); let text = text .trim() .trim_start_matches('[') @@ -770,7 +779,7 @@ impl<'a> HtmlParser<'a> { return nodes; } if depth >= self.max_depth { - let text: String = el.text().collect(); + let text = flat_text(el); let trimmed = text.trim(); if !trimmed.is_empty() { nodes.push(Node::Text(trimmed.to_string())); @@ -847,17 +856,15 @@ impl<'a> HtmlParser<'a> { exclude: &[Selector], depth: usize, ) -> Option { - let thead_sel = Selector::parse("thead").unwrap(); - let tbody_sel = Selector::parse("tbody").unwrap(); - let tr_sel = Selector::parse("tr").unwrap(); - let mut headers: Vec = Vec::new(); let mut alignments: Vec = Vec::new(); let mut rows: Vec> = Vec::new(); - if let Some(thead) = table_el.select(&thead_sel).next() - && let Some(tr) = thead.select(&tr_sel).next() - { + // Direct structure only: a descendant query would also pick up the + // rows of nested tables, duplicating them into every enclosing table + // (output and work grew 4x per nesting level). THREAT[T2c]. + let (head_row, body_rows) = table_rows(table_el); + if let Some(tr) = head_row { let ths = row_cells(tr, &["th", "td"]); alignments = column_alignments(&ths); headers = ths @@ -866,13 +873,6 @@ impl<'a> HtmlParser<'a> { .collect(); } - let body_rows: Vec> = - if let Some(tbody) = table_el.select(&tbody_sel).next() { - tbody.select(&tr_sel).collect() - } else { - table_el.select(&tr_sel).collect() - }; - for tr in body_rows { let row = row_cells(tr, &["td", "th"]); // A leading all- row is the header. @@ -933,6 +933,60 @@ impl<'a> HtmlParser<'a> { } /// Direct `th`/`td` children of a row (nested tables' cells excluded). +/// Text of a subtree flattened at `max_nesting_depth`, without the content of +/// [`NON_CONTENT`] elements (`el.text()` leaked `

    after

    {}", + "
    ".repeat(depth), + "
    ".repeat(depth) + ); + let md = convert(&html); + assert!(md.contains("ok") && md.contains("after"), "{depth}: {md}"); + for bad in [ + "SECRET_JS", + "color:red", + "tpl", + "", + ] { + assert!(!md.contains(bad), "{depth}: {bad} leaked: {md}"); + } + } +} + #[test] fn very_long_text_node() { let text = "a".repeat(100_000); diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index 3788639..5ae1ea7 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -134,12 +134,26 @@ Trimming at render time: Limit: `limits.max_nesting_depth` (default 256, clamped to `HARD_MAX_NESTING_DEPTH` = 512) Behavior at depth limit: -- `convert_children`: extracts all descendant text via `.text().collect()`, trims, emits as single `Text` node -- `convert_element`: same -- flattens subtree to trimmed text -- `collect_inline`: same -- flattens subtree to trimmed text +- `convert_children`: flattens the subtree to its text (`flat_text`: iterative, skips the + content of non-content elements such as `script`, `style`, `template`, `form`), trims, + emits a single `Text` node +- `convert_element`, `collect_inline`: same - Empty text at depth limit -> elided -Prevents stack overflow on adversarial nesting. Tested with 1000 nested `
    ` elements. +Before html5ever runs, `html::nesting::cap_nesting` pre-scans the raw HTML and drops start +tags (and paired end tags) that would nest deeper than `max_nesting_depth + 8`, keeping their +text; non-content elements past the cap are dropped whole. html5ever's tree builder is +quadratic in depth (most start tags scan the open-element stack), so this keeps parsing +linear. Comments, quoted attributes and raw-text elements are skipped; void and optional-end +elements (`p`, `li`, `td`, ...) are not counted; table parts close implicitly. Input that +never reaches the cap is passed through unchanged (no copy). + +Tables use direct structure only: header = first `` of the first direct ``; body += direct `` children and `` children of direct `` sections. Rows of nested +tables stay in the nested table. + +Prevents stack overflow and super-linear time on adversarial nesting. Tested with 64 000 +nested `
    `s, 20 000 unclosed nested tables, and script/style past the limit. ## Exclude Selector Defaults diff --git a/knowledge/log.md b/knowledge/log.md index 293615e..cfa9294 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-27 +* **Nesting cost**: HTML parsing was quadratic in nesting depth (html5ever) and exponential in nested tables (descendant row selection duplicated inner rows); text flattened past `max_nesting_depth` leaked `