diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf3d53..5807dff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). the article body of a web page, Readability-style, dropping navigation, sidebars, comments, share bars and related-link blocks. Off by default; include selectors take precedence; falls back to the full page when no article-like block is found. +- Every resource limit is configurable outside Rust: CLI flags (`--max-output-size`, `--timeout`, + `--max-nesting-depth`, `--max-nodes`, archive / sheet / redirect flags; `--max-size` kept as an + alias of `--max-input-size`), Python `markkit.Limits(...)` passed as `limits=` to every function, + and flat Node.js and WASM option keys (`maxOutputSize`, `maxNodes`, `maxNestingDepth`, plus + `timeout` in Node.js). +- Node.js and WASM options gain `baseUrl` and `mainContent`; Python `extract_html_sync_with_options` + gains `base_url=` and `main_content=`, matching the core builder. ### Security @@ -57,7 +64,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/). thousands of rows); `max_nodes` (5 000 000) caps parsed HTML DOM / DOCX XML nodes; `max_nesting_depth` (256) is configurable, with a hard ceiling of 512 that keeps deep input off the host stack. Violations return `Error::ResourceLimit` naming the limit. - - `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: loopback, private, link-local / cloud metadata and other reserved addresses, checked on the URL, after DNS resolution and on every redirect. Only `http`/`https` are fetched. Redirects are capped diff --git a/README.md b/README.md index 768ec74..e8a273d 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,9 @@ let markkit = Markkit::builder() .build(); ``` -Each limit also has a builder setter (`.max_nodes(..)`, `.timeout(..)`, ...). +Each limit also has a builder setter (`.max_nodes(..)`, `.timeout(..)`, ...), a CLI +flag (`--max-nodes`, `--timeout`, ...; see `markkit extract --help`), a +`markkit.Limits(...)` argument in Python and flat option keys in Node.js and WASM. Other options: `include_hidden_sheets`, `allow_private_networks`, `allow_host`. ## Performance diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index 36abb6d..dfef440 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -1,8 +1,10 @@ use std::io::Read; use anyhow::Result; -use clap::{Parser, Subcommand}; -use markkit::Markkit; +use std::time::Duration; + +use clap::{Args, Parser, Subcommand}; +use markkit::{Limits, Markkit}; #[derive(Parser)] #[command( @@ -38,10 +40,6 @@ enum Commands { #[arg(long = "exclude", short = 'e')] exclude_selectors: Vec, - /// Maximum input size in bytes - #[arg(long, default_value = "52428800")] - max_size: usize, - /// Output as JSON with metadata #[arg(long)] json: bool, @@ -54,9 +52,82 @@ enum Commands { /// Keep only the detected main content (article body) of HTML pages #[arg(long)] main_content: bool, + + #[command(flatten)] + limits: LimitArgs, }, } +/// Resource limits; unset flags keep the library defaults (see `markkit::Limits`). +#[derive(Args, Debug, Default)] +#[command(next_help_heading = "Limits")] +struct LimitArgs { + /// Maximum input (and fetched body) size in bytes [default: 50 MiB] + #[arg(long, alias = "max-size", value_name = "BYTES")] + max_input_size: Option, + /// Maximum Markdown output size in bytes [default: 128 MiB] + #[arg(long, value_name = "BYTES")] + max_output_size: Option, + /// Time budget for fetch + parse + render, in seconds [default: 60] + #[arg(long, value_name = "SECS", value_parser = parse_secs)] + timeout: Option, + /// Maximum element nesting, hard-capped at 512 [default: 256] + #[arg(long, value_name = "N")] + max_nesting_depth: Option, + /// Maximum parsed HTML DOM / DOCX XML nodes [default: 5000000] + #[arg(long, value_name = "N")] + max_nodes: Option, + /// Maximum entries in a ZIP container [default: 10000] + #[arg(long, value_name = "N")] + max_archive_entries: Option, + /// Maximum decompressed bytes per ZIP entry [default: 64 MiB] + #[arg(long, value_name = "BYTES")] + max_archive_entry_size: Option, + /// Maximum decompressed bytes per ZIP container [default: 256 MiB] + #[arg(long, value_name = "BYTES")] + max_archive_total_size: Option, + /// Maximum data rows rendered per sheet [default: 10000] + #[arg(long, value_name = "N")] + max_sheet_rows: Option, + /// Maximum columns rendered per sheet [default: 256] + #[arg(long, value_name = "N")] + max_sheet_cols: Option, + /// Maximum XLS/ODS cell grid [default: 4000000] + #[arg(long, value_name = "N")] + max_spreadsheet_cells: Option, + /// Maximum redirects followed when fetching a URL [default: 5] + #[arg(long, value_name = "N")] + max_redirects: Option, +} + +impl LimitArgs { + fn apply(&self, mut l: Limits) -> Limits { + macro_rules! set { + ($($f:ident),*) => {$( if let Some(v) = self.$f { l.$f = v; } )*}; + } + set!( + max_input_size, + max_output_size, + timeout, + max_nesting_depth, + max_nodes, + max_archive_entries, + max_archive_entry_size, + max_archive_total_size, + max_sheet_rows, + max_sheet_cols, + max_spreadsheet_cells, + max_redirects + ); + l + } +} + +fn parse_secs(s: &str) -> Result { + let secs: f64 = s.parse().map_err(|e| format!("{e}"))?; + Duration::try_from_secs_f64(secs).map_err(|e| format!("{e}")) +} + #[tokio::main] async fn main() -> Result<()> { let cli = Cli::parse(); @@ -68,7 +139,7 @@ async fn main() -> Result<()> { stdin, include_selectors, exclude_selectors, - max_size, + limits, json: output_json, base_url, main_content, @@ -80,7 +151,7 @@ async fn main() -> Result<()> { } let mut builder = Markkit::builder() - .max_input_size(max_size) + .limits(limits.apply(Limits::default())) .main_content(main_content); for sel in &include_selectors { @@ -141,4 +212,60 @@ mod tests { fn verify_cli() { Cli::command().debug_assert(); } + + fn limits_from(args: &[&str]) -> Limits { + let cli = Cli::try_parse_from(["markkit", "extract", "x"].iter().chain(args)).unwrap(); + let Commands::Extract { limits, .. } = cli.command; + limits.apply(Limits::default()) + } + + #[test] + fn unset_limit_flags_keep_defaults() { + assert_eq!(limits_from(&[]), Limits::default()); + } + + #[test] + fn every_limit_flag_maps() { + let l = limits_from(&[ + "--max-input-size=1", + "--max-output-size=2", + "--timeout=0.5", + "--max-nesting-depth=3", + "--max-nodes=4", + "--max-archive-entries=5", + "--max-archive-entry-size=6", + "--max-archive-total-size=7", + "--max-sheet-rows=8", + "--max-sheet-cols=9", + "--max-spreadsheet-cells=10", + "--max-redirects=11", + ]); + let want = Limits::default() + .max_input_size(1) + .max_output_size(2) + .timeout(Duration::from_millis(500)) + .max_nesting_depth(3) + .max_nodes(4) + .max_archive_entries(5) + .max_archive_entry_size(6) + .max_archive_total_size(7) + .max_sheet_rows(8) + .max_sheet_cols(9) + .max_spreadsheet_cells(10) + .max_redirects(11); + assert_eq!(l, want); + } + + #[test] + fn legacy_max_size_alias() { + assert_eq!(limits_from(&["--max-size=9"]).max_input_size, 9); + } + + #[test] + fn bad_timeout_rejected() { + for bad in ["-1", "nan", "abc"] { + let arg = format!("--timeout={bad}"); + assert!(Cli::try_parse_from(["markkit", "extract", "x", &arg]).is_err()); + } + } } diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 2d50944..1b2e9c8 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -199,3 +199,24 @@ fn missing_source_errors() { assert!(!output.status.success()); } + +#[test] +fn limit_flags_enforced() { + let mut child = markkit_bin() + .args(["extract", "--stdin", "--max-output-size", "5"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + child + .stdin + .take() + .unwrap() + .write_all(b"

hello world, this is long

") + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("max_output_size"), "stderr: {stderr}"); +} diff --git a/crates/markkit-js/README.md b/crates/markkit-js/README.md index a8d07e7..0f194a3 100644 --- a/crates/markkit-js/README.md +++ b/crates/markkit-js/README.md @@ -33,7 +33,9 @@ const doc = extractDocumentSync(fs.readFileSync('report.docx'), { fileName: 'rep console.log(doc.title, doc.format, doc.warnings, doc.markdown); ``` -Fields: `maxInputSize`, `maxProcessingSecs`, `preserveLinks`, `preserveImages`, +Fields: `maxInputSize`, `maxOutputSize`, `timeout` (seconds, covers fetch, parsing and +rendering; `maxProcessingSecs` is the whole-second form), `maxNodes`, `maxNestingDepth`, +`baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors` (added to the defaults), `maxArchiveEntries`, `maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, `maxSpreadsheetCells`, `includeHiddenSheets`, `allowPrivateNetworks` (trusted URLs only), diff --git a/crates/markkit-js/__test__/options.test.cjs b/crates/markkit-js/__test__/options.test.cjs index 1d393a8..aa60a0c 100644 --- a/crates/markkit-js/__test__/options.test.cjs +++ b/crates/markkit-js/__test__/options.test.cjs @@ -24,9 +24,29 @@ test("input size limit", () => { assert.throws(() => m.extractHtmlSync("

too long

", { maxInputSize: 4 }), /Resource limit/); }); +test("resource limit options", () => { + const html = "

hello world

".repeat(200); + assert.throws(() => m.extractHtmlSync(html, { maxOutputSize: 100 }), /max_output_size/); + assert.throws(() => m.extractHtmlSync(html, { maxNodes: 50 }), /max_nodes/); + assert.throws(() => m.extractHtmlSync(html, { timeout: 0 }), /timeout/); + assert.ok(m.extractHtmlSync(html, { timeout: 1e300, maxOutputSize: 1 << 20, maxNodes: 10000 }).includes("hello")); + const deep = "
".repeat(20) + "

Deep

" + "
".repeat(20); + assert.ok(!m.extractHtmlSync(deep, { maxNestingDepth: 5 }).includes("> ".repeat(6))); +}); + +test("baseUrl and mainContent options", () => { + const page = '

T

' + "

Long article text, with commas, and more words here.

".repeat(8) + '

rel

'; + const md = m.extractHtmlSync(page, { baseUrl: "https://x.test/d/", mainContent: true }); + assert.ok(md.includes("(https://x.test/a)")); + assert.ok(!md.includes("side")); + assert.ok(m.extractHtmlSync(page).includes("side")); +}); + test("invalid numeric options are rejected", () => { assert.throws(() => m.extractHtmlSync("

x

", { maxArchiveTotalSize: -1 }), /invalid options/); assert.throws(() => m.extractHtmlSync("

x

", { maxSpreadsheetCells: 1.5 }), /invalid options/); + assert.throws(() => m.extractHtmlSync("

x

", { timeout: -1 }), /invalid options/); + assert.throws(() => m.extractHtmlSync("

x

", { timeout: NaN }), /invalid options/); }); test("extractDocument returns metadata", async () => { diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index d7d5ea6..9f1f3bf 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -17,8 +17,22 @@ use napi_derive::napi; pub struct MarkkitOptions { /// Maximum input size in bytes (default 50 MiB). pub max_input_size: Option, - /// Upper bound for `extractUrl` in seconds (default 60). + /// Wall-clock budget per call in whole seconds (default 60). Prefer `timeout`. pub max_processing_secs: Option, + /// Wall-clock budget per call in seconds, fractions allowed (default 60); + /// covers URL fetch, parsing and rendering. Wins over `maxProcessingSecs`. + pub timeout: Option, + /// Maximum Markdown output in bytes (default 128 MiB). + pub max_output_size: Option, + /// Maximum parsed nodes (default 5 000 000). + pub max_nodes: Option, + /// Maximum element nesting (default 256, capped at 512). + pub max_nesting_depth: Option, + /// Absolute URL relative links and images resolve against (`extractUrl` + /// uses the final response URL). + pub base_url: Option, + /// Keep only the detected main content (article body) of HTML pages. + pub main_content: Option, pub preserve_links: Option, pub preserve_images: Option, /// CSS selectors to include (empty = whole document). @@ -98,6 +112,19 @@ fn to_u64(v: f64, field: &str) -> napi::Result { } } +/// Seconds as a JS number; non-negative (values too large for `Duration` mean no deadline). +fn to_duration(v: f64) -> napi::Result { + std::time::Duration::try_from_secs_f64(v).or_else(|_| { + if v.is_nan() || v < 0.0 { + Err(napi::Error::from_reason( + "invalid options: `timeout` must be a non-negative number of seconds", + )) + } else { + Ok(std::time::Duration::MAX) + } + }) +} + fn markkit(options: Option) -> napi::Result { let o = options.unwrap_or_default(); let mut b = markkit::Markkit::builder(); @@ -107,6 +134,24 @@ fn markkit(options: Option) -> napi::Result { if let Some(v) = o.max_processing_secs { b = b.max_processing_secs(v.into()); } + if let Some(v) = o.timeout { + b = b.timeout(to_duration(v)?); + } + if let Some(v) = o.max_output_size { + b = b.max_output_size(v as usize); + } + if let Some(v) = o.max_nodes { + b = b.max_nodes(v as usize); + } + if let Some(v) = o.max_nesting_depth { + b = b.max_nesting_depth(v as usize); + } + if let Some(v) = o.base_url { + b = b.base_url(v); + } + if let Some(v) = o.main_content { + b = b.main_content(v); + } if let Some(v) = o.preserve_links { b = b.preserve_links(v); } diff --git a/crates/markkit-python/README.md b/crates/markkit-python/README.md index 3b4ca3b..437da4c 100644 --- a/crates/markkit-python/README.md +++ b/crates/markkit-python/README.md @@ -17,8 +17,17 @@ import markkit print(markkit.extract_html_sync("

Hello

")) # # Hello + +# Resource limits (all optional, generous defaults); breaches raise RuntimeError +limits = markkit.Limits(timeout=10.0, max_output_size=8 << 20, max_nesting_depth=128) +markkit.extract_html_sync("

Hello

", limits=limits) ``` +`Limits` fields: `max_input_size`, `max_output_size`, `timeout` (seconds), +`max_nesting_depth`, `max_nodes`, `max_archive_entries`, `max_archive_entry_size`, +`max_archive_total_size`, `max_sheet_rows`, `max_sheet_cols`, +`max_spreadsheet_cells`, `max_redirects`. Every function accepts `limits=`. + Publication verifies wheel metadata, installs the actual wheel into a clean virtual environment and exercises native extraction before upload. diff --git a/crates/markkit-python/src/lib.rs b/crates/markkit-python/src/lib.rs index 7f3c744..d28c0f6 100644 --- a/crates/markkit-python/src/lib.rs +++ b/crates/markkit-python/src/lib.rs @@ -1,22 +1,162 @@ +use std::time::Duration; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; +/// Resource limits (mirrors `markkit::Limits`). Every argument is optional and +/// defaults to the library default; violations raise `RuntimeError` naming +/// the limit. +/// +/// Keyword arguments: max_input_size, max_output_size, timeout (seconds, +/// float), max_nesting_depth, max_nodes, max_archive_entries, +/// max_archive_entry_size, max_archive_total_size, max_sheet_rows, +/// max_sheet_cols, max_spreadsheet_cells, max_redirects. +#[pyclass(name = "Limits", module = "markkit", frozen, get_all)] +#[derive(Clone)] +struct Limits { + max_input_size: usize, + max_output_size: usize, + timeout: f64, + max_nesting_depth: usize, + max_nodes: usize, + max_archive_entries: usize, + max_archive_entry_size: u64, + max_archive_total_size: u64, + max_sheet_rows: usize, + max_sheet_cols: usize, + max_spreadsheet_cells: u64, + max_redirects: usize, +} + +#[pymethods] +impl Limits { + #[new] + #[pyo3(signature = ( + *, + max_input_size=None, + max_output_size=None, + timeout=None, + max_nesting_depth=None, + max_nodes=None, + max_archive_entries=None, + max_archive_entry_size=None, + max_archive_total_size=None, + max_sheet_rows=None, + max_sheet_cols=None, + max_spreadsheet_cells=None, + max_redirects=None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + max_input_size: Option, + max_output_size: Option, + timeout: Option, + max_nesting_depth: Option, + max_nodes: Option, + max_archive_entries: Option, + max_archive_entry_size: Option, + max_archive_total_size: Option, + max_sheet_rows: Option, + max_sheet_cols: Option, + max_spreadsheet_cells: Option, + max_redirects: Option, + ) -> PyResult { + let d = markkit::Limits::default(); + if let Some(t) = timeout { + Duration::try_from_secs_f64(t) + .map_err(|e| PyValueError::new_err(format!("timeout: {e}")))?; + } + Ok(Self { + max_input_size: max_input_size.unwrap_or(d.max_input_size), + max_output_size: max_output_size.unwrap_or(d.max_output_size), + timeout: timeout.unwrap_or(d.timeout.as_secs_f64()), + max_nesting_depth: max_nesting_depth.unwrap_or(d.max_nesting_depth), + max_nodes: max_nodes.unwrap_or(d.max_nodes), + max_archive_entries: max_archive_entries.unwrap_or(d.max_archive_entries), + max_archive_entry_size: max_archive_entry_size.unwrap_or(d.max_archive_entry_size), + max_archive_total_size: max_archive_total_size.unwrap_or(d.max_archive_total_size), + max_sheet_rows: max_sheet_rows.unwrap_or(d.max_sheet_rows), + max_sheet_cols: max_sheet_cols.unwrap_or(d.max_sheet_cols), + max_spreadsheet_cells: max_spreadsheet_cells.unwrap_or(d.max_spreadsheet_cells), + max_redirects: max_redirects.unwrap_or(d.max_redirects), + }) + } + + fn __repr__(&self) -> String { + format!( + "Limits(max_input_size={}, max_output_size={}, timeout={}, max_nesting_depth={}, \ + max_nodes={}, max_archive_entries={}, max_archive_entry_size={}, \ + max_archive_total_size={}, max_sheet_rows={}, max_sheet_cols={}, \ + max_spreadsheet_cells={}, max_redirects={})", + self.max_input_size, + self.max_output_size, + self.timeout, + self.max_nesting_depth, + self.max_nodes, + self.max_archive_entries, + self.max_archive_entry_size, + self.max_archive_total_size, + self.max_sheet_rows, + self.max_sheet_cols, + self.max_spreadsheet_cells, + self.max_redirects + ) + } +} + +impl Limits { + fn to_core(&self) -> markkit::Limits { + markkit::Limits::default() + .max_input_size(self.max_input_size) + .max_output_size(self.max_output_size) + // Validated in `new`. + .timeout(Duration::try_from_secs_f64(self.timeout).unwrap_or(Duration::ZERO)) + .max_nesting_depth(self.max_nesting_depth) + .max_nodes(self.max_nodes) + .max_archive_entries(self.max_archive_entries) + .max_archive_entry_size(self.max_archive_entry_size) + .max_archive_total_size(self.max_archive_total_size) + .max_sheet_rows(self.max_sheet_rows) + .max_sheet_cols(self.max_sheet_cols) + .max_spreadsheet_cells(self.max_spreadsheet_cells) + .max_redirects(self.max_redirects) + } +} + +fn builder(limits: Option<&Limits>) -> markkit::MarkkitBuilder { + let b = markkit::Markkit::builder(); + match limits { + Some(l) => b.limits(l.to_core()), + None => b, + } +} + +fn err(e: markkit::Error) -> PyErr { + PyRuntimeError::new_err(e.to_string()) +} + /// Extract markdown from an HTML string (async). #[pyfunction] -fn extract_html(py: Python<'_>, html: String) -> PyResult> { +#[pyo3(signature = (html, *, limits=None))] +fn extract_html<'py>( + py: Python<'py>, + html: String, + limits: Option>, +) -> PyResult> { + let m = builder(limits.as_deref()).build(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let m = markkit::Markkit::builder().build(); - m.extract_html(&html) - .await - .map_err(|e| PyErr::new::(e.to_string())) + m.extract_html(&html).await.map_err(err) }) } /// Extract markdown from an HTML string (synchronous). #[pyfunction] -fn extract_html_sync(html: String) -> PyResult { - let m = markkit::Markkit::builder().build(); - m.extract_html_sync(&html) - .map_err(|e| PyErr::new::(e.to_string())) +#[pyo3(signature = (html, *, limits=None))] +fn extract_html_sync(html: String, limits: Option>) -> PyResult { + builder(limits.as_deref()) + .build() + .extract_html_sync(&html) + .map_err(err) } /// Extract markdown from an HTML string with configuration options (synchronous). @@ -24,16 +164,25 @@ fn extract_html_sync(html: String) -> PyResult { /// Keyword arguments: /// include_selectors: list of CSS selectors to include /// exclude_selectors: list of CSS selectors to exclude -/// max_input_size: maximum input size in bytes +/// max_input_size: maximum input size in bytes (overrides limits.max_input_size) +/// base_url: absolute URL relative links and images resolve against +/// main_content: keep only the detected article body (default False) +/// limits: markkit.Limits #[pyfunction] -#[pyo3(signature = (html, *, include_selectors=None, exclude_selectors=None, max_input_size=None))] +#[pyo3(signature = (html, *, include_selectors=None, exclude_selectors=None, max_input_size=None, base_url=None, main_content=false, limits=None))] fn extract_html_sync_with_options( html: String, include_selectors: Option>, exclude_selectors: Option>, max_input_size: Option, + base_url: Option, + main_content: bool, + limits: Option>, ) -> PyResult { - let mut builder = markkit::Markkit::builder(); + let mut builder = builder(limits.as_deref()).main_content(main_content); + if let Some(base) = base_url { + builder = builder.base_url(base); + } if let Some(size) = max_input_size { builder = builder.max_input_size(size); } @@ -47,40 +196,45 @@ fn extract_html_sync_with_options( builder = builder.exclude_selector(sel); } } - let m = builder.build(); - m.extract_html_sync(&html) - .map_err(|e| PyErr::new::(e.to_string())) + builder.build().extract_html_sync(&html).map_err(err) } /// Extract markdown from a URL. #[pyfunction] -fn extract_url(py: Python<'_>, url: String) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let m = markkit::Markkit::builder().build(); - m.extract_url(&url) - .await - .map_err(|e| PyErr::new::(e.to_string())) - }) +#[pyo3(signature = (url, *, limits=None))] +fn extract_url<'py>( + py: Python<'py>, + url: String, + limits: Option>, +) -> PyResult> { + let m = builder(limits.as_deref()).build(); + pyo3_async_runtimes::tokio::future_into_py( + py, + async move { m.extract_url(&url).await.map_err(err) }, + ) } /// Extract markdown from bytes with optional content type. #[pyfunction] -fn extract_bytes( - py: Python<'_>, +#[pyo3(signature = (data, content_type=None, *, limits=None))] +fn extract_bytes<'py>( + py: Python<'py>, data: Vec, content_type: Option, -) -> PyResult> { + limits: Option>, +) -> PyResult> { + let m = builder(limits.as_deref()).build(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let m = markkit::Markkit::builder().build(); m.extract_bytes(&data, content_type.as_deref()) .await - .map_err(|e| PyErr::new::(e.to_string())) + .map_err(err) }) } #[pymodule] #[pyo3(name = "markkit")] fn markkit_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; m.add_function(wrap_pyfunction!(extract_html, m)?)?; m.add_function(wrap_pyfunction!(extract_html_sync, m)?)?; m.add_function(wrap_pyfunction!(extract_html_sync_with_options, m)?)?; diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index 76fe4c6..30c020a 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -37,10 +37,11 @@ console.log(doc.title, doc.format, doc.markdown); ## Options -`maxInputSize`, `preserveLinks`, `preserveImages`, `includeSelectors`, +`maxInputSize`, `maxOutputSize`, `maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors` (added to the defaults), `maxArchiveEntries`, `maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, -`maxSpreadsheetCells`, `includeHiddenSheets`. Unknown keys throw. +`maxSpreadsheetCells`, `includeHiddenSheets`. Unknown keys throw. There is no +`timeout`: wasm has no clock, so bound time in the host (e.g. a Worker timeout). ## Agent tool diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index 0f73332..3c80df0 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -38,6 +38,23 @@ test("input size limit", () => { assert.throws(() => mk.extractHtml("

too long

"), /Resource limit/); }); +test("output, node and depth limits", () => { + const html = "

hello world

".repeat(200); + assert.throws(() => new Markkit({ maxOutputSize: 100 }).extractHtml(html), /max_output_size/); + assert.throws(() => new Markkit({ maxNodes: 50 }).extractHtml(html), /max_nodes/); + const deep = "
".repeat(20) + "

Deep

" + "
".repeat(20); + assert.ok(!new Markkit({ maxNestingDepth: 5 }).extractHtml(deep).includes("> ".repeat(6))); + assert.throws(() => new Markkit({ timeout: 1 }), /invalid options/); +}); + +test("baseUrl and mainContent options", () => { + const page = '

T

' + "

Long article text, with commas, and more words here.

".repeat(8) + '

rel

'; + const md = new Markkit({ baseUrl: "https://x.test/d/", mainContent: true }).extractHtml(page); + assert.ok(md.includes("(https://x.test/a)")); + assert.ok(!md.includes("side")); + assert.ok(new Markkit().extractHtml(page).includes("side")); +}); + test("extractDocument returns metadata", () => { const mk = new Markkit(); const out = mk.extractDocument(new TextEncoder().encode("

bytes

"), { fileName: "a.html" }); diff --git a/crates/markkit-wasm/js/index.d.ts b/crates/markkit-wasm/js/index.d.ts index 0fb6808..b465432 100644 --- a/crates/markkit-wasm/js/index.d.ts +++ b/crates/markkit-wasm/js/index.d.ts @@ -10,6 +10,16 @@ export function version(): string; export interface MarkkitOptions { /** Maximum input size in bytes (default 50 MiB). */ maxInputSize?: number; + /** Maximum Markdown output in bytes (default 128 MiB). */ + maxOutputSize?: number; + /** Maximum parsed nodes (default 5 000 000). */ + maxNodes?: number; + /** Maximum element nesting (default 256, capped at 512). */ + maxNestingDepth?: number; + /** Absolute URL relative links and images resolve against. */ + baseUrl?: string; + /** Keep only the detected main content (article body) of HTML pages. */ + mainContent?: boolean; preserveLinks?: boolean; preserveImages?: boolean; /** CSS selectors to include (empty = whole document). */ diff --git a/crates/markkit-wasm/src/lib.rs b/crates/markkit-wasm/src/lib.rs index c49ef35..fe279da 100644 --- a/crates/markkit-wasm/src/lib.rs +++ b/crates/markkit-wasm/src/lib.rs @@ -6,6 +6,8 @@ //! so typos fail loudly instead of silently using defaults. //! - Bytes in, Markdown out. No URL fetching in wasm: the host fetches. //! - Errors are thrown as `Error` with the Rust `Display` text. +//! - No `timeout` option: wasm32-unknown-unknown has no clock, so the core +//! budget has no deadline here; hosts bound time (Worker timeout, CPU limit). use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; @@ -18,6 +20,13 @@ use markkit::{DocumentHint, MarkkitBuilder}; #[serde(rename_all = "camelCase", deny_unknown_fields)] struct Options { max_input_size: Option, + max_output_size: Option, + max_nodes: Option, + /// Capped at 512. + max_nesting_depth: Option, + /// Absolute URL relative links and images resolve against. + base_url: Option, + main_content: Option, preserve_links: Option, preserve_images: Option, /// Added to the default selector set. @@ -43,6 +52,11 @@ impl Options { } set!( max_input_size, + max_output_size, + max_nodes, + max_nesting_depth, + base_url, + main_content, preserve_links, preserve_images, max_archive_entries, diff --git a/docs/getting-started.md b/docs/getting-started.md index ab06b78..cae9b8f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -77,6 +77,12 @@ let markkit = Markkit::builder() .build(); ``` +Every resource an input can drive (input/output size, time, nesting, node count, +archives, spreadsheets, redirects) is bounded by `markkit::Limits` with generous +defaults. The same limits are available as CLI flags (`markkit extract --help`, +"Limits" section), as `markkit.Limits(...)` in Python and as flat option keys in +Node.js and WASM. + ## Next Steps - [HTML extraction](../knowledge/formats/html-extraction.md): element mapping and stripping rules diff --git a/knowledge/log.md b/knowledge/log.md index 293615e..68d7709 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -27,4 +27,5 @@ * **Accuracy pass**: [Python Package](runtimes/python-package.md) now names the shipped `markkit` module; [Examples](operations/examples.md) lists the four real examples; [Evaluation Harness](operations/eval.md) is marked as a stub; [Threat Model](security/threat-model.md) T3 records that `http_client` is a default feature and SSRF mitigations are still open. * **Maintenance contract**: [Maintenance](operations/maintenance.md) adopts the analyze, fix, ship invocation contract and a workflow credential audit. * **Benchmarks**: `markkit-bench` is no longer a stub. It measures latency, ops/s, throughput and per-op heap allocations (count, bytes, peak) plus process peak RSS over a generated corpus, and `just bench` saves JSON + Markdown runs to `crates/markkit-bench/results/`, like bashkit. Criterion moves to `just bench-criterion`. See [Performance Results](operations/performance-results.md). -* **Resource limits**: all limits moved into one `Limits` struct (bashkit `ExecutionLimits` style). New: `timeout` now bounds parsing and rendering, not just URL fetch; `max_output_size` (128 MiB, stops table amplification); `max_nodes` (5M); `max_nesting_depth` configurable (256, was hardcoded 200 HTML / 256 XML) with a 512 hard ceiling. Threat T2h, invariants S14-S16, `limits` tests. See [Threat Model](security/threat-model.md) T2. +* **Resource limits**: all limits moved into one `Limits` struct (bashkit `ExecutionLimits` style). New: `timeout` now bounds parsing and rendering, not just URL fetch; `max_output_size` (128 MiB, stops table amplification); `max_nodes` (5M); `max_nesting_depth` configurable (256, was hardcoded 200 HTML / 256 XML) with a 512 hard ceiling. Threat T2g, invariants S14-S16, `limits` tests. See [Threat Model](security/threat-model.md) T2. +* **Limits in CLI and bindings**: every `Limits` field is a CLI flag and a Python `markkit.Limits` argument; Node `MarkkitOptions` and WASM `Options` gain flat `timeout`, `maxOutputSize`, `maxNodes`, `maxNestingDepth` keys; parity table updated. See [Node.js Package](runtimes/node-package.md), [Python Package](runtimes/python-package.md), [WASM Package](runtimes/wasm-package.md). diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 9de5657..7343ced 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -30,9 +30,10 @@ const doc = m.extractDocumentSync(buf, { fileName: 'report.docx' }); Every function takes an optional trailing `options` object (`MarkkitOptions`): the camelCase builder fields shared with the [WASM Package](wasm-package.md) -(`maxInputSize`, `preserveLinks`, `preserveImages`, `includeSelectors`, +(`maxInputSize`, `maxOutputSize`, `maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet limits, `includeHiddenSheets`) plus the URL -policy fields `maxProcessingSecs`, `allowPrivateNetworks`, `allowedHosts`, +policy fields `timeout` (seconds, float; whole call), `maxProcessingSecs` +(whole-second alias, `timeout` wins), `allowPrivateNetworks`, `allowedHosts`, `maxRedirects`. Selectors add to the defaults. 64-bit limits arrive as JS numbers and must be whole and non-negative. Unlike WASM, unknown keys are ignored (napi objects do not report them). @@ -44,7 +45,8 @@ Errors reject (or throw) with the Rust error's `Display` text as the reason. | Capability | Python | Node.js | WASM | |---|---|---|---| | HTML async / sync | yes | yes | sync | -| HTML with options | `extract_html_sync_with_options` (selectors, size) | all builder options | all builder options | +| HTML with options | `extract_html_sync_with_options` (selectors, size, `base_url`, `main_content`) | all builder options | all builder options | +| Resource limits (`Limits`) | `limits=markkit.Limits(...)` on every function | flat option keys, incl. `timeout` | flat option keys, no `timeout` (no clock) | | URL | yes, defaults only | yes, with URL policy options | no (host fetches) | | Bytes | yes | yes | yes | | `extract_document` (metadata) | no | yes | yes | diff --git a/knowledge/runtimes/python-package.md b/knowledge/runtimes/python-package.md index 087a2d5..87d8485 100644 --- a/knowledge/runtimes/python-package.md +++ b/knowledge/runtimes/python-package.md @@ -33,9 +33,19 @@ markdown = markkit.extract_html_sync_with_options( include_selectors=["article"], exclude_selectors=["nav"], max_input_size=1_000_000, + base_url="https://example.com/docs/", + main_content=True, ) + +# Resource limits: every function takes keyword-only `limits=` +limits = markkit.Limits(timeout=10.0, max_output_size=8 << 20) +markdown = markkit.extract_html_sync(html, limits=limits) ``` +`markkit.Limits` mirrors `markkit::Limits` field for field (`timeout` in float +seconds); unset arguments keep library defaults and are readable as attributes. +Invalid values raise `ValueError` / `OverflowError` at construction. + Errors surface as `RuntimeError` carrying the Rust error's `Display` text. ## Build diff --git a/knowledge/runtimes/wasm-package.md b/knowledge/runtimes/wasm-package.md index 33bbb45..3887068 100644 --- a/knowledge/runtimes/wasm-package.md +++ b/knowledge/runtimes/wasm-package.md @@ -37,7 +37,8 @@ const tool = new MarkkitTool({ locale, maxOutputChars }); tool.toolDefinition(); await tool.execute(args); ``` -Options are camelCase builder fields (`maxInputSize`, `preserveLinks`, +Options are camelCase builder fields (`maxInputSize`, `maxOutputSize`, +`maxNodes`, `maxNestingDepth`, `baseUrl`, `mainContent`, `preserveLinks`, `preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet limits, `includeHiddenSheets`). Selectors add to the defaults, as in Python. Unknown keys and wrong types throw `invalid options`: values go through