diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 22de698..b4170d0 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -33,6 +33,8 @@ jobs: run: cd crates/markkit-js && npx --no-install napi build --release --platform - name: Verify build preserved the patched lockfile run: git diff --exit-code -- Cargo.lock + - name: Binding tests + run: cd crates/markkit-js && node --test __test__/*.test.cjs - name: Verify installed npm package run: | cd crates/markkit-js diff --git a/CHANGELOG.md b/CHANGELOG.md index 85fa598..3164e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `@everruns/markkit-wasm`: browser / edge WebAssembly package (`crates/markkit-wasm`), single-threaded, no special headers. `Markkit` class (HTML, DOCX, XLSX/XLS/ODS with validated camelCase options) and `MarkkitTool`. Built by `crates/markkit-wasm/scripts/build.sh`, published by `publish-wasm.yml`. +- Node (`@everruns/markkit`): every function takes an optional trailing `options` object with all + builder settings (selectors, links/images, input, archive and sheet limits) and the URL policy + (`allowPrivateNetworks`, `allowedHosts`, `maxRedirects`, `maxProcessingSecs`). New + `extractDocument` / `extractDocumentSync` return Markdown with title, format, warnings, stats and + page metadata. - `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats. ### Security diff --git a/crates/markkit-js/README.md b/crates/markkit-js/README.md index ccd1d8a..a8d07e7 100644 --- a/crates/markkit-js/README.md +++ b/crates/markkit-js/README.md @@ -1,6 +1,6 @@ # @everruns/markkit -Native Node.js bindings for Markkit HTML-to-Markdown extraction. +Native Node.js bindings for Markkit: HTML, DOCX and spreadsheets to Markdown. This npm release supports **Linux x64 with glibc**. It does not include binaries for macOS, Windows, ARM64 or musl Linux. Other platforms require a separate future @@ -15,3 +15,28 @@ const { extractHtmlSync } = require('@everruns/markkit'); console.log(extractHtmlSync('

Hello

')); // # Hello ``` + +## Options + +Every function takes an optional trailing options object: + +```js +const { extractHtml, extractUrl, extractDocumentSync } = require('@everruns/markkit'); + +await extractHtml(html, { includeSelectors: ['article'], excludeSelectors: ['.ad'], preserveImages: false }); + +// URLs: non-public addresses are blocked by default. +await extractUrl('https://docs.example.com/page', { allowedHosts: ['docs.example.com'], maxRedirects: 2 }); + +// Documents with metadata (HTML, DOCX, XLSX/XLS/ODS). +const doc = extractDocumentSync(fs.readFileSync('report.docx'), { fileName: 'report.docx' }, { maxSheetRows: 500 }); +console.log(doc.title, doc.format, doc.warnings, doc.markdown); +``` + +Fields: `maxInputSize`, `maxProcessingSecs`, `preserveLinks`, `preserveImages`, +`includeSelectors`, `excludeSelectors` (added to the defaults), `maxArchiveEntries`, +`maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, +`maxSpreadsheetCells`, `includeHiddenSheets`, `allowPrivateNetworks` (trusted URLs only), +`allowedHosts`, `maxRedirects`. + +For browsers and edge runtimes use [`@everruns/markkit-wasm`](https://www.npmjs.com/package/@everruns/markkit-wasm). diff --git a/crates/markkit-js/__test__/options.test.cjs b/crates/markkit-js/__test__/options.test.cjs new file mode 100644 index 0000000..1d393a8 --- /dev/null +++ b/crates/markkit-js/__test__/options.test.cjs @@ -0,0 +1,67 @@ +// Node binding tests (run after `napi build --platform`): options parity, +// extractDocument, URL policy. All inputs inline; the URL test uses a local server. +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const http = require("node:http"); +const m = require(".."); + +test("defaults unchanged without options", async () => { + assert.equal(m.extractHtmlSync("

Hello

"), "# Hello\n"); + assert.equal(await m.extractHtml("

a

"), "a\n"); + assert.equal(await m.extractBytes(Buffer.from("

b

"), "text/html"), "b\n"); +}); + +test("selector and link options apply to every entry point", async () => { + const html = '

Read this

'; + const options = { preserveLinks: false, includeSelectors: ["article"], excludeSelectors: [".ad"] }; + assert.equal(m.extractHtmlSync(html, options), "Read this\n"); + assert.equal(await m.extractHtml(html, options), "Read this\n"); + assert.equal(await m.extractBytes(Buffer.from(html), "text/html", options), "Read this\n"); + assert.equal(m.extractDocumentSync(Buffer.from(html), { fileName: "a.html" }, options).markdown, "Read this\n"); +}); + +test("input size limit", () => { + assert.throws(() => m.extractHtmlSync("

too long

", { maxInputSize: 4 }), /Resource limit/); +}); + +test("invalid numeric options are rejected", () => { + assert.throws(() => m.extractHtmlSync("

x

", { maxArchiveTotalSize: -1 }), /invalid options/); + assert.throws(() => m.extractHtmlSync("

x

", { maxSpreadsheetCells: 1.5 }), /invalid options/); +}); + +test("extractDocument returns metadata", async () => { + const out = await m.extractDocument(Buffer.from("

bytes

"), { contentType: "text/html" }); + assert.equal(out.markdown, "bytes\n"); + assert.equal(out.format, "html"); + assert.deepEqual(out.warnings, []); + assert.equal(out.title ?? null, null); + assert.throws(() => m.extractDocumentSync(Buffer.from("%PDF-1.7"), { fileName: "a.pdf" }), /Unsupported/); +}); + +test("extractDocument exposes page metadata", () => { + const html = 'T

b

'; + const out = m.extractDocumentSync(Buffer.from(html), { contentType: "text/html" }); + assert.equal(out.title, "T"); + assert.equal(out.metadata.siteName, "Site"); + assert.equal(out.metadata.canonicalUrl, "https://x.test/a"); + assert.equal(out.metadata.language, "en"); +}); + +function serve(body) { + return new Promise(resolve => { + const srv = http.createServer((_, res) => { res.setHeader("content-type", "text/html"); res.end(body); }); + srv.listen(0, "127.0.0.1", () => resolve(srv)); + }); +} + +test("extractUrl blocks private networks unless allowed", async () => { + const srv = await serve("

Local

"); + const url = `http://127.0.0.1:${srv.address().port}/`; + try { + await assert.rejects(m.extractUrl(url), /URL blocked/); + assert.equal(await m.extractUrl(url, { allowPrivateNetworks: true }), "# Local\n"); + await assert.rejects(m.extractUrl(url, { allowPrivateNetworks: true, allowedHosts: ["example.com"] }), /URL blocked/); + } finally { + srv.close(); + } +}); diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 08f7926..d7d5ea6 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -1,39 +1,256 @@ +//! Node.js bindings (`@everruns/markkit`). +//! +//! Decisions: +//! - Every entry point takes an optional trailing `options` object (camelCase +//! builder fields), so existing calls keep working and one options shape is +//! shared with `@everruns/markkit-wasm` and Python's keyword arguments. +//! - Selectors add to the defaults (as in Python and WASM). +//! - URL policy knobs (`allowPrivateNetworks`, `allowedHosts`, `maxRedirects`) +//! are opt-in; defaults keep the SSRF guard on. + +use napi::bindgen_prelude::Buffer; use napi_derive::napi; +/// Extraction options; every field is optional and defaults to the Rust builder. +#[napi(object)] +#[derive(Default)] +pub struct MarkkitOptions { + /// Maximum input size in bytes (default 50 MiB). + pub max_input_size: Option, + /// Upper bound for `extractUrl` in seconds (default 60). + pub max_processing_secs: Option, + pub preserve_links: Option, + pub preserve_images: Option, + /// CSS selectors to include (empty = whole document). + pub include_selectors: Option>, + /// CSS selectors to drop, added to nav/footer/header/script/style/noscript. + pub exclude_selectors: Option>, + pub max_archive_entries: Option, + pub max_archive_entry_size: Option, + pub max_archive_total_size: Option, + pub max_sheet_rows: Option, + pub max_sheet_cols: Option, + pub max_spreadsheet_cells: Option, + pub include_hidden_sheets: Option, + /// Let `extractUrl` reach loopback/private/link-local addresses. Trusted URLs only. + pub allow_private_networks: Option, + /// Hosts `extractUrl` may contact (checked on every redirect). + pub allowed_hosts: Option>, + /// Maximum redirects `extractUrl` follows (default 5). + pub max_redirects: Option, +} + +/// Format hints for `extractDocument`. +#[napi(object)] +#[derive(Default)] +pub struct DocumentHint { + /// MIME type, e.g. from a `Content-Type` header. + pub content_type: Option, + /// File name or URL path; only the extension is used. + pub file_name: Option, +} + +#[napi(object)] +pub struct ExtractionStats { + pub pages: Option, + pub sheets: Option, + pub rows: Option, + pub paragraphs: Option, + pub tables: Option, +} + +/// Document metadata (HTML meta, Open Graph, JSON-LD); fields absent when unknown. +#[napi(object)] +pub struct Metadata { + pub description: Option, + pub author: Option, + pub published: Option, + pub language: Option, + pub canonical_url: Option, + pub site_name: Option, + pub image: Option, +} + +/// Markdown plus metadata. +#[napi(object)] +pub struct Extraction { + pub markdown: String, + pub title: Option, + /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`). + pub format: String, + pub warnings: Vec, + pub stats: ExtractionStats, + pub metadata: Metadata, +} + +fn invalid(field: &str) -> napi::Error { + napi::Error::from_reason(format!( + "invalid options: `{field}` must be a non-negative integer" + )) +} + +/// JS numbers above 2^32 arrive as f64; accept only whole, non-negative values. +fn to_u64(v: f64, field: &str) -> napi::Result { + if v.is_finite() && v >= 0.0 && v.fract() == 0.0 && v <= u64::MAX as f64 { + Ok(v as u64) + } else { + Err(invalid(field)) + } +} + +fn markkit(options: Option) -> napi::Result { + let o = options.unwrap_or_default(); + let mut b = markkit::Markkit::builder(); + if let Some(v) = o.max_input_size { + b = b.max_input_size(v as usize); + } + if let Some(v) = o.max_processing_secs { + b = b.max_processing_secs(v.into()); + } + if let Some(v) = o.preserve_links { + b = b.preserve_links(v); + } + if let Some(v) = o.preserve_images { + b = b.preserve_images(v); + } + for s in o.include_selectors.unwrap_or_default() { + b = b.include_selector(s); + } + for s in o.exclude_selectors.unwrap_or_default() { + b = b.exclude_selector(s); + } + if let Some(v) = o.max_archive_entries { + b = b.max_archive_entries(v as usize); + } + if let Some(v) = o.max_archive_entry_size { + b = b.max_archive_entry_size(to_u64(v, "maxArchiveEntrySize")?); + } + if let Some(v) = o.max_archive_total_size { + b = b.max_archive_total_size(to_u64(v, "maxArchiveTotalSize")?); + } + if let Some(v) = o.max_sheet_rows { + b = b.max_sheet_rows(v as usize); + } + if let Some(v) = o.max_sheet_cols { + b = b.max_sheet_cols(v as usize); + } + if let Some(v) = o.max_spreadsheet_cells { + b = b.max_spreadsheet_cells(to_u64(v, "maxSpreadsheetCells")?); + } + if let Some(v) = o.include_hidden_sheets { + b = b.include_hidden_sheets(v); + } + if let Some(v) = o.allow_private_networks { + b = b.allow_private_networks(v); + } + for h in o.allowed_hosts.unwrap_or_default() { + b = b.allow_host(h); + } + if let Some(v) = o.max_redirects { + b = b.max_redirects(v as usize); + } + Ok(b.build()) +} + +fn err(e: markkit::Error) -> napi::Error { + napi::Error::from_reason(e.to_string()) +} + +fn count(v: Option) -> Option { + v.map(|n| u32::try_from(n).unwrap_or(u32::MAX)) +} + +fn to_js(e: markkit::Extraction) -> Extraction { + Extraction { + markdown: e.markdown, + title: e.title, + format: e.format.as_str().to_string(), + warnings: e.warnings, + stats: ExtractionStats { + pages: count(e.stats.pages), + sheets: count(e.stats.sheets), + rows: count(e.stats.rows), + paragraphs: count(e.stats.paragraphs), + tables: count(e.stats.tables), + }, + metadata: Metadata { + description: e.metadata.description, + author: e.metadata.author, + published: e.metadata.published, + language: e.metadata.language, + canonical_url: e.metadata.canonical_url, + site_name: e.metadata.site_name, + image: e.metadata.image, + }, + } +} + +fn document( + data: &[u8], + hint: Option, + options: Option, +) -> napi::Result { + let hint = hint.unwrap_or_default(); + markkit(options)? + .extract_document_sync( + data, + markkit::DocumentHint { + content_type: hint.content_type.as_deref(), + file_name: hint.file_name.as_deref(), + }, + ) + .map(to_js) + .map_err(err) +} + /// Extract markdown from an HTML string (async). #[napi] -pub async fn extract_html(html: String) -> napi::Result { - let m = markkit::Markkit::builder().build(); - m.extract_html(&html) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) +pub async fn extract_html(html: String, options: Option) -> napi::Result { + markkit(options)?.extract_html(&html).await.map_err(err) } /// Extract markdown from an HTML string (synchronous). #[napi] -pub fn extract_html_sync(html: String) -> napi::Result { - let m = markkit::Markkit::builder().build(); - m.extract_html_sync(&html) - .map_err(|e| napi::Error::from_reason(e.to_string())) +pub fn extract_html_sync(html: String, options: Option) -> napi::Result { + markkit(options)?.extract_html_sync(&html).map_err(err) } -/// Extract markdown from a URL. +/// Extract markdown from a URL. Non-public addresses are refused unless +/// `allowPrivateNetworks` is set. #[napi] -pub async fn extract_url(url: String) -> napi::Result { - let m = markkit::Markkit::builder().build(); - m.extract_url(&url) - .await - .map_err(|e| napi::Error::from_reason(e.to_string())) +pub async fn extract_url(url: String, options: Option) -> napi::Result { + markkit(options)?.extract_url(&url).await.map_err(err) } /// Extract markdown from a buffer with optional content type. #[napi] pub async fn extract_bytes( - data: napi::bindgen_prelude::Buffer, + data: Buffer, content_type: Option, + options: Option, ) -> napi::Result { - let m = markkit::Markkit::builder().build(); - m.extract_bytes(&data, content_type.as_deref()) + markkit(options)? + .extract_bytes(&data, content_type.as_deref()) .await - .map_err(|e| napi::Error::from_reason(e.to_string())) + .map_err(err) +} + +/// Convert a document (HTML, DOCX, XLSX/XLS/ODS) with metadata (async). +#[napi] +pub async fn extract_document( + data: Buffer, + hint: Option, + options: Option, +) -> napi::Result { + document(&data, hint, options) +} + +/// Convert a document (HTML, DOCX, XLSX/XLS/ODS) with metadata (synchronous). +#[napi] +pub fn extract_document_sync( + data: Buffer, + hint: Option, + options: Option, +) -> napi::Result { + document(&data, hint, options) } diff --git a/knowledge/log.md b/knowledge/log.md index f15aafd..5e83aa4 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-26 +* **Node options parity**: [Node.js Package](runtimes/node-package.md) functions take a trailing options object (all builder fields plus URL policy) and gain `extractDocument`/`extractDocumentSync`; the parity table now covers Python, Node and WASM. + * **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. * **Ship contract**: "ship" now ends with the merge commit's `main` CI green, not at merge; the `ship` skill gains existing-PR, auto-merge, and post-merge verification steps aligned with bashkit. See [Maintenance](operations/maintenance.md). * **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md). diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 519e39a..9de5657 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -18,29 +18,41 @@ via NAPI-RS. ## API ```javascript -const { extractHtml, extractHtmlSync, extractUrl, extractBytes } = require('@everruns/markkit'); - -const markdown = await extractHtml('

Hello

'); -const sync = extractHtmlSync('

Hello

'); -const fromUrl = await extractUrl('https://example.com'); -const fromBuffer = await extractBytes(Buffer.from('

Hello

'), 'text/html'); +const m = require('@everruns/markkit'); + +const markdown = await m.extractHtml('

Hello

', { includeSelectors: ['article'] }); +const sync = m.extractHtmlSync('

Hello

'); +const fromUrl = await m.extractUrl('https://example.com', { allowedHosts: ['example.com'] }); +const fromBuffer = await m.extractBytes(Buffer.from('

Hello

'), 'text/html'); +const doc = m.extractDocumentSync(buf, { fileName: 'report.docx' }); +// { markdown, title, format, warnings, stats, metadata } ``` +Every function takes an optional trailing `options` object (`MarkkitOptions`): +the camelCase builder fields shared with the [WASM Package](wasm-package.md) +(`maxInputSize`, `preserveLinks`, `preserveImages`, `includeSelectors`, +`excludeSelectors`, archive and sheet limits, `includeHiddenSheets`) plus the URL +policy fields `maxProcessingSecs`, `allowPrivateNetworks`, `allowedHosts`, +`maxRedirects`. Selectors add to the defaults. 64-bit limits arrive as JS numbers +and must be whole and non-negative. Unlike WASM, unknown keys are ignored +(napi objects do not report them). + Errors reject (or throw) with the Rust error's `Display` text as the reason. ## Binding parity -| Function | Python | Node.js | -|---|---|---| -| `extract_html` / `extractHtml` | yes | yes | -| `extract_html_sync` / `extractHtmlSync` | yes | yes | -| `extract_html_sync_with_options` | yes | **no** | -| `extract_url` / `extractUrl` | yes | yes | -| `extract_bytes` / `extractBytes` | yes | yes | - -Neither binding exposes `extract_document` (format, title, warnings, stats) or the -office-document limits yet; both use default builder settings except the Python -options call. +| Capability | Python | Node.js | WASM | +|---|---|---|---| +| HTML async / sync | yes | yes | sync | +| HTML with options | `extract_html_sync_with_options` (selectors, size) | all builder options | all builder options | +| URL | yes, defaults only | yes, with URL policy options | no (host fetches) | +| Bytes | yes | yes | yes | +| `extract_document` (metadata) | no | yes | yes | +| Agent tool | no | no | `MarkkitTool` | + +Tests: `crates/markkit-js/__test__/*.test.cjs` (`node --test`, run by `js.yml` +after the native build): options on every entry point, limits, invalid numbers, +`extractDocument`, and URL policy against a local server. ## Build