From 44c7d2e61123676bae0e84c037a5b41e0f6dc11f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:32:15 +0000 Subject: [PATCH 1/5] feat(tool): add MarkkitTool LLM agent tool (Everruns toolkit contract) Opt-in `tool` feature adds markkit::tool: MarkkitToolBuilder -> MarkkitTool -> single-use ToolExecution, mirroring bashkit's tool module. One call converts a url, html or base64 document to Markdown; args validated before execution, base64 bounded before decode, URL via the SSRF guard, output capped by max_output_chars. Adds Markkit::extract_url_document. --- .github/workflows/ci.yml | 1 + AGENTS.md | 1 + CHANGELOG.md | 8 + Cargo.lock | 3 + Cargo.toml | 5 + README.md | 1 + crates/markkit/Cargo.toml | 13 + crates/markkit/src/extract.rs | 17 +- crates/markkit/src/lib.rs | 4 + crates/markkit/src/tool.rs | 1119 +++++++++++++++++++++++ examples/agent_tool.rs | 32 + justfile | 2 + knowledge/index.md | 1 + knowledge/integrations/index.md | 3 + knowledge/integrations/tool-contract.md | 115 +++ knowledge/knowledge-contract.md | 1 + knowledge/log.md | 2 + knowledge/operations/examples.md | 7 + knowledge/security/threat-model.md | 1 + 19 files changed, 1334 insertions(+), 2 deletions(-) create mode 100644 crates/markkit/src/tool.rs create mode 100644 examples/agent_tool.rs create mode 100644 knowledge/integrations/index.md create mode 100644 knowledge/integrations/tool-contract.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 647799b..72c1b1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,7 @@ jobs: for ex in basic custom_selectors configuration ir_access; do cargo run -q -p markkit --example "$ex" done + cargo run -q -p markkit --features tool --example agent_tool check: name: Check diff --git a/AGENTS.md b/AGENTS.md index 9204780..881e4b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | foundations/ir | Canonical IR between parsers and the markdown renderer | | formats/html-extraction | HTML element mapping, stripping, whitespace, depth limits | | formats/format-support | Formats, bytes-in API, detection order, DOCX + spreadsheet mapping | +| integrations/tool-contract | `MarkkitTool` LLM tool (`tool` feature), Everruns toolkit contract | | security/security-model | One-page threat/invariant summary | | security/threat-model | Threats T1-T6, mitigations, test map, resource limits, invariants S1-S12 | | runtimes/python-package | PyO3 `markkit` module, maturin, PyPI | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..2a36203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- `tool` feature: `markkit::tool::MarkkitTool`, an LLM agent tool following the Everruns + toolkit-library contract (builder, OpenAI tool definition, input/output schemas, `tower::Service`, + single-use `ToolExecution`, `en-US`/`uk-UA` text). One call converts a `url`, `html` or base64 + document to Markdown, with output capped by `max_output_chars`. Example: `examples/agent_tool.rs`. +- `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats. + ### Security - `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: diff --git a/Cargo.lock b/Cargo.lock index 01303b3..b3aa161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1183,9 +1183,11 @@ name = "markkit" version = "0.1.1" dependencies = [ "anyhow", + "base64", "calamine", "cfb", "criterion", + "futures-core", "html5ever", "proptest", "quick-xml", @@ -1196,6 +1198,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tower", "tracing", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index a67f998..ab34c1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,11 @@ cfb = "0.15" # XLS (OLE2) container pre-scan # Logging tracing = "0.1" +# LLM tool contract (markkit `tool` feature) +tower = { version = "0.5", default-features = false, features = ["util"] } +futures-core = "0.3" +base64 = "0.23" + # Python bindings pyo3 = { version = "0.30" } pyo3-async-runtimes = { version = "0.30", features = ["tokio-runtime"] } diff --git a/README.md b/README.md index 0d9e55a..75fb4d2 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ let markkit = Markkit::builder() | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` | | `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` | +| `tool` | No | `markkit::tool::MarkkitTool` LLM agent tool (Everruns toolkit contract) | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | `http_client` is optional. Embedders that fetch content themselves can build without diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index f6f5d96..424777b 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -21,6 +21,8 @@ docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] +# LLM tool contract (`markkit::tool`): Everruns toolkit-library shape. +tool = ["dep:tower", "dep:futures-core", "dep:base64"] all_formats = ["html", "pdf", "docx", "spreadsheet"] [dependencies] @@ -52,12 +54,18 @@ reqwest = { workspace = true, optional = true } # Logging (optional) tracing = { workspace = true, optional = true } +# LLM tool (optional) +tower = { workspace = true, optional = true } +futures-core = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } + [dev-dependencies] anyhow.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "time"] } tempfile.workspace = true proptest.workspace = true criterion.workspace = true +tower = { workspace = true } zip = { workspace = true } cfb = { workspace = true } @@ -81,6 +89,11 @@ name = "configuration" path = "../../examples/configuration.rs" required-features = ["html"] +[[example]] +name = "agent_tool" +path = "../../examples/agent_tool.rs" +required-features = ["html", "tool"] + [[bench]] name = "extraction" harness = false diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 06ce32b..2f8de85 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -246,9 +246,22 @@ impl Markkit { /// by `max_processing_secs` ([`Error::ResourceLimit`]). #[cfg(feature = "http_client")] pub async fn extract_url(&self, url: &str) -> crate::Result { + self.extract_url_document(url).await.map(|e| e.markdown) + } + + /// [`Markkit::extract_url`] with metadata (format, title, warnings, stats). + /// + /// The response `Content-Type` is the format hint; same URL policy and limits. + #[cfg(feature = "http_client")] + pub async fn extract_url_document(&self, url: &str) -> crate::Result { let fetched = crate::fetch::fetch(url, &self.config).await?; - self.extract_bytes(&fetched.bytes, fetched.content_type.as_deref()) - .await + self.extract_document_sync( + &fetched.bytes, + DocumentHint { + content_type: fetched.content_type.as_deref(), + file_name: None, + }, + ) } /// PDF extraction is not implemented yet; report it explicitly rather than diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index d6453e2..e037f97 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -36,6 +36,8 @@ //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest +//! - **Agent tool** (`tool`): [`tool::MarkkitTool`], an LLM tool following the +//! Everruns toolkit-library contract (schemas, OpenAI definition, `tower::Service`) //! //! Embedders that do their own fetching can drop reqwest entirely: //! `markkit = { version = "0.1", default-features = false, features = ["docx", "spreadsheet"] }`. @@ -56,6 +58,8 @@ pub mod ir; pub mod render; #[cfg(feature = "spreadsheet")] mod spreadsheet; +#[cfg(feature = "tool")] +pub mod tool; #[cfg(feature = "docx")] mod xml; diff --git a/crates/markkit/src/tool.rs b/crates/markkit/src/tool.rs new file mode 100644 index 0000000..fbc52f9 --- /dev/null +++ b/crates/markkit/src/tool.rs @@ -0,0 +1,1119 @@ +//! LLM tool contract and [`MarkkitTool`]. +//! +//! # Public Library Contract +//! +//! Markkit follows the Everruns toolkit-library contract (same shape as +//! bashkit's `tool` module): +//! +//! ```text +//! ToolBuilder (config) -> Tool (metadata) -> ToolExecution (single-use runtime) +//! ``` +//! +//! [`MarkkitToolBuilder`] configures a reusable tool definition. [`MarkkitTool`] +//! exposes locale-aware metadata plus [`Tool::execution`] for validated, +//! single-use runs. [`ToolExecution`] returns a structured [`ToolOutput`]. +//! +//! Decisions (see `knowledge/integrations/tool-contract.md`): +//! - One tool, one call = one document. Source is exactly one of `url`, `html` +//! or `content_base64`; agents mostly pass a URL, hosts inject attachments. +//! - Tool arguments are untrusted LLM output: validated before an execution +//! exists, base64 length bounded before decoding, URL fetching goes through +//! the core SSRF guard and can be switched off per tool. +//! - Output is capped (`max_output_chars`, default 100 000 chars) so one call +//! cannot flood a context window; the model can lower it per call. +//! - Extraction failures are [`ToolError::UserFacing`] (safe for the model); +//! only I/O and serialization failures are [`ToolError::Internal`]. +//! - No incremental chunks today: the stream closes when execution ends. +//! - Lives in core behind the opt-in `tool` feature so plain library users do +//! not pull `tower`. +//! +//! # Example +//! +//! ``` +//! use markkit::tool::{MarkkitTool, Tool}; +//! +//! # #[tokio::main(flavor = "current_thread")] +//! # async fn main() { +//! let builder = MarkkitTool::builder().locale("en-US"); +//! assert!(builder.build_tool_definition()["function"]["parameters"].is_object()); +//! +//! let tool = builder.build(); +//! assert_eq!(tool.name(), "markkit"); +//! let output = tool +//! .execution(serde_json::json!({"html": "

Hi

there

"})) +//! .expect("valid args") +//! .execute() +//! .await +//! .expect("extraction succeeds"); +//! assert_eq!(output.result["markdown"], "# Hi\n\nthere\n"); +//! assert_eq!(output.result["format"], "html"); +//! # } +//! ``` + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use base64::Engine as _; +use futures_core::Stream; +use serde::{Deserialize, Serialize}; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::extract::{Extraction, Markkit}; +use crate::format::DocumentHint; + +type ToolExecutionFuture = Pin> + Send>>; +type ToolExecutionRunner = Box< + dyn FnOnce(Option>) -> ToolExecutionFuture + + Send, +>; + +/// Library version from Cargo.toml. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// Tool identifier; stable, not localized. +pub const TOOL_NAME: &str = "markkit"; + +/// Default cap on returned Markdown, in characters. +pub const DEFAULT_MAX_OUTPUT_CHARS: usize = 100_000; + +/// Standard `tower::Service` type for toolkit integrations. +pub type ToolService = + tower::util::BoxCloneService; + +/// Tool execution error. +/// +/// [`ToolError::UserFacing`] is short, localized and safe to send back to the +/// LLM. [`ToolError::Internal`] is for logs and stays English. +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] +pub enum ToolError { + /// Safe to show to the LLM/user. + #[error("{0}")] + UserFacing(String), + /// Internal failure for logs/diagnostics only. + #[error("{0}")] + Internal(String), +} + +impl ToolError { + /// Whether the message is safe to show to the LLM. + pub fn is_user_facing(&self) -> bool { + matches!(self, Self::UserFacing(_)) + } +} + +/// Image payload returned by a tool. Markkit never emits images; kept for +/// contract parity with other toolkit crates. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ToolImage { + pub base64: String, + pub media_type: String, +} + +/// Consumer-facing metadata that never goes to the LLM. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ToolOutputMetadata { + #[serde(with = "duration_millis")] + pub duration: Duration, + /// Kit-specific diagnostics: `stats` and `chars` (untruncated length). + pub extra: serde_json::Value, +} + +/// Structured execution result. +/// +/// [`ToolOutput::result`] is the JSON payload for the LLM tool result. +/// [`ToolOutput::metadata`] is reserved for the host application. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ToolOutput { + pub result: serde_json::Value, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub images: Vec, + pub metadata: ToolOutputMetadata, +} + +/// Incremental tool output chunk (`kind` is consumer-routable). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ToolOutputChunk { + pub data: serde_json::Value, + pub kind: String, +} + +/// Stream returned by [`ToolExecution::output_stream`]. Informational only; +/// the authoritative result comes from [`ToolExecution::execute`]. +pub struct ToolOutputStream { + receiver: tokio::sync::mpsc::UnboundedReceiver, +} + +impl Stream for ToolOutputStream { + type Item = ToolOutputChunk; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.receiver.poll_recv(cx) + } +} + +#[derive(Default)] +struct ToolExecutionStreamState { + sender: Option>, + receiver: Option>, +} + +/// Stateful, single-use tool execution. +/// +/// Build one with [`Tool::execution`]. Call [`ToolExecution::output_stream`] +/// before [`ToolExecution::execute`] if you need live updates. +pub struct ToolExecution { + runner: Option, + stream_state: Arc>, +} + +impl ToolExecution { + /// Wrap a runner for a [`Tool::execution`] implementation. + /// + /// `runner` receives a stream sender only when the caller asked for + /// [`Self::output_stream`] before executing. + pub fn new(runner: F) -> Self + where + F: FnOnce(Option>) -> Fut + + Send + + 'static, + Fut: Future> + Send + 'static, + { + Self { + runner: Some(Box::new(move |sender| Box::pin(runner(sender)))), + stream_state: Arc::new(Mutex::new(ToolExecutionStreamState::default())), + } + } + + /// Stream incremental output. Must be called before [`Self::execute`]. + /// Returns `None` on a second call. + pub fn output_stream(&self) -> Option { + let mut state = match self.stream_state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + if state.sender.is_none() { + let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); + state.sender = Some(sender); + state.receiver = Some(receiver); + } + state + .receiver + .take() + .map(|receiver| ToolOutputStream { receiver }) + } + + /// Run the execution to completion. + pub async fn execute(mut self) -> Result { + let sender = match self.stream_state.lock() { + Ok(mut state) => state.sender.take(), + Err(poisoned) => poisoned.into_inner().sender.take(), + }; + let Some(runner) = self.runner.take() else { + return Err(ToolError::Internal( + "tool execution may only be run once".to_string(), + )); + }; + runner(sender).await + } +} + +mod duration_millis { + use serde::{Deserialize, Deserializer, Serializer}; + use std::time::Duration; + + pub fn serialize(value: &Duration, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(value.as_millis() as u64) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let millis = u64::deserialize(deserializer)?; + Ok(Duration::from_millis(millis)) + } +} + +// ============================================================================ +// Tool Trait - Public Library Contract +// ============================================================================ + +/// Tool contract for LLM integration. +/// +/// This trait is a **public library contract**; breaking changes need a major +/// version bump. See `knowledge/integrations/tool-contract.md`. +pub trait Tool: Send + Sync { + /// Tool identifier (stable, not localized). + fn name(&self) -> &str; + /// Human-readable display name for UI. + fn display_name(&self) -> &str; + /// One-line description for tool listings. + fn short_description(&self) -> &str; + /// Token-efficient description for LLMs. + fn description(&self) -> &str; + /// Full documentation for LLMs (Markdown). + fn help(&self) -> String; + /// Condensed plain text for system prompts; starts with the tool name. + fn system_prompt(&self) -> String; + /// Locale used for user-facing text. + fn locale(&self) -> &str; + /// JSON Schema for the arguments. + fn input_schema(&self) -> serde_json::Value; + /// JSON Schema for [`ToolOutput::result`]. + fn output_schema(&self) -> serde_json::Value; + /// Library/tool version. + fn version(&self) -> &str; + /// Validate `args` and create a single-use execution. + fn execution(&self, args: serde_json::Value) -> Result; +} + +/// Pick the English or Ukrainian variant of a message for `locale`. +pub fn localized<'a>(locale: &str, en: &'a str, uk: &'a str) -> &'a str { + if locale.starts_with("uk") { uk } else { en } +} + +// ============================================================================ +// MarkkitTool +// ============================================================================ + +/// Builder for [`MarkkitTool`]. +#[derive(Debug, Clone)] +pub struct MarkkitToolBuilder { + locale: String, + config: MarkkitBuilder, + allow_url: bool, + max_output_chars: usize, +} + +impl Default for MarkkitToolBuilder { + fn default() -> Self { + Self::new() + } +} + +impl MarkkitToolBuilder { + /// Defaults: `en-US`, default [`MarkkitBuilder`], URL fetching on when the + /// `http_client` feature is built, [`DEFAULT_MAX_OUTPUT_CHARS`]. + pub fn new() -> Self { + Self { + locale: "en-US".to_string(), + config: MarkkitBuilder::default(), + allow_url: cfg!(feature = "http_client"), + max_output_chars: DEFAULT_MAX_OUTPUT_CHARS, + } + } + + /// Locale for descriptions, prompts, help and user-facing errors + /// (`en-US`, `uk-UA`; others fall back to English). + pub fn locale(mut self, locale: &str) -> Self { + self.locale = locale.to_string(); + self + } + + /// Extraction configuration (limits, selectors, SSRF policy). + pub fn config(mut self, config: MarkkitBuilder) -> Self { + self.config = config; + self + } + + /// Let the model pass `url`. Ignored (always off) without `http_client`. + pub fn allow_url(mut self, yes: bool) -> Self { + self.allow_url = yes; + self + } + + /// Hard cap on returned Markdown characters; per-call `max_chars` can only + /// lower it. + pub fn max_output_chars(mut self, chars: usize) -> Self { + self.max_output_chars = chars; + self + } + + /// Build the tool. Non-consuming so one builder can mint many tools. + pub fn build(&self) -> MarkkitTool { + let locale = self.locale.clone(); + let allow_url = self.allow_url && cfg!(feature = "http_client"); + MarkkitTool { + inner: Arc::new(ToolInner { + display_name: localized(&locale, "Markdown extractor", "Витяг Markdown") + .to_string(), + short_desc: short_description(&locale).to_string(), + description: description(&locale, allow_url), + markkit: self.config.clone().build(), + allow_url, + max_output_chars: self.max_output_chars, + locale, + }), + } + } + + /// Build a `tower::Service`. + pub fn build_service(&self) -> ToolService { + let tool = self.build(); + tower::util::BoxCloneService::new(tower::service_fn(move |args| { + let tool = tool.clone(); + async move { + let output = tool.execution(args)?.execute().await?; + Ok(output.result) + } + })) + } + + /// Build an OpenAI-compatible function tool definition. + pub fn build_tool_definition(&self) -> serde_json::Value { + let tool = self.build(); + serde_json::json!({ + "type": "function", + "function": { + "name": tool.name(), + "description": tool.description(), + "parameters": tool.input_schema(), + } + }) + } + + /// Input schema of the tool this builder makes. + pub fn build_input_schema(&self) -> serde_json::Value { + input_schema(self.allow_url && cfg!(feature = "http_client")) + } + + /// Output schema for [`ToolOutput::result`]. + pub fn build_output_schema(&self) -> serde_json::Value { + output_schema() + } +} + +struct ToolInner { + locale: String, + display_name: String, + short_desc: String, + description: String, + markkit: Markkit, + allow_url: bool, + max_output_chars: usize, +} + +/// Document-to-Markdown extraction tool implementing [`Tool`]. +/// +/// Cheap to clone; clones share configuration. +#[derive(Clone)] +pub struct MarkkitTool { + inner: Arc, +} + +impl MarkkitTool { + /// Create a tool builder. + pub fn builder() -> MarkkitToolBuilder { + MarkkitToolBuilder::new() + } +} + +impl Default for MarkkitTool { + fn default() -> Self { + MarkkitToolBuilder::new().build() + } +} + +impl std::fmt::Debug for MarkkitTool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MarkkitTool") + .field("locale", &self.inner.locale) + .field("allow_url", &self.inner.allow_url) + .field("max_output_chars", &self.inner.max_output_chars) + .finish_non_exhaustive() + } +} + +impl Tool for MarkkitTool { + fn name(&self) -> &str { + TOOL_NAME + } + + fn display_name(&self) -> &str { + &self.inner.display_name + } + + fn short_description(&self) -> &str { + &self.inner.short_desc + } + + fn description(&self) -> &str { + &self.inner.description + } + + fn help(&self) -> String { + help( + &self.inner.locale, + self.inner.allow_url, + self.inner.max_output_chars, + ) + } + + fn system_prompt(&self) -> String { + system_prompt(&self.inner.locale, self.inner.allow_url) + } + + fn locale(&self) -> &str { + &self.inner.locale + } + + fn input_schema(&self) -> serde_json::Value { + input_schema(self.inner.allow_url) + } + + fn output_schema(&self) -> serde_json::Value { + output_schema() + } + + fn version(&self) -> &str { + VERSION + } + + fn execution(&self, args: serde_json::Value) -> Result { + let req = parse_request(&self.inner, args)?; + let inner = Arc::clone(&self.inner); + // The stream sender is dropped when the runner returns, closing the + // stream; markkit emits no incremental chunks. + Ok(ToolExecution::new(move |_stream| async move { + let start = now(); + let extraction = run(&inner, req).await?; + output_from_extraction(extraction, elapsed(start)) + })) + } +} + +// ---------------------------------------------------------------------------- +// Request handling +// ---------------------------------------------------------------------------- + +enum Source { + Url(String), + Html(String), + Bytes(Vec), +} + +struct Request { + source: Source, + file_name: Option, + content_type: Option, + max_chars: usize, +} + +fn user_err(locale: &str, en: &str, uk: &str) -> ToolError { + ToolError::UserFacing(localized(locale, en, uk).to_string()) +} + +fn opt_string( + obj: &serde_json::Map, + key: &str, + locale: &str, +) -> Result, ToolError> { + match obj.get(key) { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::String(s)) => Ok(Some(s.clone())), + Some(_) => Err(ToolError::UserFacing(format!( + "`{key}` {}", + localized(locale, "must be a string", "має бути рядком") + ))), + } +} + +fn parse_request(inner: &ToolInner, args: serde_json::Value) -> Result { + let locale = inner.locale.as_str(); + let Some(obj) = args.as_object() else { + return Err(user_err( + locale, + "tool arguments must be a JSON object", + "аргументи інструмента мають бути JSON-об'єктом", + )); + }; + + let url = opt_string(obj, "url", locale)?; + let html = opt_string(obj, "html", locale)?; + let content_base64 = opt_string(obj, "content_base64", locale)?; + let file_name = opt_string(obj, "file_name", locale)?; + let content_type = opt_string(obj, "content_type", locale)?; + + let max_chars = match obj.get("max_chars") { + None | Some(serde_json::Value::Null) => inner.max_output_chars, + Some(value) => { + let n = value.as_u64().ok_or_else(|| { + user_err( + locale, + "`max_chars` must be a non-negative integer", + "поле `max_chars` має бути невід'ємним цілим числом", + ) + })?; + usize::try_from(n) + .unwrap_or(usize::MAX) + .min(inner.max_output_chars) + } + }; + + let given = [url.is_some(), html.is_some(), content_base64.is_some()] + .into_iter() + .filter(|b| *b) + .count(); + if given != 1 { + return Err(if inner.allow_url { + user_err( + locale, + "pass exactly one of `url`, `html` or `content_base64`", + "передайте рівно одне з полів `url`, `html` або `content_base64`", + ) + } else { + user_err( + locale, + "pass exactly one of `html` or `content_base64`", + "передайте рівно одне з полів `html` або `content_base64`", + ) + }); + } + + let source = if let Some(url) = url { + if !inner.allow_url { + return Err(user_err( + locale, + "URL fetching is disabled for this tool; pass `html` or `content_base64`", + "завантаження за URL вимкнено; передайте `html` або `content_base64`", + )); + } + Source::Url(url) + } else if let Some(html) = html { + Source::Html(html) + } else { + let encoded = content_base64.unwrap_or_default(); + // Bound the work before decoding: 4 base64 chars carry 3 bytes. + let max_encoded = inner + .markkit + .config() + .max_input_size + .saturating_add(2) + .saturating_div(3) + .saturating_mul(4); + if encoded.len() > max_encoded { + return Err(user_err( + locale, + "`content_base64` exceeds the input size limit", + "`content_base64` перевищує ліміт розміру вхідних даних", + )); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .map_err(|_| { + user_err( + locale, + "`content_base64` is not valid base64", + "`content_base64` не є коректним base64", + ) + })?; + Source::Bytes(bytes) + }; + + Ok(Request { + source, + file_name, + content_type, + max_chars, + }) +} + +async fn run(inner: &ToolInner, req: Request) -> Result { + let hint = DocumentHint { + content_type: req.content_type.as_deref(), + file_name: req.file_name.as_deref(), + }; + let result = match req.source { + #[cfg(feature = "http_client")] + Source::Url(url) => inner.markkit.extract_url_document(&url).await, + #[cfg(not(feature = "http_client"))] + Source::Url(_) => unreachable!("url rejected in parse_request without http_client"), + Source::Html(html) => inner.markkit.extract_document_sync( + html.as_bytes(), + DocumentHint { + content_type: Some("text/html"), + file_name: None, + }, + ), + Source::Bytes(bytes) => inner.markkit.extract_document_sync(&bytes, hint), + }; + let mut extraction = result.map_err(|err| map_error(&inner.locale, err))?; + if extraction.markdown.chars().count() > req.max_chars { + let cut = extraction + .markdown + .char_indices() + .nth(req.max_chars) + .map_or(extraction.markdown.len(), |(i, _)| i); + let full_chars = extraction.markdown.chars().count(); + extraction.markdown.truncate(cut); + extraction.warnings.push(format!( + "truncated: output cut at {} of {full_chars} chars", + req.max_chars + )); + } + Ok(extraction) +} + +fn map_error(locale: &str, err: Error) -> ToolError { + match err { + Error::Io(e) => ToolError::Internal(format!("I/O error: {e}")), + other => ToolError::UserFacing(format!( + "{}: {other}", + localized(locale, "extraction failed", "не вдалося витягти") + )), + } +} + +// `std::time::Instant` panics on wasm32-unknown-unknown; report zero there. +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +fn now() -> Option { + Some(Instant::now()) +} + +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] +fn now() -> Option { + None +} + +fn elapsed(start: Option) -> Duration { + start.map_or(Duration::ZERO, |s| s.elapsed()) +} + +fn output_from_extraction( + extraction: Extraction, + duration: Duration, +) -> Result { + let truncated = extraction + .warnings + .iter() + .any(|w| w.starts_with("truncated:")); + let stats = serde_json::to_value(&extraction.stats) + .map_err(|e| ToolError::Internal(format!("failed to serialize stats: {e}")))?; + let chars = extraction.markdown.chars().count(); + Ok(ToolOutput { + result: serde_json::json!({ + "markdown": extraction.markdown, + "title": extraction.title, + "format": extraction.format, + "warnings": extraction.warnings, + "truncated": truncated, + }), + images: Vec::new(), + metadata: ToolOutputMetadata { + duration, + extra: serde_json::json!({ "stats": stats, "chars": chars }), + }, + }) +} + +// ---------------------------------------------------------------------------- +// Schemas and localized text +// ---------------------------------------------------------------------------- + +fn input_schema(allow_url: bool) -> serde_json::Value { + let mut properties = serde_json::json!({ + "html": { + "type": "string", + "description": "HTML source to convert" + }, + "content_base64": { + "type": "string", + "description": "Document bytes, base64 (DOCX, XLSX, XLS, ODS, HTML)" + }, + "file_name": { + "type": "string", + "description": "File name hint for content_base64 format detection" + }, + "content_type": { + "type": "string", + "description": "MIME type hint for content_base64 format detection" + }, + "max_chars": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Truncate the returned Markdown to this many characters" + } + }); + if allow_url { + properties["url"] = serde_json::json!({ + "type": "string", + "description": "Public http(s) URL to fetch and convert" + }); + } + serde_json::json!({ + "type": "object", + "properties": properties, + "additionalProperties": false + }) +} + +fn output_schema() -> serde_json::Value { + serde_json::json!({ + "type": "object", + "required": ["markdown", "format", "warnings", "truncated"], + "properties": { + "markdown": { "type": "string", "description": "Extracted Markdown" }, + "title": { "type": ["string", "null"], "description": "Document title when known" }, + "format": { "type": "string", "description": "Detected format (html, docx, xlsx, xls, ods)" }, + "warnings": { + "type": "array", + "items": { "type": "string" }, + "description": "Non-fatal issues, each prefixed with a stable code" + }, + "truncated": { "type": "boolean", "description": "Markdown was cut at max_chars" } + } + }) +} + +fn short_description(locale: &str) -> &'static str { + localized( + locale, + "Convert web pages and documents to Markdown", + "Перетворює вебсторінки та документи на Markdown", + ) +} + +fn description(locale: &str, allow_url: bool) -> String { + match (locale.starts_with("uk"), allow_url) { + (false, true) => "Convert a web page (url), HTML or a document (base64 DOCX/XLSX/XLS/ODS) to clean Markdown.", + (false, false) => "Convert HTML or a document (base64 DOCX/XLSX/XLS/ODS) to clean Markdown.", + (true, true) => "Перетворює вебсторінку (url), HTML або документ (base64 DOCX/XLSX/XLS/ODS) на чистий Markdown.", + (true, false) => "Перетворює HTML або документ (base64 DOCX/XLSX/XLS/ODS) на чистий Markdown.", + } + .to_string() +} + +fn system_prompt(locale: &str, allow_url: bool) -> String { + let uk = locale.starts_with("uk"); + let sources = match (uk, allow_url) { + (false, true) => "Pass one of url, html, content_base64.", + (false, false) => "Pass one of html, content_base64.", + (true, true) => "Передайте одне з: url, html, content_base64.", + (true, false) => "Передайте одне з: html, content_base64.", + }; + let lead = localized( + locale, + "converts web pages and documents to Markdown.", + "перетворює вебсторінки та документи на Markdown.", + ); + let tail = localized( + locale, + "Use max_chars to limit output.", + "max_chars обмежує розмір відповіді.", + ); + format!("{TOOL_NAME}: {lead} {sources} {tail}") +} + +fn help(locale: &str, allow_url: bool, max_output_chars: usize) -> String { + let uk = locale.starts_with("uk"); + let mut doc = String::new(); + doc.push_str(&format!("# {TOOL_NAME}\n\n")); + doc.push_str(&description(locale, allow_url)); + doc.push_str("\n\n"); + doc.push_str(if uk { + "## Аргументи\n\n" + } else { + "## Arguments\n\n" + }); + if allow_url { + doc.push_str(if uk { + "- `url`: публічна http(s) адреса; приватні мережі заблоковано\n" + } else { + "- `url`: public http(s) address; private networks are blocked\n" + }); + } + doc.push_str(if uk { + "- `html`: HTML-код\n\ + - `content_base64`: байти документа в base64, з `file_name` або `content_type` як підказкою формату\n\ + - `max_chars`: обрізати Markdown до цієї кількості символів\n\n" + } else { + "- `html`: HTML source\n\ + - `content_base64`: document bytes as base64, with `file_name` or `content_type` as a format hint\n\ + - `max_chars`: truncate the Markdown to this many characters\n\n" + }); + doc.push_str(&if uk { + format!( + "Рівно одне джерело на виклик. Відповідь не довша за {max_output_chars} символів.\n\n" + ) + } else { + format!( + "Exactly one source per call. Output is capped at {max_output_chars} characters.\n\n" + ) + }); + doc.push_str(if uk { + "## Приклад\n\n" + } else { + "## Example\n\n" + }); + doc.push_str("```json\n{\"html\": \"

Title

Body

\"}\n```\n\n"); + doc.push_str(if uk { + "## Результат\n\n" + } else { + "## Result\n\n" + }); + doc.push_str("`markdown`, `title`, `format`, `warnings`, `truncated`.\n"); + doc +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn tool() -> MarkkitTool { + MarkkitTool::default() + } + + async fn run_ok(tool: &MarkkitTool, args: serde_json::Value) -> ToolOutput { + tool.execution(args) + .expect("valid args") + .execute() + .await + .expect("execution succeeds") + } + + fn err_of(tool: &MarkkitTool, args: serde_json::Value) -> ToolError { + match tool.execution(args) { + Ok(_) => panic!("expected validation error"), + Err(e) => e, + } + } + + #[test] + fn metadata() { + let t = tool(); + assert_eq!(t.name(), "markkit"); + assert_eq!(t.version(), VERSION); + assert_eq!(t.locale(), "en-US"); + assert!(t.system_prompt().starts_with("markkit:")); + assert!(t.help().starts_with("# markkit")); + assert!(!t.description().contains('\n')); + } + + #[test] + fn locale_localizes_text_not_names() { + let t = MarkkitTool::builder().locale("uk-UA").build(); + assert_eq!(t.name(), "markkit"); + assert_eq!(t.display_name(), "Витяг Markdown"); + assert!(t.description().starts_with("Перетворює")); + assert!(t.system_prompt().starts_with("markkit:")); + assert_eq!(t.input_schema(), tool().input_schema()); + match err_of(&t, json!([])) { + ToolError::UserFacing(msg) => assert!(msg.contains("JSON-об'єктом")), + other => panic!("unexpected {other:?}"), + } + } + + #[test] + fn unknown_locale_falls_back_to_english() { + let t = MarkkitTool::builder().locale("fr-FR").build(); + assert_eq!(t.display_name(), "Markdown extractor"); + } + + #[test] + fn builder_helpers_match_tool() { + let b = MarkkitTool::builder(); + let t = b.build(); + let def = b.build_tool_definition(); + assert_eq!(def["type"], "function"); + assert_eq!(def["function"]["name"], "markkit"); + assert_eq!(def["function"]["description"], t.description()); + assert_eq!(def["function"]["parameters"], t.input_schema()); + assert_eq!(b.build_input_schema(), t.input_schema()); + assert_eq!(b.build_output_schema(), t.output_schema()); + } + + #[test] + fn url_property_follows_allow_url() { + let off = MarkkitTool::builder().allow_url(false).build(); + assert!(off.input_schema()["properties"].get("url").is_none()); + assert!(!off.description().contains("url")); + let on = MarkkitTool::builder().allow_url(true).build(); + assert_eq!( + on.input_schema()["properties"].get("url").is_some(), + cfg!(feature = "http_client") + ); + } + + #[tokio::test] + async fn html_source() { + let out = run_ok(&tool(), json!({"html": "

Hi

there

"})).await; + assert_eq!(out.result["markdown"], "# Hi\n\nthere\n"); + assert_eq!(out.result["format"], "html"); + assert_eq!(out.result["truncated"], false); + assert!(out.images.is_empty()); + assert_eq!(out.metadata.extra["chars"], 12); + } + + #[tokio::test] + async fn base64_source_with_hint() { + let encoded = base64::engine::general_purpose::STANDARD.encode("

bytes

"); + let out = run_ok( + &tool(), + json!({"content_base64": encoded, "file_name": "page.html"}), + ) + .await; + assert_eq!(out.result["markdown"], "bytes\n"); + } + + #[tokio::test] + async fn output_matches_schema_shape() { + let out = run_ok(&tool(), json!({"html": "

x

"})).await; + let schema = tool().output_schema(); + for key in schema["required"].as_array().unwrap() { + assert!(out.result.get(key.as_str().unwrap()).is_some(), "{key}"); + } + } + + #[tokio::test] + async fn max_chars_truncates_on_char_boundary() { + let out = run_ok( + &tool(), + json!({"html": "

привіт світ

", "max_chars": 3}), + ) + .await; + assert_eq!(out.result["markdown"], "при"); + assert_eq!(out.result["truncated"], true); + assert!( + out.result["warnings"][0] + .as_str() + .unwrap() + .starts_with("truncated:") + ); + } + + #[tokio::test] + async fn per_call_max_chars_cannot_exceed_builder_cap() { + let t = MarkkitTool::builder().max_output_chars(4).build(); + let out = run_ok(&t, json!({"html": "

abcdefgh

", "max_chars": 1000})).await; + assert_eq!(out.result["markdown"], "abcd"); + } + + #[test] + fn rejects_non_object_args() { + assert!(err_of(&tool(), json!("html")).is_user_facing()); + } + + #[test] + fn rejects_missing_and_multiple_sources() { + assert!(err_of(&tool(), json!({})).is_user_facing()); + assert!( + err_of( + &tool(), + json!({"html": "

a

", "content_base64": "eA=="}) + ) + .is_user_facing() + ); + } + + #[test] + fn rejects_wrong_types() { + assert!(err_of(&tool(), json!({"html": 1})).is_user_facing()); + assert!(err_of(&tool(), json!({"html": "x", "max_chars": -1})).is_user_facing()); + assert!(err_of(&tool(), json!({"html": "x", "max_chars": "5"})).is_user_facing()); + } + + #[test] + fn rejects_invalid_base64() { + let e = err_of(&tool(), json!({"content_base64": "not base64!!"})); + assert_eq!( + e, + ToolError::UserFacing("`content_base64` is not valid base64".into()) + ); + } + + #[test] + fn rejects_oversized_base64_before_decoding() { + let t = MarkkitTool::builder() + .config(MarkkitBuilder::default().max_input_size(3)) + .build(); + // 8 chars decode to 6 bytes > 3-byte limit (max 4 encoded chars). + let e = err_of(&t, json!({"content_base64": "aGVsbG8h"})); + assert!(matches!(e, ToolError::UserFacing(m) if m.contains("size limit"))); + } + + #[test] + fn rejects_url_when_disabled() { + let t = MarkkitTool::builder().allow_url(false).build(); + let e = err_of(&t, json!({"url": "https://example.com"})); + assert!(matches!(e, ToolError::UserFacing(m) if m.contains("disabled"))); + } + + #[cfg(feature = "http_client")] + #[tokio::test] + async fn url_goes_through_ssrf_guard() { + let e = tool() + .execution(json!({"url": "http://127.0.0.1/"})) + .expect("valid args") + .execute() + .await + .expect_err("loopback blocked"); + assert!(matches!(e, ToolError::UserFacing(m) if m.contains("URL blocked"))); + } + + #[tokio::test] + async fn extraction_errors_are_user_facing() { + let t = MarkkitTool::builder() + .config(MarkkitBuilder::default().max_input_size(4)) + .build(); + let e = t + .execution(json!({"html": "

too long

"})) + .unwrap() + .execute() + .await + .expect_err("limit"); + assert!(matches!(e, ToolError::UserFacing(m) if m.contains("Resource limit"))); + } + + #[tokio::test] + async fn unsupported_format_is_user_facing() { + let encoded = base64::engine::general_purpose::STANDARD.encode("%PDF-1.7"); + let e = tool() + .execution(json!({"content_base64": encoded, "file_name": "a.pdf"})) + .unwrap() + .execute() + .await + .expect_err("pdf unsupported"); + assert!(e.is_user_facing()); + } + + #[tokio::test] + async fn stream_closes_without_chunks() { + use std::future::poll_fn; + let exec = tool().execution(json!({"html": "

x

"})).unwrap(); + let mut stream = exec.output_stream().expect("first stream"); + assert!(exec.output_stream().is_none()); + exec.execute().await.unwrap(); + let next = poll_fn(|cx| Pin::new(&mut stream).poll_next(cx)).await; + assert!(next.is_none()); + } + + #[tokio::test] + async fn service_executes() { + use tower::{Service, ServiceExt}; + let mut svc = MarkkitTool::builder().build_service(); + let value = svc + .ready() + .await + .unwrap() + .call(json!({"html": "

svc

"})) + .await + .unwrap(); + assert_eq!(value["markdown"], "svc\n"); + let err = svc + .ready() + .await + .unwrap() + .call(json!({})) + .await + .unwrap_err(); + assert!(err.is_user_facing()); + } +} diff --git a/examples/agent_tool.rs b/examples/agent_tool.rs new file mode 100644 index 0000000..e1bbf84 --- /dev/null +++ b/examples/agent_tool.rs @@ -0,0 +1,32 @@ +//! Agent tool: expose markkit to an LLM via the toolkit-library contract. +//! +//! Run: `cargo run -p markkit --features tool --example agent_tool` + +use markkit::tool::{MarkkitTool, Tool}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let builder = MarkkitTool::builder().max_output_chars(2_000); + + // OpenAI-compatible function definition to send with the model request. + println!( + "{}", + serde_json::to_string_pretty(&builder.build_tool_definition())? + ); + + // System prompt snippet. + let tool = builder.build(); + println!("\n{}\n", tool.system_prompt()); + + // Execute arguments the model returned in a tool call. + let args = serde_json::json!({ + "html": "

Release notes

  • Agent tool
  • WASM
" + }); + match tool.execution(args)?.execute().await { + Ok(output) => println!("{}", serde_json::to_string_pretty(&output.result)?), + // UserFacing errors are safe to return to the model as the tool result. + Err(e) if e.is_user_facing() => println!("tool error: {e}"), + Err(e) => return Err(e.into()), + } + Ok(()) +} diff --git a/justfile b/justfile index 85d67fd..86c7327 100644 --- a/justfile +++ b/justfile @@ -62,6 +62,8 @@ examples: echo "── example: $ex" cargo run -q -p markkit --example "$ex" done + echo "── example: agent_tool" + cargo run -q -p markkit --features tool --example agent_tool # ─── Run ───────────────────────────────────────────────────── diff --git a/knowledge/index.md b/knowledge/index.md index 6132cc0..cced66d 100644 --- a/knowledge/index.md +++ b/knowledge/index.md @@ -11,6 +11,7 @@ okf_version: "0.2" * [foundations/](foundations/) - Extraction pipeline architecture and the intermediate representation. * [formats/](formats/) - Per-format parsing: HTML extraction, format detection, DOCX and spreadsheet mapping. +* [integrations/](integrations/) - Agent-facing integrations: the LLM tool contract. * [security/](security/) - Security model and detailed threat model for untrusted input. * [runtimes/](runtimes/) - Published language bindings for Python and Node.js. * [operations/](operations/) - Testing, documentation, examples, maintenance, release, and eval. diff --git a/knowledge/integrations/index.md b/knowledge/integrations/index.md new file mode 100644 index 0000000..33bf266 --- /dev/null +++ b/knowledge/integrations/index.md @@ -0,0 +1,3 @@ +# Integrations + +* [Tool Contract](tool-contract.md) - MarkkitTool LLM tool, its schemas, limits, and error semantics under the Everruns toolkit-library contract. diff --git a/knowledge/integrations/tool-contract.md b/knowledge/integrations/tool-contract.md new file mode 100644 index 0000000..25e0562 --- /dev/null +++ b/knowledge/integrations/tool-contract.md @@ -0,0 +1,115 @@ +--- +type: Interface Contract +title: Tool Contract +description: MarkkitTool LLM tool, its schemas, limits, and error semantics under the Everruns toolkit-library contract. +tags: + - markkit + - tools + - llm + - api +--- + +# Tool Contract + +## Status +Implemented (Rust core, `tool` feature). Bindings, WASM and an MCP server are not wired yet. + +## Summary + +Markkit follows the Everruns toolkit-library contract, the same shape bashkit ships +in its `tool` module: + +```text +MarkkitToolBuilder (config) -> MarkkitTool (metadata) -> ToolExecution (single-use runtime) +``` + +Source: `crates/markkit/src/tool.rs`; example `examples/agent_tool.rs`. + +## Feature gating + +The `tool` module is behind the opt-in `tool` feature (adds `tower`, `futures-core`, +`base64`). Unlike bashkit it is **off by default**: plain library users keep a small +dependency tree. `url` support additionally needs `http_client`. + +## Public API + +`Tool` trait, `MarkkitToolBuilder`, `MarkkitTool`, `ToolExecution`, `ToolOutput`, +`ToolOutputChunk`, `ToolOutputStream`, `ToolError`, `ToolService`, `localized`. + +### Builder rules + +- `build()` is non-consuming; `build_service()` returns + `tower::Service`. +- `build_tool_definition()` emits OpenAI function JSON; `build_input_schema()` / + `build_output_schema()` match the built tool. +- Knobs: `locale`, `config(MarkkitBuilder)` (all extraction limits and the SSRF + policy), `allow_url` (default on with `http_client`), `max_output_chars` + (default 100 000). + +### Metadata rules + +- `name()` is `markkit`, never localized. `display_name()` is `Markdown extractor` / + `Витяг Markdown`. +- `description()` is one sentence; `system_prompt()` starts with `markkit:`; + `help()` is Markdown. All three omit `url` when URL fetching is off. + +### Execution rules + +- `execution()` validates arguments before an execution exists; invalid arguments + never reach a parser. +- `ToolExecution` is single-use; `output_stream()` must be called before `execute()`. + Markkit emits no chunks: the stream closes when execution ends. +- `images` is always empty. + +## Input schema + +Exactly one source per call. + +| Field | Type | Notes | +|---|---|---| +| `url` | string | Only present when `allow_url`; goes through `extract_url_document` (SSRF guard, invariant S12) | +| `html` | string | Parsed as HTML | +| `content_base64` | string | Document bytes; encoded length checked against `max_input_size` before decoding | +| `file_name`, `content_type` | string | Format hints for `content_base64` | +| `max_chars` | integer or null | Truncation; clamped to the builder's `max_output_chars` | + +`additionalProperties` is `false`. + +## Output schema + +`ToolOutput::result`: + +```json +{ "markdown": "string", "title": "string|null", "format": "html|docx|xlsx|xls|ods", + "warnings": ["string"], "truncated": false } +``` + +Truncation cuts on a character boundary and adds a `truncated:` warning. +`metadata.extra` is `{ "stats": ExtractionStats, "chars": }`. +`metadata.duration` is zero on `wasm32-unknown-unknown` (no `Instant` there). + +## Error rules + +- `UserFacing`: argument validation and every extraction `Error` except I/O, + prefixed `extraction failed:` / `не вдалося витягти:`. Safe to return to the model. +- `Internal`: I/O and serialization failures; English, for logs. + +## Locale + +`en-US` and `uk-UA`; anything else falls back to English. Locale changes display +name, descriptions, help, system prompt and `UserFacing` text, never `name()`, +schemas or `version()`. + +## Verification + +Unit tests in `tool::tests`: metadata and locale, builder helpers vs tool, schema +gating of `url`, each source, truncation (multi-byte and builder cap), argument +validation (non-object, missing or multiple sources, wrong types, bad base64, +oversized base64 before decode, disabled URL), SSRF via the tool, error mapping, +stream closing, and the `tower` service. + +## See also + +- [Markkit Architecture](../foundations/architecture.md), the extraction pipeline the tool wraps +- [Threat Model](../security/threat-model.md), T3 SSRF and the tool arguments surface +- [Examples](../operations/examples.md), `agent_tool.rs` diff --git a/knowledge/knowledge-contract.md b/knowledge/knowledge-contract.md index f7fb888..11d511a 100644 --- a/knowledge/knowledge-contract.md +++ b/knowledge/knowledge-contract.md @@ -43,6 +43,7 @@ The bundle declares `okf_version: "0.2"` in the bundle-root [`index.md`](index.m |---|---| | [`foundations/`](foundations/) | Pipeline architecture, IR | | [`formats/`](formats/) | HTML extraction, format detection, office mappings | +| [`integrations/`](integrations/) | LLM tool contract | | [`security/`](security/) | Security model, threat model | | [`runtimes/`](runtimes/) | Python and Node.js packages | | [`operations/`](operations/) | Testing, docs, examples, maintenance, release, eval | diff --git a/knowledge/log.md b/knowledge/log.md index 1b2ff07..9262787 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-26 +* **Agent tool**: new [Tool Contract](integrations/tool-contract.md) concept in a new `integrations/` domain. `MarkkitTool` (opt-in `tool` feature) follows the Everruns toolkit-library contract; tool arguments are a new untrusted surface in the [Threat Model](security/threat-model.md). + * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). diff --git a/knowledge/operations/examples.md b/knowledge/operations/examples.md index 6005845..6363268 100644 --- a/knowledge/operations/examples.md +++ b/knowledge/operations/examples.md @@ -17,6 +17,7 @@ examples/ custom_selectors.rs # include/exclude CSS selectors configuration.rs # How builder options change output ir_access.rs # Parse to IR, inspect nodes, render separately + agent_tool.rs # LLM tool definition, system prompt, execution ``` Examples live at the workspace root and are registered as `[[example]]` targets @@ -48,6 +49,12 @@ Run one with `cargo run -p markkit --example `, all with `just examples`. - Walk `Document.nodes`, match on `Node` variants - Render with `MarkdownRenderer` to show parse and render are separate stages +### agent_tool.rs + +- Needs the `tool` feature (`required-features = ["html", "tool"]`) +- Prints the OpenAI tool definition and system prompt, then executes model-style + arguments and prints `ToolOutput::result` (see [Tool Contract](../integrations/tool-contract.md)) + ## Requirements - No network or file access; inline HTML samples only diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 49d78f1..0b6315b 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -18,6 +18,7 @@ tags: | URL fetching | `extract_url()` (http_client feature) | Untrusted remote | | Byte input | `extract_bytes()` with content-type | Untrusted | | Document input | `extract_document()`, `extract_document_sync()` | Untrusted (bytes and `DocumentHint`) | +| LLM tool arguments | `MarkkitTool::execution()` (tool feature) | Untrusted (model output); validated before execution, see [Tool Contract](../integrations/tool-contract.md) | | Configuration | `MarkkitBuilder` methods | Trusted (caller-controlled) | | CSS selectors | `include_selector()`, `exclude_selector()` | Semi-trusted (caller-provided) | From 03fc1b404b267978556026298f0f4f17a9a21d78 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:33:55 +0000 Subject: [PATCH 2/5] fix(tool): silence dead-code warning without http_client --- crates/markkit/src/tool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/markkit/src/tool.rs b/crates/markkit/src/tool.rs index fbc52f9..b14ccee 100644 --- a/crates/markkit/src/tool.rs +++ b/crates/markkit/src/tool.rs @@ -494,6 +494,7 @@ impl Tool for MarkkitTool { // ---------------------------------------------------------------------------- enum Source { + #[cfg_attr(not(feature = "http_client"), allow(dead_code))] Url(String), Html(String), Bytes(Vec), From 3582fe05849fa56d4c53c6e10eb1b399dfa7a278 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:36:52 +0000 Subject: [PATCH 3/5] docs(tool): avoid feature-gated intra-doc link --- crates/markkit/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index e037f97..80319e9 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -36,7 +36,7 @@ //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest -//! - **Agent tool** (`tool`): [`tool::MarkkitTool`], an LLM tool following the +//! - **Agent tool** (`tool`): `markkit::tool::MarkkitTool`, an LLM tool following the //! Everruns toolkit-library contract (schemas, OpenAI definition, `tower::Service`) //! //! Embedders that do their own fetching can drop reqwest entirely: From d07faf2db556d4952f3bd7d4c2656ab955b0986c Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:36:47 +0000 Subject: [PATCH 4/5] feat(wasm): add @everruns/markkit-wasm browser and edge package wasm-bindgen build of markkit (html, docx, spreadsheet, tool) for wasm32-unknown-unknown: single-threaded, no COOP/COEP, no URL fetching. Markkit class with validated camelCase options plus MarkkitTool. Node tests, CI job gating Check, and an OIDC npm publisher. --- .github/workflows/ci.yml | 36 +- .github/workflows/publish-wasm.yml | 77 +++++ AGENTS.md | 4 +- CHANGELOG.md | 3 + Cargo.lock | 24 ++ Cargo.toml | 6 + README.md | 22 +- crates/markkit-wasm/.gitignore | 4 + crates/markkit-wasm/Cargo.toml | 38 +++ crates/markkit-wasm/README.md | 67 ++++ .../__test__/markkit-wasm.test.mjs | 139 ++++++++ crates/markkit-wasm/__test__/zip.mjs | 67 ++++ crates/markkit-wasm/js/index.d.ts | 118 +++++++ crates/markkit-wasm/js/index.js | 24 ++ crates/markkit-wasm/package.json | 46 +++ crates/markkit-wasm/pkg/index.d.ts | 12 + crates/markkit-wasm/pkg/markkit_wasm.d.ts | 4 +- crates/markkit-wasm/pkg/markkit_wasm.js | 14 +- .../pkg/markkit_wasm_bg.wasm.d.ts | 4 +- crates/markkit-wasm/scripts/build.sh | 41 +++ crates/markkit-wasm/src/lib.rs | 319 ++++++++++++++++++ justfile | 6 + knowledge/index.md | 2 +- knowledge/integrations/tool-contract.md | 2 +- knowledge/log.md | 2 +- knowledge/operations/release-process.md | 6 +- knowledge/runtimes/index.md | 1 + knowledge/runtimes/node-package.md | 1 + knowledge/runtimes/wasm-package.md | 87 +++++ scripts/install-wasm-bindgen.sh | 21 ++ 30 files changed, 1177 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/publish-wasm.yml create mode 100644 crates/markkit-wasm/.gitignore create mode 100644 crates/markkit-wasm/Cargo.toml create mode 100644 crates/markkit-wasm/README.md create mode 100644 crates/markkit-wasm/__test__/markkit-wasm.test.mjs create mode 100644 crates/markkit-wasm/__test__/zip.mjs create mode 100644 crates/markkit-wasm/js/index.d.ts create mode 100644 crates/markkit-wasm/js/index.js create mode 100644 crates/markkit-wasm/package.json create mode 100755 crates/markkit-wasm/scripts/build.sh create mode 100644 crates/markkit-wasm/src/lib.rs create mode 100644 knowledge/runtimes/wasm-package.md create mode 100755 scripts/install-wasm-bindgen.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72c1b1f..f4e9f4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,9 +90,40 @@ jobs: done cargo run -q -p markkit --features tool --example agent_tool + wasm: + # Browser package @everruns/markkit-wasm: build the wasm32-unknown-unknown + # bundle and run its headless Node tests. See knowledge/runtimes/wasm-package.md. + name: WASM web package + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: "24.22.1" + - name: Check core for wasm32 (no warnings) + run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,tool --target wasm32-unknown-unknown + - name: Install wasm-bindgen-cli (matches Cargo.lock) + run: bash scripts/install-wasm-bindgen.sh + - name: Build package + run: bash crates/markkit-wasm/scripts/build.sh release + - name: Integration tests (headless Node) + run: node --test crates/markkit-wasm/__test__/*.test.mjs + - name: Verify npm archive contents + run: | + cd crates/markkit-wasm/pkg + npm pack --dry-run --json --ignore-scripts > "$RUNNER_TEMP/wasm-pack.json" + node -e 'const files=new Set(require(process.argv[1])[0].files.map(f=>f.path)); for (const f of ["index.js","index.d.ts","markkit_wasm.js","markkit_wasm_bg.wasm","README.md"]) if(!files.has(f)) throw Error("archive lacks "+f)' "$RUNNER_TEMP/wasm-pack.json" + check: name: Check - needs: [lint, audit, test] + needs: [lint, audit, test, wasm] runs-on: ubuntu-latest if: always() steps: @@ -100,7 +131,8 @@ jobs: run: | if [[ "${{ needs.lint.result }}" != "success" ]] || \ [[ "${{ needs.audit.result }}" != "success" ]] || \ - [[ "${{ needs.test.result }}" != "success" ]]; then + [[ "${{ needs.test.result }}" != "success" ]] || \ + [[ "${{ needs.wasm.result }}" != "success" ]]; then echo "One or more jobs failed" exit 1 fi diff --git a/.github/workflows/publish-wasm.yml b/.github/workflows/publish-wasm.yml new file mode 100644 index 0000000..7d2d95e --- /dev/null +++ b/.github/workflows/publish-wasm.yml @@ -0,0 +1,77 @@ +# Publish @everruns/markkit-wasm (browser / edge WebAssembly package) to npm. +# +# Run from the verified release tag, like publish-js.yml. Defaults to a dry run. +# Publishes with npm trusted publishing (GitHub OIDC), no stored token: +# npmjs.com @everruns/markkit-wasm > Settings > Trusted publisher: +# GitHub Actions, everruns/markkit, workflow publish-wasm.yml, no environment. +# npm >= 11.5.1 is required for the OIDC exchange (provenance is automatic). + +name: Publish WASM + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run" + required: false + default: "true" + +permissions: + contents: read + +jobs: + publish: + name: Publish npm package (wasm) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + id-token: write # npm trusted publishing + provenance + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Verify publish source is reachable from main + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$(git rev-parse HEAD)" origin/main + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: actions/setup-node@v4 + with: + node-version: "24.22.1" + registry-url: "https://registry.npmjs.org" + - name: Require npm with trusted publishing (>= 11.5.1) + run: | + npm_ok() { + node -e 'const [a,b,c]=process.argv[1].split(".").map(Number);process.exit(a>11||(a==11&&(b>5||(b==5&&c>=1)))?0:1)' "$(npm --version)" + } + npm_ok || npm install -g npm@11.6.2 + npm_ok || { echo "::error::npm $(npm --version) lacks trusted publishing"; exit 1; } + - name: Install wasm-bindgen-cli (matches Cargo.lock) + run: bash scripts/install-wasm-bindgen.sh + - name: Build package + run: bash crates/markkit-wasm/scripts/build.sh release + - name: Verify build preserved the lockfile + run: git diff --exit-code -- Cargo.lock + - name: Integration tests (headless Node) + run: node --test crates/markkit-wasm/__test__/*.test.mjs + - name: Verify tag matches package version + run: | + PKG_VERSION=$(node -p "require('./crates/markkit-wasm/pkg/package.json').version") + if [[ "$GITHUB_REF" =~ ^refs/tags/v ]] && [ "${GITHUB_REF#refs/tags/v}" != "$PKG_VERSION" ]; then + echo "::error::tag ${GITHUB_REF#refs/tags/} does not match package version $PKG_VERSION" + exit 1 + fi + - name: Publish + working-directory: crates/markkit-wasm/pkg + env: + DRY_RUN: ${{ inputs.dry_run }} + run: | + if [ "$DRY_RUN" != "false" ]; then + npm publish --dry-run --ignore-scripts --access public + else + npm publish --ignore-scripts --access public + fi diff --git a/AGENTS.md b/AGENTS.md index 881e4b1..d167516 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | security/security-model | One-page threat/invariant summary | | security/threat-model | Threats T1-T6, mitigations, test map, resource limits, invariants S1-S12 | | runtimes/python-package | PyO3 `markkit` module, maturin, PyPI | +| runtimes/wasm-package | wasm-bindgen `@everruns/markkit-wasm`, browser/edge, build + tests | | runtimes/node-package | NAPI-RS `@everruns/markkit`, npm, binding parity table | | operations/testing | Test layout, categories, CI jobs | | operations/documentation | Doc trees and what each must cover | @@ -44,7 +45,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn - User docs: `README.md`, `docs/` - API reference: rustdoc in `crates/markkit/src/lib.rs` and modules -- Binding READMEs (`crates/markkit-python/README.md`, `crates/markkit-js/README.md`) ship to PyPI/npm, keep them in sync with exports +- Binding READMEs (`crates/markkit-python/README.md`, `crates/markkit-js/README.md`, `crates/markkit-wasm/README.md`) ship to PyPI/npm, keep them in sync with exports - Durable engineering knowledge: `knowledge/`, never `docs/` ### Maintenance invocation @@ -82,6 +83,7 @@ just pre-pr # Pre-PR checks | `markkit-eval` | LLM evaluation harness (stub) | | `markkit-python` | Python bindings (PyO3), module `markkit` | | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` | +| `markkit-wasm` | Browser/edge WebAssembly (wasm-bindgen), package `@everruns/markkit-wasm` | ### Rust diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e175..b7621e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). toolkit-library contract (builder, OpenAI tool definition, input/output schemas, `tower::Service`, single-use `ToolExecution`, `en-US`/`uk-UA` text). One call converts a `url`, `html` or base64 document to Markdown, with output capped by `max_output_chars`. Example: `examples/agent_tool.rs`. +- `@everruns/markkit-wasm`: browser / edge WebAssembly package (`crates/markkit-wasm`), single-threaded, + no special headers. `Markkit` class (HTML, DOCX, XLSX/XLS/ODS with validated camelCase options) and + `MarkkitTool`. Built by `crates/markkit-wasm/scripts/build.sh`, published by `publish-wasm.yml`. - `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats. ### Security diff --git a/Cargo.lock b/Cargo.lock index b3aa161..de65b94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1251,6 +1251,19 @@ dependencies = [ "pyo3-async-runtimes", ] +[[package]] +name = "markkit-wasm" +version = "0.1.1" +dependencies = [ + "js-sys", + "markkit", + "serde", + "serde-wasm-bindgen", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "markup5ever" version = "0.40.0" @@ -2112,6 +2125,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.229" diff --git a/Cargo.toml b/Cargo.toml index ab34c1a..dd72562 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/markkit-eval", "crates/markkit-python", "crates/markkit-js", + "crates/markkit-wasm", ] [workspace.package] @@ -75,3 +76,8 @@ lto = "thin" codegen-units = 1 panic = "abort" strip = true + +# Browser bundle (crates/markkit-wasm/scripts/build.sh): size over speed. +[profile.wasm-release] +inherits = "release" +opt-level = "z" diff --git a/README.md b/README.md index 3fdd27e..49447c3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Extract clean markdown from any format. Written in Rust. Markkit converts HTML, Word documents and spreadsheets into well-structured markdown, built for feeding untrusted documents to LLMs and agents. It ships as -a Rust library, a CLI, and native bindings for Python and Node.js. +a Rust library, a CLI, native bindings for Python and Node.js, and a WebAssembly +package for browsers and edge runtimes. ## Features @@ -101,6 +102,15 @@ const fromHtml = await extractHtml('

Hello

'); const fromUrl = await extractUrl('https://example.com'); ``` +### Browser / edge (WebAssembly) + +```javascript +import { initMarkkit, Markkit } from '@everruns/markkit-wasm'; + +await initMarkkit(); +const md = new Markkit().extractHtml('

Hello

'); +``` + ## Installation ### Rust @@ -132,6 +142,16 @@ npm install @everruns/markkit Native binary: Linux x64 (glibc). +### Browser / edge (WebAssembly) + +```bash +npm install @everruns/markkit-wasm +``` + +Any JS runtime (browsers, Cloudflare Workers, Deno, Bun, Node); no special headers. +HTML, DOCX and spreadsheets; no URL fetching. See +[`crates/markkit-wasm/README.md`](crates/markkit-wasm/README.md). + ## Configuration ```rust diff --git a/crates/markkit-wasm/.gitignore b/crates/markkit-wasm/.gitignore new file mode 100644 index 0000000..c54d13a --- /dev/null +++ b/crates/markkit-wasm/.gitignore @@ -0,0 +1,4 @@ +# wasm-bindgen build output (scripts/build.sh) +/pkg/ +node_modules/ +*.tgz diff --git a/crates/markkit-wasm/Cargo.toml b/crates/markkit-wasm/Cargo.toml new file mode 100644 index 0000000..7e00540 --- /dev/null +++ b/crates/markkit-wasm/Cargo.toml @@ -0,0 +1,38 @@ +# Markkit browser bindings via wasm-bindgen. +# +# Decision: slim, single-threaded wasm32-unknown-unknown build, separate from the +# napi `markkit-js` crate (same split as bashkit-wasm). No threads, no WASI, no +# SharedArrayBuffer and no COOP/COEP headers, so it runs in browsers, edge +# workers (Cloudflare, Vercel, Deno Deploy), Node, Deno and Bun. +# +# Feature surface: html, docx, spreadsheet, tool. `http_client` is excluded: +# reqwest needs sockets; JS hosts fetch themselves and pass bytes in, which also +# keeps SSRF policy with the host. See knowledge/runtimes/wasm-package.md. + +[package] +name = "markkit-wasm" +description = "Browser (WebAssembly) bindings for Markkit markdown extraction" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] +doc = false + +[dependencies] +markkit = { path = "../markkit", default-features = false, features = [ + "html", + "docx", + "spreadsheet", + "tool", +] } +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +js-sys = "0.3" +serde = { workspace = true } +serde_json = { workspace = true } +serde-wasm-bindgen = "0.6" diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md new file mode 100644 index 0000000..76fe4c6 --- /dev/null +++ b/crates/markkit-wasm/README.md @@ -0,0 +1,67 @@ +# @everruns/markkit-wasm + +Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX and +spreadsheets (XLSX/XLSM, XLS, ODS) in the **browser and any JavaScript runtime**: +edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. + +Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` +headers. For a native Node.js addon use +[`@everruns/markkit`](https://www.npmjs.com/package/@everruns/markkit). + +No URL fetching: the host fetches and passes bytes in, so network policy stays +with the host. + +## Install + +```bash +npm install @everruns/markkit-wasm +``` + +## Quick start + +```js +import { initMarkkit, Markkit } from "@everruns/markkit-wasm"; + +await initMarkkit(); // once + +const mk = new Markkit({ includeSelectors: ["article"], preserveImages: false }); +console.log(mk.extractHtml("

Hello

")); // # Hello + +const res = await fetch("/report.docx"); +const doc = mk.extractDocument(new Uint8Array(await res.arrayBuffer()), { + contentType: res.headers.get("content-type") ?? undefined, + fileName: "report.docx", +}); +console.log(doc.title, doc.format, doc.markdown); +``` + +## Options + +`maxInputSize`, `preserveLinks`, `preserveImages`, `includeSelectors`, +`excludeSelectors` (added to the defaults), `maxArchiveEntries`, +`maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, +`maxSpreadsheetCells`, `includeHiddenSheets`. Unknown keys throw. + +## Agent tool + +`MarkkitTool` follows the Everruns toolkit contract: tool definition, schemas, +system prompt and a validated `execute`. Sources are `html` or `content_base64`. + +```js +import { initMarkkit, MarkkitTool } from "@everruns/markkit-wasm"; +await initMarkkit(); + +const tool = new MarkkitTool({ maxOutputChars: 20000 }); +const tools = [tool.toolDefinition()]; // send with the model request + +try { + const result = await tool.execute({ html: "

Title

" }); + // { markdown, title, format, warnings, truncated } +} catch (e) { + if (e.userFacing) { /* safe to return e.message to the model */ } +} +``` + +## License + +MIT diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs new file mode 100644 index 0000000..0f73332 --- /dev/null +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -0,0 +1,139 @@ +// Headless Node tests against the built pkg/ (run scripts/build.sh first). +import { test, before } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { docx, zip } from "./zip.mjs"; +import { initMarkkit, Markkit, MarkkitTool, version } from "../pkg/index.js"; + +const pkgJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url))); + +before(async () => { + await initMarkkit(); + await initMarkkit(); // idempotent +}); + +test("version matches package.json", () => { + assert.equal(version(), pkgJson.version); +}); + +test("extractHtml with defaults", () => { + const mk = new Markkit(); + assert.equal(mk.extractHtml("

Hello

World

"), "# Hello\n\nWorld\n"); +}); + +test("options are applied", () => { + const html = ''; + const mk = new Markkit({ preserveLinks: false, includeSelectors: ["article"], excludeSelectors: [".ad"] }); + assert.equal(mk.extractHtml(html), "Read this\n"); +}); + +test("unknown option is rejected", () => { + assert.throws(() => new Markkit({ preserveLink: false }), /invalid options/); + assert.throws(() => new Markkit({ maxInputSize: "big" }), /invalid options/); + assert.throws(() => new Markkit().extractDocument(new Uint8Array(), { filename: "a.docx" }), /invalid options/); +}); + +test("input size limit", () => { + const mk = new Markkit({ maxInputSize: 4 }); + assert.throws(() => mk.extractHtml("

too long

"), /Resource limit/); +}); + +test("extractDocument returns metadata", () => { + const mk = new Markkit(); + const out = mk.extractDocument(new TextEncoder().encode("

bytes

"), { fileName: "a.html" }); + assert.equal(out.markdown, "bytes\n"); + assert.equal(out.format, "html"); + assert.deepEqual(out.warnings, []); + assert.equal(out.title, null); +}); + +test("extractDocument exposes page metadata in camelCase", () => { + const html = 'T

b

'; + const out = new Markkit().extractDocument(new TextEncoder().encode(html), { contentType: "text/html" }); + assert.equal(out.title, "T"); + assert.equal(out.metadata.siteName, "Site"); + assert.equal(out.metadata.canonicalUrl, "https://x.test/a"); + assert.equal(out.metadata.language, "en"); + assert.equal(out.metadata.site_name, undefined); +}); + +test("extractBytes and unsupported formats", () => { + const mk = new Markkit(); + assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); + assert.throws(() => mk.extractDocument(new TextEncoder().encode("%PDF-1.7"), { fileName: "a.pdf" }), /Unsupported/); +}); + +test("tool metadata and definition", () => { + const tool = new MarkkitTool(); + assert.equal(tool.name, "markkit"); + assert.equal(tool.locale, "en-US"); + assert.ok(tool.systemPrompt().startsWith("markkit:")); + const def = tool.toolDefinition(); + assert.equal(def.type, "function"); + assert.equal(def.function.name, "markkit"); + // No http_client in wasm: the url source is not offered. + assert.equal(def.function.parameters.properties.url, undefined); + assert.deepEqual(tool.inputSchema(), def.function.parameters); + assert.ok(tool.outputSchema().required.includes("markdown")); +}); + +test("tool rejects unknown options", () => { + assert.throws(() => new MarkkitTool({ maxOutputChar: 5 }), /invalid options/); + assert.throws(() => new MarkkitTool({ locale: 5 }), /invalid options/); + assert.throws(() => new MarkkitTool("x"), /invalid options/); +}); + +test("tool locale", () => { + const tool = new MarkkitTool({ locale: "uk-UA" }); + assert.equal(tool.name, "markkit"); + assert.equal(tool.displayName, "Витяг Markdown"); +}); + +test("tool executes html and base64", async () => { + const tool = new MarkkitTool({ maxOutputChars: 5 }); + const html = await tool.execute({ html: "

Hi

" }); + assert.equal(html.markdown, "# Hi\n"); + const b64 = Buffer.from("

abcdefgh

").toString("base64"); + const out = await tool.execute({ content_base64: b64, file_name: "a.html" }); + assert.equal(out.markdown, "abcde"); + assert.equal(out.truncated, true); +}); + +test("tool errors carry userFacing", async () => { + const tool = new MarkkitTool(); + await assert.rejects(tool.execute({}), e => e.userFacing === true && /exactly one/.test(e.message)); + await assert.rejects(tool.execute({ url: "https://example.com" }), e => e.userFacing === true && /disabled/.test(e.message)); + await assert.rejects(tool.execute({ content_base64: "!!" }), e => e.userFacing === true); +}); + +test("DOCX converts (zip + XML stack works in wasm)", () => { + const mk = new Markkit(); + const bytes = docx('ReportBody text'); + const out = mk.extractDocument(bytes, { fileName: "r.docx" }); + assert.equal(out.format, "docx"); + assert.match(out.markdown, /Report/); + assert.match(out.markdown, /Body text/); + assert.equal(out.stats.paragraphs, 2); +}); + +test("zip bomb style archive limits hold in wasm", () => { + const mk = new Markkit({ maxArchiveEntrySize: 1024 }); + const bytes = docx(`${"a".repeat(64 * 1024)}`); + assert.throws(() => mk.extractDocument(bytes, { fileName: "r.docx" }), /limit/i); +}); + +test("malformed office input throws, never traps", () => { + const mk = new Markkit(); + assert.throws(() => mk.extractDocument(zip([["junk", "x"]]), { fileName: "a.docx" })); + assert.throws(() => mk.extractDocument(new Uint8Array([0x50, 0x4b, 3, 4, 0, 0]), { fileName: "a.xlsx" })); + // Instance still usable after errors. + assert.equal(mk.extractHtml("

ok

"), "ok\n"); +}); + +test("tool converts base64 DOCX", async () => { + const tool = new MarkkitTool(); + const b64 = Buffer.from(docx("From tool")).toString("base64"); + const out = await tool.execute({ content_base64: b64, file_name: "a.docx" }); + assert.equal(out.format, "docx"); + assert.match(out.markdown, /From tool/); +}); diff --git a/crates/markkit-wasm/__test__/zip.mjs b/crates/markkit-wasm/__test__/zip.mjs new file mode 100644 index 0000000..c1e4c70 --- /dev/null +++ b/crates/markkit-wasm/__test__/zip.mjs @@ -0,0 +1,67 @@ +// Minimal ZIP writer for test fixtures (deflate via node:zlib, no deps). +import { deflateRawSync } from "node:zlib"; + +const CRC_TABLE = Array.from({ length: 256 }, (_, n) => { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + return c >>> 0; +}); + +function crc32(buf) { + let c = 0xffffffff; + for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; +} + +/** entries: [name, string | Uint8Array][] -> Uint8Array */ +export function zip(entries) { + const locals = []; + const centrals = []; + let offset = 0; + for (const [name, content] of entries) { + const data = typeof content === "string" ? Buffer.from(content) : Buffer.from(content); + const packed = deflateRawSync(data); + const nameBuf = Buffer.from(name); + const crc = crc32(data); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(8, 8); // deflate + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(packed.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(nameBuf.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(8, 10); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(packed.length, 20); + central.writeUInt32LE(data.length, 24); + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt32LE(offset, 42); + locals.push(local, nameBuf, packed); + centrals.push(central, nameBuf); + offset += local.length + nameBuf.length + packed.length; + } + const cd = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(cd.length, 12); + end.writeUInt32LE(offset, 16); + return new Uint8Array(Buffer.concat([...locals, cd, end])); +} + +const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'; + +/** Minimal DOCX with the given `` inner XML. */ +export function docx(body) { + return zip([ + ["[Content_Types].xml", ''], + ["_rels/.rels", ''], + ["word/document.xml", `${body}`], + ]); +} diff --git a/crates/markkit-wasm/js/index.d.ts b/crates/markkit-wasm/js/index.d.ts new file mode 100644 index 0000000..0fb6808 --- /dev/null +++ b/crates/markkit-wasm/js/index.d.ts @@ -0,0 +1,118 @@ +/** Load the WebAssembly module. Call (and await) once before using any class. */ +export function initMarkkit( + input?: RequestInfo | URL | Response | BufferSource | WebAssembly.Module, +): Promise; + +/** Library version. */ +export function version(): string; + +/** Extraction options (all optional; defaults match the Rust builder). */ +export interface MarkkitOptions { + /** Maximum input size in bytes (default 50 MiB). */ + maxInputSize?: number; + preserveLinks?: boolean; + preserveImages?: boolean; + /** CSS selectors to include (empty = whole document). */ + includeSelectors?: string[]; + /** CSS selectors to drop, added to nav/footer/header/script/style/noscript. */ + excludeSelectors?: string[]; + maxArchiveEntries?: number; + maxArchiveEntrySize?: number; + maxArchiveTotalSize?: number; + maxSheetRows?: number; + maxSheetCols?: number; + maxSpreadsheetCells?: number; + includeHiddenSheets?: boolean; +} + +export interface DocumentHint { + /** MIME type, e.g. from a `Content-Type` header. */ + contentType?: string; + /** File name or URL path; only the extension is used. */ + fileName?: string; +} + +export interface ExtractionStats { + pages: number | null; + sheets: number | null; + rows: number | null; + paragraphs: number | null; + tables: number | null; +} + +/** Document metadata (HTML meta, Open Graph, JSON-LD); fields are null when absent. */ +export interface Metadata { + description: string | null; + author: string | null; + published: string | null; + language: string | null; + canonicalUrl: string | null; + siteName: string | null; + image: string | null; +} + +export interface Extraction { + markdown: string; + title: string | null; + format: string; + warnings: string[]; + stats: ExtractionStats; + metadata: Metadata; +} + +export class Markkit { + constructor(options?: MarkkitOptions); + extractHtml(html: string): string; + extractDocument(bytes: Uint8Array, hint?: DocumentHint): Extraction; + extractBytes(bytes: Uint8Array, contentType?: string): string; + free(): void; +} + +export interface MarkkitToolOptions extends MarkkitOptions { + /** `en-US` (default) or `uk-UA`. */ + locale?: string; + /** Cap on returned Markdown characters (default 100 000). */ + maxOutputChars?: number; +} + +/** Tool call arguments (exactly one of `html` / `content_base64`). */ +export interface MarkkitToolArgs { + html?: string; + content_base64?: string; + file_name?: string; + content_type?: string; + max_chars?: number | null; +} + +export interface MarkkitToolResult { + markdown: string; + title: string | null; + format: string; + warnings: string[]; + truncated: boolean; +} + +/** Rejection reason of `MarkkitTool.execute`. */ +export interface MarkkitToolError extends Error { + /** True when the message is safe to return to the model. */ + userFacing: boolean; +} + +/** LLM agent tool (Everruns toolkit contract). No `url` source in wasm. */ +export class MarkkitTool { + constructor(options?: MarkkitToolOptions); + readonly name: string; + readonly displayName: string; + readonly shortDescription: string; + readonly description: string; + readonly version: string; + readonly locale: string; + help(): string; + systemPrompt(): string; + inputSchema(): Record; + outputSchema(): Record; + /** OpenAI-compatible function tool definition. */ + toolDefinition(): Record; + execute(args: MarkkitToolArgs): Promise; + free(): void; +} diff --git a/crates/markkit-wasm/js/index.js b/crates/markkit-wasm/js/index.js new file mode 100644 index 0000000..f5cd7d7 --- /dev/null +++ b/crates/markkit-wasm/js/index.js @@ -0,0 +1,24 @@ +// Entry point for @everruns/markkit-wasm. +// +// `initMarkkit()` loads the .wasm once. In browsers and edge runtimes the +// generated glue fetches `markkit_wasm_bg.wasm` relative to this module; in +// Node (no fetch for file: URLs) the bytes are read from disk instead. +import init, { Markkit, MarkkitTool, version } from "./markkit_wasm.js"; + +let ready; + +export function initMarkkit(input) { + if (!ready) { + ready = (async () => { + let source = input; + if (source === undefined && typeof process !== "undefined" && process.versions?.node) { + const { readFile } = await import("node:fs/promises"); + source = await readFile(new URL("./markkit_wasm_bg.wasm", import.meta.url)); + } + await init(source === undefined ? undefined : { module_or_path: source }); + })(); + } + return ready; +} + +export { Markkit, MarkkitTool, version }; diff --git a/crates/markkit-wasm/package.json b/crates/markkit-wasm/package.json new file mode 100644 index 0000000..ddf0f60 --- /dev/null +++ b/crates/markkit-wasm/package.json @@ -0,0 +1,46 @@ +{ + "name": "@everruns/markkit-wasm", + "version": "0.1.1", + "description": "Extract clean Markdown from HTML, DOCX and spreadsheets in the browser and edge runtimes (WebAssembly). Includes an LLM agent tool.", + "type": "module", + "main": "index.js", + "module": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./markkit_wasm_bg.wasm": "./markkit_wasm_bg.wasm" + }, + "sideEffects": false, + "files": [ + "index.js", + "index.d.ts", + "markkit_wasm.js", + "markkit_wasm.d.ts", + "markkit_wasm_bg.wasm", + "markkit_wasm_bg.wasm.d.ts", + "README.md" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/everruns/markkit" + }, + "keywords": [ + "markdown", + "html", + "docx", + "xlsx", + "wasm", + "webassembly", + "browser", + "ai", + "agent", + "tool" + ], + "publishConfig": { + "access": "public" + } +} diff --git a/crates/markkit-wasm/pkg/index.d.ts b/crates/markkit-wasm/pkg/index.d.ts index 5527a8e..0fb6808 100644 --- a/crates/markkit-wasm/pkg/index.d.ts +++ b/crates/markkit-wasm/pkg/index.d.ts @@ -40,12 +40,24 @@ export interface ExtractionStats { tables: number | null; } +/** Document metadata (HTML meta, Open Graph, JSON-LD); fields are null when absent. */ +export interface Metadata { + description: string | null; + author: string | null; + published: string | null; + language: string | null; + canonicalUrl: string | null; + siteName: string | null; + image: string | null; +} + export interface Extraction { markdown: string; title: string | null; format: string; warnings: string[]; stats: ExtractionStats; + metadata: Metadata; } export class Markkit { diff --git a/crates/markkit-wasm/pkg/markkit_wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm.d.ts index d265fdb..a8c75a1 100644 --- a/crates/markkit-wasm/pkg/markkit_wasm.d.ts +++ b/crates/markkit-wasm/pkg/markkit_wasm.d.ts @@ -86,8 +86,8 @@ export interface InitOutput { readonly markkittool_toolDefinition: (a: number, b: number) => void; readonly markkittool_version: (a: number, b: number) => void; readonly version: (a: number) => void; - readonly __wasm_bindgen_func_elem_1938: (a: number, b: number, c: number, d: number) => void; - readonly __wasm_bindgen_func_elem_1940: (a: number, b: number, c: number, d: number) => void; + readonly __wasm_bindgen_func_elem_2058: (a: number, b: number, c: number, d: number) => void; + readonly __wasm_bindgen_func_elem_2060: (a: number, b: number, c: number, d: number) => void; readonly __wbindgen_export: (a: number, b: number) => number; readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_export3: (a: number) => void; diff --git a/crates/markkit-wasm/pkg/markkit_wasm.js b/crates/markkit-wasm/pkg/markkit_wasm.js index 43b7b33..abf6ecd 100644 --- a/crates/markkit-wasm/pkg/markkit_wasm.js +++ b/crates/markkit-wasm/pkg/markkit_wasm.js @@ -596,7 +596,7 @@ function __wbg_get_imports() { const a = state0.a; state0.a = 0; try { - return __wasm_bindgen_func_elem_1940(a, state0.b, arg0, arg1); + return __wasm_bindgen_func_elem_2060(a, state0.b, arg0, arg1); } finally { state0.a = a; } @@ -668,8 +668,8 @@ function __wbg_get_imports() { return addHeapObject(ret); }, __wbindgen_generic_0000000000000001: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 146, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, __wasm_bindgen_func_elem_1938); + // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 145, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`. + const ret = makeMutClosure(arg0, arg1, __wasm_bindgen_func_elem_2058); return addHeapObject(ret); }, __wbindgen_generic_0000000000000002: function(arg0) { @@ -706,14 +706,14 @@ function __wbg_get_imports() { }; } -function __wasm_bindgen_func_elem_1940(arg0, arg1, arg2, arg3) { - wasm.__wasm_bindgen_func_elem_1940(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3)); +function __wasm_bindgen_func_elem_2060(arg0, arg1, arg2, arg3) { + wasm.__wasm_bindgen_func_elem_2060(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3)); } -function __wasm_bindgen_func_elem_1938(arg0, arg1, arg2) { +function __wasm_bindgen_func_elem_2058(arg0, arg1, arg2) { try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.__wasm_bindgen_func_elem_1938(retptr, arg0, arg1, addHeapObject(arg2)); + wasm.__wasm_bindgen_func_elem_2058(retptr, arg0, arg1, addHeapObject(arg2)); var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); if (r1) { diff --git a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts index 2459379..7e84982 100644 --- a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts +++ b/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts @@ -21,8 +21,8 @@ export const markkittool_systemPrompt: (a: number, b: number) => void; export const markkittool_toolDefinition: (a: number, b: number) => void; export const markkittool_version: (a: number, b: number) => void; export const version: (a: number) => void; -export const __wasm_bindgen_func_elem_1938: (a: number, b: number, c: number, d: number) => void; -export const __wasm_bindgen_func_elem_1940: (a: number, b: number, c: number, d: number) => void; +export const __wasm_bindgen_func_elem_2058: (a: number, b: number, c: number, d: number) => void; +export const __wasm_bindgen_func_elem_2060: (a: number, b: number, c: number, d: number) => void; export const __wbindgen_export: (a: number, b: number) => number; export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; export const __wbindgen_export3: (a: number) => void; diff --git a/crates/markkit-wasm/scripts/build.sh b/crates/markkit-wasm/scripts/build.sh new file mode 100755 index 0000000..004f6e5 --- /dev/null +++ b/crates/markkit-wasm/scripts/build.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Build @everruns/markkit-wasm into pkg/ (ES module, `wasm-bindgen --target web`). +# Needs no bundler and no cross-origin isolation headers to load. +# +# Requires: rustup target add wasm32-unknown-unknown; bash scripts/install-wasm-bindgen.sh +# Optional (smaller output): wasm-opt (binaryen) on PATH. +set -euo pipefail + +CRATE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKSPACE_ROOT="$(cd "$CRATE_DIR/../.." && pwd)" +OUT_DIR="$CRATE_DIR/pkg" +TARGET=wasm32-unknown-unknown +PROFILE="${1:-release}" +case "$PROFILE" in + release) PROFILE=wasm-release; TARGET_SUBDIR=wasm-release ;; + dev|debug) PROFILE=dev; TARGET_SUBDIR=debug ;; + *) echo "unsupported profile: $PROFILE" >&2; exit 1 ;; +esac + +cd "$WORKSPACE_ROOT" +echo "==> cargo build ($PROFILE) for $TARGET" +cargo build --locked -p markkit-wasm --target "$TARGET" --profile "$PROFILE" +WASM_IN="${CARGO_TARGET_DIR:-$WORKSPACE_ROOT/target}/$TARGET/$TARGET_SUBDIR/markkit_wasm.wasm" + +echo "==> wasm-bindgen --target web" +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" +wasm-bindgen "$WASM_IN" --target web --out-dir "$OUT_DIR" --out-name markkit_wasm + +if command -v wasm-opt >/dev/null 2>&1; then + echo "==> wasm-opt -Oz" + wasm-opt -Oz --enable-bulk-memory --enable-nontrapping-float-to-int --enable-sign-ext \ + "$OUT_DIR/markkit_wasm_bg.wasm" -o "$OUT_DIR/markkit_wasm_bg.wasm" +else + echo "==> wasm-opt not found; skipping size optimization" +fi + +cp "$CRATE_DIR/js/index.js" "$CRATE_DIR/js/index.d.ts" "$OUT_DIR/" +cp "$CRATE_DIR/package.json" "$CRATE_DIR/README.md" "$OUT_DIR/" +echo "==> built $OUT_DIR" +ls -lh "$OUT_DIR" diff --git a/crates/markkit-wasm/src/lib.rs b/crates/markkit-wasm/src/lib.rs new file mode 100644 index 0000000..c49ef35 --- /dev/null +++ b/crates/markkit-wasm/src/lib.rs @@ -0,0 +1,319 @@ +//! Browser / edge bindings for Markkit (`@everruns/markkit-wasm`). +//! +//! Decisions: +//! - Class API (`Markkit`, `MarkkitTool`) so one configured instance is reused; +//! options are a plain camelCase object validated with `deny_unknown_fields` +//! so typos fail loudly instead of silently using defaults. +//! - Bytes in, Markdown out. No URL fetching in wasm: the host fetches. +//! - Errors are thrown as `Error` with the Rust `Display` text. + +use serde::{Deserialize, Serialize}; +use wasm_bindgen::prelude::*; + +use markkit::tool::{MarkkitTool as CoreTool, Tool}; +use markkit::{DocumentHint, MarkkitBuilder}; + +/// Extraction options shared by `Markkit` and `MarkkitTool` (camelCase in JS). +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Options { + max_input_size: Option, + preserve_links: Option, + preserve_images: Option, + /// Added to the default selector set. + include_selectors: Option>, + /// Added to the default exclusions (nav, footer, header, script, style, noscript). + exclude_selectors: Option>, + max_archive_entries: Option, + max_archive_entry_size: Option, + max_archive_total_size: Option, + max_sheet_rows: Option, + max_sheet_cols: Option, + max_spreadsheet_cells: Option, + include_hidden_sheets: Option, +} + +impl Options { + fn builder(self) -> MarkkitBuilder { + let mut b = MarkkitBuilder::default(); + macro_rules! set { + ($($field:ident),*) => {$( + if let Some(v) = self.$field { b = b.$field(v); } + )*}; + } + set!( + max_input_size, + preserve_links, + preserve_images, + max_archive_entries, + max_archive_entry_size, + max_archive_total_size, + max_sheet_rows, + max_sheet_cols, + max_spreadsheet_cells, + include_hidden_sheets + ); + for s in self.include_selectors.unwrap_or_default() { + b = b.include_selector(s); + } + for s in self.exclude_selectors.unwrap_or_default() { + b = b.exclude_selector(s); + } + b + } +} + +/// Tool options: `locale`, `maxOutputChars`, plus every [`Options`] key. +/// (serde cannot combine `flatten` with `deny_unknown_fields`, so split by hand.) +#[derive(Debug, Default)] +struct ToolOptions { + locale: Option, + max_output_chars: Option, + extraction: Options, +} + +impl ToolOptions { + fn from_js(value: JsValue) -> Result { + let mut map = match from_js::(value)? { + serde_json::Value::Null => return Ok(Self::default()), + serde_json::Value::Object(map) => map, + _ => return Err(JsError::new("invalid options: expected an object")), + }; + let invalid = |e: serde_json::Error| JsError::new(&format!("invalid options: {e}")); + let locale = map + .remove("locale") + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let max_output_chars = map + .remove("maxOutputChars") + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let extraction = serde_json::from_value(serde_json::Value::Object(map)).map_err(invalid)?; + Ok(Self { + locale, + max_output_chars, + extraction, + }) + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Hint { + content_type: Option, + file_name: Option, +} + +/// JS value -> `T`, with `undefined`/`null` meaning defaults. +/// +/// Goes through `serde_json::Value`: serde-wasm-bindgen only reads the fields a +/// struct names, so `deny_unknown_fields` would never see a typo otherwise. +fn from_js Deserialize<'de> + Default>(value: JsValue) -> Result { + if value.is_undefined() || value.is_null() { + return Ok(T::default()); + } + let json: serde_json::Value = serde_wasm_bindgen::from_value(value) + .map_err(|e| JsError::new(&format!("invalid options: {e}")))?; + serde_json::from_value(json).map_err(|e| JsError::new(&format!("invalid options: {e}"))) +} + +fn to_js(value: &T) -> Result { + value + .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) + .map_err(|e| JsError::new(&e.to_string())) +} + +/// Rust serializes `snake_case`; JS callers get `camelCase` (`canonicalUrl`). +fn camel_case_keys(value: &mut serde_json::Value) { + if let serde_json::Value::Object(map) = value { + *map = std::mem::take(map) + .into_iter() + .map(|(k, v)| { + let mut out = String::with_capacity(k.len()); + let mut upper = false; + for c in k.chars() { + if c == '_' { + upper = true; + } else if upper { + out.extend(c.to_uppercase()); + upper = false; + } else { + out.push(c); + } + } + (out, v) + }) + .collect(); + } +} + +fn js_err(e: impl std::fmt::Display) -> JsError { + JsError::new(&e.to_string()) +} + +/// Library version. +#[wasm_bindgen] +pub fn version() -> String { + markkit::tool::VERSION.to_string() +} + +/// Configured extractor. +#[wasm_bindgen] +pub struct Markkit { + inner: markkit::Markkit, +} + +#[wasm_bindgen] +impl Markkit { + /// `new Markkit(options?)`. + #[wasm_bindgen(constructor)] + pub fn new(options: JsValue) -> Result { + let opts: Options = from_js(options)?; + Ok(Self { + inner: opts.builder().build(), + }) + } + + /// HTML string to Markdown. + #[wasm_bindgen(js_name = extractHtml)] + pub fn extract_html(&self, html: &str) -> Result { + self.inner.extract_html_sync(html).map_err(js_err) + } + + /// Document bytes to `{ markdown, title, format, warnings, stats }`. + /// `hint` is `{ contentType?, fileName? }`. + #[wasm_bindgen(js_name = extractDocument)] + pub fn extract_document(&self, bytes: &[u8], hint: JsValue) -> Result { + let hint: Hint = from_js(hint)?; + let extraction = self + .inner + .extract_document_sync( + bytes, + DocumentHint { + content_type: hint.content_type.as_deref(), + file_name: hint.file_name.as_deref(), + }, + ) + .map_err(js_err)?; + let mut value = serde_json::to_value(&extraction).map_err(js_err)?; + camel_case_keys(&mut value["metadata"]); + to_js(&value) + } + + /// Document bytes to Markdown only. + #[wasm_bindgen(js_name = extractBytes)] + pub fn extract_bytes( + &self, + bytes: &[u8], + content_type: Option, + ) -> Result { + self.inner + .extract_document_sync( + bytes, + DocumentHint { + content_type: content_type.as_deref(), + file_name: None, + }, + ) + .map(|e| e.markdown) + .map_err(js_err) + } +} + +/// LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. +#[wasm_bindgen] +pub struct MarkkitTool { + builder: markkit::tool::MarkkitToolBuilder, + tool: CoreTool, +} + +#[wasm_bindgen] +impl MarkkitTool { + /// `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. + #[wasm_bindgen(constructor)] + pub fn new(options: JsValue) -> Result { + let opts = ToolOptions::from_js(options)?; + let mut builder = CoreTool::builder().config(opts.extraction.builder()); + if let Some(locale) = opts.locale { + builder = builder.locale(&locale); + } + if let Some(n) = opts.max_output_chars { + builder = builder.max_output_chars(n); + } + let tool = builder.build(); + Ok(Self { builder, tool }) + } + + #[wasm_bindgen(getter)] + pub fn name(&self) -> String { + self.tool.name().to_string() + } + + #[wasm_bindgen(getter, js_name = displayName)] + pub fn display_name(&self) -> String { + self.tool.display_name().to_string() + } + + #[wasm_bindgen(getter, js_name = shortDescription)] + pub fn short_description(&self) -> String { + self.tool.short_description().to_string() + } + + #[wasm_bindgen(getter)] + pub fn description(&self) -> String { + self.tool.description().to_string() + } + + #[wasm_bindgen(getter)] + pub fn version(&self) -> String { + self.tool.version().to_string() + } + + #[wasm_bindgen(getter)] + pub fn locale(&self) -> String { + self.tool.locale().to_string() + } + + pub fn help(&self) -> String { + self.tool.help() + } + + #[wasm_bindgen(js_name = systemPrompt)] + pub fn system_prompt(&self) -> String { + self.tool.system_prompt() + } + + #[wasm_bindgen(js_name = inputSchema)] + pub fn input_schema(&self) -> Result { + to_js(&self.tool.input_schema()) + } + + #[wasm_bindgen(js_name = outputSchema)] + pub fn output_schema(&self) -> Result { + to_js(&self.tool.output_schema()) + } + + /// OpenAI-compatible function tool definition. + #[wasm_bindgen(js_name = toolDefinition)] + pub fn tool_definition(&self) -> Result { + to_js(&self.builder.build_tool_definition()) + } + + /// Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` + /// whose `userFacing` property says whether the message is safe for the model. + pub async fn execute(&self, args: JsValue) -> Result { + let args: serde_json::Value = serde_wasm_bindgen::from_value(args) + .map_err(|e| tool_error(&markkit::tool::ToolError::UserFacing(e.to_string())))?; + let execution = self.tool.execution(args).map_err(|e| tool_error(&e))?; + let output = execution.execute().await.map_err(|e| tool_error(&e))?; + to_js(&output.result).map_err(JsValue::from) + } +} + +fn tool_error(err: &markkit::tool::ToolError) -> JsValue { + let e = js_sys::Error::new(&err.to_string()); + let _ = js_sys::Reflect::set(&e, &"userFacing".into(), &err.is_user_facing().into()); + e.into() +} diff --git a/justfile b/justfile index 86c7327..5aa093d 100644 --- a/justfile +++ b/justfile @@ -65,6 +65,12 @@ examples: echo "── example: agent_tool" cargo run -q -p markkit --features tool --example agent_tool +# Build the WASM package and run its Node tests (needs wasm32 target + wasm-bindgen-cli) +wasm: + bash scripts/install-wasm-bindgen.sh + bash crates/markkit-wasm/scripts/build.sh release + node --test crates/markkit-wasm/__test__/*.test.mjs + # ─── Run ───────────────────────────────────────────────────── # Run the CLI with arguments diff --git a/knowledge/index.md b/knowledge/index.md index cced66d..7f945cb 100644 --- a/knowledge/index.md +++ b/knowledge/index.md @@ -13,5 +13,5 @@ okf_version: "0.2" * [formats/](formats/) - Per-format parsing: HTML extraction, format detection, DOCX and spreadsheet mapping. * [integrations/](integrations/) - Agent-facing integrations: the LLM tool contract. * [security/](security/) - Security model and detailed threat model for untrusted input. -* [runtimes/](runtimes/) - Published language bindings for Python and Node.js. +* [runtimes/](runtimes/) - Published language bindings for Python, Node.js, and WebAssembly. * [operations/](operations/) - Testing, documentation, examples, maintenance, release, and eval. diff --git a/knowledge/integrations/tool-contract.md b/knowledge/integrations/tool-contract.md index 25e0562..4bdbc68 100644 --- a/knowledge/integrations/tool-contract.md +++ b/knowledge/integrations/tool-contract.md @@ -12,7 +12,7 @@ tags: # Tool Contract ## Status -Implemented (Rust core, `tool` feature). Bindings, WASM and an MCP server are not wired yet. +Implemented (Rust core, `tool` feature) and exposed in the [WASM Package](../runtimes/wasm-package.md) as `MarkkitTool` (no `url` source there). Python/Node bindings and an MCP server are not wired yet. ## Summary diff --git a/knowledge/log.md b/knowledge/log.md index 38c2dc7..c1d9c75 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,10 +2,10 @@ ## 2026-10-26 +* **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. * **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from ``, Open Graph, `` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. * **IR v2**: footnotes, definition lists, task lists, figures, table alignment and spans; `Document.ir_version` (`IR_VERSION = 2`), `#[non_exhaustive]` IR types, `Node::is_block`/`is_inline`. HTML maps GitHub/Pandoc footnotes, `
`, checkbox items, captioned figures, `colspan`/`rowspan`/alignment and row-header ``. New threat T2e and invariant S13 (span grid budget, footnote label sanitizing). See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md), [Threat Model](security/threat-model.md). * **Agent tool**: new [Tool Contract](integrations/tool-contract.md) concept in a new `integrations/` domain. `MarkkitTool` (opt-in `tool` feature) follows the Everruns toolkit-library contract; tool arguments are a new untrusted surface in the [Threat Model](security/threat-model.md). - * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index 247e902..7915123 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -21,7 +21,8 @@ Follows [Semantic Versioning](https://semver.org/): Releases are agent-driven; CI creates the tag, GitHub Release and crates.io upload. 1. Ask the agent: "Create release v0.X.Y". -2. Agent bumps `version` in root `Cargo.toml`, moves `CHANGELOG.md` Unreleased entries +2. Agent bumps `version` in root `Cargo.toml`, `crates/markkit-js/package.json` and + `crates/markkit-wasm/package.json`, moves `CHANGELOG.md` Unreleased entries under `## [0.X.Y] - YYYY-MM-DD`, runs `just pre-pr`, and opens a PR titled `chore(release): prepare v0.X.Y`. 3. Merging that PR to `main` (squash keeps the title as the commit subject) runs @@ -35,7 +36,7 @@ Releases are agent-driven; CI creates the tag, GitHub Release and crates.io uplo Manual fallback: run the **Release** workflow from `main` (workflow_dispatch); it reads the version from `Cargo.toml`. Python and npm packages are published separately with -`publish-python.yml` / `publish-js.yml`. +`publish-python.yml` / `publish-js.yml` / `publish-wasm.yml`. ## Artifacts @@ -44,6 +45,7 @@ version from `Cargo.toml`. Python and npm packages are published separately with | Rust crate | `cargo publish` | crates.io | | Python wheel | `maturin build` | PyPI | | npm package | `napi build` | npm | +| npm WASM package | `wasm-bindgen` | npm (`@everruns/markkit-wasm`, OIDC) | ## Native package scope and verification diff --git a/knowledge/runtimes/index.md b/knowledge/runtimes/index.md index e8f93e9..1108e47 100644 --- a/knowledge/runtimes/index.md +++ b/knowledge/runtimes/index.md @@ -2,3 +2,4 @@ * [Python Package](python-package.md) - PyO3 Python bindings, exported API, maturin build, and PyPI distribution. * [Node.js Package](node-package.md) - NAPI-RS Node.js bindings, exported API, build, and npm distribution. +* [WASM Package](wasm-package.md) - wasm-bindgen browser and edge package @everruns/markkit-wasm, its API, build, tests, and npm distribution. diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 1d75a46..519e39a 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -58,4 +58,5 @@ Published to npm by `publish-js.yml` from a verified release tag; see ## See also - [Python Package](python-package.md), the sibling binding +- [WASM Package](wasm-package.md), the browser and edge build - [Release Process](../operations/release-process.md), npm publication and tarball verification diff --git a/knowledge/runtimes/wasm-package.md b/knowledge/runtimes/wasm-package.md new file mode 100644 index 0000000..b8415a5 --- /dev/null +++ b/knowledge/runtimes/wasm-package.md @@ -0,0 +1,87 @@ +--- +type: Package Design +title: WASM Package +description: wasm-bindgen browser and edge package @everruns/markkit-wasm, its API, build, tests, and npm distribution. +tags: + - markkit + - wasm + - browser + - bindings +--- + +# WASM Package + +## Overview + +`crates/markkit-wasm` builds `@everruns/markkit-wasm`: Markkit compiled to +`wasm32-unknown-unknown` with `wasm-bindgen --target web`. It mirrors bashkit-wasm: + +- **Single-threaded**: no `SharedArrayBuffer`, no `COOP`/`COEP` headers; runs in + browsers (incl. iframes), edge workers, Node, Deno and Bun. +- **Separate from the napi package** (`@everruns/markkit`), which stays the fast + native option for Node on Linux x64. +- **Features**: `html`, `docx`, `spreadsheet`, `tool`. No `http_client`: reqwest + needs sockets, and the host fetching keeps network policy (SSRF) with the host. + The tool therefore offers only `html` and `content_base64`. + +## API + +```js +import { initMarkkit, Markkit, MarkkitTool, version } from "@everruns/markkit-wasm"; +await initMarkkit(); // once; idempotent +const mk = new Markkit({ includeSelectors: ["article"] }); +mk.extractHtml(html); // string +mk.extractDocument(bytes, { fileName }); // { markdown, title, format, warnings, stats, metadata } +mk.extractBytes(bytes, contentType); // string +const tool = new MarkkitTool({ locale, maxOutputChars }); +tool.toolDefinition(); await tool.execute(args); +``` + +Options are camelCase builder fields (`maxInputSize`, `preserveLinks`, +`preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet +limits, `includeHiddenSheets`). Selectors add to the defaults, as in Python. +Unknown keys and wrong types throw `invalid options`: values go through +`serde_json::Value` because serde-wasm-bindgen only reads named fields, so +`deny_unknown_fields` alone would never see a typo. + +Errors throw `Error` with the Rust `Display` text. `MarkkitTool.execute` rejects +with an `Error` carrying `userFacing` (see [Tool Contract](../integrations/tool-contract.md)). + +`initMarkkit()` reads the `.wasm` from disk under Node (no `fetch` for `file:` +URLs) and otherwise lets the glue fetch it next to the module; callers may pass +their own bytes, `Response`, URL or `WebAssembly.Module`. + +## Build + +```bash +rustup target add wasm32-unknown-unknown +bash scripts/install-wasm-bindgen.sh # CLI pinned to Cargo.lock's wasm-bindgen +bash crates/markkit-wasm/scripts/build.sh # -> crates/markkit-wasm/pkg +``` + +Uses the `wasm-release` profile (`opt-level = "z"`, `panic = "abort"`); runs +`wasm-opt -Oz` when binaryen is on PATH. Unoptimized output is about 1.4 MB. +With `panic = "abort"` a Rust panic traps the instance; parsers are expected to +return errors, never panic (invariant S9). + +## Tests + +`crates/markkit-wasm/__test__/*.test.mjs` (`node --test`) run against `pkg/`: +options and their validation, HTML, bytes, DOCX through the zip/XML stack +(fixtures built by `__test__/zip.mjs`), archive limits, malformed input, version +parity with `package.json`, and the tool (definition, locale, execution, +`userFacing` errors). CI job `wasm` in `ci.yml` also checks the core crate for +wasm32 and the npm archive contents; it gates `Check`. + +## Distribution + +`publish-wasm.yml` (workflow_dispatch, dry run by default) builds, tests and +publishes with npm trusted publishing (OIDC). The trusted publisher must be set up +on npmjs.com for `@everruns/markkit-wasm` before the first real publish. +`package.json` `version` must equal the workspace version (tested). + +## See also + +- [Node.js Package](node-package.md), the native napi sibling +- [Tool Contract](../integrations/tool-contract.md), what `MarkkitTool` implements +- [Release Process](../operations/release-process.md), where the publisher runs diff --git a/scripts/install-wasm-bindgen.sh b/scripts/install-wasm-bindgen.sh new file mode 100755 index 0000000..333e178 --- /dev/null +++ b/scripts/install-wasm-bindgen.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Install wasm-bindgen-cli matching the wasm-bindgen crate in Cargo.lock. +# The CLI and the crate must agree exactly on the bindgen schema version. +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +VERSION=$(awk '/^name = "wasm-bindgen"$/ { getline; gsub(/"/, "", $3); print $3 }' "$REPO_ROOT/Cargo.lock" | sort -u) +if [[ -z "$VERSION" || "$VERSION" == *$'\n'* ]]; then + echo "Expected exactly one wasm-bindgen version in Cargo.lock" >&2 + exit 1 +fi + +EXPECTED="wasm-bindgen $VERSION" +if [[ "$(wasm-bindgen --version 2>/dev/null || true)" != "$EXPECTED" ]]; then + cargo install wasm-bindgen-cli --version "$VERSION" --locked --force +fi +if [[ "$(wasm-bindgen --version)" != "$EXPECTED" ]]; then + echo "wasm-bindgen on PATH does not match Cargo.lock ($VERSION)" >&2 + exit 1 +fi +echo "$EXPECTED (matches Cargo.lock)" From 442dfc2a17a7354ebd1a925813ff7e8b72910719 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:54:16 +0000 Subject: [PATCH 5/5] chore(wasm): untrack build output accidentally committed in #19 --- crates/markkit-wasm/pkg/README.md | 67 -- crates/markkit-wasm/pkg/index.d.ts | 118 -- crates/markkit-wasm/pkg/index.js | 24 - crates/markkit-wasm/pkg/markkit_wasm.d.ts | 119 -- crates/markkit-wasm/pkg/markkit_wasm.js | 1064 ----------------- .../pkg/markkit_wasm_bg.wasm.d.ts | 31 - crates/markkit-wasm/pkg/package.json | 46 - 7 files changed, 1469 deletions(-) delete mode 100644 crates/markkit-wasm/pkg/README.md delete mode 100644 crates/markkit-wasm/pkg/index.d.ts delete mode 100644 crates/markkit-wasm/pkg/index.js delete mode 100644 crates/markkit-wasm/pkg/markkit_wasm.d.ts delete mode 100644 crates/markkit-wasm/pkg/markkit_wasm.js delete mode 100644 crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts delete mode 100644 crates/markkit-wasm/pkg/package.json diff --git a/crates/markkit-wasm/pkg/README.md b/crates/markkit-wasm/pkg/README.md deleted file mode 100644 index 76fe4c6..0000000 --- a/crates/markkit-wasm/pkg/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# @everruns/markkit-wasm - -Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX and -spreadsheets (XLSX/XLSM, XLS, ODS) in the **browser and any JavaScript runtime**: -edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. - -Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` -headers. For a native Node.js addon use -[`@everruns/markkit`](https://www.npmjs.com/package/@everruns/markkit). - -No URL fetching: the host fetches and passes bytes in, so network policy stays -with the host. - -## Install - -```bash -npm install @everruns/markkit-wasm -``` - -## Quick start - -```js -import { initMarkkit, Markkit } from "@everruns/markkit-wasm"; - -await initMarkkit(); // once - -const mk = new Markkit({ includeSelectors: ["article"], preserveImages: false }); -console.log(mk.extractHtml("

Hello

")); // # Hello - -const res = await fetch("/report.docx"); -const doc = mk.extractDocument(new Uint8Array(await res.arrayBuffer()), { - contentType: res.headers.get("content-type") ?? undefined, - fileName: "report.docx", -}); -console.log(doc.title, doc.format, doc.markdown); -``` - -## Options - -`maxInputSize`, `preserveLinks`, `preserveImages`, `includeSelectors`, -`excludeSelectors` (added to the defaults), `maxArchiveEntries`, -`maxArchiveEntrySize`, `maxArchiveTotalSize`, `maxSheetRows`, `maxSheetCols`, -`maxSpreadsheetCells`, `includeHiddenSheets`. Unknown keys throw. - -## Agent tool - -`MarkkitTool` follows the Everruns toolkit contract: tool definition, schemas, -system prompt and a validated `execute`. Sources are `html` or `content_base64`. - -```js -import { initMarkkit, MarkkitTool } from "@everruns/markkit-wasm"; -await initMarkkit(); - -const tool = new MarkkitTool({ maxOutputChars: 20000 }); -const tools = [tool.toolDefinition()]; // send with the model request - -try { - const result = await tool.execute({ html: "

Title

" }); - // { markdown, title, format, warnings, truncated } -} catch (e) { - if (e.userFacing) { /* safe to return e.message to the model */ } -} -``` - -## License - -MIT diff --git a/crates/markkit-wasm/pkg/index.d.ts b/crates/markkit-wasm/pkg/index.d.ts deleted file mode 100644 index 0fb6808..0000000 --- a/crates/markkit-wasm/pkg/index.d.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** Load the WebAssembly module. Call (and await) once before using any class. */ -export function initMarkkit( - input?: RequestInfo | URL | Response | BufferSource | WebAssembly.Module, -): Promise; - -/** Library version. */ -export function version(): string; - -/** Extraction options (all optional; defaults match the Rust builder). */ -export interface MarkkitOptions { - /** Maximum input size in bytes (default 50 MiB). */ - maxInputSize?: number; - preserveLinks?: boolean; - preserveImages?: boolean; - /** CSS selectors to include (empty = whole document). */ - includeSelectors?: string[]; - /** CSS selectors to drop, added to nav/footer/header/script/style/noscript. */ - excludeSelectors?: string[]; - maxArchiveEntries?: number; - maxArchiveEntrySize?: number; - maxArchiveTotalSize?: number; - maxSheetRows?: number; - maxSheetCols?: number; - maxSpreadsheetCells?: number; - includeHiddenSheets?: boolean; -} - -export interface DocumentHint { - /** MIME type, e.g. from a `Content-Type` header. */ - contentType?: string; - /** File name or URL path; only the extension is used. */ - fileName?: string; -} - -export interface ExtractionStats { - pages: number | null; - sheets: number | null; - rows: number | null; - paragraphs: number | null; - tables: number | null; -} - -/** Document metadata (HTML meta, Open Graph, JSON-LD); fields are null when absent. */ -export interface Metadata { - description: string | null; - author: string | null; - published: string | null; - language: string | null; - canonicalUrl: string | null; - siteName: string | null; - image: string | null; -} - -export interface Extraction { - markdown: string; - title: string | null; - format: string; - warnings: string[]; - stats: ExtractionStats; - metadata: Metadata; -} - -export class Markkit { - constructor(options?: MarkkitOptions); - extractHtml(html: string): string; - extractDocument(bytes: Uint8Array, hint?: DocumentHint): Extraction; - extractBytes(bytes: Uint8Array, contentType?: string): string; - free(): void; -} - -export interface MarkkitToolOptions extends MarkkitOptions { - /** `en-US` (default) or `uk-UA`. */ - locale?: string; - /** Cap on returned Markdown characters (default 100 000). */ - maxOutputChars?: number; -} - -/** Tool call arguments (exactly one of `html` / `content_base64`). */ -export interface MarkkitToolArgs { - html?: string; - content_base64?: string; - file_name?: string; - content_type?: string; - max_chars?: number | null; -} - -export interface MarkkitToolResult { - markdown: string; - title: string | null; - format: string; - warnings: string[]; - truncated: boolean; -} - -/** Rejection reason of `MarkkitTool.execute`. */ -export interface MarkkitToolError extends Error { - /** True when the message is safe to return to the model. */ - userFacing: boolean; -} - -/** LLM agent tool (Everruns toolkit contract). No `url` source in wasm. */ -export class MarkkitTool { - constructor(options?: MarkkitToolOptions); - readonly name: string; - readonly displayName: string; - readonly shortDescription: string; - readonly description: string; - readonly version: string; - readonly locale: string; - help(): string; - systemPrompt(): string; - inputSchema(): Record; - outputSchema(): Record; - /** OpenAI-compatible function tool definition. */ - toolDefinition(): Record; - execute(args: MarkkitToolArgs): Promise; - free(): void; -} diff --git a/crates/markkit-wasm/pkg/index.js b/crates/markkit-wasm/pkg/index.js deleted file mode 100644 index f5cd7d7..0000000 --- a/crates/markkit-wasm/pkg/index.js +++ /dev/null @@ -1,24 +0,0 @@ -// Entry point for @everruns/markkit-wasm. -// -// `initMarkkit()` loads the .wasm once. In browsers and edge runtimes the -// generated glue fetches `markkit_wasm_bg.wasm` relative to this module; in -// Node (no fetch for file: URLs) the bytes are read from disk instead. -import init, { Markkit, MarkkitTool, version } from "./markkit_wasm.js"; - -let ready; - -export function initMarkkit(input) { - if (!ready) { - ready = (async () => { - let source = input; - if (source === undefined && typeof process !== "undefined" && process.versions?.node) { - const { readFile } = await import("node:fs/promises"); - source = await readFile(new URL("./markkit_wasm_bg.wasm", import.meta.url)); - } - await init(source === undefined ? undefined : { module_or_path: source }); - })(); - } - return ready; -} - -export { Markkit, MarkkitTool, version }; diff --git a/crates/markkit-wasm/pkg/markkit_wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm.d.ts deleted file mode 100644 index a8c75a1..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm.d.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ - -/** - * Configured extractor. - */ -export class Markkit { - free(): void; - [Symbol.dispose](): void; - /** - * Document bytes to Markdown only. - */ - extractBytes(bytes: Uint8Array, content_type?: string | null): string; - /** - * Document bytes to `{ markdown, title, format, warnings, stats }`. - * `hint` is `{ contentType?, fileName? }`. - */ - extractDocument(bytes: Uint8Array, hint: any): any; - /** - * HTML string to Markdown. - */ - extractHtml(html: string): string; - /** - * `new Markkit(options?)`. - */ - constructor(options: any); -} - -/** - * LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. - */ -export class MarkkitTool { - free(): void; - [Symbol.dispose](): void; - /** - * Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` - * whose `userFacing` property says whether the message is safe for the model. - */ - execute(args: any): Promise; - help(): string; - inputSchema(): any; - /** - * `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. - */ - constructor(options: any); - outputSchema(): any; - systemPrompt(): string; - /** - * OpenAI-compatible function tool definition. - */ - toolDefinition(): any; - readonly description: string; - readonly displayName: string; - readonly locale: string; - readonly name: string; - readonly shortDescription: string; - readonly version: string; -} - -/** - * Library version. - */ -export function version(): string; - -export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; - -export interface InitOutput { - readonly memory: WebAssembly.Memory; - readonly __wbg_markkit_free: (a: number, b: number) => void; - readonly __wbg_markkittool_free: (a: number, b: number) => void; - readonly markkit_extractBytes: (a: number, b: number, c: number, d: number, e: number, f: number) => void; - readonly markkit_extractDocument: (a: number, b: number, c: number, d: number, e: number) => void; - readonly markkit_extractHtml: (a: number, b: number, c: number, d: number) => void; - readonly markkit_new: (a: number, b: number) => void; - readonly markkittool_description: (a: number, b: number) => void; - readonly markkittool_displayName: (a: number, b: number) => void; - readonly markkittool_execute: (a: number, b: number) => number; - readonly markkittool_help: (a: number, b: number) => void; - readonly markkittool_inputSchema: (a: number, b: number) => void; - readonly markkittool_locale: (a: number, b: number) => void; - readonly markkittool_name: (a: number, b: number) => void; - readonly markkittool_new: (a: number, b: number) => void; - readonly markkittool_outputSchema: (a: number, b: number) => void; - readonly markkittool_shortDescription: (a: number, b: number) => void; - readonly markkittool_systemPrompt: (a: number, b: number) => void; - readonly markkittool_toolDefinition: (a: number, b: number) => void; - readonly markkittool_version: (a: number, b: number) => void; - readonly version: (a: number) => void; - readonly __wasm_bindgen_func_elem_2058: (a: number, b: number, c: number, d: number) => void; - readonly __wasm_bindgen_func_elem_2060: (a: number, b: number, c: number, d: number) => void; - readonly __wbindgen_export: (a: number, b: number) => number; - readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; - readonly __wbindgen_export3: (a: number) => void; - readonly __wbindgen_export4: (a: number, b: number) => void; - readonly __wbindgen_add_to_stack_pointer: (a: number) => number; - readonly __wbindgen_export5: (a: number, b: number, c: number) => void; -} - -export type SyncInitInput = BufferSource | WebAssembly.Module; - -/** - * Instantiates the given `module`, which can either be bytes or - * a precompiled `WebAssembly.Module`. - * - * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated. - * - * @returns {InitOutput} - */ -export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput; - -/** - * If `module_or_path` is {RequestInfo} or {URL}, makes a request and - * for everything else, calls `WebAssembly.instantiate` directly. - * - * @param {{ module_or_path: InitInput | Promise }} module_or_path - Passing `InitInput` directly is deprecated. - * - * @returns {Promise} - */ -export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise } | InitInput | Promise): Promise; diff --git a/crates/markkit-wasm/pkg/markkit_wasm.js b/crates/markkit-wasm/pkg/markkit_wasm.js deleted file mode 100644 index abf6ecd..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm.js +++ /dev/null @@ -1,1064 +0,0 @@ -/* @ts-self-types="./markkit_wasm.d.ts" */ - -/** - * Configured extractor. - */ -export class Markkit { - __destroy_into_raw() { - const ptr = this.__wbg_ptr; - this.__wbg_ptr = 0; - MarkkitFinalization.unregister(this); - return ptr; - } - free() { - const ptr = this.__destroy_into_raw(); - wasm.__wbg_markkit_free(ptr, 0); - } - /** - * Document bytes to Markdown only. - * @param {Uint8Array} bytes - * @param {string | null} [content_type] - * @returns {string} - */ - extractBytes(bytes, content_type) { - let deferred4_0; - let deferred4_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export); - const len0 = WASM_VECTOR_LEN; - var ptr1 = isLikeNone(content_type) ? 0 : passStringToWasm0(content_type, wasm.__wbindgen_export, wasm.__wbindgen_export2); - var len1 = WASM_VECTOR_LEN; - wasm.markkit_extractBytes(retptr, this.__wbg_ptr, ptr0, len0, ptr1, len1); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); - var ptr3 = r0; - var len3 = r1; - if (r3) { - ptr3 = 0; len3 = 0; - throw takeObject(r2); - } - deferred4_0 = ptr3; - deferred4_1 = len3; - return getStringFromWasm0(ptr3, len3); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred4_0, deferred4_1, 1); - } - } - /** - * Document bytes to `{ markdown, title, format, warnings, stats }`. - * `hint` is `{ contentType?, fileName? }`. - * @param {Uint8Array} bytes - * @param {any} hint - * @returns {any} - */ - extractDocument(bytes, hint) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export); - const len0 = WASM_VECTOR_LEN; - wasm.markkit_extractDocument(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(hint)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * HTML string to Markdown. - * @param {string} html - * @returns {string} - */ - extractHtml(html) { - let deferred3_0; - let deferred3_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len0 = WASM_VECTOR_LEN; - wasm.markkit_extractHtml(retptr, this.__wbg_ptr, ptr0, len0); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); - var ptr2 = r0; - var len2 = r1; - if (r3) { - ptr2 = 0; len2 = 0; - throw takeObject(r2); - } - deferred3_0 = ptr2; - deferred3_1 = len2; - return getStringFromWasm0(ptr2, len2); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred3_0, deferred3_1, 1); - } - } - /** - * `new Markkit(options?)`. - * @param {any} options - */ - constructor(options) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkit_new(retptr, addHeapObject(options)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - this.__wbg_ptr = r0; - MarkkitFinalization.register(this, this.__wbg_ptr, this); - return this; - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } -} -if (Symbol.dispose) Markkit.prototype[Symbol.dispose] = Markkit.prototype.free; - -/** - * LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. - */ -export class MarkkitTool { - __destroy_into_raw() { - const ptr = this.__wbg_ptr; - this.__wbg_ptr = 0; - MarkkitToolFinalization.unregister(this); - return ptr; - } - free() { - const ptr = this.__destroy_into_raw(); - wasm.__wbg_markkittool_free(ptr, 0); - } - /** - * @returns {string} - */ - get description() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_description(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - get displayName() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_displayName(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` - * whose `userFacing` property says whether the message is safe for the model. - * @param {any} args - * @returns {Promise} - */ - execute(args) { - const ret = wasm.markkittool_execute(this.__wbg_ptr, addHeapObject(args)); - return takeObject(ret); - } - /** - * @returns {string} - */ - help() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_help(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {any} - */ - inputSchema() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_inputSchema(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get locale() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_locale(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - get name() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_name(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. - * @param {any} options - */ - constructor(options) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_new(retptr, addHeapObject(options)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - this.__wbg_ptr = r0; - MarkkitToolFinalization.register(this, this.__wbg_ptr, this); - return this; - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {any} - */ - outputSchema() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_outputSchema(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get shortDescription() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_shortDescription(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - systemPrompt() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_systemPrompt(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * OpenAI-compatible function tool definition. - * @returns {any} - */ - toolDefinition() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_toolDefinition(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get version() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_version(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } -} -if (Symbol.dispose) MarkkitTool.prototype[Symbol.dispose] = MarkkitTool.prototype.free; - -/** - * Library version. - * @returns {string} - */ -export function version() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.version(retptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } -} -function __wbg_get_imports() { - const import0 = { - __proto__: null, - __wbg_Error_30c8987f7c2ed4e2: function(arg0, arg1) { - const ret = Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_String_8564e559799eccda: function(arg0, arg1) { - const ret = String(getObject(arg1)); - const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_bigint_get_as_i64_a2383202b9353e4c: function(arg0, arg1) { - const v = getObject(arg1); - const ret = typeof(v) === 'bigint' ? v : undefined; - getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); - }, - __wbg___wbindgen_boolean_get_5b446f51afd21013: function(arg0) { - const v = getObject(arg0); - const ret = typeof(v) === 'boolean' ? v : undefined; - return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; - }, - __wbg___wbindgen_debug_string_4687d8d8c2017d52: function(arg0, arg1) { - const ret = debugString(getObject(arg1)); - const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_in_92f62ee1427d9e49: function(arg0, arg1) { - const ret = getObject(arg0) in getObject(arg1); - return ret; - }, - __wbg___wbindgen_is_bigint_b123553bed3bb382: function(arg0) { - const ret = typeof(getObject(arg0)) === 'bigint'; - return ret; - }, - __wbg___wbindgen_is_function_1f9d30630b8b1d3d: function(arg0) { - const ret = typeof(getObject(arg0)) === 'function'; - return ret; - }, - __wbg___wbindgen_is_null_e343b7d08827ba72: function(arg0) { - const ret = getObject(arg0) === null; - return ret; - }, - __wbg___wbindgen_is_object_3c45d4f2dde4e749: function(arg0) { - const val = getObject(arg0); - const ret = typeof(val) === 'object' && val !== null; - return ret; - }, - __wbg___wbindgen_is_string_90b56bc79aad6f6c: function(arg0) { - const ret = typeof(getObject(arg0)) === 'string'; - return ret; - }, - __wbg___wbindgen_is_undefined_8865fb403f8fe9d8: function(arg0) { - const ret = getObject(arg0) === undefined; - return ret; - }, - __wbg___wbindgen_jsval_eq_02babf21faa37971: function(arg0, arg1) { - const ret = getObject(arg0) === getObject(arg1); - return ret; - }, - __wbg___wbindgen_jsval_loose_eq_677f21e468d6b461: function(arg0, arg1) { - const ret = getObject(arg0) == getObject(arg1); - return ret; - }, - __wbg___wbindgen_number_get_2e0e7dee9f701a71: function(arg0, arg1) { - const obj = getObject(arg1); - const ret = typeof(obj) === 'number' ? obj : undefined; - getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); - }, - __wbg___wbindgen_string_get_0380ccaa2f57f0d9: function(arg0, arg1) { - const obj = getObject(arg1); - const ret = typeof(obj) === 'string' ? obj : undefined; - var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - var len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_throw_41e9ee4f547fc59a: function(arg0, arg1) { - throw new Error(getStringFromWasm0(arg0, arg1)); - }, - __wbg__wbg_cb_unref_dcc1a90847f04c41: function(arg0) { - getObject(arg0)._wbg_cb_unref(); - }, - __wbg_call_187d372bd5fdd4aa: function() { return handleError(function (arg0, arg1, arg2) { - const ret = getObject(arg0).call(getObject(arg1), getObject(arg2)); - return addHeapObject(ret); - }, arguments); }, - __wbg_call_6137034ef55c9d0f: function() { return handleError(function (arg0, arg1) { - const ret = getObject(arg0).call(getObject(arg1)); - return addHeapObject(ret); - }, arguments); }, - __wbg_done_b41a1d26cdb37fb6: function(arg0) { - const ret = getObject(arg0).done; - return ret; - }, - __wbg_entries_fb6397112b1de25f: function(arg0) { - const ret = Object.entries(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_get_658f6698067d9515: function() { return handleError(function (arg0, arg1) { - const ret = Reflect.get(getObject(arg0), getObject(arg1)); - return addHeapObject(ret); - }, arguments); }, - __wbg_get_6c896e0571ddae51: function(arg0, arg1) { - const ret = getObject(arg0)[arg1 >>> 0]; - return addHeapObject(ret); - }, - __wbg_get_unchecked_288889d017702237: function(arg0, arg1) { - const ret = getObject(arg0)[arg1 >>> 0]; - return addHeapObject(ret); - }, - __wbg_instanceof_ArrayBuffer_a99f175873e5d9b8: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof ArrayBuffer; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_instanceof_Map_b2611749102d7ba3: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof Map; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_instanceof_Uint8Array_828cef2aaacafc31: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof Uint8Array; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_isArray_e15a2ff68ffdbef2: function(arg0) { - const ret = Array.isArray(getObject(arg0)); - return ret; - }, - __wbg_isSafeInteger_717808ad6a54bd9e: function(arg0) { - const ret = Number.isSafeInteger(getObject(arg0)); - return ret; - }, - __wbg_iterator_e3c31c892080e444: function() { - const ret = Symbol.iterator; - return addHeapObject(ret); - }, - __wbg_length_7f3c00c40364105e: function(arg0) { - const ret = getObject(arg0).length; - return ret; - }, - __wbg_length_d4bdea10311bd9cf: function(arg0) { - const ret = getObject(arg0).length; - return ret; - }, - __wbg_new_1dbf7428bba60a42: function(arg0) { - const ret = new Uint8Array(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_new_28744009d011f847: function() { - const ret = new Map(); - return addHeapObject(ret); - }, - __wbg_new_343a093a3c2ffb4e: function(arg0, arg1) { - const ret = new Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_new_617a8cdb8bb1130e: function() { - const ret = new Object(); - return addHeapObject(ret); - }, - __wbg_new_ee2291f50781bf1d: function() { - const ret = new Array(); - return addHeapObject(ret); - }, - __wbg_new_typed_b01cb72a8af741a3: function(arg0, arg1) { - try { - var state0 = {a: arg0, b: arg1}; - var cb0 = (arg0, arg1) => { - const a = state0.a; - state0.a = 0; - try { - return __wasm_bindgen_func_elem_2060(a, state0.b, arg0, arg1); - } finally { - state0.a = a; - } - }; - const ret = new Promise(cb0); - return addHeapObject(ret); - } finally { - state0.a = 0; - } - }, - __wbg_next_33784799010f1bbe: function(arg0) { - const ret = getObject(arg0).next; - return addHeapObject(ret); - }, - __wbg_next_f4aac29c42af995c: function() { return handleError(function (arg0) { - const ret = getObject(arg0).next(); - return addHeapObject(ret); - }, arguments); }, - __wbg_prototypesetcall_bc27214492979395: function(arg0, arg1, arg2) { - Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), getObject(arg2)); - }, - __wbg_queueMicrotask_9833f9a49df95a49: function(arg0) { - const ret = getObject(arg0).queueMicrotask; - return addHeapObject(ret); - }, - __wbg_queueMicrotask_a72f977e97f23c5f: function(arg0) { - queueMicrotask(getObject(arg0)); - }, - __wbg_resolve_0076e10020304ede: function(arg0) { - const ret = Promise.resolve(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_set_145a351398b48c65: function() { return handleError(function (arg0, arg1, arg2) { - const ret = Reflect.set(getObject(arg0), getObject(arg1), getObject(arg2)); - return ret; - }, arguments); }, - __wbg_set_6ae97e73113c4f0b: function(arg0, arg1, arg2) { - const ret = getObject(arg0).set(getObject(arg1), getObject(arg2)); - return addHeapObject(ret); - }, - __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) { - getObject(arg0)[takeObject(arg1)] = takeObject(arg2); - }, - __wbg_set_bea140a88be9b277: function(arg0, arg1, arg2) { - getObject(arg0)[arg1 >>> 0] = takeObject(arg2); - }, - __wbg_static_accessor_GLOBAL_266715b9d96ba635: function() { - const ret = typeof global === 'undefined' ? null : global; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_GLOBAL_THIS_10fb7dc1ae063179: function() { - const ret = typeof globalThis === 'undefined' ? null : globalThis; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_SELF_0b583911f537483a: function() { - const ret = typeof self === 'undefined' ? null : self; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_WINDOW_d7f903d1508cbdc4: function() { - const ret = typeof window === 'undefined' ? null : window; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_then_e71170d78fcf8954: function(arg0, arg1) { - const ret = getObject(arg0).then(getObject(arg1)); - return addHeapObject(ret); - }, - __wbg_value_f3c585ee8f5ba40c: function(arg0) { - const ret = getObject(arg0).value; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000001: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 145, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, __wasm_bindgen_func_elem_2058); - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000002: function(arg0) { - // Cast intrinsic for `F64 -> Externref`. - const ret = arg0; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000003: function(arg0) { - // Cast intrinsic for `I64 -> Externref`. - const ret = arg0; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000004: function(arg0, arg1) { - // Cast intrinsic for `Ref(String) -> Externref`. - const ret = getStringFromWasm0(arg0, arg1); - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000005: function(arg0) { - // Cast intrinsic for `U64 -> Externref`. - const ret = BigInt.asUintN(64, arg0); - return addHeapObject(ret); - }, - __wbindgen_object_clone_ref: function(arg0) { - const ret = getObject(arg0); - return addHeapObject(ret); - }, - __wbindgen_object_drop_ref: function(arg0) { - takeObject(arg0); - }, - }; - return { - __proto__: null, - "./markkit_wasm_bg.js": import0, - }; -} - -function __wasm_bindgen_func_elem_2060(arg0, arg1, arg2, arg3) { - wasm.__wasm_bindgen_func_elem_2060(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3)); -} - -function __wasm_bindgen_func_elem_2058(arg0, arg1, arg2) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.__wasm_bindgen_func_elem_2058(retptr, arg0, arg1, addHeapObject(arg2)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - if (r1) { - throw takeObject(r0); - } - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } -} - -const MarkkitFinalization = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(ptr => wasm.__wbg_markkit_free(ptr, 1)); -const MarkkitToolFinalization = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(ptr => wasm.__wbg_markkittool_free(ptr, 1)); - -function addHeapObject(obj) { - if (heap_next === heap.length) heap.push(heap.length + 1); - const idx = heap_next; - heap_next = heap[idx]; - - heap[idx] = obj; - return idx; -} - -const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(state => wasm.__wbindgen_export4(state.a, state.b)); - -function debugString(val) { - // primitive types - const type = typeof val; - if (type == 'number' || type == 'boolean' || val == null) { - return `${val}`; - } - if (type == 'string') { - return `"${val}"`; - } - if (type == 'symbol') { - const description = val.description; - if (description == null) { - return 'Symbol'; - } else { - return `Symbol(${description})`; - } - } - if (type == 'function') { - const name = val.name; - if (typeof name == 'string' && name.length > 0) { - return `Function(${name})`; - } else { - return 'Function'; - } - } - // objects - if (Array.isArray(val)) { - const length = val.length; - let debug = '['; - if (length > 0) { - debug += debugString(val[0]); - } - for(let i = 1; i < length; i++) { - debug += ', ' + debugString(val[i]); - } - debug += ']'; - return debug; - } - // Test for built-in - const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); - let className; - if (builtInMatches && builtInMatches.length > 1) { - className = builtInMatches[1]; - } else { - // Failed to match the standard '[object ClassName]' - return toString.call(val); - } - if (className == 'Object') { - // we're a user defined class or Object - // JSON.stringify avoids problems with cycles, and is generally much - // easier than looping through ownProperties of `val`. - try { - return 'Object(' + JSON.stringify(val) + ')'; - } catch (_) { - return 'Object'; - } - } - // errors - if (val instanceof Error) { - return `${val.name}: ${val.message}\n${val.stack}`; - } - // TODO we could test for more things here, like `Set`s and `Map`s. - return className; -} - -function dropObject(idx) { - if (idx < 1028) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function getArrayU8FromWasm0(ptr, len) { - ptr = ptr >>> 0; - return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); -} - -let cachedDataViewMemory0 = null; -function getDataViewMemory0() { - if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { - cachedDataViewMemory0 = new DataView(wasm.memory.buffer); - } - return cachedDataViewMemory0; -} - -function getStringFromWasm0(ptr, len) { - return decodeText(ptr >>> 0, len); -} - -let cachedUint8ArrayMemory0 = null; -function getUint8ArrayMemory0() { - if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { - cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); - } - return cachedUint8ArrayMemory0; -} - -function getObject(idx) { return heap[idx]; } - -function handleError(f, args) { - try { - return f.apply(this, args); - } catch (e) { - wasm.__wbindgen_export3(addHeapObject(e)); - } -} - -let heap = new Array(1024).fill(undefined); -heap.push(undefined, null, true, false); - -let heap_next = heap.length; - -function isLikeNone(x) { - return x === undefined || x === null; -} - -function makeMutClosure(arg0, arg1, f) { - const state = { a: arg0, b: arg1, cnt: 1 }; - const real = (...args) => { - - // First up with a closure we increment the internal reference - // count. This ensures that the Rust closure environment won't - // be deallocated while we're invoking it. - state.cnt++; - const a = state.a; - state.a = 0; - try { - return f(a, state.b, ...args); - } finally { - state.a = a; - real._wbg_cb_unref(); - } - }; - real._wbg_cb_unref = () => { - if (--state.cnt === 0) { - wasm.__wbindgen_export4(state.a, state.b); - state.a = 0; - CLOSURE_DTORS.unregister(state); - } - }; - CLOSURE_DTORS.register(real, state, state); - return real; -} - -function passArray8ToWasm0(arg, malloc) { - const ptr = malloc(arg.length * 1, 1) >>> 0; - getUint8ArrayMemory0().set(arg, ptr / 1); - WASM_VECTOR_LEN = arg.length; - return ptr; -} - -function passStringToWasm0(arg, malloc, realloc) { - if (realloc === undefined) { - const buf = cachedTextEncoder.encode(arg); - const ptr = malloc(buf.length, 1) >>> 0; - getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); - WASM_VECTOR_LEN = buf.length; - return ptr; - } - - let len = arg.length; - let ptr = malloc(len, 1) >>> 0; - - const mem = getUint8ArrayMemory0(); - - let offset = 0; - - for (; offset < len; offset++) { - const code = arg.charCodeAt(offset); - if (code > 0x7F) break; - mem[ptr + offset] = code; - } - if (offset !== len) { - if (offset !== 0) { - arg = arg.slice(offset); - } - ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; - const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); - const ret = cachedTextEncoder.encodeInto(arg, view); - - offset += ret.written; - ptr = realloc(ptr, len, offset, 1) >>> 0; - } - - WASM_VECTOR_LEN = offset; - return ptr; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - -let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); -cachedTextDecoder.decode(); -const MAX_SAFARI_DECODE_BYTES = 2146435072; -let numBytesDecoded = 0; -function decodeText(ptr, len) { - numBytesDecoded += len; - if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { - cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); - cachedTextDecoder.decode(); - numBytesDecoded = len; - } - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); -} - -const cachedTextEncoder = new TextEncoder(); - -if (!('encodeInto' in cachedTextEncoder)) { - cachedTextEncoder.encodeInto = function (arg, view) { - const buf = cachedTextEncoder.encode(arg); - view.set(buf); - return { - read: arg.length, - written: buf.length - }; - }; -} - -let WASM_VECTOR_LEN = 0; - -let wasmModule, wasmInstance, wasm; -function __wbg_finalize_init(instance, module) { - wasmInstance = instance; - wasm = instance.exports; - wasmModule = module; - cachedDataViewMemory0 = null; - cachedUint8ArrayMemory0 = null; - return wasm; -} - -async function __wbg_load(module, imports) { - if (typeof Response === 'function' && module instanceof Response) { - if (!module.ok) { - throw new Error(`failed to fetch Wasm: ${module.status} ${module.statusText} fetching '${module.url}'`); - } - - if (typeof WebAssembly.instantiateStreaming === 'function') { - try { - return await WebAssembly.instantiateStreaming(module, imports); - } catch (e) { - const validResponse = expectedResponseType(module.type); - - if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { - console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); - - } else { throw e; } - } - } - - const bytes = await module.arrayBuffer(); - return await WebAssembly.instantiate(bytes, imports); - } else { - const instance = await WebAssembly.instantiate(module, imports); - - if (instance instanceof WebAssembly.Instance) { - return { instance, module }; - } else { - return instance; - } - } - - function expectedResponseType(type) { - switch (type) { - case 'basic': case 'cors': case 'default': return true; - } - return false; - } -} - -function initSync(module) { - if (wasm !== undefined) return wasm; - - - if (module !== undefined) { - if (Object.getPrototypeOf(module) === Object.prototype) { - ({module} = module) - } else { - console.warn('using deprecated parameters for `initSync()`; pass a single object instead') - } - } - - const imports = __wbg_get_imports(); - if (!(module instanceof WebAssembly.Module)) { - module = new WebAssembly.Module(module); - } - const instance = new WebAssembly.Instance(module, imports); - return __wbg_finalize_init(instance, module); -} - -async function __wbg_init(module_or_path) { - if (wasm !== undefined) return wasm; - - - if (module_or_path !== undefined) { - if (Object.getPrototypeOf(module_or_path) === Object.prototype) { - ({module_or_path} = module_or_path) - } else { - console.warn('using deprecated parameters for the initialization function; pass a single object instead') - } - } - - if (module_or_path === undefined) { - module_or_path = new URL('markkit_wasm_bg.wasm', import.meta.url); - } - const imports = __wbg_get_imports(); - - if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { - module_or_path = fetch(module_or_path); - } - - const { instance, module } = await __wbg_load(await module_or_path, imports); - - return __wbg_finalize_init(instance, module); -} - -export { initSync, __wbg_init as default }; diff --git a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts deleted file mode 100644 index 7e84982..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -export const memory: WebAssembly.Memory; -export const __wbg_markkit_free: (a: number, b: number) => void; -export const __wbg_markkittool_free: (a: number, b: number) => void; -export const markkit_extractBytes: (a: number, b: number, c: number, d: number, e: number, f: number) => void; -export const markkit_extractDocument: (a: number, b: number, c: number, d: number, e: number) => void; -export const markkit_extractHtml: (a: number, b: number, c: number, d: number) => void; -export const markkit_new: (a: number, b: number) => void; -export const markkittool_description: (a: number, b: number) => void; -export const markkittool_displayName: (a: number, b: number) => void; -export const markkittool_execute: (a: number, b: number) => number; -export const markkittool_help: (a: number, b: number) => void; -export const markkittool_inputSchema: (a: number, b: number) => void; -export const markkittool_locale: (a: number, b: number) => void; -export const markkittool_name: (a: number, b: number) => void; -export const markkittool_new: (a: number, b: number) => void; -export const markkittool_outputSchema: (a: number, b: number) => void; -export const markkittool_shortDescription: (a: number, b: number) => void; -export const markkittool_systemPrompt: (a: number, b: number) => void; -export const markkittool_toolDefinition: (a: number, b: number) => void; -export const markkittool_version: (a: number, b: number) => void; -export const version: (a: number) => void; -export const __wasm_bindgen_func_elem_2058: (a: number, b: number, c: number, d: number) => void; -export const __wasm_bindgen_func_elem_2060: (a: number, b: number, c: number, d: number) => void; -export const __wbindgen_export: (a: number, b: number) => number; -export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; -export const __wbindgen_export3: (a: number) => void; -export const __wbindgen_export4: (a: number, b: number) => void; -export const __wbindgen_add_to_stack_pointer: (a: number) => number; -export const __wbindgen_export5: (a: number, b: number, c: number) => void; diff --git a/crates/markkit-wasm/pkg/package.json b/crates/markkit-wasm/pkg/package.json deleted file mode 100644 index ddf0f60..0000000 --- a/crates/markkit-wasm/pkg/package.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "@everruns/markkit-wasm", - "version": "0.1.1", - "description": "Extract clean Markdown from HTML, DOCX and spreadsheets in the browser and edge runtimes (WebAssembly). Includes an LLM agent tool.", - "type": "module", - "main": "index.js", - "module": "index.js", - "types": "index.d.ts", - "exports": { - ".": { - "types": "./index.d.ts", - "default": "./index.js" - }, - "./markkit_wasm_bg.wasm": "./markkit_wasm_bg.wasm" - }, - "sideEffects": false, - "files": [ - "index.js", - "index.d.ts", - "markkit_wasm.js", - "markkit_wasm.d.ts", - "markkit_wasm_bg.wasm", - "markkit_wasm_bg.wasm.d.ts", - "README.md" - ], - "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/everruns/markkit" - }, - "keywords": [ - "markdown", - "html", - "docx", - "xlsx", - "wasm", - "webassembly", - "browser", - "ai", - "agent", - "tool" - ], - "publishConfig": { - "access": "public" - } -}