diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72c1b1f..f4e9f4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,9 +90,40 @@ jobs: done cargo run -q -p markkit --features tool --example agent_tool + wasm: + # Browser package @everruns/markkit-wasm: build the wasm32-unknown-unknown + # bundle and run its headless Node tests. See knowledge/runtimes/wasm-package.md. + name: WASM web package + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: "24.22.1" + - name: Check core for wasm32 (no warnings) + run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,tool --target wasm32-unknown-unknown + - name: Install wasm-bindgen-cli (matches Cargo.lock) + run: bash scripts/install-wasm-bindgen.sh + - name: Build package + run: bash crates/markkit-wasm/scripts/build.sh release + - name: Integration tests (headless Node) + run: node --test crates/markkit-wasm/__test__/*.test.mjs + - name: Verify npm archive contents + run: | + cd crates/markkit-wasm/pkg + npm pack --dry-run --json --ignore-scripts > "$RUNNER_TEMP/wasm-pack.json" + node -e 'const files=new Set(require(process.argv[1])[0].files.map(f=>f.path)); for (const f of ["index.js","index.d.ts","markkit_wasm.js","markkit_wasm_bg.wasm","README.md"]) if(!files.has(f)) throw Error("archive lacks "+f)' "$RUNNER_TEMP/wasm-pack.json" + check: name: Check - needs: [lint, audit, test] + needs: [lint, audit, test, wasm] runs-on: ubuntu-latest if: always() steps: @@ -100,7 +131,8 @@ jobs: run: | if [[ "${{ needs.lint.result }}" != "success" ]] || \ [[ "${{ needs.audit.result }}" != "success" ]] || \ - [[ "${{ needs.test.result }}" != "success" ]]; then + [[ "${{ needs.test.result }}" != "success" ]] || \ + [[ "${{ needs.wasm.result }}" != "success" ]]; then echo "One or more jobs failed" exit 1 fi diff --git a/.github/workflows/publish-wasm.yml b/.github/workflows/publish-wasm.yml new file mode 100644 index 0000000..7d2d95e --- /dev/null +++ b/.github/workflows/publish-wasm.yml @@ -0,0 +1,77 @@ +# Publish @everruns/markkit-wasm (browser / edge WebAssembly package) to npm. +# +# Run from the verified release tag, like publish-js.yml. Defaults to a dry run. +# Publishes with npm trusted publishing (GitHub OIDC), no stored token: +# npmjs.com @everruns/markkit-wasm > Settings > Trusted publisher: +# GitHub Actions, everruns/markkit, workflow publish-wasm.yml, no environment. +# npm >= 11.5.1 is required for the OIDC exchange (provenance is automatic). + +name: Publish WASM + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run" + required: false + default: "true" + +permissions: + contents: read + +jobs: + publish: + name: Publish npm package (wasm) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + id-token: write # npm trusted publishing + provenance + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Verify publish source is reachable from main + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$(git rev-parse HEAD)" origin/main + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: actions/setup-node@v4 + with: + node-version: "24.22.1" + registry-url: "https://registry.npmjs.org" + - name: Require npm with trusted publishing (>= 11.5.1) + run: | + npm_ok() { + node -e 'const [a,b,c]=process.argv[1].split(".").map(Number);process.exit(a>11||(a==11&&(b>5||(b==5&&c>=1)))?0:1)' "$(npm --version)" + } + npm_ok || npm install -g npm@11.6.2 + npm_ok || { echo "::error::npm $(npm --version) lacks trusted publishing"; exit 1; } + - name: Install wasm-bindgen-cli (matches Cargo.lock) + run: bash scripts/install-wasm-bindgen.sh + - name: Build package + run: bash crates/markkit-wasm/scripts/build.sh release + - name: Verify build preserved the lockfile + run: git diff --exit-code -- Cargo.lock + - name: Integration tests (headless Node) + run: node --test crates/markkit-wasm/__test__/*.test.mjs + - name: Verify tag matches package version + run: | + PKG_VERSION=$(node -p "require('./crates/markkit-wasm/pkg/package.json').version") + if [[ "$GITHUB_REF" =~ ^refs/tags/v ]] && [ "${GITHUB_REF#refs/tags/v}" != "$PKG_VERSION" ]; then + echo "::error::tag ${GITHUB_REF#refs/tags/} does not match package version $PKG_VERSION" + exit 1 + fi + - name: Publish + working-directory: crates/markkit-wasm/pkg + env: + DRY_RUN: ${{ inputs.dry_run }} + run: | + if [ "$DRY_RUN" != "false" ]; then + npm publish --dry-run --ignore-scripts --access public + else + npm publish --ignore-scripts --access public + fi diff --git a/AGENTS.md b/AGENTS.md index 518f8d9..8c96d74 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | security/security-model | One-page threat/invariant summary | | security/threat-model | Threats T1-T6, mitigations, test map, resource limits, invariants S1-S12 | | runtimes/python-package | PyO3 `markkit` module, maturin, PyPI | +| runtimes/wasm-package | wasm-bindgen `@everruns/markkit-wasm`, browser/edge, build + tests | | runtimes/node-package | NAPI-RS `@everruns/markkit`, npm, binding parity table | | operations/testing | Test layout, categories, CI jobs | | operations/documentation | Doc trees and what each must cover | @@ -44,7 +45,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn - User docs: `README.md`, `docs/` - API reference: rustdoc in `crates/markkit/src/lib.rs` and modules -- Binding READMEs (`crates/markkit-python/README.md`, `crates/markkit-js/README.md`) ship to PyPI/npm, keep them in sync with exports +- Binding READMEs (`crates/markkit-python/README.md`, `crates/markkit-js/README.md`, `crates/markkit-wasm/README.md`) ship to PyPI/npm, keep them in sync with exports - Durable engineering knowledge: `knowledge/`, never `docs/` ### Shipping @@ -86,6 +87,7 @@ just pre-pr # Pre-PR checks | `markkit-eval` | LLM evaluation harness (stub) | | `markkit-python` | Python bindings (PyO3), module `markkit` | | `markkit-js` | Node.js bindings (NAPI-RS), package `@everruns/markkit` | +| `markkit-wasm` | Browser/edge WebAssembly (wasm-bindgen), package `@everruns/markkit-wasm` | ### Rust diff --git a/CHANGELOG.md b/CHANGELOG.md index b68f4fd..85fa598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). toolkit-library contract (builder, OpenAI tool definition, input/output schemas, `tower::Service`, single-use `ToolExecution`, `en-US`/`uk-UA` text). One call converts a `url`, `html` or base64 document to Markdown, with output capped by `max_output_chars`. Example: `examples/agent_tool.rs`. +- `@everruns/markkit-wasm`: browser / edge WebAssembly package (`crates/markkit-wasm`), single-threaded, + no special headers. `Markkit` class (HTML, DOCX, XLSX/XLS/ODS with validated camelCase options) and + `MarkkitTool`. Built by `crates/markkit-wasm/scripts/build.sh`, published by `publish-wasm.yml`. - `Markkit::extract_url_document`: `extract_url` with format, title, warnings and stats. ### Security diff --git a/Cargo.lock b/Cargo.lock index 2c348ce..1383fef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1252,6 +1252,19 @@ dependencies = [ "pyo3-async-runtimes", ] +[[package]] +name = "markkit-wasm" +version = "0.1.1" +dependencies = [ + "js-sys", + "markkit", + "serde", + "serde-wasm-bindgen", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "markup5ever" version = "0.40.0" @@ -2113,6 +2126,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.229" diff --git a/Cargo.toml b/Cargo.toml index de8f51b..aae37d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/markkit-eval", "crates/markkit-python", "crates/markkit-js", + "crates/markkit-wasm", ] [workspace.package] @@ -76,3 +77,8 @@ lto = "thin" codegen-units = 1 panic = "abort" strip = true + +# Browser bundle (crates/markkit-wasm/scripts/build.sh): size over speed. +[profile.wasm-release] +inherits = "release" +opt-level = "z" diff --git a/README.md b/README.md index 3fdd27e..49447c3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Extract clean markdown from any format. Written in Rust. Markkit converts HTML, Word documents and spreadsheets into well-structured markdown, built for feeding untrusted documents to LLMs and agents. It ships as -a Rust library, a CLI, and native bindings for Python and Node.js. +a Rust library, a CLI, native bindings for Python and Node.js, and a WebAssembly +package for browsers and edge runtimes. ## Features @@ -101,6 +102,15 @@ const fromHtml = await extractHtml('

Hello

'); const fromUrl = await extractUrl('https://example.com'); ``` +### Browser / edge (WebAssembly) + +```javascript +import { initMarkkit, Markkit } from '@everruns/markkit-wasm'; + +await initMarkkit(); +const md = new Markkit().extractHtml('

Hello

'); +``` + ## Installation ### Rust @@ -132,6 +142,16 @@ npm install @everruns/markkit Native binary: Linux x64 (glibc). +### Browser / edge (WebAssembly) + +```bash +npm install @everruns/markkit-wasm +``` + +Any JS runtime (browsers, Cloudflare Workers, Deno, Bun, Node); no special headers. +HTML, DOCX and spreadsheets; no URL fetching. See +[`crates/markkit-wasm/README.md`](crates/markkit-wasm/README.md). + ## Configuration ```rust diff --git a/crates/markkit-wasm/.gitignore b/crates/markkit-wasm/.gitignore new file mode 100644 index 0000000..c54d13a --- /dev/null +++ b/crates/markkit-wasm/.gitignore @@ -0,0 +1,4 @@ +# wasm-bindgen build output (scripts/build.sh) +/pkg/ +node_modules/ +*.tgz diff --git a/crates/markkit-wasm/Cargo.toml b/crates/markkit-wasm/Cargo.toml new file mode 100644 index 0000000..7e00540 --- /dev/null +++ b/crates/markkit-wasm/Cargo.toml @@ -0,0 +1,38 @@ +# Markkit browser bindings via wasm-bindgen. +# +# Decision: slim, single-threaded wasm32-unknown-unknown build, separate from the +# napi `markkit-js` crate (same split as bashkit-wasm). No threads, no WASI, no +# SharedArrayBuffer and no COOP/COEP headers, so it runs in browsers, edge +# workers (Cloudflare, Vercel, Deno Deploy), Node, Deno and Bun. +# +# Feature surface: html, docx, spreadsheet, tool. `http_client` is excluded: +# reqwest needs sockets; JS hosts fetch themselves and pass bytes in, which also +# keeps SSRF policy with the host. See knowledge/runtimes/wasm-package.md. + +[package] +name = "markkit-wasm" +description = "Browser (WebAssembly) bindings for Markkit markdown extraction" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] +doc = false + +[dependencies] +markkit = { path = "../markkit", default-features = false, features = [ + "html", + "docx", + "spreadsheet", + "tool", +] } +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +js-sys = "0.3" +serde = { workspace = true } +serde_json = { workspace = true } +serde-wasm-bindgen = "0.6" diff --git a/crates/markkit-wasm/pkg/README.md b/crates/markkit-wasm/README.md similarity index 100% rename from crates/markkit-wasm/pkg/README.md rename to crates/markkit-wasm/README.md diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs new file mode 100644 index 0000000..0f73332 --- /dev/null +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -0,0 +1,139 @@ +// Headless Node tests against the built pkg/ (run scripts/build.sh first). +import { test, before } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { docx, zip } from "./zip.mjs"; +import { initMarkkit, Markkit, MarkkitTool, version } from "../pkg/index.js"; + +const pkgJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url))); + +before(async () => { + await initMarkkit(); + await initMarkkit(); // idempotent +}); + +test("version matches package.json", () => { + assert.equal(version(), pkgJson.version); +}); + +test("extractHtml with defaults", () => { + const mk = new Markkit(); + assert.equal(mk.extractHtml("

Hello

World

"), "# Hello\n\nWorld\n"); +}); + +test("options are applied", () => { + const html = '

Read this

'; + const mk = new Markkit({ preserveLinks: false, includeSelectors: ["article"], excludeSelectors: [".ad"] }); + assert.equal(mk.extractHtml(html), "Read this\n"); +}); + +test("unknown option is rejected", () => { + assert.throws(() => new Markkit({ preserveLink: false }), /invalid options/); + assert.throws(() => new Markkit({ maxInputSize: "big" }), /invalid options/); + assert.throws(() => new Markkit().extractDocument(new Uint8Array(), { filename: "a.docx" }), /invalid options/); +}); + +test("input size limit", () => { + const mk = new Markkit({ maxInputSize: 4 }); + assert.throws(() => mk.extractHtml("

too long

"), /Resource limit/); +}); + +test("extractDocument returns metadata", () => { + const mk = new Markkit(); + const out = mk.extractDocument(new TextEncoder().encode("

bytes

"), { fileName: "a.html" }); + assert.equal(out.markdown, "bytes\n"); + assert.equal(out.format, "html"); + assert.deepEqual(out.warnings, []); + assert.equal(out.title, null); +}); + +test("extractDocument exposes page metadata in camelCase", () => { + const html = 'T

b

'; + const out = new Markkit().extractDocument(new TextEncoder().encode(html), { contentType: "text/html" }); + assert.equal(out.title, "T"); + assert.equal(out.metadata.siteName, "Site"); + assert.equal(out.metadata.canonicalUrl, "https://x.test/a"); + assert.equal(out.metadata.language, "en"); + assert.equal(out.metadata.site_name, undefined); +}); + +test("extractBytes and unsupported formats", () => { + const mk = new Markkit(); + assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); + assert.throws(() => mk.extractDocument(new TextEncoder().encode("%PDF-1.7"), { fileName: "a.pdf" }), /Unsupported/); +}); + +test("tool metadata and definition", () => { + const tool = new MarkkitTool(); + assert.equal(tool.name, "markkit"); + assert.equal(tool.locale, "en-US"); + assert.ok(tool.systemPrompt().startsWith("markkit:")); + const def = tool.toolDefinition(); + assert.equal(def.type, "function"); + assert.equal(def.function.name, "markkit"); + // No http_client in wasm: the url source is not offered. + assert.equal(def.function.parameters.properties.url, undefined); + assert.deepEqual(tool.inputSchema(), def.function.parameters); + assert.ok(tool.outputSchema().required.includes("markdown")); +}); + +test("tool rejects unknown options", () => { + assert.throws(() => new MarkkitTool({ maxOutputChar: 5 }), /invalid options/); + assert.throws(() => new MarkkitTool({ locale: 5 }), /invalid options/); + assert.throws(() => new MarkkitTool("x"), /invalid options/); +}); + +test("tool locale", () => { + const tool = new MarkkitTool({ locale: "uk-UA" }); + assert.equal(tool.name, "markkit"); + assert.equal(tool.displayName, "Витяг Markdown"); +}); + +test("tool executes html and base64", async () => { + const tool = new MarkkitTool({ maxOutputChars: 5 }); + const html = await tool.execute({ html: "

Hi

" }); + assert.equal(html.markdown, "# Hi\n"); + const b64 = Buffer.from("

abcdefgh

").toString("base64"); + const out = await tool.execute({ content_base64: b64, file_name: "a.html" }); + assert.equal(out.markdown, "abcde"); + assert.equal(out.truncated, true); +}); + +test("tool errors carry userFacing", async () => { + const tool = new MarkkitTool(); + await assert.rejects(tool.execute({}), e => e.userFacing === true && /exactly one/.test(e.message)); + await assert.rejects(tool.execute({ url: "https://example.com" }), e => e.userFacing === true && /disabled/.test(e.message)); + await assert.rejects(tool.execute({ content_base64: "!!" }), e => e.userFacing === true); +}); + +test("DOCX converts (zip + XML stack works in wasm)", () => { + const mk = new Markkit(); + const bytes = docx('ReportBody text'); + const out = mk.extractDocument(bytes, { fileName: "r.docx" }); + assert.equal(out.format, "docx"); + assert.match(out.markdown, /Report/); + assert.match(out.markdown, /Body text/); + assert.equal(out.stats.paragraphs, 2); +}); + +test("zip bomb style archive limits hold in wasm", () => { + const mk = new Markkit({ maxArchiveEntrySize: 1024 }); + const bytes = docx(`${"a".repeat(64 * 1024)}`); + assert.throws(() => mk.extractDocument(bytes, { fileName: "r.docx" }), /limit/i); +}); + +test("malformed office input throws, never traps", () => { + const mk = new Markkit(); + assert.throws(() => mk.extractDocument(zip([["junk", "x"]]), { fileName: "a.docx" })); + assert.throws(() => mk.extractDocument(new Uint8Array([0x50, 0x4b, 3, 4, 0, 0]), { fileName: "a.xlsx" })); + // Instance still usable after errors. + assert.equal(mk.extractHtml("

ok

"), "ok\n"); +}); + +test("tool converts base64 DOCX", async () => { + const tool = new MarkkitTool(); + const b64 = Buffer.from(docx("From tool")).toString("base64"); + const out = await tool.execute({ content_base64: b64, file_name: "a.docx" }); + assert.equal(out.format, "docx"); + assert.match(out.markdown, /From tool/); +}); diff --git a/crates/markkit-wasm/__test__/zip.mjs b/crates/markkit-wasm/__test__/zip.mjs new file mode 100644 index 0000000..c1e4c70 --- /dev/null +++ b/crates/markkit-wasm/__test__/zip.mjs @@ -0,0 +1,67 @@ +// Minimal ZIP writer for test fixtures (deflate via node:zlib, no deps). +import { deflateRawSync } from "node:zlib"; + +const CRC_TABLE = Array.from({ length: 256 }, (_, n) => { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + return c >>> 0; +}); + +function crc32(buf) { + let c = 0xffffffff; + for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; +} + +/** entries: [name, string | Uint8Array][] -> Uint8Array */ +export function zip(entries) { + const locals = []; + const centrals = []; + let offset = 0; + for (const [name, content] of entries) { + const data = typeof content === "string" ? Buffer.from(content) : Buffer.from(content); + const packed = deflateRawSync(data); + const nameBuf = Buffer.from(name); + const crc = crc32(data); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(8, 8); // deflate + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(packed.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(nameBuf.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(8, 10); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(packed.length, 20); + central.writeUInt32LE(data.length, 24); + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt32LE(offset, 42); + locals.push(local, nameBuf, packed); + centrals.push(central, nameBuf); + offset += local.length + nameBuf.length + packed.length; + } + const cd = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(cd.length, 12); + end.writeUInt32LE(offset, 16); + return new Uint8Array(Buffer.concat([...locals, cd, end])); +} + +const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'; + +/** Minimal DOCX with the given `` inner XML. */ +export function docx(body) { + return zip([ + ["[Content_Types].xml", ''], + ["_rels/.rels", ''], + ["word/document.xml", `${body}`], + ]); +} diff --git a/crates/markkit-wasm/pkg/index.d.ts b/crates/markkit-wasm/js/index.d.ts similarity index 90% rename from crates/markkit-wasm/pkg/index.d.ts rename to crates/markkit-wasm/js/index.d.ts index 5527a8e..0fb6808 100644 --- a/crates/markkit-wasm/pkg/index.d.ts +++ b/crates/markkit-wasm/js/index.d.ts @@ -40,12 +40,24 @@ export interface ExtractionStats { tables: number | null; } +/** Document metadata (HTML meta, Open Graph, JSON-LD); fields are null when absent. */ +export interface Metadata { + description: string | null; + author: string | null; + published: string | null; + language: string | null; + canonicalUrl: string | null; + siteName: string | null; + image: string | null; +} + export interface Extraction { markdown: string; title: string | null; format: string; warnings: string[]; stats: ExtractionStats; + metadata: Metadata; } export class Markkit { diff --git a/crates/markkit-wasm/pkg/index.js b/crates/markkit-wasm/js/index.js similarity index 100% rename from crates/markkit-wasm/pkg/index.js rename to crates/markkit-wasm/js/index.js diff --git a/crates/markkit-wasm/pkg/package.json b/crates/markkit-wasm/package.json similarity index 100% rename from crates/markkit-wasm/pkg/package.json rename to crates/markkit-wasm/package.json diff --git a/crates/markkit-wasm/pkg/markkit_wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm.d.ts deleted file mode 100644 index d265fdb..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm.d.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ - -/** - * Configured extractor. - */ -export class Markkit { - free(): void; - [Symbol.dispose](): void; - /** - * Document bytes to Markdown only. - */ - extractBytes(bytes: Uint8Array, content_type?: string | null): string; - /** - * Document bytes to `{ markdown, title, format, warnings, stats }`. - * `hint` is `{ contentType?, fileName? }`. - */ - extractDocument(bytes: Uint8Array, hint: any): any; - /** - * HTML string to Markdown. - */ - extractHtml(html: string): string; - /** - * `new Markkit(options?)`. - */ - constructor(options: any); -} - -/** - * LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. - */ -export class MarkkitTool { - free(): void; - [Symbol.dispose](): void; - /** - * Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` - * whose `userFacing` property says whether the message is safe for the model. - */ - execute(args: any): Promise; - help(): string; - inputSchema(): any; - /** - * `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. - */ - constructor(options: any); - outputSchema(): any; - systemPrompt(): string; - /** - * OpenAI-compatible function tool definition. - */ - toolDefinition(): any; - readonly description: string; - readonly displayName: string; - readonly locale: string; - readonly name: string; - readonly shortDescription: string; - readonly version: string; -} - -/** - * Library version. - */ -export function version(): string; - -export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; - -export interface InitOutput { - readonly memory: WebAssembly.Memory; - readonly __wbg_markkit_free: (a: number, b: number) => void; - readonly __wbg_markkittool_free: (a: number, b: number) => void; - readonly markkit_extractBytes: (a: number, b: number, c: number, d: number, e: number, f: number) => void; - readonly markkit_extractDocument: (a: number, b: number, c: number, d: number, e: number) => void; - readonly markkit_extractHtml: (a: number, b: number, c: number, d: number) => void; - readonly markkit_new: (a: number, b: number) => void; - readonly markkittool_description: (a: number, b: number) => void; - readonly markkittool_displayName: (a: number, b: number) => void; - readonly markkittool_execute: (a: number, b: number) => number; - readonly markkittool_help: (a: number, b: number) => void; - readonly markkittool_inputSchema: (a: number, b: number) => void; - readonly markkittool_locale: (a: number, b: number) => void; - readonly markkittool_name: (a: number, b: number) => void; - readonly markkittool_new: (a: number, b: number) => void; - readonly markkittool_outputSchema: (a: number, b: number) => void; - readonly markkittool_shortDescription: (a: number, b: number) => void; - readonly markkittool_systemPrompt: (a: number, b: number) => void; - readonly markkittool_toolDefinition: (a: number, b: number) => void; - readonly markkittool_version: (a: number, b: number) => void; - readonly version: (a: number) => void; - readonly __wasm_bindgen_func_elem_1938: (a: number, b: number, c: number, d: number) => void; - readonly __wasm_bindgen_func_elem_1940: (a: number, b: number, c: number, d: number) => void; - readonly __wbindgen_export: (a: number, b: number) => number; - readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; - readonly __wbindgen_export3: (a: number) => void; - readonly __wbindgen_export4: (a: number, b: number) => void; - readonly __wbindgen_add_to_stack_pointer: (a: number) => number; - readonly __wbindgen_export5: (a: number, b: number, c: number) => void; -} - -export type SyncInitInput = BufferSource | WebAssembly.Module; - -/** - * Instantiates the given `module`, which can either be bytes or - * a precompiled `WebAssembly.Module`. - * - * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated. - * - * @returns {InitOutput} - */ -export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput; - -/** - * If `module_or_path` is {RequestInfo} or {URL}, makes a request and - * for everything else, calls `WebAssembly.instantiate` directly. - * - * @param {{ module_or_path: InitInput | Promise }} module_or_path - Passing `InitInput` directly is deprecated. - * - * @returns {Promise} - */ -export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise } | InitInput | Promise): Promise; diff --git a/crates/markkit-wasm/pkg/markkit_wasm.js b/crates/markkit-wasm/pkg/markkit_wasm.js deleted file mode 100644 index 43b7b33..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm.js +++ /dev/null @@ -1,1064 +0,0 @@ -/* @ts-self-types="./markkit_wasm.d.ts" */ - -/** - * Configured extractor. - */ -export class Markkit { - __destroy_into_raw() { - const ptr = this.__wbg_ptr; - this.__wbg_ptr = 0; - MarkkitFinalization.unregister(this); - return ptr; - } - free() { - const ptr = this.__destroy_into_raw(); - wasm.__wbg_markkit_free(ptr, 0); - } - /** - * Document bytes to Markdown only. - * @param {Uint8Array} bytes - * @param {string | null} [content_type] - * @returns {string} - */ - extractBytes(bytes, content_type) { - let deferred4_0; - let deferred4_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export); - const len0 = WASM_VECTOR_LEN; - var ptr1 = isLikeNone(content_type) ? 0 : passStringToWasm0(content_type, wasm.__wbindgen_export, wasm.__wbindgen_export2); - var len1 = WASM_VECTOR_LEN; - wasm.markkit_extractBytes(retptr, this.__wbg_ptr, ptr0, len0, ptr1, len1); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); - var ptr3 = r0; - var len3 = r1; - if (r3) { - ptr3 = 0; len3 = 0; - throw takeObject(r2); - } - deferred4_0 = ptr3; - deferred4_1 = len3; - return getStringFromWasm0(ptr3, len3); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred4_0, deferred4_1, 1); - } - } - /** - * Document bytes to `{ markdown, title, format, warnings, stats }`. - * `hint` is `{ contentType?, fileName? }`. - * @param {Uint8Array} bytes - * @param {any} hint - * @returns {any} - */ - extractDocument(bytes, hint) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export); - const len0 = WASM_VECTOR_LEN; - wasm.markkit_extractDocument(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(hint)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * HTML string to Markdown. - * @param {string} html - * @returns {string} - */ - extractHtml(html) { - let deferred3_0; - let deferred3_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len0 = WASM_VECTOR_LEN; - wasm.markkit_extractHtml(retptr, this.__wbg_ptr, ptr0, len0); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); - var ptr2 = r0; - var len2 = r1; - if (r3) { - ptr2 = 0; len2 = 0; - throw takeObject(r2); - } - deferred3_0 = ptr2; - deferred3_1 = len2; - return getStringFromWasm0(ptr2, len2); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred3_0, deferred3_1, 1); - } - } - /** - * `new Markkit(options?)`. - * @param {any} options - */ - constructor(options) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkit_new(retptr, addHeapObject(options)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - this.__wbg_ptr = r0; - MarkkitFinalization.register(this, this.__wbg_ptr, this); - return this; - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } -} -if (Symbol.dispose) Markkit.prototype[Symbol.dispose] = Markkit.prototype.free; - -/** - * LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. - */ -export class MarkkitTool { - __destroy_into_raw() { - const ptr = this.__wbg_ptr; - this.__wbg_ptr = 0; - MarkkitToolFinalization.unregister(this); - return ptr; - } - free() { - const ptr = this.__destroy_into_raw(); - wasm.__wbg_markkittool_free(ptr, 0); - } - /** - * @returns {string} - */ - get description() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_description(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - get displayName() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_displayName(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` - * whose `userFacing` property says whether the message is safe for the model. - * @param {any} args - * @returns {Promise} - */ - execute(args) { - const ret = wasm.markkittool_execute(this.__wbg_ptr, addHeapObject(args)); - return takeObject(ret); - } - /** - * @returns {string} - */ - help() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_help(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {any} - */ - inputSchema() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_inputSchema(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get locale() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_locale(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - get name() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_name(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. - * @param {any} options - */ - constructor(options) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_new(retptr, addHeapObject(options)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - this.__wbg_ptr = r0; - MarkkitToolFinalization.register(this, this.__wbg_ptr, this); - return this; - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {any} - */ - outputSchema() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_outputSchema(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get shortDescription() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_shortDescription(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * @returns {string} - */ - systemPrompt() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_systemPrompt(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } - /** - * OpenAI-compatible function tool definition. - * @returns {any} - */ - toolDefinition() { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_toolDefinition(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); - if (r2) { - throw takeObject(r1); - } - return takeObject(r0); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } - } - /** - * @returns {string} - */ - get version() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.markkittool_version(retptr, this.__wbg_ptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } - } -} -if (Symbol.dispose) MarkkitTool.prototype[Symbol.dispose] = MarkkitTool.prototype.free; - -/** - * Library version. - * @returns {string} - */ -export function version() { - let deferred1_0; - let deferred1_1; - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.version(retptr); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - deferred1_0 = r0; - deferred1_1 = r1; - return getStringFromWasm0(r0, r1); - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export5(deferred1_0, deferred1_1, 1); - } -} -function __wbg_get_imports() { - const import0 = { - __proto__: null, - __wbg_Error_30c8987f7c2ed4e2: function(arg0, arg1) { - const ret = Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_String_8564e559799eccda: function(arg0, arg1) { - const ret = String(getObject(arg1)); - const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_bigint_get_as_i64_a2383202b9353e4c: function(arg0, arg1) { - const v = getObject(arg1); - const ret = typeof(v) === 'bigint' ? v : undefined; - getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); - }, - __wbg___wbindgen_boolean_get_5b446f51afd21013: function(arg0) { - const v = getObject(arg0); - const ret = typeof(v) === 'boolean' ? v : undefined; - return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; - }, - __wbg___wbindgen_debug_string_4687d8d8c2017d52: function(arg0, arg1) { - const ret = debugString(getObject(arg1)); - const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - const len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_in_92f62ee1427d9e49: function(arg0, arg1) { - const ret = getObject(arg0) in getObject(arg1); - return ret; - }, - __wbg___wbindgen_is_bigint_b123553bed3bb382: function(arg0) { - const ret = typeof(getObject(arg0)) === 'bigint'; - return ret; - }, - __wbg___wbindgen_is_function_1f9d30630b8b1d3d: function(arg0) { - const ret = typeof(getObject(arg0)) === 'function'; - return ret; - }, - __wbg___wbindgen_is_null_e343b7d08827ba72: function(arg0) { - const ret = getObject(arg0) === null; - return ret; - }, - __wbg___wbindgen_is_object_3c45d4f2dde4e749: function(arg0) { - const val = getObject(arg0); - const ret = typeof(val) === 'object' && val !== null; - return ret; - }, - __wbg___wbindgen_is_string_90b56bc79aad6f6c: function(arg0) { - const ret = typeof(getObject(arg0)) === 'string'; - return ret; - }, - __wbg___wbindgen_is_undefined_8865fb403f8fe9d8: function(arg0) { - const ret = getObject(arg0) === undefined; - return ret; - }, - __wbg___wbindgen_jsval_eq_02babf21faa37971: function(arg0, arg1) { - const ret = getObject(arg0) === getObject(arg1); - return ret; - }, - __wbg___wbindgen_jsval_loose_eq_677f21e468d6b461: function(arg0, arg1) { - const ret = getObject(arg0) == getObject(arg1); - return ret; - }, - __wbg___wbindgen_number_get_2e0e7dee9f701a71: function(arg0, arg1) { - const obj = getObject(arg1); - const ret = typeof(obj) === 'number' ? obj : undefined; - getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); - }, - __wbg___wbindgen_string_get_0380ccaa2f57f0d9: function(arg0, arg1) { - const obj = getObject(arg1); - const ret = typeof(obj) === 'string' ? obj : undefined; - var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); - var len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); - getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); - }, - __wbg___wbindgen_throw_41e9ee4f547fc59a: function(arg0, arg1) { - throw new Error(getStringFromWasm0(arg0, arg1)); - }, - __wbg__wbg_cb_unref_dcc1a90847f04c41: function(arg0) { - getObject(arg0)._wbg_cb_unref(); - }, - __wbg_call_187d372bd5fdd4aa: function() { return handleError(function (arg0, arg1, arg2) { - const ret = getObject(arg0).call(getObject(arg1), getObject(arg2)); - return addHeapObject(ret); - }, arguments); }, - __wbg_call_6137034ef55c9d0f: function() { return handleError(function (arg0, arg1) { - const ret = getObject(arg0).call(getObject(arg1)); - return addHeapObject(ret); - }, arguments); }, - __wbg_done_b41a1d26cdb37fb6: function(arg0) { - const ret = getObject(arg0).done; - return ret; - }, - __wbg_entries_fb6397112b1de25f: function(arg0) { - const ret = Object.entries(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_get_658f6698067d9515: function() { return handleError(function (arg0, arg1) { - const ret = Reflect.get(getObject(arg0), getObject(arg1)); - return addHeapObject(ret); - }, arguments); }, - __wbg_get_6c896e0571ddae51: function(arg0, arg1) { - const ret = getObject(arg0)[arg1 >>> 0]; - return addHeapObject(ret); - }, - __wbg_get_unchecked_288889d017702237: function(arg0, arg1) { - const ret = getObject(arg0)[arg1 >>> 0]; - return addHeapObject(ret); - }, - __wbg_instanceof_ArrayBuffer_a99f175873e5d9b8: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof ArrayBuffer; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_instanceof_Map_b2611749102d7ba3: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof Map; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_instanceof_Uint8Array_828cef2aaacafc31: function(arg0) { - let result; - try { - result = getObject(arg0) instanceof Uint8Array; - } catch (_) { - result = false; - } - const ret = result; - return ret; - }, - __wbg_isArray_e15a2ff68ffdbef2: function(arg0) { - const ret = Array.isArray(getObject(arg0)); - return ret; - }, - __wbg_isSafeInteger_717808ad6a54bd9e: function(arg0) { - const ret = Number.isSafeInteger(getObject(arg0)); - return ret; - }, - __wbg_iterator_e3c31c892080e444: function() { - const ret = Symbol.iterator; - return addHeapObject(ret); - }, - __wbg_length_7f3c00c40364105e: function(arg0) { - const ret = getObject(arg0).length; - return ret; - }, - __wbg_length_d4bdea10311bd9cf: function(arg0) { - const ret = getObject(arg0).length; - return ret; - }, - __wbg_new_1dbf7428bba60a42: function(arg0) { - const ret = new Uint8Array(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_new_28744009d011f847: function() { - const ret = new Map(); - return addHeapObject(ret); - }, - __wbg_new_343a093a3c2ffb4e: function(arg0, arg1) { - const ret = new Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_new_617a8cdb8bb1130e: function() { - const ret = new Object(); - return addHeapObject(ret); - }, - __wbg_new_ee2291f50781bf1d: function() { - const ret = new Array(); - return addHeapObject(ret); - }, - __wbg_new_typed_b01cb72a8af741a3: function(arg0, arg1) { - try { - var state0 = {a: arg0, b: arg1}; - var cb0 = (arg0, arg1) => { - const a = state0.a; - state0.a = 0; - try { - return __wasm_bindgen_func_elem_1940(a, state0.b, arg0, arg1); - } finally { - state0.a = a; - } - }; - const ret = new Promise(cb0); - return addHeapObject(ret); - } finally { - state0.a = 0; - } - }, - __wbg_next_33784799010f1bbe: function(arg0) { - const ret = getObject(arg0).next; - return addHeapObject(ret); - }, - __wbg_next_f4aac29c42af995c: function() { return handleError(function (arg0) { - const ret = getObject(arg0).next(); - return addHeapObject(ret); - }, arguments); }, - __wbg_prototypesetcall_bc27214492979395: function(arg0, arg1, arg2) { - Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), getObject(arg2)); - }, - __wbg_queueMicrotask_9833f9a49df95a49: function(arg0) { - const ret = getObject(arg0).queueMicrotask; - return addHeapObject(ret); - }, - __wbg_queueMicrotask_a72f977e97f23c5f: function(arg0) { - queueMicrotask(getObject(arg0)); - }, - __wbg_resolve_0076e10020304ede: function(arg0) { - const ret = Promise.resolve(getObject(arg0)); - return addHeapObject(ret); - }, - __wbg_set_145a351398b48c65: function() { return handleError(function (arg0, arg1, arg2) { - const ret = Reflect.set(getObject(arg0), getObject(arg1), getObject(arg2)); - return ret; - }, arguments); }, - __wbg_set_6ae97e73113c4f0b: function(arg0, arg1, arg2) { - const ret = getObject(arg0).set(getObject(arg1), getObject(arg2)); - return addHeapObject(ret); - }, - __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) { - getObject(arg0)[takeObject(arg1)] = takeObject(arg2); - }, - __wbg_set_bea140a88be9b277: function(arg0, arg1, arg2) { - getObject(arg0)[arg1 >>> 0] = takeObject(arg2); - }, - __wbg_static_accessor_GLOBAL_266715b9d96ba635: function() { - const ret = typeof global === 'undefined' ? null : global; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_GLOBAL_THIS_10fb7dc1ae063179: function() { - const ret = typeof globalThis === 'undefined' ? null : globalThis; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_SELF_0b583911f537483a: function() { - const ret = typeof self === 'undefined' ? null : self; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_static_accessor_WINDOW_d7f903d1508cbdc4: function() { - const ret = typeof window === 'undefined' ? null : window; - return isLikeNone(ret) ? 0 : addHeapObject(ret); - }, - __wbg_then_e71170d78fcf8954: function(arg0, arg1) { - const ret = getObject(arg0).then(getObject(arg1)); - return addHeapObject(ret); - }, - __wbg_value_f3c585ee8f5ba40c: function(arg0) { - const ret = getObject(arg0).value; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000001: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 146, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, __wasm_bindgen_func_elem_1938); - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000002: function(arg0) { - // Cast intrinsic for `F64 -> Externref`. - const ret = arg0; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000003: function(arg0) { - // Cast intrinsic for `I64 -> Externref`. - const ret = arg0; - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000004: function(arg0, arg1) { - // Cast intrinsic for `Ref(String) -> Externref`. - const ret = getStringFromWasm0(arg0, arg1); - return addHeapObject(ret); - }, - __wbindgen_generic_0000000000000005: function(arg0) { - // Cast intrinsic for `U64 -> Externref`. - const ret = BigInt.asUintN(64, arg0); - return addHeapObject(ret); - }, - __wbindgen_object_clone_ref: function(arg0) { - const ret = getObject(arg0); - return addHeapObject(ret); - }, - __wbindgen_object_drop_ref: function(arg0) { - takeObject(arg0); - }, - }; - return { - __proto__: null, - "./markkit_wasm_bg.js": import0, - }; -} - -function __wasm_bindgen_func_elem_1940(arg0, arg1, arg2, arg3) { - wasm.__wasm_bindgen_func_elem_1940(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3)); -} - -function __wasm_bindgen_func_elem_1938(arg0, arg1, arg2) { - try { - const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); - wasm.__wasm_bindgen_func_elem_1938(retptr, arg0, arg1, addHeapObject(arg2)); - var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); - var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); - if (r1) { - throw takeObject(r0); - } - } finally { - wasm.__wbindgen_add_to_stack_pointer(16); - } -} - -const MarkkitFinalization = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(ptr => wasm.__wbg_markkit_free(ptr, 1)); -const MarkkitToolFinalization = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(ptr => wasm.__wbg_markkittool_free(ptr, 1)); - -function addHeapObject(obj) { - if (heap_next === heap.length) heap.push(heap.length + 1); - const idx = heap_next; - heap_next = heap[idx]; - - heap[idx] = obj; - return idx; -} - -const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined') - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry(state => wasm.__wbindgen_export4(state.a, state.b)); - -function debugString(val) { - // primitive types - const type = typeof val; - if (type == 'number' || type == 'boolean' || val == null) { - return `${val}`; - } - if (type == 'string') { - return `"${val}"`; - } - if (type == 'symbol') { - const description = val.description; - if (description == null) { - return 'Symbol'; - } else { - return `Symbol(${description})`; - } - } - if (type == 'function') { - const name = val.name; - if (typeof name == 'string' && name.length > 0) { - return `Function(${name})`; - } else { - return 'Function'; - } - } - // objects - if (Array.isArray(val)) { - const length = val.length; - let debug = '['; - if (length > 0) { - debug += debugString(val[0]); - } - for(let i = 1; i < length; i++) { - debug += ', ' + debugString(val[i]); - } - debug += ']'; - return debug; - } - // Test for built-in - const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); - let className; - if (builtInMatches && builtInMatches.length > 1) { - className = builtInMatches[1]; - } else { - // Failed to match the standard '[object ClassName]' - return toString.call(val); - } - if (className == 'Object') { - // we're a user defined class or Object - // JSON.stringify avoids problems with cycles, and is generally much - // easier than looping through ownProperties of `val`. - try { - return 'Object(' + JSON.stringify(val) + ')'; - } catch (_) { - return 'Object'; - } - } - // errors - if (val instanceof Error) { - return `${val.name}: ${val.message}\n${val.stack}`; - } - // TODO we could test for more things here, like `Set`s and `Map`s. - return className; -} - -function dropObject(idx) { - if (idx < 1028) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function getArrayU8FromWasm0(ptr, len) { - ptr = ptr >>> 0; - return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); -} - -let cachedDataViewMemory0 = null; -function getDataViewMemory0() { - if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { - cachedDataViewMemory0 = new DataView(wasm.memory.buffer); - } - return cachedDataViewMemory0; -} - -function getStringFromWasm0(ptr, len) { - return decodeText(ptr >>> 0, len); -} - -let cachedUint8ArrayMemory0 = null; -function getUint8ArrayMemory0() { - if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { - cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); - } - return cachedUint8ArrayMemory0; -} - -function getObject(idx) { return heap[idx]; } - -function handleError(f, args) { - try { - return f.apply(this, args); - } catch (e) { - wasm.__wbindgen_export3(addHeapObject(e)); - } -} - -let heap = new Array(1024).fill(undefined); -heap.push(undefined, null, true, false); - -let heap_next = heap.length; - -function isLikeNone(x) { - return x === undefined || x === null; -} - -function makeMutClosure(arg0, arg1, f) { - const state = { a: arg0, b: arg1, cnt: 1 }; - const real = (...args) => { - - // First up with a closure we increment the internal reference - // count. This ensures that the Rust closure environment won't - // be deallocated while we're invoking it. - state.cnt++; - const a = state.a; - state.a = 0; - try { - return f(a, state.b, ...args); - } finally { - state.a = a; - real._wbg_cb_unref(); - } - }; - real._wbg_cb_unref = () => { - if (--state.cnt === 0) { - wasm.__wbindgen_export4(state.a, state.b); - state.a = 0; - CLOSURE_DTORS.unregister(state); - } - }; - CLOSURE_DTORS.register(real, state, state); - return real; -} - -function passArray8ToWasm0(arg, malloc) { - const ptr = malloc(arg.length * 1, 1) >>> 0; - getUint8ArrayMemory0().set(arg, ptr / 1); - WASM_VECTOR_LEN = arg.length; - return ptr; -} - -function passStringToWasm0(arg, malloc, realloc) { - if (realloc === undefined) { - const buf = cachedTextEncoder.encode(arg); - const ptr = malloc(buf.length, 1) >>> 0; - getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); - WASM_VECTOR_LEN = buf.length; - return ptr; - } - - let len = arg.length; - let ptr = malloc(len, 1) >>> 0; - - const mem = getUint8ArrayMemory0(); - - let offset = 0; - - for (; offset < len; offset++) { - const code = arg.charCodeAt(offset); - if (code > 0x7F) break; - mem[ptr + offset] = code; - } - if (offset !== len) { - if (offset !== 0) { - arg = arg.slice(offset); - } - ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; - const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); - const ret = cachedTextEncoder.encodeInto(arg, view); - - offset += ret.written; - ptr = realloc(ptr, len, offset, 1) >>> 0; - } - - WASM_VECTOR_LEN = offset; - return ptr; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - -let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); -cachedTextDecoder.decode(); -const MAX_SAFARI_DECODE_BYTES = 2146435072; -let numBytesDecoded = 0; -function decodeText(ptr, len) { - numBytesDecoded += len; - if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { - cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); - cachedTextDecoder.decode(); - numBytesDecoded = len; - } - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); -} - -const cachedTextEncoder = new TextEncoder(); - -if (!('encodeInto' in cachedTextEncoder)) { - cachedTextEncoder.encodeInto = function (arg, view) { - const buf = cachedTextEncoder.encode(arg); - view.set(buf); - return { - read: arg.length, - written: buf.length - }; - }; -} - -let WASM_VECTOR_LEN = 0; - -let wasmModule, wasmInstance, wasm; -function __wbg_finalize_init(instance, module) { - wasmInstance = instance; - wasm = instance.exports; - wasmModule = module; - cachedDataViewMemory0 = null; - cachedUint8ArrayMemory0 = null; - return wasm; -} - -async function __wbg_load(module, imports) { - if (typeof Response === 'function' && module instanceof Response) { - if (!module.ok) { - throw new Error(`failed to fetch Wasm: ${module.status} ${module.statusText} fetching '${module.url}'`); - } - - if (typeof WebAssembly.instantiateStreaming === 'function') { - try { - return await WebAssembly.instantiateStreaming(module, imports); - } catch (e) { - const validResponse = expectedResponseType(module.type); - - if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { - console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); - - } else { throw e; } - } - } - - const bytes = await module.arrayBuffer(); - return await WebAssembly.instantiate(bytes, imports); - } else { - const instance = await WebAssembly.instantiate(module, imports); - - if (instance instanceof WebAssembly.Instance) { - return { instance, module }; - } else { - return instance; - } - } - - function expectedResponseType(type) { - switch (type) { - case 'basic': case 'cors': case 'default': return true; - } - return false; - } -} - -function initSync(module) { - if (wasm !== undefined) return wasm; - - - if (module !== undefined) { - if (Object.getPrototypeOf(module) === Object.prototype) { - ({module} = module) - } else { - console.warn('using deprecated parameters for `initSync()`; pass a single object instead') - } - } - - const imports = __wbg_get_imports(); - if (!(module instanceof WebAssembly.Module)) { - module = new WebAssembly.Module(module); - } - const instance = new WebAssembly.Instance(module, imports); - return __wbg_finalize_init(instance, module); -} - -async function __wbg_init(module_or_path) { - if (wasm !== undefined) return wasm; - - - if (module_or_path !== undefined) { - if (Object.getPrototypeOf(module_or_path) === Object.prototype) { - ({module_or_path} = module_or_path) - } else { - console.warn('using deprecated parameters for the initialization function; pass a single object instead') - } - } - - if (module_or_path === undefined) { - module_or_path = new URL('markkit_wasm_bg.wasm', import.meta.url); - } - const imports = __wbg_get_imports(); - - if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { - module_or_path = fetch(module_or_path); - } - - const { instance, module } = await __wbg_load(await module_or_path, imports); - - return __wbg_finalize_init(instance, module); -} - -export { initSync, __wbg_init as default }; diff --git a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts b/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts deleted file mode 100644 index 2459379..0000000 --- a/crates/markkit-wasm/pkg/markkit_wasm_bg.wasm.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -export const memory: WebAssembly.Memory; -export const __wbg_markkit_free: (a: number, b: number) => void; -export const __wbg_markkittool_free: (a: number, b: number) => void; -export const markkit_extractBytes: (a: number, b: number, c: number, d: number, e: number, f: number) => void; -export const markkit_extractDocument: (a: number, b: number, c: number, d: number, e: number) => void; -export const markkit_extractHtml: (a: number, b: number, c: number, d: number) => void; -export const markkit_new: (a: number, b: number) => void; -export const markkittool_description: (a: number, b: number) => void; -export const markkittool_displayName: (a: number, b: number) => void; -export const markkittool_execute: (a: number, b: number) => number; -export const markkittool_help: (a: number, b: number) => void; -export const markkittool_inputSchema: (a: number, b: number) => void; -export const markkittool_locale: (a: number, b: number) => void; -export const markkittool_name: (a: number, b: number) => void; -export const markkittool_new: (a: number, b: number) => void; -export const markkittool_outputSchema: (a: number, b: number) => void; -export const markkittool_shortDescription: (a: number, b: number) => void; -export const markkittool_systemPrompt: (a: number, b: number) => void; -export const markkittool_toolDefinition: (a: number, b: number) => void; -export const markkittool_version: (a: number, b: number) => void; -export const version: (a: number) => void; -export const __wasm_bindgen_func_elem_1938: (a: number, b: number, c: number, d: number) => void; -export const __wasm_bindgen_func_elem_1940: (a: number, b: number, c: number, d: number) => void; -export const __wbindgen_export: (a: number, b: number) => number; -export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; -export const __wbindgen_export3: (a: number) => void; -export const __wbindgen_export4: (a: number, b: number) => void; -export const __wbindgen_add_to_stack_pointer: (a: number) => number; -export const __wbindgen_export5: (a: number, b: number, c: number) => void; diff --git a/crates/markkit-wasm/scripts/build.sh b/crates/markkit-wasm/scripts/build.sh new file mode 100755 index 0000000..004f6e5 --- /dev/null +++ b/crates/markkit-wasm/scripts/build.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Build @everruns/markkit-wasm into pkg/ (ES module, `wasm-bindgen --target web`). +# Needs no bundler and no cross-origin isolation headers to load. +# +# Requires: rustup target add wasm32-unknown-unknown; bash scripts/install-wasm-bindgen.sh +# Optional (smaller output): wasm-opt (binaryen) on PATH. +set -euo pipefail + +CRATE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKSPACE_ROOT="$(cd "$CRATE_DIR/../.." && pwd)" +OUT_DIR="$CRATE_DIR/pkg" +TARGET=wasm32-unknown-unknown +PROFILE="${1:-release}" +case "$PROFILE" in + release) PROFILE=wasm-release; TARGET_SUBDIR=wasm-release ;; + dev|debug) PROFILE=dev; TARGET_SUBDIR=debug ;; + *) echo "unsupported profile: $PROFILE" >&2; exit 1 ;; +esac + +cd "$WORKSPACE_ROOT" +echo "==> cargo build ($PROFILE) for $TARGET" +cargo build --locked -p markkit-wasm --target "$TARGET" --profile "$PROFILE" +WASM_IN="${CARGO_TARGET_DIR:-$WORKSPACE_ROOT/target}/$TARGET/$TARGET_SUBDIR/markkit_wasm.wasm" + +echo "==> wasm-bindgen --target web" +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" +wasm-bindgen "$WASM_IN" --target web --out-dir "$OUT_DIR" --out-name markkit_wasm + +if command -v wasm-opt >/dev/null 2>&1; then + echo "==> wasm-opt -Oz" + wasm-opt -Oz --enable-bulk-memory --enable-nontrapping-float-to-int --enable-sign-ext \ + "$OUT_DIR/markkit_wasm_bg.wasm" -o "$OUT_DIR/markkit_wasm_bg.wasm" +else + echo "==> wasm-opt not found; skipping size optimization" +fi + +cp "$CRATE_DIR/js/index.js" "$CRATE_DIR/js/index.d.ts" "$OUT_DIR/" +cp "$CRATE_DIR/package.json" "$CRATE_DIR/README.md" "$OUT_DIR/" +echo "==> built $OUT_DIR" +ls -lh "$OUT_DIR" diff --git a/crates/markkit-wasm/src/lib.rs b/crates/markkit-wasm/src/lib.rs new file mode 100644 index 0000000..c49ef35 --- /dev/null +++ b/crates/markkit-wasm/src/lib.rs @@ -0,0 +1,319 @@ +//! Browser / edge bindings for Markkit (`@everruns/markkit-wasm`). +//! +//! Decisions: +//! - Class API (`Markkit`, `MarkkitTool`) so one configured instance is reused; +//! options are a plain camelCase object validated with `deny_unknown_fields` +//! so typos fail loudly instead of silently using defaults. +//! - Bytes in, Markdown out. No URL fetching in wasm: the host fetches. +//! - Errors are thrown as `Error` with the Rust `Display` text. + +use serde::{Deserialize, Serialize}; +use wasm_bindgen::prelude::*; + +use markkit::tool::{MarkkitTool as CoreTool, Tool}; +use markkit::{DocumentHint, MarkkitBuilder}; + +/// Extraction options shared by `Markkit` and `MarkkitTool` (camelCase in JS). +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Options { + max_input_size: Option, + preserve_links: Option, + preserve_images: Option, + /// Added to the default selector set. + include_selectors: Option>, + /// Added to the default exclusions (nav, footer, header, script, style, noscript). + exclude_selectors: Option>, + max_archive_entries: Option, + max_archive_entry_size: Option, + max_archive_total_size: Option, + max_sheet_rows: Option, + max_sheet_cols: Option, + max_spreadsheet_cells: Option, + include_hidden_sheets: Option, +} + +impl Options { + fn builder(self) -> MarkkitBuilder { + let mut b = MarkkitBuilder::default(); + macro_rules! set { + ($($field:ident),*) => {$( + if let Some(v) = self.$field { b = b.$field(v); } + )*}; + } + set!( + max_input_size, + preserve_links, + preserve_images, + max_archive_entries, + max_archive_entry_size, + max_archive_total_size, + max_sheet_rows, + max_sheet_cols, + max_spreadsheet_cells, + include_hidden_sheets + ); + for s in self.include_selectors.unwrap_or_default() { + b = b.include_selector(s); + } + for s in self.exclude_selectors.unwrap_or_default() { + b = b.exclude_selector(s); + } + b + } +} + +/// Tool options: `locale`, `maxOutputChars`, plus every [`Options`] key. +/// (serde cannot combine `flatten` with `deny_unknown_fields`, so split by hand.) +#[derive(Debug, Default)] +struct ToolOptions { + locale: Option, + max_output_chars: Option, + extraction: Options, +} + +impl ToolOptions { + fn from_js(value: JsValue) -> Result { + let mut map = match from_js::(value)? { + serde_json::Value::Null => return Ok(Self::default()), + serde_json::Value::Object(map) => map, + _ => return Err(JsError::new("invalid options: expected an object")), + }; + let invalid = |e: serde_json::Error| JsError::new(&format!("invalid options: {e}")); + let locale = map + .remove("locale") + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let max_output_chars = map + .remove("maxOutputChars") + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let extraction = serde_json::from_value(serde_json::Value::Object(map)).map_err(invalid)?; + Ok(Self { + locale, + max_output_chars, + extraction, + }) + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Hint { + content_type: Option, + file_name: Option, +} + +/// JS value -> `T`, with `undefined`/`null` meaning defaults. +/// +/// Goes through `serde_json::Value`: serde-wasm-bindgen only reads the fields a +/// struct names, so `deny_unknown_fields` would never see a typo otherwise. +fn from_js Deserialize<'de> + Default>(value: JsValue) -> Result { + if value.is_undefined() || value.is_null() { + return Ok(T::default()); + } + let json: serde_json::Value = serde_wasm_bindgen::from_value(value) + .map_err(|e| JsError::new(&format!("invalid options: {e}")))?; + serde_json::from_value(json).map_err(|e| JsError::new(&format!("invalid options: {e}"))) +} + +fn to_js(value: &T) -> Result { + value + .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) + .map_err(|e| JsError::new(&e.to_string())) +} + +/// Rust serializes `snake_case`; JS callers get `camelCase` (`canonicalUrl`). +fn camel_case_keys(value: &mut serde_json::Value) { + if let serde_json::Value::Object(map) = value { + *map = std::mem::take(map) + .into_iter() + .map(|(k, v)| { + let mut out = String::with_capacity(k.len()); + let mut upper = false; + for c in k.chars() { + if c == '_' { + upper = true; + } else if upper { + out.extend(c.to_uppercase()); + upper = false; + } else { + out.push(c); + } + } + (out, v) + }) + .collect(); + } +} + +fn js_err(e: impl std::fmt::Display) -> JsError { + JsError::new(&e.to_string()) +} + +/// Library version. +#[wasm_bindgen] +pub fn version() -> String { + markkit::tool::VERSION.to_string() +} + +/// Configured extractor. +#[wasm_bindgen] +pub struct Markkit { + inner: markkit::Markkit, +} + +#[wasm_bindgen] +impl Markkit { + /// `new Markkit(options?)`. + #[wasm_bindgen(constructor)] + pub fn new(options: JsValue) -> Result { + let opts: Options = from_js(options)?; + Ok(Self { + inner: opts.builder().build(), + }) + } + + /// HTML string to Markdown. + #[wasm_bindgen(js_name = extractHtml)] + pub fn extract_html(&self, html: &str) -> Result { + self.inner.extract_html_sync(html).map_err(js_err) + } + + /// Document bytes to `{ markdown, title, format, warnings, stats }`. + /// `hint` is `{ contentType?, fileName? }`. + #[wasm_bindgen(js_name = extractDocument)] + pub fn extract_document(&self, bytes: &[u8], hint: JsValue) -> Result { + let hint: Hint = from_js(hint)?; + let extraction = self + .inner + .extract_document_sync( + bytes, + DocumentHint { + content_type: hint.content_type.as_deref(), + file_name: hint.file_name.as_deref(), + }, + ) + .map_err(js_err)?; + let mut value = serde_json::to_value(&extraction).map_err(js_err)?; + camel_case_keys(&mut value["metadata"]); + to_js(&value) + } + + /// Document bytes to Markdown only. + #[wasm_bindgen(js_name = extractBytes)] + pub fn extract_bytes( + &self, + bytes: &[u8], + content_type: Option, + ) -> Result { + self.inner + .extract_document_sync( + bytes, + DocumentHint { + content_type: content_type.as_deref(), + file_name: None, + }, + ) + .map(|e| e.markdown) + .map_err(js_err) + } +} + +/// LLM agent tool (Everruns toolkit contract). Accepts `html` or `content_base64`. +#[wasm_bindgen] +pub struct MarkkitTool { + builder: markkit::tool::MarkkitToolBuilder, + tool: CoreTool, +} + +#[wasm_bindgen] +impl MarkkitTool { + /// `new MarkkitTool({ locale?, maxOutputChars?, ...extraction options })`. + #[wasm_bindgen(constructor)] + pub fn new(options: JsValue) -> Result { + let opts = ToolOptions::from_js(options)?; + let mut builder = CoreTool::builder().config(opts.extraction.builder()); + if let Some(locale) = opts.locale { + builder = builder.locale(&locale); + } + if let Some(n) = opts.max_output_chars { + builder = builder.max_output_chars(n); + } + let tool = builder.build(); + Ok(Self { builder, tool }) + } + + #[wasm_bindgen(getter)] + pub fn name(&self) -> String { + self.tool.name().to_string() + } + + #[wasm_bindgen(getter, js_name = displayName)] + pub fn display_name(&self) -> String { + self.tool.display_name().to_string() + } + + #[wasm_bindgen(getter, js_name = shortDescription)] + pub fn short_description(&self) -> String { + self.tool.short_description().to_string() + } + + #[wasm_bindgen(getter)] + pub fn description(&self) -> String { + self.tool.description().to_string() + } + + #[wasm_bindgen(getter)] + pub fn version(&self) -> String { + self.tool.version().to_string() + } + + #[wasm_bindgen(getter)] + pub fn locale(&self) -> String { + self.tool.locale().to_string() + } + + pub fn help(&self) -> String { + self.tool.help() + } + + #[wasm_bindgen(js_name = systemPrompt)] + pub fn system_prompt(&self) -> String { + self.tool.system_prompt() + } + + #[wasm_bindgen(js_name = inputSchema)] + pub fn input_schema(&self) -> Result { + to_js(&self.tool.input_schema()) + } + + #[wasm_bindgen(js_name = outputSchema)] + pub fn output_schema(&self) -> Result { + to_js(&self.tool.output_schema()) + } + + /// OpenAI-compatible function tool definition. + #[wasm_bindgen(js_name = toolDefinition)] + pub fn tool_definition(&self) -> Result { + to_js(&self.builder.build_tool_definition()) + } + + /// Run one call. Resolves to `ToolOutput.result`; rejects with an `Error` + /// whose `userFacing` property says whether the message is safe for the model. + pub async fn execute(&self, args: JsValue) -> Result { + let args: serde_json::Value = serde_wasm_bindgen::from_value(args) + .map_err(|e| tool_error(&markkit::tool::ToolError::UserFacing(e.to_string())))?; + let execution = self.tool.execution(args).map_err(|e| tool_error(&e))?; + let output = execution.execute().await.map_err(|e| tool_error(&e))?; + to_js(&output.result).map_err(JsValue::from) + } +} + +fn tool_error(err: &markkit::tool::ToolError) -> JsValue { + let e = js_sys::Error::new(&err.to_string()); + let _ = js_sys::Reflect::set(&e, &"userFacing".into(), &err.is_user_facing().into()); + e.into() +} diff --git a/justfile b/justfile index 86c7327..5aa093d 100644 --- a/justfile +++ b/justfile @@ -65,6 +65,12 @@ examples: echo "── example: agent_tool" cargo run -q -p markkit --features tool --example agent_tool +# Build the WASM package and run its Node tests (needs wasm32 target + wasm-bindgen-cli) +wasm: + bash scripts/install-wasm-bindgen.sh + bash crates/markkit-wasm/scripts/build.sh release + node --test crates/markkit-wasm/__test__/*.test.mjs + # ─── Run ───────────────────────────────────────────────────── # Run the CLI with arguments diff --git a/knowledge/index.md b/knowledge/index.md index cced66d..7f945cb 100644 --- a/knowledge/index.md +++ b/knowledge/index.md @@ -13,5 +13,5 @@ okf_version: "0.2" * [formats/](formats/) - Per-format parsing: HTML extraction, format detection, DOCX and spreadsheet mapping. * [integrations/](integrations/) - Agent-facing integrations: the LLM tool contract. * [security/](security/) - Security model and detailed threat model for untrusted input. -* [runtimes/](runtimes/) - Published language bindings for Python and Node.js. +* [runtimes/](runtimes/) - Published language bindings for Python, Node.js, and WebAssembly. * [operations/](operations/) - Testing, documentation, examples, maintenance, release, and eval. diff --git a/knowledge/integrations/tool-contract.md b/knowledge/integrations/tool-contract.md index 25e0562..4bdbc68 100644 --- a/knowledge/integrations/tool-contract.md +++ b/knowledge/integrations/tool-contract.md @@ -12,7 +12,7 @@ tags: # Tool Contract ## Status -Implemented (Rust core, `tool` feature). Bindings, WASM and an MCP server are not wired yet. +Implemented (Rust core, `tool` feature) and exposed in the [WASM Package](../runtimes/wasm-package.md) as `MarkkitTool` (no `url` source there). Python/Node bindings and an MCP server are not wired yet. ## Summary diff --git a/knowledge/log.md b/knowledge/log.md index 97e3577..f15aafd 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. * **Ship contract**: "ship" now ends with the merge commit's `main` CI green, not at merge; the `ship` skill gains existing-PR, auto-merge, and post-merge verification steps aligned with bashkit. See [Maintenance](operations/maintenance.md). * **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md). * **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from ``, Open Graph, `` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index 247e902..7915123 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -21,7 +21,8 @@ Follows [Semantic Versioning](https://semver.org/): Releases are agent-driven; CI creates the tag, GitHub Release and crates.io upload. 1. Ask the agent: "Create release v0.X.Y". -2. Agent bumps `version` in root `Cargo.toml`, moves `CHANGELOG.md` Unreleased entries +2. Agent bumps `version` in root `Cargo.toml`, `crates/markkit-js/package.json` and + `crates/markkit-wasm/package.json`, moves `CHANGELOG.md` Unreleased entries under `## [0.X.Y] - YYYY-MM-DD`, runs `just pre-pr`, and opens a PR titled `chore(release): prepare v0.X.Y`. 3. Merging that PR to `main` (squash keeps the title as the commit subject) runs @@ -35,7 +36,7 @@ Releases are agent-driven; CI creates the tag, GitHub Release and crates.io uplo Manual fallback: run the **Release** workflow from `main` (workflow_dispatch); it reads the version from `Cargo.toml`. Python and npm packages are published separately with -`publish-python.yml` / `publish-js.yml`. +`publish-python.yml` / `publish-js.yml` / `publish-wasm.yml`. ## Artifacts @@ -44,6 +45,7 @@ version from `Cargo.toml`. Python and npm packages are published separately with | Rust crate | `cargo publish` | crates.io | | Python wheel | `maturin build` | PyPI | | npm package | `napi build` | npm | +| npm WASM package | `wasm-bindgen` | npm (`@everruns/markkit-wasm`, OIDC) | ## Native package scope and verification diff --git a/knowledge/runtimes/index.md b/knowledge/runtimes/index.md index e8f93e9..1108e47 100644 --- a/knowledge/runtimes/index.md +++ b/knowledge/runtimes/index.md @@ -2,3 +2,4 @@ * [Python Package](python-package.md) - PyO3 Python bindings, exported API, maturin build, and PyPI distribution. * [Node.js Package](node-package.md) - NAPI-RS Node.js bindings, exported API, build, and npm distribution. +* [WASM Package](wasm-package.md) - wasm-bindgen browser and edge package @everruns/markkit-wasm, its API, build, tests, and npm distribution. diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 1d75a46..519e39a 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -58,4 +58,5 @@ Published to npm by `publish-js.yml` from a verified release tag; see ## See also - [Python Package](python-package.md), the sibling binding +- [WASM Package](wasm-package.md), the browser and edge build - [Release Process](../operations/release-process.md), npm publication and tarball verification diff --git a/knowledge/runtimes/wasm-package.md b/knowledge/runtimes/wasm-package.md new file mode 100644 index 0000000..b8415a5 --- /dev/null +++ b/knowledge/runtimes/wasm-package.md @@ -0,0 +1,87 @@ +--- +type: Package Design +title: WASM Package +description: wasm-bindgen browser and edge package @everruns/markkit-wasm, its API, build, tests, and npm distribution. +tags: + - markkit + - wasm + - browser + - bindings +--- + +# WASM Package + +## Overview + +`crates/markkit-wasm` builds `@everruns/markkit-wasm`: Markkit compiled to +`wasm32-unknown-unknown` with `wasm-bindgen --target web`. It mirrors bashkit-wasm: + +- **Single-threaded**: no `SharedArrayBuffer`, no `COOP`/`COEP` headers; runs in + browsers (incl. iframes), edge workers, Node, Deno and Bun. +- **Separate from the napi package** (`@everruns/markkit`), which stays the fast + native option for Node on Linux x64. +- **Features**: `html`, `docx`, `spreadsheet`, `tool`. No `http_client`: reqwest + needs sockets, and the host fetching keeps network policy (SSRF) with the host. + The tool therefore offers only `html` and `content_base64`. + +## API + +```js +import { initMarkkit, Markkit, MarkkitTool, version } from "@everruns/markkit-wasm"; +await initMarkkit(); // once; idempotent +const mk = new Markkit({ includeSelectors: ["article"] }); +mk.extractHtml(html); // string +mk.extractDocument(bytes, { fileName }); // { markdown, title, format, warnings, stats, metadata } +mk.extractBytes(bytes, contentType); // string +const tool = new MarkkitTool({ locale, maxOutputChars }); +tool.toolDefinition(); await tool.execute(args); +``` + +Options are camelCase builder fields (`maxInputSize`, `preserveLinks`, +`preserveImages`, `includeSelectors`, `excludeSelectors`, archive and sheet +limits, `includeHiddenSheets`). Selectors add to the defaults, as in Python. +Unknown keys and wrong types throw `invalid options`: values go through +`serde_json::Value` because serde-wasm-bindgen only reads named fields, so +`deny_unknown_fields` alone would never see a typo. + +Errors throw `Error` with the Rust `Display` text. `MarkkitTool.execute` rejects +with an `Error` carrying `userFacing` (see [Tool Contract](../integrations/tool-contract.md)). + +`initMarkkit()` reads the `.wasm` from disk under Node (no `fetch` for `file:` +URLs) and otherwise lets the glue fetch it next to the module; callers may pass +their own bytes, `Response`, URL or `WebAssembly.Module`. + +## Build + +```bash +rustup target add wasm32-unknown-unknown +bash scripts/install-wasm-bindgen.sh # CLI pinned to Cargo.lock's wasm-bindgen +bash crates/markkit-wasm/scripts/build.sh # -> crates/markkit-wasm/pkg +``` + +Uses the `wasm-release` profile (`opt-level = "z"`, `panic = "abort"`); runs +`wasm-opt -Oz` when binaryen is on PATH. Unoptimized output is about 1.4 MB. +With `panic = "abort"` a Rust panic traps the instance; parsers are expected to +return errors, never panic (invariant S9). + +## Tests + +`crates/markkit-wasm/__test__/*.test.mjs` (`node --test`) run against `pkg/`: +options and their validation, HTML, bytes, DOCX through the zip/XML stack +(fixtures built by `__test__/zip.mjs`), archive limits, malformed input, version +parity with `package.json`, and the tool (definition, locale, execution, +`userFacing` errors). CI job `wasm` in `ci.yml` also checks the core crate for +wasm32 and the npm archive contents; it gates `Check`. + +## Distribution + +`publish-wasm.yml` (workflow_dispatch, dry run by default) builds, tests and +publishes with npm trusted publishing (OIDC). The trusted publisher must be set up +on npmjs.com for `@everruns/markkit-wasm` before the first real publish. +`package.json` `version` must equal the workspace version (tested). + +## See also + +- [Node.js Package](node-package.md), the native napi sibling +- [Tool Contract](../integrations/tool-contract.md), what `MarkkitTool` implements +- [Release Process](../operations/release-process.md), where the publisher runs diff --git a/scripts/install-wasm-bindgen.sh b/scripts/install-wasm-bindgen.sh new file mode 100755 index 0000000..333e178 --- /dev/null +++ b/scripts/install-wasm-bindgen.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Install wasm-bindgen-cli matching the wasm-bindgen crate in Cargo.lock. +# The CLI and the crate must agree exactly on the bindgen schema version. +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +VERSION=$(awk '/^name = "wasm-bindgen"$/ { getline; gsub(/"/, "", $3); print $3 }' "$REPO_ROOT/Cargo.lock" | sort -u) +if [[ -z "$VERSION" || "$VERSION" == *$'\n'* ]]; then + echo "Expected exactly one wasm-bindgen version in Cargo.lock" >&2 + exit 1 +fi + +EXPECTED="wasm-bindgen $VERSION" +if [[ "$(wasm-bindgen --version 2>/dev/null || true)" != "$EXPECTED" ]]; then + cargo install wasm-bindgen-cli --version "$VERSION" --locked --force +fi +if [[ "$(wasm-bindgen --version)" != "$EXPECTED" ]]; then + echo "wasm-bindgen on PATH does not match Cargo.lock ($VERSION)" >&2 + exit 1 +fi +echo "$EXPECTED (matches Cargo.lock)"