diff --git a/.gitignore b/.gitignore index fa61094..83e75a5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ test/cljs/**/*.mjs # dtype-next Panama upcall stubs generated at runtime (gensym per interface) classes/ .clj-kondo/.cache-cli/ +dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 421d7d1..926ccd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## [Unreleased] + +### Added + +- `:float32`, a C float, as an argument or a return type. +- `ffi-wasm/handler`, the worker side of ffi-wasm. `init-pool!` gives its + URL to each handler as `ffiWasmHandlerUrl`. +- `:label` for `gen-handler-source`. + +### Changed + +- ffi-wasm ships its own modules in `dist/`: the page bundle + `ffi-wasm.mjs`, and `handler.mjs` and the modules that the two share as + files of their own, one copy of each. `./handler`, `./handler-*`, + `./http-bridge`, `./fetch-worker` and `./test-runner` point at their own + files, and each other subpath at `ffi-wasm.mjs`. Node and a bundler resolve + the same subpaths as before. Of `src/`, the package ships only + `macros.cljc` and `macros.mjs`. +- A page maps `ffi-wasm` to `dist/ffi-wasm.mjs`, and each subpath that it + imports to the file of that subpath. It also maps `worker-router`, + `worker-router/worker-bootstrap`, `comlink`, `resource-tracker` and + `squint-cljs/`. +- Worker code imports `ffi-wasm/handler` or a worker-side subpath, not + `ffi-wasm`. A bundler of a `dist/` file marks `node:*` as external. +- **Breaking.** `stage-test-deps!` takes `:native-dist`, the `dist/` dir of + the installed ffi-wasm, in place of `:native-src`, and copies only + `test_runner.mjs`. +- **Breaking.** A generated handler has no static import of ffi-wasm. It + imports ffi-wasm from `ffiWasmHandlerUrl`. Its overrides module exports + `methods(ffi)`, and `init` gets `ctx.ffi`. `:runtime-import-path` is gone. +- On Node, `createSyncFetch` defaults `workerUrl` to its own + `fetch_worker.mjs`. + +### Fixed + +- The JVM writes the classes that dtype-next generates to a temp dir. A + project gets no `./classes` dir. +- A zig `-O0` build links no UBSan runtime. + ## [0.0.2] - 2026-09-28 ### Added @@ -41,3 +80,7 @@ ## [0.0.1] - 2026-07-30 First release. + +[Unreleased]: https://github.com/willcohen/clj-native/compare/0.0.2...HEAD +[0.0.2]: https://github.com/willcohen/clj-native/compare/0.0.1...0.0.2 +[0.0.1]: https://github.com/willcohen/clj-native/releases/tag/0.0.1 diff --git a/README.md b/README.md index 5a1d9f3..239bd20 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,8 @@ host. | C struct to map | `graal-wasm/read-struct` | none | `heapHelpers(getModule)` returns an object of methods. Spread it into the -handler's `methods` map, before the consumer's own keys. A consumer key with -the same name replaces the helper. +object that the `methods(ffi)` of the overrides module returns, before the +consumer's own keys. A consumer key with the same name replaces the helper. On JavaScript, a consumer writes its string-array and struct reads in its own overrides module, which knows the struct layouts of its library. On the JVM, @@ -87,10 +87,28 @@ overrides module, which knows the struct layouts of its library. On the JVM, ## Packages -The npm package [`ffi-wasm`](https://www.npmjs.com/package/ffi-wasm) holds the -hand-written `.mjs` runtime helpers and the squint output of the `.cljc` and -`.cljs` modules. The `exports` map in `package.json` lists them. The package -also holds `macros.cljc`, because squint expands the macros at compile time. +The npm package [`ffi-wasm`](https://www.npmjs.com/package/ffi-wasm) ships +its own modules in `dist/`. `ffi-wasm.mjs` is an esbuild bundle of the page +side. `handler.mjs` holds the worker side. The modules that the two share are +files of their own, and each importer imports them by a relative path, so a +page or a Node process loads one copy of each. `./handler`, `./handler-*`, +`./http-bridge`, `./fetch-worker` and `./test-runner` point at their own +files, and each other subpath points at `ffi-wasm.mjs`. `dist/` holds no other +package: a page maps `ffi-wasm`, `worker-router`, +`worker-router/worker-bootstrap`, `comlink`, `squint-cljs/` and +`resource-tracker` in its importmap, one copy of each. A page that imports a +subpath also maps it to the file of that subpath. + +A module worker does not use the importmap of the page. `handler.mjs` and the +shared modules import only each other and `node:` builtins, and `init-pool!` +gives each handler the URL of `handler.mjs`. Worker code imports +`ffi-wasm/handler` or a worker-side subpath, not `ffi-wasm`. A bundler of +worker code marks `node:*` as external. The `gen-handler-source` docstring +gives the contract of the overrides module. + +Besides `dist/`, the package holds `macros.cljc`, because squint expands the +macros at compile time, and `macros.mjs`, which the squint output of a +consumer imports by path. The jar [`net.willcohen/native`](https://clojars.org/net.willcohen/native) holds the source of these 13 namespaces, and a clj-kondo config export. It diff --git a/bb.edn b/bb.edn index 603e836..dc4368c 100644 --- a/bb.edn +++ b/bb.edn @@ -59,15 +59,16 @@ Drift fails build:js, and through prepack each npm pack." :task (nb/check-exports-sync! "package.json")} build:js - {:doc "Compile each shipped .cljc or .cljs module to the .mjs that npm ships." + {:doc "Compile each .cljc or .cljs module to .mjs, then bundle dist/ with esbuild.config.mjs." :depends [check:exports npm:install] :task (let [dir "src/cljc/net/willcohen/native"] - (doseq [f nb/shipped-module-files + (doseq [f nb/export-module-files :let [base (str/replace f (re-pattern "\\.mjs$") "") src (first (filter (fn [src] (fs/exists? (fs/path dir src))) [(str base ".cljc") (str base ".cljs")]))] :when src] - (nb/squint-compile! dir src)))} + (nb/squint-compile! dir src)) + (shell "node" "esbuild.config.mjs"))} build:js:test {:doc "Compile each cljs.test suite." diff --git a/esbuild.config.mjs b/esbuild.config.mjs new file mode 100644 index 0000000..6cc3118 --- /dev/null +++ b/esbuild.config.mjs @@ -0,0 +1,59 @@ +// Copyright (c) 2026 Will Cohen +// +// Part of clj-native, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Builds dist/: the page bundle of ffi-wasm's own modules, and next to it the +// worker entry, the modules that the page and the worker share, and the files +// that createSyncFetch and a test load. bb build:js runs it after squint +// compiles the .cljc modules. +import * as esbuild from 'esbuild'; +import { copyFileSync, mkdirSync } from 'node:fs'; + +const src = 'src/cljc/net/willcohen/native'; + +// Each is a file of its own in dist/, and each importer imports it by a +// relative path, so a realm that loads the page bundle and handler.mjs loads +// one copy of each. They import only each other and node: builtins, thus a +// module worker needs no importmap for them. +const shared = ['handler_runtime', 'handler_env', 'handler_fs', 'handler_heap', + 'handler_paths', 'http_bridge'].map((m) => `${m}.mjs`); + +const keepSharedExternal = { + name: 'keep-shared-external', + setup(build) { + build.onResolve({ filter: /^\.\/[a-z_]+\.mjs$/ }, (args) => + shared.includes(args.path.slice(2)) ? { path: args.path, external: true } : undefined); + }, +}; + +const common = { + bundle: true, + format: 'esm', + platform: 'neutral', + mainFields: ['module', 'main'], + // Node loads the builtins through dynamic imports on Node only. Each one + // has the node: prefix, so a consumer that bundles a dist file marks only + // node:* as external. + external: ['node:*'], + logLevel: 'warning', +}; + +mkdirSync('dist', { recursive: true }); + +await esbuild.build({ + ...common, + entryPoints: [`${src}/ffi_wasm.mjs`], + outfile: 'dist/ffi-wasm.mjs', + // The bundle holds ffi-wasm's own code only. A copy of another package is a + // second module instance next to the copy that a consumer imports. + external: [...common.external, 'squint-cljs', 'resource-tracker', 'worker-router'], + plugins: [keepSharedExternal], +}); + +// test_runner.mjs is out of the page bundle, so that a page does not import +// the squint test library. +for (const f of ['handler.mjs', ...shared, 'fetch_worker.mjs', 'test_runner.mjs']) { + copyFileSync(`${src}/${f}`, `dist/${f}`); +} diff --git a/eslint.config.mjs b/eslint.config.mjs index 221f8eb..2ccea14 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -5,7 +5,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // Lints the hand-written .mjs and the test fixtures. The ignored src files are -// gitignored squint outputs. test/cljs and test/cljc hold compiled test +// gitignored squint outputs, and dist/ holds the esbuild bundles. test/cljs and test/cljc hold compiled test // mirrors. import js from '@eslint/js'; import globals from 'globals'; @@ -19,6 +19,7 @@ export default [ 'src/cljc/net/willcohen/native/pool.mjs', 'src/cljc/net/willcohen/native/test_runner.mjs', 'src/cljc/net/willcohen/native/workload_pool.mjs', + 'dist/**', 'test/cljs/**', 'test/cljc/**', ], diff --git a/package-lock.json b/package-lock.json index 9e2f347..264eef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", + "esbuild": "^0.28.2", "eslint": "^10.11.0", "globals": "^17.12.0", "squint-cljs": "^0.14.210" @@ -54,6 +55,448 @@ "keyv": "^5.6.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", @@ -431,6 +874,48 @@ "dev": true, "license": "MIT" }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", diff --git a/package.json b/package.json index 01e4960..0db544c 100644 --- a/package.json +++ b/package.json @@ -3,39 +3,38 @@ "version": "0.0.2", "type": "module", "description": "Helper utilities for WASM libraries in the Squint (ClojureScript) ecosystem. The JavaScript half of clj-native (Clojars: net.willcohen/native).", - "main": "src/cljc/net/willcohen/native/handler_runtime.mjs", + "main": "dist/ffi-wasm.mjs", "exports": { - ".": "./src/cljc/net/willcohen/native/handler_runtime.mjs", - "./handler-runtime": "./src/cljc/net/willcohen/native/handler_runtime.mjs", - "./handler-fs": "./src/cljc/net/willcohen/native/handler_fs.mjs", - "./handler-heap": "./src/cljc/net/willcohen/native/handler_heap.mjs", - "./handler-paths": "./src/cljc/net/willcohen/native/handler_paths.mjs", - "./handler-env": "./src/cljc/net/willcohen/native/handler_env.mjs", - "./pool": "./src/cljc/net/willcohen/native/pool.mjs", - "./workload-pool": "./src/cljc/net/willcohen/native/workload_pool.mjs", - "./platform-state": "./src/cljc/net/willcohen/native/platform_state.mjs", - "./dispatch": "./src/cljc/net/willcohen/native/dispatch.mjs", - "./macros": "./src/cljc/net/willcohen/native/macros.mjs", - "./test-runner": "./src/cljc/net/willcohen/native/test_runner.mjs", - "./http-bridge": "./src/cljc/net/willcohen/native/http_bridge.mjs", - "./fetch-worker": "./src/cljc/net/willcohen/native/fetch_worker.mjs", + ".": "./dist/ffi-wasm.mjs", + "./handler": "./dist/handler.mjs", + "./handler-runtime": "./dist/handler_runtime.mjs", + "./handler-fs": "./dist/handler_fs.mjs", + "./handler-heap": "./dist/handler_heap.mjs", + "./handler-paths": "./dist/handler_paths.mjs", + "./handler-env": "./dist/handler_env.mjs", + "./pool": "./dist/ffi-wasm.mjs", + "./workload-pool": "./dist/ffi-wasm.mjs", + "./platform-state": "./dist/ffi-wasm.mjs", + "./dispatch": "./dist/ffi-wasm.mjs", + "./macros": "./dist/ffi-wasm.mjs", + "./test-runner": "./dist/test_runner.mjs", + "./http-bridge": "./dist/http_bridge.mjs", + "./fetch-worker": "./dist/fetch_worker.mjs", "./package.json": "./package.json" }, "files": [ - "src/cljc/net/willcohen/native/handler_runtime.mjs", - "src/cljc/net/willcohen/native/handler_fs.mjs", - "src/cljc/net/willcohen/native/handler_heap.mjs", - "src/cljc/net/willcohen/native/handler_paths.mjs", - "src/cljc/net/willcohen/native/handler_env.mjs", - "src/cljc/net/willcohen/native/pool.mjs", - "src/cljc/net/willcohen/native/workload_pool.mjs", - "src/cljc/net/willcohen/native/platform_state.mjs", - "src/cljc/net/willcohen/native/dispatch.mjs", + "dist/ffi-wasm.mjs", + "dist/handler.mjs", + "dist/handler_runtime.mjs", + "dist/handler_fs.mjs", + "dist/handler_heap.mjs", + "dist/handler_paths.mjs", + "dist/handler_env.mjs", + "dist/http_bridge.mjs", + "dist/fetch_worker.mjs", + "dist/test_runner.mjs", "src/cljc/net/willcohen/native/macros.cljc", - "src/cljc/net/willcohen/native/macros.mjs", - "src/cljc/net/willcohen/native/test_runner.mjs", - "src/cljc/net/willcohen/native/http_bridge.mjs", - "src/cljc/net/willcohen/native/fetch_worker.mjs" + "src/cljc/net/willcohen/native/macros.mjs" ], "scripts": { "prepack": "bb build:js" @@ -54,6 +53,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", + "esbuild": "^0.28.2", "eslint": "^10.11.0", "globals": "^17.12.0", "squint-cljs": "^0.14.210" diff --git a/src/bb/net/willcohen/native/build.clj b/src/bb/net/willcohen/native/build.clj index 758f0a9..837f5e8 100644 --- a/src/bb/net/willcohen/native/build.clj +++ b/src/bb/net/willcohen/native/build.clj @@ -562,54 +562,73 @@ (println out "ready at" (str dir "/" out))))) (defn stage-test-deps! - "Copy platform_state.mjs and test_runner.mjs, the clj-native test - helpers, from the dir :native-src into :dist (default + "Copy test_runner.mjs, the clj-native test helper, from :native-dist, the + dist/ dir of the installed ffi-wasm, into :dist (default \"test/cljc/dist\"), for import by relative path. An npm symlink would load a second squint-cljs instance, and deftest and run-tests would then - see different registries. Throws when a file is missing. + see different registries. Throws when the file is missing. - Each copy still imports squint-cljs by bare specifier, so the consumer's + The copy still imports squint-cljs by bare specifier, so the consumer's test node_modules must have it." - [{:keys [native-src dist] :or {dist "test/cljc/dist"}}] - (let [files ["platform_state.mjs" "test_runner.mjs"]] + [{:keys [native-dist dist] :or {dist "test/cljc/dist"}}] + (let [n "test_runner.mjs" + src (fs/path (str native-dist) n)] + (when-not (fs/exists? src) + (throw (ex-info (str "Missing clj-native artifact: " src + ". Install ffi-wasm where :native-dist points.") + {:missing (str src)}))) (fs/create-dirs dist) - (doseq [n files] - (let [src (fs/path (str native-src) n)] - (when-not (fs/exists? src) - (throw (ex-info (str "Missing clj-native artifact: " src - ". Install ffi-wasm where :native-src points.") - {:missing (str src)}))) - (fs/copy src (fs/path dist n) {:replace-existing true}))) - (println (str "Staged clj-native helpers into " dist ": " (str/join ", " files))))) + (fs/copy src (fs/path dist n) {:replace-existing true}) + (println (str "Staged " n " into " dist)))) (def ^:private npm-package-name "The npm package name. check-exports-sync! pins it to package.json." "ffi-wasm") -(def shipped-module-files - "Each compiled .mjs that the npm tarball ships. check-exports-sync! pins - it to package.json." - ;; macros.mjs is dead code in a consumer bundle, but esbuild needs it to - ;; resolve the import. - ["dispatch.mjs" "fetch_worker.mjs" "handler_env.mjs" "handler_fs.mjs" +(def export-module-files + "Each .mjs of src/ that has an export subpath: the file name with dashes + for underscores and no extension. build:js compiles the .cljc or .cljs of + each one. check-exports-sync! pins the subpaths to package.json." + ["dispatch.mjs" "fetch_worker.mjs" "handler.mjs" "handler_env.mjs" "handler_fs.mjs" "handler_heap.mjs" "handler_paths.mjs" "handler_runtime.mjs" "http_bridge.mjs" "macros.mjs" "platform_state.mjs" "pool.mjs" "test_runner.mjs" "workload_pool.mjs"]) +(def shipped-module-files + "Each .mjs of src/ that the npm tarball ships. The squint output of a + consumer imports macros.mjs by path, next to macros.cljc, and a bundler of + that output needs the file. check-exports-sync! pins it to package.json." + ["macros.mjs"]) + +(def bundle-files + "Each file of dist/ that esbuild.config.mjs writes and the npm tarball + ships. check-exports-sync! pins it to package.json." + ["fetch_worker.mjs" "ffi-wasm.mjs" "handler.mjs" "handler_env.mjs" "handler_fs.mjs" + "handler_heap.mjs" "handler_paths.mjs" "handler_runtime.mjs" "http_bridge.mjs" + "test_runner.mjs"]) + +(def ^:private own-files + "The dist/ file of each subpath outside the page bundle." + {"handler" "handler.mjs" "handler-env" "handler_env.mjs" "handler-fs" "handler_fs.mjs" + "handler-heap" "handler_heap.mjs" "handler-paths" "handler_paths.mjs" + "handler-runtime" "handler_runtime.mjs" "http-bridge" "http_bridge.mjs" + "fetch-worker" "fetch_worker.mjs" "test-runner" "test_runner.mjs"}) + (defn export-specifier-rewrites "Map each bare specifier this package exports, such as \"ffi-wasm/pool\", - to `target-prefix` plus its file name. For rewrite-import-specifiers! on - copies of shipped-module-files, where no importmap applies. The subpath is - the file name with dashes for underscores and no extension." + to `target-prefix` plus the dist/ file that it resolves to. For + rewrite-import-specifiers! on copies, where no importmap applies. The + subpath is the src/ file name with dashes for underscores and no + extension." [target-prefix] - (into {} + (into {npm-package-name (str target-prefix "ffi-wasm.mjs")} (map (fn [f] - [(str npm-package-name "/" - (-> f - (str/replace (re-pattern "\\.mjs$") "") - (str/replace "_" "-"))) - (str target-prefix f)])) - shipped-module-files)) + (let [subpath (-> f + (str/replace (re-pattern "\\.mjs$") "") + (str/replace "_" "-"))] + [(str npm-package-name "/" subpath) + (str target-prefix (get own-files subpath "ffi-wasm.mjs"))]))) + export-module-files)) (defn rewrite-import-specifiers! "Rewrite the import specifiers in the file at `target` by `rewrites`, a @@ -655,22 +674,25 @@ (defn check-exports-sync! "Throw with the drift when the file `package-json` disagrees with - npm-package-name and shipped-module-files: its name, main, each exports - entry and its files, all by full path." + npm-package-name, export-module-files, shipped-module-files and + bundle-files: its name, main, each exports entry and its files, all by + full path." [package-json] (let [pkg (json/parse-string (slurp (str package-json))) - dir "src/cljc/net/willcohen/native/" - main (str dir "handler_runtime.mjs") + src "src/cljc/net/willcohen/native/" want {"name" npm-package-name - "main" main - "exports" (into {"." (str "./" main) "./package.json" "./package.json"} + "main" "dist/ffi-wasm.mjs" + "exports" (into {"./package.json" "./package.json"} (map (fn [[spec f]] [(str "." (subs spec (count npm-package-name))) f])) - (export-specifier-rewrites (str "./" dir))) + (export-specifier-rewrites "./dist/")) ;; squint reads macros.cljc to expand the macros. - "files" (sort (cons (str dir "macros.cljc") (map #(str dir %) shipped-module-files)))} + "files" (sort (concat [(str src "macros.cljc")] + (map #(str src %) shipped-module-files) + (map #(str "dist/" %) bundle-files)))} have (update (select-keys pkg (keys want)) "files" sort) drift (into {} (remove (fn [[k v]] (= v (get have k)))) want)] (when (seq drift) (throw (ex-info "package.json disagrees with the module inventory of net.willcohen.native.build" {:build-clj drift :package-json (select-keys have (keys drift))}))) - (println (str "package.json inventory in sync: " (count shipped-module-files) " modules")))) + (println (str "package.json inventory in sync: " (count export-module-files) + " subpath modules, " (count bundle-files) " dist files")))) diff --git a/src/clj/net/willcohen/native/gen_handler.clj b/src/clj/net/willcohen/native/gen_handler.clj index 7eda4b5..4f41925 100644 --- a/src/clj/net/willcohen/native/gen_handler.clj +++ b/src/clj/net/willcohen/native/gen_handler.clj @@ -12,7 +12,7 @@ (:require [clojure.string :as str])) (def ^:private required-override-keys - [:overrides-import-path :runtime-import-path :exposed-methods :fingerprint-fields]) + [:overrides-import-path :exposed-methods :fingerprint-fields]) (defn- check-required! [overrides] (doseq [k required-override-keys] @@ -45,9 +45,8 @@ fndefs {fn-key fn-def}. Only :destroy? of a fn-def is read. overrides Keys: :overrides-import-path Required. Path of the overrides .mjs. - :runtime-import-path Required. Path of the handler runtime. - :exposed-methods Required. Method keywords, each mapped to - overrides.methods.. + :exposed-methods Required. Method keywords, each taken from the + object that overrides.methods(ffi) returns. :fingerprint-fields Required. Init-arg names for the byteLengthFingerprint of the handler. :busy-methods Subset of :exposed-methods, disjoint from @@ -55,31 +54,38 @@ :destroy-methods Subset of :exposed-methods. Default []. :destroy-fns C fn names. Default: fndefs keys with :destroy?. :fingerprint-prefix Fingerprint label for re-init errors. + :label The makeHandler label of each trace event. The busy and destroy classes pick trace events only, because the runtime - queue is serial. The fingerprint is generated here because the bundler - inlines this module but not the overrides module, so an overrides module - that imports the runtime gets a second copy with its own logging state. + queue is serial. - The overrides module can export `destroy`, which worker-router calls once - per worker when the pool terminates, to close resources that the - emscripten Module holds open, such as the http-bridge fetch worker." + The handler has no static import of ffi-wasm, because a module worker + ignores the page importmap. Its first create imports ffi-wasm/handler from + initArgs.ffiWasmHandlerUrl, which init-pool! sets, or from the bare + specifier outside a pool. The overrides module gets that one copy, so + the worker has one logging state: + + methods(ffi) Required. Returns the methods object. + init(args, ctx) Optional. ctx.ffi is the ffi-wasm/handler namespace. + destroy() Optional. worker-router calls it once per worker when + the pool terminates, to close resources that the + emscripten Module holds open, such as the http-bridge + fetch worker." [fndefs overrides] (check-required! overrides) - (let [{:keys [overrides-import-path runtime-import-path - exposed-methods busy-methods destroy-methods - destroy-fns fingerprint-fields fingerprint-prefix]} overrides + (let [{:keys [overrides-import-path exposed-methods busy-methods destroy-methods + destroy-fns fingerprint-fields fingerprint-prefix label]} overrides busy-methods (or busy-methods []) destroy-methods (or destroy-methods []) destroy-fns (or destroy-fns (destroy-fns-from-fndefs fndefs)) method-bindings (->> exposed-methods (map (fn [m] (let [n (name m)] - (str " " n ": overrides.methods." n ",")))) - (str/join "\n"))] + (str " " n ": impl." n ",")))) + (str/join "\n")) + js-string-or-null (fn [s] (if s (js-string-literal s) "null"))] (str "// Generated by clj-native gen-handler-source. Do not edit.\n\n" - "import { makeHandler, byteLengthFingerprint } from '" runtime-import-path "';\n" "import * as overrides from '" overrides-import-path "';\n" "\n" "const busyMethods = " (js-string-array busy-methods) ";\n" @@ -88,22 +94,35 @@ "// C fnNames from the fndefs :destroy? flags. Nothing reads this.\n" "const destroyFns = " (js-string-array destroy-fns) ";\n" "\n" - "const methods = {\n" + "const fingerprintFields = " (js-string-array fingerprint-fields) ";\n" + "const fingerprintPrefix = " (js-string-or-null fingerprint-prefix) ";\n" + "\n" + "// A module worker ignores the page importmap, so ffi-wasm loads from the URL\n" + "// that init-pool! puts in the init args. Outside a pool, Node resolves the\n" + "// bare specifier.\n" + "const loadFactory = async (initArgs) => {\n" + " const ffi = await import(initArgs?.ffiWasmHandlerUrl ?? 'ffi-wasm/handler');\n" + " const impl = overrides.methods(ffi);\n" + " const methods = {\n" method-bindings "\n" + " };\n" + " return ffi.makeHandler({\n" + " init: (args, ctx) => overrides.init?.(args, { ...ctx, ffi }),\n" + " methods,\n" + " busyMethods,\n" + " destroyMethods,\n" + " fingerprint: ffi.byteLengthFingerprint(fingerprintFields, fingerprintPrefix),\n" + " label: " (js-string-or-null label) ",\n" + " });\n" "};\n" "\n" - "const fingerprint = byteLengthFingerprint(\n" - " " (js-string-array fingerprint-fields) ",\n" - " " (if fingerprint-prefix (js-string-literal fingerprint-prefix) "null") ",\n" - ");\n" + "let factory = null;\n" "\n" - "export const create = makeHandler({\n" - " init: overrides.init,\n" - " methods,\n" - " busyMethods,\n" - " destroyMethods,\n" - " fingerprint,\n" - "});\n" + "export const create = async (initArgs) => {\n" + " // A failed import clears the memo, so a later create can retry.\n" + " factory ??= loadFactory(initArgs).catch((e) => { factory = null; throw e; });\n" + " return (await factory)(initArgs);\n" + "};\n" "\n" "export default create;\n" "export { destroyFns };\n" diff --git a/src/cljc/net/willcohen/native/dispatch.cljc b/src/cljc/net/willcohen/native/dispatch.cljc index 1270873..b4c701b 100644 --- a/src/cljc/net/willcohen/native/dispatch.cljc +++ b/src/cljc/net/willcohen/native/dispatch.cljc @@ -40,7 +40,7 @@ (def supported-types "The fndefs types that `library` accepts for :rettype and argtypes." - #{:pointer :pointer? :int32 :int64 :float64 :size-t :void :string :string?}) + #{:pointer :pointer? :int32 :int64 :float32 :float64 :size-t :void :string :string?}) ;; (str :pointer) is ":pointer" on the JVM and "pointer" under squint. (def ^:private supported-types-msg @@ -111,7 +111,9 @@ gives \"\" for both. :int64 is a Long on the JVM. The wasm backends send a BigInt, which a WASM_BIGINT module (the emscripten default since 4.0.0) needs. CLJS - returns the BigInt." + returns the BigInt. + :float32 is a C float. An argument narrows to f32 at the call. A result + is a Float from FFI, a Double from GraalVM and a number on CLJS." [{:keys [key fndefs impl-atom ffi-impl-ns hooks]}] {:key key :impl-atom impl-atom @@ -159,9 +161,9 @@ :int32 (if (instance? org.graalvm.polyglot.Value result) (nw/address-as-int result) result) - :float64 (if (instance? org.graalvm.polyglot.Value result) - (.asDouble ^org.graalvm.polyglot.Value result) - result) + (:float32 :float64) (if (instance? org.graalvm.polyglot.Value result) + (.asDouble ^org.graalvm.polyglot.Value result) + result) (:int64 :size-t) (if (instance? org.graalvm.polyglot.Value result) (.asLong ^org.graalvm.polyglot.Value result) result) diff --git a/src/cljc/net/willcohen/native/fetch_worker.mjs b/src/cljc/net/willcohen/native/fetch_worker.mjs index 0609485..f5b1c16 100644 --- a/src/cljc/net/willcohen/native/fetch_worker.mjs +++ b/src/cljc/net/willcohen/native/fetch_worker.mjs @@ -32,7 +32,7 @@ // the blocked caller, and a token refresh can await. If the import fails or // exports no function, the worker posts an error and never reports ready. -import { parentPort } from 'worker_threads'; +import { parentPort } from 'node:worker_threads'; const OVERFLOW_FLAG = 1; diff --git a/src/cljc/net/willcohen/native/ffi_wasm.mjs b/src/cljc/net/willcohen/native/ffi_wasm.mjs new file mode 100644 index 0000000..43401f5 --- /dev/null +++ b/src/cljc/net/willcohen/native/ffi_wasm.mjs @@ -0,0 +1,23 @@ +// Copyright (c) 2026 Will Cohen +// +// Part of clj-native, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// The page side of ffi-wasm as one module. esbuild bundles it into +// dist/ffi-wasm.mjs, and each subpath export except ./handler, ./fetch-worker +// and ./test-runner points there. A page, Node and a bundler then load one module +// instance. export * drops a name that two of these modules export, and +// bundle_test checks that none does. + +export * from './pool.mjs'; +export * from './dispatch.mjs'; +export * from './platform_state.mjs'; +export * from './macros.mjs'; +export * from './workload_pool.mjs'; +export * from './handler_runtime.mjs'; +export * from './handler_env.mjs'; +export * from './handler_fs.mjs'; +export * from './handler_heap.mjs'; +export * from './handler_paths.mjs'; +export * from './http_bridge.mjs'; diff --git a/src/cljc/net/willcohen/native/handler.mjs b/src/cljc/net/willcohen/native/handler.mjs new file mode 100644 index 0000000..4edb40c --- /dev/null +++ b/src/cljc/net/willcohen/native/handler.mjs @@ -0,0 +1,17 @@ +// Copyright (c) 2026 Will Cohen +// +// Part of clj-native, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// The worker side of ffi-wasm as one module. A module worker ignores the page +// importmap, so a generated handler imports this module by the URL that +// init-pool! puts in its init args. Each module below imports only its +// siblings and Node builtins, so that one URL loads all of them. + +export * from './handler_runtime.mjs'; +export * from './handler_env.mjs'; +export * from './handler_fs.mjs'; +export * from './handler_heap.mjs'; +export * from './handler_paths.mjs'; +export * from './http_bridge.mjs'; diff --git a/src/cljc/net/willcohen/native/http_bridge.mjs b/src/cljc/net/willcohen/native/http_bridge.mjs index 9587d7c..2bc7056 100644 --- a/src/cljc/net/willcohen/native/http_bridge.mjs +++ b/src/cljc/net/willcohen/native/http_bridge.mjs @@ -13,6 +13,7 @@ // caller must be in a Web Worker. // // Contracts: +// - On Node, workerUrl defaults to the fetch_worker.mjs next to this module. // - On Node, createSyncFetch rejects if the worker cannot become ready, for // example when the decorator import fails. A later call reuses the // running worker and ignores its own workerUrl, dataBufferSize and @@ -64,7 +65,8 @@ async function ensureWorker(opts) { const decorateUrl = opts.decorateUrl ? String(opts.decorateUrl) : null; let state = workerState; if (!state) { - workerStart ??= startWorker(opts.workerUrl, decorateUrl, opts) + workerStart ??= startWorker(opts.workerUrl ?? new URL('./fetch_worker.mjs', import.meta.url), + decorateUrl, opts) .finally(() => { workerStart = null; }); state = await workerStart; } @@ -74,9 +76,10 @@ async function ensureWorker(opts) { return state; } +// A bundle that inlines this module must pass workerUrl, because the default +// resolves next to the bundle. async function startWorker(workerUrl, decorateUrl, opts) { - if (!workerUrl) throw new Error('createSyncFetch: workerUrl is required on Node'); - const { Worker } = await import('worker_threads'); + const { Worker } = await import('node:worker_threads'); const dataBufferSize = opts.dataBufferSize ?? DEFAULT_DATA_BUFFER_SIZE; const requestTimeoutMs = opts.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; diff --git a/src/cljc/net/willcohen/native/pool.cljs b/src/cljc/net/willcohen/native/pool.cljs index 8912bae..518ee45 100644 --- a/src/cljc/net/willcohen/native/pool.cljs +++ b/src/cljc/net/willcohen/native/pool.cljs @@ -92,16 +92,29 @@ [opts] (cond-> opts (some? (:categories opts)) (update :categories vec))) +;; handler.mjs sits next to this module in dist/, so each worker runs the +;; ffi-wasm copy of the page, and a page maps no importmap entry for it. +(def ^:private ffi-wasm-handler-url + (.-href (js/URL. "./handler.mjs" (.-url js/import.meta)))) + +(defn- with-ffi-wasm-handler-url + "`handlers` with ffiWasmHandlerUrl in a copy of each :init." + [handlers] + (update-vals handlers + (fn add-url [spec] + (assoc spec :init (assoc (:init spec) :ffiWasmHandlerUrl ffi-wasm-handler-url))))) + (defn ^:async init-pool! "Spawn a WorkerPool that loads `handlers`, handler-key -> {:module url :init args}. :size is \"auto\" (default) or an integer. Each worker gets its slot index, and the :handler-runtime log config when there is one. + Each :init also gets ffiWasmHandlerUrl, which a generated handler imports. Returns the pool." [{:keys [handlers size handler-runtime]}] (let [pool (await (.create (.-WorkerPool cp) #js {:size (or size "auto") :bootstrap (.resolve js/import.meta "worker-router/worker-bootstrap") - :handlers handlers})) + :handlers (with-ffi-wasm-handler-url handlers)})) handler-keys (vec (js/Object.keys handlers))] (when (some? handler-runtime) (await (broadcast-to-handlers! pool handler-keys "__setLogConfig" diff --git a/test/bb/net/willcohen/native/build_test.clj b/test/bb/net/willcohen/native/build_test.clj index 6728ba8..0a85596 100644 --- a/test/bb/net/willcohen/native/build_test.clj +++ b/test/bb/net/willcohen/native/build_test.clj @@ -341,26 +341,45 @@ Import { (nb/check-exports-sync! copy))] (is (nil? (check! "" ""))) (is (thrown? clojure.lang.ExceptionInfo - (check! "\"./src/cljc/net/willcohen/native/pool.mjs\"" "\"./src/cljc/net/willcohen/nativ/pool.mjs\"")) - "a wrong dir in an export") + (check! "\"./handler\": \"./dist/handler.mjs\"" "\"./handler\": \"./dist/ffi-wasm.mjs\"")) + "a subpath at the wrong dist file") (is (thrown? clojure.lang.ExceptionInfo - (check! "\"main\": \"src/cljc/net/willcohen/native/handler_runtime.mjs\"" + (check! "\"main\": \"dist/ffi-wasm.mjs\"" "\"main\": \"src/cljc/net/willcohen/native/pool.mjs\"")) "a wrong main") (is (thrown? clojure.lang.ExceptionInfo (check! "\"src/cljc/net/willcohen/native/macros.cljc\",\n" "")) - "macros.cljc left out of files"))) - -(deftest stage-test-deps-copies-each-helper-or-throws - (let [src (fs/create-temp-dir) - dist (str (fs/path (fs/create-temp-dir) "dist")) - opts {:dist dist :native-src (str src)}] - (spit (str (fs/path src "platform_state.mjs")) "p") + "macros.cljc left out of files") + (is (thrown? clojure.lang.ExceptionInfo + (check! "\"dist/test_runner.mjs\",\n" "")) + "a dist file left out of files"))) + +(deftest export-specifier-rewrites-point-each-subpath-at-its-dist-file + (let [r (nb/export-specifier-rewrites "./ffi-wasm/")] + (is (= "./ffi-wasm/ffi-wasm.mjs" (get r "ffi-wasm"))) + (is (= "./ffi-wasm/ffi-wasm.mjs" (get r "ffi-wasm/pool"))) + (is (= "./ffi-wasm/ffi-wasm.mjs" (get r "ffi-wasm/platform-state"))) + (is (= "./ffi-wasm/handler.mjs" (get r "ffi-wasm/handler"))) + (is (= "./ffi-wasm/handler_env.mjs" (get r "ffi-wasm/handler-env"))) + (is (= "./ffi-wasm/http_bridge.mjs" (get r "ffi-wasm/http-bridge"))) + (is (= "./ffi-wasm/fetch_worker.mjs" (get r "ffi-wasm/fetch-worker"))) + (is (= "./ffi-wasm/test_runner.mjs" (get r "ffi-wasm/test-runner"))))) + +;; The tarball ships dist/ and the macros, not the src/ copy of each module. +(deftest an-export-subpath-needs-no-shipped-src-file + (is (not-any? #{"pool.mjs"} nb/shipped-module-files)) + (is (= "./ffi-wasm/ffi-wasm.mjs" (get (nb/export-specifier-rewrites "./ffi-wasm/") "ffi-wasm/pool")))) + +(deftest stage-test-deps-copies-the-test-runner-from-dist-or-throws + (let [native-dist (fs/create-temp-dir) + dist (str (fs/path (fs/create-temp-dir) "dist")) + opts {:dist dist :native-dist (str native-dist)}] (is (thrown-with-msg? clojure.lang.ExceptionInfo #"test_runner\.mjs" (nb/stage-test-deps! opts))) - (spit (str (fs/path src "test_runner.mjs")) "t") + (spit (str (fs/path native-dist "test_runner.mjs")) "t") (nb/stage-test-deps! opts) - (is (= ["p" "t"] (map #(slurp (str (fs/path dist %))) ["platform_state.mjs" "test_runner.mjs"]))))) + (is (= "t" (slurp (str (fs/path dist "test_runner.mjs"))))) + (is (not (fs/exists? (fs/path dist "platform_state.mjs")))))) (def ^:private dylib-deps "/tmp/libproj.dylib: diff --git a/test/clj/net/willcohen/native/gen_handler_test.clj b/test/clj/net/willcohen/native/gen_handler_test.clj index 8b681c0..2133fdc 100644 --- a/test/clj/net/willcohen/native/gen_handler_test.clj +++ b/test/clj/net/willcohen/native/gen_handler_test.clj @@ -7,6 +7,7 @@ (ns net.willcohen.native.gen-handler-test (:require [clojure.test :refer [deftest is testing]] [clojure.string :as str] + [clojure.java.io :as io] [clojure.java.shell :as sh] [net.willcohen.native.gen-handler :as g])) @@ -29,7 +30,6 @@ (def mylib-overrides {:overrides-import-path "./mylib-handler-overrides.mjs" - :runtime-import-path "ffi-wasm/handler-runtime" :exposed-methods [:context_create :set_log_level :context_destroy :ccall :malloc :free :heapf64_set :heapf64_get :read_string_array @@ -50,9 +50,12 @@ (deftest gen-handler-source--module-parts (let [source (g/gen-handler-source sample-fndefs mylib-overrides)] + ;; A module worker ignores the page importmap, so a static bare import + ;; of ffi-wasm fails in a browser worker. (testing "imports" - (is (str/includes? source "import { makeHandler, byteLengthFingerprint } from 'ffi-wasm/handler-runtime'")) - (is (str/includes? source "import * as overrides from './mylib-handler-overrides.mjs'"))) + (is (not (re-find #"(?m)^import .* from 'ffi-wasm" source))) + (is (str/includes? source "import * as overrides from './mylib-handler-overrides.mjs'")) + (is (str/includes? source "await import(initArgs?.ffiWasmHandlerUrl ?? 'ffi-wasm/handler')"))) (testing "classification arrays" (is (str/includes? source (str "const busyMethods = [\"ccall\", \"malloc\", \"context_create\", " "\"set_log_level\", \"heapf64_set\", \"heapf64_get\", " @@ -60,10 +63,11 @@ "\"string_to_utf8\", \"utf8_to_string\"];\n"))) (is (str/includes? source "const destroyMethods = [\"context_destroy\", \"free\", \"shutdown\"];\n"))) - (testing "one methods entry per exposed method" + (testing "one methods entry per exposed method, from overrides.methods(ffi)" + (is (str/includes? source "const impl = overrides.methods(ffi);")) (is (str/includes? source "const methods = {")) (doseq [m [:context_create :ccall :malloc :free :shutdown :heapf64_set]] - (is (str/includes? source (str (name m) ": overrides.methods." (name m))) + (is (str/includes? source (str (name m) ": impl." (name m) ",")) (str "method " (name m) " missing from emitted methods object")))) ;; worker-router's worker-bootstrap runs teardown only when ;; `typeof mod.destroy === 'function'`. @@ -87,7 +91,7 @@ (is (not (str/includes? block "\"mylib_destroy\""))))) (deftest gen-handler-source--required-keys - (doseq [k [:overrides-import-path :runtime-import-path :exposed-methods :fingerprint-fields]] + (doseq [k [:overrides-import-path :exposed-methods :fingerprint-fields]] (is (thrown-with-msg? Exception (re-pattern (str "missing required override key " k)) (g/gen-handler-source sample-fndefs (dissoc mylib-overrides k)))))) @@ -100,15 +104,13 @@ (is (str/includes? source "const destroyMethods = [];")))) (deftest gen-handler-source--fingerprint-fields-emit-a-generated-fingerprint - ;; The overrides stay external to the bundle, so an overrides import of the - ;; runtime would load a second copy of its logging state. (let [source (g/gen-handler-source sample-fndefs (assoc mylib-overrides :fingerprint-fields [:dbBytes :iniBytes :logLevel]))] - (is (str/includes? source "import { makeHandler, byteLengthFingerprint } from") - "the helper is imported alongside makeHandler, from the same one copy") - (is (str/includes? source "byteLengthFingerprint(\n [\"dbBytes\", \"iniBytes\", \"logLevel\"],\n null,\n)")) - (is (str/includes? source " fingerprint,\n")) + (is (str/includes? source "const fingerprintFields = [\"dbBytes\", \"iniBytes\", \"logLevel\"];\n")) + (is (str/includes? source "const fingerprintPrefix = null;\n")) + (is (str/includes? source "fingerprint: ffi.byteLengthFingerprint(fingerprintFields, fingerprintPrefix),") + "the helper comes from the same ffi-wasm copy as makeHandler") (is (not (str/includes? source "overrides.fingerprint")) "the overrides export is no longer consulted"))) @@ -118,10 +120,14 @@ :fingerprint-prefix prefix))) (deftest gen-handler-source--fingerprint-prefix-labels-the-handler - (is (str/includes? (prefixed-source "gdal") - "byteLengthFingerprint(\n [\"dbBytes\"],\n \"gdal\",\n)")) + (is (str/includes? (prefixed-source "gdal") "const fingerprintPrefix = \"gdal\";\n")) (testing "the prefix is a JS string literal, so a quote in it cannot end it" - (is (str/includes? (prefixed-source "o'b\"x") " \"o'b\\\"x\",\n")))) + (is (str/includes? (prefixed-source "o'b\"x") "const fingerprintPrefix = \"o'b\\\"x\";\n")))) + +(deftest gen-handler-source--label-goes-to-makeHandler + (is (str/includes? (g/gen-handler-source sample-fndefs mylib-overrides) " label: null,\n")) + (is (str/includes? (g/gen-handler-source sample-fndefs (assoc mylib-overrides :label "cg.wasmts")) + " label: \"cg.wasmts\",\n"))) (deftest gen-handler-source--output-parses-under-node ;; With a prefix, the output holds every form the generator emits. @@ -146,3 +152,68 @@ (is (= (.getAbsolutePath tmp) path)) (is (= (g/gen-handler-source sample-fndefs mylib-overrides) (slurp path)))) (finally (.delete tmp))))) + +(def ^:private run-overrides + {:overrides-import-path "./overrides.mjs" + :exposed-methods [:probe] + :busy-methods [:probe] + :fingerprint-fields [:dbBytes] + :label "fixture"}) + +;; With a URL argument, the first create gets a URL that does not load, and +;; the second gets the good one. +(def ^:private run-driver + "import { create, destroy } from './handler.mjs'; +const good = process.argv[2]; +const log = { handlerRuntime: { logLevel: 'debug', logCategories: ['busy'] } }; +const out = {}; +if (good) { + out.bad = await create({ ffiWasmHandlerUrl: 'file:///nonexistent/ffi-wasm-handler.mjs', dbBytes: new Uint8Array(1) }) + .then(() => 'resolved', () => 'rejected'); +} +const h = await create({ ...(good ? { ffiWasmHandlerUrl: good } : {}), dbBytes: new Uint8Array(1), tag: 't', ...log }); +out.probe = await h.probe(); +out.destroy = await destroy(); +console.log(JSON.stringify(out)); +") + +(defn- run-generated-handler + "Write a generated handler, the fixture overrides and a driver into a new + dir under target/, and run the driver with node. The dir is inside the + repo, so the bare specifier ffi-wasm/handler resolves to this package." + [& driver-args] + (let [dir (.toFile (java.nio.file.Files/createTempDirectory + (.toPath (doto (io/file "target") .mkdirs)) + "gen-handler-run-" + (make-array java.nio.file.attribute.FileAttribute 0)))] + (try + (io/copy (io/file "test/fixtures/gen-handler-overrides.mjs") (io/file dir "overrides.mjs")) + (g/write-handler! {} run-overrides (io/file dir "handler.mjs")) + (spit (io/file dir "driver.mjs") run-driver) + (let [{:keys [exit out err]} (apply sh/sh "node" "driver.mjs" (concat driver-args [:dir dir]))] + {:exit exit :out out :err err + :result (last (str/split-lines out))}) + (finally + (doseq [f (reverse (file-seq dir))] (.delete f)))))) + +;; The worker bundle that npm ships. test:clj runs build:js first. +(def ^:private ffi-wasm-handler-url + (str (.toUri (.toPath (io/file "dist/handler.mjs"))))) + +(def ^:private expected-probe + (str "\"probe\":{\"sameFfi\":true,\"makeHandler\":\"function\",\"stageFiles\":\"function\"," + "\"createSyncFetch\":\"function\",\"tag\":\"t\"}")) + +(deftest generated-handler--loads-ffi-wasm-from-the-init-url + (let [{:keys [exit out err result]} (run-generated-handler ffi-wasm-handler-url)] + (is (zero? exit) (str "node failed\nstdout: " out "\nstderr: " err)) + (is (= (str "{\"bad\":\"rejected\"," expected-probe ",\"destroy\":\"destroyed\"}") result) + (str "a URL that does not load rejects create, a later create retries, " + "and init and methods get one ffi-wasm namespace")) + (is (str/includes? out "BUSY-INC] fn=probe label=fixture") + "the :label reaches the trace events of makeHandler"))) + +(deftest generated-handler--falls-back-to-the-bare-specifier-with-no-url + (let [{:keys [exit out err result]} (run-generated-handler)] + (is (zero? exit) (str "node failed\nstdout: " out "\nstderr: " err)) + (is (= (str "{" expected-probe ",\"destroy\":\"destroyed\"}") result)))) diff --git a/test/clj/net/willcohen/native/graal_wasm_test.clj b/test/clj/net/willcohen/native/graal_wasm_test.clj index 1fae7ba..09abe76 100644 --- a/test/clj/net/willcohen/native/graal_wasm_test.clj +++ b/test/clj/net/willcohen/native/graal_wasm_test.clj @@ -459,6 +459,16 @@ (is (instance? Long r)) (is (= 3000000000 r) "all 64 bits, above 2^31"))) +(deftest call!-on-graal-passes-and-returns-a-float32 + ;; The f32 export narrows the number, so 0.1 comes back as the nearest f32. + (let [lib (dispatch/library {:key lib-key + :fndefs {:f32_id {:rettype :float32 + :argtypes [[:v :float32]]}} + :impl-atom (atom :graal)}) + r (on-module (dispatch/call! lib :f32_id [0.1]))] + (is (instance? Double r) "a Double, not a Polyglot Value") + (is (= (double (float 0.1)) r)))) + (deftest call!-on-graal-sends-an-int64-argument-as-a-bigint ;; A WASM_BIGINT module rejects a JS number for an i64 parameter. (let [lib (dispatch/library {:key lib-key diff --git a/test/clj/net/willcohen/native/platform_test.clj b/test/clj/net/willcohen/native/platform_test.clj index 4710c61..e320595 100644 --- a/test/clj/net/willcohen/native/platform_test.clj +++ b/test/clj/net/willcohen/native/platform_test.clj @@ -12,6 +12,7 @@ [clojure.test :refer [deftest is testing]] [clojure.tools.logging.test :as lt] [net.willcohen.native.callbacks :as cb] + [net.willcohen.native.dispatch :as dispatch] [net.willcohen.native.platform :as platform] [tech.v3.datatype.ffi :as dt-ffi]) (:import [java.io File] @@ -240,6 +241,38 @@ (println "SKIP init-jdk-library!-leaves-no-class-file-in-the-compile-path:" "no known C library on this OS"))) +(defn- system-math-library + "The C math library of this process, or nil on an OS that this test does + not know. macOS keeps libm in libSystem." + ^File [] + (case (platform/get-os) + :darwin (File. "/usr/lib/libSystem.B.dylib") + :linux (some-> (re-find #"(?m)/\S*/libm\.so\.6$" (#'platform/process-maps)) + (File.)) + nil)) + +(def ^:private math-fndefs + {:fabsf {:rettype :float32 :argtypes [[:x :float32]]}}) + +(def ^:private math-fn-defs (platform/rehydrate-fn-defs math-fndefs)) + +(def ^:private math-state + (atom {:singleton (dt-ffi/library-singleton #'math-fn-defs)})) + +(platform/define-library-fns! math-fn-defs math-state) + +(deftest call!-on-ffi-passes-and-returns-a-float32 + (if-let [libm (system-math-library)] + (let [lib (dispatch/library {:key ::math + :fndefs math-fndefs + :impl-atom (atom :ffi) + :ffi-impl-ns 'net.willcohen.native.platform-test})] + (platform/init-jdk-library! (:singleton @math-state) libm) + (is (= (double (float 0.1)) (dispatch/call! lib :fabsf [-0.1])) + "the argument narrows to f32, and the f32 result comes back")) + (println "SKIP call!-on-ffi-passes-and-returns-a-float32:" + "no known C math library on this OS"))) + (deftest define-callback-interface-leaves-no-class-file-in-the-compile-path (dt-ffi/set-ffi-impl! :jdk) (let [iface (atom nil)] diff --git a/test/cljc/net/willcohen/native/dispatch_test.cljc b/test/cljc/net/willcohen/native/dispatch_test.cljc index bab6173..69d7a9e 100644 --- a/test/cljc/net/willcohen/native/dispatch_test.cljc +++ b/test/cljc/net/willcohen/native/dispatch_test.cljc @@ -24,8 +24,13 @@ ;; squint compiles :number to the string "number", so one assertion holds on ;; both platforms. (deftest argtype->ccall-type-maps-strings-and-numbers - (is (= [:string :string :number :number] - (mapv d/argtype->ccall-type [:string :string? :int64 :pointer?])))) + (is (= [:string :string :number :number :number] + (mapv d/argtype->ccall-type [:string :string? :int64 :pointer? :float32])))) + +(deftest library-accepts-float32 + (is (some? (d/library {:key :f32-lib + :fndefs {:lib_f32 {:rettype :float32 + :argtypes [[:v :float32]]}}})))) (deftest library-validates-fndefs-types-at-build-time ;; argtype->ccall-type maps an unknown type to :number with no error, so @@ -150,6 +155,25 @@ (is (= (js/BigInt "3000000000") (aget @seen 0)) "the :int64 argument is a BigInt") (is (= 1 (aget @seen 1)) "an :int32 argument stays a number")))) +#?(:cljs + (deftest ^:async call!-on-cljs-sends-a-float32-as-a-number + ;; The wasm export narrows a JS number to f32, so ccall takes "number". + (let [seen (atom nil) + fake #js {:worker (fn [_idx] + (js-obj "f32-lib" + #js {:ccall (fn [_fn ret types args] + (reset! seen [ret types args]) + 0.5)}))} + lib (d/library {:key :f32-lib + :fndefs {:lib_f32 {:rettype :float32 + :argtypes [[:v :float32]]}}}) + r (await (d/call! lib :lib_f32 [0.1] #js {:pool fake})) + [ret types args] @seen] + (is (= 0.5 r)) + (is (= "number" ret)) + (is (= "number" (aget types 0))) + (is (= 0.1 (aget args 0)))))) + #?(:cljs (deftest ^:async call!-on-cljs-runs-the-hooks-in-order (let [seen (atom nil) diff --git a/test/cljs/net/willcohen/native/bundle_test.cljc b/test/cljs/net/willcohen/native/bundle_test.cljc new file mode 100644 index 0000000..7682a54 --- /dev/null +++ b/test/cljs/net/willcohen/native/bundle_test.cljc @@ -0,0 +1,110 @@ +;; Copyright (c) 2026 Will Cohen +;; +;; Part of clj-native, under the Apache License v2.0 with LLVM Exceptions. +;; See LICENSE for license information. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(ns net.willcohen.native.bundle-test + "ffi-wasm ships one page bundle, and the worker side and the modules that + it shares with the page as files of their own. One file is one module + instance, so each consumer of ffi-wasm shares the state of pool." + (:require [cljs.test :refer [deftest is]] + ["ffi-wasm/test-runner" :as tr] + ["node:fs" :refer [existsSync readFileSync]] + ["node:path" :refer [dirname join]] + ["node:url" :refer [fileURLToPath]])) + +(def ^:private dist-dir + (dirname (fileURLToPath (.resolve js/import.meta "ffi-wasm")))) + +(def ^:private page-modules + ["pool" "dispatch" "platform_state" "macros" "workload_pool" "handler_runtime" + "handler_env" "handler_fs" "handler_heap" "handler_paths" "http_bridge"]) + +(def ^:private shared-modules + ["handler_runtime" "handler_env" "handler_fs" "handler_heap" "handler_paths" "http_bridge"]) + +(defn- src-module-url [m] + (.-href (js/URL. (str "../../../../../src/cljc/net/willcohen/native/" m ".mjs") + (.-url js/import.meta)))) + +;; A dynamic import gives the same namespace object as a static import. +(defn- ^:async import-ns [spec] + (await (js/import (.resolve js/import.meta spec)))) + +(deftest ^:async every-page-subpath-is-the-page-bundle + (let [ffi (await (import-ns "ffi-wasm"))] + (doseq [spec ["ffi-wasm/pool" "ffi-wasm/dispatch" "ffi-wasm/workload-pool"]] + (is (identical? ffi (await (import-ns spec))) spec)) + (is (not (identical? ffi (await (import-ns "ffi-wasm/test-runner")))) + "the test runner is a file of its own"))) + +(deftest ^:async the-page-bundle-exports-every-name-of-its-modules + ;; export * drops a name that two modules export, with no error. + (let [ffi (await (import-ns "ffi-wasm"))] + (doseq [m page-modules] + (let [mod (await (js/import (src-module-url m)))] + (doseq [k (js/Object.keys mod)] + (is (js/Object.hasOwn ffi k) (str m " exports " k))))))) + +(deftest ^:async the-worker-entry-holds-the-worker-side + (let [ffi (await (import-ns "ffi-wasm")) + h (await (import-ns "ffi-wasm/handler"))] + (is (fn? (.-makeHandler h))) + (is (fn? (.-stageFiles h))) + (is (fn? (.-createSyncFetch h))) + (is (not (identical? ffi h)) "the worker entry is a file of its own"))) + +;; A page or a Node process that loads the page bundle and the worker entry +;; gets one copy of each module that the two share. +(deftest ^:async the-page-and-the-worker-share-one-copy-of-each-module + (let [ffi (await (import-ns "ffi-wasm")) + h (await (import-ns "ffi-wasm/handler"))] + (doseq [m shared-modules] + (let [mod (await (import-ns (str "ffi-wasm/" (.replaceAll m "_" "-"))))] + (doseq [k (js/Object.keys mod)] + (is (identical? (aget mod k) (aget ffi k)) (str m " " k " in ffi-wasm.mjs")) + (is (identical? (aget mod k) (aget h k)) (str m " " k " in handler.mjs"))))))) + +;; A module worker does not use the importmap of the page. +(deftest the-worker-side-imports-no-bare-specifier + (doseq [f (cons "handler.mjs" (map #(str % ".mjs") shared-modules)) + :let [text (readFileSync (join dist-dir f) "utf8")] + [_ spec] (.matchAll text (js/RegExp. "(?:from|import\\()\\s*[\"']([^\"']+)[\"']" "g"))] + (is (or (.startsWith spec "./") (.startsWith spec "node:")) (str f ": " spec)))) + +;; init-pool! and createSyncFetch find these next to the bundle. +(deftest the-bundle-files-sit-together + (doseq [f ["handler.mjs" "fetch_worker.mjs" "test_runner.mjs"]] + (is (existsSync (join dist-dir f)) f))) + +;; Each library ships its own code only. A page loads worker-router and +;; comlink from their own packages, one copy each. +(deftest dist-holds-no-copy-of-another-library + (doseq [f ["worker-bootstrap.mjs" "comlink.mjs"]] + (is (not (existsSync (join dist-dir f))) f))) + +;; The bundle holds ffi-wasm's own modules only. A copy of another package +;; is a second module instance next to the copy that a consumer imports. +(deftest the-bundles-inline-no-other-package + (doseq [f ["ffi-wasm.mjs" "handler.mjs"] + :let [text (readFileSync (join dist-dir f) "utf8")]] + (is (not (.includes text "// node_modules/")) f)) + (let [text (readFileSync (join dist-dir "ffi-wasm.mjs") "utf8")] + (is (.includes text "from \"squint-cljs/core.js\"")) + (is (.includes text "from \"worker-router\"")) + (is (not (.includes text "squint-cljs/src/squint/test.js")) + "a page maps no importmap entry for the squint test library"))) + +;; A consumer that bundles a dist file marks node:* as external, so a Node +;; builtin with no node: prefix fails its build. +(deftest the-dist-files-import-each-node-builtin-with-the-node-prefix + (doseq [f (concat ["ffi-wasm.mjs" "handler.mjs" "fetch_worker.mjs"] + (map #(str % ".mjs") shared-modules)) + :let [text (readFileSync (join dist-dir f) "utf8")] + b ["worker_threads" "fs" "path" "url"] + q ["\"" "'"] + form [(str "import(" q b q ")") (str "from " q b q)]] + (is (not (.includes text form)) (str f ": " form)))) + +(tr/run-tests-and-exit! "net.willcohen.native.bundle-test") diff --git a/test/cljs/net/willcohen/native/http_bridge_test.cljc b/test/cljs/net/willcohen/native/http_bridge_test.cljc index 0558344..9a9b3c8 100644 --- a/test/cljs/net/willcohen/native/http_bridge_test.cljc +++ b/test/cljs/net/willcohen/native/http_bridge_test.cljc @@ -221,4 +221,10 @@ (is (false? (await (shutdown))) "the release of the crashed consumer keeps the new worker") (is (= 200 (.-status (fresh (str base "/plain"))))))))))) +(deftest ^:async with-no-workerUrl-node-starts-the-fetch-worker-next-to-the-bridge + (await (with-echo-server + (fn ^:async default-worker [base] + (let [sync-fetch (await (createSyncFetch))] + (is (= "/default" (.-url (echoed (sync-fetch (str base "/default") #js {})))))))))) + (tr/run-tests-and-exit! "net.willcohen.native.http-bridge-test") diff --git a/test/cljs/net/willcohen/native/pool_test.cljc b/test/cljs/net/willcohen/native/pool_test.cljc index dee785f..09fc1ab 100644 --- a/test/cljs/net/willcohen/native/pool_test.cljc +++ b/test/cljs/net/willcohen/native/pool_test.cljc @@ -26,7 +26,9 @@ get_pool_stats fire_and_capture_dispose_BANG_ flush_pending_disposes_BANG_ - set_log_config_BANG_]] + set_log_config_BANG_ + init_pool_BANG_ + worker_call]] ["ffi-wasm/handler-runtime" :refer [isEnabled]] ["ffi-wasm/test-runner" :as tr])) @@ -209,4 +211,25 @@ (is (isEnabled "BUSY-INC")) (set_log_config_BANG_ nil)) +(def ^:private registry-handler-url + (.-href (js/URL. "../../../../fixtures/registry-handler.mjs" (.-url js/import.meta)))) + +;; A module worker ignores the page importmap, so the init args carry the URL. +(deftest ^:async init-pool!-gives-each-handler-the-url-of-ffi-wasm-handler + (let [init #js {:tag "a"} + p (await (init_pool_BANG_ {:handlers #js {"lib-a" #js {:module registry-handler-url + :init init}} + :size 1}))] + (try + (let [url (await (worker_call p "lib-a" "init_arg" #js ["ffiWasmHandlerUrl"] 0))] + (is (= (.-href (js/URL. "./handler.mjs" (.resolve js/import.meta "ffi-wasm/pool"))) url) + "the handler.mjs next to the pool.mjs of the page") + (is (fn? (.-makeHandler (await (js/import url)))))) + (is (= "pong:a" (await (worker_call p "lib-a" "ping" #js [] 0))) + "the rest of the init arrives") + (is (not (js/Object.hasOwn init "ffiWasmHandlerUrl")) + "the caller's init object does not change") + (finally + (await (.terminate p)))))) + (tr/run-tests-and-exit! "net.willcohen.native.pool-test") diff --git a/test/fixtures/gen-handler-overrides.mjs b/test/fixtures/gen-handler-overrides.mjs new file mode 100644 index 0000000..e9ab88a --- /dev/null +++ b/test/fixtures/gen-handler-overrides.mjs @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Will Cohen +// +// Part of clj-native, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Test fixture: the overrides module of a generated handler. probe reports +// what the handler gave the overrides, so gen_handler_test can check it. +let initFfi = null; +let initArgs = null; + +export async function init(args, ctx) { + initFfi = ctx.ffi; + initArgs = args; +} + +export const methods = (ffi) => ({ + probe: async () => ({ + sameFfi: ffi === initFfi, + makeHandler: typeof ffi.makeHandler, + stageFiles: typeof ffi.stageFiles, + createSyncFetch: typeof ffi.createSyncFetch, + tag: initArgs?.tag ?? null, + }), +}); + +export function destroy() { + return 'destroyed'; +} diff --git a/test/fixtures/registry-handler.mjs b/test/fixtures/registry-handler.mjs index 0545cad..30d56bd 100644 --- a/test/fixtures/registry-handler.mjs +++ b/test/fixtures/registry-handler.mjs @@ -6,10 +6,12 @@ // Test fixture: the smallest worker-router handler module. ping echoes the // :args tag, so a test can tell which library's init payload arrived. +// init_arg reads one field of the init args that the worker got. export function create(initArgs) { const tag = (initArgs && initArgs.tag) || ''; return { ping: () => `pong:${tag}`, + init_arg: (k) => initArgs?.[k] ?? null, }; } diff --git a/test/fixtures/wasm-heap-loader.mjs b/test/fixtures/wasm-heap-loader.mjs index 63fb069..a5fae74 100644 --- a/test/fixtures/wasm-heap-loader.mjs +++ b/test/fixtures/wasm-heap-loader.mjs @@ -71,8 +71,19 @@ function utf8Encode(str, heap, ptr, maxBytes) { return n; } +// (module (func (export "f32_id") (param f32) (result f32) local.get 0)) +// The export narrows a JS number to f32, as an f32 parameter of emscripten does. +const F32_BYTES = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, // magic, version 1 + 0x01, 0x06, 0x01, 0x60, 0x01, 0x7d, 0x01, 0x7d, // type 0 = (f32)->f32 + 0x03, 0x02, 0x01, 0x00, // func 0 has type 0 + 0x07, 0x0a, 0x01, 0x06, 0x66, 0x33, 0x32, 0x5f, 0x69, 0x64, 0x00, 0x00, // export "f32_id" + 0x0a, 0x06, 0x01, 0x04, 0x00, 0x20, 0x00, 0x0b, // body: local.get 0 +]); + function makeModule() { const instance = new WebAssembly.Instance(new WebAssembly.Module(MODULE_BYTES)); + const f32 = new WebAssembly.Instance(new WebAssembly.Module(F32_BYTES)); const { memory, malloc, free } = instance.exports; const buffer = memory.buffer; @@ -148,6 +159,8 @@ function makeModule() { } return rettype === 'string' ? M.UTF8ToString(ptr) : ptr; } + case 'f32_id': + return f32.exports.f32_id(vals[0]); // An i64 result, as a WASM_BIGINT module gives it: a BigInt. case 'i64_ret': return 3000000000n;