diff --git a/flake.nix b/flake.nix index 5f12f3c6e..ffe869943 100644 --- a/flake.nix +++ b/flake.nix @@ -296,11 +296,13 @@ ''; }; # `just ts bench` adds Chrome and Firefox as engines, and `just ffi browsers` runs the - # web binding's test in them. Apart for their size. + # web binding's test in them. Apart for their size. Binaryen's wasm2js lowers the + # bench's wasm32 build to JavaScript for its wasm2js route. devShells.bench = self.devShells.${system}.default.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.chromium pkgs.firefox + pkgs.binaryen ]; # `just ffi bench` builds moq's rs/moq-c, whose VAAPI bindings run bindgen. LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; diff --git a/quest/m2/wasm/routes.md b/quest/m2/wasm/routes.md index 505beac02..c5cc72aeb 100644 --- a/quest/m2/wasm/routes.md +++ b/quest/m2/wasm/routes.md @@ -1,49 +1,42 @@ -# [M] Every route on the browser benchmark +# [M] The binding benchmark's cases through the translation ## Goal -`just ts bench` ([the TS benchmark](/ts/README.md#bench)) times each workload through two -more routes, in Chrome, Firefox, Node, and Bun: the fixture's `pub` API exported by -[rust2-wasm](/ffi/README.md#typescript-web), called through its generated binding, and the -same `.wasm` lowered to JavaScript by Binaryen's `wasm2js`. Each route's output must match -native Rust's first, and the report gives each route's shipped bytes, raw and gzipped, -with its `.wasm` or runtime included, its cold start, from a fresh worker to its first -result with `WebAssembly.compile` or the module's evaluation inside the time, and how long -it takes to build. [The binding benchmark](/bench/ffi/README.md)'s cases, which cross -handles, records, objects, reads, and payloads, run through the translation too, beside -rust2-wasm's TypeScript column there. +[The binding benchmark](/bench/ffi/README.md)'s cases, which cross handles, records, +objects, reads, and payloads, run through rust2-ts's translation of the same moq API too, +beside rust2-wasm's TypeScript column there, in Node, Chromium, and Firefox, each case's +result matching the export's first. With [the TS benchmark](/ts/README.md#bench)'s routes, +this is the measurement [the gate](/quest/m2/wasm/README.md) reads. ## Plan -Decided with the maintainer on 2026-11-09: this is the measurement -[the gate](/quest/m2/wasm/README.md) reads. - -- The benchmark's existing wasm32 route stays, as compiled Rust with a hand-written - boundary whose copies are timed apart: against it, the rust2-wasm route shows what the - generated binding costs. -- `wasm2js` is the control no one has measured: it needs no translator and no WebAssembly - at run time, which is what moq's - [`rs2ts-wasm2js`](https://github.com/moq-dev/moq/blob/main/quest/m5/rs2ts-wasm2js.md) - planned to try first. No published speed or size figures for it were found. If it - matches the translation on speed and size, that is a finding about the translator, to - report plainly. -- What is published points both ways, so only this benchmark decides: a Rust and WASM - parser lost 2.2 to 4.6 times to a TypeScript rewrite on boundary costs - ([OpenUI, 2026-03](https://www.openui.com/blog/rust-wasm-parser)), while Scala.js on - Wasm won up to 5.8 times on hashing and lost on object-heavy workloads - ([VirtusLab, 2024-07](https://virtuslab.com/blog/scala/scala-to-web-assembly/)). -- Decided with the maintainer on 2026-11-12, in the quest audit: ✅ the routes are built - alike. Today `ts/Cargo.toml`'s `wasm-release` profile is release at `opt-level = "z"`, - where overflow wraps and debug assertions are off, while `ts/src/cli.rs` translates - from the dev profile, with both on. Add a wasm route built with overflow checks and - debug assertions, or a translation with release semantics, so each pair compared runs - the same checks; the report says which pair it compares. -- The existing benchmark crosses the boundary once per sample, since every route is one - `run(&[u8]) -> Vec` (`ts/bench/entry.ts`), which is why the binding benchmark's - cases join it. -- moq's `js/net/bench/frames.ts` workload is [the model epic](/quest/m2/ts-model/README.md)'s - to add once the model translates, on every route here. -- `wasm2js` needs Binaryen, which the flake does not have yet, and it legalizes `i64` - exports, so the generated binding may need its own entry for that route. +Decided with the maintainer on 2026-11-09: this is the measurement the gate reads. + +Decided with the maintainer on 2026-11-17, on +[#860](https://github.com/kixel-ai/rust2/pull/860#issuecomment-6947175990): + +- ✅ `just ts bench`'s routes land in #860, and this quest keeps only the binding benchmark's + cases through the translation, blocked on the model epic (rejected: holding #860 until the + model translates; dropping these cases). +- ✅ The `wasm2js` route lowers the hand-boundary `wasm checked` build, which isolates the + lowering's cost (rejected: teaching the generated binding to load a lowered rust2-wasm + module, which needs a new `load` source and `i64` legalization on the binding). + +- [#860](https://github.com/kixel-ai/rust2/pull/860) gave `just ts bench` its rust2-wasm and + `wasm2js` routes, each route's build time and cold start, and the `wasm-checked` profile, + so the codec's and the handoff's routes run the same checks. What is left needs moq's + model translated: the cases call `TrackProducer`, `GroupProducer`, `GroupConsumer`, + `FrameFrame`, and `Timestamp` (`ffi/moq/bench/export.ts`), which no translation reaches + until [the model epic](/quest/m2/ts-model/README.md) lands. +- Translate with the model fixture's roots (`ts/moq/model.toml`) and port `export.ts`'s + cases onto the translation's API, which [the epic](/quest/m2/wasm/README.md)'s 2026-11-12 + decision made the binding's too, so the port renames imports rather than rewriting calls. +- Report the translation as one more column in `bench/ffi/report.py`'s TypeScript table. + The translation keeps the overflow checks the export's release build drops, so the + report says which checks each column runs, as `just ts bench`'s does. Public API: none. + +## Required + +- [Translate moq-net's model](/quest/m2/ts-model/README.md) - the translation reaches the handles, groups, and frames the cases call diff --git a/ts/Cargo.lock b/ts/Cargo.lock index cc9ad1121..2cbe11e3c 100644 --- a/ts/Cargo.lock +++ b/ts/Cargo.lock @@ -74,6 +74,13 @@ dependencies = [ "tracing", ] +[[package]] +name = "codec-web" +version = "0.0.0" +dependencies = [ + "codec-fixture", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -705,6 +712,13 @@ dependencies = [ name = "task-kio" version = "0.0.0" +[[package]] +name = "task-web" +version = "0.0.0" +dependencies = [ + "task-fixture", +] + [[package]] name = "thiserror" version = "2.0.21" diff --git a/ts/Cargo.toml b/ts/Cargo.toml index e1172f0c0..3be801683 100644 --- a/ts/Cargo.toml +++ b/ts/Cargo.toml @@ -19,6 +19,7 @@ members = [ "fixtures/bind", "fixtures/census", "fixtures/codec", + "fixtures/codec/web", "fixtures/crates", # A fixture per module of the language, beside what they share. "fixtures/lang/*", @@ -27,6 +28,7 @@ members = [ "fixtures/share", "fixtures/stock", "fixtures/task", + "fixtures/task/web", "fixtures/tokio", ] resolver = "3" @@ -67,3 +69,10 @@ lto = "fat" codegen-units = 1 strip = "symbols" opt-level = "z" + +# wasm-release with the checks the translation keeps, overflow checks and debug assertions, +# so the bench's wasm32 routes built with it run what the translated route runs. +[profile.wasm-checked] +inherits = "wasm-release" +overflow-checks = true +debug-assertions = true diff --git a/ts/README.md b/ts/README.md index 2cdec2509..0922d0fda 100644 --- a/ts/README.md +++ b/ts/README.md @@ -1436,28 +1436,41 @@ does, whose products pass 2^53, then added by its `checked_add` and ordered by i whose cross products do too, integers sorted, searched for their greatest and least, and kept in a `BinaryHeap`, TS packets numbered through `as_chunks_mut` as moq-mux numbers them, each also read by value, a copy of its bytes, and frames' `Bytes` split off a -group's, cloned, sliced, and dropped, as moq-net's model hands them to its consumers. Each runs four ways: natively, as wasm32 at the -size-optimized profile moq's WASM shipped with, hand-written (moq's `js/net` in `bench/moq.ts` and its `Timestamp` +group's, cloned, sliced, and dropped, as moq-net's model hands them to its consumers. Each runs seven ways: natively; as wasm32 at the +size-optimized profile moq's WASM shipped with, through `fixtures/wasm.rs`'s hand-written +boundary; the same at `wasm-checked`, that profile with the overflow checks and debug +assertions the translation keeps; that build lowered to JavaScript by Binaryen's `wasm2js`; +through [rust2-wasm](../ffi/README.md#typescript-web), the export of a facade whose `pub` +API is `bench::run` alone (`fixtures/codec/web`), built at `wasm-checked` for +`wasm32-wasip1` and called through its generated TypeScript; hand-written (moq's `js/net` in `bench/moq.ts` and its `Timestamp` in `bench/timestamp.ts`, ES iterator helpers in `bench/iter.ts`, a sorted array in `bench/sorted.ts`, `bigint` in `bench/rescale.ts`, as moq's `js/hang` converts timescales, typed arrays and a heap of its own in `bench/order.ts`, offsets into one -`Uint8Array` in `bench/packets.ts`, and `subarray`s in `bench/frames.ts`), and translated; every route's output must match -native Rust's first. The JS routes run minified, the same bundles whose size it reports, -in Node, Bun, Chrome, and Firefox, with the WASM route's copies across the boundary timed +`Uint8Array` in `bench/packets.ts`, and `subarray`s in `bench/frames.ts`); and translated. Every route's output must match +native Rust's first. The four routes the translation is compared with run the same checks, +so rust2-wasm against `wasm checked` is the generated binding's cost, `wasm2js` against it +the lowering's, and `wasm checked` against `wasm` the checks'. The JS routes run minified, the same bundles whose size it reports, +in Node, Bun, Chrome, and Firefox, with the reference WASM route's copies across the boundary timed apart from its compute, and an A/A copy of the -translation to show the noise. Each route runs alone in a worker of its own, since routes +translation to show the noise. `bench/build.ts` builds every route from the fixtures' +source, timing each build from an emptied target directory without a compiler cache, and +each round's fresh worker times its route's cold start, from loading its files to its +first output, compiling its `.wasm` or evaluating its JavaScript included; each `.wasm` is +compiled with a custom section of its own, since V8 shares a module's code across a +process. Each route runs alone in a worker of its own, since routes in one realm share type profiles and run slower loaded second, and each round starts every route's worker afresh: a worker keeps the inlining and deopts it settles into for the whole run, which put identical copies of a route up to 1.6x apart in Chrome, so a round is one sample of each workload from fresh workers. Each round warms up and times the routes and workloads in a new random order, so no route keeps a slot that runs it faster or slower than identical code in another; the report records the seed. It also records the host and engine versions, -and each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow +and each route's shipped bytes raw and gzipped, rust2-wasm's `.wasm` and binding together. It sets no threshold; the nightly workflow runs it and posts the report to its summary. Each browser's page reports each step it starts, and one that runs 30 s fails the run, naming it. The task fixture's `bench::run` times an async handoff: a value sent to a task parked on -its channel, read through 1, 4, or 16 async functions. Its translation runs polled at +its channel, read through 1, 4, or 16 async functions, through the same wasm32 routes, its +facade in `fixtures/task/web`. Its translation runs polled at once, as natively, and on `rs.spawn`; its hand-written route is the same handoff with promises, as moq's TypeScript writes it. @@ -1476,4 +1489,4 @@ workloads still settle into one of two speeds per worker, up to 2x apart, so on their A/A stayed 6-35% wide while the median workload's was 1-3%. The report's A/A line also gives the noise of a family's geometric mean, the steadier gauge. -The bench shell adds Chromium and Firefox. +The bench shell adds Chromium, Firefox, and Binaryen. diff --git a/ts/bench/build.ts b/ts/bench/build.ts new file mode 100644 index 000000000..2b3b20524 --- /dev/null +++ b/ts/bench/build.ts @@ -0,0 +1,145 @@ +// Builds each fixture's routes from its Rust source, timing each step as a consumer's first +// build: from an emptied target directory, without the dev shell's compiler cache, in wall +// and CPU seconds, its processes' children included. + +import { copyFile, rm } from "node:fs/promises"; +import { join } from "node:path"; + +/** A step's cost, in seconds. */ +export interface Step { + step: string; + wall: number; + cpu: number; +} + +/** What `build` made, as paths, and what each step cost. */ +export interface Built { + /** The wasm32 build at the profile moq's WASM shipped with. */ + wasm: string; + /** The same at `wasm-checked`, which keeps the translation's checks. */ + checked: string; + /** `checked` as wasm2js lowered it: an ES module of its exports. */ + lowered: string; + /** rust2-ffi's export of the fixture's facade, whose `web/` the rust2-wasm route imports. */ + export: string; + /** The export's module, built at `wasm-checked`. */ + exported: string; + /** The translation. */ + translated: string; + steps: Step[]; +} + +/** The translators, and where the builds go. */ +export interface Tools { + /** rust2-ts's release binary. */ + ts: string; + /** rust2-ffi's release binary. */ + ffi: string; + /** ts/, which `fixtures/` is in. */ + root: string; + /** Where the routes' files go. */ + out: string; + /** The target directory, emptied before each step. */ + scratch: string; +} + +// The drivers link the toolchain's own libraries, as rustc does. +const sysroot = Bun.spawnSync(["rustc", "--print", "sysroot"], { stderr: "inherit" }).stdout.toString().trim(); +const libpath = process.platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"; + +// wasm2js validates against the MVP unless told what rustc turns on for wasm32. +const features = [ + "--enable-bulk-memory", + "--enable-bulk-memory-opt", + "--enable-nontrapping-float-to-int", + "--enable-sign-ext", + "--enable-mutable-globals", + "--enable-reference-types", + "--enable-multivalue", +]; + +/** + * Builds the routes of `fixtures/` (`codec` or `task`), the crate `-fixture` and + * its facade in `web/`, into `tools.out`, and its translation into the directory `translated`. + */ +export async function build(name: string, translated: string, tools: Tools): Promise { + const { root, out, scratch } = tools; + const steps: Step[] = []; + const step = async (step: string, cmd: string[]) => steps.push({ step, ...(await timed(cmd, root, scratch)) }); + const built: Built = { + wasm: join(out, `${name}.wasm`), + checked: join(out, `${name}-checked.wasm`), + lowered: join(out, `${name}-wasm2js.mjs`), + export: join(out, `${name}-web`), + exported: join(out, `${name}-web.wasm`), + translated, + steps, + }; + + const wasm = (profile: string) => + step(profile === "wasm-release" ? "wasm" : "wasm checked", [ + "cargo", + "build", + "--locked", + "--quiet", + "--package", + `${name}-fixture`, + "--features", + "bench", + "--target", + "wasm32-unknown-unknown", + "--profile", + profile, + ]); + await wasm("wasm-release"); + await copyFile(join(scratch, "wasm32-unknown-unknown/wasm-release", `${name}_fixture.wasm`), built.wasm); + await wasm("wasm-checked"); + await copyFile(join(scratch, "wasm32-unknown-unknown/wasm-checked", `${name}_fixture.wasm`), built.checked); + await step("wasm2js", ["wasm2js", "-O", ...features, built.checked, "-o", built.lowered]); + + const facade = join(root, "fixtures", name, "web"); + await rm(built.export, { recursive: true, force: true }); + await step("rust2-ffi", [tools.ffi, "--config", join(facade, "ffi.toml"), "--target", "wasm32-wasip1", "--out", built.export]); + await step("export's build", [ + "cargo", + "build", + "--locked", + "--quiet", + "--manifest-path", + join(built.export, "Cargo.toml"), + "--target", + "wasm32-wasip1", + "--profile", + "wasm-checked", + "--features", + "web", + ]); + await copyFile(join(scratch, "wasm32-wasip1/wasm-checked", `${name}_web.wasm`), built.exported); + + await rm(built.translated, { recursive: true, force: true }); + await step("rust2-ts", [tools.ts, "--config", join(root, "fixtures", name, "ts.toml"), "--out", built.translated]); + return built; +} + +/** Wall and CPU seconds of `cmd` in `cwd`, from an emptied `scratch` target, uncached. */ +async function timed(cmd: string[], cwd: string, scratch: string): Promise> { + // Removed, not emptied: cargo cleans only a target directory it made. + await rm(scratch, { recursive: true, force: true }); + const env: Record = { ...process.env, CARGO_TARGET_DIR: scratch, [libpath]: join(sysroot, "lib") }; + // mbx's Cargo shim keeps a compiler cache that emptying the target does not empty: the + // wrapper is its, and CARGO would name the shim to the drivers. + const shim = process.env.RUST2_CARGO_SHIM; + env.PATH = (process.env.PATH ?? "") + .split(":") + .filter((dir) => dir !== shim) + .join(":"); + delete env.RUSTC_WRAPPER; + delete env.CARGO; + const start = performance.now(); + const proc = Bun.spawn(cmd, { cwd, env, stdout: "inherit", stderr: "inherit" }); + if ((await proc.exited) !== 0) throw new Error(`${cmd.join(" ")} failed`); + const wall = (performance.now() - start) / 1000; + const usage = proc.resourceUsage(); + if (!usage) throw new Error(`${cmd[0]} reported no resource usage`); + return { wall, cpu: Number(usage.cpuTime.total) / 1e6 }; +} diff --git a/ts/bench/entry.ts b/ts/bench/entry.ts index 604367bce..262010659 100644 --- a/ts/bench/entry.ts +++ b/ts/bench/entry.ts @@ -88,7 +88,7 @@ function work() { request, reply, (file) => import(new URL(file, import.meta.url).href), - async (file) => new Wasm(await WebAssembly.compile(await load(file))), + load, ); if (threads) threads.parentPort?.once("message", start); else onmessage = (e: MessageEvent) => start(e.data); @@ -115,20 +115,29 @@ async function main() { // another, and runs in a worker of its own. The A/A route is the translation again: its // spread from the first is the measurement's noise. const route = (route: string, file: string, exported = "run", slice = false): Subject => ({ route, file, exported, slice }); + // The wasm32 builds: the reference, the build with the translation's checks, and that + // build lowered to JavaScript. + const builds = (prefix: string): Subject[] => [ + { route: "wasm", wasm: `${prefix}.wasm`, lowered: false, copies: true }, + { route: "wasm checked", wasm: `${prefix}-checked.wasm`, lowered: false, copies: false }, + { route: "wasm2js", wasm: `${prefix}-wasm2js.js`, lowered: true, copies: false }, + ]; const wasm = new Wasm(await WebAssembly.compile(await load("bench.wasm"))); const routes = [ - { wasm: "bench.wasm" }, + ...builds("bench"), + route("rust2-wasm", "rust2-wasm.js"), route("hand-written", "moq.js"), route("translated", "translated.js", "run", true), route("A/A", "translated-aa.js", "run", true), ]; if (config.route) routes.push(route(config.route, "route.js", "run", true)); - const timings = await measure(wasm, routes, parse(config.workloads), spawn, options); + const codec = await measure(wasm, routes, parse(config.workloads), spawn, options); // The async handoff: the translation polled at once by a one-task executor, as the // native and WASM routes are, and on the runtime's microtask executor. const handoffRoutes = [ - { wasm: "task.wasm" }, + ...builds("task"), + route("rust2-wasm", "task-rust2-wasm.js"), route("hand-written", "task-hand.js", "handWritten"), route("translated", "task-translated.js"), route("spawned", "task-spawned.js", "spawn"), @@ -136,8 +145,8 @@ async function main() { ]; if (config.route) handoffRoutes.push(route(config.route, "task-route.js")); const taskWasm = new Wasm(await WebAssembly.compile(await load("task.wasm"))); - timings.push(...(await measure(taskWasm, handoffRoutes, parse(config.handoffs), spawn, options))); - return { engine: engine(), timings }; + const handoff = await measure(taskWasm, handoffRoutes, parse(config.handoffs), spawn, options); + return { engine: engine(), timings: [...codec.timings, ...handoff.timings], cold: { codec: codec.cold, handoff: handoff.cold } }; } if (worker) { diff --git a/ts/bench/fresh.ts b/ts/bench/fresh.ts new file mode 100644 index 000000000..ad12fe477 --- /dev/null +++ b/ts/bench/fresh.ts @@ -0,0 +1,17 @@ +// What every route that compiles a `.wasm` compiles, in the bench's workers and in the +// rust2-wasm route's bundle alike. + +/** + * `module` with a custom section of its own, so no engine serves code it compiled from the + * same bytes in another worker, as V8 shares a module's code across a process. + */ +export function fresh(module: Uint8Array): Uint8Array { + const nonce = crypto.getRandomValues(new Uint8Array(8)); + // A custom section: id 0, its size, then its name's length and name, then its bytes. + const name = new TextEncoder().encode("fresh"); + const section = Uint8Array.of(0, 1 + name.length + nonce.length, name.length, ...name, ...nonce); + const out = new Uint8Array(module.length + section.length); + out.set(module); + out.set(section, module.length); + return out; +} diff --git a/ts/bench/main.ts b/ts/bench/main.ts index b5198953a..d3519cd05 100644 --- a/ts/bench/main.ts +++ b/ts/bench/main.ts @@ -1,9 +1,10 @@ -// `just ts bench`: bundles and minifies each route, times every workload natively and in -// Node, Bun, Chrome, and Firefox, and reports the numbers with each route's bundle size. +// `just ts bench`: builds each route from the fixtures' Rust, timing each build, bundles and +// minifies the JS routes, times every workload natively and in Node, Bun, Chrome, and +// Firefox, and reports the numbers with each route's size, build, and cold start. // -// usage: bun main.ts --translated --wasm --native --moq -// --task --task-wasm --task-native -// --out [--sample-ms 2] [--rounds 25] [--seed ] [--route =] [--filter ] +// usage: bun main.ts --rust2-ts --rust2-ffi --native --task-native +// --moq --out +// [--sample-ms 2] [--rounds 25] [--seed ] [--route =] [--filter ] // // The task fixture's async handoff runs beside the codec, against moq's way with promises. // `--route` times the translations in another run's `--out` as one more route, for A/B runs. @@ -15,23 +16,23 @@ import { parseArgs } from "node:util"; import type { BunPlugin } from "bun"; import { Wasm } from "../tests/wasm.ts"; +import { type Built, build, type Step } from "./build.ts"; import type { Config } from "./entry.ts"; -import { hex, type Timing } from "./time.ts"; +import { type Cold, hex, type Timing } from "./time.ts"; import { handoffs, type Workload, workloads } from "./workloads.ts"; interface Engine { engine: string; timings: Timing[]; + cold: { codec: Cold[]; handoff: Cold[] }; } const { values: args } = parseArgs({ options: { - translated: { type: "string" }, - wasm: { type: "string" }, + "rust2-ts": { type: "string" }, + "rust2-ffi": { type: "string" }, native: { type: "string" }, moq: { type: "string" }, - task: { type: "string" }, - "task-wasm": { type: "string" }, "task-native": { type: "string" }, out: { type: "string" }, "sample-ms": { type: "string", default: "2" }, @@ -55,8 +56,9 @@ const rounds = Number(args.rounds); const seed = args.seed === undefined ? crypto.getRandomValues(new Uint32Array(1))[0] : Number(args.seed); if (!Number.isInteger(seed) || seed < 0 || seed >= 2 ** 32) throw new Error(`--seed takes a u32, not ${args.seed}`); -const codecRoutes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"]; -const handoffRoutes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"]; +const wasmRoutes = ["wasm", "wasm copy-in", "wasm copy-out", "wasm checked", "wasm2js", "rust2-wasm"]; +const codecRoutes = ["native", ...wasmRoutes, "hand-written", "translated", "A/A"]; +const handoffRoutes = ["native", ...wasmRoutes.filter((r) => !r.includes("copy")), "hand-written", "translated", "spawned", "A/A"]; const extra = args.route === undefined ? undefined : await extraRoute(args.route); /** `--route`'s name, and the bundles of the translations in its directory, which must both be there. */ @@ -82,9 +84,16 @@ const moq: BunPlugin = { }, }; -/** `entry` bundled and minified, as a consumer ships it: each route is timed and sized as this. */ -async function bundle(entry: string, plugins: BunPlugin[] = []): Promise> { +/** + * `entry` bundled and minified, as a consumer ships it: each route is timed and sized as + * this. Its wall and CPU time go to `steps` as the step `step`. + */ +async function bundle(entry: string, plugins: BunPlugin[] = [], steps?: Step[], step = ""): Promise> { + const start = performance.now(); + const cpu = process.cpuUsage(); const result = await Bun.build({ entrypoints: [entry], target: "browser", external: ["node:*"], minify: true, plugins }); + const used = process.cpuUsage(cpu); + steps?.push({ step, wall: (performance.now() - start) / 1000, cpu: (used.user + used.system) / 1e6 }); if (result.outputs.length !== 1) throw new Error(`bundling ${entry} made ${result.outputs.length} files`); return new Uint8Array(await result.outputs[0].arrayBuffer()); } @@ -116,10 +125,6 @@ export const run = (input: Uint8Array) => bench(rs.byteSliceFrom(input)); ); return entry; } -const translated = await codecEntry(arg("translated")); -const wasmBytes = await read(arg("wasm")); -const translation = await bundle(translated); -const handBundle = await bundle(handWritten, [moq]); // The task fixture's translation, as `bench/task.ts` drives it. async function taskEntry(dir: string): Promise { @@ -138,50 +143,130 @@ export const spawn = spawned(fixture); ); return entry; } -const taskHand = join(import.meta.dir, "task.ts"); -const taskTranslated = await taskEntry(arg("task")); -const taskWasmBytes = await read(arg("task-wasm")); -const taskTranslation = await bundle(taskTranslated); -const taskHandBundle = await bundle(taskHand); + +// A fixture's export through rust2-ffi, as a consumer calls it: the module, `wasm` beside +// the bundle, loaded once, and `bench::run` through the generated binding, its output +// copied out of the bytes Rust returns, which are then freed. +async function webEntry(built: Built, wasm: string): Promise { + const entry = join(built.export, "bench-entry.ts"); + await Bun.write( + entry, + `import { load, run as call } from "./web/index.ts"; +import { fresh } from ${JSON.stringify(join(import.meta.dir, "fresh.ts"))}; + +const url = new URL(${JSON.stringify(wasm)}, import.meta.url); +const bytes = url.protocol === "file:" ? (await import("node:fs/promises")).readFile(url) : fetch(url).then((r) => r.arrayBuffer()); +await load(fresh(new Uint8Array(await bytes))); + +export function run(input: Uint8Array): Uint8Array { + const out = call(input); + try { + return out.slice(); + } finally { + out[Symbol.dispose](); + } +} +`, + ); + return entry; +} + +// Each fixture's routes, built from its Rust source, each step timed. +const tools = { ts: arg("rust2-ts"), ffi: arg("rust2-ffi"), root: resolve(import.meta.dir, ".."), out: join(out, "build"), scratch: join(out, "scratch") }; +await mkdir(tools.out, { recursive: true }); +const codec = await build("codec", join(out, "translated"), tools); +const task = await build("task", join(out, "task"), tools); +await rm(tools.scratch, { recursive: true, force: true }); + +/** A fixture's routes as they ship, each its files, and what made them. */ +interface Shipped { + wasm: Uint8Array; + checked: Uint8Array; + lowered: Uint8Array; + exported: Uint8Array; + binding: Uint8Array; + hand: Uint8Array; + translation: Uint8Array; + steps: Step[]; +} + +async function ship(built: Built, prefix: string, hand: string, translated: string, plugins: BunPlugin[]): Promise { + const { steps } = built; + return { + wasm: await read(built.wasm), + checked: await read(built.checked), + lowered: await bundle(built.lowered, [], steps, "wasm2js's bundle"), + exported: await read(built.exported), + binding: await bundle(await webEntry(built, `${prefix}rust2-wasm.wasm`), [], steps, "rust2-wasm's bundle"), + hand: await bundle(hand, plugins, steps, "hand-written's bundle"), + translation: await bundle(translated, [], steps, "translated's bundle"), + steps, + }; +} +const codecShip = await ship(codec, "", handWritten, await codecEntry(codec.translated), [moq]); +// The task fixture: its channel, its async functions, and their runtime, as each route ships them. +const taskShip = await ship(task, "task-", join(import.meta.dir, "task.ts"), await taskEntry(task.translated), []); + await Promise.all([ Bun.write(join(out, "entry.js"), await bundle(join(import.meta.dir, "entry.ts"))), - Bun.write(join(out, "moq.js"), handBundle), - Bun.write(join(out, "translated.js"), translation), - Bun.write(join(out, "translated-aa.js"), translation), - Bun.write(join(out, "bench.wasm"), wasmBytes), - Bun.write(join(out, "task-hand.js"), taskHandBundle), - Bun.write(join(out, "task-translated.js"), taskTranslation), - Bun.write(join(out, "task-spawned.js"), taskTranslation), - Bun.write(join(out, "task-translated-aa.js"), taskTranslation), - Bun.write(join(out, "task.wasm"), taskWasmBytes), + ...files("bench", "", codecShip), + ...files("task", "task-", taskShip), + Bun.write(join(out, "moq.js"), codecShip.hand), + Bun.write(join(out, "translated.js"), codecShip.translation), + Bun.write(join(out, "translated-aa.js"), codecShip.translation), + Bun.write(join(out, "task-hand.js"), taskShip.hand), + Bun.write(join(out, "task-translated.js"), taskShip.translation), + Bun.write(join(out, "task-spawned.js"), taskShip.translation), + Bun.write(join(out, "task-translated-aa.js"), taskShip.translation), Bun.write(join(out, "package.json"), '{ "type": "module" }\n'), Bun.write(join(out, "index.html"), page), extra && Bun.write(join(out, "route.js"), extra.codec), extra && Bun.write(join(out, "task-route.js"), extra.task), ]); -const sizes = [ - { route: "wasm", files: [wasmBytes] }, - { route: "hand-written", files: [handBundle] }, - { route: "translated", files: [translation] }, - ...(extra ? [{ route: extra.name, files: [extra.codec] }] : []), -].map(measureSize); +/** The wasm32 builds' files, as `entry.ts` names them. */ +function files(wasm: string, prefix: string, s: Shipped): Promise[] { + return [ + Bun.write(join(out, `${wasm}.wasm`), s.wasm), + Bun.write(join(out, `${wasm}-checked.wasm`), s.checked), + Bun.write(join(out, `${wasm}-wasm2js.js`), s.lowered), + Bun.write(join(out, `${prefix}rust2-wasm.js`), s.binding), + Bun.write(join(out, `${prefix}rust2-wasm.wasm`), s.exported), + ]; +} -// The task fixture: its channel, its async functions, and their runtime, as each route ships them. -const taskSizes = [ - { route: "wasm", files: [taskWasmBytes] }, - { route: "hand-written", files: [taskHandBundle] }, - { route: "translated", files: [taskTranslation] }, - ...(extra ? [{ route: extra.name, files: [extra.task] }] : []), -].map(measureSize); - -function measureSize({ route, files }: { route: string; files: Uint8Array[] }) { - const sum = (f: (b: Uint8Array) => number) => files.reduce((n, b) => n + f(b), 0); - return { route, raw: sum((b) => b.byteLength), gzip: sum((b) => Bun.gzipSync(b, { level: 9 }).byteLength) }; +/** Each route's files, and the build steps that make them. */ +function shipped(s: Shipped, extraFile?: Uint8Array) { + return [ + { route: "wasm", files: [s.wasm], steps: ["wasm"] }, + { route: "wasm checked", files: [s.checked], steps: ["wasm checked"] }, + { route: "wasm2js", files: [s.lowered], steps: ["wasm checked", "wasm2js", "wasm2js's bundle"] }, + { route: "rust2-wasm", files: [s.exported, s.binding], steps: ["rust2-ffi", "export's build", "rust2-wasm's bundle"] }, + { route: "hand-written", files: [s.hand], steps: ["hand-written's bundle"] }, + { route: "translated", files: [s.translation], steps: ["rust2-ts", "translated's bundle"] }, + ...(extra && extraFile ? [{ route: extra.name, files: [extraFile], steps: [] }] : []), + ].map(({ route, files, steps }) => { + const sum = (f: (b: Uint8Array) => number) => files.reduce((n, b) => n + f(b), 0); + const cost = steps.map((name) => { + const step = s.steps.find((t) => t.step === name); + if (!step) throw new Error(`no build step ${name}`); + return step; + }); + return { + route, + raw: sum((b) => b.byteLength), + gzip: sum((b) => Bun.gzipSync(b, { level: 9 }).byteLength), + wall: cost.reduce((n, t) => n + t.wall, 0), + cpu: cost.reduce((n, t) => n + t.cpu, 0), + built: steps.length > 0, + }; + }); } +const sizes = shipped(codecShip, extra?.codec); +const taskSizes = shipped(taskShip, extra?.task); // The workloads, made by the wasm32 build, which also gives each its expected output. -const wasm = new Wasm(await WebAssembly.compile(wasmBytes)); +const wasm = new Wasm(await WebAssembly.compile(codecShip.wasm)); function expected(input: Uint8Array): Uint8Array { const output = wasm.call(input); if (!(output instanceof Uint8Array)) throw new Error(`native Rust traps on ${hex(input)}`); @@ -217,18 +302,14 @@ async function nativeRoute(bin: string, list: Workload[], expected: (input: Uint }); } -async function native(): Promise { - const taskWasm = new Wasm(await WebAssembly.compile(taskWasmBytes)); +async function native(): Promise { + const taskWasm = new Wasm(await WebAssembly.compile(taskShip.wasm)); const handoff = (input: Uint8Array) => { const output = taskWasm.call(input); if (!(output instanceof Uint8Array)) throw new Error(`native Rust traps on ${hex(input)}`); return output; }; - const timings = [ - ...(await nativeRoute(arg("native"), list, expected)), - ...(await nativeRoute(arg("task-native"), handoffList, handoff)), - ]; - return { engine: "native", timings }; + return [...(await nativeRoute(arg("native"), list, expected)), ...(await nativeRoute(arg("task-native"), handoffList, handoff))]; } async function script(command: string): Promise { @@ -301,8 +382,13 @@ async function browser({ name, command, argv, setup }: Browser): Promise const file = Bun.file(join(out, path === "/" ? "index.html" : path.slice(1))); if (!(await file.exists())) return new Response(null, { status: 404 }); // Cross-origin isolated, so performance.now() is precise to 5µs in Chrome and 20µs - // in Firefox, rather than 100µs and 1 ms. - const headers = { "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp" }; + // in Firefox, rather than 100µs and 1 ms. Never stored, so each worker's load of a + // route is cold: no engine caches code it compiled for an earlier one by the response. + const headers = { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", + "Cache-Control": "no-store", + }; return new Response(file, { headers }); }, }); @@ -358,6 +444,7 @@ function render(): string { "", `- Host: ${model}, ${cpu.length} threads, ${(totalmem() / 2 ** 30).toFixed(0)} GiB, ${process.platform} ${release()} ${process.arch}`, `- Rust: ${text(["rustc", "-V"])}`, + `- ${text(["wasm2js", "--version"])}`, `- moq: ${text(["git", "-C", checkout, "rev-parse", "HEAD"])}`, ...engines.map((e) => `- ${e.engine}`), "", @@ -366,30 +453,45 @@ function render(): string { `the JS routes and workloads in a new random order, from seed ${seed}. \`native\` is a release`, "build on the host, its workloads in a fixed order. `wasm` is the same", "Rust at the size-optimized profile moq's WASM shipped with, run with its input already in memory;", - "copying the input in and the output out are timed apart. `hand-written` is moq's", + "copying the input in and the output out are timed apart. `wasm checked` is it at the", + "`wasm-checked` profile, with the overflow checks and debug assertions the translation keeps,", + "and `wasm2js` that build lowered to JavaScript by Binaryen's wasm2js, each run as `wasm` is.", + "`rust2-wasm` is the fixture's `bench::run` exported by rust2-ffi, built at `wasm-checked`, and", + "called through its generated TypeScript, which copies the input in and the output out. So", + "`translated`, `rust2-wasm`, `wasm2js`, and `wasm checked` run the same checks: `rust2-wasm`", + "against `wasm checked` plus `wasm`'s copies is the binding's cost, `wasm2js` against `wasm checked`", + "the lowering's, and `wasm checked` against `wasm` the checks'. `hand-written` is moq's", "`js/net`, called as its own benchmarks call it, for `iter/` the same chain with ES", "iterator helpers (`bench/iter.ts`), for `btree/` a sorted array (`bench/sorted.ts`), for `hash/` a `Map` (`bench/hashed.ts`), and", "for `time/` conversions `bigint`, as moq's `js/hang` converts timescales (`bench/rescale.ts`),", "for `time/checked-add` and `time/ord` moq's own `Timestamp` (`bench/timestamp.ts`), for `fmt/` a template literal (`bench/fmt.ts`), for `parts/` a plain array (`bench/parts.ts`), for `bytes/` subarrays (`bench/frames.ts`), and for `str/`", "moq's Writer and Cursor, a template literal, and a `Map` (`bench/strings.ts`).", - "Each JS route and `wasm` runs alone in a worker of its own, a fresh one each round. `A/A` is the translation again: its", + "Each route runs alone in a worker of its own, a fresh one each round. `A/A` is the translation again: its", "gap from `translated` is the noise. A `reads` decode is the frame headers above as they", "arrive that many bytes at a time, so a header cut by a read comes up short and is", "decoded again on the next: its gap from the whole decode is the error path's price.", ...(extra ? [`\`${extra.name}\` is the translation in another run's output, given by \`--route\`.`] : []), "", - "## Bundle size", + "## Size and build", "", - "Bytes for the same workloads, as timed: minified by Bun where JS.", + "Bytes of each route's own files for the same workloads, as timed: minified by Bun where JS,", + "and for `rust2-wasm` its `.wasm`, its binding, and the loader that fetches the module, while", + "the hand-boundary wasm32 rows are the `.wasm` alone, without the few lines of JS a caller", + "needs to copy its input in and its output out. Each build step is timed from an emptied target", + "directory of its own, without a compiler cache, its dependencies included, in wall and CPU", + "seconds, so a route of two cargo steps pays for its dependencies twice where a consumer's", + "one pipeline would share them: `wasm2js`'s includes `wasm checked`'s build, `rust2-wasm`'s", + "rust2-ffi's export and the export's build, `translated`'s rust2-ts's translation, and every", + "JS route's its bundling by Bun.", "", - "| route | raw | gzip |", - "| --- | ---: | ---: |", - ...sizes.map((s) => `| ${s.route} | ${s.raw} | ${s.gzip} |`), + "| route | raw | gzip | build wall s | build CPU s |", + "| --- | ---: | ---: | ---: | ---: |", + ...sizes.map(sizeRow), ]; const routes = [...codecRoutes, ...(extra ? [extra.name] : [])]; for (const engine of engines) { - const all = [...host.timings, ...engine.timings]; + const all = [...host, ...engine.timings]; const median = (w: Workload, route: string) => stats(find(all, w, route).ns).median; const ratio = (w: Workload) => median(w, "A/A") / median(w, "translated"); const aa = list.map((w) => Math.abs(ratio(w) - 1)).sort((a, b) => a - b); @@ -403,19 +505,54 @@ function render(): string { "", `A/A differs from translated by ${percent(aa[aa.length >> 1])} at the median workload, ${percent(aa[aa.length - 1])} at most, and ${percent(family)} at most over a family's geometric mean.`, "", - `| workload | op | ${routes.join(" | ")} | translated ÷ hand-written |`, - `| --- | --- |${" ---: |".repeat(routes.length + 1)}`, + `| workload | op | ${routes.join(" | ")} | translated ÷ rust2-wasm | translated ÷ hand-written |`, + `| --- | --- |${" ---: |".repeat(routes.length + 2)}`, ...list.map((w) => { const cells = routes.map((r) => cell(find(all, w, r).ns)); - const ratio = (median(w, "translated") / median(w, "hand-written")).toFixed(2); - return `| ${w.name} | ${w.op} | ${cells.join(" | ")} | ${ratio} |`; + const over = (route: string) => (median(w, "translated") / median(w, route)).toFixed(2); + return `| ${w.name} | ${w.op} | ${cells.join(" | ")} | ${over("rust2-wasm")} | ${over("hand-written")} |`; }), ); } - lines.push(...renderHandoffs()); + lines.push(...renderCold(), ...renderHandoffs()); return `${lines.join("\n")}\n`; } +function sizeRow(s: ReturnType[number]): string { + const build = s.built ? `${s.wall.toFixed(2)} | ${s.cpu.toFixed(2)}` : " | "; + return `| ${s.route} | ${s.raw} | ${s.gzip} | ${build} |`; +} + +/** Each route's cold start in each engine. */ +function renderCold(): string[] { + const lines = [ + "", + "## Cold start", + "", + "Milliseconds from a fresh worker's load of a route to its first output, the codec's first", + "workload and the handoff's: fetching or reading its files, compiling a `.wasm` or", + "evaluating its JavaScript, and the first call, the median of a round's worker each, ± half", + "their interquartile range. Each `.wasm` is compiled with a custom section of its own, and", + "a browser is told to store nothing, so no engine reuses code it compiled for another worker.", + ]; + for (const engine of engines) { + const of = (cold: Cold[], route: string) => { + const found = cold.find((c) => c.route === route); + return found ? cell(found.ms) : ""; + }; + const routes = [...new Set([...engine.cold.codec, ...engine.cold.handoff].map((c) => c.route))]; + lines.push( + "", + `### ${engine.engine}`, + "", + "| route | codec | handoff |", + "| --- | ---: | ---: |", + ...routes.map((r) => `| ${r} | ${of(engine.cold.codec, r)} | ${of(engine.cold.handoff, r)} |`), + ); + } + return lines; +} + /** The async handoff on every route, and each route's size. */ function renderHandoffs(): string[] { const routes = [...handoffRoutes, ...(extra ? [extra.name] : [])]; @@ -425,30 +562,36 @@ function renderHandoffs(): string[] { "", "Nanoseconds per handoff: a value sent to a task parked on a kio-shaped channel, which", "wakes it, polls it through `depth` async functions to the channel, and parks it again.", - "`native` and `wasm` run the task fixture's `bench.rs`, which polls the task at once on", - "each wake, and `translated` its translation polled the same way. `spawned` runs the", + "`native` and the wasm32 routes run the task fixture's `bench.rs`, which polls the task at", + "once on each wake, and `translated` its translation polled the same way. `spawned` runs the", "translation on the runtime's `spawn`, which polls woken tasks from a microtask.", "`hand-written` is the handoff with promises and `async` functions, as moq's TypeScript", - "writes it, where each handoff waits for the reader to park again. Bundle sizes are of", - "the channel, the async functions, and their runtime.", + "writes it, where each handoff waits for the reader to park again. Sizes are of", + "the channel, the async functions, and their runtime, and builds as above.", + "`rust2-wasm` can run the handoff faster than `wasm checked`, the same Rust at the same", + "profile. Neither its target nor its allocator does it, since the hand-boundary build", + "for `wasm32-wasip1` runs as the `wasm32-unknown-unknown` one, nor this harness, since", + "both reproduce called directly. The fixture is the cdylib's root in one and a dependency", + "behind the export's `extern \"C\"` function in the other, and the gap narrows at", + "`opt-level = 3`, so it reads as their code generation differing, not pinned down further.", "", - "| route | raw | gzip |", - "| --- | ---: | ---: |", - ...taskSizes.map((s) => `| ${s.route} | ${s.raw} | ${s.gzip} |`), + "| route | raw | gzip | build wall s | build CPU s |", + "| --- | ---: | ---: | ---: | ---: |", + ...taskSizes.map(sizeRow), ]; for (const engine of engines) { - const all = [...host.timings, ...engine.timings]; + const all = [...host, ...engine.timings]; lines.push( "", `### ${engine.engine}`, "", - `| depth | ${routes.join(" | ")} | translated ÷ hand-written |`, - `| --- |${" ---: |".repeat(routes.length + 1)}`, + `| depth | ${routes.join(" | ")} | translated ÷ rust2-wasm | translated ÷ hand-written |`, + `| --- |${" ---: |".repeat(routes.length + 2)}`, ...handoffList.map((w) => { const cells = routes.map((r) => cell(find(all, w, r).ns)); const median = (route: string) => stats(find(all, w, route).ns).median; - const ratio = (median("translated") / median("hand-written")).toFixed(2); - return `| ${w.name.replace("depth ", "")} | ${cells.join(" | ")} | ${ratio} |`; + const over = (route: string) => (median("translated") / median(route)).toFixed(2); + return `| ${w.name.replace("depth ", "")} | ${cells.join(" | ")} | ${over("rust2-wasm")} | ${over("hand-written")} |`; }), ); } diff --git a/ts/bench/time.ts b/ts/bench/time.ts index 59b9cca25..a6f4991de 100644 --- a/ts/bench/time.ts +++ b/ts/bench/time.ts @@ -1,16 +1,18 @@ // Times every workload on every route in one engine, after checking that each route's -// output matches the wasm32 build's, which is native Rust's. The WASM route is timed in -// three parts: copying the input in, running, and copying the output out. +// output matches the wasm32 build's, which is native Rust's. A wasm32 build's route times +// its run alone, and the reference's also copying the input in and the output out. // // Each route runs alone in a realm of its own, a worker, which `measure` drives one // sample at a time. Routes in one realm share its state: JavaScriptCore merges the type // profiles of modules with the same text, a module two routes call sees both routes' // types, and V8 ran a route 2-12% slower loaded after an identical copy than before it. +// Each round's worker is fresh, so its first result is the route's cold start. -import type { Wasm } from "../tests/wasm.ts"; +import { type Exports, Wasm } from "../tests/wasm.ts"; +import { fresh } from "./fresh.ts"; import type { Workload } from "./workloads.ts"; -/** What one realm times: a route's bundle, or the wasm32 build's three steps. */ +/** What one realm times: a route's bundle, or a wasm32 build through `fixtures/wasm.rs`. */ export type Subject = | { route: string; @@ -21,7 +23,14 @@ export type Subject = /** Whether `exported` returns the translation's `{ data, length }` rather than bytes. */ slice: boolean; } - | { wasm: string }; + | { + route: string; + /** The `.wasm`, or with `lowered`, the bundle of wasm2js's lowering of one. */ + wasm: string; + lowered: boolean; + /** Whether copying the input in and the output out are timed apart, as ` copy-in` and `copy-out`. */ + copies: boolean; + }; /** What a realm is told first: what to load, and where later requests arrive. */ export interface Load { @@ -77,6 +86,18 @@ export interface Timing { ns: number[]; } +/** A route's cold starts: milliseconds from a fresh worker's load of it to its first output, one per round. */ +export interface Cold { + route: string; + ms: number[]; +} + +/** What `measure` found. */ +export interface Measured { + timings: Timing[]; + cold: Cold[]; +} + interface Body { workload: Workload; route: string; @@ -107,7 +128,7 @@ export async function measure( workloads: Workload[], spawn: () => Link, options: Options, -): Promise { +): Promise { const { progress, sampleMs } = options; const want = workloads.map((workload) => { const output = wasm.call(workload.input); @@ -116,6 +137,7 @@ export async function measure( }); // Each subject's timings, one per body, which every round's worker lists alike. const timings: Timing[][] = []; + const cold: Cold[] = subjects.map(({ route }) => ({ route, ms: [] })); const span = `${workloads[0].name} through ${workloads[workloads.length - 1].name}`; const random = prng(options.seed); for (let round = 0; round < options.rounds; round++) { @@ -130,7 +152,9 @@ export async function measure( links.push(link); const control = new Int32Array(new SharedArrayBuffer(12)); link.post({ subject, workloads, sampleMs, control }); - const list = (await link.reply()) as { route: string; workload: number }[]; + const loaded = (await link.reply()) as { bodies: { route: string; workload: number }[]; cold: number }; + const list = loaded.bodies; + cold[s].ms.push(loaded.cold); const realm = (op: Op, arg: number) => { control[1] = op; control[2] = arg; @@ -142,7 +166,6 @@ export async function measure( list.forEach(({ workload: w }, index) => { bodies.push({ realm, index, timing: timings[s][index], items: workloads[w].items, calls: 0 }); }); - if (!("route" in subject)) continue; progress(`checking ${subject.route} for ${of}`); for (const [i, workload] of workloads.entries()) { const got = await realm(Op.Output, i); @@ -172,34 +195,45 @@ export async function measure( for (const link of links) link.close(); } } - return timings.flat(); + return { timings: timings.flat(), cold }; } /** - * A realm's side of `measure`: loads `load.subject`, importing a route's bundle with - * `bundle` and instantiating the wasm32 build with `wasm`, then answers each request - * through `reply` until the worker is closed. + * A realm's side of `measure`: loads `load.subject`, importing a bundle with `bundle` and + * compiling a `.wasm`'s bytes from `wasm`, then answers each request through `reply` + * until the worker is closed. Its first reply gives the cold start: from the load to the + * first workload's output. */ export async function serve( load: Load, reply: (reply: Reply) => void, bundle: (file: string) => Promise>, - wasm: (file: string) => Promise, + wasm: (file: string) => Promise, ): Promise { const { subject, workloads, sampleMs, control } = load; const bodies: Body[] = []; const body = (workload: Workload, route: string, call: () => unknown, setup?: () => void) => bodies.push({ workload, route, setup, call, async: false, calls: 1 }); - let output: ((workload: number) => Promise) | undefined; + let output: (workload: number) => Promise; + let cold: number; try { + const start = performance.now(); if ("wasm" in subject) { - const instance = await wasm(subject.wasm); + const instance = subject.lowered + ? new Wasm((await bundle(subject.wasm)) as unknown as Exports) + : new Wasm(await WebAssembly.compile(fresh(await wasm(subject.wasm)))); + output = async (i) => { + const out = instance.call(workloads[i].input); + if (!(out instanceof Uint8Array)) throw new Error(`${subject.route} traps: ${JSON.stringify(out)}`); + return out; + }; for (const workload of workloads) { const { input } = workload; let len = 0; - body(workload, "wasm", () => instance.run(), () => instance.input(input)); - body(workload, "wasm copy-in", () => instance.input(input)); - body(workload, "wasm copy-out", () => instance.output(len), () => { + body(workload, subject.route, () => instance.run(), () => instance.input(input)); + if (!subject.copies) continue; + body(workload, `${subject.route} copy-in`, () => instance.input(input)); + body(workload, `${subject.route} copy-out`, () => instance.output(len), () => { instance.input(input); len = instance.run(); }); @@ -212,15 +246,16 @@ export async function serve( output = async (i) => view((await run(workloads[i].input)) as never); for (const workload of workloads) body(workload, subject.route, () => run(workload.input)); } + await output(0); + cold = performance.now() - start; } catch (err) { return reply(failure(err)); } - reply({ value: bodies.map((b) => ({ route: b.route, workload: workloads.indexOf(b.workload) })) }); + reply({ value: { bodies: bodies.map((b) => ({ route: b.route, workload: workloads.indexOf(b.workload) })), cold } }); async function answer(op: Op, arg: number): Promise { switch (op) { case Op.Output: - if (!output) throw new Error("the wasm32 build is the reference, which has no output to check"); return hex(await output(arg)); case Op.Warm: { const body = bodies[arg]; diff --git a/ts/fixtures/codec/src/lib.rs b/ts/fixtures/codec/src/lib.rs index b605e39a4..7c14210a9 100644 --- a/ts/fixtures/codec/src/lib.rs +++ b/ts/fixtures/codec/src/lib.rs @@ -27,13 +27,15 @@ mod lite { pub use version::Version; } mod version; -#[cfg(target_arch = "wasm32")] +// wasm32-unknown-unknown alone: on wasm32-wasip1, rust2-ffi exports the crate, and a module +// built from that export would carry these exports too. +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] #[path = "../../wasm.rs"] mod wasm; -/// What the wasm32 build exports: the benchmark's workloads with `bench`, else the harness. -#[cfg(all(target_arch = "wasm32", feature = "bench"))] +/// What the wasm32-unknown-unknown build exports: the benchmark's workloads with `bench`, else the harness. +#[cfg(all(target_arch = "wasm32", target_os = "unknown", feature = "bench"))] use bench::run as entry; -#[cfg(all(target_arch = "wasm32", not(feature = "bench")))] +#[cfg(all(target_arch = "wasm32", target_os = "unknown", not(feature = "bench")))] use harness::run as entry; pub use version::*; diff --git a/ts/fixtures/codec/web/Cargo.toml b/ts/fixtures/codec/web/Cargo.toml new file mode 100644 index 000000000..faa1b8fef --- /dev/null +++ b/ts/fixtures/codec/web/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "codec-web" +description = "codec-fixture's benchmark entry alone, which rust2-ffi exports for the bench's rust2-wasm route" +version = "0.0.0" +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +codec-fixture = { path = ".." } + +[lints] +workspace = true diff --git a/ts/fixtures/codec/web/ffi.toml b/ts/fixtures/codec/web/ffi.toml new file mode 100644 index 000000000..35122f437 --- /dev/null +++ b/ts/fixtures/codec/web/ffi.toml @@ -0,0 +1 @@ +# Empty: rust2-ffi takes a config until /quest/m1/ffi-refusals/no-config.md drops it. diff --git a/ts/fixtures/codec/web/src/lib.rs b/ts/fixtures/codec/web/src/lib.rs new file mode 100644 index 000000000..1fb84bc9a --- /dev/null +++ b/ts/fixtures/codec/web/src/lib.rs @@ -0,0 +1,4 @@ +//! `codec-fixture`'s benchmark entry as the whole `pub` API, so rust2-ffi's export of it +//! holds what the bench's other wasm32 routes build and no more. + +pub use codec_fixture::bench::run; diff --git a/ts/fixtures/task/src/lib.rs b/ts/fixtures/task/src/lib.rs index d1fd96767..e1f75ba58 100644 --- a/ts/fixtures/task/src/lib.rs +++ b/ts/fixtures/task/src/lib.rs @@ -23,11 +23,13 @@ pub mod futures; pub mod harness; pub mod log; pub mod model; -#[cfg(target_arch = "wasm32")] +// wasm32-unknown-unknown alone: on wasm32-wasip1, rust2-ffi exports the crate, and a module +// built from that export would carry these exports too. +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] #[path = "../../wasm.rs"] mod wasm; -/// What the wasm32 build exports: the benchmark's handoff with `bench`, else the harness. -#[cfg(all(target_arch = "wasm32", feature = "bench"))] +/// What the wasm32-unknown-unknown build exports: the benchmark's handoff with `bench`, else the harness. +#[cfg(all(target_arch = "wasm32", target_os = "unknown", feature = "bench"))] use bench::run as entry; -#[cfg(all(target_arch = "wasm32", not(feature = "bench")))] +#[cfg(all(target_arch = "wasm32", target_os = "unknown", not(feature = "bench")))] use harness::run as entry; diff --git a/ts/fixtures/task/web/Cargo.toml b/ts/fixtures/task/web/Cargo.toml new file mode 100644 index 000000000..3ff9880a0 --- /dev/null +++ b/ts/fixtures/task/web/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "task-web" +description = "task-fixture's benchmark entry alone, which rust2-ffi exports for the bench's rust2-wasm route" +version = "0.0.0" +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +task-fixture = { path = ".." } + +[lints] +workspace = true diff --git a/ts/fixtures/task/web/ffi.toml b/ts/fixtures/task/web/ffi.toml new file mode 100644 index 000000000..35122f437 --- /dev/null +++ b/ts/fixtures/task/web/ffi.toml @@ -0,0 +1 @@ +# Empty: rust2-ffi takes a config until /quest/m1/ffi-refusals/no-config.md drops it. diff --git a/ts/fixtures/task/web/src/lib.rs b/ts/fixtures/task/web/src/lib.rs new file mode 100644 index 000000000..aab5909e4 --- /dev/null +++ b/ts/fixtures/task/web/src/lib.rs @@ -0,0 +1,4 @@ +//! `task-fixture`'s benchmark entry as the whole `pub` API, so rust2-ffi's export of it +//! holds what the bench's other wasm32 routes build and no more. + +pub use task_fixture::bench::run; diff --git a/ts/justfile b/ts/justfile index 1041c2207..292b275ff 100644 --- a/ts/justfile +++ b/ts/justfile @@ -114,32 +114,28 @@ moq-pin: GIT_NO_LAZY_FETCH=1 git -C target/moq cat-file -e {{ moq_rev }}^{commit} 2> /dev/null || git -C target/moq fetch --quiet origin {{ moq_rev }} git -C target/moq checkout --quiet --detach {{ moq_rev }} -# Time the codec natively, as moq's WASM, as moq's hand-written TS, and translated, in Node, Bun, Chrome, and Firefox; `--route name=dir` adds another run's translations. +# Time the codec natively, as moq's WASM, through rust2-ffi's export, lowered by wasm2js, as moq's hand-written TS, and translated, in Node, Bun, Chrome, and Firefox, with each route's size, build, and cold start; `--route name=dir` adds another run's translations. [arg("route", long)] bench checkout="" route="": (moq checkout) #!/usr/bin/env bash set -euo pipefail - for tool in bun chromium firefox; do + for tool in bun chromium firefox wasm2js; do command -v "$tool" > /dev/null || { echo "$tool is missing: run this in the bench shell, nix develop .#bench" >&2; exit 1; } done src="{{ checkout }}" [ -n "$src" ] || src=target/moq route=() [ -z "{{ route }}" ] || route=(--route "{{ route }}") - out=target/bench - cargo run --locked --quiet -- --config fixtures/codec/ts.toml --out "$out/translated" - cargo build --locked --quiet --package codec-fixture --features bench --target wasm32-unknown-unknown --profile wasm-release + # The translators, built as a consumer installs them; bench/build.ts times each route's + # build with them. + cargo build --locked --quiet --release + cargo build --locked --quiet --release --manifest-path ../ffi/Cargo.toml cargo build --locked --quiet --package codec-fixture --example bench --release - cargo run --locked --quiet -- --config fixtures/task/ts.toml --out "$out/task" - cargo build --locked --quiet --package task-fixture --features bench --target wasm32-unknown-unknown --profile wasm-release cargo build --locked --quiet --package task-fixture --example handoff --release # moq's hand-written TS imports its npm dependencies; its postinstall builds libav. (cd "$src" && bun install --frozen-lockfile --ignore-scripts --filter @moq/net) - bun bench/main.ts --translated "$out/translated" --moq "$src" --out "$out" \ - --wasm target/wasm32-unknown-unknown/wasm-release/codec_fixture.wasm --native target/release/examples/bench \ - --task "$out/task" \ - --task-wasm target/wasm32-unknown-unknown/wasm-release/task_fixture.wasm --task-native target/release/examples/handoff \ - "${route[@]}" + bun bench/main.ts --rust2-ts target/release/rust2-ts --rust2-ffi ../ffi/target/release/rust2-ffi --moq "$src" --out target/bench \ + --native target/release/examples/bench --task-native target/release/examples/handoff "${route[@]}" # Run cargo on the pinned nightly. cargo *args: diff --git a/ts/tests/wasm.ts b/ts/tests/wasm.ts index 0a02f884d..d02d4a962 100644 --- a/ts/tests/wasm.ts +++ b/ts/tests/wasm.ts @@ -1,9 +1,11 @@ // A fixture's wasm32 build (`fixtures/wasm.rs`): the host writes an input into the // instance's memory, runs the crate's entry on it, and copies the output back out. -// `differential.ts` runs it beside the translation, and `bench/time.ts` times each step. +// `differential.ts` runs it beside the translation, and `bench/time.ts` times each step, +// also of the build Binaryen's wasm2js lowered to JavaScript, whose exports are the same. -interface Exports { - memory: WebAssembly.Memory; +/** The module's exports, or a lowered module's, whose memory is an object with a buffer too. */ +export interface Exports { + memory: { readonly buffer: ArrayBuffer }; input(len: number): number; run(): number; output(): number; @@ -30,7 +32,7 @@ export type Glue = Record & { __wbg_set_wasm(exports: unknown): export type Outcome = Uint8Array | { panic: string } | { thrown: string } | "abort"; export class Wasm { - #module: WebAssembly.Module; + #module: WebAssembly.Module | Exports; #exports!: Exports; #glued: boolean; /** What a fixture's panic hooks wrote before the last call trapped, if it exports `note`. */ @@ -38,13 +40,19 @@ export class Wasm { /** Whether the last call left the instance mid-call, which only `renew` replaces for a module with glue. */ stale = false; - constructor(module: WebAssembly.Module, glue?: Glue) { + /** An instance of `module`, or of a lowered module, whose evaluation made its one instance. */ + constructor(module: WebAssembly.Module | Exports, glue?: Glue) { this.#module = module; this.#glued = glue !== undefined; this.#instantiate(glue); } #instantiate(glue?: Glue) { + if (!(this.#module instanceof WebAssembly.Module)) { + if (this.#exports) throw new Error("a lowered module has one instance, which a trap left mid-call"); + this.#exports = this.#module; + return; + } const imports: WebAssembly.Imports = {}; for (const { module } of WebAssembly.Module.imports(this.#module)) { if (glue) imports[module] = glue as WebAssembly.ModuleImports;