diff --git a/quest/m2/README.md b/quest/m2/README.md index 024118fea..abe1b534c 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -20,7 +20,6 @@ is not a goal, and types must be solid at the API boundary ([design](/ts/README. - [MIR lowering spike](/quest/m2/ts-mir-spike.md) - the whole lowering ported to MIR on two fixtures decides whether the translator moves - [Fail the TS benchmark when Chrome hangs](/quest/m2/ts-bench-chrome-timeout.md) - a hung renderer fails the run loudly -- [A/B routes in the TS benchmark](/quest/m2/ts-bench-routes.md) - `--route name=dir` loads another translation for reproducible A/B runs - [Stable names](/quest/m2/ts-stable-names.md) - generic instances and trait-impl methods always carry their suffix, so callers' names never shift - [Diverging calls through closures and generics](/quest/m2/ts-never-edges.md) - closures returning `!` and generics at `!` pass `tsc` - [Cheaper byte slices](/quest/m2/ts-byte-views.md) - splitting a `&[u8]` stops making a `Uint8Array` view per split diff --git a/quest/m2/ts-bench-routes.md b/quest/m2/ts-bench-routes.md deleted file mode 100644 index 3882e8b03..000000000 --- a/quest/m2/ts-bench-routes.md +++ /dev/null @@ -1,15 +0,0 @@ -# [S] A/B routes in the TS benchmark - -## Goal - -`just ts bench --route name=dir` loads another translation as a route beside the -current one, so an A/B comparison of two translator versions is one reproducible -command. - -## Plan - -[#15](https://github.com/kixelated/rust2/pull/15) and -[#26](https://github.com/kixelated/rust2/pull/26) both patched `bench/main.ts` by -hand to time another translation (main's, or a rejected representation's). - -Public API: a `just ts bench` option. diff --git a/quest/m2/ts-wide-fast.md b/quest/m2/ts-wide-fast.md index 8ec5ecb5b..0a6c719aa 100644 --- a/quest/m2/ts-wide-fast.md +++ b/quest/m2/ts-wide-fast.md @@ -17,8 +17,6 @@ From [#26](https://github.com/kixelated/rust2/pull/26)'s follow-ups: multiplying; multiplying their halves directly would cut the allocations `rescale` measures. -Public API: none; the runtime's internals. - -## Related +`just ts bench --route` times each against main's translation in one run. -- [A/B routes in the TS benchmark](/quest/m2/ts-bench-routes.md) - compares each path against the current one +Public API: none; the runtime's internals. diff --git a/ts/README.md b/ts/README.md index 7c6f6565c..5c8be3395 100644 --- a/ts/README.md +++ b/ts/README.md @@ -256,4 +256,14 @@ its channel, read through 1, 4, or 16 async functions. Its translation runs poll once, as natively, and on `rt.spawn`; its hand-written route is the same handoff with promises, as moq's TypeScript writes it. +An A/B run times another translator's output as one more route: run `just ts bench` in +the other checkout, then pass its output directory, absolute or relative to `ts/`. + +```bash +nix develop .#bench --command just ts bench --route main="$HOME/rust2-main/ts/target/bench" +``` + +The route runs both fixtures' translations from that directory's `translated/` and +`task/`, so a hand-edited translation can be timed by copying the directory and editing it. + The bench shell adds Bun, Chromium, and the wasm-bindgen that builds moq-wasm's package. diff --git a/ts/bench/entry.ts b/ts/bench/entry.ts index ed2fcada1..a0253c351 100644 --- a/ts/bench/entry.ts +++ b/ts/bench/entry.ts @@ -13,6 +13,8 @@ export interface Config { workloads: Workloads; /** The async handoff's, on the task fixture. */ handoffs: Workloads; + /** The name of `--route`'s translations, bundled as `route.js` and `task-route.js`. */ + route?: string; } const browser = typeof window !== "undefined"; @@ -55,6 +57,7 @@ async function main() { await route("translated", "translated.js", translated), await route("A/A", "translated-aa.js", translated), ]; + if (config.route) routes.push(await route(config.route, "route.js", translated)); const timings = await measure(wasm, routes, workloads, config); // The async handoff: the translation polled at once by a one-task executor, as the @@ -66,6 +69,7 @@ async function main() { await route("spawned", "task-translated.js", bytes, "spawn"), await route("A/A", "task-translated-aa.js", bytes), ]; + if (config.route) handoffRoutes.push(await route(config.route, "task-route.js", bytes)); const taskWasm = new Wasm(await WebAssembly.compile(await load("task.wasm"))); timings.push(...(await measure(taskWasm, handoffRoutes, parse(config.handoffs), config))); return { engine: engine(), timings }; diff --git a/ts/bench/main.ts b/ts/bench/main.ts index 0a5aca525..cf0d1401b 100644 --- a/ts/bench/main.ts +++ b/ts/bench/main.ts @@ -3,9 +3,10 @@ // // usage: bun main.ts --translated --wasm --native --moq // --moq-wasm --task --task-wasm --task-native -// --out [--sample-ms 10] [--rounds 15] +// --out [--sample-ms 10] [--rounds 15] [--route =] // // The task fixture's async handoff runs beside the codec, against moq's way with promises. +// `--route` times the translations in another run's `--out` as one more route, for A/B runs. import { cpus, release, totalmem } from "node:os"; import { join, resolve } from "node:path"; @@ -35,6 +36,7 @@ const { values: args } = parseArgs({ out: { type: "string" }, "sample-ms": { type: "string", default: "10" }, rounds: { type: "string", default: "15" }, + route: { type: "string" }, }, strict: true, }); @@ -48,6 +50,25 @@ const out = arg("out"); const sampleMs = Number(args["sample-ms"]); const rounds = Number(args.rounds); +const codecRoutes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"]; +const handoffRoutes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"]; +const extra = args.route === undefined ? undefined : await extraRoute(args.route); + +/** `--route`'s name, and the entries of the translations in its directory, which must both be there. */ +async function extraRoute(arg: string): Promise<{ name: string; codec: string; task: string }> { + const at = arg.indexOf("="); + const name = arg.slice(0, at); + if (at < 1 || at === arg.length - 1) throw new Error(`--route takes name=dir, not ${arg}`); + if ([...codecRoutes, ...handoffRoutes].includes(name)) throw new Error(`--route ${name} is already a route`); + const dir = resolve(arg.slice(at + 1)); + const codec = join(dir, "translated", "bench.ts"); + const task = join(dir, "task"); + for (const file of [codec, join(task, "model.ts")]) { + if (!(await Bun.file(file).exists())) throw new Error(`--route ${name}: ${file} is missing`); + } + return { name, codec, task: await taskEntry(task) }; +} + // `moq/` is the checkout, whose own imports resolve through its install. const moq: BunPlugin = { name: "moq", @@ -111,6 +132,8 @@ await Promise.all([ Bun.write(join(out, "task.wasm"), taskWasmBytes), Bun.write(join(out, "package.json"), '{ "type": "module" }\n'), Bun.write(join(out, "index.html"), page), + extra && Bun.write(join(out, "route.js"), await bundle(extra.codec)), + extra && Bun.write(join(out, "task-route.js"), await bundle(extra.task)), ]); const moqWasm = arg("moq-wasm"); @@ -118,6 +141,7 @@ const sizes = [ { route: "wasm", files: [wasmBytes] }, { route: "hand-written", files: [await bundle(handWritten, { minify: true, plugins: [moq] })] }, { route: "translated", files: [await bundle(translated, { minify: true })] }, + ...(extra ? [{ route: extra.name, files: [await bundle(extra.codec, { minify: true })] }] : []), { route: "moq-wasm package", files: [await read(join(moqWasm, "moq_bg.wasm")), await read(join(moqWasm, "moq.js"))] }, ].map(measureSize); @@ -126,6 +150,7 @@ const taskSizes = [ { route: "wasm", files: [taskWasmBytes] }, { route: "hand-written", files: [await bundle(taskHand, { minify: true })] }, { route: "translated", files: [await bundle(taskTranslated, { minify: true })] }, + ...(extra ? [{ route: extra.name, files: [await bundle(extra.task, { minify: true })] }] : []), ].map(measureSize); function measureSize({ route, files }: { route: string; files: Uint8Array[] }) { @@ -147,6 +172,7 @@ const config: Config = { rounds, workloads: list.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), handoffs: handoffList.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), + route: extra?.name, }; await Bun.write(join(out, "workloads.json"), JSON.stringify(config)); @@ -259,6 +285,7 @@ function render(): string { "`js/net`, called as its own benchmarks call it, and for `iter/` the same chain with ES", "iterator helpers (`bench/iter.ts`). `A/A` is the translation loaded a second time: its", "gap from `translated` is the noise.", + ...(extra ? [`\`${extra.name}\` is the translation in another run's output, given by \`--route\`.`] : []), "", "## Bundle size", "", @@ -270,7 +297,7 @@ function render(): string { ...sizes.map((s) => `| ${s.route} | ${s.raw} | ${s.gzip} |`), ]; - const routes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"]; + const routes = [...codecRoutes, ...(extra ? [extra.name] : [])]; for (const engine of engines) { const all = [...host.timings, ...engine.timings]; const median = (w: Workload, route: string) => stats(find(all, w, route).ns).median; @@ -296,7 +323,7 @@ function render(): string { /** The async handoff on every route, and each route's size. */ function renderHandoffs(): string[] { - const routes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"]; + const routes = [...handoffRoutes, ...(extra ? [extra.name] : [])]; const lines = [ "", "## Async handoff", diff --git a/ts/justfile b/ts/justfile index 580570991..d204fa697 100644 --- a/ts/justfile +++ b/ts/justfile @@ -71,8 +71,9 @@ moq-pin: git -C target/moq fetch --quiet --depth 1 origin {{ moq_rev }} git -C target/moq checkout --quiet --detach {{ moq_rev }} -# Time the codec natively, as moq's WASM, as moq's hand-written TS, and translated, in Node, Bun, and Chrome. -bench checkout="": (moq checkout) +# Time the codec natively, as moq's WASM, as moq's hand-written TS, and translated, in Node, Bun, and Chrome; `--route name=dir` adds another run's translations. +[arg("route", long)] +bench checkout="" route="": (moq checkout) #!/usr/bin/env bash set -euo pipefail for tool in bun chromium wasm-bindgen; do @@ -80,6 +81,8 @@ bench checkout="": (moq checkout) done src="{{ checkout }}" [ -n "$src" ] || src=target/moq + route=() + [ -z "{{ route }}" ] || route=(--route "{{ route }}") out=target/bench cargo run --locked --quiet -- --config fixtures/codec/ts.toml --out "$out/translated" cargo build --locked --quiet --package codec-fixture --features bench --target wasm32-unknown-unknown --profile wasm-release @@ -95,7 +98,8 @@ bench checkout="": (moq checkout) bun bench/main.ts --translated "$out/translated" --moq "$src" --moq-wasm "$out/moq-wasm" --out "$out" \ --wasm target/wasm32-unknown-unknown/wasm-release/codec_fixture.wasm --native target/release/examples/bench \ --task "$out/task" \ - --task-wasm target/wasm32-unknown-unknown/wasm-release/task_fixture.wasm --task-native target/release/examples/handoff + --task-wasm target/wasm32-unknown-unknown/wasm-release/task_fixture.wasm --task-native target/release/examples/handoff \ + "${route[@]}" # Run cargo on the pinned nightly. cargo *args: