From 5584281f3e2bb535781655b631ab82514dbd5043 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:27:23 -0700 Subject: [PATCH] wip(ts): shuffle the bench's bodies each round from a recorded seed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- ts/README.md | 9 +++++++-- ts/bench/entry.ts | 1 + ts/bench/main.ts | 13 +++++++++---- ts/bench/time.ts | 39 +++++++++++++++++++++++++++++++++------ 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/ts/README.md b/ts/README.md index 5c8be3395..b8ca084d0 100644 --- a/ts/README.md +++ b/ts/README.md @@ -247,8 +247,10 @@ size-optimized profile, hand-written (moq's `js/net` in `bench/moq.ts`, and ES i helpers in `bench/iter.ts`), and translated; every route's output must match native Rust's first. The JS routes run in Node, Bun, and Chrome, with the WASM route's copies across the boundary timed apart from its compute, and an A/A copy of the -translation to show the noise. The report records the host and engine versions, and -each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow +translation to show the noise. Each round runs the JS routes and workloads in a new +random order, so no route keeps a slot that runs it faster or slower than identical code +in another; the report records the seed. It also records the host and engine versions, +and each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow runs it and posts the report to its summary. The task fixture's `bench::run` times an async handoff: a value sent to a task parked on @@ -265,5 +267,8 @@ nix develop .#bench --command just ts bench --route main="$HOME/rust2-main/ts/ta The route runs both fixtures' translations from that directory's `translated/` and `task/`, so a hand-edited translation can be timed by copying the directory and editing it. +Read its gap from `translated` against A/A's, which is the noise of the same run: no +route holds a fixed slot, so one forward run measures the change, with no reversed run +to cancel a slot's bias. The bench shell adds Bun, Chromium, and the wasm-bindgen that builds moq-wasm's package. diff --git a/ts/bench/entry.ts b/ts/bench/entry.ts index a0253c351..448768873 100644 --- a/ts/bench/entry.ts +++ b/ts/bench/entry.ts @@ -10,6 +10,7 @@ type Workloads = { name: string; op: Workload["op"]; items: number; input: strin export interface Config { sampleMs: number; rounds: number; + seed: number; workloads: Workloads; /** The async handoff's, on the task fixture. */ handoffs: Workloads; diff --git a/ts/bench/main.ts b/ts/bench/main.ts index cf0d1401b..a6ae9301e 100644 --- a/ts/bench/main.ts +++ b/ts/bench/main.ts @@ -3,7 +3,7 @@ // // usage: bun main.ts --translated --wasm --native --moq // --moq-wasm --task --task-wasm --task-native -// --out [--sample-ms 10] [--rounds 15] [--route =] +// --out [--sample-ms 10] [--rounds 15] [--seed ] [--route =] // // The task fixture's async handoff runs beside the codec, against moq's way with promises. // `--route` times the translations in another run's `--out` as one more route, for A/B runs. @@ -36,6 +36,7 @@ const { values: args } = parseArgs({ out: { type: "string" }, "sample-ms": { type: "string", default: "10" }, rounds: { type: "string", default: "15" }, + seed: { type: "string" }, route: { type: "string" }, }, strict: true, @@ -49,6 +50,9 @@ const checkout = arg("moq"); const out = arg("out"); const sampleMs = Number(args["sample-ms"]); const rounds = Number(args.rounds); +// Each run orders its rounds anew unless `--seed` replays one. +const seed = args.seed === undefined ? crypto.getRandomValues(new Uint32Array(1))[0] : Number(args.seed); +if (!Number.isInteger(seed) || seed < 0 || seed >= 2 ** 32) throw new Error(`--seed takes a u32, not ${args.seed}`); const codecRoutes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"]; const handoffRoutes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"]; @@ -170,6 +174,7 @@ const handoffList = handoffs(); const config: Config = { sampleMs, rounds, + seed, workloads: list.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), handoffs: handoffList.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), route: extra?.name, @@ -261,7 +266,7 @@ for (const run of [() => script("node"), () => script(process.execPath), chrome] const report = render(); await Bun.write(join(out, "report.md"), report); -await Bun.write(join(out, "report.json"), JSON.stringify({ native: host, engines, sizes, taskSizes }, null, "\t")); +await Bun.write(join(out, "report.json"), JSON.stringify({ seed, native: host, engines, sizes, taskSizes }, null, "\t")); console.log(report); function render(): string { @@ -278,8 +283,8 @@ function render(): string { ...engines.map((e) => `- ${e.engine}`), "", "Each cell is nanoseconds per item (a varint, a frame header, a message, or a byte iterated): the median", - `of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round interleaves`, - "every route and workload. `native` is a release build on the host. `wasm` is the same", + `of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round runs`, + `every route and workload in its own random order, from seed ${seed}. \`native\` is a release build on the host. \`wasm\` is the same`, "Rust at moq-wasm's size-optimized profile, run with its input already in memory;", "copying the input in and the output out are timed apart. `hand-written` is moq's", "`js/net`, called as its own benchmarks call it, and for `iter/` the same chain with ES", diff --git a/ts/bench/time.ts b/ts/bench/time.ts index 4d8638355..4851a7bd9 100644 --- a/ts/bench/time.ts +++ b/ts/bench/time.ts @@ -18,6 +18,8 @@ export interface Options { sampleMs: number; /** Samples per timed body, interleaved with every other body's. */ rounds: number; + /** Seeds the order of the bodies in each round. */ + seed: number; } export interface Timing { @@ -28,6 +30,7 @@ export interface Timing { calls: number; /** Nanoseconds per item, one per round. */ ns: number[]; + after?: string[]; } interface Body { @@ -39,6 +42,7 @@ interface Body { async: boolean; calls: number; ns: number[]; + after: string[]; } // Holds the last result, so no engine can drop a call as dead. @@ -63,7 +67,7 @@ export async function measure(wasm: Wasm, routes: Route[], workloads: Workload[] let len = 0; const body = (route: string, call: () => unknown, setup?: () => void) => - bodies.push({ workload, route, setup, call, async: false, calls: 1, ns: [] }); + bodies.push({ workload, route, setup, call, async: false, calls: 1, ns: [], after: [] }); body("wasm", () => wasm.run(), () => wasm.input(input)); body("wasm copy-in", () => wasm.input(input)); body("wasm copy-out", () => wasm.output(len), () => { @@ -82,17 +86,22 @@ export async function measure(wasm: Wasm, routes: Route[], workloads: Workload[] while ((await sample(body)) < options.sampleMs) body.calls *= 2; } - // Interleave the bodies in each round, starting each round one later, so drift and - // position spread over all of them. + // Each round runs every body in a new order. In a fixed order each body would always + // follow the same one and pay for what it leaves behind, such as garbage to collect. + const random = prng(options.seed); + const order = [...bodies]; + let prev = ""; for (let round = 0; round < options.rounds; round++) { - for (let i = 0; i < bodies.length; i++) { - const body = bodies[(i + round) % bodies.length]; + shuffle(order, random); + for (const body of order) { const ms = await sample(body); body.ns.push((ms * 1e6) / (body.calls * body.workload.items)); + body.after.push(prev); + prev = body.route; } } - return bodies.map(({ workload, route, calls, ns }) => ({ workload: workload.name, op: workload.op, route, calls, ns })); + return bodies.map(({ workload, route, calls, ns, after }) => ({ workload: workload.name, op: workload.op, route, calls, ns, after })); } /** Milliseconds for one sample of `body.calls` calls. */ @@ -107,3 +116,21 @@ async function sample(body: Body): Promise { } return performance.now() - start; } + +/** Shuffles `items` in place, Fisher-Yates. */ +function shuffle(items: T[], random: () => number) { + for (let i = items.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + [items[i], items[j]] = [items[j], items[i]]; + } +} + +/** Uniform numbers in [0, 1) from a 32-bit seed: mulberry32. */ +function prng(seed: number): () => number { + return () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), seed | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 2 ** 32; + }; +}