diff --git a/ts/README.md b/ts/README.md index 8c751ec97..2fd9a7816 100644 --- a/ts/README.md +++ b/ts/README.md @@ -251,8 +251,10 @@ size-optimized profile, hand-written (moq's `js/net` in `bench/moq.ts`, and ES i helpers in `bench/iter.ts`), and translated; every route's output must match native Rust's first. The JS routes run in Node, Bun, and Chrome, with the WASM route's copies across the boundary timed apart from its compute, and an A/A copy of the -translation to show the noise. The report records the host and engine versions, and -each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow +translation to show the noise. Each round runs the JS routes and workloads in a new +random order, so no route keeps a slot that runs it faster or slower than identical code +in another; the report records the seed. It also records the host and engine versions, +and each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow runs it and posts the report to its summary. Chrome's page reports each step it starts, and one that runs 30 s fails the run, naming it. @@ -270,5 +272,7 @@ nix develop .#bench --command just ts bench --route main="$HOME/rust2-main/ts/ta The route runs both fixtures' translations from that directory's `translated/` and `task/`, so a hand-edited translation can be timed by copying the directory and editing it. +Read its gap from `translated` against A/A's, the noise of the same run. No route holds a +fixed slot, so a gap past that noise is the change, and no reversed run is needed. The bench shell adds Bun, Chromium, and the wasm-bindgen that builds moq-wasm's package. diff --git a/ts/bench/entry.ts b/ts/bench/entry.ts index 34f982e88..815dfc3bc 100644 --- a/ts/bench/entry.ts +++ b/ts/bench/entry.ts @@ -10,6 +10,7 @@ type Workloads = { name: string; op: Workload["op"]; items: number; input: strin export interface Config { sampleMs: number; rounds: number; + seed: number; workloads: Workloads; /** The async handoff's, on the task fixture. */ handoffs: Workloads; diff --git a/ts/bench/main.ts b/ts/bench/main.ts index 391fe7e57..bd8adcaee 100644 --- a/ts/bench/main.ts +++ b/ts/bench/main.ts @@ -3,7 +3,7 @@ // // usage: bun main.ts --translated --wasm --native --moq // --moq-wasm --task --task-wasm --task-native -// --out [--sample-ms 10] [--rounds 15] [--route =] +// --out [--sample-ms 10] [--rounds 15] [--seed ] [--route =] // // The task fixture's async handoff runs beside the codec, against moq's way with promises. // `--route` times the translations in another run's `--out` as one more route, for A/B runs. @@ -36,6 +36,7 @@ const { values: args } = parseArgs({ out: { type: "string" }, "sample-ms": { type: "string", default: "10" }, rounds: { type: "string", default: "15" }, + seed: { type: "string" }, route: { type: "string" }, }, strict: true, @@ -49,6 +50,9 @@ const checkout = arg("moq"); const out = arg("out"); const sampleMs = Number(args["sample-ms"]); const rounds = Number(args.rounds); +// Each run orders its rounds anew unless `--seed` replays one. +const seed = args.seed === undefined ? crypto.getRandomValues(new Uint32Array(1))[0] : Number(args.seed); +if (!Number.isInteger(seed) || seed < 0 || seed >= 2 ** 32) throw new Error(`--seed takes a u32, not ${args.seed}`); const codecRoutes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"]; const handoffRoutes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"]; @@ -170,6 +174,7 @@ const handoffList = handoffs(); const config: Config = { sampleMs, rounds, + seed, workloads: list.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), handoffs: handoffList.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })), route: extra?.name, @@ -283,7 +288,7 @@ for (const run of [() => script("node"), () => script(process.execPath), chrome] const report = render(); await Bun.write(join(out, "report.md"), report); -await Bun.write(join(out, "report.json"), JSON.stringify({ native: host, engines, sizes, taskSizes }, null, "\t")); +await Bun.write(join(out, "report.json"), JSON.stringify({ seed, native: host, engines, sizes, taskSizes }, null, "\t")); console.log(report); function render(): string { @@ -300,8 +305,9 @@ function render(): string { ...engines.map((e) => `- ${e.engine}`), "", "Each cell is nanoseconds per item (a varint, a frame header, a message, or a byte iterated): the median", - `of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round interleaves`, - "every route and workload. `native` is a release build on the host. `wasm` is the same", + `of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round runs`, + `the JS routes and workloads in a new random order, from seed ${seed}. \`native\` is a release`, + "build on the host, its workloads in a fixed order. `wasm` is the same", "Rust at moq-wasm's size-optimized profile, run with its input already in memory;", "copying the input in and the output out are timed apart. `hand-written` is moq's", "`js/net`, called as its own benchmarks call it, and for `iter/` the same chain with ES", diff --git a/ts/bench/time.ts b/ts/bench/time.ts index 138711da0..2cba64539 100644 --- a/ts/bench/time.ts +++ b/ts/bench/time.ts @@ -18,6 +18,8 @@ export interface Options { sampleMs: number; /** Samples per timed body, interleaved with every other body's. */ rounds: number; + /** Seeds the order of the bodies in each round. */ + seed: number; /** Told what runs next, between timed regions. */ progress(step: string): void; } @@ -87,14 +89,17 @@ export async function measure(wasm: Wasm, routes: Route[], workloads: Workload[] while ((await sample(body)) < options.sampleMs) body.calls *= 2; } - // Interleave the bodies in each round, starting each round one later, so drift and - // position spread over all of them. Progress is per round: a report per sample slows - // Chrome's samples by about 10%. + // Each round runs the bodies in a new random order, so drift spreads over all of them + // and no body keeps a slot: a fixed order runs identical code slower in one slot than + // in another. Progress is per round: a report per sample slows Chrome's samples by + // about 10%. const span = `${workloads[0].name} through ${workloads[workloads.length - 1].name}`; + const random = prng(options.seed); + const order = [...bodies]; for (let round = 0; round < options.rounds; round++) { progress(`timing round ${round + 1} of ${options.rounds} on ${span}`); - for (let i = 0; i < bodies.length; i++) { - const body = bodies[(i + round) % bodies.length]; + shuffle(order, random); + for (const body of order) { const ms = await sample(body); body.ns.push((ms * 1e6) / (body.calls * body.workload.items)); } @@ -115,3 +120,21 @@ async function sample(body: Body): Promise { } return performance.now() - start; } + +/** Shuffles `items` in place: Fisher-Yates. */ +function shuffle(items: T[], random: () => number) { + for (let i = items.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + [items[i], items[j]] = [items[j], items[i]]; + } +} + +/** Uniform numbers in [0, 1) from a 32-bit seed: mulberry32. */ +function prng(seed: number): () => number { + return () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), seed | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 2 ** 32; + }; +}