Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions ts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,10 @@ size-optimized profile, hand-written (moq's `js/net` in `bench/moq.ts`, and ES i
helpers in `bench/iter.ts`), and translated; every route's output must match
native Rust's first. The JS routes run in Node, Bun, and Chrome, with the WASM route's
copies across the boundary timed apart from its compute, and an A/A copy of the
translation to show the noise. The report records the host and engine versions, and
each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow
translation to show the noise. Each round runs the JS routes and workloads in a new
random order, so no route keeps a slot that runs it faster or slower than identical code
in another; the report records the seed. It also records the host and engine versions,
and each route's bundle size raw and gzipped. It sets no threshold; the nightly workflow
runs it and posts the report to its summary. Chrome's page reports each step it starts,
and one that runs 30 s fails the run, naming it.

Expand All @@ -270,5 +272,7 @@ nix develop .#bench --command just ts bench --route main="$HOME/rust2-main/ts/ta

The route runs both fixtures' translations from that directory's `translated/` and
`task/`, so a hand-edited translation can be timed by copying the directory and editing it.
Read its gap from `translated` against A/A's, the noise of the same run. No route holds a
fixed slot, so a gap past that noise is the change, and no reversed run is needed.

The bench shell adds Bun, Chromium, and the wasm-bindgen that builds moq-wasm's package.
1 change: 1 addition & 0 deletions ts/bench/entry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type Workloads = { name: string; op: Workload["op"]; items: number; input: strin
export interface Config {
sampleMs: number;
rounds: number;
seed: number;
workloads: Workloads;
/** The async handoff's, on the task fixture. */
handoffs: Workloads;
Expand Down
14 changes: 10 additions & 4 deletions ts/bench/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
// usage: bun main.ts --translated <dir> --wasm <file> --native <bin> --moq <checkout>
// --moq-wasm <dir> --task <dir> --task-wasm <file> --task-native <bin>
// --out <dir> [--sample-ms 10] [--rounds 15] [--route <name>=<dir>]
// --out <dir> [--sample-ms 10] [--rounds 15] [--seed <u32>] [--route <name>=<dir>]
//
// The task fixture's async handoff runs beside the codec, against moq's way with promises.
// `--route` times the translations in another run's `--out` as one more route, for A/B runs.
Expand Down Expand Up @@ -36,6 +36,7 @@ const { values: args } = parseArgs({
out: { type: "string" },
"sample-ms": { type: "string", default: "10" },
rounds: { type: "string", default: "15" },
seed: { type: "string" },
route: { type: "string" },
},
strict: true,
Expand All @@ -49,6 +50,9 @@ const checkout = arg("moq");
const out = arg("out");
const sampleMs = Number(args["sample-ms"]);
const rounds = Number(args.rounds);
// Each run orders its rounds anew unless `--seed` replays one.
const seed = args.seed === undefined ? crypto.getRandomValues(new Uint32Array(1))[0] : Number(args.seed);
if (!Number.isInteger(seed) || seed < 0 || seed >= 2 ** 32) throw new Error(`--seed takes a u32, not ${args.seed}`);

const codecRoutes = ["native", "wasm", "wasm copy-in", "wasm copy-out", "hand-written", "translated", "A/A"];
const handoffRoutes = ["native", "wasm", "hand-written", "translated", "spawned", "A/A"];
Expand Down Expand Up @@ -170,6 +174,7 @@ const handoffList = handoffs();
const config: Config = {
sampleMs,
rounds,
seed,
workloads: list.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })),
handoffs: handoffList.map((w) => ({ name: w.name, op: w.op, items: w.items, input: hex(w.input) })),
route: extra?.name,
Expand Down Expand Up @@ -283,7 +288,7 @@ for (const run of [() => script("node"), () => script(process.execPath), chrome]

const report = render();
await Bun.write(join(out, "report.md"), report);
await Bun.write(join(out, "report.json"), JSON.stringify({ native: host, engines, sizes, taskSizes }, null, "\t"));
await Bun.write(join(out, "report.json"), JSON.stringify({ seed, native: host, engines, sizes, taskSizes }, null, "\t"));
console.log(report);

function render(): string {
Expand All @@ -300,8 +305,9 @@ function render(): string {
...engines.map((e) => `- ${e.engine}`),
"",
"Each cell is nanoseconds per item (a varint, a frame header, a message, or a byte iterated): the median",
`of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round interleaves`,
"every route and workload. `native` is a release build on the host. `wasm` is the same",
`of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round runs`,
`the JS routes and workloads in a new random order, from seed ${seed}. \`native\` is a release`,
"build on the host, its workloads in a fixed order. `wasm` is the same",
"Rust at moq-wasm's size-optimized profile, run with its input already in memory;",
"copying the input in and the output out are timed apart. `hand-written` is moq's",
"`js/net`, called as its own benchmarks call it, and for `iter/` the same chain with ES",
Expand Down
33 changes: 28 additions & 5 deletions ts/bench/time.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ export interface Options {
sampleMs: number;
/** Samples per timed body, interleaved with every other body's. */
rounds: number;
/** Seeds the order of the bodies in each round. */
seed: number;
/** Told what runs next, between timed regions. */
progress(step: string): void;
}
Expand Down Expand Up @@ -87,14 +89,17 @@ export async function measure(wasm: Wasm, routes: Route[], workloads: Workload[]
while ((await sample(body)) < options.sampleMs) body.calls *= 2;
}

// Interleave the bodies in each round, starting each round one later, so drift and
// position spread over all of them. Progress is per round: a report per sample slows
// Chrome's samples by about 10%.
// Each round runs the bodies in a new random order, so drift spreads over all of them
// and no body keeps a slot: a fixed order runs identical code slower in one slot than
// in another. Progress is per round: a report per sample slows Chrome's samples by
// about 10%.
const span = `${workloads[0].name} through ${workloads[workloads.length - 1].name}`;
const random = prng(options.seed);
const order = [...bodies];
for (let round = 0; round < options.rounds; round++) {
progress(`timing round ${round + 1} of ${options.rounds} on ${span}`);
for (let i = 0; i < bodies.length; i++) {
const body = bodies[(i + round) % bodies.length];
shuffle(order, random);
for (const body of order) {
const ms = await sample(body);
body.ns.push((ms * 1e6) / (body.calls * body.workload.items));
}
Expand All @@ -115,3 +120,21 @@ async function sample(body: Body): Promise<number> {
}
return performance.now() - start;
}

/** Shuffles `items` in place: Fisher-Yates. */
function shuffle<T>(items: T[], random: () => number) {
for (let i = items.length - 1; i > 0; i--) {
const j = Math.floor(random() * (i + 1));
[items[i], items[j]] = [items[j], items[i]];
}
}

/** Uniform numbers in [0, 1) from a 32-bit seed: mulberry32. */
function prng(seed: number): () => number {
return () => {
seed = (seed + 0x6d2b79f5) | 0;
let t = Math.imul(seed ^ (seed >>> 15), seed | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 2 ** 32;
};
}