From ad540d268d18cb4676b3b4a1d210c9a24dcc8a6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 22:22:19 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m1/bench-ci.md Quest-Claim: 410d11cf09c88a2d From ccf207d5a53d548f8fd37dd1dc99e8150365d33f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 23:05:51 -0700 Subject: [PATCH 2/5] bench: pair nightly Criterion runs against the previous nightly Each Rust bench group builds the previous scheduled Nightly's commit beside its own and runs both on one runner, alternating order. A quick screen flags slowdowns past 10%, alternating rounds confirm them, and a confirmed regression fails the job after its results upload, so the site publishes the comparison the alert links to. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 31 +++++- bench/README.md | 45 +++++++- bench/compare.py | 123 ++++++++++++++++++++-- bench/compare.test.py | 71 +++++++++++++ bench/run.sh | 142 +++++++++++++++++++++++++- quest/m1/README.md | 2 +- quest/m1/bench-ci.md | 99 ++++++------------ quest/m1/release-perf/README.md | 2 +- quest/m2/audit-perf.md | 12 +-- quest/m2/bench-sources.md | 5 +- quest/m2/performance-survey/README.md | 2 +- sh/gh/bench-publish.sh | 9 +- sh/gh/bench-report.py | 123 +++++++++++++++++++++- sh/gh/bench-report.test.py | 78 +++++++++++++- sh/gh/nightly.sh | 45 ++++++-- sh/gh/nightly.test.py | 50 +++++++-- 16 files changed, 715 insertions(+), 124 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index edca5a31179..d393ebd9ffc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -40,6 +40,11 @@ permissions: on: workflow_dispatch: + inputs: + bench-base: + description: Revision the paired Rust benchmarks compare against; the run's own commit gives an A/A run. Empty is the previous scheduled Nightly's commit. + type: string + default: "" schedule: - cron: "0 7 * * *" @@ -80,12 +85,19 @@ jobs: # build environment, so a benchmark reuses the dependencies its group already # compiled. The Rust benchmarks outgrew one hour, so they split by harness: # nextest's ignored benchmark tests and cargo's bench targets share few builds. + # A paired group also builds and runs the previous nightly's commit beside this + # one on the same runner, which is what fails the job on a regression; the + # cross-night history on the benchmark site stays informational. nightly: name: Nightly (${{ matrix.name }}) runs-on: ${{ matrix.runner || 'ubuntu-24.04-arm' }} # `main` and its merge queue write the Rust cache; see .github/actions/rust-cache. environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/gh-readonly-queue/main/')) && 'cache-write' || '' }} timeout-minutes: 60 + permissions: + contents: read + # Paired groups look up the previous scheduled Nightly's commit. + actions: read strategy: fail-fast: false matrix: @@ -124,16 +136,25 @@ jobs: artifact: bench-rust artifact-path: target/bench/nightly/rust-bench benchmark: true + paired: true + - name: Rust shared bench targets + recipe: gh nightly rust-shared + artifact: bench-rust-shared + artifact-path: target/bench/nightly/rust-shared + benchmark: true + paired: true - name: Rust supplemental benchmarks recipe: gh nightly rust-extra artifact: bench-rust-extra artifact-path: target/bench/nightly/rust-extra benchmark: true + paired: true - name: io_uring benchmarks recipe: gh nightly rust-uring artifact: bench-rust-uring artifact-path: target/bench/nightly/rust-uring benchmark: true + paired: true - name: JS benchmarks recipe: gh nightly js artifact: bench-js @@ -205,6 +226,8 @@ jobs: run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} env: NEXTEST_PROFILE: default + MOQ_BENCH_BASE: ${{ matrix.paired && (inputs.bench-base || 'previous-nightly') || '' }} + GH_TOKEN: ${{ matrix.paired && github.token || '' }} - name: Record benchmark outcome if: ${{ matrix.benchmark && !cancelled() }} @@ -218,6 +241,12 @@ jobs: path: ${{ matrix.artifact-path }} retention-days: 45 + # After the upload, so the benchmark site still records tonight's history and + # publishes the comparison that failed this job before the alert links to it. + - name: Fail on paired regressions + if: ${{ matrix.paired && steps.workload.outcome == 'success' }} + run: nix develop --command just gh benchmark-report gate "${{ matrix.artifact-path }}" + # gh-pages is the one Pages site, and each directory has one writer: this job owns # bench/ (the regression table and its history) and the root index listing every # report, smoke.yml owns relays/, and interop.yml owns interop/. Each commits only @@ -290,7 +319,7 @@ jobs: comment-on-alert: false fail-on-alert: false - name: Publish shared history - run: nix shell --inputs-from . nixpkgs#just --command just gh benchmark-publish finish benchmark-site + run: nix shell --inputs-from . nixpkgs#just --command just gh benchmark-publish finish benchmark-site benchmark-reports/paired.json - name: Drop a stale Pages artifact run: nix shell --inputs-from . nixpkgs#just --command just gh pages-clean env: diff --git a/bench/README.md b/bench/README.md index 8be337740f5..61183e645e6 100644 --- a/bench/README.md +++ b/bench/README.md @@ -398,15 +398,15 @@ The table includes these measurements: same host, including for throughput, where the ratio is Rust/wrapper. - Rust Criterion mean ns per iteration; the history also keeps the confidence interval. The `net-cpu` source and the moq-quic targets time thread CPU; the others - time elapsed time. The Rust bench targets job measures them with a short warmup + time elapsed time. The Rust bench target jobs measure them with a short warmup and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates - trade precision for nightly runtime; use full paired runs to assess a change. + trade precision for nightly runtime; the paired comparison below assesses a change. - JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation variants) and `size_bytes` columns, retaining every other column as a workload dimension in each series name. -Hosted runners vary across nights, so a flag prompts a paired local run rather -than proving a regression. Binding ratios compare languages within one run. +Hosted runners vary across nights, so a trend flag prompts a look at the paired +comparison rather than proving a regression. Binding ratios compare languages within one run. Malformed, empty, nonfinite, or duplicate measurements fail validation. The nightly artifacts retain the reports, raw JS output, and each Criterion run's metadata and estimates for 45 days. Relay load, audio quality, and the Rust @@ -445,4 +445,39 @@ within a source; dimensions belong in the name. Supported units are `ns`, `us`, `ops/s`, and `frames/s`. Optional `range` and `extra` fields retain uncertainty and context. Validate and split sources into cost and throughput histories with `just gh benchmark-report prepare INPUT_DIRECTORY OUTPUT_DIRECTORY`; extending -the table requires no new renderer or data branch. +the table requires no new renderer or data branch. A source's `paired.json`, as +`just gh benchmark-report rust` writes for a paired run, joins the paired comparison. + +### Paired comparison + +Each Nightly's Rust bench groups (`rust-bench`, `rust-shared`, `rust-extra` and +`rust-uring`) also build the previous scheduled Nightly's commit beside their +own, on the same runner, so night-to-night runner variance drops out. Each +built target runs once per side in the trend's quick mode, alternating which +side goes first; the current side is the trend point. A case more than 10% +slower in that screen reruns, worst first and at most 20 per group, in three +alternating rounds with the BASE comparison's settings (1 s warmup, 2 s +measurement, 30 samples). A case whose rounds are all slower with a median +beyond 10% is a confirmed regression and fails its job after the upload, so the +Alert workflow posts the failed Nightly. A target the base cannot build or run +fails the job too, rather than passing uncompared; a bench target the base does +not have yet is only reported as new. The threshold is a starting point until +A/A runs on the hosted runner measure its noise floor. + +The table page shows tonight's comparison above the trend: confirmed +regressions, targets not compared, cases faster in the screen, slowdowns the +rounds rejected or the budget skipped, and the rest. `bench/paired.json` on the +site holds every case. A dispatched Nightly's `bench-base` input compares +against another revision; its own commit gives an A/A run. + +Reproduce a paired group locally against any revision; `previous-nightly` +resolves the last scheduled Nightly's commit through `gh`: + +```bash +nix develop --command env MOQ_BENCH_BASE=previous-nightly just gh nightly rust-bench +nix develop --command just gh benchmark-report gate target/bench/nightly/rust-bench +``` + +Each run's evidence directory keeps both builds' estimates (`criterion` and +`criterion-base`), the confirmation rounds under `confirm/`, and the run's +`paired.json`. diff --git a/bench/compare.py b/bench/compare.py index 77bb7e38786..4a5247e55dd 100644 --- a/bench/compare.py +++ b/bench/compare.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -"""Reduce paired relay rounds; retain dispersion instead of implying single-run precision.""" +"""Reduce paired relay rounds and Criterion runs; retain dispersion instead of implying single-run precision.""" import argparse +from collections import Counter import json import math import shutil @@ -14,6 +15,15 @@ def value(row, metric): return row.get('generator', {}).get('cpu_cores') if metric == 'generator_cpu_cores' else row.get(metric) +def summarize(deltas): + """The median paired percent delta, its median absolute deviation, and whether every pair agreed.""" + median = statistics.median(deltas) + mad = statistics.median(abs(value - median) for value in deltas) + return {"delta_pct": median, "mad_pct": mad, "range_pct": [min(deltas), max(deltas)], "paired_deltas_pct": deltas, + "status": "insufficient samples" if len(deltas) < 3 else + "sign varies across pairs" if min(deltas) < 0 < max(deltas) else "consistent sign"} + + def reduce(pairs): if not pairs: raise ValueError("no paired rounds") @@ -37,12 +47,7 @@ def reduce(pairs): if not deltas: result["metrics"][metric] = {"delta_pct": None, "status": "undefined ratio to zero", "paired_absolute_deltas": absolute} continue - median = statistics.median(deltas) - mad = statistics.median(abs(value - median) for value in deltas) - result["metrics"][metric] = {"delta_pct": median, "mad_pct": mad, - "range_pct": [min(deltas), max(deltas)], "paired_deltas_pct": deltas, "paired_absolute_deltas": absolute, - "status": "insufficient samples" if len(deltas) < 3 else - "sign varies across pairs" if min(deltas) < 0 < max(deltas) else "consistent sign"} + result["metrics"][metric] = {**summarize(deltas), "paired_absolute_deltas": absolute} # Keep offered load and delivery/loss next to CPU; no normalized CPU claim when windows differ. result["cpu_efficiency"] = "not estimated: verify offered load, delivery/loss and host/load window alignment before normalizing CPU" return result @@ -84,15 +89,119 @@ def criterion_baseline(directory, discovery, benchmark): shutil.copytree(sources[0], destination, dirs_exist_ok=True) +# A paired case beyond this change is flagged, and fails the nightly only when its rounds agree. +# Loose until A/A runs on the hosted runner measure its noise floor. +THRESHOLD_PCT = 10 +# Each alternating round costs seconds per case, so a noisy night confirms only its worst slowdowns. +CONFIRMATIONS = 20 + + +def criterion_means(home): + """Map each (target, full ID) measured under HOME/// to its mean nanoseconds.""" + means = {} + for path in home.glob("*/*/**/new/estimates.json"): + benchmark = json.loads(path.with_name("benchmark.json").read_text())["full_id"] + if not isinstance(benchmark, str) or not benchmark or any(c in benchmark for c in "\t\r\n"): + raise ValueError(f"invalid Criterion full_id in {path}: {benchmark!r}") + key = ("/".join(path.relative_to(home).parts[:2]), benchmark) + if key in means: + raise ValueError(f"duplicate Criterion case {key} in {home}") + mean = json.loads(path.read_text())["mean"]["point_estimate"] + if isinstance(mean, bool) or not isinstance(mean, (int, float)) or not math.isfinite(mean) or mean <= 0: + raise ValueError(f"invalid Criterion mean in {path}: {mean!r}") + means[key] = mean + return means + + +def percent(base, current): + return 100 * (current / base - 1) + + +def criterion_screen(run): + """The quick screen's slowdowns past the threshold, worst first, up to the confirmation budget.""" + base, current = criterion_means(run / "criterion-base"), criterion_means(run / "criterion") + slower = sorted(((percent(base[key], current[key]), key) for key in base.keys() & current.keys()), reverse=True) + return [key for delta, key in slower if delta > THRESHOLD_PCT][:CONFIRMATIONS] + + +def criterion_paired(run): + """Classify every case from the quick screen and the alternating confirmation rounds.""" + metadata = json.loads((run / "metadata.json").read_text()) + rounds = metadata["rounds"] + base, current = criterion_means(run / "criterion-base"), criterion_means(run / "criterion") + unavailable = [] + if (run / "paired-unavailable.tsv").exists(): + for line in (run / "paired-unavailable.tsv").read_text().splitlines(): + target, reason = line.split("\t", 1) + unavailable.append({"target": target, "reason": reason}) + blocked = {entry["target"] for entry in unavailable} + confirmed = {} + lines = (run / "confirm.tsv").read_text().splitlines() if (run / "confirm.tsv").exists() else [] + for index, line in enumerate(lines, 1): + key = tuple(line.split("\t", 1)) + if key[0] in blocked: + continue + deltas = [] + for number in range(1, rounds + 1): + sides = [criterion_means(run / "confirm" / str(index) / f"round-{number}" / side).get(key) + for side in ("base", "current")] + if None in sides: + raise ValueError(f"confirmation round {number} did not measure {key}") + deltas.append(percent(*sides)) + confirmed[key] = summarize(deltas) + cases = [] + for key in sorted(base.keys() | current.keys()): + target, benchmark = key + case = {"target": target, "id": benchmark, "base_ns": base.get(key), "current_ns": current.get(key)} + if key not in current: + case["verdict"] = "removed" + elif key not in base: + case["verdict"] = "unavailable" if target in blocked else "new" + else: + case["delta_pct"] = delta = percent(base[key], current[key]) + if key in confirmed: + case["rounds"] = confirmed[key] + agreed = case["rounds"]["status"] == "consistent sign" and case["rounds"]["delta_pct"] > THRESHOLD_PCT + case["verdict"] = "regression" if agreed else "noise" + elif delta > THRESHOLD_PCT: + case["verdict"] = "unavailable" if target in blocked else "unconfirmed" + elif delta < -THRESHOLD_PCT: + case["verdict"] = "faster" + else: + case["verdict"] = "unchanged" + cases.append(case) + return {"commit": metadata["revision"], "base": metadata["base"], "threshold_pct": THRESHOLD_PCT, + "rounds": rounds, "unavailable": unavailable, "cases": cases} + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--criterion-cases", action="store_true", help="list full IDs from isolated discovery samples") parser.add_argument("--criterion-baseline", nargs=2, metavar=("DISCOVERY", "ID"), help="align an exact baseline with current discovery") + parser.add_argument("--criterion-screen", action="store_true", help="list a paired run's slowdowns to confirm") + parser.add_argument("--criterion-paired", action="store_true", help="classify a paired run's cases into paired.json") parser.add_argument("directory", type=Path) args = parser.parse_args() if args.criterion_cases: print("\n".join(criterion_cases(args.directory))) return + if args.criterion_screen: + for target, benchmark in criterion_screen(args.directory): + print(f"{target}\t{benchmark}") + return + if args.criterion_paired: + result = criterion_paired(args.directory) + (args.directory / "paired.json").write_text(json.dumps(result, indent=2) + "\n") + verdicts = Counter(case["verdict"] for case in result["cases"]) + print(f"\nPaired comparison against {result['base']}: " + + ", ".join(f"{count} {verdict}" for verdict, count in sorted(verdicts.items()))) + for case in result["cases"]: + if case["verdict"] == "regression": + print(f"REGRESSION {case['target']} {case['id']}: " + f"{case['rounds']['delta_pct']:+.1f}% over {result['rounds']} rounds") + for entry in result["unavailable"]: + print(f"UNAVAILABLE {entry['target']}: {entry['reason']}") + return if args.criterion_baseline: discovery, benchmark = args.criterion_baseline criterion_baseline(args.directory, Path(discovery), benchmark) diff --git a/bench/compare.test.py b/bench/compare.test.py index 4097e3c1991..630bb977355 100644 --- a/bench/compare.test.py +++ b/bench/compare.test.py @@ -115,6 +115,77 @@ def test_baseline_follows_full_id_when_directory_suffix_changes(self): with self.assertRaises(ValueError): compare.criterion_baseline(samples, current, "unmatched") + def measure(self, home, target, benchmark, mean): + path = home / target / benchmark.replace("/", "_") / "new" + path.mkdir(parents=True) + (path / "benchmark.json").write_text(json.dumps({"full_id": benchmark})) + (path / "estimates.json").write_text(json.dumps({"mean": {"point_estimate": mean}})) + + def paired_run(self, root, cases, rounds=3): + """Write a paired run's quick screen from {(target, id): (base, current)}; None skips a side.""" + (root / "metadata.json").write_text(json.dumps({"revision": "current", "base": "base", "rounds": rounds})) + for (target, benchmark), sides in cases.items(): + for home, mean in zip(("criterion-base", "criterion"), sides): + if mean is not None: + self.measure(root / home, target, benchmark, mean) + + def confirm(self, root, index, key, rounds): + for number, (base, current) in enumerate(rounds, 1): + for side, mean in (("base", base), ("current", current)): + self.measure(root / "confirm" / str(index) / f"round-{number}" / side, *key, mean) + + def test_screen_confirms_the_worst_slowdowns_within_budget(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases = {("moq-net/session", f"case/{index}"): (100, 111 + index) for index in range(compare.CONFIRMATIONS + 5)} + cases[("moq-net/session", "steady")] = (100, 109) + cases[("kio/lock", "faster")] = (100, 50) + self.paired_run(root, cases) + screen = compare.criterion_screen(root) + self.assertEqual(len(screen), compare.CONFIRMATIONS) + self.assertEqual(screen[0], ("moq-net/session", f"case/{compare.CONFIRMATIONS + 4}")) + self.assertNotIn(("moq-net/session", "steady"), screen) + + def test_rounds_decide_each_flagged_slowdown(self): + regressed, noisy, beyond = ("t/a", "regressed"), ("t/a", "noisy"), ("t/a", "beyond") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.paired_run(root, {regressed: (100, 150), noisy: (100, 150), beyond: (100, 115), + ("t/a", "faster"): (100, 80), ("t/a", "same"): (100, 105), + ("t/a", "added"): (None, 10), ("t/a", "dropped"): (10, None), + ("t/broken", "base failed"): (None, 10)}) + (root / "confirm.tsv").write_text("t/a\tregressed\nt/a\tnoisy\n") + self.confirm(root, 1, regressed, [(100, 120), (100, 130), (100, 125)]) + # A bimodal case: slower in two rounds, faster in one. + self.confirm(root, 2, noisy, [(100, 150), (150, 100), (100, 150)]) + (root / "paired-unavailable.tsv").write_text("t/broken\tbase run failed\n") + result = compare.criterion_paired(root) + verdicts = {case["id"]: case["verdict"] for case in result["cases"]} + self.assertEqual(verdicts, {"regressed": "regression", "noisy": "noise", "beyond": "unconfirmed", + "faster": "faster", "same": "unchanged", "added": "new", "dropped": "removed", + "base failed": "unavailable"}) + regression = next(case for case in result["cases"] if case["id"] == "regressed") + self.assertEqual([round(delta) for delta in regression["rounds"]["paired_deltas_pct"]], [20, 30, 25]) + self.assertEqual(result["unavailable"], [{"target": "t/broken", "reason": "base run failed"}]) + + def test_missing_confirmation_round_fails(self): + key = ("t/a", "slow") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.paired_run(root, {key: (100, 150)}) + (root / "confirm.tsv").write_text("t/a\tslow\n") + self.confirm(root, 1, key, [(100, 150), (100, 150)]) + with self.assertRaises(ValueError): + compare.criterion_paired(root) + + def test_invalid_means_fail(self): + for mean in (0, -1, float("nan"), None, True): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.measure(root, "t/a", "case", mean) + with self.subTest(mean=mean), self.assertRaises(ValueError): + compare.criterion_means(root) + def test_empty_or_unrepresentable_inventory_fails(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/bench/run.sh b/bench/run.sh index 38da8aaab22..d7dd5463d88 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -48,7 +48,11 @@ usable_cpus() { } case $MODE in --criterion) + # MOQ_BENCH_BASE pairs each selected target with the same target built at that revision. BASE= + if [[ -n ${MOQ_BENCH_BASE:-} ]]; then + BASE=$(git rev-parse --verify "$MOQ_BENCH_BASE^{commit}") + fi CRITERION_CARGO_ARGS=() CRITERION_FILTER_ARGS=() parsing_filters=false @@ -265,6 +269,139 @@ run_criterion_target() { ) } +# The nightly trend's settings, which also screen every case of a paired run. +CRITERION_QUICK=(--quick --warm-up-time 0.05 --measurement-time 0.2 --noplot) + +# Build the selected bench targets in a checkout, listing each in OUT as +# "package-directory/targetpackage pathexecutable". OUT is written here rather +# than by redirecting the call: an errexit inside a redirected function would leave the +# shell's saved stdout open, and cleanup would wait forever on the log copier. +criterion_build() { + local out=$1 checkout=$2 + shift 2 + ( + cd "$checkout" || exit 1 + CARGO_TARGET_DIR=$(target_dir "$checkout") \ + cargo bench --locked --no-run --message-format=json-render-diagnostics "$@" + ) | jq -r 'select(.reason == "compiler-artifact" and .executable != null and any(.target.kind[]; . == "bench")) + | (.manifest_path | rtrimstr("/Cargo.toml")) as $package + | [($package | split("/") | last) + "/" + .target.name, $package, .executable] | @tsv' | sort >"$out" +} + +# Build the base with the current cargo arguments, minus bench targets it does not have yet: +# naming one would fail the whole build. Lists nothing in OUT when every selected target is new. +criterion_build_base() { + local out=$1 names name index selected=0 count=${#CRITERION_CARGO_ARGS[@]} + local args=() + : >"$out" + names=$(criterion_targets "$WORKTREE" | cut -f2) || return 1 + for ((index = 0; index < count; index++)); do + name=${CRITERION_CARGO_ARGS[index]} + if [[ $name == --bench ]] && ((index + 1 < count)); then + index=$((index + 1)) + name=${CRITERION_CARGO_ARGS[index]} + selected=$((selected + 1)) + # A glob selects whatever the base has. + if [[ $name == *[*?[]* ]] || grep -qxF -- "$name" <<<"$names"; then + args+=(--bench "$name") + fi + else + args+=("$name") + fi + done + if ((selected)) && [[ " ${args[*]} " != *" --bench "* ]]; then + return 0 + fi + criterion_build "$out" "$WORKTREE" ${args[@]+"${args[@]}"} +} + +# Run one built target as cargo bench would, from its package directory, with results under HOME/KEY. +criterion_exec() { + local list=$1 home=$2 key=$3 package executable + shift 3 + IFS=$'\t' read -r _ package executable < <(target_line "$list" "$key") || return 1 + printf '\nCriterion %s: %s\n' "${home#"$RUN"/}" "$key" + (cd "$package" && CRITERION_HOME=$home/$key "$executable" --bench "$@" >"$RUN/paired-unavailable.tsv" + printf 'Paired comparison unavailable for %s: %s\n' "$1" "$2" >&2 +} + +# Measure the selected targets in quick mode. With a base, run each target's base build +# beside it, alternating which goes first, then confirm the screen's worst slowdowns. +# A failed current target fails the run; the base only marks its targets unavailable. +measure_criterion() { + local current=$RUN/criterion-current.targets base=$RUN/criterion-base.targets + local key side targets index=0 status=0 + local filters=(${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"}) + criterion_build "$current" "$ROOT" ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} + mapfile -t targets < <(cut -f1 "$current") + if ((${#targets[@]} == 0)); then + printf 'no Criterion bench targets selected\n' >&2 + return 1 + fi + : >"$base" + if [[ -n $BASE ]]; then + WORKTREE=$RUN/base + git worktree add --detach "$WORKTREE" "$BASE" >/dev/null + mkdir -p "$RUN/base-context" + (cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" comparison-base) + printf 'Criterion paired with base %s\n' "$BASE" + if ! criterion_build_base "$base"; then + : >"$base" + for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done + fi + fi + for key in "${targets[@]}"; do + local order=(current) + if [[ -n $(target_line "$base" "$key") ]]; then + if ((index++ % 2)); then order=(current base); else order=(base current); fi + fi + for side in "${order[@]}"; do + if [[ $side == current ]]; then + criterion_exec "$current" "$RUN/criterion" "$key" "${CRITERION_QUICK[@]}" \ + ${filters[@]+"${filters[@]}"} || status=1 + else + criterion_exec "$base" "$RUN/criterion-base" "$key" "${CRITERION_QUICK[@]}" \ + ${filters[@]+"${filters[@]}"} || unavailable "$key" "base run failed" + fi + done + done + if [[ -n $BASE ]] && ((status == 0)); then + confirm_criterion || return 1 + fi + return "$status" +} + +# One quick run per side can land on noise, so each slowdown the screen flags is +# re-measured in alternating rounds with the comparison settings before it can fail. +confirm_criterion() { + local key benchmark round side index=0 + python3 "$ROOT/bench/compare.py" --criterion-screen "$RUN" >"$RUN/confirm.tsv" || return 1 + while IFS=$'\t' read -r key benchmark; do + index=$((index + 1)) + for ((round = 1; round <= ROUNDS; round++)); do + local order=(base current) + if ((round % 2 == 0)); then order=(current base); fi + for side in "${order[@]}"; do + if ! criterion_exec "$RUN/criterion-$side.targets" "$RUN/confirm/$index/round-$round/$side" "$key" \ + "${CRITERION_ARGS[@]}" "$benchmark" --exact; then + unavailable "$key" "confirmation failed" + continue 3 + fi + done + done + done <"$RUN/confirm.tsv" + python3 "$ROOT/bench/compare.py" --criterion-paired "$RUN" +} + criterion_cases() { local checkout=$1 local manifest target benchmark discovery @@ -744,9 +881,8 @@ if [[ $MODE == --criterion ]]; then if [[ -n ${MOQ_BENCH_RUNS:-} ]]; then printf '%s\n' "$RUN" >>"$MOQ_BENCH_RUNS" fi - CARGO_TARGET_DIR=$CURRENT_TARGET cargo bench --locked ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} -- \ - --quick --warm-up-time 0.05 --measurement-time 0.2 --noplot ${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"} - exit 0 + measure_criterion + exit elif [[ $MODE == --audio ]]; then run_audio_comparison exit 0 diff --git a/quest/m1/README.md b/quest/m1/README.md index c0bb4fbd6cf..bd165455bd2 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -166,7 +166,7 @@ tracks, so per-group stats stop costing cluster announces. - [Peer count bounds](/quest/m1/peer-count-bounds.md) - namespace interests, a legacy announce list, a WebSocket upgrade, and public HTTP admission are bounded by what the peer sent or by an existing budget - [Caption store bounds](/quest/m1/caption-store-bounds.md) - a hostile caption track cannot grow a viewer's cue and region store or its per-frame work without bound - [Auth contract](/quest/m1/auth-contract.md) - the auth server receives the certificate-selecting SNI, documented as client-chosen, and a certificate grants nothing alone -- [Nightly benchmark regressions](/quest/m1/bench-ci.md) - each nightly compares the Rust Criterion benches against the previous nightly's commit on the same hosted runner, publishes it to the git.moq.dev report, and alerts on a regression +- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - A/A and deliberately slowed Nightly runs set the paired comparison's threshold, so its regression alerts can be trusted - [Interop audit](/quest/m1/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down ## Related diff --git a/quest/m1/bench-ci.md b/quest/m1/bench-ci.md index 22eaa6dd3db..3695bfed6f2 100644 --- a/quest/m1/bench-ci.md +++ b/quest/m1/bench-ci.md @@ -1,79 +1,38 @@ -# [M] Nightly benchmark regression report +# [S] Calibrate the paired nightly benchmarks ## Goal -Each Nightly run on `main` compares every measured Rust Criterion benchmark -against the previous nightly's commit, built and run on the same runner, and -publishes the comparison to the git.moq.dev nightly report beside the -[regression table](https://git.moq.dev/bench/). A regression past a loose threshold -fails the job, so the existing Alert workflow posts it to Discord. - -Benchmarks never run in the merge queue or on PRs. Rust Criterion targets -only: browser benchmarks stay under `just bench-browser` (see -`bench/README.md`), and relay load stays local under `just bench BASE`. +The paired Rust Criterion comparison each Nightly runs (see +[Paired comparison](/bench/README.md#paired-comparison)) fails only on real +regressions: its threshold, confirmation budget, and round count come from A/A +runs on the hosted ARM runners, and a deliberately slowed bench fails the job. ## Plan -Decided 2026-10-20 (maintainer): benchmarks go into the git.moq.dev report -each night, not the merge queue, because the queue runs only what gates a -merge, and main PRs skip cloud CI since #6038. This drops the per-PR sticky -comment design and its `workflow_run` follow-up. - -Decided 2026-10-22 (maintainer): the paired run stays on the hosted ARM -runners. Moved from m2 to m1 so the [performance audit](/quest/m2/audit-perf.md) -reads paired flags instead of raw history. Rejected hosts: - -- The merge queue runner (`nixos-moq-ci`) is an nspawn container on a - desktop with `CPUWeight=10`, a floating 8-core quota, and the `powersave` - governor beside unrelated GUI, simulation, and agent load. A benchmark - there is the first workload starved, and nightly's benchmark jobs would - serialize on the one queue runner and hold merges for hours. -- moq.pro's two load nodes are shared 4-vCore, 8 GB OVH VPSes with measured - steal, and moq.pro decided they run nothing but the load generator. -- A dedicated bare-metal bench host would be the most stable but costs money - and adds a persistent exec surface for a public repository. Revisit it only - if A/A runs show the hosted noise floor hides regressions worth catching. - -The paired run removes runner-to-runner variance; what remains is noise -within one VM, which alternating rounds and A/A-derived thresholds handle. -The first three nights of hosted history (2026-10-20 to 22, 95-129 commits -apart) put the median night-to-night swing at about 4%, with 13% of Rust -series moving more than 10%, many up about 50% and back the next night. -Those nights mix code changes with noise, so they bound it rather than -measure it; the A/A run measures it. - -Rust Criterion only (decided 2026-10-22): `bench-ffi` stays history-only, -since its JVM and Dart rows are the noisiest (Kotlin about 18% a night), -and JS on bun only stands in for the browser. - -Researched 2026-09: CodSpeed's instruction counting skips every -`iter_custom` bench (all of `moq-uring`, plus `moq-net` origin and track) and -can't see syscall time. Bencher compares against history recorded on other -VMs, so its thresholds fire on runner noise. - -- Paired comparison: build the previous nightly's commit beside `main` on the - same runner, run the selected targets for both, and compare saved - Criterion baselines. Only that paired comparison fails the job; the - cross-run history stays informational on the regression table. -- Extend `bench/run.sh` and `sh/gh/nightly.sh` (`just gh nightly rust-bench`, - `rust-extra`, and `rust-uring`) with the paired mode instead of writing a second runner, and publish its - table through `just gh benchmark-report` into the report. `nightly.sh` - exits on a failure before it publishes today; publish first, so the table - exists when the regression alert fires. -- The nightly jobs measure in Criterion's quick mode for runtime; the paired - run needs enough samples for its threshold. Weigh a full run of the - compared targets against the nightly's time budget. -- Decided 2026-10-16: moq-quic's Criterion targets (#5812) are included. - They time thread CPU rather than wall time, which already cut most host - noise; on a shared hybrid-core host, identical builds still swing about 5% - on small cases and 20-130% on large-working-set ones, so the noise floor - must come from A/A runs on the CI runner itself, never from a laptop. -- The `moq-uring` socket benches skip loudly when the runner's kernel is too - old. -- Validate with an A/A run (the same commit twice) and a deliberately slowed - bench before trusting the signal. -- Document the report and local reproduction in `bench/README.md`. +Waits on maintainer-dispatched Nightly runs on `main` after the paired mode +lands; agents never dispatch Nightly. + +- A/A: dispatch Nightly on `main` with `bench-base` set to the run's own + commit, at least twice. Read each paired group's `paired.json` (the + `bench-rust*` artifacts, or `bench/paired.json` on the site): the spread of + screen deltas sets the screen threshold, and any confirmed regression in an + A/A run means the rounds or threshold are too loose. +- Slowed bench: dispatch with `bench-base` on a branch commit that speeds up + one cheap bench (so tonight's `main` reads as slower), and check the job fails + with that case alone. +- Tune `THRESHOLD_PCT`, `CONFIRMATIONS` (`bench/compare.py`) and the round count + (`MOQ_BENCH_RUNTIME_ROUNDS`, default three) from what the runs show, and + update the README's numbers. Bimodal series (about 50% up and back across + nights) are the expected false positives; three alternating rounds pass one + only when all three land on the slow mode. +- Check the paired legs' durations against their hour; the base build relies + on the mbx cache for the previous night's artifacts, and a cold one roughly + doubles each leg's build time. + +Rejected hosts (decided 2026-10-22, maintainer): the merge queue runner, the +moq.pro load nodes, and a dedicated bare-metal host; revisit the last only if +A/A runs show the hosted noise floor hides regressions worth catching. ## Related -- [Benchmark reporting](/bench/README.md) - the existing `bench/run.sh` retains repeated paired rounds +- [Benchmark reporting](/bench/README.md) - the paired comparison and the trend table diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index c7e702cfddc..769d36da9f3 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -117,4 +117,4 @@ outside the worktree to keep the next one. ## Related - [Performance audit](/quest/m2/audit-perf.md) - compares nightly to nightly; this epic compares main to release -- [Nightly benchmark regressions](/quest/m1/bench-ci.md) - paired nightly-to-nightly comparison on one runner +- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - noise floor of the paired nightly-to-nightly comparison on one runner diff --git a/quest/m2/audit-perf.md b/quest/m2/audit-perf.md index 0d4315d8f5b..cb5a9649491 100644 --- a/quest/m2/audit-perf.md +++ b/quest/m2/audit-perf.md @@ -78,9 +78,9 @@ did not move. - History: compare at least five nights by commit, now that it exists. Nightly stores every series on `gh-pages` (`bench/costs`, `bench/rates`); re-runs add - duplicate points for one commit. Read the - [nightly benchmark regression report](/quest/m1/bench-ci.md)'s paired flags - instead of raw history once it lands. The first JS series (841) appeared on + duplicate points for one commit. Read the paired comparison on the table + page instead of raw history, once + [its calibration](/quest/m1/bench-ci.md) lands. The first JS series (841) appeared on Oct 22, and Dart still needs a recheck against the bindings history. - The per-group write cost above, then the resume reader: at 16 single-frame groups `track::Subscriber::poll_group` is 44% of the Annex B export @@ -166,9 +166,9 @@ Single-axis fan-out benches to widen: `session_delivery_paced` (viewers only), Fan-out paths need a sweep over both axes (AGENTS.md). JS perf is judged in a browser, not node or bun. Benchmarks never gate the merge queue (decided -2026-10-20, maintainer): each Nightly compares them into the git.moq.dev -report, which the [nightly benchmark regression report](/quest/m1/bench-ci.md) -owns; don't duplicate it. +2026-10-20, maintainer): each Nightly compares them in the git.moq.dev +report's [paired comparison](/bench/README.md#paired-comparison); don't +duplicate it. ## Required diff --git a/quest/m2/bench-sources.md b/quest/m2/bench-sources.md index 1e35963e171..a8f5f0aecb3 100644 --- a/quest/m2/bench-sources.md +++ b/quest/m2/bench-sources.md @@ -26,7 +26,8 @@ don't already emit one. - The relay load benches (`bench-audio`, `bench-profile`) and the audio-quality job's benchmarks can join later through the same input format. -- [Nightly benchmark regression report](/quest/m1/bench-ci.md) records its nightly - Criterion history on the same `gh-pages` branch instead of creating its own. +- The paired nightly comparison publishes beside the table from the same + `gh-pages` branch (`bench/paired.json`); a new Rust source joins it through + `just gh benchmark-report rust`. Public API: none. Wire: none. diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 5b79f8d7f87..d5452ef2911 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -198,4 +198,4 @@ No duplicate quest or workflow edit belongs here. - [Performance audit](/quest/m2/audit-perf.md) - nightly regression attribution and remaining coverage - [Benchmark sources](/quest/m2/bench-sources.md) - publishing existing measurements to the trend table -- [Nightly comparisons](/quest/m1/bench-ci.md) - paired nightly regression reporting +- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - noise floor of the paired nightly regression report diff --git a/sh/gh/bench-publish.sh b/sh/gh/bench-publish.sh index 2fed6bcc254..dc2c01cd468 100755 --- a/sh/gh/bench-publish.sh +++ b/sh/gh/bench-publish.sh @@ -16,13 +16,14 @@ case ${1:-} in fi ;; finish) - if [[ $# != 2 ]]; then - echo 'usage: benchmark-publish finish DIRECTORY' >&2 + if [[ $# != 3 ]]; then + echo 'usage: benchmark-publish finish DIRECTORY PAIRED' >&2 exit 2 fi + paired=$(realpath "$3") git worktree add "$2" gh-pages # The checkout is the nightly's commit; a source without a point for it failed tonight. - just gh benchmark-report index "$2" --commit "$(git rev-parse HEAD)" + just gh benchmark-report index "$2" --commit "$(git rev-parse HEAD)" --paired "$paired" git -C "$2" add index.html .nojekyll bench if ! git -C "$2" diff --cached --quiet; then git -C "$2" -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ @@ -31,7 +32,7 @@ case ${1:-} in git -C "$2" push origin HEAD:gh-pages ;; *) - echo 'usage: benchmark-publish {init|finish DIRECTORY}' >&2 + echo 'usage: benchmark-publish {init|finish DIRECTORY PAIRED}' >&2 exit 2 ;; esac diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 6c32c45d721..88319665f75 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -77,6 +77,51 @@ def rust(directory): return checked(entries) +def paired(directory): + """Combine each run's paired comparison under its trend series names; None when no run was paired.""" + runs = [Path(line) for line in (directory / 'criterion-runs.txt').read_text().splitlines() if line] + metadata = [json.loads((run / 'metadata.json').read_text()) for run in runs] + bases = {data['base'] for data in metadata} + if bases == {''}: + return None + if len(bases) != 1: + raise ValueError(f'Criterion runs disagree on their paired base: {sorted(bases)}') + combined = {'unavailable': [], 'cases': []} + for run, data in zip(runs, metadata): + comparison = json.loads((run / 'paired.json').read_text()) + for key in ('commit', 'base', 'threshold_pct', 'rounds'): + if combined.setdefault(key, comparison[key]) != comparison[key]: + raise ValueError(f'paired run disagrees on {key}: {run}') + label = data['environment']['MOQ_BENCH_LABEL'] + combined['unavailable'] += [{'name': f'rust/{label}/{entry["target"]}', 'reason': entry['reason']} + for entry in comparison['unavailable']] + for case in comparison['cases']: + case = dict(case) + combined['cases'].append({'name': f'rust/{label}/{case.pop("id")}', **case}) + return combined + + +def gate(directory): + """Print confirmed paired regressions and comparisons that could not run; nonzero when any exist.""" + path = directory / 'paired.json' + if not path.exists(): + raise ValueError(f'no paired comparison in {directory}') + comparison = json.loads(path.read_text()) + regressions = [case for case in comparison['cases'] if case['verdict'] == 'regression'] + lines = [f'Paired comparison of {comparison["commit"][:9]} against {comparison["base"][:9]}: ' + f'{len(regressions)} confirmed regressions, {len(comparison["unavailable"])} targets not compared', ''] + for case in regressions: + rounds = ', '.join(f'{delta:+.1f}%' for delta in case['rounds']['paired_deltas_pct']) + lines.append(f'- `{case["name"]}`: {case["rounds"]["delta_pct"]:+.1f}% (rounds {rounds})') + lines += [f'- `{entry["name"]}` not compared: {entry["reason"]}' for entry in comparison['unavailable']] + text = '\n'.join(lines) + '\n' + print(text, end='') + if summary := os.environ.get('GITHUB_STEP_SUMMARY'): + with open(summary, 'a') as stream: + stream.write(text) + return 1 if regressions or comparison['unavailable'] else 0 + + def measurement_unit(column): if column == 'size_bytes': return 'bytes' @@ -123,7 +168,7 @@ def javascript(directory): def prepare(directory, output): - costs, rates = [], [] + costs, rates, comparisons = [], [], {} for source in sorted(directory.glob('bench-*')): # Other nightly artifacts share the prefix; only a recorded outcome marks a chart source. status = source / 'outcome.json' @@ -143,10 +188,14 @@ def prepare(directory, output): target = rates if entry['unit'] in BIGGER else costs target.append({**entry, 'name': f'{source.name}/{entry["name"]}'}) print(f'{source.name}: {len(entries)} measured entries') + if (source / 'paired.json').exists(): + comparisons[source.name] = json.loads((source / 'paired.json').read_text()) if not costs and not rates: # The table still rebuilds from stored history and marks every source stale. print('no completed benchmark reports; no history point') output.mkdir(parents=True, exist_ok=True) + # Tonight's paired comparisons replace the published ones, even when none finished. + (output / 'paired.json').write_text(json.dumps({'sources': comparisons}, indent=2) + '\n') for name, entries in [('costs', costs), ('rates', rates)]: (output / f'{name}.json').write_text(json.dumps(entries, indent=2) + '\n') if entries: @@ -235,6 +284,63 @@ def table(rows): + '\n'.join(body) + '') +def paired_table(cases): + body = [] + for case in sorted(cases, key=lambda case: -abs(case.get('delta_pct') or 0)): + sides = [number(case[side], 'ns') if case[side] is not None else '' for side in ('base_ns', 'current_ns')] + status = {'regression': 'worse', 'faster': 'better'}.get(case['verdict'], '') + change = f'{case["delta_pct"]:+.1f}%' if 'delta_pct' in case else case['verdict'] + rounds = ' '.join(f'{delta:+.1f}%' for delta in case['rounds']['paired_deltas_pct']) if 'rounds' in case else '' + body.append(f'{html.escape(case["name"])}{sides[0]}{sides[1]}' + f'{change}{rounds}') + return ('' + + '\n'.join(body) + '
SeriesBaseCurrentScreenRounds
') + + +# Paired verdicts by section: open sections hold what needs a look tonight. +PAIRED_SECTIONS = [ + ({'regression'}, 'Confirmed regressions', True), + ({'unavailable'}, 'Not compared', True), + ({'faster'}, 'Faster in the screen', True), + ({'unconfirmed', 'noise'}, 'Slower in the screen, not confirmed', False), + ({'new', 'removed'}, 'Measured on one side only', False), + ({'unchanged'}, 'Unchanged', False), +] + + +def paired_section(directory): + """Render the published paired comparisons: tonight's commit against the previous nightly's, per source.""" + path = directory / 'bench' / 'paired.json' + if not path.exists(): + return '' + sources = json.loads(path.read_text())['sources'] + if not sources: + return '

Paired comparison

No paired comparison finished on the latest nightly.

' + cases, notes, unavailable = [], [], [] + for source, comparison in sorted(sources.items()): + commit, base = comparison['commit'], comparison['base'] + notes.append(f'{html.escape(source)}: ' + f'{base[:9]}...{commit[:9]}') + cases += [{**case, 'name': f'{source}/{case["name"]}'} for case in comparison['cases']] + unavailable += [f'
  • {html.escape(source)}/{html.escape(entry["name"])}: {html.escape(entry["reason"])}' + for entry in comparison['unavailable']] + threshold, rounds = comparison['threshold_pct'], comparison['rounds'] + parts = [f'''

    Paired comparison

    +

    Each source built the previous nightly's commit beside this one and ran both on one runner, alternating +which went first. A quick screen flags a change beyond {threshold}%; each slowdown then reruns in {rounds} +alternating rounds of longer measurements, and fails the nightly when every round is slower and their median +exceeds {threshold}%. {"; ".join(notes)}

    '''] + if unavailable: + parts.append(f'

    Targets not compared:

      {"".join(unavailable)}
    ') + for verdicts, label, flagged in PAIRED_SECTIONS: + rows = [case for case in cases if case['verdict'] in verdicts] + if rows or flagged: + attributes = ' open class="flagged"' if flagged else '' + parts.append(f'{label} ({len(rows)})' + f'{paired_table(rows) if rows else ""}') + return '\n'.join(parts) + '\n

    Nightly trend

    \n' + + def regressions(directory, commit=None, now=None): """Render every series' newest point against its own baseline as one static table.""" series = history(directory) @@ -293,7 +399,7 @@ def regressions(directory, commit=None, now=None): Metrics and local reproduction

    -{''.join(parts)} +{paired_section(directory)}{''.join(parts)}