From 452c52c63d1f033107e0094c04ad90fa4751a5be Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:22:45 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m1/release-perf/compare-harness.md Quest-Claim: 627453889f0cac08 From 6ddfce624017cd856715629d72a9f895fe874c6e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:34:14 -0700 Subject: [PATCH 2/4] fix(bench): pair Criterion samples by their full IDs Co-Authored-By: GPT-6 --- bench/README.md | 10 +++-- bench/compare.py | 36 +++++++++++++++ bench/compare.test.py | 47 +++++++++++++++++++ bench/run.sh | 57 +++++++++++++----------- quest/m1/release-perf/compare-harness.md | 13 ++++++ 5 files changed, 135 insertions(+), 28 deletions(-) diff --git a/bench/README.md b/bench/README.md index bc7e21663f5..9563074d662 100644 --- a/bench/README.md +++ b/bench/README.md @@ -200,12 +200,16 @@ them as `affinity` in the metadata. A CPU with one core type runs unpinned. BASE comparisons run paired rounds (default three, `MOQ_BENCH_RUNTIME_ROUNDS`), alternating base/current order. Startup remains outside the final five-second throughput window. `MOQ_BENCH_ARTIFACTS` chooses the evidence parent (default -`target/bench/runs`), which must be outside the repository, or git-ignored and -outside `bench/`; each invocation retains a new run directory even on failure. +the current worktree's `.scratch/bench`), which must be outside the repository, +or git-ignored and outside `bench/`; each invocation retains a new run directory even on failure. It contains raw load/host JSONL, process diagnostics, Criterion statistics, execution order, source revision and tracked dirty patch, tool/build/machine metadata, workload files, complete stdout/stderr, and a separate copy of untracked -source files. +source files. The retained path is printed at the end, including on failure. +Choose an evidence parent outside disposable worktrees when the report must survive +worktree removal. Criterion case discovery uses isolated short measurements and +their full JSON IDs, since `--list` truncates display names; discovery samples +are retained separately and never used in the comparison. Each comparison reports the median of the paired percentage deltas, their median absolute deviation and range, and flags fewer than three observations or a sign diff --git a/bench/compare.py b/bench/compare.py index 3bd105cbb3d..77bb7e38786 100644 --- a/bench/compare.py +++ b/bench/compare.py @@ -3,6 +3,7 @@ import argparse import json import math +import shutil import statistics from pathlib import Path @@ -57,10 +58,45 @@ def load(directory, workload): for name in sorted(rounds["base"], key=lambda name: int(name.removeprefix("round-")))] +def criterion_cases(directory): + """Read exact IDs from a fresh target's discovery measurements.""" + ids = set() + for path in directory.glob("**/discovery/benchmark.json"): + benchmark = json.loads(path.read_text())["full_id"] + if not isinstance(benchmark, str) or not benchmark or any(c in benchmark for c in "\t\r\n"): + raise ValueError(f"invalid Criterion full_id in {path}: {benchmark!r}") + ids.add(benchmark) + if not ids: + raise ValueError(f"no Criterion benchmark cases found in {directory}") + return sorted(ids) + + +def criterion_baseline(directory, discovery, benchmark): + """Place the exact baseline at the current revision's Criterion directory.""" + sources = [path.parent for path in directory.glob("**/base/benchmark.json") + if json.loads(path.read_text())["full_id"] == benchmark] + targets = [path.parent.parent.relative_to(discovery) for path in discovery.glob("**/discovery/benchmark.json") + if json.loads(path.read_text())["full_id"] == benchmark] + if len(sources) != 1 or len(targets) != 1: + raise ValueError(f"expected one baseline and discovery case for {benchmark!r}") + destination = directory / targets[0] / "base" + if sources[0] != destination: + shutil.copytree(sources[0], destination, dirs_exist_ok=True) + + def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--criterion-cases", action="store_true", help="list full IDs from isolated discovery samples") + parser.add_argument("--criterion-baseline", nargs=2, metavar=("DISCOVERY", "ID"), help="align an exact baseline with current discovery") parser.add_argument("directory", type=Path) args = parser.parse_args() + if args.criterion_cases: + print("\n".join(criterion_cases(args.directory))) + return + if args.criterion_baseline: + discovery, benchmark = args.criterion_baseline + criterion_baseline(args.directory, Path(discovery), benchmark) + return for workload in ("video", "fanout"): result = reduce(load(args.directory, workload)) (args.directory / f"comparison-{workload}.json").write_text(json.dumps(result, indent=2) + "\n") diff --git a/bench/compare.test.py b/bench/compare.test.py index 704ee714c74..4097e3c1991 100644 --- a/bench/compare.test.py +++ b/bench/compare.test.py @@ -81,4 +81,51 @@ def test_unpaired_rounds(self): compare.load(root, "video") +class Criterion(unittest.TestCase): + def test_full_ids_with_the_same_truncated_title(self): + prefix = "session_fanout/" + "a" * 100 + ids = [prefix + "/publishers=1/readers=16", prefix + "/publishers=16/readers=1"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index, benchmark in enumerate(ids): + path = root / f"truncated_{index}" / "discovery" + path.mkdir(parents=True) + (path / "benchmark.json").write_text(json.dumps({"full_id": benchmark})) + # Stale real measurements must not join the discovery inventory. + path = root / "old" / "new" + path.mkdir(parents=True) + (path / "benchmark.json").write_text(json.dumps({"full_id": "stale"})) + self.assertEqual(compare.criterion_cases(root), sorted(ids)) + + def test_baseline_follows_full_id_when_directory_suffix_changes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + samples = root / "samples" + old = samples / "truncated" / "base" + current = root / "discovery" + new = current / "truncated_2" / "discovery" + old.mkdir(parents=True) + new.mkdir(parents=True) + benchmark = "session/" + "long-" * 30 + for path in (old, new): + (path / "benchmark.json").write_text(json.dumps({"full_id": benchmark})) + (old / "sample.json").write_text('{"measured": true}') + compare.criterion_baseline(samples, current, benchmark) + self.assertEqual((samples / "truncated_2" / "base" / "sample.json").read_text(), '{"measured": true}') + with self.assertRaises(ValueError): + compare.criterion_baseline(samples, current, "unmatched") + + def test_empty_or_unrepresentable_inventory_fails(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(ValueError): + compare.criterion_cases(root) + path = root / "case" / "discovery" + path.mkdir(parents=True) + for value in [None, "", "bad\tcase", "bad\ncase"]: + (path / "benchmark.json").write_text(json.dumps({"full_id": value})) + with self.subTest(value=value), self.assertRaises(ValueError): + compare.criterion_cases(root) + + unittest.main() diff --git a/bench/run.sh b/bench/run.sh index 82353be9b56..afbdf51b4db 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -122,7 +122,7 @@ fi BASE_TARGET=$TARGET/base CURRENT_TARGET=$TARGET/current -ARTIFACTS=${MOQ_BENCH_ARTIFACTS:-$TARGET/runs} +ARTIFACTS=${MOQ_BENCH_ARTIFACTS:-$ROOT/.scratch/bench} mkdir -p "$ARTIFACTS" ARTIFACTS=$(cd "$ARTIFACTS" && pwd -P) # Each run copies bench/ and untracked sources, so it must land in neither. @@ -267,9 +267,12 @@ run_criterion_target() { criterion_cases() { local checkout=$1 - local manifest target features benchmark - local listing=$RUN/criterion-listing + local manifest target benchmark discovery local target_list=$RUN/criterion-targets + local label=base + if [[ $checkout == "$ROOT" ]]; then + label=current + fi if ! criterion_targets "$checkout" >"$target_list"; then return 1 @@ -278,21 +281,18 @@ criterion_cases() { printf 'no Criterion benchmark targets found\n' >&2 return 1 fi - while IFS=$'\t' read -r manifest target features; do - if ! ( - cd "$checkout" || exit 1 - CARGO_TARGET_DIR=$(target_dir "$checkout") \ - cargo bench --locked --manifest-path "$manifest" --bench "$target" ${features:+--features "$features"} -- --list - ) | sed -n 's/: benchmark$//p' >"$listing"; then - return 1 - fi - if [[ ! -s $listing ]]; then - printf 'Criterion target has no benchmark cases: %s/%s\n' "$manifest" "$target" >&2 - return 1 - fi + while IFS=$'\t' read -r manifest target _; do + discovery=$RUN/criterion-discovery/$label/$manifest/$target + # --list prints truncated display titles. A separate short measurement + # writes full IDs without polluting the actual comparison samples. + ( + CRITERION_ARGS=(--warm-up-time 0.001 --measurement-time 0.001 --sample-size 10 --nresamples 1 --noplot) + CRITERION_HOME=$discovery run_criterion_target "$checkout" "$manifest" "$target" --save-baseline discovery + ) >&2 || return 1 + python3 "$ROOT/bench/compare.py" --criterion-cases "$discovery" >"$RUN/criterion-listing" || return 1 while IFS= read -r benchmark; do printf '%s\t%s\t%s\n' "$manifest" "$target" "$benchmark" - done <"$listing" + done <"$RUN/criterion-listing" done <"$target_list" } @@ -302,31 +302,38 @@ compare_criterion() { local base_cases=$RUN/criterion-base.cases local current_cases=$RUN/criterion-current.cases local keys=$RUN/criterion-case.keys - local manifest target benchmark base_match current_match + local manifest target benchmark base_match current_match home + local index=0 criterion_cases "$base_checkout" >"$base_cases" criterion_cases "$current_checkout" >"$current_cases" sort -u "$base_cases" "$current_cases" >"$keys" while IFS=$'\t' read -r manifest target benchmark; do - base_match=$(awk -F '\t' -v manifest="$manifest" -v target="$target" -v benchmark="$benchmark" \ - '$1 == manifest && $2 == target && $3 == benchmark { print; exit }' "$base_cases") - current_match=$(awk -F '\t' -v manifest="$manifest" -v target="$target" -v benchmark="$benchmark" \ - '$1 == manifest && $2 == target && $3 == benchmark { print; exit }' "$current_cases") + index=$((index + 1)) + home=$RUN/criterion/$index + base_match=$(MOQ_CASE_MANIFEST=$manifest MOQ_CASE_TARGET=$target MOQ_CASE_ID=$benchmark awk -F '\t' \ + '$1 == ENVIRON["MOQ_CASE_MANIFEST"] && $2 == ENVIRON["MOQ_CASE_TARGET"] && $3 == ENVIRON["MOQ_CASE_ID"] { print; exit }' "$base_cases") + current_match=$(MOQ_CASE_MANIFEST=$manifest MOQ_CASE_TARGET=$target MOQ_CASE_ID=$benchmark awk -F '\t' \ + '$1 == ENVIRON["MOQ_CASE_MANIFEST"] && $2 == ENVIRON["MOQ_CASE_TARGET"] && $3 == ENVIRON["MOQ_CASE_ID"] { print; exit }' "$current_cases") if [[ -n $base_match ]]; then printf '\nCriterion baseline: %s\n' "$benchmark" - run_criterion_target "$base_checkout" "$manifest" "$target" \ + CRITERION_HOME=$home run_criterion_target "$base_checkout" "$manifest" "$target" \ --save-baseline base "$benchmark" --exact fi if [[ -n $current_match ]]; then printf '\nCriterion current: %s\n' "$benchmark" if [[ -n $base_match ]]; then - run_criterion_target "$current_checkout" "$manifest" "$target" \ + # Truncated directory names can receive different uniqueness suffixes + # when cases are added or reordered between revisions. + python3 "$ROOT/bench/compare.py" --criterion-baseline \ + "$RUN/criterion-discovery/current/$manifest/$target" "$benchmark" "$home" + CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ --baseline-lenient base "$benchmark" --exact else - run_criterion_target "$current_checkout" "$manifest" "$target" \ - --discard-baseline "$benchmark" --exact + CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ + --save-baseline current "$benchmark" --exact fi fi done <"$keys" diff --git a/quest/m1/release-perf/compare-harness.md b/quest/m1/release-perf/compare-harness.md index 9ae3dd0aec6..c0fc0b52182 100644 --- a/quest/m1/release-perf/compare-harness.md +++ b/quest/m1/release-perf/compare-harness.md @@ -20,3 +20,16 @@ when that worktree was removed. Write the report under a path the caller chooses. Default to the current worktree's `.scratch/`, and print its path at the end. A durable copy belongs with the epic's final run, not in the repo. Extend `bench/compare.test.py` to cover a case ID longer than 100 characters. + +Implementation is complete in [moq-dev/moq#6297](https://github.com/moq-dev/moq/pull/6297). +The runner discovers isolated JSON `full_id` values, keeps comparison samples +separate per case, and aligns baseline storage when truncated directory suffixes +change. A real Criterion smoke pairs two IDs longer than 100 characters with +opposite registration order; focused tests cover identity and directory matching. +`MOQ_BENCH_ARTIFACTS` remains the caller override, now defaulting to this +worktree's `.scratch/bench`, and the retained report path prints on exit. + +Only completion bookkeeping remains: remove this file and its links after the +maintainer authorizes it. Automatic approval review refused `quest rm` as +abandonment under the AFK no-abandon instruction; no deletion was performed. +The epic's final comparison uses the harness after its candidate changes land. From b244d68adaf6cbbdb3e3954fb1679919fc7d0617 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:47:59 -0700 Subject: [PATCH 3/4] test(bench): gate exact-ID pairing and report inventory changes Co-Authored-By: GPT-6 --- bench/run.sh | 16 +++++----------- sh/dispatch.sh | 5 ++++- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/bench/run.sh b/bench/run.sh index afbdf51b4db..38da8aaab22 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -330,7 +330,7 @@ compare_criterion() { python3 "$ROOT/bench/compare.py" --criterion-baseline \ "$RUN/criterion-discovery/current/$manifest/$target" "$benchmark" "$home" CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ - --baseline-lenient base "$benchmark" --exact + --baseline base "$benchmark" --exact else CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ --save-baseline current "$benchmark" --exact @@ -339,19 +339,13 @@ compare_criterion() { done <"$keys" } -collect_ids() { - local sample=$1 - while IFS= read -r -d '' benchmark; do - jq -r .full_id "$benchmark" - done < <(find "$CRITERION_HOME" -type f -path "*/$sample/benchmark.json" -print0) | - sort -u -} - report_set_changes() { local base_ids=$RUN/criterion-base.ids local current_ids=$RUN/criterion-current.ids - collect_ids base >"$base_ids" - collect_ids new >"$current_ids" + # Baseline measurements also write new/, so saved sample directories cannot + # distinguish a removed case from a case measured on the current revision. + cut -f3- "$RUN/criterion-base.cases" | sort -u >"$base_ids" + cut -f3- "$RUN/criterion-current.cases" | sort -u >"$current_ids" local new removed new=$(comm -13 "$base_ids" "$current_ids") diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 41e24667ece..a3b1da27946 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -280,7 +280,10 @@ for module in "${selected[@]}"; do # The preamble keeps docs and target-specific lints; the test build # below enforces default-feature clippy without a second compile. check:rs) just rs lint-changed "$rs_list" ;; - check:bench) cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring ;; + check:bench) + python3 bench/compare.test.py + cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring + ;; check:quest) quest check ;; check:drill) just test drill-sensitivity --apply-only ;; check:obs_compile) just obs compile ;; From f8ed0419d61555cc993d8885b1991e4376164855 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:11:27 -0700 Subject: [PATCH 4/4] chore(quest): retire the compare-harness quest Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 7 ++--- quest/m1/release-perf/compare-harness.md | 35 ------------------------ 2 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 quest/m1/release-perf/compare-harness.md diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index a35788fa4bb..1a88f6f425d 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -86,9 +86,9 @@ and 12% less on fanout, with the same delivery. - JS `net/bench/frames.ts` on bun: lite rows 1.2 to 1.3x, IETF rows 1.3 to 2.2x ([JS frames](/quest/m1/release-perf/js-frames.md)). -The raw report lived in a scratch directory that has since been removed. The -[comparison harness](/quest/m1/release-perf/compare-harness.md) keeps the next -one. +The raw report lived in a scratch directory that has since been removed. +`just bench ` now pairs full Criterion IDs; point `MOQ_BENCH_ARTIFACTS` +outside the worktree to keep the next one. ## Required @@ -99,7 +99,6 @@ one. - [Stats report](/quest/m1/release-perf/stats-report.md) - the stats drain locks each row once and its bench counts the wire walk; per-label bytes stay, at an accepted O(live sessions) - [Re-check](/quest/m1/release-perf/recheck.md) - the screen-only moves are re-measured with instruction counts, and the ones that hold are fixed or accepted - [JS frames](/quest/m1/release-perf/js-frames.md) - the js/net frame path is no slower than release in a browser, or the gap is accepted -- [Comparison harness](/quest/m1/release-perf/compare-harness.md) - `just bench ` pairs full Criterion IDs and keeps its report ## Related diff --git a/quest/m1/release-perf/compare-harness.md b/quest/m1/release-perf/compare-harness.md deleted file mode 100644 index c0fc0b52182..00000000000 --- a/quest/m1/release-perf/compare-harness.md +++ /dev/null @@ -1,35 +0,0 @@ -# [S] Comparison harness - -## Goal - -`just bench ` (`bench/run.sh`) pairs every Criterion case by its full -ID, and writes its report somewhere that survives the worktree. The -[epic](/quest/m1/release-perf/README.md)'s final main against release run -uses it without hand-written joins. - -## Plan - -`bench/run.sh` discovers cases from `cargo bench -- --list`, which truncates -display titles at 100 characters. That loses long `session_*` IDs. The -2026-10-21 run worked around it by joining the saved `benchmark.json` -`full_id` values instead (`run.sh` already reads them after the run). Discover -and pair cases from those full IDs. - -The same run kept its report in another worktree's `.scratch/`, and lost it -when that worktree was removed. Write the report under a path the caller -chooses. Default to the current worktree's `.scratch/`, and print its path at -the end. A durable copy belongs with the epic's final run, not in the repo. -Extend `bench/compare.test.py` to cover a case ID longer than 100 characters. - -Implementation is complete in [moq-dev/moq#6297](https://github.com/moq-dev/moq/pull/6297). -The runner discovers isolated JSON `full_id` values, keeps comparison samples -separate per case, and aligns baseline storage when truncated directory suffixes -change. A real Criterion smoke pairs two IDs longer than 100 characters with -opposite registration order; focused tests cover identity and directory matching. -`MOQ_BENCH_ARTIFACTS` remains the caller override, now defaulting to this -worktree's `.scratch/bench`, and the retained report path prints on exit. - -Only completion bookkeeping remains: remove this file and its links after the -maintainer authorizes it. Automatic approval review refused `quest rm` as -abandonment under the AFK no-abandon instruction; no deletion was performed. -The epic's final comparison uses the harness after its candidate changes land.