diff --git a/bench/README.md b/bench/README.md index 0783d096181..cc1c71ffe80 100644 --- a/bench/README.md +++ b/bench/README.md @@ -200,12 +200,16 @@ them as `affinity` in the metadata. A CPU with one core type runs unpinned. BASE comparisons run paired rounds (default three, `MOQ_BENCH_RUNTIME_ROUNDS`), alternating base/current order. Startup remains outside the final five-second throughput window. `MOQ_BENCH_ARTIFACTS` chooses the evidence parent (default -`target/bench/runs`), which must be outside the repository, or git-ignored and -outside `bench/`; each invocation retains a new run directory even on failure. +the current worktree's `.scratch/bench`), which must be outside the repository, +or git-ignored and outside `bench/`; each invocation retains a new run directory even on failure. It contains raw load/host JSONL, process diagnostics, Criterion statistics, execution order, source revision and tracked dirty patch, tool/build/machine metadata, workload files, complete stdout/stderr, and a separate copy of untracked -source files. +source files. The retained path is printed at the end, including on failure. +Choose an evidence parent outside disposable worktrees when the report must survive +worktree removal. Criterion case discovery uses isolated short measurements and +their full JSON IDs, since `--list` truncates display names; discovery samples +are retained separately and never used in the comparison. Each comparison reports the median of the paired percentage deltas, their median absolute deviation and range, and flags fewer than three observations or a sign diff --git a/bench/compare.py b/bench/compare.py index 3bd105cbb3d..77bb7e38786 100644 --- a/bench/compare.py +++ b/bench/compare.py @@ -3,6 +3,7 @@ import argparse import json import math +import shutil import statistics from pathlib import Path @@ -57,10 +58,45 @@ def load(directory, workload): for name in sorted(rounds["base"], key=lambda name: int(name.removeprefix("round-")))] +def criterion_cases(directory): + """Read exact IDs from a fresh target's discovery measurements.""" + ids = set() + for path in directory.glob("**/discovery/benchmark.json"): + benchmark = json.loads(path.read_text())["full_id"] + if not isinstance(benchmark, str) or not benchmark or any(c in benchmark for c in "\t\r\n"): + raise ValueError(f"invalid Criterion full_id in {path}: {benchmark!r}") + ids.add(benchmark) + if not ids: + raise ValueError(f"no Criterion benchmark cases found in {directory}") + return sorted(ids) + + +def criterion_baseline(directory, discovery, benchmark): + """Place the exact baseline at the current revision's Criterion directory.""" + sources = [path.parent for path in directory.glob("**/base/benchmark.json") + if json.loads(path.read_text())["full_id"] == benchmark] + targets = [path.parent.parent.relative_to(discovery) for path in discovery.glob("**/discovery/benchmark.json") + if json.loads(path.read_text())["full_id"] == benchmark] + if len(sources) != 1 or len(targets) != 1: + raise ValueError(f"expected one baseline and discovery case for {benchmark!r}") + destination = directory / targets[0] / "base" + if sources[0] != destination: + shutil.copytree(sources[0], destination, dirs_exist_ok=True) + + def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--criterion-cases", action="store_true", help="list full IDs from isolated discovery samples") + parser.add_argument("--criterion-baseline", nargs=2, metavar=("DISCOVERY", "ID"), help="align an exact baseline with current discovery") parser.add_argument("directory", type=Path) args = parser.parse_args() + if args.criterion_cases: + print("\n".join(criterion_cases(args.directory))) + return + if args.criterion_baseline: + discovery, benchmark = args.criterion_baseline + criterion_baseline(args.directory, Path(discovery), benchmark) + return for workload in ("video", "fanout"): result = reduce(load(args.directory, workload)) (args.directory / f"comparison-{workload}.json").write_text(json.dumps(result, indent=2) + "\n") diff --git a/bench/compare.test.py b/bench/compare.test.py index 704ee714c74..4097e3c1991 100644 --- a/bench/compare.test.py +++ b/bench/compare.test.py @@ -81,4 +81,51 @@ def test_unpaired_rounds(self): compare.load(root, "video") +class Criterion(unittest.TestCase): + def test_full_ids_with_the_same_truncated_title(self): + prefix = "session_fanout/" + "a" * 100 + ids = [prefix + "/publishers=1/readers=16", prefix + "/publishers=16/readers=1"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index, benchmark in enumerate(ids): + path = root / f"truncated_{index}" / "discovery" + path.mkdir(parents=True) + (path / "benchmark.json").write_text(json.dumps({"full_id": benchmark})) + # Stale real measurements must not join the discovery inventory. + path = root / "old" / "new" + path.mkdir(parents=True) + (path / "benchmark.json").write_text(json.dumps({"full_id": "stale"})) + self.assertEqual(compare.criterion_cases(root), sorted(ids)) + + def test_baseline_follows_full_id_when_directory_suffix_changes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + samples = root / "samples" + old = samples / "truncated" / "base" + current = root / "discovery" + new = current / "truncated_2" / "discovery" + old.mkdir(parents=True) + new.mkdir(parents=True) + benchmark = "session/" + "long-" * 30 + for path in (old, new): + (path / "benchmark.json").write_text(json.dumps({"full_id": benchmark})) + (old / "sample.json").write_text('{"measured": true}') + compare.criterion_baseline(samples, current, benchmark) + self.assertEqual((samples / "truncated_2" / "base" / "sample.json").read_text(), '{"measured": true}') + with self.assertRaises(ValueError): + compare.criterion_baseline(samples, current, "unmatched") + + def test_empty_or_unrepresentable_inventory_fails(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(ValueError): + compare.criterion_cases(root) + path = root / "case" / "discovery" + path.mkdir(parents=True) + for value in [None, "", "bad\tcase", "bad\ncase"]: + (path / "benchmark.json").write_text(json.dumps({"full_id": value})) + with self.subTest(value=value), self.assertRaises(ValueError): + compare.criterion_cases(root) + + unittest.main() diff --git a/bench/run.sh b/bench/run.sh index 82353be9b56..38da8aaab22 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -122,7 +122,7 @@ fi BASE_TARGET=$TARGET/base CURRENT_TARGET=$TARGET/current -ARTIFACTS=${MOQ_BENCH_ARTIFACTS:-$TARGET/runs} +ARTIFACTS=${MOQ_BENCH_ARTIFACTS:-$ROOT/.scratch/bench} mkdir -p "$ARTIFACTS" ARTIFACTS=$(cd "$ARTIFACTS" && pwd -P) # Each run copies bench/ and untracked sources, so it must land in neither. @@ -267,9 +267,12 @@ run_criterion_target() { criterion_cases() { local checkout=$1 - local manifest target features benchmark - local listing=$RUN/criterion-listing + local manifest target benchmark discovery local target_list=$RUN/criterion-targets + local label=base + if [[ $checkout == "$ROOT" ]]; then + label=current + fi if ! criterion_targets "$checkout" >"$target_list"; then return 1 @@ -278,21 +281,18 @@ criterion_cases() { printf 'no Criterion benchmark targets found\n' >&2 return 1 fi - while IFS=$'\t' read -r manifest target features; do - if ! ( - cd "$checkout" || exit 1 - CARGO_TARGET_DIR=$(target_dir "$checkout") \ - cargo bench --locked --manifest-path "$manifest" --bench "$target" ${features:+--features "$features"} -- --list - ) | sed -n 's/: benchmark$//p' >"$listing"; then - return 1 - fi - if [[ ! -s $listing ]]; then - printf 'Criterion target has no benchmark cases: %s/%s\n' "$manifest" "$target" >&2 - return 1 - fi + while IFS=$'\t' read -r manifest target _; do + discovery=$RUN/criterion-discovery/$label/$manifest/$target + # --list prints truncated display titles. A separate short measurement + # writes full IDs without polluting the actual comparison samples. + ( + CRITERION_ARGS=(--warm-up-time 0.001 --measurement-time 0.001 --sample-size 10 --nresamples 1 --noplot) + CRITERION_HOME=$discovery run_criterion_target "$checkout" "$manifest" "$target" --save-baseline discovery + ) >&2 || return 1 + python3 "$ROOT/bench/compare.py" --criterion-cases "$discovery" >"$RUN/criterion-listing" || return 1 while IFS= read -r benchmark; do printf '%s\t%s\t%s\n' "$manifest" "$target" "$benchmark" - done <"$listing" + done <"$RUN/criterion-listing" done <"$target_list" } @@ -302,49 +302,50 @@ compare_criterion() { local base_cases=$RUN/criterion-base.cases local current_cases=$RUN/criterion-current.cases local keys=$RUN/criterion-case.keys - local manifest target benchmark base_match current_match + local manifest target benchmark base_match current_match home + local index=0 criterion_cases "$base_checkout" >"$base_cases" criterion_cases "$current_checkout" >"$current_cases" sort -u "$base_cases" "$current_cases" >"$keys" while IFS=$'\t' read -r manifest target benchmark; do - base_match=$(awk -F '\t' -v manifest="$manifest" -v target="$target" -v benchmark="$benchmark" \ - '$1 == manifest && $2 == target && $3 == benchmark { print; exit }' "$base_cases") - current_match=$(awk -F '\t' -v manifest="$manifest" -v target="$target" -v benchmark="$benchmark" \ - '$1 == manifest && $2 == target && $3 == benchmark { print; exit }' "$current_cases") + index=$((index + 1)) + home=$RUN/criterion/$index + base_match=$(MOQ_CASE_MANIFEST=$manifest MOQ_CASE_TARGET=$target MOQ_CASE_ID=$benchmark awk -F '\t' \ + '$1 == ENVIRON["MOQ_CASE_MANIFEST"] && $2 == ENVIRON["MOQ_CASE_TARGET"] && $3 == ENVIRON["MOQ_CASE_ID"] { print; exit }' "$base_cases") + current_match=$(MOQ_CASE_MANIFEST=$manifest MOQ_CASE_TARGET=$target MOQ_CASE_ID=$benchmark awk -F '\t' \ + '$1 == ENVIRON["MOQ_CASE_MANIFEST"] && $2 == ENVIRON["MOQ_CASE_TARGET"] && $3 == ENVIRON["MOQ_CASE_ID"] { print; exit }' "$current_cases") if [[ -n $base_match ]]; then printf '\nCriterion baseline: %s\n' "$benchmark" - run_criterion_target "$base_checkout" "$manifest" "$target" \ + CRITERION_HOME=$home run_criterion_target "$base_checkout" "$manifest" "$target" \ --save-baseline base "$benchmark" --exact fi if [[ -n $current_match ]]; then printf '\nCriterion current: %s\n' "$benchmark" if [[ -n $base_match ]]; then - run_criterion_target "$current_checkout" "$manifest" "$target" \ - --baseline-lenient base "$benchmark" --exact + # Truncated directory names can receive different uniqueness suffixes + # when cases are added or reordered between revisions. + python3 "$ROOT/bench/compare.py" --criterion-baseline \ + "$RUN/criterion-discovery/current/$manifest/$target" "$benchmark" "$home" + CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ + --baseline base "$benchmark" --exact else - run_criterion_target "$current_checkout" "$manifest" "$target" \ - --discard-baseline "$benchmark" --exact + CRITERION_HOME=$home run_criterion_target "$current_checkout" "$manifest" "$target" \ + --save-baseline current "$benchmark" --exact fi fi done <"$keys" } -collect_ids() { - local sample=$1 - while IFS= read -r -d '' benchmark; do - jq -r .full_id "$benchmark" - done < <(find "$CRITERION_HOME" -type f -path "*/$sample/benchmark.json" -print0) | - sort -u -} - report_set_changes() { local base_ids=$RUN/criterion-base.ids local current_ids=$RUN/criterion-current.ids - collect_ids base >"$base_ids" - collect_ids new >"$current_ids" + # Baseline measurements also write new/, so saved sample directories cannot + # distinguish a removed case from a case measured on the current revision. + cut -f3- "$RUN/criterion-base.cases" | sort -u >"$base_ids" + cut -f3- "$RUN/criterion-current.cases" | sort -u >"$current_ids" local new removed new=$(comm -13 "$base_ids" "$current_ids") diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index a35788fa4bb..1a88f6f425d 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -86,9 +86,9 @@ and 12% less on fanout, with the same delivery. - JS `net/bench/frames.ts` on bun: lite rows 1.2 to 1.3x, IETF rows 1.3 to 2.2x ([JS frames](/quest/m1/release-perf/js-frames.md)). -The raw report lived in a scratch directory that has since been removed. The -[comparison harness](/quest/m1/release-perf/compare-harness.md) keeps the next -one. +The raw report lived in a scratch directory that has since been removed. +`just bench ` now pairs full Criterion IDs; point `MOQ_BENCH_ARTIFACTS` +outside the worktree to keep the next one. ## Required @@ -99,7 +99,6 @@ one. - [Stats report](/quest/m1/release-perf/stats-report.md) - the stats drain locks each row once and its bench counts the wire walk; per-label bytes stay, at an accepted O(live sessions) - [Re-check](/quest/m1/release-perf/recheck.md) - the screen-only moves are re-measured with instruction counts, and the ones that hold are fixed or accepted - [JS frames](/quest/m1/release-perf/js-frames.md) - the js/net frame path is no slower than release in a browser, or the gap is accepted -- [Comparison harness](/quest/m1/release-perf/compare-harness.md) - `just bench ` pairs full Criterion IDs and keeps its report ## Related diff --git a/quest/m1/release-perf/compare-harness.md b/quest/m1/release-perf/compare-harness.md deleted file mode 100644 index 9ae3dd0aec6..00000000000 --- a/quest/m1/release-perf/compare-harness.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] Comparison harness - -## Goal - -`just bench ` (`bench/run.sh`) pairs every Criterion case by its full -ID, and writes its report somewhere that survives the worktree. The -[epic](/quest/m1/release-perf/README.md)'s final main against release run -uses it without hand-written joins. - -## Plan - -`bench/run.sh` discovers cases from `cargo bench -- --list`, which truncates -display titles at 100 characters. That loses long `session_*` IDs. The -2026-10-21 run worked around it by joining the saved `benchmark.json` -`full_id` values instead (`run.sh` already reads them after the run). Discover -and pair cases from those full IDs. - -The same run kept its report in another worktree's `.scratch/`, and lost it -when that worktree was removed. Write the report under a path the caller -chooses. Default to the current worktree's `.scratch/`, and print its path at -the end. A durable copy belongs with the epic's final run, not in the repo. -Extend `bench/compare.test.py` to cover a case ID longer than 100 characters. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 0b011cd38f3..af88ecf0359 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -281,7 +281,10 @@ for module in "${selected[@]}"; do # The preamble keeps docs and target-specific lints; the test build # below enforces default-feature clippy without a second compile. check:rs) just rs lint-changed "$rs_list" ;; - check:bench) cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring ;; + check:bench) + python3 bench/compare.test.py + cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring + ;; check:quest) quest check ;; check:drill) just test drill-sensitivity --apply-only ;; check:obs_compile) just obs compile ;;