diff --git a/.config/nextest.toml b/.config/nextest.toml index 993a7ad512a..49d5913fe55 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -60,6 +60,12 @@ slow-timeout = { period = "60s", terminate-after = 3 } io-uring = { max-threads = 4 } v4l2 = { max-threads = 1 } +# Nightly's ignored benchmark tests (`just gh nightly rust-test`) keep each test's +# printed tables in target/nextest/bench/junit.xml for the benchmark report. +[profile.bench.junit] +path = "junit.xml" +store-success-output = true + # CI has noisier neighbours and cold caches, so give a test longer before # calling it wedged. [profile.ci] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cbec471b82d..e9f16cc96d1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -109,6 +109,9 @@ jobs: include: - name: Rust benchmark tests recipe: gh nightly rust-test + artifact: bench-rust-test + artifact-path: target/bench/nightly/rust-test + benchmark: true - name: Binding benchmark runners recipe: gh nightly bindings - name: Rust bench targets @@ -244,7 +247,7 @@ jobs: # Explicit Pages deployment works with GITHUB_TOKEN; a branch push alone does not. benchmark-site: name: Benchmark site - needs: nightly + needs: [nightly, sizes] if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04-arm timeout-minutes: 15 @@ -671,6 +674,21 @@ jobs: size-report/js/*.metafile.json retention-days: 45 if-no-files-found: error + # The benchmark table records tonight's sizes even when the comparison fails on growth. + - name: Convert sizes for the benchmark table + id: size-table + if: ${{ !cancelled() && hashFiles('size-report/sizes.json') != '' }} + run: nix develop --command just gh benchmark-report size size-report target/bench/size/report.json + - name: Record benchmark outcome + if: ${{ !cancelled() }} + run: nix develop --command just gh benchmark-report status "${{ steps.size-table.outcome }}" target/bench/size + - name: Preserve benchmark report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: bench-size + path: target/bench/size + retention-days: 45 fuzz: name: Fuzz (${{ matrix.target }}) diff --git a/bench/README.md b/bench/README.md index cbf97c7778d..ca8d8904759 100644 --- a/bench/README.md +++ b/bench/README.md @@ -81,8 +81,8 @@ throttled one: nix develop --command just rs test -p moq-tokio --release -E 'test(benchmark_datagram_delivery)' --run-ignored only --success-output immediate ``` -It reports offered, received and lost datagrams, bottleneck queue drops, reordered -arrivals and append-to-receive latency per viewer, and fails if the loopback +It reports offered and lost datagrams, bottleneck queue drops, QUIC DATAGRAM frames +that delivered no datagram, reordered arrivals and append-to-receive latency per viewer, and fails if the loopback viewer loses any. It runs nightly. Measure 50 fps audio at per-packet, 100 and 200 ms grouping, plus 400 fps @@ -620,16 +620,27 @@ The table includes these measurements: time elapsed time. The Rust bench target jobs measure them with a short warmup and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates trade precision for nightly runtime; the paired comparison below assesses a change. -- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation - variants) and `size_bytes` columns, retaining every other column as a workload - dimension in each series name. +- The CSV tables the JS benchmarks and the Rust benchmark tests print. A header + line names the columns; the time columns (`ns`, `us`, `ms`, and their + per-operation variants such as `ns_per_frame` or `plain_ns_per_read`), + `size_bytes` and `peak_bytes`, `mib_per_s`, and `*_packets` columns are + measurements, and every other column is a workload dimension in each series + name, so print outcomes only in measured columns. The Rust benchmark tests + (`just gh nightly rust-test`) run in release builds, one at a time, under + nextest's `bench` profile, which keeps each test's output in a JUnit report; + their series are named by package and test path, such as + `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. +- Shipping sizes, in bytes: the release artifacts, minimal FFI build, and JS + bundles the [size report](../sh/size/README.md) measures, recorded even on a + night its growth check fails. Release build times stay out. Hosted runners vary across nights, so a trend flag prompts a look at the paired comparison rather than proving a regression. Malformed, empty, nonfinite, or duplicate measurements fail validation. The -nightly artifacts retain the reports, raw JS output, and each Criterion run's -metadata and estimates for 45 days. Relay load, audio quality, and the Rust -benchmark tests job's nextest output do not enter the table. +nightly artifacts retain the reports, raw JS output, nextest JUnit reports, and +each Criterion run's metadata and estimates for 45 days. Relay load, audio +quality, and the binding runners (measured in rust2, above) do not enter the +table. Render the table from the published history: diff --git a/quest/m2/bench-sources.md b/quest/m2/bench-sources.md index 66182dbcce2..ebb29b517b9 100644 --- a/quest/m2/bench-sources.md +++ b/quest/m2/bench-sources.md @@ -9,24 +9,23 @@ is a smoke run that measures nothing. ## Plan Split from the benchmark trend site on 2026-10-20, when that quest became -the m0 table. Sources decided with the maintainer on 2026-10-15: the binding -benchmark's JSON and the "Rust benchmark tests", "Rust bench targets" and "JS -benchmarks" nightly jobs, converted to the history input format where they -don't already emit one. +the m0 table. The Criterion targets, the JS benchmarks, the Rust benchmark +tests and the nightly shipping sizes feed it now +([moq-dev/moq#7510](https://github.com/moq-dev/moq/pull/7510) added the last +two). The binding runners stay an unmeasured smoke run: rust2 measures them, +and [moq-dev/moq#6629](https://github.com/moq-dev/moq/pull/6629) retired their +series. The relay load benches left Nightly in +[moq-dev/moq#7603](https://github.com/moq-dev/moq/pull/7603). What remains joins +through the same input format; the sources decided with the maintainer on +2026-10-15 left it for later: -- Criterion targets now measure through `just gh criterion` in - `sh/gh/nightly.sh` (`rust-bench`, `rust-shared`, `rust-extra`, `rust-uring`), and the - comparison harness uses quick mode. Check their published histories - rather than converting them again; the unreported sources below remain. -- The "Rust benchmark tests" job is not recorded yet: its nextest benchmarks - print no structured measurements. -- Check each JS bench reports a measurement before recording it. -- Nightly size history (today a 45-day artifact) is a natural extra series - if it fits cheaply. -- The audio-quality job's benchmarks can join later through the same input - format. -- The paired nightly comparison publishes beside the table from the same - `gh-pages` branch (`bench/paired.json`); a new Rust source joins it through - `just gh benchmark-report rust`. +- The audio-quality job's benchmarks: `just js qmux-bench`, `serve-bench` and + `ring-bench`. Check each prints a CSV table the `js` converter reads before + recording it. + +A source joins as a `bench-*` artifact with a recorded outcome, which the +benchmark-site job collects. The paired nightly comparison publishes beside the +table from the same `gh-pages` branch (`bench/paired.json`); a new Rust source +joins it through `just gh benchmark-report rust`. Public API: none. Wire: none. diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index e4b433a5d4d..c112a218ae2 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -707,6 +707,7 @@ mod tests { #[tokio::test] #[ignore = "measures real codec CPU; run with just rs bench-playback"] async fn benchmark_raw_audio() { + println!("tracks,retention_secs,always_on,idle_ms,catchup_ms"); for tracks in [1, 8] { for seconds in [1, 30] { for active in [true, false] { @@ -747,7 +748,9 @@ mod tests { } let catchup = start.elapsed(); println!( - "tracks={tracks} retention={seconds}s always_on={active}: idle={idle:?} catchup={catchup:?}" + "{tracks},{seconds},{active},{:.3},{:.3}", + idle.as_secs_f64() * 1e3, + catchup.as_secs_f64() * 1e3 ); for producer in &mut producers { producer.finish().unwrap(); diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index a53ea229d85..d1828c24262 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -497,6 +497,7 @@ mod tests { reason = "the benchmark measures mixer callbacks on the host clock" )] fn benchmark_mixer_callbacks() { + println!("sinks,frames,ns_per_callback"); for sinks in [1, 8, 32, MAX_SINKS] { for frames in [128, 480, 1024, 2048] { let mut harness = Harness::with_depth(Layout::Stereo, MAX_SINKS); @@ -514,10 +515,7 @@ mod tests { elapsed += start.elapsed(); std::hint::black_box(&out); } - println!( - "sinks={sinks} frames={frames}: {:.1} ns/callback", - elapsed.as_nanos() as f64 / 1000.0 - ); + println!("{sinks},{frames},{:.1}", elapsed.as_nanos() as f64 / 1000.0); } } } diff --git a/rs/moq-audio/src/resample.rs b/rs/moq-audio/src/resample.rs index 04f6eabf4a2..85f1f318022 100644 --- a/rs/moq-audio/src/resample.rs +++ b/rs/moq-audio/src/resample.rs @@ -539,6 +539,7 @@ mod tests { #[test] #[ignore = "run with just rs bench-playback"] fn benchmark_resampler() { + println!("input_rate,output_rate,channels,chunk,packet,ns_per_input_frame"); for (input_rate, output_rate) in [(44_100, 48_000), (48_000, 44_100), (96_000, 48_000)] { for channels in [1, 2, 6] { let mut chunks = vec![input_rate as usize / 50, (input_rate as usize / 50).clamp(1, 960)]; @@ -567,7 +568,7 @@ mod tests { assert_eq!(count, output_rate as usize * channels as usize); } println!( - "{input_rate}->{output_rate} channels={channels} chunk={chunk} packet={packet}: {:.1} ns/input-frame", + "{input_rate},{output_rate},{channels},{chunk},{packet},{:.1}", elapsed.as_nanos() as f64 / (10 * frames) as f64 ); } diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 2396104f57a..210caa9b2b2 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -292,4 +292,5 @@ to the results. `just rs bench-latency` measures latency recording with 1, 4 and 16 concurrent writers and 0, 10 and 100 snapshots per run. It checks final sample conservation -and prints throughput for each case. The nightly workflow runs the same matrix. +and prints the cost per sample for each case. The nightly workflow runs the same +matrix and records it on the benchmark table. diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 05ffb39a3f3..05409c94558 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -766,6 +766,7 @@ mod tests { reason = "the benchmark measures real establishment time" )] fn benchmark_readiness_establishment() { + println!("connections,broadcasts_per_connection,elapsed_ns"); for connections in [1, 16, 128] { for broadcasts in [1, 16, 128] { let count = connections * broadcasts; @@ -793,10 +794,7 @@ mod tests { } } assert!(std::hint::black_box(stats.ready())); - println!( - "readiness connections={connections} broadcasts_per_connection={broadcasts} elapsed_ns={}", - start.elapsed().as_nanos() - ); + println!("{connections},{broadcasts},{}", start.elapsed().as_nanos()); assert_eq!(stats.broadcasts_established.load(Ordering::Relaxed), count); } } diff --git a/rs/moq-bench/src/stats.rs b/rs/moq-bench/src/stats.rs index a489a96883e..45e121418e5 100644 --- a/rs/moq-bench/src/stats.rs +++ b/rs/moq-bench/src/stats.rs @@ -788,6 +788,7 @@ mod tests { use std::sync::Barrier; use std::time::Instant; + println!("writers,snapshots,ns_per_sample"); for writers in [1, 4, 16] { for snapshots in [0, 10, 100] { let stats = Stats::default(); @@ -821,10 +822,8 @@ mod tests { let elapsed = elapsed.elapsed(); assert_eq!(Snapshot::take(&stats).latency_samples, (writers * samples) as u64); println!( - "writers={writers} snapshots={snapshots} samples={} elapsed_ns={} samples_per_second={:.0}", - writers * samples, - elapsed.as_nanos(), - (writers * samples) as f64 / elapsed.as_secs_f64() + "{writers},{snapshots},{:.2}", + elapsed.as_nanos() as f64 / (writers * samples) as f64 ); } } diff --git a/rs/moq-c/src/id.rs b/rs/moq-c/src/id.rs index 820a360bd9b..30210f988b5 100644 --- a/rs/moq-c/src/id.rs +++ b/rs/moq-c/src/id.rs @@ -148,6 +148,7 @@ mod tests { fn benchmark_handles() { use std::hint::black_box; + println!("live,wrap,ns_per_iteration"); for live in [0, 64, 4096] { for wrap in [false, true] { let next = AtomicU32::new(live + 1); @@ -166,8 +167,8 @@ mod tests { let id = slab.insert_next(black_box(42), &next).unwrap(); black_box(slab.remove(id).unwrap()); } - eprintln!( - "handles live={live} wrap={wrap}: {:.0} ns/iteration", + println!( + "{live},{wrap},{:.0}", start.elapsed().as_nanos() as f64 / f64::from(iterations) ); } diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 7d3fa239934..024a8b29c2e 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1988,6 +1988,7 @@ pub(crate) mod tests { let mut null = [0xff; 188]; null[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); let padding = null.repeat(INPUT.len() / 188); + println!("copies,ns_per_byte"); for copies in [1, 16, 128] { let mut file = tempfile::NamedTempFile::new().unwrap(); file.write_all(INPUT).unwrap(); @@ -2012,10 +2013,7 @@ pub(crate) mod tests { decode(decoder, catalog, input, Publication::default()).await.unwrap(); let elapsed = start.elapsed(); let bytes = INPUT.len() * copies; - println!( - "bytes={bytes}: {elapsed:?}, {:.2} Mbit/s", - bytes as f64 * 8.0 / elapsed.as_secs_f64() / 1e6 - ); + println!("{copies},{:.3}", elapsed.as_nanos() as f64 / bytes as f64); } } diff --git a/rs/moq-congestion/src/bbr3/bench.rs b/rs/moq-congestion/src/bbr3/bench.rs index 6cc91f6b01c..4d8e7b79120 100644 --- a/rs/moq-congestion/src/bbr3/bench.rs +++ b/rs/moq-congestion/src/bbr3/bench.rs @@ -90,11 +90,10 @@ fn measure(flight: usize, batch: usize, steady: bool) -> (Duration, usize) { #[test] #[ignore = "controller benchmark: run with --release and --success-output immediate"] fn benchmark_per_packet() { - println!( - "bbr_packet controller_stack_bytes={} packet_state_bytes={}", - size_of::(), - size_of::() - ); + println!("state,size_bytes"); + println!("controller_stack,{}", size_of::()); + println!("packet,{}", size_of::()); + println!("steady,flight,batch,cpu_ns,peak_bytes"); for steady in [false, true] { for flight in [16, 64, 1000, 4000, 16000] { for batch in [1, 8, 32] { @@ -107,7 +106,7 @@ fn benchmark_per_packet() { } samples.sort(); println!( - "bbr_packet steady={steady} flight={flight} batch={batch} cpu_ns={} peak_bytes={bytes}", + "{steady},{flight},{batch},{},{bytes}", samples[1].as_nanos() ); } diff --git a/rs/moq-congestion/src/gcc/bench.rs b/rs/moq-congestion/src/gcc/bench.rs index 1fa6ca9e01d..99cb69839a1 100644 --- a/rs/moq-congestion/src/gcc/bench.rs +++ b/rs/moq-congestion/src/gcc/bench.rs @@ -75,7 +75,9 @@ fn measure(flight: usize, batch: usize, steady: bool) -> Duration { #[test] #[ignore = "controller benchmark: run with --release and --success-output immediate"] fn benchmark_per_packet() { - println!("gcc_packet controller_stack_bytes={}", size_of::()); + println!("state,size_bytes"); + println!("controller_stack,{}", size_of::()); + println!("steady,flight,batch,cpu_ns"); for steady in [false, true] { for flight in [16, 64, 1000, 4000, 16000] { for batch in [1, 8, 32] { @@ -84,10 +86,7 @@ fn benchmark_per_packet() { *sample = measure(flight, batch, steady); } samples.sort(); - println!( - "gcc_packet steady={steady} flight={flight} batch={batch} cpu_ns={}", - samples[1].as_nanos() - ); + println!("{steady},{flight},{batch},{}", samples[1].as_nanos()); } } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index ef89eaa2d1f..335f0017253 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -2086,7 +2086,7 @@ mod timebase_tests { #[test] #[ignore = "benchmark, runs in Nightly"] fn benchmark_timebase_reservation_fanout() { - eprintln!("timebases,containers,ns_per_container"); + println!("timebases,containers,ns_per_container"); for entries in [1, 32, 1024] { let owner = MoqBroadcastProducer::new().unwrap(); let timebases: Vec<_> = (0..entries) @@ -2115,7 +2115,7 @@ mod timebase_tests { } } let nanos = start.elapsed().as_nanos() as f64 / (1000 * touched) as f64; - eprintln!("{entries},{touched},{nanos:.1}"); + println!("{entries},{touched},{nanos:.1}"); } owner.close().unwrap(); } diff --git a/rs/moq-mux/src/container/ts/damage.rs b/rs/moq-mux/src/container/ts/damage.rs index 9676a913628..fbbdb9f570a 100644 --- a/rs/moq-mux/src/container/ts/damage.rs +++ b/rs/moq-mux/src/container/ts/damage.rs @@ -89,7 +89,7 @@ mod tests { )] async fn benchmark_damage_pid_fanout() { let error = anyhow::anyhow!("broken PES"); - eprintln!("pids,damaged_units,ns_per_unit"); + println!("pids,damaged_units,ns_per_unit"); for pids in [1_u16, 64, 8192] { for units in [1, 16, 128] { let mut log = Log::default(); @@ -108,7 +108,7 @@ mod tests { } } let nanos = start.elapsed().as_nanos() as f64 / (10_000 * units) as f64; - eprintln!("{pids},{units},{nanos:.1}"); + println!("{pids},{units},{nanos:.1}"); } } } diff --git a/rs/moq-tokio/tests/integration/datagram_delivery.rs b/rs/moq-tokio/tests/integration/datagram_delivery.rs index 164aa1d1aee..a0035758069 100644 --- a/rs/moq-tokio/tests/integration/datagram_delivery.rs +++ b/rs/moq-tokio/tests/integration/datagram_delivery.rs @@ -290,12 +290,11 @@ async fn measure(version: &str, cell: Cell) { 0 }; println!( - "{version},{},{},{offered},{},{},{overflowed},{},{},{:.1},{:.1},{:.1}", + "{version},{},{},{offered},{},{overflowed},{},{},{:.1},{:.1},{:.1}", cell.name, viewer.name, - report.received, offered as usize - report.received, - report.frames, + report.frames as i64 - report.received as i64, report.reordered, report.percentile(50), report.percentile(99), @@ -313,9 +312,12 @@ async fn measure(version: &str, cell: Cell) { } /// Datagram delivery to a loopback viewer and a bufferbloated one, below and above the -/// bottleneck's rate. Reports per viewer the datagrams offered, received, lost, those the +/// bottleneck's rate. Reports per viewer the datagrams offered and lost, those the /// bottleneck's full queue dropped (UDP packets, so approximate), QUIC DATAGRAM frames -/// received, reordered arrivals, and append-to-receive latency percentiles. Nothing on the +/// received that delivered no datagram (padding, or ones the session discarded), +/// reordered arrivals, and append-to-receive latency percentiles. Only `offered` is a +/// workload dimension; the counts are `_packets` columns, all smaller-is-better, so the +/// nightly benchmark table records them as measurements (see bench/README.md). Nothing on the /// path expires a datagram by age: what the sender's QUIC datagram buffer holds waits /// there until it is sent or displaced. Fails if a datagram arrives twice or corrupted, or /// if the loopback viewer loses any. Run by nightly. @@ -334,7 +336,9 @@ async fn benchmark_datagram_delivery() { interval: Duration::from_millis(2), }, ]; - println!("version,cell,viewer,offered,received,lost,overflowed,quic_frames,reordered,p50_ms,p99_ms,max_ms"); + println!( + "version,cell,viewer,offered,lost_packets,overflowed_packets,undelivered_frame_packets,reordered_packets,p50_ms,p99_ms,max_ms" + ); for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { for cell in cells { measure(version, cell).await; diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 475dd3650ee..4c1c52ae3c5 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -173,7 +173,7 @@ pub(crate) fn rand_seed() -> u64 { .saturating_duration_since(kio::clock::Wall::UNIX_EPOCH) .as_nanos() as u64, }; - println!("MOQ_SHAPER_SEED={seed}"); + eprintln!("MOQ_SHAPER_SEED={seed}"); seed } @@ -1088,7 +1088,7 @@ async fn benchmark_receive_windows() { ("loopback", None, BENCH_BYTES), ("bufferbloat", Some(bufferbloat(BENCH_BLOAT.0)), BENCH_BLOAT.1), ]; - println!("path,axis,streams,frame,window_mib,stream_window,peak_mib,mib_per_s,p50_ms,p99_ms,overflowed"); + println!("path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,overflowed_packets"); for (cell, (path, profile, bytes)) in cells.flat_map(|cell| paths.clone().map(|path| (cell, path))) { let mut runs = Vec::with_capacity(RUNS); for _ in 0..RUNS { @@ -1109,13 +1109,12 @@ async fn benchmark_receive_windows() { let throughput = median(runs.iter().map(|run| run.throughput).collect()); let overflowed = median(runs.iter().map(|run| run.overflowed as f64).collect()); println!( - "{path},{},{},{},{},{},{:.1},{:.0},{:.2},{:.2},{overflowed}", + "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{overflowed}", cell.axis, cell.streams, cell.frame, cell.window >> 20, if cell.follow { "follow" } else { "finite" }, - peak / (1 << 20) as f64, throughput / (1 << 20) as f64, percentile(50), percentile(99), diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 679c8c9a744..6e7f55c5ea6 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -10,12 +10,13 @@ import re import statistics import time +import xml.etree.ElementTree as ElementTree from datetime import datetime, timezone from pathlib import Path from urllib.parse import quote BIGGER = {"MB/s", "MiB/s", "B/s", "ops/s", "frames/s"} -SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB"} +SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB", "packets"} def value(number): @@ -131,45 +132,89 @@ def gate(directory): def measurement_unit(column): - if column == "size_bytes": + """The unit a benchmark CSV column measures, or None for a workload dimension.""" + if column in ("size_bytes", "peak_bytes"): return "bytes" + if column == "mib_per_s" or column.endswith("_mib_per_s"): + return "MiB/s" + if column.endswith("_packets"): + return "packets" for unit in ("ns", "us", "ms"): - if column == unit or column.endswith(f"_{unit}") or column.startswith(f"{unit}_per_"): + if re.search(f"(^|_){unit}($|_per_)", column): return unit return None -def javascript_file(path): +def printed(text, name, where): + """Read the CSV tables a benchmark printed: a header naming measured columns, then its rows.""" entries, headers = [], None - for line_number, line in enumerate(path.read_text().splitlines(), 1): + for line_number, line in enumerate(text.splitlines(), 1): row = [field.strip() for field in next(csv.reader([line]))] if len(row) > 1 and any(measurement_unit(field) for field in row): if len(set(row)) != len(row): - raise ValueError(f"duplicate CSV headers: {path}:{line_number}") + raise ValueError(f"duplicate CSV headers: {where}:{line_number}") headers = row continue if not headers or not line.strip(): continue if len(row) != len(headers): - raise ValueError(f"malformed benchmark CSV: {path}:{line_number}") + raise ValueError(f"malformed benchmark CSV: {where}:{line_number}") dimensions = "/".join( f"{key}={quote(field, safe='-_.~')}" for key, field in zip(headers, row) if measurement_unit(key) is None ) for key, field in zip(headers, row): if unit := measurement_unit(key): - entries.append( - {"name": f"js/{path.stem}/{dimensions}/{key}", "unit": unit, "value": value(float(field))} - ) + entries.append({"name": f"{name}/{dimensions}/{key}", "unit": unit, "value": value(float(field))}) if not entries: - raise ValueError(f"JS benchmark produced no measurements: {path}") - return checked(entries) + raise ValueError(f"benchmark produced no measurements: {where}") + return entries def javascript(directory): - entries = [entry for path in sorted((directory / "raw").glob("*.csv")) for entry in javascript_file(path)] + entries = [ + entry + for path in sorted((directory / "raw").glob("*.csv")) + for entry in printed(path.read_text(), f"js/{path.stem}", path) + ] return checked(entries) +# The lines libtest's harness prints around, and during, each test's own output. +LIBTEST = re.compile( + r"running \d+ tests?|test \S+ \.\.\. \w+|test \S+ has been running for over \d+ seconds" + r"|test result: .*" +) + + +def rust_tests(directory): + """Read each benchmark test's tables from the JUnit reports of nextest's bench profile.""" + reports = sorted((directory / "raw").glob("*.xml")) + if not reports: + raise ValueError(f"no nextest JUnit reports in {directory / 'raw'}") + entries = [] + for path in reports: + for case in ElementTree.parse(path).getroot().iter("testcase"): + # A test binary is named `package` or `package::kind/target`. The test's module + # path stays in its name, since modules of one package can name a test alike. + package = case.get("classname").split("::", 1)[0] + test = case.get("name").replace("::", "/") + output = case.findtext("system-out") or "" + text = "\n".join(line for line in output.splitlines() if not LIBTEST.fullmatch(line)) + entries += printed(text, f"rust-test/{package}/{test}", f"{path.name} {package} {test}") + return checked(entries) + + +def sizes(directory): + """Record the nightly size report's shipping artifact bytes; its build times stay out, being noise.""" + report = json.loads((directory / "sizes.json").read_text()) + return checked( + [ + {"name": f"size/{key}", "unit": "bytes", "value": value(size)} + for key, size in sorted(report["bytes"].items()) + ] + ) + + def prepare(directory, output): costs, rates, comparisons = [], [], {} for source in sorted(directory.glob("bench-*")): @@ -488,7 +533,7 @@ def index(directory, commit=None, paired_path=None): def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) - for command in ("rust", "js", "prepare"): + for command in ("rust", "rust-test", "js", "size", "prepare"): sub = commands.add_parser(command) sub.add_argument("directory", type=Path) sub.add_argument("output", type=Path) @@ -508,8 +553,12 @@ def main(): args.output.with_name("paired.json").write_text(json.dumps(comparison, indent=2) + "\n") elif args.command == "gate": raise SystemExit(gate(args.directory)) + elif args.command == "rust-test": + write(args.output, rust_tests(args.directory)) elif args.command == "js": write(args.output, javascript(args.directory)) + elif args.command == "size": + write(args.output, sizes(args.directory)) elif args.command == "prepare": prepare(args.directory, args.output) elif args.command == "status": diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 6337d2c35c0..363d000c51a 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -35,21 +35,63 @@ def test_values_refuse_smoke_and_malformed_reports(self): with self.assertRaises(ValueError): report.checked(entries) + def javascript(self, name, text): + source = self.root / "raw" / f"{name}.csv" + source.parent.mkdir(exist_ok=True) + source.write_text(text) + return report.javascript(self.root) + def test_js_multiple_schemas(self): - source = self.root / "requests.csv" - source.write_text("bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\noperation,ns_per_op\nclose,0\n") - entries = report.javascript_file(source) + entries = self.javascript( + "requests", "bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\noperation,ns_per_op\nclose,0\n" + ) self.assertEqual([entry["value"] for entry in entries], [1.25, 2.5, 0]) self.assertEqual(entries[0]["name"], "js/requests/paths=4/active_us") for text in ("Testing case\nSuccess\n", "paths,time_us\n4\n", "paths,time_us\n4,nan\n"): - source.write_text(text) with self.assertRaises(ValueError): - report.javascript_file(source) + self.javascript("requests", text) + + def test_measured_columns(self): + units = { + column: report.measurement_unit(column) + for column in ( + "us", + "elapsed_ms", + "ns_per_frame", + "plain_ns_per_read", + "size_bytes", + "peak_bytes", + "mib_per_s", + "overflowed_packets", + "payload_bytes", + "bytes", + "frames_per_chunk", + "nsamples", + "runs", + ) + } + self.assertEqual( + units, + { + "us": "us", + "elapsed_ms": "ms", + "ns_per_frame": "ns", + "plain_ns_per_read": "ns", + "size_bytes": "bytes", + "peak_bytes": "bytes", + "mib_per_s": "MiB/s", + "overflowed_packets": "packets", + # Workload dimensions, as the JS benchmarks name them. + "payload_bytes": None, + "bytes": None, + "frames_per_chunk": None, + "nsamples": None, + "runs": None, + }, + ) def test_js_byte_measurements_do_not_become_dimensions(self): - source = self.root / "compression.csv" - source.write_text("bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n") - entries = report.javascript_file(source) + entries = self.javascript("compression", "bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n") self.assertEqual( [(entry["name"], entry["unit"], entry["value"]) for entry in entries], [ @@ -58,9 +100,109 @@ def test_js_byte_measurements_do_not_become_dimensions(self): ("js/compression/measurement=bundle/size_bytes", "bytes", 512), ], ) - source.write_text("bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n") + later = self.javascript("compression", "bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n") + self.assertEqual([entry["name"] for entry in later], [entry["name"] for entry in entries]) + + def junit(self, name, cases): + """Write a nextest JUnit report of (binary, test, stdout) cases.""" + raw = self.root / "raw" + raw.mkdir(exist_ok=True) + body = "".join( + f'{stdout}' + "" + for binary, test, stdout in cases + ) + (raw / name).write_text(f'{body}') + + def test_rust_tests_record_each_benchmark_by_package_and_test(self): + with self.assertRaises(ValueError): + report.rust_tests(self.root) + self.junit( + "nextest-1.xml", + [ + ( + "moq-congestion", + "bbr3::bench::benchmark_per_packet", + "\nrunning 1 test\nstate,size_bytes\npacket,40\n" + "steady,flight,cpu_ns,peak_bytes\nfalse,16,3170,1336\n" + "test bbr3::bench::benchmark_per_packet ... ok\n\n" + "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 97 filtered out; finished in 0.05s\n", + ), + # Another controller's benchmark, named alike in the same package. + ( + "moq-congestion", + "gcc::bench::benchmark_per_packet", + "steady,flight,cpu_ns\nfalse,16,2210\n", + ), + ], + ) + self.junit( + "nextest-2.xml", + [ + ( + "moq-tokio::integration", + "receive_credit::benchmark_receive_windows", + "path,mib_per_s,overflowed_packets\nloopback,812,0\n" + "test receive_credit::benchmark_receive_windows has been running for over 60 seconds\n", + ), + ( + "moq-bench::bin/moq-bench", + "stats::tests::benchmark_observation_contention", + "writers,snapshots,ns_per_sample\n1,0,57.39\n", + ), + ], + ) + entries = report.rust_tests(self.root) + self.assertEqual( + [(entry["name"], entry["unit"], entry["value"]) for entry in entries], + [ + ("rust-test/moq-congestion/bbr3/bench/benchmark_per_packet/state=packet/size_bytes", "bytes", 40), + ("rust-test/moq-congestion/bbr3/bench/benchmark_per_packet/steady=false/flight=16/cpu_ns", "ns", 3170), + ( + "rust-test/moq-congestion/bbr3/bench/benchmark_per_packet/steady=false/flight=16/peak_bytes", + "bytes", + 1336, + ), + ("rust-test/moq-congestion/gcc/bench/benchmark_per_packet/steady=false/flight=16/cpu_ns", "ns", 2210), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/mib_per_s", + "MiB/s", + 812, + ), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/overflowed_packets", + "packets", + 0, + ), + ( + "rust-test/moq-bench/stats/tests/benchmark_observation_contention/writers=1/snapshots=0/ns_per_sample", + "ns", + 57.39, + ), + ], + ) + # A benchmark that prints no table, or a line its table cannot read, is refused. + for stdout in ("\nrunning 1 test\nwriters=1 elapsed_ns=5\n", "writers,ns_per_sample\n1,2\nwarming up\n"): + self.junit("nextest-2.xml", [("moq-bench", "benchmark_observation_contention", stdout)]) + with self.assertRaises(ValueError): + report.rust_tests(self.root) + + def test_sizes_record_shipping_bytes(self): + (self.root / "sizes.json").write_text( + json.dumps( + { + "version": 1, + "bytes": {"release/moq-relay/moq-relay.tar.gz": 900, "js/net/gzip": 120}, + "seconds": {"release-moq-relay": 300}, + } + ) + ) self.assertEqual( - [entry["name"] for entry in report.javascript_file(source)], [entry["name"] for entry in entries] + report.sizes(self.root), + [ + {"name": "size/js/net/gzip", "unit": "bytes", "value": 120}, + {"name": "size/release/moq-relay/moq-relay.tar.gz", "unit": "bytes", "value": 900}, + ], ) def test_criterion_requires_fresh_measured_estimates(self): diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 30fa14f8392..d13ac7d5b18 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -7,10 +7,12 @@ if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 fi -# The measured groups write a report for the benchmark site; rust-test and bindings only run. +# The measured groups write a report for the benchmark site; bindings only run, since +# rust2 measures them (see bench/README.md). case "$1" in - rust-test | bindings) report= ;; + bindings) report= ;; rust-bench | rust-shared | rust-relay | rust-extra | rust-uring) report=rust ;; + rust-test) report=rust-test ;; js) report=js ;; *) echo "$usage" >&2 @@ -48,7 +50,8 @@ if [[ -n $report ]]; then mkdir -p "$out" export BENCH_RUNS="$out/criterion-runs.txt" : >"$BENCH_RUNS" - rm -f "$out/report.json" "$out/paired.json" + rm -rf "$out/report.json" "$out/paired.json" "$out/raw" + mkdir -p "$out/raw" fi failed=() @@ -66,34 +69,48 @@ criterion() { javascript() { local label=$1 shift - mkdir -p "$out/raw" echo "nightly: $*" if ! "$@" | tee "$out/raw/$label.csv"; then failed+=("$*"); fi } +# The bench nextest profile stores each test's printed tables in its JUnit report, +# which every run overwrites; keep each run's copy. +nextest() { + local junit=${CARGO_TARGET_DIR:-$root/target}/nextest/bench/junit.xml + rm -f "$junit" + run "$@" + if [[ -s $junit ]]; then + mv "$junit" "$out/raw/nextest-$((++nextest_runs)).xml" + else + failed+=("no JUnit report from: $*") + fi +} +nextest_runs=0 + case "$1" in - # Ignored benchmark tests under nextest: the release builds first, then the test-profile ones. + # Ignored benchmark tests under nextest, all in release builds and one at a time, so + # none measures beside another. Each prints CSV tables whose unit-suffixed columns + # the report records (see bench/README.md). rust-test) - run just rs bench-latency - run just rs test -p moq-bench --release --run-ignored only \ + export NEXTEST_PROFILE=bench NEXTEST_TEST_THREADS=1 + nextest just rs bench-latency + nextest just rs test -p moq-bench --release --run-ignored only \ -E 'test(benchmark_readiness_establishment)' --success-output immediate - run just rs bench-playback - run just rs test --release -p moq-congestion -E 'test(benchmark_per_packet)' \ + nextest just rs bench-playback + nextest just rs test --release -p moq-congestion -E 'test(benchmark_per_packet)' \ --run-ignored only --success-output immediate - run just rs bench-feed-fetch - run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate - run just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate - run just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate - run just rs test -p moq-tokio --release -E 'test(benchmark_datagram_delivery)' --run-ignored only --success-output immediate - run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate - run just rs test -p moq-c -E 'test(benchmark_handles)' --run-ignored only --success-output immediate - run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ - -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_reference_handoff_fanout)' \ + nextest just rs bench-feed-fetch + nextest just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate + nextest just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate + nextest just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate + nextest just rs test -p moq-tokio --release -E 'test(benchmark_datagram_delivery)' --run-ignored only --success-output immediate + # One build for the crates whose benchmarks need no features of their own. + nextest just rs test --release --no-fail-fast -p moq-relay -p moq-c -p moq-mux -p moq-cli \ + -E 'test(benchmark_stun_sources) | test(benchmark_handles) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_stream_import)' \ --run-ignored only --success-output immediate # Codecs are irrelevant to timebase reservation, so skip building them. - run just rs test -p moq-ffi --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ + nextest just rs test -p moq-ffi --release --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ --run-ignored only --success-output immediate - run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate ;; # Measured Cargo bench targets, all in the bench profile. A paired run builds and # measures each group twice, so the groups split across jobs to fit an hour each. diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index a3e88f90fe2..196e57c7ad9 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -26,6 +26,11 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): "#!/usr/bin/env bash\n" 'printf "%s %s\n" "${0##*/}" "$*" >> "$NIGHTLY_CALLS"\n' 'printf "%s\n" "${BENCH_BASE:-}" >> "$NIGHTLY_CALLS.base"\n' + # Nextest's bench profile writes a JUnit report, unless the test withholds it. + 'if [[ ${NEXTEST_PROFILE:-} == bench && "$*" != "$NIGHTLY_NO_JUNIT" ]]; then\n' + ' mkdir -p "$CARGO_TARGET_DIR/nextest/bench"\n' + ' printf "" > "$CARGO_TARGET_DIR/nextest/bench/junit.xml"\n' + "fi\n" '[[ "$*" != "$NIGHTLY_FAIL" ]]\n' ) stub.chmod(0o755) @@ -43,12 +48,16 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null", + CARGO_TARGET_DIR=str(scratch / "target"), ) - env.pop("BENCH_BASE", None) + for name in ("BENCH_BASE", "NEXTEST_PROFILE", "NIGHTLY_NO_JUNIT"): + env.pop(name, None) env.update(environment) result = subprocess.run( ["bash", ROOT / "sh/gh/nightly.sh", group], cwd=cwd, env=env, capture_output=True, text=True ) + raw = scratch / "results" / "raw" + self.raw = sorted(path.name for path in raw.iterdir()) if raw.exists() else [] self.bases = set(Path(f"{calls}.base").read_text().splitlines()) if calls.exists() else set() return result, calls.read_text().splitlines() if calls.exists() else [] @@ -220,12 +229,34 @@ def test_dev_shell_jobs_trust_the_nix_cache(self): def test_benchmark_tests_stay_out_of_cargo_bench(self): result, calls = self.run_group("rust-test") self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in calls)) - # Release builds run before the test-profile ones. - profiles = ["--release" in call or " bench-" in call for call in calls] - self.assertEqual(profiles, sorted(profiles, reverse=True)) + *runs, publish = calls + self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in runs)) + # Every benchmark measures a release build, and each run keeps its JUnit report. + self.assertTrue(all("--release" in call or " bench-" in call for call in runs), runs) + self.assertIn("just rs bench-feed-fetch", runs) + self.assertEqual(set(self.raw), {f"nextest-{index}.xml" for index in range(1, len(runs) + 1)}) + self.assertTrue(publish.startswith("just gh benchmark-report rust-test "), publish) + + def test_benchmark_test_without_its_report_fails(self): + result, calls = self.run_group("rust-test", NIGHTLY_NO_JUNIT="rs bench-latency") + self.assertEqual(result.returncode, 1) + self.assertIn("no JUnit report from: just rs bench-latency", result.stderr) self.assertIn("just rs bench-feed-fetch", calls) - self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) + self.assertFalse(any(call.startswith("just gh") for call in calls)) + + def test_every_nightly_benchmark_test_exists(self): + # A filter naming a removed test matches nothing, and nextest still passes. + recipes = (ROOT / "rs/justfile").read_text() + _, calls = self.run_group("rust-test") + named = set() + for call in calls: + if match := re.fullmatch(r"just rs (bench-[\w-]+)", call): + call = re.search(rf"^{match[1]}\b.*:\n (.+)$", recipes, re.M)[1] + named |= set(re.findall(r"test\((\w+)\)", call)) + self.assertTrue(named) + sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path)) + missing = {name for name in named if not re.search(rf"\bfn {name}\b", sources)} + self.assertEqual(missing, set(), "Nightly benchmark tests that no longer exist") def test_failed_release_build_does_not_hide_later_work(self): _, calls = self.run_group("rust-bench") diff --git a/sh/size/README.md b/sh/size/README.md index bd1142914ba..fde6b67aba1 100644 --- a/sh/size/README.md +++ b/sh/size/README.md @@ -29,4 +29,6 @@ gh api --paginate 'repos/{owner}/{repo}/actions/artifacts?name=nightly-size-hist A regression fails Nightly and reaches its existing alert workflow. The job summary carries the byte counts, build times and FFI crate attribution, the -latter from the symbol-preserving `profiling` profile. +latter from the symbol-preserving `profiling` profile. Each night's byte counts +also join the [benchmark table](https://git.moq.dev/bench/), whose history +outlives the 45-day artifacts.