From 57d9213fb4bcef2013f80080fdc5ff5df9bb6ff2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:16:52 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m2/bench-sources.md Quest-Claim: 5866126e3504e8e0 From 311be813febe1d3e86251bb344d6703efc5fb06d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:32:33 -0700 Subject: [PATCH 2/5] bench: record the Rust benchmark tests and shipping sizes on the nightly table The nightly benchmark tests print CSV tables that nextest's bench profile keeps in a JUnit report, run one at a time in release builds, and the size job's byte counts join the table beside them. Co-Authored-By: Claude Opus 5.5 --- .config/nextest.toml | 6 ++ .github/workflows/nightly.yml | 20 ++++- bench/README.md | 22 +++-- quest/m2/bench-sources.md | 40 +++++----- rs/moq-audio/src/playback/mixer.rs | 6 +- rs/moq-audio/src/resample.rs | 3 +- rs/moq-bench/README.md | 3 +- rs/moq-bench/src/connection.rs | 6 +- rs/moq-bench/src/stats.rs | 7 +- rs/moq-c/src/id.rs | 5 +- rs/moq-cli/src/publish.rs | 6 +- rs/moq-congestion/src/bbr3/bench.rs | 11 ++- rs/moq-ffi/src/media.rs | 4 +- rs/moq-mux/src/container/ts/damage.rs | 4 +- .../tests/integration/receive_credit.rs | 5 +- sh/gh/bench-report.py | 65 ++++++++++++--- sh/gh/bench-report.test.py | 80 ++++++++++++++++--- sh/gh/nightly.sh | 53 +++++++----- sh/gh/nightly.test.py | 45 +++++++++-- sh/size/README.md | 4 +- 20 files changed, 286 insertions(+), 109 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 1a5f8ed28b2..1db52a106d5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -53,6 +53,12 @@ slow-timeout = { period = "60s", terminate-after = 5 } io-uring = { max-threads = 4 } v4l2 = { max-threads = 1 } +# Nightly's ignored benchmark tests (`just gh nightly rust-test`) keep each test's +# printed tables in target/nextest/bench/junit.xml for the benchmark report. +[profile.bench.junit] +path = "junit.xml" +store-success-output = true + # CI has noisier neighbours and cold caches, so give a test longer before # calling it wedged. [profile.ci] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index aeee3cebf3e..b24975a225a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -121,6 +121,9 @@ jobs: runner: ubuntu-24.04 - name: Rust benchmark tests recipe: gh nightly rust-test + artifact: bench-rust-test + artifact-path: target/bench/nightly/rust-test + benchmark: true - name: Binding benchmark runners recipe: gh nightly bindings - name: Rust bench targets @@ -261,7 +264,7 @@ jobs: # Explicit Pages deployment works with GITHUB_TOKEN; a branch push alone does not. benchmark-site: name: Benchmark site - needs: nightly + needs: [nightly, sizes] if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04-arm timeout-minutes: 15 @@ -673,6 +676,21 @@ jobs: size-report/js/*.metafile.json retention-days: 45 if-no-files-found: error + # The benchmark table records tonight's sizes even when the comparison fails on growth. + - name: Convert sizes for the benchmark table + id: size-table + if: ${{ !cancelled() && hashFiles('size-report/sizes.json') != '' }} + run: nix develop --command just gh benchmark-report size size-report target/bench/size/report.json + - name: Record benchmark outcome + if: ${{ !cancelled() }} + run: nix develop --command just gh benchmark-report status "${{ steps.size-table.outcome }}" target/bench/size + - name: Preserve benchmark report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: bench-size + path: target/bench/size + retention-days: 45 fuzz: name: Fuzz (${{ matrix.target }}) diff --git a/bench/README.md b/bench/README.md index 82427ba0cb9..341761389e5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -480,16 +480,26 @@ The table includes these measurements: time elapsed time. The Rust bench target jobs measure them with a short warmup and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates trade precision for nightly runtime; the paired comparison below assesses a change. -- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation - variants) and `size_bytes` columns, retaining every other column as a workload - dimension in each series name. +- The CSV tables the JS benchmarks and the Rust benchmark tests print. A header + line names the columns; the time columns (`ns`, `us`, `ms`, and their + per-operation variants such as `ns_per_frame` or `plain_ns_per_read`), + `size_bytes` and `peak_bytes`, `mib_per_s`, and `*_packets` columns are + measurements, and every other column is a workload dimension in each series + name, so print outcomes only in measured columns. The Rust benchmark tests + (`just gh nightly rust-test`) run in release builds, one at a time, under + nextest's `bench` profile, which keeps each test's output in a JUnit report; + their series are named by package and test function. +- Shipping sizes, in bytes: the release artifacts, minimal FFI build, and JS + bundles the [size report](../sh/size/README.md) measures, recorded even on a + night its growth check fails. Release build times stay out. Hosted runners vary across nights, so a trend flag prompts a look at the paired comparison rather than proving a regression. Malformed, empty, nonfinite, or duplicate measurements fail validation. The -nightly artifacts retain the reports, raw JS output, and each Criterion run's -metadata and estimates for 45 days. Relay load, audio quality, and the Rust -benchmark tests job's nextest output do not enter the table. +nightly artifacts retain the reports, raw JS output, nextest JUnit reports, and +each Criterion run's metadata and estimates for 45 days. Relay load, audio +quality, and the binding runners (measured in rust2, above) do not enter the +table. Render the table from the published history: diff --git a/quest/m2/bench-sources.md b/quest/m2/bench-sources.md index 407cf4a0bac..35738f56763 100644 --- a/quest/m2/bench-sources.md +++ b/quest/m2/bench-sources.md @@ -9,25 +9,27 @@ is a smoke run that measures nothing. ## Plan Split from the benchmark trend site on 2026-10-20, when that quest became -the m0 table. Sources decided with the maintainer on 2026-10-15: the binding -benchmark's JSON and the "Rust benchmark tests", "Rust bench targets" and "JS -benchmarks" nightly jobs, converted to the history input format where they -don't already emit one. +the m0 table. The Criterion targets, the JS benchmarks, the Rust benchmark +tests and the nightly shipping sizes feed it now +([moq-dev/moq#7510](https://github.com/moq-dev/moq/pull/7510) added the last +two). The binding runners stay an unmeasured smoke run: rust2 measures them, +and [moq-dev/moq#6629](https://github.com/moq-dev/moq/pull/6629) retired their +series. What remains joins through the same input format; the sources decided +with the maintainer on 2026-10-15 left these for later: -- Criterion targets now measure through `just gh criterion` in - `sh/gh/nightly.sh` (`rust-bench`, `rust-shared`, `rust-extra`, `rust-uring`), and the - comparison harness uses quick mode. Check their published histories - rather than converting them again; the unreported sources below remain. -- The "Rust benchmark tests" job is not recorded yet: its nextest benchmarks - print no structured measurements. -- Check each JS bench reports a measurement before recording it. -- Nightly size history (today a 45-day artifact) is a natural extra series - if it fits cheaply. -- The relay load benches (`bench-audio`, `bench-profile`) and the - audio-quality job's benchmarks can join later through the same input - format. -- The paired nightly comparison publishes beside the table from the same - `gh-pages` branch (`bench/paired.json`); a new Rust source joins it through - `just gh benchmark-report rust`. +- The relay load benches, `bench-audio` and `bench-shard` (the "Audio grouping + relay benchmark" and "Relay shard sweep" Nightly jobs). Each writes a JSON + summary per workload; pick the metrics a trend should follow (delivered + throughput, relay CPU, loss, latency) and convert them in + `sh/gh/bench-report.py`. "Relay profiles" and "Browser benchmark smoke" only + check their capture paths and stay out. +- The audio-quality job's benchmarks: `just js qmux-bench`, `serve-bench` and + `ring-bench`. Check each prints a CSV table the `js` converter reads before + recording it. + +A source joins as a `bench-*` artifact with a recorded outcome, which the +benchmark-site job collects. The paired nightly comparison publishes beside the +table from the same `gh-pages` branch (`bench/paired.json`); a new Rust source +joins it through `just gh benchmark-report rust`. Public API: none. Wire: none. diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index a53ea229d85..d1828c24262 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -497,6 +497,7 @@ mod tests { reason = "the benchmark measures mixer callbacks on the host clock" )] fn benchmark_mixer_callbacks() { + println!("sinks,frames,ns_per_callback"); for sinks in [1, 8, 32, MAX_SINKS] { for frames in [128, 480, 1024, 2048] { let mut harness = Harness::with_depth(Layout::Stereo, MAX_SINKS); @@ -514,10 +515,7 @@ mod tests { elapsed += start.elapsed(); std::hint::black_box(&out); } - println!( - "sinks={sinks} frames={frames}: {:.1} ns/callback", - elapsed.as_nanos() as f64 / 1000.0 - ); + println!("{sinks},{frames},{:.1}", elapsed.as_nanos() as f64 / 1000.0); } } } diff --git a/rs/moq-audio/src/resample.rs b/rs/moq-audio/src/resample.rs index ba1c5ccf578..2ff8379fdb3 100644 --- a/rs/moq-audio/src/resample.rs +++ b/rs/moq-audio/src/resample.rs @@ -539,6 +539,7 @@ mod tests { #[test] #[ignore = "run with just rs bench-playback"] fn benchmark_resampler() { + println!("input_rate,output_rate,channels,chunk,packet,ns_per_input_frame"); for (input_rate, output_rate) in [(44_100, 48_000), (48_000, 44_100), (96_000, 48_000)] { for channels in [1, 2, 6] { let mut chunks = vec![input_rate as usize / 50, (input_rate as usize / 50).clamp(1, 960)]; @@ -567,7 +568,7 @@ mod tests { assert_eq!(count, output_rate as usize * channels as usize); } println!( - "{input_rate}->{output_rate} channels={channels} chunk={chunk} packet={packet}: {:.1} ns/input-frame", + "{input_rate},{output_rate},{channels},{chunk},{packet},{:.1}", elapsed.as_nanos() as f64 / (10 * frames) as f64 ); } diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 8ab1ee9b759..26930d4ae64 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -275,4 +275,5 @@ to the results. `just rs bench-latency` measures latency recording with 1, 4 and 16 concurrent writers and 0, 10 and 100 snapshots per run. It checks final sample conservation -and prints throughput for each case. The nightly workflow runs the same matrix. +and prints the cost per sample for each case. The nightly workflow runs the same +matrix and records it on the benchmark table. diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 84ff24c907b..5d6a2c73469 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -766,6 +766,7 @@ mod tests { reason = "the benchmark measures real establishment time" )] fn benchmark_readiness_establishment() { + println!("connections,broadcasts_per_connection,elapsed_ns"); for connections in [1, 16, 128] { for broadcasts in [1, 16, 128] { let count = connections * broadcasts; @@ -793,10 +794,7 @@ mod tests { } } assert!(std::hint::black_box(stats.ready())); - println!( - "readiness connections={connections} broadcasts_per_connection={broadcasts} elapsed_ns={}", - start.elapsed().as_nanos() - ); + println!("{connections},{broadcasts},{}", start.elapsed().as_nanos()); assert_eq!(stats.broadcasts_established.load(Ordering::Relaxed), count); } } diff --git a/rs/moq-bench/src/stats.rs b/rs/moq-bench/src/stats.rs index ec4382ec12a..a5816a812ba 100644 --- a/rs/moq-bench/src/stats.rs +++ b/rs/moq-bench/src/stats.rs @@ -789,6 +789,7 @@ mod tests { use std::sync::Barrier; use std::time::Instant; + println!("writers,snapshots,ns_per_sample"); for writers in [1, 4, 16] { for snapshots in [0, 10, 100] { let stats = Stats::default(); @@ -822,10 +823,8 @@ mod tests { let elapsed = elapsed.elapsed(); assert_eq!(Snapshot::take(&stats).latency_samples, (writers * samples) as u64); println!( - "writers={writers} snapshots={snapshots} samples={} elapsed_ns={} samples_per_second={:.0}", - writers * samples, - elapsed.as_nanos(), - (writers * samples) as f64 / elapsed.as_secs_f64() + "{writers},{snapshots},{:.2}", + elapsed.as_nanos() as f64 / (writers * samples) as f64 ); } } diff --git a/rs/moq-c/src/id.rs b/rs/moq-c/src/id.rs index 420fc6796fb..73f2e622207 100644 --- a/rs/moq-c/src/id.rs +++ b/rs/moq-c/src/id.rs @@ -148,6 +148,7 @@ mod tests { fn benchmark_handles() { use std::hint::black_box; + println!("live,wrap,ns_per_iteration"); for live in [0, 64, 4096] { for wrap in [false, true] { let next = AtomicU32::new(live + 1); @@ -166,8 +167,8 @@ mod tests { let id = slab.insert_next(black_box(42), &next).unwrap(); black_box(slab.remove(id).unwrap()); } - eprintln!( - "handles live={live} wrap={wrap}: {:.0} ns/iteration", + println!( + "{live},{wrap},{:.0}", start.elapsed().as_nanos() as f64 / f64::from(iterations) ); } diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 8d2982a69c8..372bb48f0db 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1943,6 +1943,7 @@ pub(crate) mod tests { let mut null = [0xff; 188]; null[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); let padding = null.repeat(INPUT.len() / 188); + println!("copies,ns_per_byte"); for copies in [1, 16, 128] { let mut file = tempfile::NamedTempFile::new().unwrap(); file.write_all(INPUT).unwrap(); @@ -1967,10 +1968,7 @@ pub(crate) mod tests { decode(decoder, catalog, input, Publication::default()).await.unwrap(); let elapsed = start.elapsed(); let bytes = INPUT.len() * copies; - println!( - "bytes={bytes}: {elapsed:?}, {:.2} Mbit/s", - bytes as f64 * 8.0 / elapsed.as_secs_f64() / 1e6 - ); + println!("{copies},{:.3}", elapsed.as_nanos() as f64 / bytes as f64); } } diff --git a/rs/moq-congestion/src/bbr3/bench.rs b/rs/moq-congestion/src/bbr3/bench.rs index db58d731f40..68b59310003 100644 --- a/rs/moq-congestion/src/bbr3/bench.rs +++ b/rs/moq-congestion/src/bbr3/bench.rs @@ -89,11 +89,10 @@ fn measure(flight: usize, batch: usize, steady: bool) -> (Duration, usize) { #[test] #[ignore = "controller benchmark: run with --release and --success-output immediate"] fn benchmark_per_packet() { - println!( - "bbr_packet controller_stack_bytes={} packet_state_bytes={}", - size_of::(), - size_of::() - ); + println!("state,size_bytes"); + println!("controller_stack,{}", size_of::()); + println!("packet,{}", size_of::()); + println!("steady,flight,batch,cpu_ns,peak_bytes"); for steady in [false, true] { for flight in [16, 64, 1000, 4000, 16000] { for batch in [1, 8, 32] { @@ -106,7 +105,7 @@ fn benchmark_per_packet() { } samples.sort(); println!( - "bbr_packet steady={steady} flight={flight} batch={batch} cpu_ns={} peak_bytes={bytes}", + "{steady},{flight},{batch},{},{bytes}", samples[1].as_nanos() ); } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 930f09d0b38..9b81ef40141 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -2016,7 +2016,7 @@ mod timebase_tests { #[test] #[ignore = "benchmark, runs in Nightly"] fn benchmark_timebase_reservation_fanout() { - eprintln!("timebases,containers,ns_per_container"); + println!("timebases,containers,ns_per_container"); for entries in [1, 32, 1024] { let owner = MoqBroadcastProducer::new().unwrap(); let timebases: Vec<_> = (0..entries) @@ -2045,7 +2045,7 @@ mod timebase_tests { } } let nanos = start.elapsed().as_nanos() as f64 / (1000 * touched) as f64; - eprintln!("{entries},{touched},{nanos:.1}"); + println!("{entries},{touched},{nanos:.1}"); } owner.close().unwrap(); } diff --git a/rs/moq-mux/src/container/ts/damage.rs b/rs/moq-mux/src/container/ts/damage.rs index be10d694e54..189fa333de2 100644 --- a/rs/moq-mux/src/container/ts/damage.rs +++ b/rs/moq-mux/src/container/ts/damage.rs @@ -89,7 +89,7 @@ mod tests { )] async fn benchmark_damage_pid_fanout() { let error = anyhow::anyhow!("broken PES"); - eprintln!("pids,damaged_units,ns_per_unit"); + println!("pids,damaged_units,ns_per_unit"); for pids in [1_u16, 64, 8192] { for units in [1, 16, 128] { let mut log = Log::default(); @@ -108,7 +108,7 @@ mod tests { } } let nanos = start.elapsed().as_nanos() as f64 / (10_000 * units) as f64; - eprintln!("{pids},{units},{nanos:.1}"); + println!("{pids},{units},{nanos:.1}"); } } } diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index e2ac41fdb0f..fee4d0f87b7 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -900,7 +900,7 @@ async fn benchmark_receive_windows() { ("loopback", None, BENCH_BYTES), ("bufferbloat", Some(bufferbloat(BENCH_BLOAT.0)), BENCH_BLOAT.1), ]; - println!("path,axis,streams,frame,window_mib,stream_window,peak_mib,mib_per_s,p50_ms,p99_ms,overflowed"); + println!("path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,overflowed_packets"); for (cell, (path, profile, bytes)) in cells.flat_map(|cell| paths.clone().map(|path| (cell, path))) { let mut runs = Vec::with_capacity(RUNS); for _ in 0..RUNS { @@ -921,13 +921,12 @@ async fn benchmark_receive_windows() { let throughput = median(runs.iter().map(|run| run.throughput).collect()); let overflowed = median(runs.iter().map(|run| run.overflowed as f64).collect()); println!( - "{path},{},{},{},{},{},{:.1},{:.0},{:.2},{:.2},{overflowed}", + "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{overflowed}", cell.axis, cell.streams, cell.frame, cell.window >> 20, if cell.follow { "follow" } else { "finite" }, - peak / (1 << 20) as f64, throughput / (1 << 20) as f64, percentile(50), percentile(99), diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 68ac551d34f..3346cbb48b1 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -13,10 +13,11 @@ import statistics import time from urllib.parse import quote +import xml.etree.ElementTree as ElementTree BIGGER = {'MB/s', 'MiB/s', 'B/s', 'ops/s', 'frames/s'} -SMALLER = {'ns', 'us', 'µs', 'ms', 's', 'ms/MB', 'x', 'bytes', 'B', 'MiB'} +SMALLER = {'ns', 'us', 'µs', 'ms', 's', 'ms/MB', 'x', 'bytes', 'B', 'MiB', 'packets'} def value(number): @@ -123,43 +124,77 @@ def gate(directory): def measurement_unit(column): - if column == 'size_bytes': + """The unit a benchmark CSV column measures, or None for a workload dimension.""" + if column in ('size_bytes', 'peak_bytes'): return 'bytes' + if column == 'mib_per_s' or column.endswith('_mib_per_s'): + return 'MiB/s' + if column.endswith('_packets'): + return 'packets' for unit in ('ns', 'us', 'ms'): - if column == unit or column.endswith(f'_{unit}') or column.startswith(f'{unit}_per_'): + if re.search(f'(^|_){unit}($|_per_)', column): return unit return None -def javascript_file(path): +def printed(text, name, where): + """Read the CSV tables a benchmark printed: a header naming measured columns, then its rows.""" entries, headers = [], None - for line_number, line in enumerate(path.read_text().splitlines(), 1): + for line_number, line in enumerate(text.splitlines(), 1): row = [field.strip() for field in next(csv.reader([line]))] if len(row) > 1 and any(measurement_unit(field) for field in row): if len(set(row)) != len(row): - raise ValueError(f'duplicate CSV headers: {path}:{line_number}') + raise ValueError(f'duplicate CSV headers: {where}:{line_number}') headers = row continue if not headers or not line.strip(): continue if len(row) != len(headers): - raise ValueError(f'malformed benchmark CSV: {path}:{line_number}') + raise ValueError(f'malformed benchmark CSV: {where}:{line_number}') dimensions = '/'.join(f'{key}={quote(field, safe="-_.~")}' for key, field in zip(headers, row) if measurement_unit(key) is None) for key, field in zip(headers, row): if unit := measurement_unit(key): - entries.append({'name': f'js/{path.stem}/{dimensions}/{key}', - 'unit': unit, 'value': value(float(field))}) + entries.append({'name': f'{name}/{dimensions}/{key}', 'unit': unit, 'value': value(float(field))}) if not entries: - raise ValueError(f'JS benchmark produced no measurements: {path}') - return checked(entries) + raise ValueError(f'benchmark produced no measurements: {where}') + return entries def javascript(directory): - entries = [entry for path in sorted((directory / 'raw').glob('*.csv')) for entry in javascript_file(path)] + entries = [entry for path in sorted((directory / 'raw').glob('*.csv')) + for entry in printed(path.read_text(), f'js/{path.stem}', path)] + return checked(entries) + + +# The lines libtest's harness frames each test's output with. +LIBTEST = re.compile(r'running \d+ tests?|test \S+ \.\.\. \w+|test result: .*') + + +def rust_tests(directory): + """Read each benchmark test's tables from the JUnit reports of nextest's bench profile.""" + reports = sorted((directory / 'raw').glob('*.xml')) + if not reports: + raise ValueError(f'no nextest JUnit reports in {directory / "raw"}') + entries = [] + for path in reports: + for case in ElementTree.parse(path).getroot().iter('testcase'): + # A test binary is named `package` or `package::kind/target`. + package = case.get('classname').split('::', 1)[0] + test = case.get('name').rsplit('::', 1)[-1] + output = case.findtext('system-out') or '' + text = '\n'.join(line for line in output.splitlines() if not LIBTEST.fullmatch(line)) + entries += printed(text, f'rust-test/{package}/{test}', f'{path.name} {package} {test}') return checked(entries) +def sizes(directory): + """Record the nightly size report's shipping artifact bytes; its build times stay out, being noise.""" + report = json.loads((directory / 'sizes.json').read_text()) + return checked([{'name': f'size/{key}', 'unit': 'bytes', 'value': value(size)} + for key, size in sorted(report['bytes'].items())]) + + def prepare(directory, output): costs, rates, comparisons = [], [], {} for source in sorted(directory.glob('bench-*')): @@ -442,7 +477,7 @@ def index(directory, commit=None, paired_path=None): def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) - for command in ('rust', 'js', 'prepare'): + for command in ('rust', 'rust-test', 'js', 'size', 'prepare'): sub = commands.add_parser(command) sub.add_argument('directory', type=Path) sub.add_argument('output', type=Path) @@ -462,8 +497,12 @@ def main(): args.output.with_name('paired.json').write_text(json.dumps(comparison, indent=2) + '\n') elif args.command == 'gate': raise SystemExit(gate(args.directory)) + elif args.command == 'rust-test': + write(args.output, rust_tests(args.directory)) elif args.command == 'js': write(args.output, javascript(args.directory)) + elif args.command == 'size': + write(args.output, sizes(args.directory)) elif args.command == 'prepare': prepare(args.directory, args.output) elif args.command == 'status': diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index d47b27e193b..b28d5839516 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -35,29 +35,87 @@ def test_values_refuse_smoke_and_malformed_reports(self): with self.assertRaises(ValueError): report.checked(entries) + def javascript(self, name, text): + source = self.root / 'raw' / f'{name}.csv' + source.parent.mkdir(exist_ok=True) + source.write_text(text) + return report.javascript(self.root) + def test_js_multiple_schemas(self): - source = self.root / 'requests.csv' - source.write_text('bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\noperation,ns_per_op\nclose,0\n') - entries = report.javascript_file(source) + entries = self.javascript('requests', 'bun install v1.3\npaths,active_us,settled_us\n4,1.25,2.5\n' + 'operation,ns_per_op\nclose,0\n') self.assertEqual([entry['value'] for entry in entries], [1.25, 2.5, 0]) self.assertEqual(entries[0]['name'], 'js/requests/paths=4/active_us') for text in ('Testing case\nSuccess\n', 'paths,time_us\n4\n', 'paths,time_us\n4,nan\n'): - source.write_text(text) with self.assertRaises(ValueError): - report.javascript_file(source) + self.javascript('requests', text) + + def test_measured_columns(self): + units = {column: report.measurement_unit(column) for column in ( + 'us', 'elapsed_ms', 'ns_per_frame', 'plain_ns_per_read', 'size_bytes', 'peak_bytes', 'mib_per_s', + 'overflowed_packets', 'payload_bytes', 'bytes', 'frames_per_chunk', 'nsamples', 'runs')} + self.assertEqual(units, { + 'us': 'us', 'elapsed_ms': 'ms', 'ns_per_frame': 'ns', 'plain_ns_per_read': 'ns', 'size_bytes': 'bytes', + 'peak_bytes': 'bytes', 'mib_per_s': 'MiB/s', 'overflowed_packets': 'packets', + # Workload dimensions, as the JS benchmarks name them. + 'payload_bytes': None, 'bytes': None, 'frames_per_chunk': None, 'nsamples': None, 'runs': None}) def test_js_byte_measurements_do_not_become_dimensions(self): - source = self.root / 'compression.csv' - source.write_text('bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n') - entries = report.javascript_file(source) + entries = self.javascript('compression', 'bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n') self.assertEqual([(entry['name'], entry['unit'], entry['value']) for entry in entries], [ ('js/compression/bytes=4/us', 'us', 1.5), ('js/compression/bytes=4/size_bytes', 'bytes', 64), ('js/compression/measurement=bundle/size_bytes', 'bytes', 512), ]) - source.write_text('bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n') - self.assertEqual([entry['name'] for entry in report.javascript_file(source)], - [entry['name'] for entry in entries]) + later = self.javascript('compression', 'bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n') + self.assertEqual([entry['name'] for entry in later], [entry['name'] for entry in entries]) + + def junit(self, name, cases): + """Write a nextest JUnit report of (binary, test, stdout) cases.""" + raw = self.root / 'raw' + raw.mkdir(exist_ok=True) + body = ''.join(f'{stdout}' + '' for binary, test, stdout in cases) + (raw / name).write_text(f'{body}') + + def test_rust_tests_record_each_benchmark_by_package_and_test(self): + with self.assertRaises(ValueError): + report.rust_tests(self.root) + self.junit('nextest-1.xml', [ + ('moq-congestion', 'bbr3::bench::benchmark_per_packet', + '\nrunning 1 test\nstate,size_bytes\npacket,40\nsteady,flight,cpu_ns,peak_bytes\nfalse,16,3170,1336\n' + 'test bbr3::bench::benchmark_per_packet ... ok\n\n' + 'test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 97 filtered out; finished in 0.05s\n'), + ]) + self.junit('nextest-2.xml', [ + ('moq-tokio::integration', 'receive_credit::benchmark_receive_windows', + 'path,mib_per_s,overflowed_packets\nloopback,812,0\n'), + ('moq-bench::bin/moq-bench', 'stats::tests::benchmark_observation_contention', + 'writers,snapshots,ns_per_sample\n1,0,57.39\n'), + ]) + entries = report.rust_tests(self.root) + self.assertEqual([(entry['name'], entry['unit'], entry['value']) for entry in entries], [ + ('rust-test/moq-congestion/benchmark_per_packet/state=packet/size_bytes', 'bytes', 40), + ('rust-test/moq-congestion/benchmark_per_packet/steady=false/flight=16/cpu_ns', 'ns', 3170), + ('rust-test/moq-congestion/benchmark_per_packet/steady=false/flight=16/peak_bytes', 'bytes', 1336), + ('rust-test/moq-tokio/benchmark_receive_windows/path=loopback/mib_per_s', 'MiB/s', 812), + ('rust-test/moq-tokio/benchmark_receive_windows/path=loopback/overflowed_packets', 'packets', 0), + ('rust-test/moq-bench/benchmark_observation_contention/writers=1/snapshots=0/ns_per_sample', 'ns', 57.39), + ]) + # A benchmark that prints no table, or a line its table cannot read, is refused. + for stdout in ('\nrunning 1 test\nwriters=1 elapsed_ns=5\n', 'writers,ns_per_sample\n1,2\nwarming up\n'): + self.junit('nextest-2.xml', [('moq-bench', 'benchmark_observation_contention', stdout)]) + with self.assertRaises(ValueError): + report.rust_tests(self.root) + + def test_sizes_record_shipping_bytes(self): + (self.root / 'sizes.json').write_text(json.dumps({ + 'version': 1, 'bytes': {'release/moq-relay/moq-relay.tar.gz': 900, 'js/net/gzip': 120}, + 'seconds': {'release-moq-relay': 300}})) + self.assertEqual(report.sizes(self.root), [ + {'name': 'size/js/net/gzip', 'unit': 'bytes', 'value': 120}, + {'name': 'size/release/moq-relay/moq-relay.tar.gz', 'unit': 'bytes', 'value': 900}, + ]) def test_criterion_requires_fresh_measured_estimates(self): run = self.root / 'runs' / 'one' diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index ae8312bc9dd..5d18686b61b 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -7,10 +7,12 @@ if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 fi -# The measured groups write a report for the benchmark site; rust-test and bindings only run. +# The measured groups write a report for the benchmark site; bindings only run, since +# rust2 measures them (see bench/README.md). case "$1" in - rust-test | bindings) report= ;; + bindings) report= ;; rust-bench | rust-shared | rust-relay | rust-extra | rust-uring) report=rust ;; + rust-test) report=rust-test ;; js) report=js ;; *) echo "$usage" >&2 @@ -48,7 +50,8 @@ if [[ -n $report ]]; then mkdir -p "$out" export BENCH_RUNS="$out/criterion-runs.txt" : >"$BENCH_RUNS" - rm -f "$out/report.json" "$out/paired.json" + rm -rf "$out/report.json" "$out/paired.json" "$out/raw" + mkdir -p "$out/raw" fi failed=() @@ -66,32 +69,46 @@ criterion() { javascript() { local label=$1 shift - mkdir -p "$out/raw" echo "nightly: $*" if ! "$@" | tee "$out/raw/$label.csv"; then failed+=("$*"); fi } +# The bench nextest profile stores each test's printed tables in its JUnit report, +# which every run overwrites; keep each run's copy. +nextest() { + local junit=${CARGO_TARGET_DIR:-$root/target}/nextest/bench/junit.xml + rm -f "$junit" + run "$@" + if [[ -s $junit ]]; then + mv "$junit" "$out/raw/nextest-$((++nextest_runs)).xml" + else + failed+=("no JUnit report from: $*") + fi +} +nextest_runs=0 + case "$1" in - # Ignored benchmark tests under nextest: the release builds first, then the test-profile ones. + # Ignored benchmark tests under nextest, all in release builds and one at a time, so + # none measures beside another. Each prints CSV tables whose unit-suffixed columns + # the report records (see bench/README.md). rust-test) - run just rs bench-latency - run just rs test -p moq-bench --release --run-ignored only \ + export NEXTEST_PROFILE=bench NEXTEST_TEST_THREADS=1 + nextest just rs bench-latency + nextest just rs test -p moq-bench --release --run-ignored only \ -E 'test(benchmark_readiness_establishment)' --success-output immediate - run just rs bench-playback - run just rs test --release -p moq-congestion -E 'test(benchmark_per_packet)' \ + nextest just rs bench-playback + nextest just rs test --release -p moq-congestion -E 'test(benchmark_per_packet)' \ --run-ignored only --success-output immediate - run just rs bench-feed-fetch - run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate - run just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate - run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate - run just rs test -p moq-c -E 'test(benchmark_handles)' --run-ignored only --success-output immediate - run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ - -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_reference_handoff_fanout)' \ + nextest just rs bench-feed-fetch + nextest just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate + nextest just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate + # One build for the crates whose benchmarks need no features of their own. + nextest just rs test --release --no-fail-fast -p moq-relay -p moq-c -p moq-mux -p moq-cli \ + -E 'test(benchmark_stun_sources) | test(benchmark_handles) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_stream_import)' \ --run-ignored only --success-output immediate # Codecs are irrelevant to timebase reservation, so skip building them. - run just rs test -p moq-ffi --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ + nextest just rs test -p moq-ffi --release --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ --run-ignored only --success-output immediate - run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate ;; # Measured Cargo bench targets, all in the bench profile. A paired run builds and # measures each group twice, so the groups split across jobs to fit an hour each. diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 5594722dca6..e5532e1939f 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -26,6 +26,11 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): '#!/usr/bin/env bash\n' 'printf "%s %s\n" "${0##*/}" "$*" >> "$NIGHTLY_CALLS"\n' 'printf "%s\n" "${BENCH_BASE:-}" >> "$NIGHTLY_CALLS.base"\n' + # Nextest's bench profile writes a JUnit report, unless the test withholds it. + 'if [[ ${NEXTEST_PROFILE:-} == bench && "$*" != "$NIGHTLY_NO_JUNIT" ]]; then\n' + ' mkdir -p "$CARGO_TARGET_DIR/nextest/bench"\n' + ' printf "" > "$CARGO_TARGET_DIR/nextest/bench/junit.xml"\n' + 'fi\n' '[[ "$*" != "$NIGHTLY_FAIL" ]]\n' ) stub.chmod(0o755) @@ -35,11 +40,15 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): 'printf "%s" "$NIGHTLY_RUNS" | jq -r "$2"\n') gh.chmod(0o755) env = dict(os.environ, PATH=f"{scratch}:{os.environ['PATH']}", - BENCH_RESULTS=str(scratch / "results"), NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null") - env.pop("BENCH_BASE", None) + BENCH_RESULTS=str(scratch / "results"), NIGHTLY_CALLS=str(calls), NIGHTLY_FAIL=fail, BASH_ENV="/dev/null", + CARGO_TARGET_DIR=str(scratch / "target")) + for name in ("BENCH_BASE", "NEXTEST_PROFILE", "NIGHTLY_NO_JUNIT"): + env.pop(name, None) env.update(environment) result = subprocess.run(["bash", ROOT / "sh/gh/nightly.sh", group], cwd=cwd, env=env, capture_output=True, text=True) + raw = scratch / "results" / "raw" + self.raw = sorted(path.name for path in raw.iterdir()) if raw.exists() else [] self.bases = set(Path(f"{calls}.base").read_text().splitlines()) if calls.exists() else set() return result, calls.read_text().splitlines() if calls.exists() else [] @@ -179,12 +188,34 @@ def test_dev_shell_jobs_trust_the_nix_cache(self): def test_benchmark_tests_stay_out_of_cargo_bench(self): result, calls = self.run_group("rust-test") self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in calls)) - # Release builds run before the test-profile ones. - profiles = ["--release" in call or " bench-" in call for call in calls] - self.assertEqual(profiles, sorted(profiles, reverse=True)) + *runs, publish = calls + self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in runs)) + # Every benchmark measures a release build, and each run keeps its JUnit report. + self.assertTrue(all("--release" in call or " bench-" in call for call in runs), runs) + self.assertIn("just rs bench-feed-fetch", runs) + self.assertEqual(self.raw, [f"nextest-{index}.xml" for index in range(1, len(runs) + 1)]) + self.assertTrue(publish.startswith("just gh benchmark-report rust-test "), publish) + + def test_benchmark_test_without_its_report_fails(self): + result, calls = self.run_group("rust-test", NIGHTLY_NO_JUNIT="rs bench-latency") + self.assertEqual(result.returncode, 1) + self.assertIn("no JUnit report from: just rs bench-latency", result.stderr) self.assertIn("just rs bench-feed-fetch", calls) - self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) + self.assertFalse(any(call.startswith("just gh") for call in calls)) + + def test_every_nightly_benchmark_test_exists(self): + # A filter naming a removed test matches nothing, and nextest still passes. + recipes = (ROOT / "rs/justfile").read_text() + _, calls = self.run_group("rust-test") + named = set() + for call in calls: + if match := re.fullmatch(r"just rs (bench-[\w-]+)", call): + call = re.search(rf"^{match[1]}\b.*:\n (.+)$", recipes, re.M)[1] + named |= set(re.findall(r"test\((\w+)\)", call)) + self.assertTrue(named) + sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path)) + missing = {name for name in named if not re.search(rf"\bfn {name}\b", sources)} + self.assertEqual(missing, set(), "Nightly benchmark tests that no longer exist") def test_failed_release_build_does_not_hide_later_work(self): _, calls = self.run_group("rust-bench") diff --git a/sh/size/README.md b/sh/size/README.md index bd1142914ba..fde6b67aba1 100644 --- a/sh/size/README.md +++ b/sh/size/README.md @@ -29,4 +29,6 @@ gh api --paginate 'repos/{owner}/{repo}/actions/artifacts?name=nightly-size-hist A regression fails Nightly and reaches its existing alert workflow. The job summary carries the byte counts, build times and FFI crate attribution, the -latter from the symbol-preserving `profiling` profile. +latter from the symbol-preserving `profiling` profile. Each night's byte counts +also join the [benchmark table](https://git.moq.dev/bench/), whose history +outlives the 45-day artifacts. From 4845a566e9990f594cfe96f40e6f571a91b8c149 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:02:21 -0700 Subject: [PATCH 3/5] bench: keep the shaper seed and libtest's slow notice out of benchmark tables Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/tests/integration/receive_credit.rs | 2 +- sh/gh/bench-report.py | 5 +++-- sh/gh/bench-report.test.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index fee4d0f87b7..4577027ebde 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -160,7 +160,7 @@ fn rand_seed() -> u64 { .saturating_duration_since(moq_clock::Wall::UNIX_EPOCH) .as_nanos() as u64, }; - println!("MOQ_SHAPER_SEED={seed}"); + eprintln!("MOQ_SHAPER_SEED={seed}"); seed } diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 3346cbb48b1..864abb72785 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -167,8 +167,9 @@ def javascript(directory): return checked(entries) -# The lines libtest's harness frames each test's output with. -LIBTEST = re.compile(r'running \d+ tests?|test \S+ \.\.\. \w+|test result: .*') +# The lines libtest's harness prints around, and during, each test's own output. +LIBTEST = re.compile(r'running \d+ tests?|test \S+ \.\.\. \w+|test \S+ has been running for over \d+ seconds' + r'|test result: .*') def rust_tests(directory): diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index b28d5839516..91fd3e12fb9 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -89,7 +89,8 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): ]) self.junit('nextest-2.xml', [ ('moq-tokio::integration', 'receive_credit::benchmark_receive_windows', - 'path,mib_per_s,overflowed_packets\nloopback,812,0\n'), + 'path,mib_per_s,overflowed_packets\nloopback,812,0\n' + 'test receive_credit::benchmark_receive_windows has been running for over 60 seconds\n'), ('moq-bench::bin/moq-bench', 'stats::tests::benchmark_observation_contention', 'writers,snapshots,ns_per_sample\n1,0,57.39\n'), ]) From 6baa07ee0e76606dd0217dac1128382302b06e3a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 20:45:46 -0800 Subject: [PATCH 4/5] bench: print the raw-audio benchmark as CSV benchmark_raw_audio, added on main and selected by bench-playback, printed prose, so the rust-test report would fail with no measurements. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/encode/producer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 4dd085fcd93..f45ee8e05cc 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -683,6 +683,7 @@ mod tests { #[tokio::test] #[ignore = "measures real codec CPU; run with just rs bench-playback"] async fn benchmark_raw_audio() { + println!("tracks,retention_secs,always_on,idle_ms,catchup_ms"); for tracks in [1, 8] { for seconds in [1, 30] { for active in [true, false] { @@ -723,7 +724,9 @@ mod tests { } let catchup = start.elapsed(); println!( - "tracks={tracks} retention={seconds}s always_on={active}: idle={idle:?} catchup={catchup:?}" + "{tracks},{seconds},{active},{:.3},{:.3}", + idle.as_secs_f64() * 1e3, + catchup.as_secs_f64() * 1e3 ); for producer in &mut producers { producer.finish().unwrap(); From 5a632cc514834bb0014b3146b76eac2ac73b78b3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 23:03:39 -0800 Subject: [PATCH 5/5] bench: count the datagram frames that delivered nothing, not every frame The table reads `*_packets` as smaller-is-better, but a received-frame count grows with delivery, so a delivery drop would read as an improvement. Report QUIC DATAGRAM frames beyond the datagrams delivered instead; every frame count remains derivable as offered - lost + undelivered. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 ++-- rs/moq-tokio/tests/integration/datagram_delivery.rs | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bench/README.md b/bench/README.md index 4b476b41efc..f03872c78ce 100644 --- a/bench/README.md +++ b/bench/README.md @@ -81,8 +81,8 @@ throttled one: nix develop --command just rs test -p moq-tokio --release -E 'test(benchmark_datagram_delivery)' --run-ignored only --success-output immediate ``` -It reports offered and lost datagrams, bottleneck queue drops, QUIC DATAGRAM frames, -reordered arrivals and append-to-receive latency per viewer, and fails if the loopback +It reports offered and lost datagrams, bottleneck queue drops, QUIC DATAGRAM frames +that delivered no datagram, reordered arrivals and append-to-receive latency per viewer, and fails if the loopback viewer loses any. It runs nightly. Measure 50 fps audio at per-packet, 100 and 200 ms grouping, plus 400 fps diff --git a/rs/moq-tokio/tests/integration/datagram_delivery.rs b/rs/moq-tokio/tests/integration/datagram_delivery.rs index 55b96ab81c2..a0035758069 100644 --- a/rs/moq-tokio/tests/integration/datagram_delivery.rs +++ b/rs/moq-tokio/tests/integration/datagram_delivery.rs @@ -294,7 +294,7 @@ async fn measure(version: &str, cell: Cell) { cell.name, viewer.name, offered as usize - report.received, - report.frames, + report.frames as i64 - report.received as i64, report.reordered, report.percentile(50), report.percentile(99), @@ -314,9 +314,10 @@ async fn measure(version: &str, cell: Cell) { /// Datagram delivery to a loopback viewer and a bufferbloated one, below and above the /// bottleneck's rate. Reports per viewer the datagrams offered and lost, those the /// bottleneck's full queue dropped (UDP packets, so approximate), QUIC DATAGRAM frames -/// received, reordered arrivals, and append-to-receive latency percentiles. Only `offered` -/// is a workload dimension; the counts are `_packets` columns so the nightly benchmark -/// table records them as measurements (see bench/README.md). Nothing on the +/// received that delivered no datagram (padding, or ones the session discarded), +/// reordered arrivals, and append-to-receive latency percentiles. Only `offered` is a +/// workload dimension; the counts are `_packets` columns, all smaller-is-better, so the +/// nightly benchmark table records them as measurements (see bench/README.md). Nothing on the /// path expires a datagram by age: what the sender's QUIC datagram buffer holds waits /// there until it is sent or displaced. Fails if a datagram arrives twice or corrupted, or /// if the loopback viewer loses any. Run by nightly. @@ -336,7 +337,7 @@ async fn benchmark_datagram_delivery() { }, ]; println!( - "version,cell,viewer,offered,lost_packets,overflowed_packets,quic_datagram_packets,reordered_packets,p50_ms,p99_ms,max_ms" + "version,cell,viewer,offered,lost_packets,overflowed_packets,undelivered_frame_packets,reordered_packets,p50_ms,p99_ms,max_ms" ); for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { for cell in cells {