From bd05e845eb5d64879f6a9619abb134ca76febb1f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:39:32 -0700 Subject: [PATCH 1/2] chore: claim pattern and container benchmarks Co-Authored-By: GPT-6 From 71d1951d317b62124841ccf109ec93d2c0b76f39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:04:38 -0700 Subject: [PATCH 2/2] bench: cover pattern matching and CMAF and Annex-B paths Co-Authored-By: GPT-6 --- Cargo.lock | 1 + bench/README.md | 11 ++ quest/m2/README.md | 1 - quest/m2/bench-ci.md | 1 - quest/m2/bench-coverage.md | 33 ---- rs/moq-mux/Cargo.toml | 4 + rs/moq-mux/benches/containers.rs | 262 +++++++++++++++++++++++++++++ rs/moq-pattern/Cargo.toml | 5 + rs/moq-pattern/benches/matching.rs | 40 +++++ sh/gh/nightly.sh | 5 +- sh/gh/nightly.test.py | 3 + 11 files changed, 329 insertions(+), 37 deletions(-) delete mode 100644 quest/m2/bench-coverage.md create mode 100644 rs/moq-mux/benches/containers.rs create mode 100644 rs/moq-pattern/benches/matching.rs diff --git a/Cargo.lock b/Cargo.lock index 552b18a9deb..bf460d9a135 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4754,6 +4754,7 @@ dependencies = [ name = "moq-pattern" version = "0.1.2" dependencies = [ + "criterion", "moq-pattern", "serde", "serde_json", diff --git a/bench/README.md b/bench/README.md index 5c8323a2ab1..d04a9894d23 100644 --- a/bench/README.md +++ b/bench/README.md @@ -154,3 +154,14 @@ ratio is what moq-ffi's handles and runtime cost. Ratios hold only within one machine. The nightly runs it, prints the table in the job summary, and keeps the results as the `bench-ffi` artifact. Each language also has its own recipe (`just go bench`, `just py bench`, `just kt bench`). + +## Pattern and container coverage + +Rust Criterion coverage includes `moq-pattern`'s +`matching` (pattern count and path depth, last match and miss) and `moq-mux`'s +`containers` (CMAF and Annex-B import/export). Container cases vary frame count +and payload size; CMAF also varies track count. Separate frame and byte rates +use the same fixtures, with byte rates counting the encoded stream size. +Fixture generation, publisher setup and destruction stay outside timing. +The existing shared Rust benchmark nightly invocation runs both targets; their `--test` mode +exercises every case without making timing claims. diff --git a/quest/m2/README.md b/quest/m2/README.md index afa2c84fa54..a06038eda16 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -115,7 +115,6 @@ ranked just below deterministic TS legs, which it requires. - [#3199](/quest/m2/3199-moq-uring-remove-sq-indirection-and-per-enter-ring-fd.md) - moq-uring: remove SQ indirection and per-enter ring fd lookup, if a profile shows the win - [#3200](/quest/m2/3200-moq-uring-batch-completion-wakeups-with-min-timeout.md) - moq-uring: batch completion wakeups with MIN_TIMEOUT - [Send depth](/quest/m2/send-depth.md) - moq-net futures prove Send at the default recursion limit, so downstream crates see no nightly lint -- [Bench coverage](/quest/m2/bench-coverage.md) - Criterion targets for moq-pattern matching first, then moq-mux containers - [Browser benchmarks](/quest/m2/browser-benchmarks.md) - measure JS transport, container, decode, and render costs in an identified browser - [Browser media report](/quest/m2/rs2ts-browser-report.md) - compare Rust-backed and handwritten @moq/net through full media workloads, supplementing the pre-adoption comparisons - [mTLS on tls://](/quest/m2/tls-listener-mtls.md) - a `tls://` listener can identify a cluster peer by its client certificate diff --git a/quest/m2/bench-ci.md b/quest/m2/bench-ci.md index cbd034dbd65..2c706421e1c 100644 --- a/quest/m2/bench-ci.md +++ b/quest/m2/bench-ci.md @@ -48,4 +48,3 @@ a GitHub App: - [Benchmark reporting](/bench/README.md) - the existing `bench/run.sh` retains repeated paired rounds - [Benchmark trend site](/quest/m1/bench-site.md) - where the nightly history is charted -- [Bench coverage](/quest/m2/bench-coverage.md) - more targets for this job to track diff --git a/quest/m2/bench-coverage.md b/quest/m2/bench-coverage.md deleted file mode 100644 index 3e7b189f68b..00000000000 --- a/quest/m2/bench-coverage.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] Benchmarks for patterns and containers - -## Goal - -Hot paths that have no benchmark get Criterion targets, so CI tracks them: -`moq-pattern` path matching, and `moq-mux` container import and export per -frame. - -## Plan - -Deferred to m2 in the 2026-09-30 audit. The `moq-pattern` benchmark is the -one worth doing first, and could return to m1 as its own quest. - -Decided 2026-10-08: shrunk to patterns and containers, the paths that run -per announce or per frame. `hang` catalog updates and `moq-auth` token -verification run per update or per connection. - -- `moq-pattern`: matching swept over pattern count and path depth. Patterns - shipped as `rs/moq-pattern` and `js/pattern`, so the matcher gets one bench - here, not a second one elsewhere. -- `moq-mux`: fMP4/CMAF and Annex-B import and export over checked-in or - generated fixtures, with throughput in frames and bytes. Keep fixture - generation out of the timed region. `rs/moq-mux/benches` holds only the - catalog churn bench today. - -Anything that fans out gets a sweep over both axes. Name each target after -what it measures, so the CI comment reads without opening the file. - -Public API: none. Wire: none. - -## Related - -- [Benchmark regressions in CI](/quest/m2/bench-ci.md) - tracks these targets on PRs and nightly diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 6f04203df88..a757eea2ac6 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -58,3 +58,7 @@ harness = false [[bench]] name = "export" harness = false + +[[bench]] +name = "containers" +harness = false diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs new file mode 100644 index 00000000000..2a2aea35289 --- /dev/null +++ b/rs/moq-mux/benches/containers.rs @@ -0,0 +1,262 @@ +//! Complete fMP4/CMAF and Annex-B imports and exports, with setup outside timing. +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_mux::catalog::{CatalogFormat, Stream}; +use moq_mux::codec::h264; +use moq_mux::container::{Frame, Kind}; +use moq_net::Timestamp; + +// A real baseline SPS, also exercised by the H.264 import regression. +const SPS: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, 0x01, + 0xd4, 0xc0, 0x80, +]; +const PPS: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + +fn payload(size: usize) -> Bytes { + let mut payload = BytesMut::new(); + for nal in [SPS, PPS, &[0x65, 0x88, 0x84, 0x21]] { + payload.extend_from_slice(&[0, 0, 0, 1]); + payload.extend_from_slice(nal); + } + // Nonzero padding cannot accidentally add an Annex-B start code. + payload.resize(size, 0xff); + payload.freeze() +} + +fn timestamp(frame: usize) -> Timestamp { + Timestamp::from_millis(frame as u64 * 20).unwrap() +} + +struct Fixture { + _published: Box, + _origin: moq_net::origin::Producer, + driver: tokio::task::JoinHandle, + requests: tokio::task::JoinHandle<()>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.driver.abort(); + self.requests.abort(); + } +} + +async fn prepare(tracks: usize, frames: usize, payload: &Bytes) -> (moq_mux::Source, impl Stream, Fixture) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut publishers = Vec::new(); + for index in 0..tracks { + let track = broadcast + .create_track( + format!("video{index}"), + catalog.track_info(hang::catalog::PRIORITY.video), + ) + .unwrap(); + let config = h264::config(payload).unwrap(); + let mut publisher = catalog + .video(track, moq_mux::catalog::hang::Container::Legacy(Kind::Video), config) + .unwrap(); + for index in 0..frames { + publisher + .write(Frame { + timestamp: timestamp(index), + duration: Some(Timestamp::from_millis(20).unwrap()), + payload: payload.clone(), + keyframe: true, + }) + .unwrap(); + } + publisher.finish().unwrap(); + publishers.push(publisher); + } + catalog.finish().unwrap(); + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + let driver = tokio::spawn(moq_net::time::run(driver)); + let dynamic = origin.dynamic("", Default::default()).unwrap(); + let requests = tokio::spawn(async move { + while let Ok(request) = dynamic.requested_broadcast().await { + request.accept(consumer.clone()); + } + }); + let resolved = origin.consume().request_broadcast("bench", None).await.unwrap(); + let source = moq_mux::Source::new(origin.consume(), resolved); + let stream = source.catalog::<()>(CatalogFormat::Hang).await.unwrap(); + let fixture = Fixture { + _published: Box::new((broadcast, catalog, publishers)), + _origin: origin, + driver, + requests, + }; + (source, stream, fixture) +} + +async fn export(format: &str, source: moq_mux::Source, stream: impl Stream, collect: bool) -> Bytes { + let mut data = BytesMut::new(); + if format == "cmaf" { + let mut export = moq_mux::container::fmp4::Export::new(source, stream) + .with_max_delay(Duration::from_secs(30)) + .with_fragment_duration(Duration::ZERO); + while let Some(chunk) = export.next().await.unwrap() { + if collect { + data.extend_from_slice(&chunk); + } else { + std::hint::black_box(chunk); + } + } + } else { + let mut export = h264::Export::new(source, stream).with_max_delay(Duration::from_secs(30)); + while let Some(chunk) = export.next().await.unwrap() { + if collect { + data.extend_from_slice(&chunk); + } else { + std::hint::black_box(chunk); + } + } + } + data.freeze() +} + +fn cmaf_import() -> (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let import = moq_mux::container::fmp4::Import::new(broadcast, catalog.reserve()); + (catalog, import) +} + +fn import_cmaf( + (catalog, mut import): (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import), + data: &Bytes, +) -> (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import) { + import.decode(data).unwrap(); + import.finish().unwrap(); + (catalog, import) +} + +struct Annexb { + _broadcast: moq_net::broadcast::Producer, + catalog: moq_mux::catalog::Producer, + import: h264::Import, + split: h264::Split, +} + +impl Annexb { + fn new() -> Self { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let import = h264::Import::new(track, catalog.reserve(), Default::default()).unwrap(); + Self { + _broadcast: broadcast, + catalog, + import, + split: h264::Split::new(), + } + } + + fn decode(&mut self, payload: &Bytes, frames: usize) { + for index in 0..frames { + let pts = timestamp(index); + let mut complete = self.split.decode(payload, pts).unwrap(); + complete.extend(self.split.flush(pts).unwrap()); + self.import.decode(complete).unwrap(); + } + self.import.finish().unwrap(); + } +} + +fn containers(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + for format in ["cmaf", "annexb"] { + for tracks in if format == "cmaf" { &[1, 4][..] } else { &[1][..] } { + for frames in [1, 16, 64] { + for size in [256, 4096] { + let payload = payload(size); + let (source, stream, fixture) = runtime.block_on(prepare(*tracks, frames, &payload)); + let data = runtime.block_on(export(format, source, stream, true)); + drop(fixture); + if format == "cmaf" { + let (catalog, mut import) = cmaf_import(); + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!(catalog.snapshot().video.renditions.len(), *tracks); + use mp4_atom::DecodeMaybe; + let mut cursor = std::io::Cursor::new(data.as_ref()); + let mut fragments = 0; + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).unwrap() { + if matches!(atom, mp4_atom::Any::Moof(_)) { + fragments += 1; + } + } + assert_eq!(fragments, tracks * frames); + } else { + let mut import = Annexb::new(); + import.decode(&payload, frames); + assert_eq!(import.catalog.snapshot().video.renditions.len(), 1); + } + assert!(!data.is_empty()); + if format == "annexb" { + assert_eq!(data.len(), size * frames); + } + for unit in ["frames", "bytes"] { + let throughput = if unit == "frames" { + Throughput::Elements((*tracks * frames) as u64) + } else { + Throughput::Bytes(data.len() as u64) + }; + for direction in ["import", "export"] { + let mut group = c.benchmark_group(format!("{format}-{direction}-{unit}")); + group.throughput(throughput.clone()); + group.bench_function( + BenchmarkId::new(format!("{tracks}tracks-{frames}frames"), size), + |b| { + if direction == "import" { + if format == "cmaf" { + b.iter_batched( + cmaf_import, + |import| import_cmaf(import, std::hint::black_box(&data)), + BatchSize::PerIteration, + ); + } else { + b.iter_batched( + Annexb::new, + |mut import| { + import.decode(std::hint::black_box(&payload), frames); + import + }, + BatchSize::PerIteration, + ); + } + } else { + b.iter_batched( + || runtime.block_on(prepare(*tracks, frames, &payload)), + |(source, stream, fixture)| { + std::hint::black_box( + runtime.block_on(export(format, source, stream, false)), + ); + fixture + }, + BatchSize::PerIteration, + ); + } + }, + ); + group.finish(); + } + } + } + } + } + } +} + +criterion_group!(benches, containers); +criterion_main!(benches); diff --git a/rs/moq-pattern/Cargo.toml b/rs/moq-pattern/Cargo.toml index bd2f54d7d02..5cdfbd32b4d 100644 --- a/rs/moq-pattern/Cargo.toml +++ b/rs/moq-pattern/Cargo.toml @@ -20,6 +20,11 @@ serde = ["dep:serde"] serde = { workspace = true, optional = true } [dev-dependencies] +criterion = { workspace = true } # Enable serde so the crate's own tests cover the optional impls in the merge gate. moq-pattern = { path = ".", features = ["serde"] } serde_json = { workspace = true } + +[[bench]] +name = "matching" +harness = false diff --git a/rs/moq-pattern/benches/matching.rs b/rs/moq-pattern/benches/matching.rs new file mode 100644 index 00000000000..9b34070f136 --- /dev/null +++ b/rs/moq-pattern/benches/matching.rs @@ -0,0 +1,40 @@ +//! Matching a pattern union, swept over union size and path depth. +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_pattern::{Pattern, Patterns}; + +fn matching(c: &mut Criterion) { + let mut group = c.benchmark_group("pattern-matching"); + for count in [1, 16, 256] { + for depth in [1, 8, 32] { + let tail = vec!["node"; depth - 1].join("/"); + let path = |tenant: String| { + if tail.is_empty() { + tenant + } else { + format!("{tenant}/{tail}") + } + }; + let patterns: Patterns = (0..count) + .map(|index| path(format!("tenant{index:03}")).parse::().unwrap()) + .collect(); + assert_eq!(patterns.len(), count); + for (case, input, expected) in [ + ("last-match", path(format!("tenant{:03}", count - 1)), true), + ("miss", path("other".into()), false), + ] { + assert_eq!(patterns.matches(&input), expected); + group.throughput(Throughput::Elements(1)); + group.bench_function( + BenchmarkId::new(case, format!("{count}patterns-{depth}segments")), + |b| { + b.iter(|| std::hint::black_box(patterns.matches(std::hint::black_box(&input)))); + }, + ); + } + } + } + group.finish(); +} + +criterion_group!(benches, matching); +criterion_main!(benches); diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 11515b19632..04cdfa10a05 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -27,10 +27,11 @@ case "${1:-}" in run cargo bench --locked --no-fail-fast -p kio --bench '*' -- --test # One invocation unifies this release build instead of rebuilding shared crates # independently for each target. Keep the explicit targets: '*' would add coverage. - run cargo bench --locked --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring \ + run cargo bench --locked --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ --features moq-stats/bench \ --bench http_idle --bench auth --bench producer_tick --bench decode --bench aggregate \ - --bench catalog --bench ts_schedule --bench export --bench chunk_interleaving --bench worker_turns -- --test + --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ + --bench chunk_interleaving --bench worker_turns -- --test run just rs test -p moq-archive -p moq-mux --no-fail-fast \ -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging)' \ --run-ignored only --success-output immediate diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index efd188b482b..81de50dec90 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -47,6 +47,9 @@ def test_release_benchmarks_share_one_invocation(self): self.assertIn("--bench http_idle", shared[0]) self.assertIn("--bench chunk_interleaving", shared[0]) self.assertIn("--bench worker_turns", shared[0]) + self.assertIn("-p moq-pattern", shared[0]) + self.assertIn("--bench matching", shared[0]) + self.assertIn("--bench containers", shared[0]) self.assertIn("just rs bench-gst-catalog -- --quick", calls) self.assertTrue(calls[-1].startswith("just rs test -p moq-cli"))