diff --git a/Cargo.lock b/Cargo.lock index b2ff0c9b6d2..76c299e1558 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4625,7 +4625,9 @@ dependencies = [ "axum-server", "bytes", "bytesize", + "criterion", "futures", + "h2", "hex", "http-cache-reqwest", "humantime", diff --git a/rs/kio/benches/channel.rs b/rs/kio/benches/channel.rs index 562710ab557..98996a7ad00 100644 --- a/rs/kio/benches/channel.rs +++ b/rs/kio/benches/channel.rs @@ -127,5 +127,30 @@ fn bench_notify(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_create_parallel, bench_notify); +/// Measure writes nobody waits on after a burst of readers parked and left, swept over +/// the burst size. A write must not keep paying for waiters that are already gone. +fn bench_write_abandoned(c: &mut Criterion) { + let mut group = c.benchmark_group("channel_write_abandoned"); + for abandoned in [0, 128, 4_096] { + group.bench_with_input(BenchmarkId::from_parameter(abandoned), &abandoned, |b, &abandoned| { + let producer = Producer::new(0u64); + let consumer = producer.consume(); + let waiters: Vec<_> = (0..abandoned).map(|_| Waiter::new(Waker::noop().clone())).collect(); + for waiter in &waiters { + assert!(consumer.poll(waiter, |_| Poll::<()>::Pending).is_pending()); + } + drop(waiters); + + b.iter(|| { + let Ok(mut state) = producer.write() else { + panic!("benchmark channel is open"); + }; + *state += 1; + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_create_parallel, bench_notify, bench_write_abandoned); criterion_main!(benches); diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index d8f71c4a07c..c4fc56ad8f9 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -55,3 +55,7 @@ harness = false [[bench]] name = "containers" harness = false + +[[bench]] +name = "parameter_sets" +harness = false diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs index 8f6654410f4..bff2ecaa873 100644 --- a/rs/moq-mux/benches/containers.rs +++ b/rs/moq-mux/benches/containers.rs @@ -45,6 +45,16 @@ impl Drop for Fixture { } async fn prepare(tracks: usize, frames: usize, payload: &Bytes) -> (moq_mux::Source, impl Stream, Fixture) { + prepare_grouped(tracks, frames, 1, payload).await +} + +/// [`prepare`] with a keyframe every `frames_per_group` frames. +async fn prepare_grouped( + tracks: usize, + frames: usize, + frames_per_group: usize, + payload: &Bytes, +) -> (moq_mux::Source, impl Stream, Fixture) { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); @@ -66,7 +76,7 @@ async fn prepare(tracks: usize, frames: usize, payload: &Bytes) -> (moq_mux::Sou timestamp: timestamp(index), duration: Some(Timestamp::from_millis(20).unwrap()), payload: payload.clone(), - keyframe: true, + keyframe: index % frames_per_group == 0, }) .unwrap(); } @@ -290,5 +300,432 @@ fn mkv_chunks(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, containers, mkv_chunks); +/// Feed `data` to the importer `chunk` bytes at a time, then finish it. +fn import_cmaf_chunks( + (catalog, mut import): (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import), + data: &Bytes, + chunk: usize, +) -> (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import) { + for chunk in data.chunks(chunk) { + import.decode(chunk).unwrap(); + } + import.finish().unwrap(); + (catalog, import) +} + +/// `fragments` moofs whose `tracks` trafs each split their samples into `runs` one-sample runs, +/// interleaved in the mdat run by run, after the init of an exported `tracks`-track broadcast. +fn interleaved(init: &[u8], tracks: usize, runs: usize, fragments: usize, payload: &Bytes) -> Bytes { + use mp4_atom::{Decode, Encode}; + let mut cursor = std::io::Cursor::new(init); + mp4_atom::Ftyp::decode(&mut cursor).unwrap(); + let moov = mp4_atom::Moov::decode(&mut cursor).unwrap(); + let mut out = BytesMut::from(&init[..cursor.position() as usize]); + for fragment in 0..fragments { + let mut moof = mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { + sequence_number: fragment as u32 + 1, + }, + traf: moov.trak[..tracks] + .iter() + .map(|trak| mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id: trak.tkhd.track_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: (fragment * runs * 20) as u64 * trak.mdia.mdhd.timescale as u64 / 1000, + }), + trun: (0..runs) + .map(|run| mp4_atom::Trun { + data_offset: Some(0), + entries: vec![mp4_atom::TrunEntry { + duration: Some(trak.mdia.mdhd.timescale / 50), + size: Some(payload.len() as u32), + flags: Some(if run == 0 { 0 } else { 0x0001_0000 }), + cts: None, + }], + }) + .collect(), + ..Default::default() + }) + .collect(), + }; + let mut buf = Vec::new(); + moof.encode(&mut buf).unwrap(); + let data = buf.len() + 8; + for (track, traf) in moof.traf.iter_mut().enumerate() { + for (run, trun) in traf.trun.iter_mut().enumerate() { + trun.data_offset = Some((data + (run * tracks + track) * payload.len()) as i32); + } + } + buf.clear(); + moof.encode(&mut buf).unwrap(); + mp4_atom::Mdat { + data: payload.repeat(tracks * runs), + } + .encode(&mut buf) + .unwrap(); + out.extend_from_slice(&buf); + } + out.freeze() +} + +/// fMP4 import of fragments whose tracks interleave their runs in one mdat, across track and +/// run count. Only `main`'s `import` direction: `release`'s importer always republishes +/// fragments whole, so it has no `unwrap` direction to pair. +fn interleaved_runs(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let payload = payload(4096); + let frames = 16; + for tracks in [1, 4] { + let (source, stream, fixture) = runtime.block_on(prepare(tracks, 1, &payload)); + let init = runtime.block_on(export("cmaf", source, stream, true)); + drop(fixture); + for size in [1, payload.len()] { + let sample = payload.slice(..size); + for runs in [1, 16] { + let data = interleaved(&init, tracks, runs, frames, &sample); + let (catalog, import) = import_cmaf(cmaf_import(), &data); + assert_eq!(catalog.snapshot().video.renditions.len(), tracks); + drop(import); + let mut group = c.benchmark_group("cmaf-interleaved-import"); + group.throughput(Throughput::Bytes(data.len() as u64)); + // The whole-input row names the input length, which differs from `main`'s, so it + // cannot pair. + for chunk in [1, 1024] { + group.bench_function( + BenchmarkId::new( + format!("{tracks}tracks"), + format!("{runs}runs-{size}bytes-{chunk}chunk"), + ), + |b| { + b.iter_batched( + cmaf_import, + |import| import_cmaf_chunks(import, std::hint::black_box(&data), chunk), + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); + } + } + } +} + +fn export_queued_groups(runtime: &tokio::runtime::Runtime, source: moq_mux::Source, stream: impl Stream) { + std::hint::black_box(runtime.block_on(export("annexb", source, stream, false))); +} + +// Keep both dimensions independent: per-frame scans over queued groups grow with +// frames per group as well as group count, unlike one constant cost per group. +fn queued_groups(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let payload = payload(256); + let mut group = c.benchmark_group("annexb-export-queued-groups"); + for groups in [1, 16, 64] { + for frames_per_group in [1, 16] { + let frames = groups * frames_per_group; + let (source, stream, fixture) = runtime.block_on(prepare_grouped(1, frames, frames_per_group, &payload)); + let data = runtime.block_on(export("annexb", source, stream, true)); + assert_eq!(data.len(), 256 * frames); + drop(fixture); + group.throughput(Throughput::Elements(frames as u64)); + group.bench_function(BenchmarkId::new(groups.to_string(), frames_per_group), |b| { + b.iter_batched( + || runtime.block_on(prepare_grouped(1, frames, frames_per_group, &payload)), + |(source, stream, fixture)| { + export_queued_groups(&runtime, source, stream); + fixture + }, + BatchSize::PerIteration, + ); + }); + } + } + group.finish(); +} + +/// A WebM file of `total` bytes of `payload`-byte VP9 blocks. +fn mkv_payload_file(payload: usize, total: usize) -> Vec { + use webm_iterable::WebmWriter; + use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; + + let frame = vec![0x5a; payload]; + let blocks: Vec = (0..total / payload) + .map(|index| SimpleBlock::new_uncheked(&frame, 1, (index % 1024) as i16, false, None, false, true).into()) + .collect(); + let mut data = Vec::new(); + let mut writer = WebmWriter::new(&mut data); + writer + .write(&MatroskaSpec::Ebml(Master::Full(vec![MatroskaSpec::DocType( + "webm".into(), + )]))) + .unwrap(); + writer.write(&MatroskaSpec::Segment(Master::Start)).unwrap(); + writer + .write(&MatroskaSpec::Tracks(Master::Full(vec![MatroskaSpec::TrackEntry( + Master::Full(vec![ + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackType(1), + MatroskaSpec::CodecID("V_VP9".into()), + ]), + )]))) + .unwrap(); + // Relative block timestamps are 16-bit, so each cluster holds a bounded run of them. + for (cluster, blocks) in blocks.chunks(1024).enumerate() { + let mut children = vec![MatroskaSpec::Timestamp(cluster as u64 * 1024)]; + children.extend_from_slice(blocks); + writer.write(&MatroskaSpec::Cluster(Master::Full(children))).unwrap(); + } + writer.write(&MatroskaSpec::Segment(Master::End)).unwrap(); + writer.flush().unwrap(); + drop(writer); + data +} + +/// MKV block payloads across payload and transport chunk sizes. +fn mkv_payloads(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let _enter = runtime.enter(); + let mut group = c.benchmark_group("mkv-payloads"); + for payload in [1024, 64 * 1024, 1024 * 1024] { + let data = mkv_payload_file(payload, 4 * 1024 * 1024); + group.throughput(Throughput::Bytes(data.len() as u64)); + for chunk in [1500, 64 * 1024, data.len()] { + let id = BenchmarkId::new(format!("payload-{payload}"), format!("chunk-{chunk}")); + group.bench_function(id, |b| { + b.iter_batched( + || { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let import = moq_mux::container::mkv::Import::new(broadcast, catalog.reserve()); + (catalog, import) + }, + |(catalog, mut import)| { + for bytes in std::hint::black_box(&data).chunks(chunk) { + import.decode(bytes).unwrap(); + } + import.finish().unwrap(); + (catalog, import) + }, + BatchSize::PerIteration, + ); + }); + } + } + group.finish(); +} + +struct SiImport { + inner: moq_mux::container::ts::Import, + cc: [u8; 4], + clock: u64, + pending: bool, +} + +impl SiImport { + const PIDS: [u16; 4] = [0x10, 0x11, 0x12, 0x14]; + + fn new() -> Self { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new( + &mut broadcast, + moq_mux::catalog::Config::default() + .with_catalog(moq_mux::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut this = Self { + inner: moq_mux::container::ts::Import::new(broadcast, catalog.reserve()), + cc: [0; 4], + clock: 0, + pending: false, + }; + // PAT and an empty PMT declaring a dedicated PCR PID. Together they also + // provide the framer's initial second packet for confirming its sync byte. + let program = [ + Self::packet(0, 0, this.section(0, 1, &[0, 1, 0xe0, 0x20])), + Self::packet(0x20, 0, this.section(2, 1, &[0xe1, 0xff, 0xf0, 0])), + ] + .concat(); + this.inner.decode(&program).unwrap(); + this.tick(); + this + } + + fn section(&self, table: u8, id: u16, body: &[u8]) -> Vec { + let length = 9 + body.len(); + let mut section = vec![ + table, + 0xb0, + length as u8, + (id >> 8) as u8, + id as u8, + 0xc1, + 0, + u8::from(self.pending && table == 0x42), + ]; + section.extend_from_slice(body); + let crc = crc::Crc::::new(&crc::CRC_32_MPEG_2).checksum(§ion); + section.extend_from_slice(&crc.to_be_bytes()); + section + } + + fn packet(pid: u16, cc: u8, section: Vec) -> Vec { + let mut packet = vec![0x47, 0x40 | (pid >> 8) as u8, pid as u8, 0x10 | cc, 0]; + packet.extend(section); + packet.resize(188, 0xff); + packet + } + + /// A 40 ms PCR interval, safely below the importer's accepted jump bound. + fn tick(&mut self) { + let base = self.clock / 300; + let mut packet = vec![0x47, 0x01, 0xff, 0x20, 183, 0x10]; + packet.extend_from_slice(&[ + (base >> 25) as u8, + (base >> 17) as u8, + (base >> 9) as u8, + (base >> 1) as u8, + ((base & 1) << 7) as u8 | 0x7e, + 0, + ]); + packet.resize(188, 0xff); + self.inner.decode(&packet).unwrap(); + self.clock += 40 * 27_000; + } + + fn repeat(&mut self, entries: impl IntoIterator) { + let mut packets = Vec::new(); + for (pid, table, id) in entries { + let cc = self.cc[pid]; + self.cc[pid] = (cc + 1) & 15; + packets.extend(Self::packet(Self::PIDS[pid], cc, self.section(table, id, &[0; 6]))); + } + self.inner.decode(&packets).unwrap(); + } + + fn identities(count: u16) -> Self { + let mut this = Self::new(); + this.repeat((0..count).map(|id| (2, 0x80, id))); + this + } + + fn pending(count: u16) -> Self { + let mut this = Self::new(); + this.pending = true; + this.inner = this.inner.with_program(1); + this.repeat((0..count).map(|id| (1, 0x42, id))); + this + } + + fn table_key(index: u16) -> (usize, u8, u16) { + (usize::from(index / 256), index as u8, 1) + } + + fn history(count: u16, touches: u16) -> Self { + let mut this = Self::new(); + this.repeat((0..count).map(Self::table_key)); + // All table generations expire. The existing continuity policy retains + // their Entry/track owners; re-activate just the entries the timed call touches. + for _ in 0..=775 { + this.tick(); + } + this.repeat((0..touches).map(|key| Self::table_key(key % count))); + this + } +} + +/// Time one accepted PCR interval plus `touches` SI arrivals against `retained` state, +/// swept over both axes so a cost that grows with the retained set shows up as a slope. +fn ts_si_sweep( + c: &mut Criterion, + name: &str, + retained: [u16; 3], + setup: impl Fn(u16, u16) -> SiImport, + touch: impl Fn(u16, u16) -> (usize, u8, u16), +) { + let mut group = c.benchmark_group(name); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + for retained in retained { + for touches in [1, 16, 64] { + group.throughput(Throughput::Elements(u64::from(touches))); + group.bench_with_input( + BenchmarkId::new(retained.to_string(), touches), + &touches, + |b, &touches| { + b.iter_batched_ref( + || setup(retained, touches), + |import| { + import.tick(); + import.repeat((0..touches).map(|key| touch(retained, key))); + }, + BatchSize::SmallInput, + ); + }, + ); + } + } + group.finish(); +} + +/// SI renewal across retained sub-table count and touched identities. Accepted +/// PCR advances the deadlines during timing without expiring the retained set. +fn ts_si_renewal(c: &mut Criterion) { + ts_si_sweep( + c, + "ts_si_renewal_pcr", + [1, 64, 4096], + |identities, _| SiImport::identities(identities), + |identities, id| (2, 0x80, id % identities), + ); +} + +/// Pending SDT generations retain only their first section; the selected service +/// prevents a repeated partial set from committing as a sparse complete cycle. +fn ts_si_pending(c: &mut Criterion) { + ts_si_sweep( + c, + "ts_si_pending_pcr", + [1, 64, 4096], + |identities, _| SiImport::pending(identities), + |identities, id| (1, 0x42, id % identities), + ); +} + +/// Distinct expired PID/table-key history versus current touched entries, using +/// only the four SI PIDs the importer actually captures and the full table-ID space. +fn ts_si_table_history(c: &mut Criterion) { + ts_si_sweep( + c, + "ts_si_table_history_pcr", + [1, 64, 1024], + SiImport::history, + |history, key| SiImport::table_key(key % history), + ); +} + +criterion_group!( + benches, + containers, + mkv_chunks, + mkv_payloads, + interleaved_runs, + queued_groups, + ts_si_renewal, + ts_si_pending, + ts_si_table_history +); criterion_main!(benches); diff --git a/rs/moq-mux/benches/parameter_sets.rs b/rs/moq-mux/benches/parameter_sets.rs new file mode 100644 index 00000000000..ec89f937f22 --- /dev/null +++ b/rs/moq-mux/benches/parameter_sets.rs @@ -0,0 +1,76 @@ +//! A keyframe carrying many H.264 PPS, swept over the PPS count, with distinct or repeated ids. +//! +//! The splitter keeps one parameter set per id, so a per-set scan of the ones before it would +//! show up as a quadratic slope over the count. Every PPS has distinct bytes, so a byte-wise +//! dedup can't skip the scan either. +//! +//! Run with `cargo bench -p moq-mux --bench parameter_sets`. + +use std::hint::black_box; +use std::time::Duration; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_mux::codec::h264::Split; + +const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; +const IDR: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + +/// A PPS defining `id`, as ue(v), followed by a 16-bit `tag` that keeps its bytes distinct. +fn pps(id: u32, tag: u16) -> Vec { + let code = id + 1; + let len = 32 - code.leading_zeros(); + let mut bits: Vec = std::iter::repeat_n(false, len as usize - 1).collect(); + bits.extend((0..len).rev().map(|bit| code >> bit & 1 == 1)); + // An always-set bit before each tag byte keeps any two bytes from both being zero. + for byte in tag.to_be_bytes() { + bits.push(true); + bits.extend((0..8).rev().map(|bit| byte >> bit & 1 == 1)); + } + bits.push(true); // rbsp_stop_one_bit + let mut out = vec![0x68]; + for chunk in bits.chunks(8) { + out.push( + chunk + .iter() + .enumerate() + .fold(0, |byte, (i, &bit)| byte | u8::from(bit) << (7 - i)), + ); + } + out +} + +fn keyframe(count: u32, distinct: bool) -> Vec { + let mut nals = vec![SPS.to_vec()]; + nals.extend((0..count).map(|n| pps(if distinct { n } else { 0 }, n as u16))); + nals.push(IDR.to_vec()); + nals.into_iter() + .flat_map(|nal| [0, 0, 0, 1].into_iter().chain(nal)) + .collect() +} + +fn parameter_sets(c: &mut Criterion) { + let mut group = c.benchmark_group("h264-split-pps"); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + for count in [1, 16, 256, 4096] { + group.throughput(Throughput::Elements(count.into())); + for distinct in [true, false] { + let data = keyframe(count, distinct); + let ids = if distinct { "distinct" } else { "repeated" }; + group.bench_function(BenchmarkId::new(ids, count), |b| { + b.iter(|| { + let mut split = Split::new(); + let mut frames = split.decode(&data, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + black_box(frames) + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, parameter_sets); +criterion_main!(benches); diff --git a/rs/moq-net/benches/frame.rs b/rs/moq-net/benches/frame.rs index 59a1a6ef919..3d8be4480f4 100644 --- a/rs/moq-net/benches/frame.rs +++ b/rs/moq-net/benches/frame.rs @@ -6,6 +6,9 @@ //! `budget` is the session default, which allocates up front until the frames in //! flight would pass it. Every stream reads a packet-sized chunk per turn. //! +//! `frame_recv_large` sweeps frames up to the group's byte cap over fewer streams, +//! where a frame spans many storage segments. +//! //! Run with `cargo bench -p moq-net --features fuzz --bench frame`. use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; @@ -17,12 +20,15 @@ const CHUNK: usize = 1200; const SIZES: [usize; 3] = [16 * 1024, 256 * 1024, 2 * 1024 * 1024]; const STREAMS: [usize; 3] = [1, 16, 64]; +const LARGE_SIZES: [usize; 3] = [4 * 1024 * 1024, 16 * 1024 * 1024, 32 * 1024 * 1024]; +const LARGE_STREAMS: [usize; 2] = [1, 4]; + const BUDGETS: [(&str, Option); 3] = [("upfront", Some(usize::MAX)), ("grow", Some(0)), ("budget", None)]; -fn bench(c: &mut Criterion) { - let mut group = c.benchmark_group("frame_recv"); - for size in SIZES { - for streams in STREAMS { +fn sweep(c: &mut Criterion, name: &str, sizes: &[usize], streams: &[usize]) { + let mut group = c.benchmark_group(name); + for &size in sizes { + for &streams in streams { group.throughput(Throughput::Bytes((size * streams) as u64)); for (name, budget) in BUDGETS { let id = format!("size={size}/streams={streams}"); @@ -39,5 +45,10 @@ fn bench(c: &mut Criterion) { group.finish(); } +fn bench(c: &mut Criterion) { + sweep(c, "frame_recv", &SIZES, &STREAMS); + sweep(c, "frame_recv_large", &LARGE_SIZES, &LARGE_STREAMS); +} + criterion_group!(benches, bench); criterion_main!(benches); diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 0cfbcc04996..cec79da42e1 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -16,6 +16,12 @@ use std::{ hint::black_box, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + task::Poll, + thread, time::{Duration, Instant}, }; @@ -36,8 +42,8 @@ const PAYLOAD: usize = 64; /// Frame counts to sweep. 8192 is the largest legal group (`MAX_GROUP_FRAMES`); /// the 8193rd write returns `GroupTooLarge`. The top end is that full group of -/// tiny frames. -const COUNTS: [usize; 5] = [4, 5, 512, 2_048, 8_192]; +/// tiny frames. Counts 32 and 33 straddle a full write batch and its next flush. +const COUNTS: [usize; 7] = [4, 5, 32, 33, 512, 2_048, 8_192]; /// Cached group counts to sweep for the track-level delivery benchmarks. A track /// publishing one group per frame at the default 5s retention sits in the hundreds, @@ -378,12 +384,225 @@ fn bench_stamp(c: &mut Criterion) { g.finish(); } +/// Worker threads the contended readers are spread over. +const THREADS: [usize; 3] = [1, 4, 16]; + +/// Readers of the one contended group. +const READERS: [usize; 3] = [16, 256, 1_024]; + +/// Frames appended per contended round: one at the edge, or a burst that readers drain +/// several at a time. +const BURSTS: [usize; 2] = [1, 8]; + +/// Frames a contended group takes before the driver rolls to the next one. +const ROLL: usize = 4_096; + +/// One pass of a worker over its readers. +type Work = Box; + +/// Worker threads that each run one pass when the driver releases a round. +struct Crew { + control: Arc, + threads: Vec>, +} + +#[derive(Default)] +struct Control { + /// Rounds released so far, or [`STOP`]. + round: AtomicU64, + /// Workers done with the current round. + done: AtomicUsize, +} + +const STOP: u64 = u64::MAX; + +impl Crew { + fn new(work: Vec) -> Self { + let control = Arc::new(Control::default()); + let threads = work + .into_iter() + .map(|mut work| { + let control = control.clone(); + thread::spawn(move || { + let mut seen = 0; + loop { + seen = spin(|| { + let round = control.round.load(Ordering::Acquire); + (round != seen).then_some(round) + }); + if seen == STOP { + return; + } + work(); + control.done.fetch_add(1, Ordering::AcqRel); + } + }) + }) + .collect(); + Self { control, threads } + } + + /// Run one pass on every worker and wait for all of them. + fn run(&self) { + self.control.done.store(0, Ordering::Relaxed); + self.control.round.fetch_add(1, Ordering::Release); + let threads = self.threads.len(); + spin(|| { + // A panicked worker never reports done, so fail instead of spinning forever. + assert!(!self.threads.iter().any(|t| t.is_finished()), "worker panicked"); + (self.control.done.load(Ordering::Acquire) == threads).then_some(()) + }); + } +} + +impl Drop for Crew { + fn drop(&mut self) { + self.control.round.store(STOP, Ordering::Release); + for thread in self.threads.drain(..) { + let result = thread.join(); + // Already unwinding from `run`'s check: a second panic would abort. + if !thread::panicking() { + result.expect("worker panicked"); + } + } + } +} + +/// Busy-wait for `f`, yielding once spinning has gone on long enough that the machine +/// may be oversubscribed. +fn spin(mut f: impl FnMut() -> Option) -> R { + let mut spins = 0u32; + loop { + if let Some(r) = f() { + return r; + } + if spins < 1_000 { + spins += 1; + std::hint::spin_loop(); + } else { + thread::yield_now(); + } + } +} + +/// Read one live group from readers on several threads. +/// +/// Each round the driver appends a burst of frames; every reader drains them with +/// [`group::Consumer::poll_read_frame`] and parks at the edge again. Every [`ROLL`] +/// frames the driver publishes the next group and finishes the current one, and each +/// reader moves on once it reads the end, so the group stays under its frame cap. +/// +/// A `subscribed` reader receives each group from its own [`track::Subscriber`], so the +/// group carries the subscription's expiry and every park judges it. +fn bench_contended(c: &mut Criterion) { + let payload = Bytes::from(vec![0u8; PAYLOAD]); + let mut g = c.benchmark_group("group_contended"); + g.warm_up_time(Duration::from_secs(1)); + g.measurement_time(Duration::from_secs(3)); + for subscribed in [false, true] { + for burst in BURSTS { + for threads in THREADS { + for readers in READERS { + g.throughput(Throughput::Elements((readers * burst) as u64)); + let source = if subscribed { "subscribed/" } else { "" }; + let id = BenchmarkId::new(format!("{source}burst_{burst}/threads_{threads}"), readers); + g.bench_function(id, |b| contended(b, &payload, subscribed, burst, threads, readers)); + } + } + } + } + g.finish(); +} + +/// One [`bench_contended`] case. +fn contended( + b: &mut criterion::Bencher, + payload: &Bytes, + subscribed: bool, + burst: usize, + threads: usize, + readers: usize, +) { + let broadcast = broadcast::Producer::new(broadcast::Info::default()); + let track = broadcast.create_track("bench", track::Info::default()).unwrap(); + let mut group = track.append_group().unwrap(); + // The group readers move to once theirs ends. + let next = Arc::new(Mutex::new(group.consume())); + let mut sets: Vec, group::Consumer)>> = (0..threads).map(|_| Vec::new()).collect(); + for i in 0..readers { + let reader = match subscribed { + true => { + let mut subscriber = track.subscribe(None); + let group = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(group.sequence, 0, "the subscriber starts at the live group"); + (Some(subscriber), group) + } + false => (None, group.consume()), + }; + sets[i % threads].push(reader); + } + let work = sets + .into_iter() + .map(|mut set| -> Work { + let next = next.clone(); + let waiter = kio::Waiter::noop(); + Box::new(move || { + for (subscriber, reader) in &mut set { + let mut read = 0; + loop { + match reader.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(frame))) => { + black_box(frame); + read += 1; + } + Poll::Ready(Ok(None)) => { + *reader = match subscriber { + Some(subscriber) => { + let next = subscriber.recv_group().now_or_never(); + next.unwrap().unwrap().expect("the next group is live") + } + None => next.lock().unwrap().clone(), + } + } + Poll::Ready(Err(err)) => panic!("read failed: {err}"), + Poll::Pending => break, + } + } + assert_eq!(read, burst, "every appended frame is ready"); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut written = 0; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + if written + burst > ROLL { + let rolled = track.append_group().unwrap(); + *next.lock().unwrap() = rolled.consume(); + let done = std::mem::replace(&mut group, rolled); + done.finish().unwrap(); + written = 0; + } + for _ in 0..burst { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + written += burst; + crew.run(); + } + start.elapsed() + }); +} + criterion_group!( benches, bench_write, bench_read, bench_roundtrip, bench_track_recv, - bench_stamp + bench_stamp, + bench_contended ); criterion_main!(benches); diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 2eace4d78d8..17ed5a46e99 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -588,34 +588,42 @@ fn bench_subscribe(c: &mut Criterion) { /// depend on `publishers`. fn bench_request(c: &mut Criterion) { let mut group = c.benchmark_group("origin/request"); - // A request costs nothing per subscriber, so this sweeps the publisher counts - // in `SHAPES` rather than its shapes, whose last two share one. for publishers in [100, 1_000] { - let mut fleet = fanout(publishers, 0); - // A broadcast above the misses covers them without serving them. - let _covering = fleet.producer.publish("room", origin::Route::default()).unwrap(); - let waiter = kio::Waiter::noop(); - group.bench_function(BenchmarkId::new("local", publishers), |b| { - b.iter(|| { - let pending = fleet.consumer.request_broadcast("room/0"); - // The front's driver resolves the first request; later ones join it. - fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); - pending - .now_or_never() - .expect("resolves once driven") - .expect("local broadcast"); + for requesters in [1, 8, 32] { + let mut fleet = fanout(publishers, 0); + // A broadcast above the misses covers them without serving them. + let _covering = fleet.producer.publish("room", origin::Route::default()).unwrap(); + let waiter = kio::Waiter::noop(); + let shape = format!("{publishers}p_{requesters}r"); + group.throughput(Throughput::Elements(requesters)); + group.bench_function(BenchmarkId::new("local", &shape), |b| { + b.iter(|| { + let pending: Vec<_> = (0..requesters) + .map(|_| fleet.consumer.request_broadcast("room/0")) + .collect(); + // The front's driver resolves the first request; later ones join it. + fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); + for pending in pending { + pending + .now_or_never() + .expect("resolves once driven") + .expect("local broadcast"); + } + }); }); - }); - group.bench_function(BenchmarkId::new("unroutable", publishers), |b| { - b.iter(|| { - let result = fleet - .consumer - .request_broadcast("room/missing") - .now_or_never() - .expect("fails synchronously"); - assert!(matches!(result, Err(moq_net::Error::Unroutable))); + group.bench_function(BenchmarkId::new("unroutable", &shape), |b| { + b.iter(|| { + for _ in 0..requesters { + let result = fleet + .consumer + .request_broadcast("room/missing") + .now_or_never() + .expect("fails synchronously"); + assert!(matches!(result, Err(moq_net::Error::Unroutable))); + } + }); }); - }); + } } group.finish(); } @@ -1241,7 +1249,7 @@ fn bench_copy_walk(c: &mut Criterion) { /// `(readers, retained, routes)` for [`bench_resume_overlap`]: readers of one track, /// the groups every route holds, and the routes it swapped through. -const RESUME_OVERLAP: [(usize, usize, usize); 15] = [ +const RESUME_OVERLAP: [(usize, usize, usize); 17] = [ (1, 1, 1), (1, 64, 1), (32, 64, 1), @@ -1257,6 +1265,8 @@ const RESUME_OVERLAP: [(usize, usize, usize); 15] = [ (1, 64, 8), (1, 64, 32), (32, 64, 8), + (32, 64, 32), + (256, 64, 8), ]; /// A track that swapped through its routes while its readers' local cap held every @@ -1282,7 +1292,10 @@ impl ResumeReader { } } -async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: bool) -> ResumeOverlap { +/// +/// With `waiting`, one more route answers last without producing a group, so it has not +/// bounded the overlapping copies yet. +async fn resume_overlap(readers: usize, retained: usize, routes: usize, waiting: bool, ordered: bool) -> ResumeOverlap { let (producer, driver) = origin::Producer::new(origin::Config::default()); tokio::spawn(moq_net::time::run(driver)); let generation = |order: &mut dyn Iterator| { @@ -1319,8 +1332,12 @@ async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: subscribers.push(subscriber); } let waiter = kio::Waiter::noop(); - for _ in 1..routes { - generations.push(generation(&mut (0..retained).rev())); + for route in 1..routes + usize::from(waiting) { + generations.push(if waiting && route == routes { + generation(&mut std::iter::empty()) + } else { + generation(&mut (0..retained).rev()) + }); let (_, latest) = generations.last().unwrap(); tokio::time::timeout(COPY_WAIT, latest.demand().used()) .await @@ -1357,8 +1374,16 @@ async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: fn bench_resume_overlap(c: &mut Criterion) { let mut group = c.benchmark_group("origin/resume_overlap"); group.sample_size(10); - for (order, ordered) in [("arrival", false), ("sequence", true)] { + for (order, ordered, waiting) in [ + ("arrival", false, false), + ("sequence", true, false), + ("arrival_waiting", false, true), + ("sequence_waiting", true, true), + ] { for (readers, retained, routes) in RESUME_OVERLAP { + if waiting && !(readers == 1 && retained == 64 && routes >= 2) { + continue; + } group.throughput(Throughput::Elements((readers * retained) as u64)); group.bench_function( BenchmarkId::new(order, format!("{readers}r_{retained}g_{routes}c")), @@ -1369,7 +1394,7 @@ fn bench_resume_overlap(c: &mut Criterion) { .unwrap(); let waiter = kio::Waiter::noop(); b.iter_batched_ref( - || runtime.block_on(resume_overlap(readers, retained, routes, ordered)), + || runtime.block_on(resume_overlap(readers, retained, routes, waiting, ordered)), |rig| { for reader in &mut rig.readers { let mut delivered = 0; @@ -1388,6 +1413,64 @@ fn bench_resume_overlap(c: &mut Criterion) { group.finish(); } +/// `(readers, open)` shapes for [`bench_passthrough`]: readers of one track, and the +/// groups each holds open at once. +const PASSTHROUGH: [(usize, usize); 4] = [(1, 1), (1, 64), (100, 1), (100, 64)]; + +/// One-frame groups read through a front that only ever had one route. Each reader holds +/// `open` groups at once. +fn bench_passthrough(c: &mut Criterion) { + let mut group = c.benchmark_group("origin/passthrough"); + for (readers, open) in PASSTHROUGH { + group.throughput(Throughput::Elements((readers * open) as u64)); + group.bench_function(BenchmarkId::from_parameter(format!("{readers}r_{open}g")), |b| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (producer, driver) = origin::Producer::new(origin::Config::default()); + runtime.spawn(moq_net::time::run(driver)); + let broadcast = producer.publish("room/live", origin::Route::default()).unwrap(); + let source = broadcast.create_track("0", timed()).unwrap(); + let mut subscriptions = runtime.block_on(async { + let resolved = producer.consume().request_broadcast("room/live").await.unwrap(); + let track = resolved.track("0").unwrap(); + let mut subscriptions = Vec::with_capacity(readers); + for _ in 0..readers { + subscriptions.push(track.subscribe(None).await.unwrap()); + } + subscriptions + }); + let mut held: Vec> = (0..readers).map(|_| Vec::with_capacity(open)).collect(); + + b.iter_custom(|iterations| { + runtime.block_on(async { + let started = std::time::Instant::now(); + for _ in 0..iterations { + // Received as each is written: a reader behind the live edge skips ahead. + for _ in 0..open { + let mut group = source.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); + group.finish().unwrap(); + for (subscription, held) in subscriptions.iter_mut().zip(&mut held) { + held.push(subscription.recv_group().await.unwrap().expect("group")); + } + } + for held in &mut held { + for group in held.iter_mut() { + group.read_frame().await.unwrap().expect("frame"); + } + held.clear(); + } + } + started.elapsed() + }) + }); + }); + } + group.finish(); +} + criterion_group!( benches, bench_announce, @@ -1411,6 +1494,7 @@ criterion_group!( bench_relay, bench_parked, bench_copy_walk, - bench_resume_overlap + bench_resume_overlap, + bench_passthrough ); criterion_main!(benches); diff --git a/rs/moq-net/benches/stats.rs b/rs/moq-net/benches/stats.rs index cd9b61d380a..051532de889 100644 --- a/rs/moq-net/benches/stats.rs +++ b/rs/moq-net/benches/stats.rs @@ -77,9 +77,13 @@ fn session_report(c: &mut Criterion) { }) .collect(); let mut report = stats::Report::default(); - group.bench_function(BenchmarkId::from_parameter(format!("{labels}x{sessions}")), |b| { - b.iter(|| registry.report(&mut report)); - }); + // `main`'s `connections=0` row: `release` has no transport wires to walk. + group.bench_function( + BenchmarkId::from_parameter(format!("{labels}x{sessions}/connections=0")), + |b| { + b.iter(|| registry.report(&mut report)); + }, + ); std::hint::black_box(contexts); } } diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index f19888672ba..ad409818bf2 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -56,7 +56,7 @@ use moq_net::time::Instant; use bytes::Bytes; use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_net::{Error, Timestamp, broadcast, cache, track}; +use moq_net::{Error, Timestamp, broadcast, cache, group, track}; /// A millisecond track: the frames written here carry timestamps. /// @@ -273,6 +273,15 @@ fn bench_gc(c: &mut Criterion) { let swept = Swept::new(tracks); b.iter(|| black_box(swept.pool.gc(swept.now))); }); + // Also advance supplied time, so reusing an identical instant cannot hide clock work. + group.bench_with_input(BenchmarkId::new("advancing", tracks), &tracks, |b, &tracks| { + let swept = Swept::new(tracks); + let mut now = swept.now; + b.iter(|| { + now += Duration::from_nanos(1); + black_box(swept.pool.gc(now)) + }); + }); // A due pass, stepping one sweep interval per call. group.throughput(Throughput::Elements(tracks as u64)); group.bench_with_input(BenchmarkId::new("sweep", tracks), &tracks, |b, &tracks| { @@ -820,7 +829,7 @@ impl GapFlush { fn bench_gap_flush(c: &mut Criterion) { let mut group = c.benchmark_group("track_gap_flush"); group.sample_size(10); - for gaps in [64u64, 512, 4096] { + for gaps in [16u64, 64, 256, 512, 4096] { group.throughput(Throughput::Elements(gaps)); group.bench_function(BenchmarkId::from_parameter(gaps), |b| { b.iter_batched_ref(|| GapFlush::new(gaps), GapFlush::flush, BatchSize::PerIteration); @@ -925,6 +934,239 @@ fn bench_empty_run(c: &mut Criterion) { group.finish(); } +/// Groups written between reads in `track_evict_fanout`, each evicting one. +const EVICTED: [usize; 3] = [1, 8, 64]; + +fn bench_evict_fanout(c: &mut Criterion) { + let mut group = c.benchmark_group("track_evict_fanout"); + for subscribers in FANOUT { + for evicted in EVICTED { + group.throughput(Throughput::Elements(evicted as u64)); + group.bench_function(BenchmarkId::new(format!("{subscribers}r"), evicted), |b| { + let mut fanout = Fanout::new(subscribers); + // Twice what the pool holds, so every write evicts. + for _ in 0..2 * CACHE_CAPACITY / PAYLOAD as u64 { + fanout.cycle(); + } + // Only the writes and the evictions they cause are timed, not the reads that + // judge each written group cached again. + b.iter_custom(|iterations| { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let start = WallInstant::now(); + for _ in 0..evicted { + let mut group = fanout.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, fanout.payload.clone()).unwrap(); + group.finish().unwrap(); + } + elapsed += start.elapsed(); + for (subscriber, waiter) in fanout.subscribers.iter_mut().zip(&fanout.waiters) { + while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(waiter) { + black_box(group); + } + } + } + elapsed + }); + }); + } + } + group.finish(); +} + +/// Tracks sharing one bounded pool, each holding `queued` datagrams for a reader that never +/// reads, while one datagram is appended to the next track and read by its fast reader. +/// +/// Only `main`'s `plain` payload: `release` has no track compression, so its `zeros` rows +/// are left out. +fn bench_datagram_pressure(c: &mut Criterion) { + let mut group = c.benchmark_group("track_datagram_pressure"); + group.throughput(Throughput::Elements(1)); + let name = "plain"; + let payload = Bytes::from(vec![0u8; PAYLOAD]); + for tracks in [1, 16, 64] { + for queued in [1, 8, 64] { + let mut info = broadcast::Info::default(); + info.pool = cache::Pool::new(cache::Config::default().with_capacity(CACHE_CAPACITY)); + let pool = info.pool.clone(); + let broadcast = broadcast::Producer::new(info); + let mut held: Vec<_> = (0..tracks) + .map(|index| { + let mut track = broadcast + .create_track(format!("bench{index}").as_str(), timed()) + .unwrap(); + let slow = track.subscribe(None); + let fast = track.subscribe(None); + for _ in 0..queued { + track.append_datagram(Timestamp::ZERO, payload.clone()).unwrap(); + } + (track, slow, fast) + }) + .collect(); + let waiter = kio::Waiter::noop(); + for (_, _, fast) in &mut held { + while let Poll::Ready(Ok(Some(_))) = fast.poll_recv_datagram(&waiter) {} + } + eprintln!( + "track_datagram_pressure/{name}/{tracks}t/{queued}q: {} bytes retained", + pool.used() + ); + + let mut next = 0; + group.bench_function(BenchmarkId::new(format!("{name}/{tracks}t"), queued), |b| { + b.iter(|| { + let (track, _, fast) = &mut held[next % tracks]; + next += 1; + track.append_datagram(Timestamp::ZERO, payload.clone()).unwrap(); + let Poll::Ready(Ok(Some(datagram))) = fast.poll_recv_datagram(&waiter) else { + panic!("the fast reader takes each datagram"); + }; + black_box(datagram); + }); + }); + drop((held, broadcast)); + } + } + group.finish(); +} + +/// A timed sparse feed holding `gaps` remembered gaps, as in `track_sparse_read`. +struct Sparse { + _broadcast: broadcast::Producer, + track: track::Producer, + sub: track::Subscriber, + next: u64, + missing: u64, +} + +impl Sparse { + fn new(gaps: u64) -> Self { + let broadcast = broadcast::Info::default().produce(); + let track = broadcast.create_track("bench", timed()).unwrap(); + let sub = track.subscribe(track::Subscription::default().with_max_age(Duration::from_secs(3600))); + let mut sparse = Self { + _broadcast: broadcast, + track, + sub, + next: gaps * 2, + missing: 1, + }; + for gap in 0..gaps { + sparse.hand_out(gap * 2); + } + // Fill beyond the cache queue's initial power-of-two capacity outside timing. + if gaps != 0 { + sparse.hand_out(sparse.next); + sparse.hand_out(sparse.missing); + sparse.next += 2; + sparse.missing += 2; + } + sparse + } + + fn hand_out(&mut self, sequence: u64) { + let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_micros(sequence).unwrap(), Bytes::from_static(b"x")) + .unwrap(); + group.finish().unwrap(); + let Poll::Ready(Ok(Some(read))) = self.sub.poll_recv_group(&kio::Waiter::noop()) else { + panic!("the group is cached"); + }; + black_box(read); + } +} + +/// A read with nothing new on a sparse feed, swept over its remembered gaps. Only `main`'s +/// `timed` rows: `release` reads no untimed sparse feed. +fn bench_sparse_poll(c: &mut Criterion) { + let mut group = c.benchmark_group("track_sparse_poll"); + for gaps in [16u64, 256, 4096] { + group.bench_function(BenchmarkId::new("timed", gaps), |b| { + b.iter_custom(|iterations| { + let mut sparse = Sparse::new(gaps); + let waiter = kio::Waiter::noop(); + let start = WallInstant::now(); + for _ in 0..iterations { + assert!(black_box(sparse.sub.poll_recv_group(&waiter)).is_pending()); + } + start.elapsed() + }); + }); + } + group.finish(); +} + +/// The budget `track_idle_poll`'s readers hold, long enough that nothing ages out. +const AGED_BUDGET: Duration = Duration::from_secs(3600); + +/// Readers parked on one track with nothing new, each having taken the one cached group. +struct Idle { + _broadcast: broadcast::Producer, + _track: track::Producer, + _open: group::Producer, + recv: Vec, + next: Vec, + waiter: kio::Waiter, +} + +impl Idle { + fn new(readers: usize, ordered: bool) -> Self { + let broadcast = broadcast::Info::default().produce(); + let track = broadcast.create_track("bench", timed()).unwrap(); + let mut open = track.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); + let waiter = kio::Waiter::noop(); + let (mut recv, mut next) = (Vec::new(), Vec::new()); + for _ in 0..readers { + let reader = track.subscribe(track::Subscription::default().with_max_age(AGED_BUDGET)); + if ordered { + let mut reader = reader.ordered(); + assert!(matches!(reader.poll_next_group(&waiter), Poll::Ready(Ok(Some(_))))); + next.push(reader); + } else { + let mut reader = reader; + assert!(matches!(reader.poll_recv_group(&waiter), Poll::Ready(Ok(Some(_))))); + recv.push(reader); + } + } + let mut idle = Self { + _broadcast: broadcast, + _track: track, + _open: open, + recv, + next, + waiter, + }; + // The first park walks the read path; only the repeats are measured. + idle.poll(); + idle + } + + fn poll(&mut self) { + for reader in &mut self.recv { + assert!(black_box(reader.poll_recv_group(&self.waiter)).is_pending()); + } + for reader in &mut self.next { + assert!(black_box(reader.poll_next_group(&self.waiter)).is_pending()); + } + } +} + +fn bench_idle_poll(c: &mut Criterion) { + let mut group = c.benchmark_group("track_idle_poll"); + for (name, ordered) in [("recv", false), ("next", true)] { + for readers in FANOUT { + group.throughput(Throughput::Elements(readers as u64)); + group.bench_function(BenchmarkId::new(name, readers), |b| { + let mut idle = Idle::new(readers, ordered); + b.iter(|| idle.poll()); + }); + } + } + group.finish(); +} + criterion_group!( benches, bench_fanout, @@ -941,6 +1183,10 @@ criterion_group!( bench_parked_read, bench_sparse_read, bench_gap_flush, - bench_empty_run + bench_empty_run, + bench_evict_fanout, + bench_datagram_pressure, + bench_sparse_poll, + bench_idle_poll ); criterion_main!(benches); diff --git a/rs/moq-pattern/benches/matching.rs b/rs/moq-pattern/benches/matching.rs index 9b34070f136..c99b851ac8a 100644 --- a/rs/moq-pattern/benches/matching.rs +++ b/rs/moq-pattern/benches/matching.rs @@ -36,5 +36,86 @@ fn matching(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, matching); +/// Whole-union operations, swept over member kind and the size of each union. +/// +/// `AUTH_OK` carries as many patterns as the peer chooses, so none of these may grow +/// with the product of two unions' sizes. Literals share one specificity tier; partials +/// share another; two-tier pairs each one-segment literal with a two-segment literal +/// under the same first segment, so the tiers meet on every shared prefix. +/// +/// The pairwise operations sweep the held union (indexed) and the queried union +/// independently, so a cost that follows either one shows up as its own slope. The +/// unions never contain each other's members, so no query ends early. `covers` must +/// succeed on every query to visit them all: per kind it queries a subset of the held +/// union, and `covers-subtree` holds `sNNNN/**` members that cover any number of +/// literals beneath them, so it sweeps both sizes. +fn algebra(c: &mut Criterion) { + let mut group = c.benchmark_group("pattern-algebra"); + let union = |kind: &str, name: &str, count: usize| -> Patterns { + let union: Patterns = (0..count) + .map(|index| match kind { + "literal" => format!("{name}{index:04}"), + "partial" => format!("{name}{index:04}*tail"), + _ if index % 2 == 0 => format!("{name}{:04}", index / 2), + _ => format!("{name}{:04}/x", index / 2), + }) + .map(|text| text.parse::().unwrap()) + .collect(); + assert_eq!(union.len(), count); + union + }; + for kind in ["literal", "partial", "two-tier"] { + for count in [16, 256, 4096] { + let inputs: Vec = union(kind, "room", count).into_iter().collect(); + group.throughput(Throughput::Elements(count as u64)); + group.bench_function(BenchmarkId::new(format!("collect-{kind}"), count), |b| { + b.iter(|| std::hint::black_box(inputs.iter().cloned().collect::().len())); + }); + } + for (held, queried) in [(16, 16), (4096, 16), (16, 4096), (4096, 4096)] { + let ours = union(kind, "room", held); + let theirs = union(kind, "other", queried); + let sizes = format!("{held}x{queried}"); + group.throughput(Throughput::Elements((held + queried) as u64)); + + group.bench_function(BenchmarkId::new(format!("extend-{kind}"), &sizes), |b| { + b.iter(|| { + let mut union = ours.clone(); + union.extend(theirs.iter().cloned()); + std::hint::black_box(union.len()) + }); + }); + group.bench_function(BenchmarkId::new(format!("intersect-{kind}"), &sizes), |b| { + b.iter(|| std::hint::black_box(ours.intersect(&theirs).unwrap().len())); + }); + if queried <= held { + let subset = union(kind, "room", queried); + assert!(ours.covers(&subset)); + group.bench_function(BenchmarkId::new(format!("covers-{kind}"), &sizes), |b| { + b.iter(|| std::hint::black_box(ours.covers(&subset))); + }); + } + } + } + for (held, queried) in [(16, 16), (4096, 16), (16, 4096), (4096, 4096)] { + let subtrees: Patterns = (0..held) + .map(|index| format!("s{index:04}/**").parse::().unwrap()) + .collect(); + let literals: Patterns = (0..queried) + .map(|index| { + format!("s{:04}/room{index:04}", index % held) + .parse::() + .unwrap() + }) + .collect(); + assert!(subtrees.covers(&literals)); + group.throughput(Throughput::Elements((held + queried) as u64)); + group.bench_function(BenchmarkId::new("covers-subtree", format!("{held}x{queried}")), |b| { + b.iter(|| std::hint::black_box(subtrees.covers(&literals))); + }); + } + group.finish(); +} + +criterion_group!(benches, matching, algebra); criterion_main!(benches); diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index c44940f824f..369396bc46e 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -92,6 +92,9 @@ sd-notify = { workspace = true } [dev-dependencies] aws-lc-rs = { workspace = true } +criterion = { workspace = true } +# The http_auth bench drives the web listener over HTTP/2. +h2 = "0.4" hex = { workspace = true } moq-auth = { path = "../moq-auth", features = ["client", "serve"] } moq-shaper = { path = "../moq-shaper" } @@ -109,3 +112,7 @@ socket2 = { workspace = true, features = ["all"] } # path rather than the trigger behind it. [target.'cfg(unix)'.dev-dependencies] libc = "0.2" + +[[bench]] +name = "http_auth" +harness = false diff --git a/rs/moq-relay/benches/http_auth.rs b/rs/moq-relay/benches/http_auth.rs new file mode 100644 index 00000000000..4365d15a5d1 --- /dev/null +++ b/rs/moq-relay/benches/http_auth.rs @@ -0,0 +1,148 @@ +//! One public HTTP read's admission, swept over connections and the HTTP/2 streams each +//! holds waiting on the decider. A slope means admitting one request costs more as others +//! wait, rather than what the request itself touches. +//! The real web listener runs over loopback TCP; timings include one request round trip. + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use moq_auth::{Grant, lease}; +use moq_relay::{auth, cluster, web}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; + +type Send = h2::client::SendRequest; + +struct Fixture { + measured: Send, + _held: Vec<(Send, Vec)>, + tasks: Vec>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + for task in &self.tasks { + task.abort(); + } + } +} + +impl Fixture { + async fn new(connections: usize, streams: usize) -> Self { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let (auth, mut admissions) = auth::Auth::embedded("bench"); + let waiting = Arc::new(AtomicUsize::new(0)); + let decider = tokio::spawn({ + let waiting = waiting.clone(); + async move { + let mut held = Vec::new(); + while let Some(admission) = admissions.next().await { + if admission.request.path.starts_with("/held") { + held.push(admission); + waiting.store(held.len(), Ordering::Release); + continue; + } + let grant = Grant::new(moq_auth::Patterns::new(), moq_auth::Pattern::all().into()); + admission.grant(lease::Consumer::fixed(grant)); + } + } + }); + + let pem = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .unwrap() + .cert + .pem(); + let certificates = moq_tokio::tls::Certificates::from_pem(pem.as_bytes()).unwrap(); + let mut config = web::Config::default(); + config.http.listen = Some("127.0.0.1:0".parse().unwrap()); + let cluster = cluster::Cluster::new(Default::default()).unwrap(); + // No admission timeout, so held requests stay waiting however long setup and the + // measurement take. + let web = web::Web::new(auth, cluster, certificates, config) + .with_timeout(None) + .bind() + .unwrap(); + let addr = web.addrs().http.unwrap(); + let routes = web.routes(); + let server = tokio::spawn(async move { + let _ = web.serve(routes).await; + }); + + let mut tasks = vec![decider, server]; + let mut held = Vec::new(); + for connection in 0..connections { + let mut send = connect(addr, &mut tasks).await; + let mut responses = Vec::new(); + for _ in 0..streams { + send = send.ready().await.unwrap(); + let (response, _) = send.send_request(get(&format!("held{connection}")), true).unwrap(); + responses.push(response); + } + held.push((send, responses)); + } + let measured = connect(addr, &mut tasks).await; + while waiting.load(Ordering::Acquire) < connections * streams { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + Self { + measured, + _held: held, + tasks, + } + } + + async fn round(&mut self) { + let mut send = self.measured.clone().ready().await.unwrap(); + let (response, _) = send.send_request(get("bench"), true).unwrap(); + let mut body = response.await.unwrap().into_body(); + while let Some(chunk) = body.data().await { + let chunk = chunk.unwrap(); + body.flow_control().release_capacity(chunk.len()).unwrap(); + } + } +} + +async fn connect(addr: std::net::SocketAddr, tasks: &mut Vec>) -> Send { + let tcp = tokio::net::TcpStream::connect(addr).await.unwrap(); + let (send, connection) = h2::client::handshake(tcp).await.unwrap(); + tasks.push(tokio::spawn(async move { + let _ = connection.await; + })); + send +} + +fn get(prefix: &str) -> axum::http::Request<()> { + axum::http::Request::get(format!("http://localhost/announced/{prefix}")) + .body(()) + .unwrap() +} + +fn admission(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("relay_http_admission"); + group.sample_size(10); + group.warm_up_time(Duration::from_millis(100)); + group.measurement_time(Duration::from_millis(300)); + // Each connection may hold up to the verification budget (128 by default) at once. + for connections in [1, 16, 64] { + for streams in [0, 16, 64] { + let mut fixture = runtime.block_on(Fixture::new(connections, streams)); + group.bench_with_input( + BenchmarkId::new(format!("connections={connections}"), streams), + &(connections, streams), + |b, _| { + b.iter(|| runtime.block_on(fixture.round())); + }, + ); + } + } + group.finish(); +} + +criterion_group!(benches, admission); +criterion_main!(benches); diff --git a/rs/moq-rtmp/benches/chunk_interleaving.rs b/rs/moq-rtmp/benches/chunk_interleaving.rs index 0917f00c61b..b80c342fd63 100644 --- a/rs/moq-rtmp/benches/chunk_interleaving.rs +++ b/rs/moq-rtmp/benches/chunk_interleaving.rs @@ -56,5 +56,78 @@ fn bench(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench); +/// Decode an AMF0 value sequence. `release` decodes it with `amf0::deserialize`, which +/// `main` replaced with `amf0::Values::decode`. +fn amf0(c: &mut Criterion) { + let mut group = c.benchmark_group("rtmp_amf0"); + for count in [1024usize, 65536, 1048576] { + for (name, value) in [("nulls", vec![5]), ("properties", vec![0, 1, b'a', 5])] { + let mut bytes = Vec::new(); + if name == "properties" { + bytes.push(3); + } + for _ in 0..count { + bytes.extend_from_slice(&value); + } + if name == "properties" { + bytes.extend_from_slice(&[0, 0, 9]); + } + group.throughput(Throughput::Bytes(bytes.len() as u64)); + group.bench_function(BenchmarkId::new(name, count), |b| { + b.iter(|| black_box(rml::amf0::deserialize(&mut black_box(bytes.as_slice())).unwrap())); + }); + } + } + group.finish(); +} + +fn connect(c: &mut Criterion) { + use rml::amf0::Amf0Value; + use rml::chunk_io::ChunkSerializer; + use rml::messages::RtmpMessage; + use rml::sessions::{ServerSession, ServerSessionConfig}; + use rml::time::RtmpTimestamp; + let mut group = c.benchmark_group("rtmp_connect_fourccs"); + for count in [16usize, 256, 4096] { + for shape in ["array", "object", "info_map"] { + let names = (0..count).rev().map(|id| format!("{id:04x}")); + let value = match shape { + "array" => Amf0Value::StrictArray(names.map(Amf0Value::Utf8String).collect()), + "object" => Amf0Value::Object(names.map(|name| (name.clone(), Amf0Value::Utf8String(name))).collect()), + _ => Amf0Value::Object(names.map(|name| (name, Amf0Value::Number(1.0))).collect()), + }; + let field = if shape == "info_map" { + "videoFourCcInfoMap" + } else { + "fourCcList" + }; + let message = RtmpMessage::Amf0Command { + command_name: "connect".into(), + transaction_id: 1.0, + command_object: Amf0Value::Object( + [ + ("app".into(), Amf0Value::Utf8String("live".into())), + (field.into(), value), + ] + .into_iter() + .collect(), + ), + additional_arguments: Vec::new(), + }; + let payload = message.into_message_payload(RtmpTimestamp::new(0), 0).unwrap(); + let packet = ChunkSerializer::new().serialize(&payload, false, false).unwrap(); + group.throughput(Throughput::Elements(count as u64)); + group.bench_function(BenchmarkId::new(shape, count), |b| { + b.iter_batched( + || ServerSession::new(ServerSessionConfig::new()).unwrap().0, + |mut session| black_box(session.handle_input(black_box(&packet.bytes)).unwrap()), + criterion::BatchSize::SmallInput, + ); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench, amf0, connect); criterion_main!(benches);