diff --git a/Cargo.lock b/Cargo.lock index b30c0b4a72c..85b771881dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5781,6 +5781,7 @@ dependencies = [ "arrayvec", "bench-harness", "bytes", + "crossbeam-epoch", "flate2", "futures", "hashbrown 0.17.1", diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index c25cce3d3b9..25f32f480a0 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -176,7 +176,6 @@ No duplicate quest or workflow edit belongs here. - [Task capacity after bursts](/quest/m2/performance-survey/task-capacity.md) - polling and memory after peak task occupancy subsides - [QUIC receive-state reuse](/quest/m2/performance-survey/quic-receive-reuse.md) - reassembly metadata retained by recycled completed streams - [Cache metadata](/quest/m2/performance-survey/cache-metadata.md) - bookkeeping capacity after cached groups expire -- [Cache backing amplification](/quest/m2/performance-survey/cache-backing.md) - small retained samples pinning large shared fragments - [Bandwidth grants](/quest/m2/performance-survey/bandwidth-grants.md) - repeated full allocation in Rust and JS - [Cached JS group joins](/quest/m2/performance-survey/js-cached-groups.md) - repeated sorting while replaying retained groups - [PCM batch processing](/quest/m2/performance-survey/audio-batches.md) - repeated tail movement during encoding and resampling diff --git a/quest/m2/performance-survey/cache-backing.md b/quest/m2/performance-survey/cache-backing.md deleted file mode 100644 index 82eb01f58bc..00000000000 --- a/quest/m2/performance-survey/cache-backing.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] Measure shared cache backing amplification - -## Goal - -Small retained media samples do not pin disproportionately large backing -allocations when a measured alternative earns its copying cost. - -## Plan - -[Group storage](/rs/moq-net/src/model/group.rs) bills `Bytes::len()` and stores -the supplied view. [fMP4 import](/rs/moq-mux/src/container/fmp4/import.rs) -slices samples from a fragment. Selective expiry across groups or tracks can -leave a small sample holding a large fragment after most logical charges -disappear. Cache capacity is not a promised hard RSS limit; this is a backing -amplification hypothesis, not proof of a broken capacity contract. - -Benchmark the actual importer and cache, sweeping fragment size × retained -sample fraction × tracks. Include mixed audio/video, several keyframe groups -in a fragment, selective expiry and independently allocated samples as a -control. Measure uniquely owned backing bytes versus `Pool::used()`, copy -volume, allocations, CPU and delivery latency. Do not double count shared -backing or assume each slice owns a fresh allocation. Compare continued -sharing with copying/compaction only where the workload warrants it. - -This covers completed reliable groups. Coordinate with -receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) -and [browser media copies](/quest/m2/performance-survey/browser-media-copies.md). -Decoded datagrams are already billed to the cache pool -([moq-dev/moq#6413](https://github.com/moq-dev/moq/pull/6413)) and the media -reorder queues are bounded ([moq-dev/moq#6453](https://github.com/moq-dev/moq/pull/6453)). - -Follow the [survey benchmark contract](/quest/m2/performance-survey/README.md): -baseline first, paired evidence, equivalent work and nightly coverage. A -measured no-win is valid. No API or wire change is expected; report actual impact. diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 405dc55c83e..f487413b9e9 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -45,6 +45,8 @@ uuid = { workspace = true } [dev-dependencies] criterion = { package = "bench-harness", path = "../bench-harness" } +# Runs the group log's deferred frees before the backing bench reads the live heap. +crossbeam-epoch = "0.9.21" futures = { workspace = true } # The tests' executor, with simulated time. kio = { path = "../kio", features = ["runtime", "sim"] } @@ -120,6 +122,12 @@ name = "frame" harness = false required-features = ["fuzz"] +# Receives frames through the private session reader via the hidden `fuzz` module. +[[bench]] +name = "backing" +harness = false +required-features = ["fuzz"] + [[bench]] name = "group" harness = false diff --git a/rs/moq-net/benches/backing.rs b/rs/moq-net/benches/backing.rs new file mode 100644 index 00000000000..fbde04be899 --- /dev/null +++ b/rs/moq-net/benches/backing.rs @@ -0,0 +1,436 @@ +//! What cached frames received from the network keep alive beyond the bytes the cache bills. +//! +//! A receiving session hands each frame's payload to the cache as the transport delivered +//! it. A frame that arrives whole in one chunk would be cached as that chunk, a slice of +//! whatever receive allocation the packet was copied into, while a frame spanning packets is +//! copied into storage of its own. The pool bills each frame's length, so a small frame could +//! keep a much larger receive allocation alive, including the other frames' bytes that were +//! copied out of it. The session's frame reader therefore copies a whole frame of at most +//! 2 KiB out of its chunk, and this bench guards that every backing stays near the control. +//! +//! Each case receives [`SECONDS`] of media on `tracks` video and audio track pairs sharing one +//! pool, through the session's frame reader ([`moq_net::fuzz::GroupRecv`]). Frames are +//! interleaved by timestamp and packed back to back into packets of [`PACKET`] stream bytes, +//! which arrive in one of three backings: +//! +//! - `exact`: every chunk in an allocation of its own, the control. +//! - `datagram`: one allocation per datagram, as `moq-uring` copies a completion without GRO. +//! - `recv64k`: consecutive datagrams share a 64 KiB allocation, as `moq-tokio`'s endpoint +//! driver copies each receive into one reused `BytesMut` and splits datagrams off it. +//! A `moq-uring` GRO completion is one allocation of up to 64 KiB as well. +//! +//! Then the oldest video groups are evicted, keeping `keep` of them, while every audio group +//! stays cached: the selective expiry that leaves a small audio frame holding a receive +//! allocation whose video bytes the cache already dropped. +//! +//! `BACKING_ALLOCS=1` prints, for every case, the live heap the cache retains against +//! [`cache::Pool::used`], the payload bytes retained, how many frames arrived whole, and the +//! allocations and bytes allocated per frame received, instead of timing. The live heap +//! counts each allocation once, however many frames slice it. An argument filters the cases +//! by name. Timing covers only the cases that keep every group, since eviction adds a few +//! aborts. The first time a timed case runs, it prints the same line for itself and for the +//! two cases that evict its video groups, so a timed run such as the nightly one logs every +//! case's memory. The ratio is reported, not asserted: the reader's unit test pins the copy. +//! +//! Run with `cargo bench -p moq-net --features fuzz --bench backing`. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}; + +use bytes::{Bytes, BytesMut}; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_net::{Timescale, Timestamp, broadcast, cache, fuzz::GroupRecv, group, track}; + +/// Stream bytes per packet: a 1200-byte datagram less its QUIC overhead. +const PACKET: usize = 1150; + +/// What `moq-tokio`'s receive buffer reallocates once a datagram split off it is still held: +/// `bytes` keeps at most 64 KiB of a buffer's original capacity. +const RECV_CAPACITY: usize = 64 * 1024; + +/// Media received per case. +const SECONDS: u64 = 10; + +/// One group per track every two seconds: a video GOP, and the audio beside it. +const GROUP_MS: u64 = 2000; + +const VIDEO_FPS: u64 = 30; +/// A keyframe is this many delta frames. +const KEYFRAME: u64 = 10; + +/// 20 ms Opus frames at 64 kbps. +const AUDIO_FPS: u64 = 50; +const AUDIO_FRAME: usize = 160; + +/// Video bitrates, in kbps. Only the lowest has delta frames small enough to arrive whole. +const VIDEO_KBPS: [u64; 3] = [150, 1000, 4000]; +const TRACKS: [usize; 2] = [1, 4]; + +#[derive(Clone, Copy, Debug)] +enum Backing { + Exact, + Datagram, + Recv, +} + +impl Backing { + const ALL: [Self; 3] = [Self::Exact, Self::Datagram, Self::Recv]; + + fn name(self) -> &'static str { + match self { + Self::Exact => "exact", + Self::Datagram => "datagram", + Self::Recv => "recv64k", + } + } +} + +/// How many of each video track's groups stay cached: all, the newest quarter, or none. +const KEEP: [(&str, u64, u64); 3] = [("all", 1, 1), ("quarter", 1, 4), ("none", 0, 1)]; + +struct Counter; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +static ALLOCATED: AtomicUsize = AtomicUsize::new(0); +static LIVE: AtomicIsize = AtomicIsize::new(0); + +#[global_allocator] +static ALLOCATOR: Counter = Counter; + +impl Counter { + fn count(size: usize) { + if COUNTING.load(Ordering::Relaxed) { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED.fetch_add(size, Ordering::Relaxed); + } + } +} + +// Counting only. Every call forwards to the system allocator unchanged. +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + Self::count(new_size); + LIVE.fetch_add(new_size as isize - layout.size() as isize, Ordering::Relaxed); + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size() as isize, Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) } + } +} + +/// One frame of the workload, and the packets its payload landed in. +struct Frame { + /// Index into [`Media::tracks`]: video tracks first, then audio. + track: usize, + timestamp: u64, + /// Opens a group. + start: bool, + /// `(packet, offset, len)` per chunk, in order. + chunks: Vec<(usize, usize, usize)>, +} + +/// The workload of one case, packetized once. +struct Media { + tracks: usize, + frames: Vec, + /// Stream bytes per packet. + packets: Vec, + payload: usize, +} + +impl Media { + fn new(tracks: usize, video_kbps: u64) -> Self { + // A GOP of one keyframe and the deltas after it carries the bitrate. + let gop = VIDEO_FPS * GROUP_MS / 1000; + let delta = (video_kbps * 1000 / 8 * GROUP_MS / 1000 / (KEYFRAME + gop - 1)) as usize; + + let mut frames = Vec::new(); + for track in 0..tracks { + for index in 0..VIDEO_FPS * SECONDS { + let timestamp = index * 1000 / VIDEO_FPS; + let start = index % gop == 0; + let size = if start { delta * KEYFRAME as usize } else { delta }; + frames.push((timestamp, track, start, size)); + } + for index in 0..AUDIO_FPS * SECONDS { + let timestamp = index * 1000 / AUDIO_FPS; + let start = timestamp % GROUP_MS < 1000 / AUDIO_FPS; + frames.push((timestamp, tracks + track, start, AUDIO_FRAME)); + } + } + frames.sort_by_key(|(timestamp, track, ..)| (*timestamp, *track)); + + // Back to back, each packet filled before the next. + let mut packets = vec![0]; + let mut payload = 0; + let frames = frames + .into_iter() + .map(|(timestamp, track, start, mut size)| { + payload += size; + let mut chunks = Vec::new(); + while size > 0 { + let packet = packets.len() - 1; + let offset = packets[packet]; + let len = size.min(PACKET - offset); + chunks.push((packet, offset, len)); + size -= len; + packets[packet] += len; + if packets[packet] == PACKET { + packets.push(0); + } + } + Frame { + track, + timestamp, + start, + chunks, + } + }) + .collect(); + + Self { + tracks: 2 * tracks, + frames, + packets, + payload, + } + } + + fn is_video(&self, track: usize) -> bool { + track < self.tracks / 2 + } +} + +/// The transport's receive path: copies each datagram into its backing and hands out slices. +struct Driver { + backing: Backing, + recv: BytesMut, + /// The packet the last chunk came from. + current: Option<(usize, Bytes)>, +} + +/// What every datagram's bytes are copied from: the content does not matter. +static WIRE: [u8; PACKET] = [0; PACKET]; + +impl Driver { + fn new(backing: Backing) -> Self { + Self { + backing, + recv: match backing { + Backing::Recv => BytesMut::with_capacity(RECV_CAPACITY), + _ => BytesMut::new(), + }, + current: None, + } + } + + fn chunk(&mut self, media: &Media, (packet, offset, len): (usize, usize, usize)) -> Bytes { + let bytes = media.packets[packet]; + match self.backing { + Backing::Exact => Bytes::copy_from_slice(&WIRE[offset..offset + len]), + Backing::Datagram | Backing::Recv => { + if self.current.as_ref().is_none_or(|(current, _)| *current != packet) { + let datagram = match self.backing { + // As `moq-uring` copies a completion out of its receive pool. + Backing::Datagram => BytesMut::from(&WIRE[..bytes]).freeze(), + // As `moq-tokio` receives: reserve, copy in, split off. + _ => { + self.recv.reserve(bytes); + self.recv.extend_from_slice(&WIRE[..bytes]); + self.recv.split_to(bytes).freeze() + } + }; + self.current = Some((packet, datagram)); + } + let (_, datagram) = self.current.as_ref().expect("set above"); + datagram.slice(offset..offset + len) + } + } + } +} + +/// A receiving broadcast, its tracks sharing one pool. +struct Cache { + pool: cache::Pool, + _broadcast: broadcast::Producer, + tracks: Vec, + /// Each track's finished groups, oldest first, with their payload bytes. + groups: Vec>, +} + +impl Cache { + fn new(media: &Media) -> Self { + let mut info = broadcast::Info::new(); + info.pool = cache::Pool::unbounded(); + let pool = info.pool.clone(); + let broadcast = info.produce(); + let tracks = (0..media.tracks) + .map(|index| { + let info = track::Info::default().with_timescale(Timescale::MILLI); + broadcast.create_track(format!("{index}").as_str(), info).unwrap() + }) + .collect(); + Self { + pool, + _broadcast: broadcast, + tracks, + groups: (0..media.tracks).map(|_| Vec::new()).collect(), + } + } + + /// Receive every frame, then evict all but `keep` of each video track's groups. + fn run(&mut self, media: &Media, backing: Backing, (_, num, den): (&str, u64, u64)) { + let mut driver = Driver::new(backing); + let mut open: Vec> = (0..media.tracks).map(|_| None).collect(); + for frame in &media.frames { + let slot = &mut open[frame.track]; + if frame.start { + if let Some((recv, bytes)) = slot.take() { + self.groups[frame.track].push((recv.finish().unwrap(), bytes)); + } + *slot = Some((GroupRecv::new(self.tracks[frame.track].append_group().unwrap()), 0)); + } + let (recv, bytes) = slot.as_mut().expect("every track opens with a group"); + let chunks: Vec<_> = frame.chunks.iter().map(|chunk| driver.chunk(media, *chunk)).collect(); + *bytes += chunks.iter().map(Bytes::len).sum::(); + let timestamp = Timestamp::new(frame.timestamp, Timescale::MILLI).unwrap(); + recv.frame(Some(timestamp), chunks).unwrap(); + } + for (track, recv) in open.into_iter().enumerate() { + if let Some((recv, bytes)) = recv { + self.groups[track].push((recv.finish().unwrap(), bytes)); + } + } + drop(driver); + + for track in (0..media.tracks).filter(|track| media.is_video(*track)) { + let groups = &mut self.groups[track]; + let keep = groups.len() as u64 * num / den; + let evict = groups.len() - keep as usize; + for (group, _) in groups.drain(..evict) { + group.abort(moq_net::Error::Evicted).unwrap(); + } + } + } + + /// Payload bytes still cached. + fn payload(&self) -> usize { + self.groups.iter().flatten().map(|(_, bytes)| bytes).sum() + } +} + +/// Every `(name, tracks, video kbps, backing, keep)` case. +fn cases() -> impl Iterator { + TRACKS.into_iter().flat_map(|tracks| { + VIDEO_KBPS.into_iter().flat_map(move |kbps| { + KEEP.into_iter().flat_map(move |keep| { + Backing::ALL + .into_iter() + .map(move |backing| (name(tracks, kbps, keep, backing), tracks, kbps, backing, keep)) + }) + }) + }) +} + +/// A case's name within the `backing` group. +fn name(tracks: usize, kbps: u64, keep: (&str, u64, u64), backing: Backing) -> String { + format!("tracks={tracks}/video={kbps}k/keep={}/{}", keep.0, backing.name()) +} + +/// The live heap, once the cache's deferred frees have run. +fn live() -> isize { + // The group log frees evicted entries through epoch-based reclamation. + for _ in 0..64 { + crossbeam_epoch::pin().flush(); + } + LIVE.load(Ordering::Relaxed) +} + +/// Receive one case once, describing the live heap it retains against the pool's bill, the +/// payload retained, and the allocations and bytes allocated per frame received. +fn measure(media: &Media, backing: Backing, keep: (&str, u64, u64)) -> String { + let before = live(); + let mut cache = Cache::new(media); + let empty = live(); + + ALLOCATIONS.store(0, Ordering::Relaxed); + ALLOCATED.store(0, Ordering::Relaxed); + COUNTING.store(true, Ordering::Relaxed); + cache.run(media, backing, keep); + COUNTING.store(false, Ordering::Relaxed); + + let retained = live() - empty; + let used = cache.pool.used(); + let payload = cache.payload(); + drop(cache); + let left = live() - before; + + // The frames the cache can keep as the chunk they arrived in. + let whole = media.frames.iter().filter(|frame| frame.chunks.len() == 1).count(); + let frames = media.frames.len(); + let allocations = ALLOCATIONS.load(Ordering::Relaxed) as f64 / frames as f64; + let allocated = ALLOCATED.load(Ordering::Relaxed) as f64 / frames as f64; + format!( + "retained {retained} B, pool {used} B ({:.2}x), payload {payload} B, {whole} of {frames} frames whole, \ + {allocations:.2} allocations and {allocated:.0} B allocated per frame, {left} B left after drop", + retained as f64 / used as f64, + ) +} + +fn bench(c: &mut Criterion) { + // Every case's memory, without timing. + if std::env::var_os("BACKING_ALLOCS").is_some() { + let filter = std::env::args().skip(1).find(|arg| !arg.starts_with('-')); + for (name, tracks, kbps, backing, keep) in cases() { + if filter.as_deref().is_none_or(|filter| name.contains(filter)) { + println!("{name}: {}", measure(&Media::new(tracks, kbps), backing, keep)); + } + } + return; + } + + // Eviction is a handful of aborts, so only the cases keeping every group are timed. The + // first run of one, which Criterion's filter allows only for a selected case, prints the + // memory of every case sharing its media and backing, untimed. + let mut group = c.benchmark_group("backing"); + for (id, tracks, kbps, backing, keep) in cases().filter(|(.., keep)| keep.0 == "all") { + let media = Media::new(tracks, kbps); + let mut measured = false; + group.throughput(Throughput::Bytes(media.payload as u64)); + group.bench_function(BenchmarkId::from_parameter(id), |b| { + if !std::mem::replace(&mut measured, true) { + for other in KEEP { + let case = name(tracks, kbps, other, backing); + eprintln!("backing/{case}: {}", measure(&media, backing, other)); + } + } + b.iter_batched( + || Cache::new(&media), + |mut cache| { + cache.run(&media, backing, keep); + cache + }, + BatchSize::PerIteration, + ) + }); + } + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index ac38145f937..eb47277624f 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -40,6 +40,17 @@ pub struct Reader { /// enough that an ordinary burst is still one wake. const WAKE_BUDGET: usize = 64 * 1024; +/// The largest frame [`Reader::poll_read_frame`] copies out of a chunk holding all of it, +/// rather than caching the chunk as it arrived. +/// +/// A transport hands over slices of its receive allocations: a QUIC datagram, or a whole +/// receive batch of them. A frame that arrives in one such slice and is cached as-is keeps +/// that allocation alive, the other frames' bytes included, while the cache bills only the +/// frame. This covers a QUIC datagram's stream payload (about 1.45 KB at the default +/// `max_udp_payload_size`), not a jumbo path's, so a larger frame from the in-memory +/// transport still crosses as the publisher's allocation. +const COPY_WHOLE: usize = 2 * 1024; + /// The largest [`Reader::scratch`] kept between decodes; one past this is freed. const SCRATCH_KEEP: usize = 64 * 1024; @@ -399,14 +410,22 @@ impl Reader { // One chunk per lease: a lease may hold the transport's lock, so the frame is // written, and its consumers woken, only once the lease is gone. A chunk holding - // the rest of the frame is taken whole, so a frame delivered in one piece keeps - // its allocation; any other stops where the segment being written seals. + // the rest of the frame is taken whole, so a large frame delivered in one piece + // keeps its allocation, and one of at most [`COPY_WHOLE`] is copied out of it; + // any other stops where the segment being written seals. let remaining = frame.remaining(); let take = match lease.chunk().len() >= remaining { true => remaining, false => lease.chunk().len().min(frame.unsealed()), }; - let chunk = lease.copy_to_bytes(take); + let chunk = match take as u64 == frame.size && take <= COPY_WHOLE { + true => { + let chunk = Bytes::copy_from_slice(&lease.chunk()[..take]); + lease.advance(take); + chunk + } + false => lease.copy_to_bytes(take), + }; let complete = chunk.len() == remaining; let ended = lease.is_fin() && !lease.has_remaining(); if complete || chunk.len() == frame.unsealed() { @@ -887,6 +906,36 @@ mod tests { )); } + /// A frame that arrives whole in a larger chunk, as in a datagram it shares with other + /// frames, is copied out of it when it is at most [`COPY_WHOLE`], so caching it does not + /// keep the rest of the receive allocation alive. A larger one is cached as the chunk. + #[test] + fn a_small_whole_frame_is_copied_out_of_its_chunk() { + let datagram: &'static [u8] = vec![7u8; 2 * COPY_WHOLE + 2].leak(); + for size in [1, COPY_WHOLE, COPY_WHOLE + 1] { + let (mut frame, mut payload, waiter, _wakes) = parked_frame(size); + let mut reader = queue(&[datagram], crate::lite::Version::Lite05); + let noop = &kio::Waiter::noop(); + assert!(matches!(reader.poll_read_frame(noop, &mut frame), Poll::Ready(Ok(())))); + let _group = frame.finish().unwrap(); + let Poll::Ready(Ok(Some(chunk))) = payload.poll_read_chunk(&waiter) else { + panic!("the committed frame has no payload"); + }; + assert_eq!(chunk, datagram[..size], "size={size}"); + assert_eq!( + chunk.as_ptr() == datagram.as_ptr(), + size > COPY_WHOLE, + "size={size}: copied {}", + size <= COPY_WHOLE + ); + assert_eq!( + leased(&mut reader), + datagram.len() - size, + "size={size}: the rest stays leased" + ); + } + } + /// A consumer parked on the group cannot run until this task yields, so a burst of /// chunks owes exactly one wake: at the boundary, from whoever wrote them. Every /// other exit hands the wake to the caller's `finish` or `abort`. diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index c223068bb1b..d1722414d7f 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1670,6 +1670,87 @@ impl crate::transport::Error for NoError { } } +/// One group's stream on a receiving session, as the backing bench feeds it: each frame's +/// payload arrives as the chunks given, exactly as a transport hands them over, and passes +/// through the session's frame reader into the cache. +/// +/// A chunk keeps whatever allocation it was sliced from, so the bench can see which +/// receive buffers the cached frames keep alive. +pub struct GroupRecv { + reader: coding::Reader, lite::Version>, + queue: std::rc::Rc>>, + group: Option, +} + +impl GroupRecv { + /// Receive frames into `group`. + pub fn new(group: group::Producer) -> Self { + let queue = std::rc::Rc::default(); + let stream = Queue(std::rc::Rc::clone(&queue)); + Self { + reader: coding::Reader::new(crate::transport::poll::Buffered::new(stream), lite::Version::Lite05), + queue, + group: Some(group), + } + } + + /// Receive one frame whose payload is `chunks`, in order, and commit it. + pub fn frame( + &mut self, + timestamp: Option, + chunks: impl IntoIterator, + ) -> crate::Result<()> { + let mut size = 0; + self.queue + .borrow_mut() + .extend(chunks.into_iter().inspect(|chunk| size += chunk.len())); + let group = self.group.take().ok_or(crate::Error::Closed)?; + let mut frame = group.create_frame(frame::Info { + size: size as u64, + timestamp, + })?; + match self.reader.poll_read_frame(&kio::Waiter::noop(), &mut frame) { + std::task::Poll::Ready(Ok(())) => {} + std::task::Poll::Ready(Err(err)) => return Err(err), + std::task::Poll::Pending => unreachable!("every chunk of the frame is queued"), + } + self.group = Some(frame.finish()?); + Ok(()) + } + + /// Finish the group, returning its producer, which keeps it cached until dropped or aborted. + pub fn finish(mut self) -> crate::Result { + let mut group = self.group.take().ok_or(crate::Error::Closed)?; + group.finish()?; + Ok(group) + } +} + +/// The chunks queued by [`GroupRecv::frame`], handed over one at a time without copying. +struct Queue(std::rc::Rc>>); + +impl crate::transport::poll::Chunks for Queue { + type Error = NoError; + + fn poll_read_chunk( + &mut self, + _waiter: &kio::Waiter, + max: usize, + ) -> std::task::Poll, NoError>> { + let mut queue = self.0.borrow_mut(); + let Some(front) = queue.front_mut() else { + return std::task::Poll::Pending; + }; + let chunk = match front.len() > max { + true => front.split_to(max), + false => queue.pop_front().expect("front exists"), + }; + std::task::Poll::Ready(Ok(Some(chunk))) + } + + fn stop(&mut self, _code: u32) {} +} + /// A source-selection watch for the private request-lifetime benchmark. pub struct RequestValidity { watch: crate::model::broadcast::RequestValidity, diff --git a/rs/moq-net/tests/integration/zero_copy.rs b/rs/moq-net/tests/integration/zero_copy.rs index b1780917af4..b8888660444 100644 --- a/rs/moq-net/tests/integration/zero_copy.rs +++ b/rs/moq-net/tests/integration/zero_copy.rs @@ -2,7 +2,8 @@ //! A frame crosses an in-memory session as the same allocation it was written as: the //! publisher hands the payload to the transport, and the subscriber reads it back, -//! without a copy on either side. +//! without a copy on either side. Only a frame of at most 2 KiB is copied, so a small +//! frame never keeps a larger receive allocation alive. use crate::support;