From ac4677082d1592d3ec7e9595b50843b54e4bc842 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:24:41 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m2/performance-survey/cache-backing.md Quest-Claim: 5714f9ecc7a5db3b From fb1f340472b30fb9475b616f540111dc93f24191 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:17:43 -0800 Subject: [PATCH 2/5] bench(net): measure the receive buffers cached frames keep alive A frame that arrives whole in one transport chunk is cached as that chunk, a slice of the datagram's receive allocation. The new `backing` bench receives mixed audio and video through the session's frame reader with per-chunk, per-datagram and moq-tokio-style 64 KiB receive allocations, evicts video groups while audio stays, and prints the live heap retained against Pool::used(). With 64 KiB allocations it retains 1.9x with nothing evicted and up to 33x once the video is gone. The quest is rewritten to the remaining fix, which waits on choosing the copy rule. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + quest/m2/performance-survey/README.md | 2 +- quest/m2/performance-survey/cache-backing.md | 51 ++- rs/moq-net/Cargo.toml | 8 + rs/moq-net/benches/backing.rs | 422 +++++++++++++++++++ rs/moq-net/src/fuzz.rs | 81 ++++ 6 files changed, 538 insertions(+), 27 deletions(-) create mode 100644 rs/moq-net/benches/backing.rs diff --git a/Cargo.lock b/Cargo.lock index df17b147ca5..308595b2f7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5769,6 +5769,7 @@ dependencies = [ "arrayvec", "bench-harness", "bytes", + "crossbeam-epoch", "flate2", "futures", "hashbrown 0.17.1", diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index c25cce3d3b9..fea932a30a2 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -176,7 +176,7 @@ No duplicate quest or workflow edit belongs here. - [Task capacity after bursts](/quest/m2/performance-survey/task-capacity.md) - polling and memory after peak task occupancy subsides - [QUIC receive-state reuse](/quest/m2/performance-survey/quic-receive-reuse.md) - reassembly metadata retained by recycled completed streams - [Cache metadata](/quest/m2/performance-survey/cache-metadata.md) - bookkeeping capacity after cached groups expire -- [Cache backing amplification](/quest/m2/performance-survey/cache-backing.md) - small retained samples pinning large shared fragments +- [Cache backing amplification](/quest/m2/performance-survey/cache-backing.md) - small received frames pinning shared receive buffers - [Bandwidth grants](/quest/m2/performance-survey/bandwidth-grants.md) - repeated full allocation in Rust and JS - [Cached JS group joins](/quest/m2/performance-survey/js-cached-groups.md) - repeated sorting while replaying retained groups - [PCM batch processing](/quest/m2/performance-survey/audio-batches.md) - repeated tail movement during encoding and resampling diff --git a/quest/m2/performance-survey/cache-backing.md b/quest/m2/performance-survey/cache-backing.md index 82eb01f58bc..8ef98808c7c 100644 --- a/quest/m2/performance-survey/cache-backing.md +++ b/quest/m2/performance-survey/cache-backing.md @@ -1,34 +1,33 @@ -# [M] Measure shared cache backing amplification +# [S] Copy small received frames out of shared receive buffers ## Goal -Small retained media samples do not pin disproportionately large backing -allocations when a measured alternative earns its copying cost. +A small frame a session receives whole is not cached as a slice of the +transport's receive allocation, so the live heap behind cached received frames +stays close to what `Pool::used()` bills. Large frames crossing the in-memory +transport still arrive without a copy. ## Plan -[Group storage](/rs/moq-net/src/model/group.rs) bills `Bytes::len()` and stores -the supplied view. [fMP4 import](/rs/moq-mux/src/container/fmp4/import.rs) -slices samples from a fragment. Selective expiry across groups or tracks can -leave a small sample holding a large fragment after most logical charges -disappear. Cache capacity is not a promised hard RSS limit; this is a backing -amplification hypothesis, not proof of a broken capacity contract. +[`backing`](/rs/moq-net/benches/backing.rs) measured this in +[moq-dev/moq#7522](https://github.com/moq-dev/moq/pull/7522). Run it with +`BACKING_ALLOCS=1 cargo bench -p moq-net --features fuzz --bench backing`. +[`Reader::poll_read_frame`](/rs/moq-net/src/coding/reader.rs) caches a frame +that arrives in one transport chunk as that chunk. Under QUIC, that is a slice +of a datagram. `moq-tokio` copies datagrams into shared 64 KiB receive +allocations, and a `moq-uring` GRO completion is a single allocation as well. +With 64 KiB receive allocations, 4 Mbps video and 64 kbps audio retained 1.9x +the pool's bill with nothing evicted. With the video groups evicted and the +audio kept, it retained 33x. Copying such frames brought every case back to +the control (1.01-1.19x). The copy cost 880-1,120 user instructions per frame +copied, adding 7-16% to ingest. -Benchmark the actual importer and cache, sweeping fragment size × retained -sample fraction × tracks. Include mixed audio/video, several keyframe groups -in a fragment, selective expiry and independently allocated samples as a -control. Measure uniquely owned backing bytes versus `Pool::used()`, copy -volume, allocations, CPU and delivery latency. Do not double count shared -backing or assume each slice owns a fresh allocation. Compare continued -sharing with copying/compaction only where the workload warrants it. +Pick the copy rule in moq-dev/moq#7522, then land it with the bench as its +evidence. The size threshold decides how far the copy reaches into +in-memory transport frames, which currently cross as the publisher's +allocation. -This covers completed reliable groups. Coordinate with -receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) -and [browser media copies](/quest/m2/performance-survey/browser-media-copies.md). -Decoded datagrams are already billed to the cache pool -([moq-dev/moq#6413](https://github.com/moq-dev/moq/pull/6413)) and the media -reorder queues are bounded ([moq-dev/moq#6453](https://github.com/moq-dev/moq/pull/6453)). - -Follow the [survey benchmark contract](/quest/m2/performance-survey/README.md): -baseline first, paired evidence, equivalent work and nightly coverage. A -measured no-win is valid. No API or wire change is expected; report actual impact. +The fMP4 importer is not affected: every sample is copied into its own frame, +so its retained heap beyond the bill is a constant ~0.3 MB of importer state. +Its passthrough fragments do keep up to 2x capacity slack, but +[moq-dev/moq#7232](https://github.com/moq-dev/moq/pull/7232) removes it. diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index d0f3460b673..0deb2e34f6c 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -45,6 +45,8 @@ uuid = { workspace = true } [dev-dependencies] criterion = { package = "bench-harness", path = "../bench-harness" } +# Runs the group log's deferred frees before the backing bench reads the live heap. +crossbeam-epoch = "0.9.21" futures = { workspace = true } # The tests' executor, with simulated time. kio = { path = "../kio", features = ["runtime", "sim"] } @@ -120,6 +122,12 @@ name = "frame" harness = false required-features = ["fuzz"] +# Receives frames through the private session reader via the hidden `fuzz` module. +[[bench]] +name = "backing" +harness = false +required-features = ["fuzz"] + [[bench]] name = "group" harness = false diff --git a/rs/moq-net/benches/backing.rs b/rs/moq-net/benches/backing.rs new file mode 100644 index 00000000000..fdac092f591 --- /dev/null +++ b/rs/moq-net/benches/backing.rs @@ -0,0 +1,422 @@ +//! What cached frames received from the network keep alive beyond the bytes the cache bills. +//! +//! A receiving session hands each frame's payload to the cache as the transport delivered +//! it. A frame that arrives whole in one chunk is cached as that chunk, a slice of whatever +//! receive allocation the packet was copied into, while a frame spanning packets is copied +//! into storage of its own. The pool bills each frame's length, so a small frame can keep a +//! much larger receive allocation alive, including the other frames' bytes that were copied +//! out of it. +//! +//! Each case receives [`SECONDS`] of media on `tracks` video and audio track pairs sharing one +//! pool, through the session's frame reader ([`moq_net::fuzz::GroupRecv`]). Frames are +//! interleaved by timestamp and packed back to back into packets of [`PACKET`] stream bytes, +//! which arrive in one of three backings: +//! +//! - `exact`: every chunk in an allocation of its own, the control. +//! - `datagram`: one allocation per datagram, as `moq-uring` copies a completion without GRO. +//! - `recv64k`: consecutive datagrams share a 64 KiB allocation, as `moq-tokio`'s endpoint +//! driver copies each receive into one reused `BytesMut` and splits datagrams off it. +//! A `moq-uring` GRO completion is one allocation of up to 64 KiB as well. +//! +//! Then the oldest video groups are evicted, keeping `keep` of them, while every audio group +//! stays cached: the selective expiry that leaves a small audio frame holding a receive +//! allocation whose video bytes the cache already dropped. +//! +//! `BACKING_ALLOCS=1` prints, for every case, the live heap the cache retains against +//! [`cache::Pool::used`], the payload bytes retained, how many frames arrived whole, and the +//! allocations and bytes allocated per frame received, instead of timing. The live heap +//! counts each allocation once, however many frames slice it. An argument filters the cases +//! by name. Timing covers only the cases that keep every group, since eviction adds a few +//! aborts, and prints the same line for each before timing it. +//! +//! Run with `cargo bench -p moq-net --features fuzz --bench backing`. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}; + +use bytes::{Bytes, BytesMut}; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_net::{Timescale, Timestamp, broadcast, cache, fuzz::GroupRecv, group, track}; + +/// Stream bytes per packet: a 1200-byte datagram less its QUIC overhead. +const PACKET: usize = 1150; + +/// What `moq-tokio`'s receive buffer reallocates once a datagram split off it is still held: +/// `bytes` keeps at most 64 KiB of a buffer's original capacity. +const RECV_CAPACITY: usize = 64 * 1024; + +/// Media received per case. +const SECONDS: u64 = 10; + +/// One group per track every two seconds: a video GOP, and the audio beside it. +const GROUP_MS: u64 = 2000; + +const VIDEO_FPS: u64 = 30; +/// A keyframe is this many delta frames. +const KEYFRAME: u64 = 10; + +/// 20 ms Opus frames at 64 kbps. +const AUDIO_FPS: u64 = 50; +const AUDIO_FRAME: usize = 160; + +/// Video bitrates, in kbps. Only the lowest has delta frames small enough to arrive whole. +const VIDEO_KBPS: [u64; 3] = [150, 1000, 4000]; +const TRACKS: [usize; 2] = [1, 4]; + +#[derive(Clone, Copy, Debug)] +enum Backing { + Exact, + Datagram, + Recv, +} + +impl Backing { + const ALL: [Self; 3] = [Self::Exact, Self::Datagram, Self::Recv]; + + fn name(self) -> &'static str { + match self { + Self::Exact => "exact", + Self::Datagram => "datagram", + Self::Recv => "recv64k", + } + } +} + +/// How many of each video track's groups stay cached: all, the newest quarter, or none. +const KEEP: [(&str, u64, u64); 3] = [("all", 1, 1), ("quarter", 1, 4), ("none", 0, 1)]; + +struct Counter; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +static ALLOCATED: AtomicUsize = AtomicUsize::new(0); +static LIVE: AtomicIsize = AtomicIsize::new(0); + +#[global_allocator] +static ALLOCATOR: Counter = Counter; + +impl Counter { + fn count(size: usize) { + if COUNTING.load(Ordering::Relaxed) { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED.fetch_add(size, Ordering::Relaxed); + } + } +} + +// Counting only. Every call forwards to the system allocator unchanged. +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + Self::count(new_size); + LIVE.fetch_add(new_size as isize - layout.size() as isize, Ordering::Relaxed); + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size() as isize, Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) } + } +} + +/// One frame of the workload, and the packets its payload landed in. +struct Frame { + /// Index into [`Media::tracks`]: video tracks first, then audio. + track: usize, + timestamp: u64, + /// Opens a group. + start: bool, + /// `(packet, offset, len)` per chunk, in order. + chunks: Vec<(usize, usize, usize)>, +} + +/// The workload of one case, packetized once. +struct Media { + tracks: usize, + frames: Vec, + /// Stream bytes per packet. + packets: Vec, + payload: usize, +} + +impl Media { + fn new(tracks: usize, video_kbps: u64) -> Self { + // A GOP of one keyframe and the deltas after it carries the bitrate. + let gop = VIDEO_FPS * GROUP_MS / 1000; + let delta = (video_kbps * 1000 / 8 * GROUP_MS / 1000 / (KEYFRAME + gop - 1)) as usize; + + let mut frames = Vec::new(); + for track in 0..tracks { + for index in 0..VIDEO_FPS * SECONDS { + let timestamp = index * 1000 / VIDEO_FPS; + let start = index % gop == 0; + let size = if start { delta * KEYFRAME as usize } else { delta }; + frames.push((timestamp, track, start, size)); + } + for index in 0..AUDIO_FPS * SECONDS { + let timestamp = index * 1000 / AUDIO_FPS; + let start = timestamp % GROUP_MS < 1000 / AUDIO_FPS; + frames.push((timestamp, tracks + track, start, AUDIO_FRAME)); + } + } + frames.sort_by_key(|(timestamp, track, ..)| (*timestamp, *track)); + + // Back to back, each packet filled before the next. + let mut packets = vec![0]; + let mut payload = 0; + let frames = frames + .into_iter() + .map(|(timestamp, track, start, mut size)| { + payload += size; + let mut chunks = Vec::new(); + while size > 0 { + let packet = packets.len() - 1; + let offset = packets[packet]; + let len = size.min(PACKET - offset); + chunks.push((packet, offset, len)); + size -= len; + packets[packet] += len; + if packets[packet] == PACKET { + packets.push(0); + } + } + Frame { + track, + timestamp, + start, + chunks, + } + }) + .collect(); + + Self { + tracks: 2 * tracks, + frames, + packets, + payload, + } + } + + fn is_video(&self, track: usize) -> bool { + track < self.tracks / 2 + } +} + +/// The transport's receive path: copies each datagram into its backing and hands out slices. +struct Driver { + backing: Backing, + recv: BytesMut, + /// The packet the last chunk came from. + current: Option<(usize, Bytes)>, +} + +/// What every datagram's bytes are copied from: the content does not matter. +static WIRE: [u8; PACKET] = [0; PACKET]; + +impl Driver { + fn new(backing: Backing) -> Self { + Self { + backing, + recv: match backing { + Backing::Recv => BytesMut::with_capacity(RECV_CAPACITY), + _ => BytesMut::new(), + }, + current: None, + } + } + + fn chunk(&mut self, media: &Media, (packet, offset, len): (usize, usize, usize)) -> Bytes { + let bytes = media.packets[packet]; + match self.backing { + Backing::Exact => Bytes::copy_from_slice(&WIRE[offset..offset + len]), + Backing::Datagram | Backing::Recv => { + if self.current.as_ref().is_none_or(|(current, _)| *current != packet) { + let datagram = match self.backing { + // As `moq-uring` copies a completion out of its receive pool. + Backing::Datagram => BytesMut::from(&WIRE[..bytes]).freeze(), + // As `moq-tokio` receives: reserve, copy in, split off. + _ => { + self.recv.reserve(bytes); + self.recv.extend_from_slice(&WIRE[..bytes]); + self.recv.split_to(bytes).freeze() + } + }; + self.current = Some((packet, datagram)); + } + let (_, datagram) = self.current.as_ref().expect("set above"); + datagram.slice(offset..offset + len) + } + } + } +} + +/// A receiving broadcast, its tracks sharing one pool. +struct Cache { + pool: cache::Pool, + _broadcast: broadcast::Producer, + tracks: Vec, + /// Each track's finished groups, oldest first, with their payload bytes. + groups: Vec>, +} + +impl Cache { + fn new(media: &Media) -> Self { + let mut info = broadcast::Info::new(); + info.pool = cache::Pool::unbounded(); + let pool = info.pool.clone(); + let broadcast = info.produce(); + let tracks = (0..media.tracks) + .map(|index| { + let info = track::Info::default().with_timescale(Timescale::MILLI); + broadcast.create_track(format!("{index}").as_str(), info).unwrap() + }) + .collect(); + Self { + pool, + _broadcast: broadcast, + tracks, + groups: (0..media.tracks).map(|_| Vec::new()).collect(), + } + } + + /// Receive every frame, then evict all but `keep` of each video track's groups. + fn run(&mut self, media: &Media, backing: Backing, (_, num, den): (&str, u64, u64)) { + let mut driver = Driver::new(backing); + let mut open: Vec> = (0..media.tracks).map(|_| None).collect(); + for frame in &media.frames { + let slot = &mut open[frame.track]; + if frame.start { + if let Some((recv, bytes)) = slot.take() { + self.groups[frame.track].push((recv.finish().unwrap(), bytes)); + } + *slot = Some((GroupRecv::new(self.tracks[frame.track].append_group().unwrap()), 0)); + } + let (recv, bytes) = slot.as_mut().expect("every track opens with a group"); + let chunks: Vec<_> = frame.chunks.iter().map(|chunk| driver.chunk(media, *chunk)).collect(); + *bytes += chunks.iter().map(Bytes::len).sum::(); + let timestamp = Timestamp::new(frame.timestamp, Timescale::MILLI).unwrap(); + recv.frame(Some(timestamp), chunks).unwrap(); + } + for (track, recv) in open.into_iter().enumerate() { + if let Some((recv, bytes)) = recv { + self.groups[track].push((recv.finish().unwrap(), bytes)); + } + } + drop(driver); + + for track in (0..media.tracks).filter(|track| media.is_video(*track)) { + let groups = &mut self.groups[track]; + let keep = groups.len() as u64 * num / den; + let evict = groups.len() - keep as usize; + for (group, _) in groups.drain(..evict) { + group.abort(moq_net::Error::Evicted).unwrap(); + } + } + } + + /// Payload bytes still cached. + fn payload(&self) -> usize { + self.groups.iter().flatten().map(|(_, bytes)| bytes).sum() + } +} + +/// Every `(name, tracks, video kbps, backing, keep)` case. +fn cases() -> impl Iterator { + TRACKS.into_iter().flat_map(|tracks| { + VIDEO_KBPS.into_iter().flat_map(move |kbps| { + KEEP.into_iter().flat_map(move |keep| { + Backing::ALL.into_iter().map(move |backing| { + let name = format!("tracks={tracks}/video={kbps}k/keep={}/{}", keep.0, backing.name()); + (name, tracks, kbps, backing, keep) + }) + }) + }) + }) +} + +/// The live heap, once the cache's deferred frees have run. +fn live() -> isize { + // The group log frees evicted entries through epoch-based reclamation. + for _ in 0..64 { + crossbeam_epoch::pin().flush(); + } + LIVE.load(Ordering::Relaxed) +} + +/// Receive one case once, describing the live heap it retains against the pool's bill, the +/// payload retained, and the allocations and bytes allocated per frame received. +fn measure(media: &Media, backing: Backing, keep: (&str, u64, u64)) -> String { + let before = live(); + let mut cache = Cache::new(media); + let empty = live(); + + ALLOCATIONS.store(0, Ordering::Relaxed); + ALLOCATED.store(0, Ordering::Relaxed); + COUNTING.store(true, Ordering::Relaxed); + cache.run(media, backing, keep); + COUNTING.store(false, Ordering::Relaxed); + + let retained = live() - empty; + let used = cache.pool.used(); + let payload = cache.payload(); + drop(cache); + let left = live() - before; + + // The frames the cache can keep as the chunk they arrived in. + let whole = media.frames.iter().filter(|frame| frame.chunks.len() == 1).count(); + let frames = media.frames.len(); + let allocations = ALLOCATIONS.load(Ordering::Relaxed) as f64 / frames as f64; + let allocated = ALLOCATED.load(Ordering::Relaxed) as f64 / frames as f64; + format!( + "retained {retained} B, pool {used} B ({:.2}x), payload {payload} B, {whole} of {frames} frames whole, \ + {allocations:.2} allocations and {allocated:.0} B allocated per frame, {left} B left after drop", + retained as f64 / used as f64, + ) +} + +fn bench(c: &mut Criterion) { + // Every case's memory, without timing. + if std::env::var_os("BACKING_ALLOCS").is_some() { + let filter = std::env::args().skip(1).find(|arg| !arg.starts_with('-')); + for (name, tracks, kbps, backing, keep) in cases() { + if filter.as_deref().is_none_or(|filter| name.contains(filter)) { + println!("{name}: {}", measure(&Media::new(tracks, kbps), backing, keep)); + } + } + return; + } + + // Eviction is a handful of aborts, so only the cases keeping every group are timed, each + // with its memory printed beside it. + let mut group = c.benchmark_group("backing"); + for (name, tracks, kbps, backing, keep) in cases().filter(|(.., keep)| keep.0 == "all") { + let media = Media::new(tracks, kbps); + eprintln!("backing/{name}: {}", measure(&media, backing, keep)); + group.throughput(Throughput::Bytes(media.payload as u64)); + group.bench_function(BenchmarkId::from_parameter(name), |b| { + b.iter_batched( + || Cache::new(&media), + |mut cache| { + cache.run(&media, backing, keep); + cache + }, + BatchSize::PerIteration, + ) + }); + } + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index e302e571d63..dc22777a732 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1673,6 +1673,87 @@ impl crate::transport::Error for NoError { } } +/// One group's stream on a receiving session, as the backing bench feeds it: each frame's +/// payload arrives as the chunks given, exactly as a transport hands them over, and passes +/// through the session's frame reader into the cache. +/// +/// A chunk keeps whatever allocation it was sliced from, so the bench can see which +/// receive buffers the cached frames keep alive. +pub struct GroupRecv { + reader: coding::Reader, lite::Version>, + queue: std::rc::Rc>>, + group: Option, +} + +impl GroupRecv { + /// Receive frames into `group`. + pub fn new(group: group::Producer) -> Self { + let queue = std::rc::Rc::default(); + let stream = Queue(std::rc::Rc::clone(&queue)); + Self { + reader: coding::Reader::new(crate::transport::poll::Buffered::new(stream), lite::Version::Lite05), + queue, + group: Some(group), + } + } + + /// Receive one frame whose payload is `chunks`, in order, and commit it. + pub fn frame( + &mut self, + timestamp: Option, + chunks: impl IntoIterator, + ) -> crate::Result<()> { + let mut size = 0; + self.queue + .borrow_mut() + .extend(chunks.into_iter().inspect(|chunk| size += chunk.len())); + let group = self.group.take().ok_or(crate::Error::Closed)?; + let mut frame = group.create_frame(frame::Info { + size: size as u64, + timestamp, + })?; + match self.reader.poll_read_frame(&kio::Waiter::noop(), &mut frame) { + std::task::Poll::Ready(Ok(())) => {} + std::task::Poll::Ready(Err(err)) => return Err(err), + std::task::Poll::Pending => unreachable!("every chunk of the frame is queued"), + } + self.group = Some(frame.finish()?); + Ok(()) + } + + /// Finish the group, returning its producer, which keeps it cached until dropped or aborted. + pub fn finish(mut self) -> crate::Result { + let mut group = self.group.take().ok_or(crate::Error::Closed)?; + group.finish()?; + Ok(group) + } +} + +/// The chunks queued by [`GroupRecv::frame`], handed over one at a time without copying. +struct Queue(std::rc::Rc>>); + +impl crate::transport::poll::Chunks for Queue { + type Error = NoError; + + fn poll_read_chunk( + &mut self, + _waiter: &kio::Waiter, + max: usize, + ) -> std::task::Poll, NoError>> { + let mut queue = self.0.borrow_mut(); + let Some(front) = queue.front_mut() else { + return std::task::Poll::Pending; + }; + let chunk = match front.len() > max { + true => front.split_to(max), + false => queue.pop_front().expect("front exists"), + }; + std::task::Poll::Ready(Ok(Some(chunk))) + } + + fn stop(&mut self, _code: u32) {} +} + /// A source-selection watch for the private request-lifetime benchmark. pub struct RequestValidity(crate::model::broadcast::RequestValidity); From 00d28a78ad9cf1a18a756e2edb2f5b3d4f361a48 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:42:50 -0800 Subject: [PATCH 3/5] perf(net): copy a small whole frame out of its receive buffer Reader::poll_read_frame cached a frame that arrived in one transport chunk as that chunk, a slice of the QUIC receive allocation it shared with other datagrams. A small audio frame kept up to 64 KiB alive while the pool billed only the frame: 1.9x the bill with nothing evicted, 33x once the video groups beside it were evicted. A whole frame of at most COPY_WHOLE (2 KiB) is now copied out, which covers a QUIC datagram's stream payload (about 1.45 KB). The backing bench returns to the control in every case. Larger frames, including the in-memory transport's, still cross as the publisher's allocation. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/backing.rs | 11 ++--- rs/moq-net/src/coding/reader.rs | 55 +++++++++++++++++++++-- rs/moq-net/tests/integration/zero_copy.rs | 3 +- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/benches/backing.rs b/rs/moq-net/benches/backing.rs index fdac092f591..52c2d35f6b0 100644 --- a/rs/moq-net/benches/backing.rs +++ b/rs/moq-net/benches/backing.rs @@ -1,11 +1,12 @@ //! What cached frames received from the network keep alive beyond the bytes the cache bills. //! //! A receiving session hands each frame's payload to the cache as the transport delivered -//! it. A frame that arrives whole in one chunk is cached as that chunk, a slice of whatever -//! receive allocation the packet was copied into, while a frame spanning packets is copied -//! into storage of its own. The pool bills each frame's length, so a small frame can keep a -//! much larger receive allocation alive, including the other frames' bytes that were copied -//! out of it. +//! it. A frame that arrives whole in one chunk would be cached as that chunk, a slice of +//! whatever receive allocation the packet was copied into, while a frame spanning packets is +//! copied into storage of its own. The pool bills each frame's length, so a small frame could +//! keep a much larger receive allocation alive, including the other frames' bytes that were +//! copied out of it. The session's frame reader therefore copies a whole frame of at most +//! 2 KiB out of its chunk, and this bench guards that every backing stays near the control. //! //! Each case receives [`SECONDS`] of media on `tracks` video and audio track pairs sharing one //! pool, through the session's frame reader ([`moq_net::fuzz::GroupRecv`]). Frames are diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index ac38145f937..eb47277624f 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -40,6 +40,17 @@ pub struct Reader { /// enough that an ordinary burst is still one wake. const WAKE_BUDGET: usize = 64 * 1024; +/// The largest frame [`Reader::poll_read_frame`] copies out of a chunk holding all of it, +/// rather than caching the chunk as it arrived. +/// +/// A transport hands over slices of its receive allocations: a QUIC datagram, or a whole +/// receive batch of them. A frame that arrives in one such slice and is cached as-is keeps +/// that allocation alive, the other frames' bytes included, while the cache bills only the +/// frame. This covers a QUIC datagram's stream payload (about 1.45 KB at the default +/// `max_udp_payload_size`), not a jumbo path's, so a larger frame from the in-memory +/// transport still crosses as the publisher's allocation. +const COPY_WHOLE: usize = 2 * 1024; + /// The largest [`Reader::scratch`] kept between decodes; one past this is freed. const SCRATCH_KEEP: usize = 64 * 1024; @@ -399,14 +410,22 @@ impl Reader { // One chunk per lease: a lease may hold the transport's lock, so the frame is // written, and its consumers woken, only once the lease is gone. A chunk holding - // the rest of the frame is taken whole, so a frame delivered in one piece keeps - // its allocation; any other stops where the segment being written seals. + // the rest of the frame is taken whole, so a large frame delivered in one piece + // keeps its allocation, and one of at most [`COPY_WHOLE`] is copied out of it; + // any other stops where the segment being written seals. let remaining = frame.remaining(); let take = match lease.chunk().len() >= remaining { true => remaining, false => lease.chunk().len().min(frame.unsealed()), }; - let chunk = lease.copy_to_bytes(take); + let chunk = match take as u64 == frame.size && take <= COPY_WHOLE { + true => { + let chunk = Bytes::copy_from_slice(&lease.chunk()[..take]); + lease.advance(take); + chunk + } + false => lease.copy_to_bytes(take), + }; let complete = chunk.len() == remaining; let ended = lease.is_fin() && !lease.has_remaining(); if complete || chunk.len() == frame.unsealed() { @@ -887,6 +906,36 @@ mod tests { )); } + /// A frame that arrives whole in a larger chunk, as in a datagram it shares with other + /// frames, is copied out of it when it is at most [`COPY_WHOLE`], so caching it does not + /// keep the rest of the receive allocation alive. A larger one is cached as the chunk. + #[test] + fn a_small_whole_frame_is_copied_out_of_its_chunk() { + let datagram: &'static [u8] = vec![7u8; 2 * COPY_WHOLE + 2].leak(); + for size in [1, COPY_WHOLE, COPY_WHOLE + 1] { + let (mut frame, mut payload, waiter, _wakes) = parked_frame(size); + let mut reader = queue(&[datagram], crate::lite::Version::Lite05); + let noop = &kio::Waiter::noop(); + assert!(matches!(reader.poll_read_frame(noop, &mut frame), Poll::Ready(Ok(())))); + let _group = frame.finish().unwrap(); + let Poll::Ready(Ok(Some(chunk))) = payload.poll_read_chunk(&waiter) else { + panic!("the committed frame has no payload"); + }; + assert_eq!(chunk, datagram[..size], "size={size}"); + assert_eq!( + chunk.as_ptr() == datagram.as_ptr(), + size > COPY_WHOLE, + "size={size}: copied {}", + size <= COPY_WHOLE + ); + assert_eq!( + leased(&mut reader), + datagram.len() - size, + "size={size}: the rest stays leased" + ); + } + } + /// A consumer parked on the group cannot run until this task yields, so a burst of /// chunks owes exactly one wake: at the boundary, from whoever wrote them. Every /// other exit hands the wake to the caller's `finish` or `abort`. diff --git a/rs/moq-net/tests/integration/zero_copy.rs b/rs/moq-net/tests/integration/zero_copy.rs index b1780917af4..b8888660444 100644 --- a/rs/moq-net/tests/integration/zero_copy.rs +++ b/rs/moq-net/tests/integration/zero_copy.rs @@ -2,7 +2,8 @@ //! A frame crosses an in-memory session as the same allocation it was written as: the //! publisher hands the payload to the transport, and the subscriber reads it back, -//! without a copy on either side. +//! without a copy on either side. Only a frame of at most 2 KiB is copied, so a small +//! frame never keeps a larger receive allocation alive. use crate::support; From de08552b6c47a39fd42a176971995fa24b34dc67 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:44:57 -0800 Subject: [PATCH 4/5] quest: finish cache-backing Co-Authored-By: Claude Opus 5.5 --- quest/m2/performance-survey/README.md | 1 - quest/m2/performance-survey/cache-backing.md | 33 -------------------- 2 files changed, 34 deletions(-) delete mode 100644 quest/m2/performance-survey/cache-backing.md diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index fea932a30a2..25f32f480a0 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -176,7 +176,6 @@ No duplicate quest or workflow edit belongs here. - [Task capacity after bursts](/quest/m2/performance-survey/task-capacity.md) - polling and memory after peak task occupancy subsides - [QUIC receive-state reuse](/quest/m2/performance-survey/quic-receive-reuse.md) - reassembly metadata retained by recycled completed streams - [Cache metadata](/quest/m2/performance-survey/cache-metadata.md) - bookkeeping capacity after cached groups expire -- [Cache backing amplification](/quest/m2/performance-survey/cache-backing.md) - small received frames pinning shared receive buffers - [Bandwidth grants](/quest/m2/performance-survey/bandwidth-grants.md) - repeated full allocation in Rust and JS - [Cached JS group joins](/quest/m2/performance-survey/js-cached-groups.md) - repeated sorting while replaying retained groups - [PCM batch processing](/quest/m2/performance-survey/audio-batches.md) - repeated tail movement during encoding and resampling diff --git a/quest/m2/performance-survey/cache-backing.md b/quest/m2/performance-survey/cache-backing.md deleted file mode 100644 index 8ef98808c7c..00000000000 --- a/quest/m2/performance-survey/cache-backing.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] Copy small received frames out of shared receive buffers - -## Goal - -A small frame a session receives whole is not cached as a slice of the -transport's receive allocation, so the live heap behind cached received frames -stays close to what `Pool::used()` bills. Large frames crossing the in-memory -transport still arrive without a copy. - -## Plan - -[`backing`](/rs/moq-net/benches/backing.rs) measured this in -[moq-dev/moq#7522](https://github.com/moq-dev/moq/pull/7522). Run it with -`BACKING_ALLOCS=1 cargo bench -p moq-net --features fuzz --bench backing`. -[`Reader::poll_read_frame`](/rs/moq-net/src/coding/reader.rs) caches a frame -that arrives in one transport chunk as that chunk. Under QUIC, that is a slice -of a datagram. `moq-tokio` copies datagrams into shared 64 KiB receive -allocations, and a `moq-uring` GRO completion is a single allocation as well. -With 64 KiB receive allocations, 4 Mbps video and 64 kbps audio retained 1.9x -the pool's bill with nothing evicted. With the video groups evicted and the -audio kept, it retained 33x. Copying such frames brought every case back to -the control (1.01-1.19x). The copy cost 880-1,120 user instructions per frame -copied, adding 7-16% to ingest. - -Pick the copy rule in moq-dev/moq#7522, then land it with the bench as its -evidence. The size threshold decides how far the copy reaches into -in-memory transport frames, which currently cross as the publisher's -allocation. - -The fMP4 importer is not affected: every sample is copied into its own frame, -so its retained heap beyond the bill is a constant ~0.3 MB of importer state. -Its passthrough fragments do keep up to 2x capacity slack, but -[moq-dev/moq#7232](https://github.com/moq-dev/moq/pull/7232) removes it. From 26f0edea76dc16306e27e512eb23721ef7e1e2ca Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 15:28:44 -0800 Subject: [PATCH 5/5] bench(moq-net): log every backing case's memory in timed runs The timed path printed memory only for the keep=all cases, so the nightly run never showed the eviction cases where the shared receive buffers amplified most. The first run of each timed case now prints the memory of its keep=all, keep=quarter, and keep=none cases. Running inside the bench closure follows Criterion's own filter, so a filtered or --exact run prints only the selected case's lines. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/backing.rs | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/rs/moq-net/benches/backing.rs b/rs/moq-net/benches/backing.rs index 52c2d35f6b0..fbde04be899 100644 --- a/rs/moq-net/benches/backing.rs +++ b/rs/moq-net/benches/backing.rs @@ -28,7 +28,9 @@ //! allocations and bytes allocated per frame received, instead of timing. The live heap //! counts each allocation once, however many frames slice it. An argument filters the cases //! by name. Timing covers only the cases that keep every group, since eviction adds a few -//! aborts, and prints the same line for each before timing it. +//! aborts. The first time a timed case runs, it prints the same line for itself and for the +//! two cases that evict its video groups, so a timed run such as the nightly one logs every +//! case's memory. The ratio is reported, not asserted: the reader's unit test pins the copy. //! //! Run with `cargo bench -p moq-net --features fuzz --bench backing`. @@ -337,15 +339,19 @@ fn cases() -> impl Iterator String { + format!("tracks={tracks}/video={kbps}k/keep={}/{}", keep.0, backing.name()) +} + /// The live heap, once the cache's deferred frees have run. fn live() -> isize { // The group log frees evicted entries through epoch-based reclamation. @@ -398,14 +404,21 @@ fn bench(c: &mut Criterion) { return; } - // Eviction is a handful of aborts, so only the cases keeping every group are timed, each - // with its memory printed beside it. + // Eviction is a handful of aborts, so only the cases keeping every group are timed. The + // first run of one, which Criterion's filter allows only for a selected case, prints the + // memory of every case sharing its media and backing, untimed. let mut group = c.benchmark_group("backing"); - for (name, tracks, kbps, backing, keep) in cases().filter(|(.., keep)| keep.0 == "all") { + for (id, tracks, kbps, backing, keep) in cases().filter(|(.., keep)| keep.0 == "all") { let media = Media::new(tracks, kbps); - eprintln!("backing/{name}: {}", measure(&media, backing, keep)); + let mut measured = false; group.throughput(Throughput::Bytes(media.payload as u64)); - group.bench_function(BenchmarkId::from_parameter(name), |b| { + group.bench_function(BenchmarkId::from_parameter(id), |b| { + if !std::mem::replace(&mut measured, true) { + for other in KEEP { + let case = name(tracks, kbps, other, backing); + eprintln!("backing/{case}: {}", measure(&media, backing, other)); + } + } b.iter_batched( || Cache::new(&media), |mut cache| {