From 2ce6b11cafd342c1dece7e588cda62bd63affe09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:16:09 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m2/perf/frame-read-owners.md Quest-Claim: 819e5b01da838433 From 5fa268fcc9b6675f34888dcb370e59e6940c3065 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:54:14 -0700 Subject: [PATCH 2/4] perf(moq-net): readers of a streaming frame share its byte owners Readers slicing a frame still arriving shared nothing: every read boxed a fresh owner over the segment. The first read past each publication now builds one owner over every published byte, cached beside the segment, and later readers slice it. Growth drops the cached owner with the old segment, so it never outlives the slices cut from it. Segments are left uninitialized, since only their published prefix is read and every byte of it was written first. A frame chain is boxed so the cached owner adds 8 bytes to each frame's content, not 32. The frame bench sweeps publishers, readers, and arrival chunks, with FRAME_ALLOCS=1 printing allocations per read; the group bench reads a streaming frame from readers across threads. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/frame.rs | 144 +++++++++++++++++++ rs/moq-net/benches/group.rs | 121 ++++++++++++++++ rs/moq-net/src/fuzz.rs | 84 +++++++++-- rs/moq-net/src/model/frame.rs | 257 ++++++++++++++++++++++++---------- 4 files changed, 521 insertions(+), 85 deletions(-) diff --git a/rs/moq-net/benches/frame.rs b/rs/moq-net/benches/frame.rs index 3d8be4480f4..a9d9291423a 100644 --- a/rs/moq-net/benches/frame.rs +++ b/rs/moq-net/benches/frame.rs @@ -9,8 +9,20 @@ //! `frame_recv_large` sweeps frames up to the group's byte cap over fewer streams, //! where a frame spans many storage segments. //! +//! `frame_read` adds subscribers reading each frame after every chunk that arrives, as +//! a relay forwards a frame still streaming in, swept over publishers, readers, and +//! arrival chunks per frame. Readers keep every chunk until the frame ends, so the +//! chunks read before a `grow` buffer outgrew itself still hold the old one. A read +//! should cost nothing that grows with the readers: `FRAME_ALLOCS=1` prints the +//! allocations and bytes allocated per read instead of timing, past what the same +//! frames cost to receive with no readers. +//! //! Run with `cargo bench -p moq-net --features fuzz --bench frame`. +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; + use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use moq_net::fuzz::FrameRecv; @@ -25,6 +37,54 @@ const LARGE_STREAMS: [usize; 2] = [1, 4]; const BUDGETS: [(&str, Option); 3] = [("upfront", Some(usize::MAX)), ("grow", Some(0)), ("budget", None)]; +/// One buffer, or a chain of segments, for the frames read while they stream in. +const READ_SIZES: [usize; 2] = [16 * 1024, 2 * 1024 * 1024]; +const READ_BUDGETS: [(&str, Option); 2] = [("upfront", Some(usize::MAX)), ("grow", Some(0))]; +const PUBLISHERS: [usize; 2] = [1, 4]; +const READERS: [usize; 3] = [1, 16, 128]; +/// Chunks each frame arrives in, every one read by every reader. +const ARRIVALS: [usize; 2] = [4, 32]; + +struct Counter; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +static ALLOCATED: AtomicUsize = AtomicUsize::new(0); + +#[global_allocator] +static ALLOCATOR: Counter = Counter; + +impl Counter { + fn count(size: usize) { + if COUNTING.load(Ordering::Relaxed) { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED.fetch_add(size, Ordering::Relaxed); + } + } +} + +// Counting only. Every call forwards to the system allocator unchanged. +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + Self::count(layout.size()); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + Self::count(new_size); + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } +} + fn sweep(c: &mut Criterion, name: &str, sizes: &[usize], streams: &[usize]) { let mut group = c.benchmark_group(name); for &size in sizes { @@ -45,9 +105,93 @@ fn sweep(c: &mut Criterion, name: &str, sizes: &[usize], streams: &[usize]) { group.finish(); } +/// One `frame_read` case: its id and the frames it receives. +struct ReadCase { + id: String, + publishers: usize, + size: usize, + chunk: usize, + budget: Option, + readers: usize, +} + +impl ReadCase { + fn recv(&self, readers: usize) -> FrameRecv { + FrameRecv::new(self.publishers, self.size, self.chunk, self.budget).with_readers(readers) + } +} + +fn read_cases() -> impl Iterator { + READ_SIZES.into_iter().flat_map(|size| { + READ_BUDGETS.into_iter().flat_map(move |(name, budget)| { + PUBLISHERS.into_iter().flat_map(move |publishers| { + READERS.into_iter().flat_map(move |readers| { + ARRIVALS.into_iter().map(move |arrivals| ReadCase { + id: format!("{name}/size={size}/publishers={publishers}/readers={readers}/arrivals={arrivals}"), + publishers, + size, + chunk: size / arrivals, + budget, + readers, + }) + }) + }) + }) + }) +} + +/// Allocations and bytes allocated while `recv` runs, and the chunks its readers read. +fn count(recv: FrameRecv) -> (usize, usize, usize) { + ALLOCATIONS.store(0, Ordering::Relaxed); + ALLOCATED.store(0, Ordering::Relaxed); + COUNTING.store(true, Ordering::Relaxed); + let reads = recv.run(); + COUNTING.store(false, Ordering::Relaxed); + ( + ALLOCATIONS.load(Ordering::Relaxed), + ALLOCATED.load(Ordering::Relaxed), + reads, + ) +} + +/// Print each `frame_read` case's allocations and bytes allocated per read, past what +/// receiving its frames costs alone, instead of timing. An argument filters the cases. +fn allocations() { + let filter = std::env::args().skip(1).find(|arg| !arg.starts_with('-')); + for case in read_cases().filter(|case| filter.as_ref().is_none_or(|filter| case.id.contains(filter))) { + let (base_allocations, base_bytes, _) = count(case.recv(0)); + let (allocations, bytes, reads) = count(case.recv(case.readers)); + let per_read = |total: usize, base: usize| total.saturating_sub(base) as f64 / reads as f64; + println!( + "frame_read/{}: {reads} reads, {:.2} allocations, {:.1} bytes allocated per read", + case.id, + per_read(allocations, base_allocations), + per_read(bytes, base_bytes), + ); + } +} + +fn bench_read(c: &mut Criterion) { + let mut group = c.benchmark_group("frame_read"); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(2)); + for case in read_cases() { + group.throughput(Throughput::Elements((case.publishers * case.readers) as u64)); + group.bench_function(BenchmarkId::from_parameter(&case.id), |b| { + b.iter_batched(|| case.recv(case.readers), FrameRecv::run, BatchSize::PerIteration) + }); + } + group.finish(); +} + fn bench(c: &mut Criterion) { + if std::env::var_os("FRAME_ALLOCS").is_some() { + allocations(); + return; + } sweep(c, "frame_recv", &SIZES, &STREAMS); sweep(c, "frame_recv_large", &LARGE_SIZES, &LARGE_STREAMS); + bench_read(c); } criterion_group!(benches, bench); diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 6b838ff303c..8919f1779ce 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -18,6 +18,10 @@ //! hands each reader the group through its own track subscription, as a relay does, so //! every read that parks at the edge also judges the group's expiry. //! +//! `group_stream_read` reads one frame while it arrives from readers spread over worker +//! threads, swept over arrival chunks, threads, and readers, so contention on the bytes +//! the readers share shows up as a slope across cores. +//! //! `group_shed` writes one live group far past what it holds, swept over the frames it //! holds (a small pool's debt, or the 32 MiB live tail) and the frames it sheds, so //! shedding the front stays a flat cost per frame. @@ -705,6 +709,122 @@ fn contended( }); } +/// Bytes each round of [`bench_streaming`] adds to the frame in flight. +const STREAM_CHUNK: usize = 1024; + +/// Chunks a streamed frame arrives in. +const STREAM_ARRIVALS: [usize; 2] = [4, 16]; + +/// Readers of the one streamed frame. +const STREAM_READERS: [usize; 2] = [16, 256]; + +/// Read one frame while it arrives from readers on several threads, as a relay forwards +/// a large frame to its subscribers. +/// +/// Each round the driver writes one chunk of the frame in flight, and every reader reads +/// it with [`frame::Consumer::poll_read_chunk`] and parks again. Readers at the same +/// published length share one owner of its bytes, so contention on that owner shows up +/// as a slope across cores. A finished frame is followed by the next in the same group, +/// rolling to a new group every [`ROLL`] frames. +fn bench_streaming(c: &mut Criterion) { + let chunk = Bytes::from(vec![0u8; STREAM_CHUNK]); + let mut g = c.benchmark_group("group_stream_read"); + g.warm_up_time(Duration::from_secs(1)); + g.measurement_time(Duration::from_secs(3)); + for arrivals in STREAM_ARRIVALS { + for threads in THREADS { + for readers in STREAM_READERS { + g.throughput(Throughput::Elements(readers as u64)); + let id = BenchmarkId::new(format!("arrivals_{arrivals}/threads_{threads}"), readers); + g.bench_function(id, |b| streaming(b, &chunk, arrivals, threads, readers)); + } + } + } + g.finish(); +} + +/// One [`bench_streaming`] case. +fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads: usize, readers: usize) { + let broadcast = broadcast::Producer::new(broadcast::Info::default()); + let track = broadcast.create_track("bench", track::Info::default()).unwrap(); + let info = frame::Info { + size: (arrivals * chunk.len()) as u64, + timestamp: None, + }; + let group = track.append_group().unwrap(); + // The group readers move to once theirs ends. + let next = Arc::new(Mutex::new(group.consume())); + let mut sets: Vec)>> = (0..threads).map(|_| Vec::new()).collect(); + for i in 0..readers { + sets[i % threads].push((group.consume(), None)); + } + let work = sets + .into_iter() + .map(|mut set| -> Work { + let next = next.clone(); + let waiter = kio::Waiter::noop(); + Box::new(move || { + for (group, frame) in &mut set { + let mut read = 0; + loop { + let reading = match frame { + Some(frame) => frame, + None => match group.poll_next_frame(&waiter) { + Poll::Ready(Ok(Some(next))) => frame.insert(next), + Poll::Ready(Ok(None)) => { + *group = next.lock().unwrap().clone(); + continue; + } + Poll::Ready(Err(err)) => panic!("next frame failed: {err}"), + Poll::Pending => break, + }, + }; + match reading.poll_read_chunk(&waiter) { + Poll::Ready(Ok(Some(chunk))) => { + black_box(chunk); + read += 1; + } + Poll::Ready(Ok(None)) => *frame = None, + Poll::Ready(Err(err)) => panic!("read failed: {err}"), + Poll::Pending => break, + } + } + assert_eq!(read, 1, "every written chunk is ready"); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut frames = 0; + let mut writing = Some(group.create_frame(info).unwrap()); + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let mut frame = writing.take().unwrap(); + frame.write(chunk.clone()).unwrap(); + writing = Some(match frame.remaining() { + 0 => { + let mut group = frame.finish().unwrap(); + frames += 1; + if frames == ROLL { + let rolled = track.append_group().unwrap(); + *next.lock().unwrap() = rolled.consume(); + group.finish().unwrap(); + frames = 0; + rolled.create_frame(info).unwrap() + } else { + group.create_frame(info).unwrap() + } + } + _ => frame, + }); + crew.run(); + } + start.elapsed() + }); +} + /// A frame for the shedding benchmark: large enough that a group outgrows a small pool /// in a few hundred frames. const SHED_PAYLOAD: usize = 16 * 1024; @@ -768,6 +888,7 @@ criterion_group!( bench_relay_frames, bench_inflate, bench_contended, + bench_streaming, bench_shed ); criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 052d2d938ce..dd4c4b6d930 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1552,12 +1552,49 @@ type FrameReader = coding::Reader, lite /// every declared size when the first byte lands and `Some(0)` grows every buffer with /// the bytes received. pub struct FrameRecv { - streams: Vec>, + streams: Vec>, + chunk: usize, // Keeps the groups' track alive. _track: track::Producer, _broadcast: broadcast::Producer, } +struct Stream { + reader: FrameReader, + frame: frame::Producer, + readers: Vec, +} + +/// A subscriber reading a stream's frame while it arrives, as a relay forwards it. +/// +/// It keeps every chunk until the run ends, as a send queue would, so the chunks it read +/// before the frame's buffer grew still hold the buffer they were cut from. +struct FrameRead { + group: group::Consumer, + frame: Option, + held: Vec, +} + +impl FrameRead { + /// Read every chunk published so far, returning how many. + fn read(&mut self, waiter: &kio::Waiter) -> usize { + let frame = match &mut self.frame { + Some(frame) => frame, + None => match self.group.poll_next_frame(waiter) { + std::task::Poll::Ready(res) => self.frame.insert(res.unwrap().expect("the frame is in flight")), + std::task::Poll::Pending => return 0, + }, + }; + let mut reads = 0; + while let std::task::Poll::Ready(chunk) = frame.poll_read_chunk(waiter) { + let Some(chunk) = chunk.unwrap() else { break }; + self.held.push(chunk); + reads += 1; + } + reads + } +} + impl FrameRecv { /// `streams` frames of `size` bytes, delivered `chunk` bytes at a time. /// `None` gives the session's default budget. @@ -1588,36 +1625,63 @@ impl FrameRecv { chunk, ready: true, }; - Some(( - coding::Reader::new(crate::transport::poll::Buffered::new(stream), lite::Version::Lite05), + Some(Stream { + reader: coding::Reader::new(crate::transport::poll::Buffered::new(stream), lite::Version::Lite05), frame, - )) + readers: Vec::new(), + }) }) .collect(); Self { streams, + chunk, _track: track, _broadcast: broadcast, } } - /// Receive every frame, round-robin a chunk at a time, then drop them. - pub fn run(mut self) { + /// Add `readers` subscribers to each stream, reading its frame after every chunk it + /// receives. + pub fn with_readers(mut self, readers: usize) -> Self { + for stream in self.streams.iter_mut().flatten() { + let size = stream.frame.size as usize; + // A read ends at the bytes published or its segment's end, whichever is first. + let reads = size.div_ceil(self.chunk) + size.div_ceil(frame::SEGMENT); + stream.readers = (0..readers) + .map(|_| FrameRead { + group: stream.frame.group().consume(), + frame: None, + held: Vec::with_capacity(reads), + }) + .collect(); + } + self + } + + /// Receive every frame, round-robin a chunk at a time, then drop them. Returns the + /// chunks the readers read. + pub fn run(mut self) -> usize { let waiter = &kio::Waiter::noop(); let mut open = self.streams.len(); + let mut reads = 0; while open > 0 { for slot in &mut self.streams { - let Some((reader, frame)) = slot else { continue }; - if let std::task::Poll::Ready(res) = reader.poll_read_frame(waiter, frame) { + let Some(stream) = slot else { continue }; + let received = stream.reader.poll_read_frame(waiter, &mut stream.frame); + for reader in &mut stream.readers { + reads += reader.read(waiter); + } + if let std::task::Poll::Ready(res) = received { res.unwrap(); - let (_, frame) = slot.take().unwrap(); - let mut group = frame.finish().unwrap(); + let stream = slot.take().unwrap(); + let mut group = stream.frame.finish().unwrap(); group.finish().unwrap(); open -= 1; } } } + reads } } diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 2b78458e3dd..7f939daf608 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -5,6 +5,7 @@ //! through [`Producer`], which borrows its parent [`group::Producer`] exclusively so //! the borrow checker enforces that only one frame is open at a time. A [`Consumer`] //! reads one frame, sharing the group's channel rather than a per-frame one. +use std::mem::MaybeUninit; use std::ops::Range; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Weak}; @@ -621,29 +622,37 @@ impl FrameContent { if start >= end { return Bytes::new(); } - let (segment, base) = match self.storage.get() { + // Loaded only to build a new owner, under the storage's lock, so every byte it + // covers is already in the segment that lock guards. + let published = || self.written.load(Ordering::Acquire); + match self.storage.get() { Some(FrameStorage::Shared(payload)) => { let chunk = payload.chunk_at(start); - return chunk.slice(..chunk.len().min(end - start)); + chunk.slice(..chunk.len().min(end - start)) + } + Some(FrameStorage::Fixed { segment, shared }) => { + let mut shared = shared.lock().unwrap_or_else(PoisonError::into_inner); + share(&mut shared, segment, start..end, published) + } + Some(FrameStorage::Growing(open)) => { + let mut open = open.lock().unwrap_or_else(PoisonError::into_inner); + let Open { segment, shared } = &mut *open; + share(shared, segment, start..end, published) } - Some(FrameStorage::Fixed(segment)) => (segment.clone(), 0), - Some(FrameStorage::Growing(current)) => (current.lock().unwrap_or_else(PoisonError::into_inner).clone(), 0), Some(FrameStorage::Chain(segments)) => { - let segments = segments.lock().unwrap_or_else(PoisonError::into_inner); + let mut segments = segments.lock().unwrap_or_else(PoisonError::into_inner); let index = start / SEGMENT; let base = index * SEGMENT; + let range = start - base..end.min(base + SEGMENT) - base; if let Some(sealed) = segments.sealed.get(index) { - return sealed.slice(start - base..end.min(base + SEGMENT) - base); + return sealed.slice(range); } - let open = segments.open.clone().expect("written bytes are sealed or open"); - (open, base) + let Open { segment, shared } = segments.open.as_mut().expect("written bytes are sealed or open"); + share(shared, segment, range, || published().min(base + SEGMENT) - base) } // Nothing written, so the range is empty. - None => return Bytes::new(), - }; - let len = end.min(base + SEGMENT) - base; - debug_assert!(len <= segment.capacity); - Bytes::from_owner(Filled { segment, len }).slice(start - base..) + None => Bytes::new(), + } } } @@ -690,12 +699,77 @@ impl Drop for Admission { enum FrameStorage { Shared(Payload), - Fixed(Arc), - // Replaced by a larger copy whenever a write outgrows it. A reader clones the current - // segment under the lock, so a slice it already handed out keeps the old one alive. - Growing(Mutex>), - // A frame larger than one segment. - Chain(Mutex), + // Allocated at the frame's size, so a write never takes the lock; only readers share + // the owner through it. + Fixed { + segment: Arc, + shared: Mutex>, + }, + // Replaced by a larger copy whenever a write outgrows it. + Growing(Mutex), + // A frame larger than one segment. Boxed: it is rare, and every frame's content + // otherwise pays for its size. + Chain(Box>), +} + +/// A segment that is still being written, and may still grow. +struct Open { + segment: Arc, + // The readers' owner of its published bytes; see [`share`]. + shared: Option, +} + +impl Open { + fn new(capacity: usize) -> Self { + Self { + segment: Arc::new(Segment::new(capacity)), + shared: None, + } + } + + /// Replace the segment with a larger copy of its first `filled` bytes if it holds fewer + /// than `needed` of the `len` it grows to. + /// + /// Doubling keeps the copies linear in `len`, and the cap makes the last copy exactly + /// `len`, so it seals without another. The readers' owner goes with the old segment, + /// which then lives only as long as the slices already cut from it. + fn reserve(&mut self, filled: usize, needed: usize, len: usize) { + if self.segment.capacity >= needed { + return; + } + let next = Segment::new(needed.max(self.segment.capacity * 2).min(len)); + let filled = Filled { + segment: self.segment.clone(), + len: filled, + }; + // Safety: `next` is new, so nothing reads it yet. + unsafe { next.write(0, filled.as_ref()) }; + self.segment = Arc::new(next); + self.shared = None; + } +} + +/// Slice `range` of `segment`, from the owner in `shared` if it covers the range, or else +/// from a new one over the segment's first `published()` bytes that replaces it. +/// +/// Readers reading up to the same published length share one owner, so reading a frame +/// still arriving allocates once per publication rather than once per reader per read. An +/// owner covers only published bytes, since the writer is still filling the rest. +fn share( + shared: &mut Option, + segment: &Arc, + range: Range, + published: impl FnOnce() -> usize, +) -> Bytes { + if shared.as_ref().is_none_or(|owner| owner.len() < range.end) { + let len = published(); + debug_assert!(range.end <= len && len <= segment.capacity); + *shared = Some(Bytes::from_owner(Filled { + segment: segment.clone(), + len, + })); + } + shared.as_ref().expect("covers the range").slice(range) } /// A frame larger than one [`SEGMENT`], as it streams in. @@ -705,7 +779,7 @@ struct Segments { sealed: Vec, // The segment being written, allocated by its first byte and grown like // [`FrameStorage::Growing`]. - open: Option>, + open: Option, } impl Segments { @@ -716,14 +790,12 @@ impl Segments { let len = (size - base).min(SEGMENT); let within = offset - base; let take = src.len().min(len - within); - let open = self - .open - .get_or_insert_with(|| Arc::new(Segment::new(if grow { 0 } else { len }))); - Segment::reserve(open, within, within + take, len); + let open = self.open.get_or_insert_with(|| Open::new(if grow { 0 } else { len })); + open.reserve(within, within + take, len); // Safety: sole writer, within the segment's length, past every byte handed out. - unsafe { open.write(within, &src[..take]) }; + unsafe { open.segment.write(within, &src[..take]) }; if within + take == len { - let segment = self.open.take().expect("written above"); + let Open { segment, .. } = self.open.take().expect("written above"); self.sealed.push(Bytes::from_owner(Filled { segment, len })); } offset += take; @@ -732,7 +804,11 @@ impl Segments { } } -/// An owned, zero-initialized heap allocation that chunked writes are copied into. +/// An owned heap allocation that chunked writes are copied into. +/// +/// Left uninitialized, since only its published prefix is ever read, and every byte of +/// that was written before it was published. Reading past it would already race the +/// writer filling those bytes. struct Segment { data: *mut u8, capacity: usize, @@ -745,10 +821,10 @@ unsafe impl Sync for Segment {} impl Drop for Segment { fn drop(&mut self) { - // Safety: data was obtained from `Box::into_raw` of a `Box<[u8]>` of length - // `capacity` and is not aliased at drop (Arc refcount hit 0). + // Safety: data was obtained from `Box::into_raw` of a `Box<[MaybeUninit]>` of + // length `capacity` and is not aliased at drop (Arc refcount hit 0). unsafe { - let slice = std::ptr::slice_from_raw_parts_mut(self.data, self.capacity); + let slice = std::ptr::slice_from_raw_parts_mut(self.data.cast::>(), self.capacity); drop(Box::from_raw(slice)); } } @@ -756,9 +832,9 @@ impl Drop for Segment { impl Segment { fn new(capacity: usize) -> Self { - let boxed: Box<[u8]> = vec![0u8; capacity].into_boxed_slice(); + let boxed = Box::<[u8]>::new_uninit_slice(capacity); let capacity = boxed.len(); - let data = Box::into_raw(boxed) as *mut u8; + let data = Box::into_raw(boxed).cast::(); Self { data, capacity } } @@ -770,25 +846,6 @@ impl Segment { debug_assert!(offset + src.len() <= self.capacity); unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), self.data.add(offset), src.len()) }; } - - /// Replace `current` with a larger copy of its first `filled` bytes if it holds fewer - /// than `needed` of the `len` it grows to. - /// - /// Doubling keeps the copies linear in `len`, and the cap makes the last copy exactly - /// `len`, so it seals without another. - fn reserve(current: &mut Arc, filled: usize, needed: usize, len: usize) { - if current.capacity >= needed { - return; - } - let next = Segment::new(needed.max(current.capacity * 2).min(len)); - let filled = Filled { - segment: current.clone(), - len: filled, - }; - // Safety: `next` is new, so nothing reads it yet. - unsafe { next.write(0, filled.as_ref()) }; - *current = Arc::new(next); - } } /// The first `len` bytes of a [`Segment`], all written, so it can back a @@ -1013,9 +1070,12 @@ impl FrameBuf { let content = &self.content; let grow = content.grow.load(Ordering::Relaxed); let storage = content.storage.get_or_init(|| match grow { - _ if size > SEGMENT => FrameStorage::Chain(Mutex::default()), - true => FrameStorage::Growing(Mutex::new(Arc::new(Segment::new(0)))), - false => FrameStorage::Fixed(Arc::new(Segment::new(size))), + _ if size > SEGMENT => FrameStorage::Chain(Box::default()), + true => FrameStorage::Growing(Mutex::new(Open::new(0))), + false => FrameStorage::Fixed { + segment: Arc::new(Segment::new(size)), + shared: Mutex::default(), + }, }); // Safety (every write below): the publication mutex makes this the sole writer; // the caller checked the remaining size, and consumers only read `[..written]`. @@ -1023,11 +1083,11 @@ impl FrameBuf { // Only reachable if the frame is already complete via shared storage, which // `Producer::write` rejects for a non-empty chunk. Nothing to copy. FrameStorage::Shared(_) => return, - FrameStorage::Fixed(segment) => unsafe { segment.write(prev, src) }, - FrameStorage::Growing(current) => { - let mut current = current.lock().unwrap_or_else(PoisonError::into_inner); - Segment::reserve(&mut current, prev, prev + src.len(), size); - unsafe { current.write(prev, src) }; + FrameStorage::Fixed { segment, .. } => unsafe { segment.write(prev, src) }, + FrameStorage::Growing(open) => { + let mut open = open.lock().unwrap_or_else(PoisonError::into_inner); + open.reserve(prev, prev + src.len(), size); + unsafe { open.segment.write(prev, src) }; } FrameStorage::Chain(segments) => { let mut segments = segments.lock().unwrap_or_else(PoisonError::into_inner); @@ -1059,12 +1119,12 @@ impl FrameBuf { pub(super) fn allocated(&self) -> usize { match self.content.storage.get() { Some(FrameStorage::Shared(payload)) => payload.len(), - Some(FrameStorage::Fixed(segment)) => segment.capacity, - Some(FrameStorage::Growing(current)) => current.lock().unwrap_or_else(PoisonError::into_inner).capacity, + Some(FrameStorage::Fixed { segment, .. }) => segment.capacity, + Some(FrameStorage::Growing(open)) => open.lock().unwrap_or_else(PoisonError::into_inner).segment.capacity, Some(FrameStorage::Chain(segments)) => { let segments = segments.lock().unwrap_or_else(PoisonError::into_inner); let sealed: usize = segments.sealed.iter().map(Bytes::len).sum(); - sealed + segments.open.as_ref().map_or(0, |open| open.capacity) + sealed + segments.open.as_ref().map_or(0, |open| open.segment.capacity) } None => 0, } @@ -1490,21 +1550,21 @@ impl Consumer { if self.expired { return Poll::Ready(Err(Error::Old)); } - let buf = match &self.source { - Source::Complete(payload) => { - if self.read_idx >= payload.len() { - return Poll::Ready(Ok(None)); - } - let out = payload.chunk_at(self.read_idx); - self.read_idx += out.len(); - self.stats.bytes(out.len() as u64); - return Poll::Ready(Ok(Some(out))); - } - Source::Partial(buf) => buf.clone(), - }; - let size = self.info.size as usize; loop { + // Borrowed rather than cloned: every reader of the frame shares its count. + let buf = match &self.source { + Source::Complete(payload) => { + if self.read_idx >= payload.len() { + return Poll::Ready(Ok(None)); + } + let out = payload.chunk_at(self.read_idx); + self.read_idx += out.len(); + self.stats.bytes(out.len() as u64); + return Poll::Ready(Ok(Some(out))); + } + Source::Partial(buf) => buf, + }; let written = buf.written(Ordering::Acquire); if written > self.read_idx { let out = buf.slice(self.read_idx, written); @@ -1520,6 +1580,9 @@ impl Consumer { if self.poll_expired(waiter) { return Poll::Ready(Err(Error::Old)); } + let Source::Partial(buf) = &self.source else { + unreachable!("only a partial frame waits for its bytes"); + }; let read_idx = self.read_idx; // Park on the group's channel; the producer notifies it on each write and // on abort. Re-check the atomic on wake. @@ -1952,6 +2015,50 @@ mod tests { assert_eq!(buf.freeze().contiguous(0), payload); } + /// The segment a buffer is writing into, wherever its storage keeps it. + fn open_segment(buf: &FrameBuf) -> Weak { + let segment = match buf.content.storage.get() { + Some(FrameStorage::Fixed { segment, .. }) => segment.clone(), + Some(FrameStorage::Growing(open)) => open.lock().unwrap().segment.clone(), + Some(FrameStorage::Chain(segments)) => segments.lock().unwrap().open.as_ref().unwrap().segment.clone(), + _ => panic!("no segment is being written"), + }; + Arc::downgrade(&segment) + } + + /// Readers reading up to the same published length share one owner of its segment, so + /// a read allocates per publication rather than per reader. Growth drops that owner, + /// so the old segment lives only as long as the slices already cut from it. + #[test] + fn readers_share_one_owner_per_publication() { + for (size, grow) in [(16, false), (16, true), (SEGMENT + 16, false), (SEGMENT + 16, true)] { + let case = format!("size={size} grow={grow}"); + let buf = FrameBuf::with_growth(size, grow); + buf.append(b"abcd").unwrap(); + let segment = open_segment(&buf); + let first: Vec = (0..8).map(|_| buf.slice(0, 4)).collect(); + assert!(first.iter().all(|slice| slice == &b"abcd"[..]), "{case}"); + // A reader behind the published length shares the owner too. + assert_eq!(buf.slice(1, 3), &b"bc"[..], "{case}"); + // The storage's reference and the readers' one owner. + assert_eq!(segment.strong_count(), 2, "{case}"); + + buf.append(b"efgh").unwrap(); + let second: Vec = (0..8).map(|_| buf.slice(4, 8)).collect(); + assert!(second.iter().all(|slice| slice == &b"efgh"[..]), "{case}"); + if grow { + assert!(!open_segment(&buf).ptr_eq(&segment), "{case}: outgrew its segment"); + // Only the owner the first readers hold. + assert_eq!(segment.strong_count(), 1, "{case}"); + drop(first); + assert_eq!(segment.strong_count(), 0, "{case}"); + } else { + // The storage's, the first readers' owner, and the second readers' owner. + assert_eq!(segment.strong_count(), 3, "{case}"); + } + } + } + /// A frame larger than a segment allocates only the segment being written, whatever it /// declared, and seals each at the same fixed boundary. Reads never cross one, and a /// whole read copies the chain into one buffer. From 750853f4075e4c38cad8a3c87f49c536022e0c91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:59:45 -0700 Subject: [PATCH 3/4] quest: finish streaming frame read owners Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/README.md | 1 - quest/m2/perf/frame-read-owners.md | 50 ------------------------------ quest/m2/quic-io-boundary.md | 7 ++--- 3 files changed, 3 insertions(+), 55 deletions(-) delete mode 100644 quest/m2/perf/frame-read-owners.md diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 5086788bc2a..57f2012b74a 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -86,7 +86,6 @@ revisions on the same host and worker count, and profiles with - [Borrow-free QUIC state](/quest/m2/perf/quic-borrow-free.md) - a prototyped report on sharing connection state with stream handles without `RefCell` or `Mutex`: context passing vs driver-owned state with per-stream channels - [Run to quiescence](/quest/m2/perf/uring-quiescence.md) - a received packet's reply is staged in the same turn, under a pass and train budget that keeps the fairness rule - [Ingest batch](/quest/m2/perf/ingest-batch.md) - relay ingest pays one lock, wake, and clock read per burst of whole frames instead of per frame -- [Streaming frame read owners](/quest/m2/perf/frame-read-owners.md) - readers of a frame still arriving share one byte owner per publication instead of allocating one each per read - [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux - [Profile the io_uring clock read](/quest/m2/perf/uring-clock-profile.md) - a `perf` profile shows whether #5926's per-turn clock read cut the io_uring relay's `[vdso]` share and CPU - [io_uring buffer pool memory](/quest/m2/perf/uring-pool-memory.md) - measure reserved versus resident pool memory before any shrink policy diff --git a/quest/m2/perf/frame-read-owners.md b/quest/m2/perf/frame-read-owners.md deleted file mode 100644 index 923959a79f2..00000000000 --- a/quest/m2/perf/frame-read-owners.md +++ /dev/null @@ -1,50 +0,0 @@ -# [M] Readers of a streaming frame share its byte owners - -## Goal - -Reading a frame that is still arriving allocates per publication of new -bytes, not per reader per read, so a relay fanning a large frame out to many -subscribers pays no allocation that grows with its subscriber count. - -## Plan - -`FrameContent::slice` (`rs/moq-net/src/model/frame.rs`) wraps the `Fixed`, -`Growing`, and open `Chain` segment in a fresh `Bytes::from_owner(Filled { .. })` -on every call, and `from_owner` boxes its owner. Complete frames are already -free: a frozen frame hands out clones of one `Payload`. - -Measured in the 2026-10-27 audit on main -[`651c30fee9`](https://github.com/moq-dev/moq/commit/651c30fee9d88fd56f4079433dd3e1f98d77f21c) -with a counting allocator: a 16 KiB frame arriving in sixteen 1 KiB chunks, -read by 128 readers after each chunk, cost exactly one 24-byte allocation per -reader per chunk (2,048 allocations, 48 KiB). Four publishers × 128 readers -cost 8,192. The same reads of complete frames allocated nothing. - -Direction: readers that slice at the same written length share one owner, -for example the last `Bytes` built for the current watermark, cached beside -the storage and replaced when `written` advances. Constraints: - -- An owner must never cover bytes past `written`: the writer is still - filling them, so a shared slice over them is a data race, not just stale - data. -- A `Growing` segment is replaced when it outgrows itself. A cached owner - must not pin a superseded segment past the readers that hold slices of it. -- Sealed `Chain` segments already share one `Bytes` each; keep that. - -`Segment::new` zero-fills its storage (`vec![0u8; capacity]`) before the -copy from the wire overwrites it. It's the same storage, so measure that fill -here too. Skipping it needs `MaybeUninit` storage and proof that every exported -slice is initialized, including overlapping writes and growth. Land it only if -the profile shows the fill. - -Extend the frame and group benches with publishers × readers × arrival chunks, -including readers that hold old chunks across growth, and schedule them -nightly. Report allocations and instructions per read, with the counting -allocator from `rs/moq-net/benches/memory.rs`. - -Public API: none expected. Wire: none. - -## Related - -- [Ingest batch](/quest/m2/perf/ingest-batch.md) - the write side of the same frames -- [Relay allocations](/quest/m2/perf/relay-allocs.md) - the relay-wide allocation profile this site belongs to diff --git a/quest/m2/quic-io-boundary.md b/quest/m2/quic-io-boundary.md index 87de27fc0b2..1afb9f14522 100644 --- a/quest/m2/quic-io-boundary.md +++ b/quest/m2/quic-io-boundary.md @@ -44,10 +44,9 @@ including a stalled reader: - moq-quic's lease `extend` copies out of datagram backing to bound what a slice retains (deliberate), and `fill` builds a temporary `Vec` of chunks per call. -- A partial MoQ frame copies into canonical frame storage, which - `Segment::new` zero-fills first - ([streaming frame read owners](/quest/m2/perf/frame-read-owners.md) owns - that fill). Whole-frame `Bytes` writes skip this copy. +- A partial MoQ frame copies into canonical frame storage, left + uninitialized rather than zero-filled first. Whole-frame `Bytes` writes + skip this copy. Decided 2026-10-08: `SENDMSG_ZC` ([#3201](/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md)) From 18588e5dd301bb17034d85c73ed318380dad13eb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 09:28:57 -0800 Subject: [PATCH 4/4] docs(moq-net): say what makes a fixed segment's owner bytes initialized Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/frame.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 7f939daf608..9a72ea22d2d 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -622,8 +622,11 @@ impl FrameContent { if start >= end { return Bytes::new(); } - // Loaded only to build a new owner, under the storage's lock, so every byte it - // covers is already in the segment that lock guards. + // Loaded only to build a new owner. The Acquire pairs with `store_written`'s + // Release, so every byte it covers is initialized. `Growing`/`Chain` also load it + // under the lock that holds their segment steady, so those bytes are in the + // segment being sliced; a `Fixed` segment never moves, and its writers never take + // the `shared` lock. let published = || self.written.load(Ordering::Acquire); match self.storage.get() { Some(FrameStorage::Shared(payload)) => {