Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m2/perf/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ revisions on the same host and worker count, and profiles with
- [QUIC lock contention](/quest/m2/perf/quic-lock-contention.md) - a measured verdict on the tokio driver's connection-wide mutex and per-packet endpoint-to-connection hop, with levers built only on evidence
- [Run to quiescence](/quest/m2/perf/uring-quiescence.md) - a received packet's reply is staged in the same turn, under a pass and train budget that keeps the fairness rule
- [Ingest batch](/quest/m2/perf/ingest-batch.md) - relay ingest pays one lock, wake, and clock read per burst of whole frames instead of per frame
- [Streaming frame read owners](/quest/m2/perf/frame-read-owners.md) - readers of a frame still arriving share one byte owner per publication instead of allocating one each per read
- [kio waiter costs](/quest/m2/perf/kio-waiter-costs.md) - long-lived waiters stop being rebuilt every poll, IETF fetch keeps one close watch per stream, and a short `kio::log` costs less
- [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux
- [io_uring buffer pool memory](/quest/m2/perf/uring-pool-memory.md) - measure reserved versus resident pool memory before any shrink policy
Expand Down
50 changes: 0 additions & 50 deletions quest/m2/perf/frame-read-owners.md

This file was deleted.

7 changes: 3 additions & 4 deletions quest/m2/quic-io-boundary.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,9 @@ including a stalled reader:
- moq-quic's lease `extend` copies out of datagram backing to bound what a
slice retains (deliberate), and `fill` builds a temporary `Vec` of chunks
per call.
- A partial MoQ frame copies into canonical frame storage, which
`Segment::new` zero-fills first
([streaming frame read owners](/quest/m2/perf/frame-read-owners.md) owns
that fill). Whole-frame `Bytes` writes skip this copy.
- A partial MoQ frame copies into canonical frame storage, left
uninitialized rather than zero-filled first. Whole-frame `Bytes` writes
skip this copy.

Decided 2026-10-08: `SENDMSG_ZC`
([#3201](/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md))
Expand Down
144 changes: 144 additions & 0 deletions rs/moq-net/benches/frame.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,20 @@
//! `frame_recv_large` sweeps frames up to the group's byte cap over fewer streams,
//! where a frame spans many storage segments.
//!
//! `frame_read` adds subscribers reading each frame after every chunk that arrives, as
//! a relay forwards a frame still streaming in, swept over publishers, readers, and
//! arrival chunks per frame. Readers keep every chunk until the frame ends, so the
//! chunks read before a `grow` buffer outgrew itself still hold the old one. A read
//! should cost nothing that grows with the readers: `FRAME_ALLOCS=1` prints the
//! allocations and bytes allocated per read instead of timing, past what the same
//! frames cost to receive with no readers.
//!
//! Run with `cargo bench -p moq-net --features fuzz --bench frame`.

use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;

use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use moq_net::fuzz::FrameRecv;

Expand All @@ -25,6 +37,54 @@ const LARGE_STREAMS: [usize; 2] = [1, 4];

const BUDGETS: [(&str, Option<usize>); 3] = [("upfront", Some(usize::MAX)), ("grow", Some(0)), ("budget", None)];

/// One buffer, or a chain of segments, for the frames read while they stream in.
const READ_SIZES: [usize; 2] = [16 * 1024, 2 * 1024 * 1024];
const READ_BUDGETS: [(&str, Option<usize>); 2] = [("upfront", Some(usize::MAX)), ("grow", Some(0))];
const PUBLISHERS: [usize; 2] = [1, 4];
const READERS: [usize; 3] = [1, 16, 128];
/// Chunks each frame arrives in, every one read by every reader.
const ARRIVALS: [usize; 2] = [4, 32];

struct Counter;

static COUNTING: AtomicBool = AtomicBool::new(false);
static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0);
static ALLOCATED: AtomicUsize = AtomicUsize::new(0);

#[global_allocator]
static ALLOCATOR: Counter = Counter;

impl Counter {
fn count(size: usize) {
if COUNTING.load(Ordering::Relaxed) {
ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
ALLOCATED.fetch_add(size, Ordering::Relaxed);
}
}
}

// Counting only. Every call forwards to the system allocator unchanged.
unsafe impl GlobalAlloc for Counter {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
Self::count(layout.size());
unsafe { System.alloc(layout) }
}

unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
Self::count(layout.size());
unsafe { System.alloc_zeroed(layout) }
}

unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
Self::count(new_size);
unsafe { System.realloc(ptr, layout, new_size) }
}

unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { System.dealloc(ptr, layout) }
}
}

fn sweep(c: &mut Criterion, name: &str, sizes: &[usize], streams: &[usize]) {
let mut group = c.benchmark_group(name);
for &size in sizes {
Expand All @@ -45,9 +105,93 @@ fn sweep(c: &mut Criterion, name: &str, sizes: &[usize], streams: &[usize]) {
group.finish();
}

/// One `frame_read` case: its id and the frames it receives.
struct ReadCase {
id: String,
publishers: usize,
size: usize,
chunk: usize,
budget: Option<usize>,
readers: usize,
}

impl ReadCase {
fn recv(&self, readers: usize) -> FrameRecv {
FrameRecv::new(self.publishers, self.size, self.chunk, self.budget).with_readers(readers)
}
}

fn read_cases() -> impl Iterator<Item = ReadCase> {
READ_SIZES.into_iter().flat_map(|size| {
READ_BUDGETS.into_iter().flat_map(move |(name, budget)| {
PUBLISHERS.into_iter().flat_map(move |publishers| {
READERS.into_iter().flat_map(move |readers| {
ARRIVALS.into_iter().map(move |arrivals| ReadCase {
id: format!("{name}/size={size}/publishers={publishers}/readers={readers}/arrivals={arrivals}"),
publishers,
size,
chunk: size / arrivals,
budget,
readers,
})
})
})
})
})
}

/// Allocations and bytes allocated while `recv` runs, and the chunks its readers read.
fn count(recv: FrameRecv) -> (usize, usize, usize) {
ALLOCATIONS.store(0, Ordering::Relaxed);
ALLOCATED.store(0, Ordering::Relaxed);
COUNTING.store(true, Ordering::Relaxed);
let reads = recv.run();
COUNTING.store(false, Ordering::Relaxed);
(
ALLOCATIONS.load(Ordering::Relaxed),
ALLOCATED.load(Ordering::Relaxed),
reads,
)
}

/// Print each `frame_read` case's allocations and bytes allocated per read, past what
/// receiving its frames costs alone, instead of timing. An argument filters the cases.
fn allocations() {
let filter = std::env::args().skip(1).find(|arg| !arg.starts_with('-'));
for case in read_cases().filter(|case| filter.as_ref().is_none_or(|filter| case.id.contains(filter))) {
let (base_allocations, base_bytes, _) = count(case.recv(0));
let (allocations, bytes, reads) = count(case.recv(case.readers));
let per_read = |total: usize, base: usize| total.saturating_sub(base) as f64 / reads as f64;
println!(
"frame_read/{}: {reads} reads, {:.2} allocations, {:.1} bytes allocated per read",
case.id,
per_read(allocations, base_allocations),
per_read(bytes, base_bytes),
);
}
}

fn bench_read(c: &mut Criterion) {
let mut group = c.benchmark_group("frame_read");
group.warm_up_time(Duration::from_secs(1));
group.measurement_time(Duration::from_secs(2));
for case in read_cases() {
group.throughput(Throughput::Elements((case.publishers * case.readers) as u64));
group.bench_function(BenchmarkId::from_parameter(&case.id), |b| {
b.iter_batched(|| case.recv(case.readers), FrameRecv::run, BatchSize::PerIteration)
});
}
group.finish();
}

fn bench(c: &mut Criterion) {
if std::env::var_os("FRAME_ALLOCS").is_some() {
allocations();
return;
}
sweep(c, "frame_recv", &SIZES, &STREAMS);
sweep(c, "frame_recv_large", &LARGE_SIZES, &LARGE_STREAMS);
bench_read(c);
}

criterion_group!(benches, bench);
Expand Down
121 changes: 121 additions & 0 deletions rs/moq-net/benches/group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
//! hands each reader the group through its own track subscription, as a relay does, so
//! every read that parks at the edge also judges the group's expiry.
//!
//! `group_stream_read` reads one frame while it arrives from readers spread over worker
//! threads, swept over arrival chunks, threads, and readers, so contention on the bytes
//! the readers share shows up as a slope across cores.
//!
//! `group_shed` writes one live group far past what it holds, swept over the frames it
//! holds (a small pool's debt, or the 32 MiB live tail) and the frames it sheds, so
//! shedding the front stays a flat cost per frame.
Expand Down Expand Up @@ -709,6 +713,122 @@ fn contended(
});
}

/// Bytes each round of [`bench_streaming`] adds to the frame in flight.
const STREAM_CHUNK: usize = 1024;

/// Chunks a streamed frame arrives in.
const STREAM_ARRIVALS: [usize; 2] = [4, 16];

/// Readers of the one streamed frame.
const STREAM_READERS: [usize; 2] = [16, 256];

/// Read one frame while it arrives from readers on several threads, as a relay forwards
/// a large frame to its subscribers.
///
/// Each round the driver writes one chunk of the frame in flight, and every reader reads
/// it with [`frame::Consumer::poll_read_chunk`] and parks again. Readers at the same
/// published length share one owner of its bytes, so contention on that owner shows up
/// as a slope across cores. A finished frame is followed by the next in the same group,
/// rolling to a new group every [`ROLL`] frames.
fn bench_streaming(c: &mut Criterion) {
let chunk = Bytes::from(vec![0u8; STREAM_CHUNK]);
let mut g = c.benchmark_group("group_stream_read");
g.warm_up_time(Duration::from_secs(1));
g.measurement_time(Duration::from_secs(3));
for arrivals in STREAM_ARRIVALS {
for threads in THREADS {
for readers in STREAM_READERS {
g.throughput(Throughput::Elements(readers as u64));
let id = BenchmarkId::new(format!("arrivals_{arrivals}/threads_{threads}"), readers);
g.bench_function(id, |b| streaming(b, &chunk, arrivals, threads, readers));
}
}
}
g.finish();
}

/// One [`bench_streaming`] case.
fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads: usize, readers: usize) {
let broadcast = broadcast::Producer::new(broadcast::Info::default());
let track = broadcast.create_track("bench", track::Info::default()).unwrap();
let info = frame::Info {
size: (arrivals * chunk.len()) as u64,
timestamp: None,
};
let group = track.append_group().unwrap();
// The group readers move to once theirs ends.
let next = Arc::new(Mutex::new(group.consume()));
let mut sets: Vec<Vec<(group::Consumer, Option<frame::Consumer>)>> = (0..threads).map(|_| Vec::new()).collect();
for i in 0..readers {
sets[i % threads].push((group.consume(), None));
}
let work = sets
.into_iter()
.map(|mut set| -> Work {
let next = next.clone();
let waiter = kio::Waiter::noop();
Box::new(move || {
for (group, frame) in &mut set {
let mut read = 0;
loop {
let reading = match frame {
Some(frame) => frame,
None => match group.poll_next_frame(&waiter) {
Poll::Ready(Ok(Some(next))) => frame.insert(next),
Poll::Ready(Ok(None)) => {
*group = next.lock().unwrap().clone();
continue;
}
Poll::Ready(Err(err)) => panic!("next frame failed: {err}"),
Poll::Pending => break,
},
};
match reading.poll_read_chunk(&waiter) {
Poll::Ready(Ok(Some(chunk))) => {
black_box(chunk);
read += 1;
}
Poll::Ready(Ok(None)) => *frame = None,
Poll::Ready(Err(err)) => panic!("read failed: {err}"),
Poll::Pending => break,
}
}
assert_eq!(read, 1, "every written chunk is ready");
}
})
})
.collect();
let crew = Crew::new(work);

let mut frames = 0;
let mut writing = Some(group.create_frame(info).unwrap());
b.iter_custom(|iters| {
let start = Instant::now();
for _ in 0..iters {
let mut frame = writing.take().unwrap();
frame.write(chunk.clone()).unwrap();
writing = Some(match frame.remaining() {
0 => {
let mut group = frame.finish().unwrap();
frames += 1;
if frames == ROLL {
let rolled = track.append_group().unwrap();
*next.lock().unwrap() = rolled.consume();
group.finish().unwrap();
frames = 0;
rolled.create_frame(info).unwrap()
} else {
group.create_frame(info).unwrap()
}
}
_ => frame,
});
crew.run();
}
start.elapsed()
});
}

/// A frame for the shedding benchmark: large enough that a group outgrows a small pool
/// in a few hundred frames.
const SHED_PAYLOAD: usize = 16 * 1024;
Expand Down Expand Up @@ -823,6 +943,7 @@ criterion_group!(
bench_relay_frames,
bench_inflate,
bench_contended,
bench_streaming,
bench_shed,
bench_paced
);
Expand Down
Loading
Loading