From 4d7bf1a1eea17a67943885c4a31b60cbc299093e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 07:00:51 -0700 Subject: [PATCH 01/16] chore(quest): claim quest/m1/unbounded-groups/fastest-reader.md Quest-Claim: c2e55399ea0d5d06 From 44a637e91ae0023a06ac3bc207de5d77e007bf43 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 07:10:52 -0700 Subject: [PATCH 02/16] feat(net): pace relay ingest by the group's fastest reader A session's ingest stops reading a group's stream once it is more than 16 MB (half the 32 MB tail) past the first frame the group's fastest reader has not taken, so stream flow control pushes back to the publisher. Readers report the frames they take; each log entry records its cumulative byte end, so the writer measures bytes, lock-free. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 13 +- rs/moq-net/src/model/frame.rs | 8 + rs/moq-net/src/model/group.rs | 103 ++++++++- rs/moq-net/src/model/mod.rs | 2 + rs/moq-net/src/model/pace.rs | 215 ++++++++++++++++++ .../tests/integration/fastest_reader.rs | 144 ++++++++++++ rs/moq-net/tests/integration/main.rs | 1 + 7 files changed, 480 insertions(+), 6 deletions(-) create mode 100644 rs/moq-net/src/model/pace.rs create mode 100644 rs/moq-net/tests/integration/fastest_reader.rs diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index ac38145f937..126742310ea 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -359,7 +359,9 @@ impl Reader { } } - /// Fill a frame's payload from the stream, returning `Pending` once it would block. + /// Fill a frame's payload from the stream, returning `Pending` once it would block or + /// the group's writer is paced by its fastest reader (see + /// [`crate::frame::Producer::poll_ready`]). /// /// The payload reaches the frame as it arrives, but keeps its receive credit until the /// segment it lands in seals, so an incomplete frame holds at most one segment of the @@ -386,7 +388,14 @@ impl Reader { let mut owed = 0; let result = loop { - let mut lease = match self.poll_lease(waiter, frame.remaining().min(1)) { + // A writer too far ahead of the group's fastest reader leaves the rest in the + // stream, so flow control pushes back upstream. + let paced = frame.poll_ready(waiter); + let lease = match paced { + Poll::Ready(()) => self.poll_lease(waiter, frame.remaining().min(1)), + Poll::Pending => Poll::Pending, + }; + let mut lease = match lease { Poll::Pending => { if owed > 0 { frame.notify(); diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 2b78458e3dd..f2e3e01a8b0 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1188,6 +1188,14 @@ impl Producer { } } + /// Poll until the group's writer may take more of this frame from upstream; see + /// [`group::Producer::poll_ready`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn poll_ready(&mut self, waiter: &kio::Waiter) -> Poll<()> { + let written = self.buf.written(Ordering::Acquire) as u64; + self.group.as_mut().expect("frame owns its group").poll_ready(waiter, written) + } + /// Publish what has been written so far, waking consumers parked on the group. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn notify(&self) { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index c779dc81435..8b7c4d6eec1 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -19,6 +19,7 @@ //! //! The stream is closed with [Error] when all writers or readers are dropped. use super::flate; +use super::pace::{self, Pace}; mod content; use crate::cache; use crate::frame::{self, Cached, FrameBuf, Payload}; @@ -57,6 +58,9 @@ pub(super) const CACHE_OVERHEAD: u64 = (kio::Producer::::HEAP // straight after. + 2 * size_of::() + size_of::() + // The `Pace` its writer and readers share, behind its own `Arc`. + + 2 * size_of::() + + size_of::() + kio::log::Writer::::HEAP) as u64; /// A group contains a sequence number because they can arrive out of order. @@ -143,6 +147,8 @@ pub(super) struct Entry { fields: Option>, // A compressed group's inflated size of the stored slice; zero on a plain group. inflated: u64, + // Where the frame ends on its copy's [`Pace`] scale, set as it enters the log. + end: u64, } impl Clone for Entry { @@ -152,6 +158,7 @@ impl Clone for Entry { frame: self.frame.clone(), fields: self.fields.clone(), inflated: self.inflated, + end: self.end, } } } @@ -164,6 +171,7 @@ impl Entry { frame, fields: None, inflated, + end: 0, } } @@ -258,6 +266,9 @@ pub(super) struct GroupState { // Whether an adopted head (see [`Handle::adopt_head`]) begins at the group's first // object, for a wire header that arrives after it. adopted: Option, + + // How far readers got, which each frame entering the log is measured against. + pace: Arc, } /// The frames a copy of a group holds, from where it starts, which a successor copy of @@ -652,6 +663,7 @@ impl Inflate { }, fields: slice.fields.clone(), inflated: 0, + end: 0, }; let cost = Plain::cost(&entry); self.plain.bill.add(cost); @@ -953,8 +965,12 @@ impl GroupState { } /// Append a completed frame, returning the slot growth it charged. - fn push(&mut self, entry: Entry) -> u64 { + /// + /// The frame's place on the pace scale costs its payload and its slot, as the live + /// tail does, so a paced writer of tiny frames is bounded too. + fn push(&mut self, mut entry: Entry) -> u64 { let slots = self.reserve_frame(); + entry.end = self.pace.commit(entry.frame.payload.len() as u64 + size_of::() as u64); self.log.push(entry); slots } @@ -1366,6 +1382,39 @@ fn modify(state: &kio::Producer) -> Result> pub struct Producer { handle: Handle, stats: stats::Meter, + // Where the fastest reader's next frame starts, for [`Self::poll_ready`]. + ahead: Ahead, +} + +/// A paced writer's view of the frame log, measuring how far it runs ahead of its +/// fastest reader without the group's lock. +struct Ahead { + log: kio::log::Reader, + // The fastest reader's next frame and where it starts, while the copy holds it. + start: Option<(usize, u64)>, +} + +impl Ahead { + /// Whether the writer, `partial` bytes into its open frame, is more than + /// [`pace::AHEAD`] past the start of frame `read`. + /// + /// Not while the log holds no frame `read`: either it is not committed yet, so the + /// fastest reader is waiting on the writer, or it was shed, so that reader fell off. + fn paced(&mut self, pace: &Pace, read: usize, partial: u64) -> bool { + let start = match self.start { + Some((index, start)) if index == read => start, + _ => { + let mut log = self.log.read(); + let Ok(Some(entry)) = log.get(read as u64) else { + return false; + }; + let start = entry.end - (entry.frame.payload.len() + size_of::()) as u64; + self.start = Some((read, start)); + start + } + }; + pace.written().saturating_add(partial).saturating_sub(start) > pace::AHEAD + } } /// Shared group state without permission to write frames or finish the group. @@ -1446,6 +1495,8 @@ struct Alive { // A reader of the frame log, cloned into each cursor so it reads frames without the // group lock. log: kio::log::Reader, + // The `GroupState::pace` cursors report their reads to. + pace: Arc, // When the group was created here, on the ambient clock; see [`Consumer::arrived`]. arrived: crate::time::Instant, } @@ -1558,7 +1609,7 @@ impl Producer { let state = kio::Producer::::default(); let charge = cache.charge(); let access = charge.access(); - let log = { + let (log, pace) = { let mut state = state.write().ok().expect("a new group is open"); if track.compress { state.flate = Some(Box::new(Flate::new( @@ -1567,7 +1618,7 @@ impl Producer { ))); } state.charge = charge; - state.log.reader() + (state.log.reader(), state.pace.clone()) }; let alive = Arc::new(Alive { aborted: AtomicBool::new(false), @@ -1576,7 +1627,8 @@ impl Producer { failed: AtomicBool::new(false), started: AtomicBool::new(false), access, - log, + log: log.clone(), + pace, arrived: super::clock::now(), }); Self { @@ -1588,6 +1640,7 @@ impl Producer { alive, }, stats: stats::Meter::default(), + ahead: Ahead { log, start: None }, } } @@ -1652,6 +1705,8 @@ impl Producer { state.offset = index; state.next_index = index; state.log.evict(index as u64); + // The frames below are not this copy's, so no reader of it is behind them. + state.pace.took(index); Ok(()) } @@ -2090,6 +2145,7 @@ impl Producer { frame, fields, inflated, + end: 0, }; if !owned { state.canonicalize(state.next_index - 1, &mut entry)?; @@ -2181,6 +2237,33 @@ impl Producer { pub async fn closed(&self) -> Result<()> { kio::wait(|waiter| self.handle.poll_closed(waiter)).await } + /// Poll until the writer may take more from upstream, `partial` bytes into a frame + /// it has not committed: until it is at most [`pace::AHEAD`] past the start of the + /// first frame its fastest reader has not taken. + /// + /// Ready at once while no reader holds the group, so a group nobody watches keeps + /// writing and sheds its front, and once it is closed, so the next write surfaces why. + /// A reader that took every committed frame is waiting on the writer, so a frame + /// larger than the window never stalls it, and one whose next frame was shed has + /// fallen off the group and is not waited on. + /// + /// Only a paced writer, a session's ingest, waits on this; a local write never does. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn poll_ready(&mut self, waiter: &kio::Waiter, partial: u64) -> Poll<()> { + let pace = &self.handle.alive.pace; + if !self.ahead.paced(pace, pace.read(), partial) { + return Poll::Ready(()); + } + // Either wakes the writer: the last reader leaving, or the group closing. + if self.handle.poll_unused(waiter).is_ready() || self.handle.state.poll_closed(waiter).is_ready() { + return Poll::Ready(()); + } + match self.ahead.paced(pace, pace.park(waiter), partial) { + true => Poll::Pending, + false => Poll::Ready(()), + } + } + /// The non-writing observer used by the cache and built-in sessions. // Sessions need an abortable observer while public callers use Demand. pub(crate) fn handle(&self) -> Handle { @@ -2792,6 +2875,7 @@ impl Handle { cache: self.cache.clone(), access: self.alive.access.clone(), refreshed: self.cache.pool().now(), + pace: self.alive.pace.clone(), }, // Untagged: a tagged track attaches the egress meter via `with_meter` // when it hands the consumer to a subscriber/fetch. @@ -3281,6 +3365,9 @@ struct Cursor { cache: Arc, access: Arc, refreshed: u64, + + // Where this cursor reports the frames it takes, pacing the group's writer. + pace: Arc, } impl Clone for Cursor { @@ -3298,6 +3385,7 @@ impl Clone for Cursor { cache: self.cache.clone(), access: self.access.clone(), refreshed: self.refreshed, + pace: self.pace.clone(), } } } @@ -4030,6 +4118,9 @@ impl Cursor { }; self.index += 1; + // Taking the in-flight frame counts as taking it whole: a reader streaming it is + // at the writer's edge, which a paced writer never waits on. + self.pace.took(self.index); // A direct read (not prefetched): count the frame here; the frame::Consumer // counts its bytes per chunk as they're read out. stats.frames(1); @@ -4065,6 +4156,8 @@ impl Cursor { // The batch is drained: refill it from the log without the lock, unless a plain // read of a compressed group has to inflate under it. if (!self.compressed || self.raw) && self.prefetch.fill_log(&mut self.log, index, budget, self.compressed) > 0 { + // The batch is this cursor's whatever the group sheds, so it counts as taken. + self.pace.took(index + self.prefetch.len); self.refresh_if_stale(); let (frames, bytes) = self.prefetch.buffered(); stats.frames(frames); @@ -4106,6 +4199,7 @@ impl Cursor { // The refill already updated the eviction rank under the group lock. self.refreshed = self.cache.pool().now(); + self.pace.took(index + self.prefetch.len); // A fresh batch was just filled (empty only on a clean end). Count the whole // batch once here, under no lock, so the drained pops that follow stay free. @@ -4863,6 +4957,7 @@ mod test { frame: Frame::new(None, Bytes::from_static(b"b")).into(), fields: fields(1), inflated: 0, + end: 0, }, ], compressed: false, diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 00a715a1efb..9123fd25f95 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -23,6 +23,8 @@ pub(crate) mod datagram; mod delivered; mod expiry; mod skipped; +// How far a group's writer reads ahead of its fastest reader. +mod pace; // The DEFLATE codec behind compressed tracks. mod flate; pub(crate) mod object; diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs new file mode 100644 index 00000000000..b9572d5660c --- /dev/null +++ b/rs/moq-net/src/model/pace.rs @@ -0,0 +1,215 @@ +//! Fastest-reader pacing: how far a group's writer reads ahead of its furthest reader. +//! +//! Readers report the frames they take; a paced writer (a session's ingest) stops reading +//! its upstream stream while it is more than [`AHEAD`] past the first frame the fastest +//! reader has not taken, so stream flow control pushes back to the publisher. Only the +//! fastest reader counts: a slower one is never waited on, and falls off the group's +//! front as the live tail sheds. + +use std::sync::{ + Mutex, PoisonError, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, +}; + +use super::group::MAX_CACHE_BYTES; + +/// How far a paced writer runs ahead of its group's fastest reader: half the live tail, +/// so the other half stays behind the fastest reader for slower ones. +pub(super) const AHEAD: u64 = MAX_CACHE_BYTES / 2; + +/// One group copy's reader progress, shared by its writer and every cursor. +#[derive(Default)] +pub(super) struct Pace { + // One past the furthest frame any reader of the copy has taken. + read: AtomicUsize, + // What the frames committed to the copy's log cost, cumulatively: each entry records + // where it ends on this scale. + written: AtomicU64, + // Set while a writer waits on `read`, so a read wakes it without locking otherwise. + parked: AtomicBool, + waiters: Mutex, +} + +impl Pace { + /// A reader took every frame below `index`. + pub fn took(&self, index: usize) { + // The common case, a reader trailing the fastest one, writes nothing. + if self.read.load(Ordering::Relaxed) >= index { + return; + } + // SeqCst pairs with `park`: either this sees the writer parked, or it sees the read. + self.read.fetch_max(index, Ordering::SeqCst); + if self.parked.load(Ordering::SeqCst) { + let mut waiters = self.waiters.lock().unwrap_or_else(PoisonError::into_inner); + self.parked.store(false, Ordering::SeqCst); + let mut woken = waiters.take(); + drop(waiters); + woken.wake(); + } + } + + /// One past the furthest frame any reader has taken. + pub fn read(&self) -> usize { + self.read.load(Ordering::SeqCst) + } + + /// Count a committed frame costing `cost`, returning where it ends. + /// + /// Called under the group's lock, so commits never race. + pub fn commit(&self, cost: u64) -> u64 { + let end = self.written.load(Ordering::Relaxed) + cost; + self.written.store(end, Ordering::Relaxed); + end + } + + /// Where the last committed frame ends. + pub fn written(&self) -> u64 { + self.written.load(Ordering::Relaxed) + } + + /// Register `waiter` for the next read, returning how far readers got, read after + /// registering so a read in between is never missed. + pub fn park(&self, waiter: &kio::Waiter) -> usize { + let mut waiters = self.waiters.lock().unwrap_or_else(PoisonError::into_inner); + waiter.register(&mut waiters); + self.parked.store(true, Ordering::SeqCst); + self.read.load(Ordering::SeqCst) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::task::Poll; + + use bytes::Bytes; + + use super::*; + use crate::{Error, frame, group}; + + const MB: usize = 1 << 20; + + struct Flag(AtomicBool); + + impl std::task::Wake for Flag { + fn wake(self: Arc) { + self.0.store(true, Ordering::SeqCst); + } + } + + impl Flag { + fn woken(&self) -> bool { + self.0.swap(false, Ordering::SeqCst) + } + } + + fn flag() -> (Arc, kio::Waiter) { + let flag = Arc::new(Flag(AtomicBool::new(false))); + let waiter = kio::Waiter::new(std::task::Waker::from(flag.clone())); + (flag, waiter) + } + + /// An untimed group with `frames` frames of a megabyte each. + fn written(frames: usize) -> group::Producer { + let mut group = group::Info { sequence: 0 }.produce_untimed(); + let payload = Bytes::from(vec![0u8; MB]); + for _ in 0..frames { + group.write_frame(None, payload.clone()).unwrap(); + } + group + } + + fn read(reader: &mut group::Consumer) { + let Poll::Ready(Ok(Some(_))) = reader.poll_read_frame(&kio::Waiter::noop()) else { + panic!("a committed frame is ready"); + }; + } + + /// A writer more than the window ahead of its fastest reader waits, and that reader + /// taking frames lets it on; a slower reader is never waited on. + #[test] + fn the_fastest_reader_paces_the_writer() { + let (flag, waiter) = flag(); + let mut group = written(15); + let mut fast = group.consume(); + let _slow = group.consume(); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(()), "within the window"); + + group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending, "past the window"); + assert!(!flag.woken()); + + // A batch read takes several frames at once, which the writer counts as read. + read(&mut fast); + assert!(flag.woken(), "the fastest reader's read wakes the writer"); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } + + /// Bytes measure the window, not frames, so an open frame counts as it fills. + #[test] + fn an_open_frame_counts_toward_the_window() { + let (_flag, waiter) = flag(); + let group = written(15); + let _reader = group.consume(); + let mut frame = group.create_frame(frame::Info { size: 2 * MB as u64, timestamp: None }).unwrap(); + assert_eq!(frame.poll_ready(&waiter), Poll::Ready(())); + frame.write(Bytes::from(vec![0u8; MB])).unwrap(); + assert_eq!(frame.poll_ready(&waiter), Poll::Pending); + } + + /// A group nobody reads keeps writing, and a parked writer moves on once its last + /// reader leaves. + #[test] + fn a_group_without_readers_never_waits() { + let (flag, waiter) = flag(); + let mut group = written(16); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + + let reader = group.consume(); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending, "a reader that took nothing"); + drop(reader); + assert!(flag.woken(), "the last reader leaving wakes the writer"); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } + + /// A reader that took every committed frame is waiting on the writer, so a frame + /// larger than the window never stalls against it. + #[test] + fn a_reader_at_the_edge_never_stalls_a_large_frame() { + let (_flag, waiter) = flag(); + let group = written(1); + let mut reader = group.consume(); + read(&mut reader); + assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); + + let mut frame = group.create_frame(frame::Info { size: 20 * MB as u64, timestamp: None }).unwrap(); + frame.write(Bytes::from(vec![0u8; 18 * MB])).unwrap(); + assert_eq!(frame.poll_ready(&waiter), Poll::Ready(())); + } + + /// A reader whose next frame the tail shed fell off the group, so the writer stops + /// waiting on it. + #[test] + fn a_reader_that_fell_off_is_not_waited_on() { + let (_flag, waiter) = flag(); + let mut group = written(1); + let _reader = group.consume(); + // A local writer never waits, so it sheds the front past the 32 MB tail. + for _ in 0..40 { + group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + } + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } + + /// An abort wakes a parked writer, so its next write surfaces why. + #[test] + fn an_abort_wakes_a_parked_writer() { + let (flag, waiter) = flag(); + let mut group = written(17); + let _reader = group.consume(); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending); + group.handle().abort(Error::Cancel).unwrap(); + assert!(flag.woken()); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } +} diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs new file mode 100644 index 00000000000..747b20bfdd6 --- /dev/null +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -0,0 +1,144 @@ +#![cfg(feature = "async")] + +//! A relay's ingest buffers at most a window past the group's fastest reader, then leaves +//! the rest in the stream, so flow control pushes back to the publisher. A slower reader +//! is never waited on: it falls off the group's front. + +use std::time::Duration; + +use bytes::Bytes; +use moq_net::{Hop, Timestamp, Version, origin, track}; + +use crate::support::harness::{MockPair, peer, spawn}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +const MB: usize = 1 << 20; + +const VERSIONS: &[&str] = &["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"]; + +fn origin(hop: u64) -> origin::Producer { + let (origin, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + origin +} + +/// A publisher's track and a relay's subscription to it, over one peering. +struct Relay { + track: track::Producer, + subscriber: track::Subscriber, + _broadcast: origin::Publication, + _pair: MockPair, + _origins: [origin::Producer; 2], +} + +async fn relay(version: &str) -> Relay { + let publisher = origin(1); + let relay = origin(2); + let pair = peer(version.parse::().unwrap(), &publisher, &relay).await; + + let broadcast = publisher.create_broadcast("live").unwrap(); + let track = broadcast + .create_track("data", track::Info::default().with_timescale(moq_net::Timescale::MILLI)) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let remote = moq_clock::timeout(TIMEOUT, relay.consume().follow("live", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + let subscriber = moq_clock::timeout( + TIMEOUT, + remote + .track("data") + .unwrap() + .subscribe(track::Subscription::default().with_max_delay(Duration::from_secs(3600))), + ) + .await + .expect("subscribe timeout") + .expect("subscribe"); + Relay { + track, + subscriber, + _broadcast: broadcast, + _pair: pair, + _origins: [publisher, relay], + } +} + +#[moq_clock::test] +async fn an_idle_reader_paces_the_relay_and_loses_nothing() { + let payload = Bytes::from(vec![7u8; MB]); + for version in VERSIONS { + let mut relay = relay(version).await; + + let mut group = relay.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("group 0"); + // Within the publisher's own 32 MB tail, so only the relay's pacing holds it back. + for _ in 1..30 { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + group.finish().unwrap(); + + // Simulated time moves only once every task is parked: the relay's ingest stopped. + moq_clock::sleep(Duration::from_secs(1)).await; + let held = reader.frame_count(); + assert!( + (16..=18).contains(&held), + "{version}: the relay read {held} MB ahead of a reader that took nothing" + ); + + for index in 0..30 { + let frame = moq_clock::timeout(TIMEOUT, reader.read_frame()) + .await + .unwrap_or_else(|_| panic!("{version}: frame {index} never arrived")) + .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) + .expect("a frame"); + assert_eq!(frame.payload.len(), MB, "{version}: frame {index}"); + } + let end = moq_clock::timeout(TIMEOUT, reader.read_frame()).await.expect("end timeout"); + assert!(matches!(end, Ok(None)), "{version}: {end:?}"); + } +} + +#[moq_clock::test] +async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { + let payload = Bytes::from(vec![7u8; MB]); + for version in VERSIONS { + let mut relay = relay(version).await; + + let mut group = relay.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut fast = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("group 0"); + let mut slow = fast.clone(); + + // Well past the 32 MB tail: the fast reader keeps up the whole way. + for index in 0..48 { + if index > 0 { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + let frame = moq_clock::timeout(TIMEOUT, fast.read_frame()) + .await + .unwrap_or_else(|_| panic!("{version}: the fast reader stalled at frame {index}")) + .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) + .expect("a frame"); + assert_eq!(frame.payload.len(), MB, "{version}: frame {index}"); + } + group.finish().unwrap(); + + let fell = moq_clock::timeout(TIMEOUT, slow.read_frame()).await.expect("read timeout"); + assert!( + matches!(fell, Err(moq_net::Error::Evicted)), + "{version}: the slow reader got {fell:?}" + ); + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 94b3330cdc3..a2c5ac84c24 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -21,6 +21,7 @@ mod dial_split_horizon; mod drained_claim; mod empty_group; mod extensions; +mod fastest_reader; mod fetch_cancel; mod fetch_fanout; mod fetch_from_feed; From 05231f0e64a77f7d9208a83efb8eef7f50ff0fbb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 07:25:01 -0700 Subject: [PATCH 03/16] refactor(net): keep fastest-reader progress in lock-free byte atomics Readers report the byte end of the last frame they take, shedding records the byte front, and the writer compares the three without touching the log, so group::Producer keeps its size. Docs and the quest's remainder follow. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 6 + doc/concept/moq-lite.md | 9 ++ quest/m1/unbounded-groups/fastest-reader.md | 41 +++++-- rs/moq-net/src/model/frame.rs | 6 +- rs/moq-net/src/model/group.rs | 88 +++++--------- rs/moq-net/src/model/pace.rs | 114 ++++++++++++------ rs/moq-net/src/model/track.rs | 4 +- .../tests/integration/fastest_reader.rs | 8 +- 8 files changed, 166 insertions(+), 110 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 407ca90be6c..76839683392 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -346,6 +346,12 @@ newest 32 MB. A reader of a track's newest group that reaches a shed frame asks for a new group on tracks that opt into refresh. Any other reader of a shed frame sees `Evicted`. +While a group has readers, the relay receives at most 16 MB of it past the +fastest one, then stops reading its stream until that reader catches up, so +flow control slows the upstream hop and the publisher. A slower reader is never +waited on: one that trails the fastest by more than the rest of the 32 MB +reaches a shed frame. A group nobody reads keeps arriving and shedding. + ## \[stats] ```toml diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 26055fc1856..28ef313ff12 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -380,6 +380,15 @@ Put together, a conference might use: Under light congestion video drops the tail of a group; under heavy congestion video stops and audio lags by at most 500 ms. No protocol change, just knobs. +A long group is paced by its fastest reader instead. Each hop receives at most +16 MB of a group past its fastest reader, then stops reading the group's stream, +so stream flow control pushes back hop by hop to the publisher, and a reader +that keeps up loses nothing. A slower reader is never waited on: one that trails +the fastest by more than the group's 32 MB tail reaches a shed frame, and asks +for a new group on tracks that opt into [refresh](#refresh) or ends with +`Evicted`. A hop with no reader keeps receiving and shedding, so pacing never +stalls an unwatched track, and the publisher's own writes never wait. + ### Refresh A publisher can opt a track into refresh by setting its byte ratio. A new diff --git a/quest/m1/unbounded-groups/fastest-reader.md b/quest/m1/unbounded-groups/fastest-reader.md index 61cb59975bd..1fdb340bf2a 100644 --- a/quest/m1/unbounded-groups/fastest-reader.md +++ b/quest/m1/unbounded-groups/fastest-reader.md @@ -72,15 +72,36 @@ control pace a single client; Node `pipe()` to several destinations paces the slowest. This is `tee()` with a bounded buffer that drops the branch that falls off it ([whatwg/streams#1235](https://github.com/whatwg/streams/issues/1235)). -Open, for the implementer to measure and record: the readiness API's shape, -which is an API decision to bring to the maintainer before coding. qmux transports carry the same rule through their -own stream credit ([qmux receive lease](/quest/m2/qmux-receive-lease.md)). +Landed in [moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388) (Rust, +session ingest only): readers report the byte end of each frame they take +(`Pace` in `rs/moq-net/src/model/pace.rs`), and the shared transport reader's +payload loop (`coding::Reader::poll_read_frame`) waits on +`group::Producer::poll_ready` before taking more, so moq-lite and +moq-transport ingest both pause past the 16 MB window. A reader that took every +committed frame counts as waiting on the writer, so a frame larger than the +window never stalls it. Tests cover one hop on lite-06, lite-07-wip, and +moq-transport-22: an idle reader holds the relay copy near the window and then +receives every frame, and a slow reader ends `Evicted` while the fast one keeps +up. -Tests: a relay chain where a fast reader paces a publisher that writes faster -than the link, with memory held near the window; a second, slower reader ending -`Evicted` without slowing the fast one; a track with no readers never blocking -its publisher; Rust and JS parity. Benchmark the hot path in `group.rs` and -record any regression. +What remains: -Docs: the moq-lite concept page's congestion section and the relay cache docs -change inline. +- The receive-credit cap for paused streams above, which has to land before + pacing can reset nothing it means to keep lossless. Bring the mechanism to + the maintainer before coding. +- The readiness future for origin producers: its shape is an API decision to + bring to the maintainer before coding. Until it lands, a local byte-stream + writer more than 32 MB ahead of its session's egress sheds its own front. +- JS parity. +- A two-relay chain test with memory measured near the window. +- Benchmarks of the hot path in `group.rs`: the cursor's report is one + relaxed load per frame or batch, and a store only for the fastest reader. + They were not run on the busy host that landed the first slice. +- Fills that write a whole batch of groups at once evict their own earlier + groups before the reader takes them (the follow-up from + [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)). Pacing within + a group does not cover this; decide whether it belongs here or with + [flights](/quest/m1/unbounded-groups/flights.md). + +qmux transports carry the same rule through their own stream credit +([qmux receive lease](/quest/m2/qmux-receive-lease.md)). diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index f2e3e01a8b0..b0890923726 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1191,9 +1191,9 @@ impl Producer { /// Poll until the group's writer may take more of this frame from upstream; see /// [`group::Producer::poll_ready`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_ready(&mut self, waiter: &kio::Waiter) -> Poll<()> { - let written = self.buf.written(Ordering::Acquire) as u64; - self.group.as_mut().expect("frame owns its group").poll_ready(waiter, written) + pub(crate) fn poll_ready(&self, waiter: &kio::Waiter) -> Poll<()> { + self.group() + .poll_ready(waiter, self.buf.written(Ordering::Acquire) as u64) } /// Publish what has been written so far, waking consumers parked on the group. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 8b7c4d6eec1..1121e371c4f 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -19,7 +19,7 @@ //! //! The stream is closed with [Error] when all writers or readers are dropped. use super::flate; -use super::pace::{self, Pace}; +use super::pace::Pace; mod content; use crate::cache; use crate::frame::{self, Cached, FrameBuf, Payload}; @@ -663,7 +663,7 @@ impl Inflate { }, fields: slice.fields.clone(), inflated: 0, - end: 0, + end: slice.end, }; let cost = Plain::cost(&entry); self.plain.bill.add(cost); @@ -883,13 +883,15 @@ impl GroupState { } let base = self.log.base() as usize; if base < end { - let (mut charged, mut payload, mut fields) = (0, 0, 0); + let (mut charged, mut payload, mut fields, mut shed) = (0, 0, 0, 0); for entry in (base..end).map(|index| self.log.get(index as u64).expect("committed below the end")) { + shed = entry.end; charged += self.entry_bytes(entry); payload += entry.frame.payload.len() as u64; fields += entry.fields.as_ref().map_or(0, |fields| fields.heap()); } self.forget(payload, fields); + self.pace.shed(shed); let heap = kio::log::Writer::::HEAP; let before = self.log.heap().max(heap); self.log.evict(end as u64); @@ -970,7 +972,9 @@ impl GroupState { /// tail does, so a paced writer of tiny frames is bounded too. fn push(&mut self, mut entry: Entry) -> u64 { let slots = self.reserve_frame(); - entry.end = self.pace.commit(entry.frame.payload.len() as u64 + size_of::() as u64); + entry.end = self + .pace + .commit(entry.frame.payload.len() as u64 + size_of::() as u64); self.log.push(entry); slots } @@ -1078,6 +1082,7 @@ impl GroupState { /// /// A compressed group hands a plain reader each frame whole once it completes, as the /// slices to inflate, and a `raw` one the stored slice, streaming one still arriving. + /// A frame handed out counts as taken, pacing the writer (see [`Pace`]). fn poll_frame_source(&self, index: usize, raw: bool) -> Poll>> { self.check_read(index, raw)?; if index < self.held() @@ -1091,6 +1096,7 @@ impl GroupState { // walk spare a group a consumer is actively draining. self.charge.refresh(); if let Some(flate) = self.flate.as_ref().filter(|_| !raw) { + self.pace.took(entry.end); return Poll::Ready(Ok(Some(Source::Inflate { job: self.inflate(flate, index, index)?, local: index, @@ -1099,6 +1105,7 @@ impl GroupState { }))); } let (header, source) = entry.read(self.flate.is_some()); + self.pace.took(entry.end); return Poll::Ready(Ok(Some(Source::Ready(header, source)))); } if index == self.committed() @@ -1107,6 +1114,8 @@ impl GroupState { && (raw || self.flate.is_none()) { self.charge.refresh(); + // A reader streaming the open frame is at the writer's edge. + self.pace.took_committed(); let info = frame::Info { size: p.buf.size() as u64, timestamp: p.timestamp, @@ -1382,39 +1391,6 @@ fn modify(state: &kio::Producer) -> Result> pub struct Producer { handle: Handle, stats: stats::Meter, - // Where the fastest reader's next frame starts, for [`Self::poll_ready`]. - ahead: Ahead, -} - -/// A paced writer's view of the frame log, measuring how far it runs ahead of its -/// fastest reader without the group's lock. -struct Ahead { - log: kio::log::Reader, - // The fastest reader's next frame and where it starts, while the copy holds it. - start: Option<(usize, u64)>, -} - -impl Ahead { - /// Whether the writer, `partial` bytes into its open frame, is more than - /// [`pace::AHEAD`] past the start of frame `read`. - /// - /// Not while the log holds no frame `read`: either it is not committed yet, so the - /// fastest reader is waiting on the writer, or it was shed, so that reader fell off. - fn paced(&mut self, pace: &Pace, read: usize, partial: u64) -> bool { - let start = match self.start { - Some((index, start)) if index == read => start, - _ => { - let mut log = self.log.read(); - let Ok(Some(entry)) = log.get(read as u64) else { - return false; - }; - let start = entry.end - (entry.frame.payload.len() + size_of::()) as u64; - self.start = Some((read, start)); - start - } - }; - pace.written().saturating_add(partial).saturating_sub(start) > pace::AHEAD - } } /// Shared group state without permission to write frames or finish the group. @@ -1627,7 +1603,7 @@ impl Producer { failed: AtomicBool::new(false), started: AtomicBool::new(false), access, - log: log.clone(), + log, pace, arrived: super::clock::now(), }); @@ -1640,7 +1616,6 @@ impl Producer { alive, }, stats: stats::Meter::default(), - ahead: Ahead { log, start: None }, } } @@ -1705,8 +1680,6 @@ impl Producer { state.offset = index; state.next_index = index; state.log.evict(index as u64); - // The frames below are not this copy's, so no reader of it is behind them. - state.pace.took(index); Ok(()) } @@ -2238,8 +2211,8 @@ impl Producer { kio::wait(|waiter| self.handle.poll_closed(waiter)).await } /// Poll until the writer may take more from upstream, `partial` bytes into a frame - /// it has not committed: until it is at most [`pace::AHEAD`] past the start of the - /// first frame its fastest reader has not taken. + /// it has not committed: until it is at most [`super::pace::AHEAD`] past the end of the last + /// frame its fastest reader took. /// /// Ready at once while no reader holds the group, so a group nobody watches keeps /// writing and sheds its front, and once it is closed, so the next write surfaces why. @@ -2249,16 +2222,16 @@ impl Producer { /// /// Only a paced writer, a session's ingest, waits on this; a local write never does. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_ready(&mut self, waiter: &kio::Waiter, partial: u64) -> Poll<()> { + pub(crate) fn poll_ready(&self, waiter: &kio::Waiter, partial: u64) -> Poll<()> { let pace = &self.handle.alive.pace; - if !self.ahead.paced(pace, pace.read(), partial) { + if !pace.ahead(partial) { return Poll::Ready(()); } // Either wakes the writer: the last reader leaving, or the group closing. if self.handle.poll_unused(waiter).is_ready() || self.handle.state.poll_closed(waiter).is_ready() { return Poll::Ready(()); } - match self.ahead.paced(pace, pace.park(waiter), partial) { + match pace.paced(pace.park(waiter), partial) { true => Poll::Pending, false => Poll::Ready(()), } @@ -2478,6 +2451,8 @@ impl Handle { if state.canonicalize(index, &mut entry).is_err() { return; } + // Below the log, so off this copy's pace scale: taking it reports nothing. + entry.end = 0; adopted.push(entry); } // Below the log's first frame, so these never enter it: readers find them here. @@ -3055,6 +3030,8 @@ struct Prefetch { inflated: Option>, pos: usize, len: usize, + // Where the last frame buffered ends on the group's pace scale. + end: u64, } impl Prefetch { @@ -3160,6 +3137,7 @@ impl Prefetch { if inflated { self.inflated.get_or_insert_default()[at] = entry.inflated; } + self.end = self.end.max(entry.end); self.len += 1; } @@ -4075,13 +4053,10 @@ impl Cursor { // A committed frame comes from the log without the lock, unless a plain read of a // compressed group has to inflate it under the lock. let committed = match !self.compressed || raw { - true => self - .log - .read() - .get(index as u64) - .ok() - .flatten() - .map(|entry| entry.read(self.compressed)), + true => self.log.read().get(index as u64).ok().flatten().map(|entry| { + self.pace.took(entry.end); + entry.read(self.compressed) + }), false => None, }; let (info, source) = match committed { @@ -4118,9 +4093,6 @@ impl Cursor { }; self.index += 1; - // Taking the in-flight frame counts as taking it whole: a reader streaming it is - // at the writer's edge, which a paced writer never waits on. - self.pace.took(self.index); // A direct read (not prefetched): count the frame here; the frame::Consumer // counts its bytes per chunk as they're read out. stats.frames(1); @@ -4157,7 +4129,7 @@ impl Cursor { // read of a compressed group has to inflate under it. if (!self.compressed || self.raw) && self.prefetch.fill_log(&mut self.log, index, budget, self.compressed) > 0 { // The batch is this cursor's whatever the group sheds, so it counts as taken. - self.pace.took(index + self.prefetch.len); + self.pace.took(self.prefetch.end); self.refresh_if_stale(); let (frames, bytes) = self.prefetch.buffered(); stats.frames(frames); @@ -4199,7 +4171,7 @@ impl Cursor { // The refill already updated the eviction rank under the group lock. self.refreshed = self.cache.pool().now(); - self.pace.took(index + self.prefetch.len); + self.pace.took(self.prefetch.end); // A fresh batch was just filled (empty only on a clean end). Count the whole // batch once here, under no lock, so the drained pops that follow stay free. diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index b9572d5660c..bda7f1949cc 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -1,15 +1,21 @@ //! Fastest-reader pacing: how far a group's writer reads ahead of its furthest reader. //! //! Readers report the frames they take; a paced writer (a session's ingest) stops reading -//! its upstream stream while it is more than [`AHEAD`] past the first frame the fastest -//! reader has not taken, so stream flow control pushes back to the publisher. Only the +//! its upstream stream while it is more than [`AHEAD`] past the end of the last frame the +//! fastest reader took, so stream flow control pushes back to the publisher. Only the //! fastest reader counts: a slower one is never waited on, and falls off the group's //! front as the live tail sheds. +//! +//! Progress is in bytes on one scale per group copy: each frame entering the copy's log +//! ends where the frames before it, payload and slot, add up to. A frame larger than the +//! window never stalls its writer, since a reader that took every committed frame is +//! waiting on the writer. use std::sync::{ Mutex, PoisonError, - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }; +use std::task::Waker; use super::group::MAX_CACHE_BYTES; @@ -20,58 +26,82 @@ pub(super) const AHEAD: u64 = MAX_CACHE_BYTES / 2; /// One group copy's reader progress, shared by its writer and every cursor. #[derive(Default)] pub(super) struct Pace { - // One past the furthest frame any reader of the copy has taken. - read: AtomicUsize, - // What the frames committed to the copy's log cost, cumulatively: each entry records - // where it ends on this scale. + // Where the last frame the fastest reader took ends. + read: AtomicU64, + // Where the last committed frame ends. written: AtomicU64, - // Set while a writer waits on `read`, so a read wakes it without locking otherwise. + // Where the last frame shed from the copy's front ends: a reader short of it fell off. + front: AtomicU64, + // Set while the writer waits on `read`, so a read wakes it without locking otherwise. parked: AtomicBool, - waiters: Mutex, + // The writer's waker: a group has one writer, so one slot, far smaller than a list. + writer: Mutex>, } impl Pace { - /// A reader took every frame below `index`. - pub fn took(&self, index: usize) { + /// A reader took the frame ending at `end`, and every frame before it. + pub fn took(&self, end: u64) { // The common case, a reader trailing the fastest one, writes nothing. - if self.read.load(Ordering::Relaxed) >= index { + if self.read.load(Ordering::Relaxed) >= end { return; } // SeqCst pairs with `park`: either this sees the writer parked, or it sees the read. - self.read.fetch_max(index, Ordering::SeqCst); - if self.parked.load(Ordering::SeqCst) { - let mut waiters = self.waiters.lock().unwrap_or_else(PoisonError::into_inner); - self.parked.store(false, Ordering::SeqCst); - let mut woken = waiters.take(); - drop(waiters); - woken.wake(); + self.read.fetch_max(end, Ordering::SeqCst); + if self.parked.swap(false, Ordering::SeqCst) { + let writer = self.writer.lock().unwrap_or_else(PoisonError::into_inner).take(); + if let Some(writer) = writer { + writer.wake(); + } } } - /// One past the furthest frame any reader has taken. - pub fn read(&self) -> usize { - self.read.load(Ordering::SeqCst) + /// A reader took every frame committed so far, such as by taking the open one. + pub fn took_committed(&self) { + self.took(self.written()); } /// Count a committed frame costing `cost`, returning where it ends. /// /// Called under the group's lock, so commits never race. pub fn commit(&self, cost: u64) -> u64 { - let end = self.written.load(Ordering::Relaxed) + cost; - self.written.store(end, Ordering::Relaxed); + let end = self.written() + cost; + self.written.store(end, Ordering::Release); end } - /// Where the last committed frame ends. - pub fn written(&self) -> u64 { - self.written.load(Ordering::Relaxed) + /// The copy shed every frame up to the one ending at `end`. + pub fn shed(&self, end: u64) { + self.front.fetch_max(end, Ordering::Release); + } + + fn written(&self) -> u64 { + self.written.load(Ordering::Acquire) + } + + /// Whether a writer `partial` bytes into an uncommitted frame is more than [`AHEAD`] + /// past the fastest reader, given it read up to `read`. + /// + /// Never while that reader took every committed frame, since it is waiting on the + /// writer, nor once the frames it has yet to take were shed, since it fell off. + pub fn paced(&self, read: u64, partial: u64) -> bool { + let written = self.written(); + read < written && read >= self.front.load(Ordering::Acquire) && written.saturating_add(partial) - read > AHEAD + } + + /// Whether the writer is paced, as [`Self::paced`] judges it now. + pub fn ahead(&self, partial: u64) -> bool { + self.paced(self.read.load(Ordering::SeqCst), partial) } /// Register `waiter` for the next read, returning how far readers got, read after /// registering so a read in between is never missed. - pub fn park(&self, waiter: &kio::Waiter) -> usize { - let mut waiters = self.waiters.lock().unwrap_or_else(PoisonError::into_inner); - waiter.register(&mut waiters); + pub fn park(&self, waiter: &kio::Waiter) -> u64 { + let mut writer = self.writer.lock().unwrap_or_else(PoisonError::into_inner); + match &mut *writer { + Some(waker) => waker.clone_from(waiter.waker()), + None => *writer = Some(waiter.waker().clone()), + } + drop(writer); self.parked.store(true, Ordering::SeqCst); self.read.load(Ordering::SeqCst) } @@ -151,7 +181,12 @@ mod tests { let (_flag, waiter) = flag(); let group = written(15); let _reader = group.consume(); - let mut frame = group.create_frame(frame::Info { size: 2 * MB as u64, timestamp: None }).unwrap(); + let mut frame = group + .create_frame(frame::Info { + size: 2 * MB as u64, + timestamp: None, + }) + .unwrap(); assert_eq!(frame.poll_ready(&waiter), Poll::Ready(())); frame.write(Bytes::from(vec![0u8; MB])).unwrap(); assert_eq!(frame.poll_ready(&waiter), Poll::Pending); @@ -162,11 +197,15 @@ mod tests { #[test] fn a_group_without_readers_never_waits() { let (flag, waiter) = flag(); - let mut group = written(16); + let group = written(16); assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); let reader = group.consume(); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending, "a reader that took nothing"); + assert_eq!( + group.poll_ready(&waiter, 0), + Poll::Pending, + "a reader that took nothing" + ); drop(reader); assert!(flag.woken(), "the last reader leaving wakes the writer"); assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); @@ -182,7 +221,12 @@ mod tests { read(&mut reader); assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); - let mut frame = group.create_frame(frame::Info { size: 20 * MB as u64, timestamp: None }).unwrap(); + let mut frame = group + .create_frame(frame::Info { + size: 20 * MB as u64, + timestamp: None, + }) + .unwrap(); frame.write(Bytes::from(vec![0u8; 18 * MB])).unwrap(); assert_eq!(frame.poll_ready(&waiter), Poll::Ready(())); } @@ -205,7 +249,7 @@ mod tests { #[test] fn an_abort_wakes_a_parked_writer() { let (flag, waiter) = flag(); - let mut group = written(17); + let group = written(17); let _reader = group.consume(); assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending); group.handle().abort(Error::Cancel).unwrap(); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index da4102ad639..9a7dafa892a 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -16698,7 +16698,7 @@ mod test { #[moq_clock::test] #[cfg(feature = "async")] async fn fetch_refresh_survives_eviction() { - let (mut producer, _pool) = pooled_producer(10_000); + let (mut producer, _pool) = pooled_producer(11_000); let consumer = producer.consume(); finished_group(&mut producer, 3_000); // seq 0 @@ -17463,7 +17463,7 @@ mod test { #[cfg(feature = "async")] async fn same_tick_fetch_protects() { // No time advances at all: every timestamp lands in the same tick. - let (mut producer, _pool) = pooled_producer(10_000); + let (mut producer, _pool) = pooled_producer(11_000); let consumer = producer.consume(); finished_group(&mut producer, 3_000); // seq 0 diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index 747b20bfdd6..7bc2fed84b3 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -101,7 +101,9 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { .expect("a frame"); assert_eq!(frame.payload.len(), MB, "{version}: frame {index}"); } - let end = moq_clock::timeout(TIMEOUT, reader.read_frame()).await.expect("end timeout"); + let end = moq_clock::timeout(TIMEOUT, reader.read_frame()) + .await + .expect("end timeout"); assert!(matches!(end, Ok(None)), "{version}: {end:?}"); } } @@ -135,7 +137,9 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { } group.finish().unwrap(); - let fell = moq_clock::timeout(TIMEOUT, slow.read_frame()).await.expect("read timeout"); + let fell = moq_clock::timeout(TIMEOUT, slow.read_frame()) + .await + .expect("read timeout"); assert!( matches!(fell, Err(moq_net::Error::Evicted)), "{version}: the slow reader got {fell:?}" From 4305b8e7f42660747400b6c74df7de44e6d6bffb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:25:55 -0700 Subject: [PATCH 04/16] refactor(net): reach the group's pace through the cursor's Alive Keeps Cursor and Prefetch at their old size, so the session state machines' layouts (and their large_enum_variant expectations on wasm32) are unchanged. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 57 +++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 1121e371c4f..fd7fbda6ce5 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2848,9 +2848,8 @@ impl Handle { raw: false, compressed: self.track.compress, cache: self.cache.clone(), - access: self.alive.access.clone(), + alive: self.alive.clone(), refreshed: self.cache.pool().now(), - pace: self.alive.pace.clone(), }, // Untagged: a tagged track attaches the egress meter via `with_meter` // when it hands the consumer to a subscriber/fetch. @@ -3030,8 +3029,6 @@ struct Prefetch { inflated: Option>, pos: usize, len: usize, - // Where the last frame buffered ends on the group's pace scale. - end: u64, } impl Prefetch { @@ -3106,8 +3103,9 @@ impl Prefetch { } } - /// Buffer `entry` at the end of the batch, with its inflated size if `inflated`. - fn push(&mut self, entry: &Entry, inflated: bool) { + /// Buffer `entry` at the end of the batch, with its inflated size if `inflated`, + /// returning where it ends on the group's pace scale. + fn push(&mut self, entry: &Entry, inflated: bool) -> u64 { let at = self.len; let frames = self .frames @@ -3137,8 +3135,8 @@ impl Prefetch { if inflated { self.inflated.get_or_insert_default()[at] = entry.inflated; } - self.end = self.end.max(entry.end); self.len += 1; + entry.end } /// Refill with up to `budget` of the completed frames from `index` on, under the @@ -3175,25 +3173,39 @@ impl Prefetch { Some(flate) if !raw && count > 0 => Some(state.inflate(flate, index, index + count - 1)?), _ => None, }; + let mut end = 0; for entry in state.entries(index, index + count) { - self.push(entry, compressed && raw); + end = self.push(entry, compressed && raw); } + // The batch is the reader's whatever the group sheds, so it counts as taken. + state.pace.took(end); Ok(job) } /// Refill with up to `budget` of the completed frames from `index` on, reading the /// log without the group's guard, and return how many. A plain read of a `compressed` /// group inflates under the guard instead, through [`Self::fill`]. - fn fill_log(&mut self, log: &mut kio::log::Reader, index: usize, budget: usize, compressed: bool) -> usize { + /// + /// The batch is the reader's whatever the group sheds, so it reports it taken to `pace`. + fn fill_log( + &mut self, + log: &mut kio::log::Reader, + pace: &Pace, + index: usize, + budget: usize, + compressed: bool, + ) -> usize { self.reset(compressed); let mut read = log.read(); + let mut end = 0; for index in (index as u64..).take(budget.min(Self::CAP)) { // Evicted or not yet committed: the guard resolves which. let Ok(Some(entry)) = read.get(index) else { break; }; - self.push(entry, compressed); + end = self.push(entry, compressed); } + pace.took(end); self.len } @@ -3339,13 +3351,11 @@ struct Cursor { // Whether the group is compressed, so a plain read inflates under the state's lock. compressed: bool, - // Record prefetched reads without entering the group's state on every frame. + // Record prefetched reads without entering the group's state on every frame, through + // the group's access stamp, and report the frames taken to its pace. cache: Arc, - access: Arc, + alive: Arc, refreshed: u64, - - // Where this cursor reports the frames it takes, pacing the group's writer. - pace: Arc, } impl Clone for Cursor { @@ -3361,9 +3371,8 @@ impl Clone for Cursor { raw: self.raw, compressed: self.compressed, cache: self.cache.clone(), - access: self.access.clone(), + alive: self.alive.clone(), refreshed: self.refreshed, - pace: self.pace.clone(), } } } @@ -3969,7 +3978,7 @@ impl Cursor { /// Record prefetched reads, updating the eviction rank once per sampled tick. fn refresh_if_stale(&mut self) { - self.access.touch(); + self.alive.access.touch(); let tick = self.cache.pool().now(); if tick != self.refreshed { self.state.read().charge.refresh(); @@ -4054,7 +4063,7 @@ impl Cursor { // compressed group has to inflate it under the lock. let committed = match !self.compressed || raw { true => self.log.read().get(index as u64).ok().flatten().map(|entry| { - self.pace.took(entry.end); + self.alive.pace.took(entry.end); entry.read(self.compressed) }), false => None, @@ -4127,9 +4136,12 @@ impl Cursor { // The batch is drained: refill it from the log without the lock, unless a plain // read of a compressed group has to inflate under it. - if (!self.compressed || self.raw) && self.prefetch.fill_log(&mut self.log, index, budget, self.compressed) > 0 { - // The batch is this cursor's whatever the group sheds, so it counts as taken. - self.pace.took(self.prefetch.end); + if (!self.compressed || self.raw) + && self + .prefetch + .fill_log(&mut self.log, &self.alive.pace, index, budget, self.compressed) + > 0 + { self.refresh_if_stale(); let (frames, bytes) = self.prefetch.buffered(); stats.frames(frames); @@ -4171,7 +4183,6 @@ impl Cursor { // The refill already updated the eviction rank under the group lock. self.refreshed = self.cache.pool().now(); - self.pace.took(self.prefetch.end); // A fresh batch was just filled (empty only on a clean end). Count the whole // batch once here, under no lock, so the drained pops that follow stay free. From c8b1a5e36638cb5d548584cba9a6d39fa21435a9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:59:02 -0700 Subject: [PATCH 05/16] fix(net): never pace a writer against a reader waiting on it or shed A reader parked on a frame the writer has yet to commit, wherever it skipped to, reports the edge, and shedding past the fastest reader under cache pressure wakes a parked writer. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 9 ++++++- rs/moq-net/src/model/pace.rs | 48 +++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index fd7fbda6ce5..f6e204ff875 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1144,7 +1144,14 @@ impl GroupState { (Some(total), Some(err)) if index < total => Poll::Ready(Err(err.clone())), (Some(_), _) => Poll::Ready(Ok(())), (None, Some(err)) => Poll::Ready(Err(err.clone())), - (None, None) => Poll::Pending, + (None, None) => { + // Waiting on frames the writer has yet to commit, wherever the reader skipped + // to, so it holds the writer back from none of them. + if index >= self.committed() { + self.pace.took_committed(); + } + Poll::Pending + } } } diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index bda7f1949cc..0baf039c84d 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -47,6 +47,11 @@ impl Pace { } // SeqCst pairs with `park`: either this sees the writer parked, or it sees the read. self.read.fetch_max(end, Ordering::SeqCst); + self.wake(); + } + + /// Wake the writer, if parked, since what it waits on changed. + fn wake(&self) { if self.parked.swap(false, Ordering::SeqCst) { let writer = self.writer.lock().unwrap_or_else(PoisonError::into_inner).take(); if let Some(writer) = writer { @@ -69,9 +74,12 @@ impl Pace { end } - /// The copy shed every frame up to the one ending at `end`. + /// The copy shed every frame up to the one ending at `end`, which may leave the + /// fastest reader fallen off, so the writer no longer waits on it. pub fn shed(&self, end: u64) { - self.front.fetch_max(end, Ordering::Release); + // SeqCst pairs with `park`, as a read does. + self.front.fetch_max(end, Ordering::SeqCst); + self.wake(); } fn written(&self) -> u64 { @@ -85,7 +93,7 @@ impl Pace { /// writer, nor once the frames it has yet to take were shed, since it fell off. pub fn paced(&self, read: u64, partial: u64) -> bool { let written = self.written(); - read < written && read >= self.front.load(Ordering::Acquire) && written.saturating_add(partial) - read > AHEAD + read < written && read >= self.front.load(Ordering::SeqCst) && written.saturating_add(partial) - read > AHEAD } /// Whether the writer is paced, as [`Self::paced`] judges it now. @@ -93,8 +101,9 @@ impl Pace { self.paced(self.read.load(Ordering::SeqCst), partial) } - /// Register `waiter` for the next read, returning how far readers got, read after - /// registering so a read in between is never missed. + /// Register `waiter` for the next read or shed, returning how far readers got, read + /// after registering so a read in between is never missed; [`Self::paced`] reads the + /// front after it too. pub fn park(&self, waiter: &kio::Waiter) -> u64 { let mut writer = self.writer.lock().unwrap_or_else(PoisonError::into_inner); match &mut *writer { @@ -245,6 +254,35 @@ mod tests { assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); } + /// A reader that skipped past what the writer has written is waiting on it, so it + /// never holds the writer back from the frame it wants. + #[test] + fn a_reader_ahead_of_the_writer_never_stalls_it() { + let (_flag, waiter) = flag(); + let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut reader = group.consume().with_range(20..); + assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); + for _ in 0..17 { + group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + } + // Each commit wakes the parked reader, which polls again and reports the edge. + assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } + + /// Shedding past the fastest reader under cache pressure wakes a parked writer: that + /// reader fell off. + #[test] + fn shedding_past_the_reader_wakes_the_writer() { + let (flag, waiter) = flag(); + let group = written(17); + let _reader = group.consume(); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending); + group.handle().shed(u64::MAX, u64::MAX); + assert!(flag.woken()); + assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + } + /// An abort wakes a parked writer, so its next write surfaces why. #[test] fn an_abort_wakes_a_parked_writer() { From 33f2f89e14ef5a2de303732fb405a4fb08c30fac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:00:02 -0700 Subject: [PATCH 06/16] feat(net): cap paused receive credit per session; public group readiness Each stream a session's ingest pauses to pace a group's writer reserves its stream window from a per-session budget of half the connection window, read live from the new poll::Session::receive_window. A stream that would exceed it keeps reading and shedding, and a transport that reports no window never pauses. moq-quic reports its windows through moq-tokio and moq-uring. group::Producer::poll_ready and ready() are now public: pending until a reader holds the group and the 16 MB window has room. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 3 + doc/concept/moq-lite.md | 5 +- quest/m1/unbounded-groups/fastest-reader.md | 29 ++- quest/m2/receive-pool.md | 9 + rs/moq-net/src/coding/reader.rs | 49 +++- rs/moq-net/src/ietf/subscriber.rs | 4 +- rs/moq-net/src/lite/subscriber.rs | 2 +- rs/moq-net/src/lite/test_transport.rs | 12 + rs/moq-net/src/model/frame.rs | 4 +- rs/moq-net/src/model/group.rs | 24 +- rs/moq-net/src/model/pace.rs | 49 +++- rs/moq-net/src/transport.rs | 38 +++ rs/moq-net/src/transport/boxed.rs | 237 +++++++++++++++--- rs/moq-net/src/transport/memory.rs | 5 + .../tests/integration/fastest_reader.rs | 33 ++- rs/moq-net/tests/integration/support/mock.rs | 15 ++ rs/moq-quic/src/connection/mod.rs | 11 + rs/moq-quic/src/connection/streams/state.rs | 11 + rs/moq-tokio/src/quic/driver/connection.rs | 7 + rs/moq-tokio/src/transport.rs | 4 + rs/moq-tokio/src/transport/grouped.rs | 6 + rs/moq-tokio/src/transport/owned.rs | 3 + rs/moq-uring/src/quic/driver/connection.rs | 6 + rs/moq-uring/src/quic/web.rs | 5 + rs/moq-uring/src/transport.rs | 4 + rs/moq-uring/src/transport/adapter.rs | 9 + rs/moq-uring/src/transport/local.rs | 4 + 27 files changed, 515 insertions(+), 73 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 76839683392..6beea8127c2 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -351,6 +351,9 @@ fastest one, then stops reading its stream until that reader catches up, so flow control slows the upstream hop and the publisher. A slower reader is never waited on: one that trails the fastest by more than the rest of the 32 MB reaches a shed frame. A group nobody reads keeps arriving and shedding. +A stopped stream keeps its receive credit, so the relay stops reading at most as +many streams per connection as fit in half of `quic.receive_window`; past that, +groups keep arriving and shedding as if unpaced. ## \[stats] diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 28ef313ff12..81665f9de76 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -387,7 +387,10 @@ that keeps up loses nothing. A slower reader is never waited on: one that trails the fastest by more than the group's 32 MB tail reaches a shed frame, and asks for a new group on tracks that opt into [refresh](#refresh) or ends with `Evicted`. A hop with no reader keeps receiving and shedding, so pacing never -stalls an unwatched track, and the publisher's own writes never wait. +stalls an unwatched track, and the publisher's own writes never wait unless it +asks to. A stopped stream keeps its receive credit, so a hop stops at most as +many streams per connection as fit in half its connection window, and the rest +keep receiving and shedding. ### Refresh diff --git a/quest/m1/unbounded-groups/fastest-reader.md b/quest/m1/unbounded-groups/fastest-reader.md index 1fdb340bf2a..f0f04b0ee72 100644 --- a/quest/m1/unbounded-groups/fastest-reader.md +++ b/quest/m1/unbounded-groups/fastest-reader.md @@ -65,6 +65,18 @@ Decided 2026-10-28 (maintainer, after an audit of the plan): lossless. Bring the mechanism to the maintainer before coding, and align it with the [receive pool](/quest/m2/receive-pool.md). +Decided 2026-10-30 (maintainer, in the +[moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388) iteration): + +- moq-net caps the receive credit paused groups hold per session at about half + the connection window, read live from the transport + (`poll::Session::receive_window`). A group that would push the total past + the cap keeps reading and shedding instead of pausing. This stands until the + [receive pool](/quest/m2/receive-pool.md) folds paused credit into its + per-worker budget. +- The readiness future is `group::Producer::poll_ready` and `async fn ready()`, + pending until a reader exists and the 16 MB window has room. + Prior art splits the same way (2026-10-28 survey): Varnish streaming, ATS read-while-writer, and WHATWG `tee()` fan out at the writer's pace and buffer everything; Varnish `transit_buffer`, nginx unbuffered proxying, and ATS flow @@ -81,17 +93,18 @@ moq-transport ingest both pause past the 16 MB window. A reader that took every committed frame counts as waiting on the writer, so a frame larger than the window never stalls it. Tests cover one hop on lite-06, lite-07-wip, and moq-transport-22: an idle reader holds the relay copy near the window and then -receives every frame, and a slow reader ends `Evicted` while the fast one keeps -up. +receives every frame, a slow reader ends `Evicted` while the fast one keeps +up, and a session past its credit cap reads on. The same PR landed the credit +cap (each paused stream reserves its stream window from a per-session budget +in `rs/moq-net/src/transport/boxed.rs`; moq-quic reports its windows through +moq-tokio and moq-uring, and a transport that reports none never pauses) and +the public readiness future. What remains: -- The receive-credit cap for paused streams above, which has to land before - pacing can reset nothing it means to keep lossless. Bring the mechanism to - the maintainer before coding. -- The readiness future for origin producers: its shape is an API decision to - bring to the maintainer before coding. Until it lands, a local byte-stream - writer more than 32 MB ahead of its session's egress sheds its own front. +- Receive-window reporting for the transports that report none yet (quinn, + iroh, qmux, WebTransport in the browser, and the FFI adapters), whose ingest + never pauses until they do. - JS parity. - A two-relay chain test with memory measured near the window. - Benchmarks of the hot path in `group.rs`: the cursor's report is one diff --git a/quest/m2/receive-pool.md b/quest/m2/receive-pool.md index ee1333748f4..25af3e931dc 100644 --- a/quest/m2/receive-pool.md +++ b/quest/m2/receive-pool.md @@ -46,6 +46,15 @@ Open, for the implementer to bring to the maintainer: to move to the cache without a window where the bytes are billed twice or not at all. +Idea for later (maintainer, 2026-10-30, from the +[fastest-reader pacing](/quest/m1/unbounded-groups/fastest-reader.md) iteration +in [moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388)): instead of +moq-quic granting `MAX_STREAM_DATA` automatically as a stream is read, the +application could pause a stream, or grant it an explicit +`allow(fastest + 16 MB)`, so flow control never grants credit past the pacing +window. Paused streams would then hold no credit beyond that window, replacing +moq-net's per-session cap on paused credit. + Lessons from other stacks: QUICHE frees 8 KiB blocks as they are consumed, msquic never advertises credit it could not back, and quinn and lsquic both let a slice pin its packet buffer. #5878 already handles the last with compaction. diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 126742310ea..b36891ddb8d 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -40,6 +40,31 @@ pub struct Reader { /// enough that an ordinary burst is still one wake. const WAKE_BUDGET: usize = 64 * 1024; +/// A stream ingest may stop reading to pace a group's writer, which holds its receive +/// credit meanwhile. +pub(crate) trait Pause { + /// Reserve the credit this stream may hold while paused, returning whether it may pause. + /// A stream outside any session always may. + fn pause(&mut self) -> bool { + true + } + + /// Return what [`Self::pause`] reserved, as the stream is read again. + fn resume(&mut self) {} +} + +impl Pause for crate::transport::boxed::RecvStream { + fn pause(&mut self) -> bool { + crate::transport::boxed::RecvStream::pause(self) + } + + fn resume(&mut self) { + crate::transport::boxed::RecvStream::resume(self) + } +} + +impl Pause for crate::transport::poll::Buffered {} + /// The largest [`Reader::scratch`] kept between decodes; one past this is freed. const SCRATCH_KEEP: usize = 64 * 1024; @@ -361,7 +386,7 @@ impl Reader { /// Fill a frame's payload from the stream, returning `Pending` once it would block or /// the group's writer is paced by its fastest reader (see - /// [`crate::frame::Producer::poll_ready`]). + /// [`crate::frame::Producer::poll_ready`]) and the stream may pause (see [`Pause`]). /// /// The payload reaches the frame as it arrives, but keeps its receive credit until the /// segment it lands in seals, so an incomplete frame holds at most one segment of the @@ -383,17 +408,27 @@ impl Reader { &mut self, waiter: &kio::Waiter, frame: &mut crate::frame::Producer, - ) -> Poll> { + ) -> Poll> + where + S: Pause, + { self.want = 0; let mut owed = 0; let result = loop { // A writer too far ahead of the group's fastest reader leaves the rest in the - // stream, so flow control pushes back upstream. - let paced = frame.poll_ready(waiter); + // stream, so flow control pushes back upstream, if the session has room for the + // credit the paused stream holds; otherwise it reads on and sheds. + let paced = match frame.poll_ready(waiter) { + Poll::Ready(()) => { + self.stream.resume(); + false + } + Poll::Pending => self.stream.pause(), + }; let lease = match paced { - Poll::Ready(()) => self.poll_lease(waiter, frame.remaining().min(1)), - Poll::Pending => Poll::Pending, + false => self.poll_lease(waiter, frame.remaining().min(1)), + true => Poll::Pending, }; let mut lease = match lease { Poll::Pending => { @@ -993,6 +1028,8 @@ mod tests { struct CreditedLease<'a>(&'a mut Credited); + impl Pause for Credited {} + impl RecvStream for Credited { type Error = crate::lite::test_transport::SinkError; type Lease<'a> = CreditedLease<'a>; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index f70b2ec62b2..346b725578b 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -5855,7 +5855,7 @@ struct FetchedObject { /// /// The `first` object on a stream has no prior to inherit from, so a field it leaves to /// the prior object is a protocol violation. -async fn decode_fetch_object( +async fn decode_fetch_object( stream: &mut Reader, first: bool, ) -> Result, Error> { @@ -5983,7 +5983,7 @@ impl GroupIngest { /// `Ready(Ok(_))` once the stream FINs on an object boundary, or an explicit /// end-of-group or end-of-track status arrives. The caller finishes or aborts the /// group; an object cut short mid-payload was already aborted here with the reason. - fn poll( + fn poll( &mut self, reader: &mut Reader, group: &mut crate::recv::Group, diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 8dd35ae830f..7b3fc33ab26 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1129,7 +1129,7 @@ impl FrameIngest { /// `Ready(Ok(()))` once the stream FINs on a frame boundary. The caller /// finishes or aborts the group; a frame cut short mid-payload was already /// aborted here with the reason. - fn poll( + fn poll( &mut self, reader: &mut Reader, group: &mut crate::recv::Group, diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index a17bf539c68..21d2fc592de 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -613,6 +613,8 @@ pub struct SinkSession { stats: Arc>, /// Set by [`Self::with_probe`]. probe: Option, + /// Set by [`Self::with_receive_window`]. + receive_window: Option, } impl SinkSession { @@ -642,6 +644,12 @@ impl SinkSession { self } + /// Report `window` as the receive windows in force. + pub fn with_receive_window(mut self, window: crate::transport::ReceiveWindow) -> Self { + self.receive_window = Some(window); + self + } + /// Give the transport a sender for the session to hand its probe target. pub fn with_probe(mut self) -> Self { self.probe = Some(crate::transport::Probe::new()); @@ -827,6 +835,10 @@ impl poll::Session for SinkSession { fn probe(&self) -> Option { self.probe.clone() } + + fn receive_window(&self) -> Option { + self.receive_window + } } /// Connection statistics a test can dictate. Every metric defaults to unknown, diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index b0890923726..3e3141a7a5e 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1189,11 +1189,11 @@ impl Producer { } /// Poll until the group's writer may take more of this frame from upstream; see - /// [`group::Producer::poll_ready`]. + /// [`group::Producer::poll_paced`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_ready(&self, waiter: &kio::Waiter) -> Poll<()> { self.group() - .poll_ready(waiter, self.buf.written(Ordering::Acquire) as u64) + .poll_paced(waiter, self.buf.written(Ordering::Acquire) as u64) } /// Publish what has been written so far, waking consumers parked on the group. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index f6e204ff875..3b144491ce0 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2217,6 +2217,25 @@ impl Producer { pub async fn closed(&self) -> Result<()> { kio::wait(|waiter| self.handle.poll_closed(waiter)).await } + /// Poll until a reader holds the group and the writer is at most 16 MB past the end of + /// the last frame its fastest reader took, so a local writer runs no further ahead than + /// the live tail keeps behind that reader. + /// + /// Ready once the group is closed, so the next write surfaces why. A reader that took + /// every committed frame is waiting on the writer, so it never holds the writer back, + /// and one whose next frame was shed has fallen off the group and is not waited on. + /// Writing without waiting never fails: the group sheds its front instead. + pub fn poll_ready(&self, waiter: &kio::Waiter) -> Poll<()> { + ready!(self.poll_paced(waiter, 0)); + // A closed group reports `Closed`, which is ready too. + self.handle.state.poll_used(waiter).map(|_| ()) + } + + /// Wait until a reader holds the group and the writer has room; see [`Self::poll_ready`]. + pub async fn ready(&self) { + kio::wait(|waiter| self.poll_ready(waiter)).await + } + /// Poll until the writer may take more from upstream, `partial` bytes into a frame /// it has not committed: until it is at most [`super::pace::AHEAD`] past the end of the last /// frame its fastest reader took. @@ -2227,9 +2246,8 @@ impl Producer { /// larger than the window never stalls it, and one whose next frame was shed has /// fallen off the group and is not waited on. /// - /// Only a paced writer, a session's ingest, waits on this; a local write never does. - #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_ready(&self, waiter: &kio::Waiter, partial: u64) -> Poll<()> { + /// A session's ingest waits on this; a local writer waits on [`Self::poll_ready`]. + pub(crate) fn poll_paced(&self, waiter: &kio::Waiter, partial: u64) -> Poll<()> { let pace = &self.handle.alive.pace; if !pace.ahead(partial) { return Poll::Ready(()); diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 0baf039c84d..13114219e10 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -172,16 +172,16 @@ mod tests { let mut group = written(15); let mut fast = group.consume(); let _slow = group.consume(); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(()), "within the window"); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(()), "within the window"); group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending, "past the window"); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Pending, "past the window"); assert!(!flag.woken()); // A batch read takes several frames at once, which the writer counts as read. read(&mut fast); assert!(flag.woken(), "the fastest reader's read wakes the writer"); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } /// Bytes measure the window, not frames, so an open frame counts as it fills. @@ -207,17 +207,42 @@ mod tests { fn a_group_without_readers_never_waits() { let (flag, waiter) = flag(); let group = written(16); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); let reader = group.consume(); assert_eq!( - group.poll_ready(&waiter, 0), + group.poll_paced(&waiter, 0), Poll::Pending, "a reader that took nothing" ); drop(reader); assert!(flag.woken(), "the last reader leaving wakes the writer"); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); + } + + /// A local writer waits for a reader before it writes, then for room in the window, and + /// a reader arriving or reading wakes it. + #[test] + fn a_local_writer_waits_for_a_reader_and_room() { + let (flag, waiter) = flag(); + let mut group = group::Info { sequence: 0 }.produce_untimed(); + assert_eq!(group.poll_ready(&waiter), Poll::Pending, "nobody reads yet"); + let mut reader = group.consume(); + assert!(flag.woken(), "a reader arriving wakes the writer"); + assert_eq!(group.poll_ready(&waiter), Poll::Ready(())); + + for _ in 0..17 { + group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + } + assert_eq!(group.poll_ready(&waiter), Poll::Pending, "past the window"); + read(&mut reader); + assert!(flag.woken(), "the reader's read wakes the writer"); + assert_eq!(group.poll_ready(&waiter), Poll::Ready(())); + + drop(reader); + assert_eq!(group.poll_ready(&waiter), Poll::Pending, "the reader left"); + group.handle().abort(Error::Cancel).unwrap(); + assert_eq!(group.poll_ready(&waiter), Poll::Ready(()), "a closed group never waits"); } /// A reader that took every committed frame is waiting on the writer, so a frame @@ -251,7 +276,7 @@ mod tests { for _ in 0..40 { group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); } - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } /// A reader that skipped past what the writer has written is waiting on it, so it @@ -267,7 +292,7 @@ mod tests { } // Each commit wakes the parked reader, which polls again and reports the edge. assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } /// Shedding past the fastest reader under cache pressure wakes a parked writer: that @@ -277,10 +302,10 @@ mod tests { let (flag, waiter) = flag(); let group = written(17); let _reader = group.consume(); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Pending); group.handle().shed(u64::MAX, u64::MAX); assert!(flag.woken()); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } /// An abort wakes a parked writer, so its next write surfaces why. @@ -289,9 +314,9 @@ mod tests { let (flag, waiter) = flag(); let group = written(17); let _reader = group.consume(); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Pending); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Pending); group.handle().abort(Error::Cancel).unwrap(); assert!(flag.woken()); - assert_eq!(group.poll_ready(&waiter, 0), Poll::Ready(())); + assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } } diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 72177a1044e..ed227c0a864 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -166,6 +166,30 @@ pub trait Metrics { struct Untracked; impl Metrics for Untracked {} +/// The receive credit a transport grants its peer, which bounds what a stream the session +/// stops reading still holds. +/// +/// A session's ingest stops reading a group's stream while the group's fastest reader is far +/// behind, so flow control pushes back to the publisher. Such a paused stream keeps up to +/// `stream` bytes counted against the shared `connection` window, so the session pauses only +/// as many streams as fit in half of it; the rest keep reading. See +/// [`poll::Session::receive_window`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct ReceiveWindow { + /// The connection-wide receive window, in bytes, or [`u64::MAX`] when streams share none. + pub connection: u64, + /// The most receive credit one stream may hold, in bytes. + pub stream: u64, +} + +impl ReceiveWindow { + /// A connection-wide window of `connection` bytes, of which one stream holds at most `stream`. + pub fn new(connection: u64, stream: u64) -> Self { + Self { connection, stream } + } +} + /// The smallest connection receive window a transport that leases may offer. /// /// A session keeps each unit it is still assembling leased, and a unit larger than the @@ -355,6 +379,16 @@ pub mod poll { None } + /// Return the receive windows now in force, or `None` when the transport cannot say. + /// + /// Read whenever the session considers pausing a stream, so a window changed on a live + /// connection is followed. Without one, the session cannot bound the credit a paused + /// stream holds, so it never pauses one: ingest reads every stream as it arrives and + /// sheds what its readers fall behind on. + fn receive_window(&self) -> Option { + None + } + /// Accept the next unidirectional stream opened by the peer. fn accept_uni(&self) -> AcceptUni<'_, Self> { AcceptUni::new(self) @@ -563,6 +597,10 @@ pub mod poll { fn probe(&self) -> Option { (**self).probe() } + + fn receive_window(&self) -> Option { + (**self).receive_window() + } } /// A transport that the async client and server can box: diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 4752b98a2e8..777788a7e6b 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -6,17 +6,19 @@ //! awaits them. Each transport only instantiates the thin forwarding impls below. use std::any::Any; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; use std::task::Poll; use bytes::{Buf, Bytes}; use super::poll::{self, SendOptions}; -use super::{Address, CloseKind, MaybeSend, MaybeSync, Probe, Wire}; +use super::{Address, CloseKind, MaybeSend, MaybeSync, Probe, ReceiveWindow, Wire}; /// A boxed [`poll::Boxable`] session, shared by every task that uses it. #[derive(Clone)] pub(crate) struct Session { - inner: std::sync::Arc, + inner: Arc>, /// Shared by every clone, so whichever task closes the transport first says why. closing: std::sync::Arc, } @@ -24,11 +26,19 @@ pub(crate) struct Session { impl Session { pub(crate) fn new(session: S) -> Self { Self { - inner: std::sync::Arc::new(session), + inner: Arc::new(Shared { + paused: AtomicU64::new(0), + session, + }), closing: Default::default(), } } + /// The handle each stream this session yields reserves its paused credit through. + fn budget(&self) -> Budget { + Arc::downgrade(&self.inner) + } + /// How this side closed the transport. pub(crate) fn closing(&self) -> &std::sync::Arc { &self.closing @@ -37,10 +47,38 @@ impl Session { /// Close with `code` over `cause`, this side's reason for ending the session. pub(crate) fn abort(&self, cause: &crate::Error, code: u32, reason: &str) { self.closing.sent(code, Some(cause)); - self.inner.close(code, reason); + self.inner.session.close(code, reason); + } +} + +/// A boxed transport and the receive credit its paused streams hold. +struct Shared { + /// The stream credit reserved by streams this session stopped reading to pace a group's + /// writer, at most half the connection window; see [`ReceiveWindow`]. + paused: AtomicU64, + session: S, +} + +impl Shared { + /// Reserve the credit of one more paused stream, unless it would push the paused total + /// over half the connection window, returning how much it reserved. + fn reserve(&self) -> Option { + let window = self.session.receive_window()?; + let charge = window.stream.min(window.connection); + let cap = window.connection / 2; + self.paused + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |paused| { + paused.checked_add(charge).filter(|total| *total <= cap) + }) + .ok()?; + Some(charge) } } +/// The session a stream reserves paused credit from, which a stream outliving it never keeps +/// open: once it is gone, nothing is paused. +type Budget = Weak>; + /// A boxed transport error, keeping both code registries and the message. #[derive(Debug)] pub(crate) struct Error(Box); @@ -107,9 +145,56 @@ impl SendStream { pub(crate) struct RecvStream(Box); impl RecvStream { + /// Box a stream outside any session, which therefore never pauses. #[cfg(test)] pub(crate) fn new(stream: impl poll::RecvStream + MaybeSend + MaybeSync) -> Self { - Self(Box::new(stream)) + Self::budgeted(stream, Weak::>::new()) + } + + fn budgeted(stream: impl poll::RecvStream + MaybeSend + MaybeSync, session: Budget) -> Self { + Self(Box::new(Budgeted { + stream, + session, + held: 0, + })) + } + + /// Reserve the receive credit this stream may hold while ingest stops reading it, returning + /// whether it may stop: not once the session's paused streams hold half its connection + /// window, nor on a transport that reports no window. + pub(crate) fn pause(&mut self) -> bool { + self.0.pause() + } + + /// Return the credit [`Self::pause`] reserved, as ingest reads the stream again. + pub(crate) fn resume(&mut self) { + self.0.resume() + } +} + +/// A transport's receive stream and the paused credit it reserved from its session. +struct Budgeted { + stream: T, + session: Budget, + /// The credit reserved while paused, zero while read. + held: u64, +} + +impl Budgeted { + /// Return the credit reserved while paused, if any. + fn release(&mut self) { + let held = std::mem::take(&mut self.held); + if held > 0 + && let Some(shared) = self.session.upgrade() + { + shared.paused.fetch_sub(held, Ordering::AcqRel); + } + } +} + +impl Drop for Budgeted { + fn drop(&mut self) { + self.release(); } } @@ -167,18 +252,20 @@ impl super::Metrics for Metrics { } trait ErasedSession: MaybeSend + MaybeSync { - fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll>; - fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll>; + fn poll_accept_uni(&self, waiter: &kio::Waiter, budget: Budget) -> Poll>; + fn poll_accept_bi(&self, waiter: &kio::Waiter, budget: Budget) -> Poll>; fn poll_open_uni(&self, waiter: &kio::Waiter, options: SendOptions<'_, Group>) -> Poll>; fn poll_open_bi( &self, waiter: &kio::Waiter, options: SendOptions<'_, Group>, + budget: Budget, ) -> Poll>; fn poll_open_request( &self, waiter: &kio::Waiter, options: SendOptions<'_, Group>, + budget: Budget, ) -> Poll, Error>>; fn send_group(&self, priority: i64) -> Option; fn send_datagram(&self, payload: Bytes, options: SendOptions<'_, Group>) -> Result<(), Error>; @@ -191,6 +278,7 @@ trait ErasedSession: MaybeSend + MaybeSync { fn metrics(&self) -> Metrics; fn wire(&self) -> Option; fn probe(&self) -> Option; + fn receive_window(&self) -> Option; } trait ErasedGroup: MaybeSend + MaybeSync { @@ -214,6 +302,8 @@ trait ErasedRecv: MaybeSend + MaybeSync { fn poll_read_chunk(&mut self, waiter: &kio::Waiter, max: usize) -> Poll, Error>>; fn set_priority(&mut self, priority: i64); fn stop(&mut self, code: u32); + fn pause(&mut self) -> bool; + fn resume(&mut self); } // Not `MaybeSend`: a lease may hold its transport's lock, and never outlives a poll. @@ -239,18 +329,18 @@ fn unbox<'a, G: 'static>(options: SendOptions<'a, Group>) -> SendOptions<'a, G> } } -fn bi((send, recv): poll::BiStreams) -> (SendStream, RecvStream) { - (SendStream(Box::new(send)), RecvStream(Box::new(recv))) +fn bi((send, recv): poll::BiStreams, budget: Budget) -> (SendStream, RecvStream) { + (SendStream(Box::new(send)), RecvStream::budgeted(recv, budget)) } impl ErasedSession for S { - fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll> { + fn poll_accept_uni(&self, waiter: &kio::Waiter, budget: Budget) -> Poll> { poll::Session::poll_accept_uni(self, waiter) - .map(|res| res.map(|recv| RecvStream(Box::new(recv))).map_err(Error::new)) + .map(|res| res.map(|recv| RecvStream::budgeted(recv, budget)).map_err(Error::new)) } - fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll> { - poll::Session::poll_accept_bi(self, waiter).map(|res| res.map(bi::).map_err(Error::new)) + fn poll_accept_bi(&self, waiter: &kio::Waiter, budget: Budget) -> Poll> { + poll::Session::poll_accept_bi(self, waiter).map(|res| res.map(|bis| bi::(bis, budget)).map_err(Error::new)) } fn poll_open_uni(&self, waiter: &kio::Waiter, options: SendOptions<'_, Group>) -> Poll> { @@ -262,17 +352,20 @@ impl ErasedSession for S { &self, waiter: &kio::Waiter, options: SendOptions<'_, Group>, + budget: Budget, ) -> Poll> { - poll::Session::poll_open_bi(self, waiter, unbox(options)).map(|res| res.map(bi::).map_err(Error::new)) + poll::Session::poll_open_bi(self, waiter, unbox(options)) + .map(|res| res.map(|bis| bi::(bis, budget)).map_err(Error::new)) } fn poll_open_request( &self, waiter: &kio::Waiter, options: SendOptions<'_, Group>, + budget: Budget, ) -> Poll, Error>> { poll::Session::poll_open_request(self, waiter, unbox(options)) - .map(|res| res.map(|streams| streams.map(bi::)).map_err(Error::new)) + .map(|res| res.map(|streams| streams.map(|bis| bi::(bis, budget))).map_err(Error::new)) } fn send_group(&self, priority: i64) -> Option { @@ -318,6 +411,10 @@ impl ErasedSession for S { fn probe(&self) -> Option { poll::Session::probe(self) } + + fn receive_window(&self) -> Option { + poll::Session::receive_window(self) + } } impl ErasedGroup for G { @@ -370,23 +467,38 @@ impl + MaybeSend + MaybeSync> } } -impl ErasedRecv for T { +impl ErasedRecv for Budgeted { fn poll_lease(&mut self, waiter: &kio::Waiter, want: usize) -> Poll, Error>> { - poll::RecvStream::poll_lease(self, waiter, want) + poll::RecvStream::poll_lease(&mut self.stream, waiter, want) .map(|res| res.map(|lease| Lease(Box::new(lease))).map_err(Error::new)) } // Forwarded rather than left to the provided method, so reading a chunk never boxes a lease. fn poll_read_chunk(&mut self, waiter: &kio::Waiter, max: usize) -> Poll, Error>> { - poll::RecvStream::poll_read_chunk(self, waiter, max).map_err(Error::new) + poll::RecvStream::poll_read_chunk(&mut self.stream, waiter, max).map_err(Error::new) } fn set_priority(&mut self, priority: i64) { - poll::RecvStream::set_priority(self, priority) + poll::RecvStream::set_priority(&mut self.stream, priority) } fn stop(&mut self, code: u32) { - poll::RecvStream::stop(self, code) + poll::RecvStream::stop(&mut self.stream, code) + } + + fn pause(&mut self) -> bool { + if self.held > 0 { + return true; + } + let Some(held) = self.session.upgrade().and_then(|shared| shared.reserve()) else { + return false; + }; + self.held = held; + true + } + + fn resume(&mut self) { + self.release(); } } @@ -410,15 +522,15 @@ impl poll::Session for Session { type Error = Error; fn poll_accept_uni(&self, waiter: &kio::Waiter) -> Poll> { - self.inner.poll_accept_uni(waiter) + self.inner.session.poll_accept_uni(waiter, self.budget()) } fn poll_accept_bi(&self, waiter: &kio::Waiter) -> Poll> { - self.inner.poll_accept_bi(waiter) + self.inner.session.poll_accept_bi(waiter, self.budget()) } fn poll_open_uni(&self, waiter: &kio::Waiter, options: SendOptions<'_, Group>) -> Poll> { - self.inner.poll_open_uni(waiter, options) + self.inner.session.poll_open_uni(waiter, options) } fn poll_open_bi( @@ -426,7 +538,7 @@ impl poll::Session for Session { waiter: &kio::Waiter, options: SendOptions<'_, Group>, ) -> Poll> { - self.inner.poll_open_bi(waiter, options) + self.inner.session.poll_open_bi(waiter, options, self.budget()) } fn poll_open_request( @@ -434,52 +546,56 @@ impl poll::Session for Session { waiter: &kio::Waiter, options: SendOptions<'_, Group>, ) -> Poll, Error>> { - self.inner.poll_open_request(waiter, options) + self.inner.session.poll_open_request(waiter, options, self.budget()) } fn send_group(&self, priority: i64) -> Option { - self.inner.send_group(priority) + self.inner.session.send_group(priority) } fn send_datagram(&self, payload: Bytes, options: SendOptions<'_, Group>) -> Result<(), Error> { - self.inner.send_datagram(payload, options) + self.inner.session.send_datagram(payload, options) } fn poll_recv_datagram(&self, waiter: &kio::Waiter) -> Poll> { - self.inner.poll_recv_datagram(waiter) + self.inner.session.poll_recv_datagram(waiter) } fn max_datagram_size(&self) -> usize { - self.inner.max_datagram_size() + self.inner.session.max_datagram_size() } fn protocol(&self) -> Option<&str> { - self.inner.protocol() + self.inner.session.protocol() } fn address(&self) -> Address { - self.inner.address() + self.inner.session.address() } fn close(&self, code: u32, reason: &str) { self.closing.sent(code, None); - self.inner.close(code, reason) + self.inner.session.close(code, reason) } fn poll_closed(&self, waiter: &kio::Waiter) -> Poll { - self.inner.poll_closed(waiter) + self.inner.session.poll_closed(waiter) } fn metrics(&self) -> impl super::Metrics { - self.inner.metrics() + self.inner.session.metrics() } fn wire(&self) -> Option { - self.inner.wire() + self.inner.session.wire() } fn probe(&self) -> Option { - self.inner.probe() + self.inner.session.probe() + } + + fn receive_window(&self) -> Option { + self.inner.session.receive_window() } } @@ -569,3 +685,52 @@ impl poll::Lease for Lease<'_> { self.0.release() } } + +#[cfg(test)] +mod test { + use super::*; + use crate::lite::test_transport::SinkSession; + + fn accept(session: &Session) -> RecvStream { + let Poll::Ready(Ok((_, recv))) = poll::Session::poll_accept_bi(session, &kio::Waiter::noop()) else { + panic!("the sink accepts at once"); + }; + recv + } + + /// Each paused stream reserves its stream window, and the session pauses no more than + /// fit in half the connection window; a stream resuming or dropped returns its share. + #[test] + fn paused_streams_hold_at_most_half_the_connection_window() { + let gate = kio::Producer::new(true); + let session = Session::new( + SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20)), + ); + let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); + assert!(streams[0].pause()); + assert!(streams[0].pause(), "a paused stream reserves once"); + assert!(streams[1].pause()); + assert!(!streams[2].pause(), "a third would hold 60 of 100"); + + streams[0].resume(); + assert!(streams[2].pause(), "a resumed stream returned its share"); + drop(streams.remove(1)); + assert!(accept(&session).pause(), "so did a dropped one"); + } + + /// A transport that reports no window never pauses a stream, since nothing bounds what + /// it would hold; nor does a stream that outlived its session. + #[test] + fn an_unbounded_stream_never_pauses() { + let gate = kio::Producer::new(true); + let session = Session::new(SinkSession::accepted_bi(gate.consume())); + assert!(!accept(&session).pause()); + + let session = Session::new( + SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20)), + ); + let mut stream = accept(&session); + drop(session); + assert!(!stream.pause()); + } +} diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 09ea66b6fcf..fb273b4ea8e 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -404,6 +404,11 @@ impl poll::Session for Session { fn poll_closed(&self, waiter: &kio::Waiter) -> Poll { poll_close(&self.side.conn.close, waiter) } + + /// Streams share no connection window, and one holds at most its buffer. + fn receive_window(&self) -> Option { + Some(super::ReceiveWindow::new(u64::MAX, self.conn().config.stream_buffer as u64)) + } } /// One direction of a stream: the messages its writer queued for its reader. diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index 7bc2fed84b3..b1b8e72fb4f 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -28,7 +28,7 @@ struct Relay { track: track::Producer, subscriber: track::Subscriber, _broadcast: origin::Publication, - _pair: MockPair, + pair: MockPair, _origins: [origin::Producer; 2], } @@ -61,7 +61,7 @@ async fn relay(version: &str) -> Relay { track, subscriber, _broadcast: broadcast, - _pair: pair, + pair, _origins: [publisher, relay], } } @@ -108,6 +108,35 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { } } +/// A session whose paused streams would hold more than half its connection window reads +/// on instead, and its idle reader keeps what the live tail holds. +#[moq_clock::test] +async fn a_session_past_its_credit_cap_reads_on() { + let payload = Bytes::from(vec![7u8; MB]); + for version in VERSIONS { + let mut relay = relay(version).await; + // One paused stream's credit is already more than half the connection window. + let window = moq_net::transport::ReceiveWindow::new(MB as u64, MB as u64); + relay.pair.server_transport.set_receive_window(Some(window)); + + let mut group = relay.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("group 0"); + for _ in 1..30 { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + group.finish().unwrap(); + + moq_clock::sleep(Duration::from_secs(1)).await; + let held = reader.frame_count(); + assert_eq!(held, 30, "{version}: the relay paused past its credit cap"); + } +} + #[moq_clock::test] async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { let payload = Bytes::from(vec![7u8; MB]); diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 1b96c550eae..4f2188cbb9e 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -454,6 +454,8 @@ struct Side { panic_on_stats: AtomicBool, /// The wire counters [`poll::Session::wire`] exposes, which a test updates as a transport would. wire: moq_net::transport::Wire, + /// What [`poll::Session::receive_window`] reports instead of the memory transport's, if set. + receive_window: Mutex>>, } /// The connection statistics a [`MockSession`] reports. Unavailable by default. @@ -643,6 +645,13 @@ impl poll::Session for MockSession { fn wire(&self) -> Option { Some(self.side.wire.clone()) } + + fn receive_window(&self) -> Option { + match *self.side.receive_window.lock().unwrap() { + Some(window) => window, + None => self.inner.receive_window(), + } + } } #[allow(dead_code, reason = "only some test binaries steer delivery")] @@ -676,6 +685,11 @@ impl MockSession { *self.side.stats.lock().unwrap() = stats; } + /// Report `window` as the receive windows in force, as a transport whose limits changed. + pub fn set_receive_window(&self, window: Option) { + *self.side.receive_window.lock().unwrap() = Some(window); + } + /// How many times the stats were read. pub fn stats_reads(&self) -> usize { self.side.stats_reads.load(Ordering::Relaxed) @@ -871,6 +885,7 @@ pub fn create_mock_session_pair(config: memory::Config) -> (MockSession, MockSes stats: Default::default(), stats_reads: Default::default(), panic_on_stats: Default::default(), + receive_window: Default::default(), wire: Default::default(), }), }; diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 53f8616e277..7fa3be2e6da 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1559,6 +1559,17 @@ impl Connection { } } + /// The connection-wide receive window now in force; see [`Limits::receive_window`] + pub fn receive_window(&self) -> u64 { + self.streams.receive_window() + } + + /// The most receive credit one stream may hold: the per-stream window now in force, or the + /// larger one a stream may have started with; see [`Limits::stream_receive_window`] + pub fn stream_receive_window(&self) -> u64 { + self.streams.stream_receive_window() + } + /// Change every limit `limits` sets, leaving the others as they are /// /// Errors, changing nothing, on limits [`Limits::validate`] refuses. diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index ce66d36c3c6..1f207b744bd 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -1063,6 +1063,17 @@ impl StreamsState { self.send_window = send_window; } + /// The connection receive window now in force. + pub(crate) fn receive_window(&self) -> u64 { + self.receive_window + } + + /// The most credit one receive stream may hold: the window now in force, or the larger one + /// a new stream started with. + pub(crate) fn stream_receive_window(&self) -> u64 { + self.stream_receive_window.max(self.initial_stream_receive_window) + } + /// Set the receive window of each stream, which applies as each stream's credit is next /// extended: credit already granted cannot be taken back. pub(crate) fn set_stream_receive_window(&mut self, window: u64) { diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 1772bcb3046..b9cd4a99d86 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -503,6 +503,13 @@ impl Connection { self.0.state.lock().inner.stats() } + /// The receive windows now in force: the connection's, and the most one stream holds. + #[cfg(feature = "net")] + pub(crate) fn receive_window(&self) -> moq_net::transport::ReceiveWindow { + let state = self.0.state.lock(); + moq_net::transport::ReceiveWindow::new(state.inner.receive_window(), state.inner.stream_receive_window()) + } + /// How many send streams the connection still holds state for #[cfg(test)] pub(crate) fn send_streams(&self) -> usize { diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 4d1ca90fd27..93feb836175 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -828,6 +828,10 @@ impl Grouped for crate::quic::web::Session { Some(crate::quic::driver::connection::Connection::stats(self).credit.evicted) } + fn receive_window(&self) -> Option { + Some(crate::quic::driver::connection::Connection::receive_window(self)) + } + fn send_group(&self, priority: i64) -> Option { Some(Self::send_group(self, priority)) } diff --git a/rs/moq-tokio/src/transport/grouped.rs b/rs/moq-tokio/src/transport/grouped.rs index 4a10d7ecddd..89cc0c54089 100644 --- a/rs/moq-tokio/src/transport/grouped.rs +++ b/rs/moq-tokio/src/transport/grouped.rs @@ -42,6 +42,12 @@ pub trait Grouped: web_transport_trait::Session { None } + /// The receive windows now in force, or `None` when the backend cannot say; see + /// [`poll::Session::receive_window`](moq_net::transport::poll::Session::receive_window). + fn receive_window(&self) -> Option { + None + } + /// Open a send group at `priority`, or `None` when the backend has none. fn send_group(&self, priority: i64) -> Option<::Group> { let _ = priority; diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 9ded0bbba8b..b76dc375504 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -200,6 +200,9 @@ impl> poll::Session fn probe(&self) -> Option { super::Grouped::probe(&self.session) } + fn receive_window(&self) -> Option { + super::Grouped::receive_window(&self.session) + } } impl poll::SendStream for BridgedSend { type Error = Error; diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 0ac224dcdbd..97312e343ea 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -375,6 +375,12 @@ impl Connection { pub(crate) fn streams_evicted(&self) -> u64 { self.shared.conn.borrow().stats().credit.evicted } + + /// The receive windows now in force: the connection's, and the most one stream holds. + pub(crate) fn receive_window(&self) -> moq_net::transport::ReceiveWindow { + let conn = self.shared.conn.borrow(); + moq_net::transport::ReceiveWindow::new(conn.receive_window(), conn.stream_receive_window()) + } } impl Clone for Connection { diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 8970f1ab407..f6a2815cd2e 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -459,6 +459,11 @@ impl Session { self.conn.streams_evicted() } + /// The receive windows now in force: the connection's, and the most one stream holds. + pub(crate) fn receive_window(&self) -> moq_net::transport::ReceiveWindow { + self.conn.receive_window() + } + /// Open a send group at `priority`, for streams to join with /// [`SendStream::set_group`]. The HTTP/3 streams stay outside every /// group, so they go first. diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 6b043479fd7..13e9bf5e977 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -37,6 +37,10 @@ impl Grouped for quic::web::Session { Some(quic::web::Session::streams_evicted(self)) } + fn receive_window(&self) -> Option { + Some(quic::web::Session::receive_window(self)) + } + fn send_group(&mut self, priority: i64) -> Option { Some(quic::web::Session::send_group(self, priority)) } diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index d54a97220f1..db65d110dc6 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -89,6 +89,12 @@ pub trait Grouped: backend::Session { None } + /// The receive windows now in force, or `None` when the backend cannot say; see + /// [`poll::Session::receive_window`](moq_net::transport::poll::Session::receive_window). + fn receive_window(&self) -> Option { + None + } + /// Open a send group at `priority`, or `None` when the backend has none. fn send_group(&mut self, priority: i64) -> Option<::Group> { let _ = priority; @@ -254,6 +260,9 @@ impl + 'static> poll::Sessi fn probe(&self) -> Option { Grouped::probe(&self.session) } + fn receive_window(&self) -> Option { + Grouped::receive_window(&self.session) + } } /// An outgoing WebTransport stream adapted to moq-net. /// diff --git a/rs/moq-uring/src/transport/local.rs b/rs/moq-uring/src/transport/local.rs index 571cbc2b246..ed476d36937 100644 --- a/rs/moq-uring/src/transport/local.rs +++ b/rs/moq-uring/src/transport/local.rs @@ -119,6 +119,10 @@ impl poll::Session for Local { fn probe(&self) -> Option { self.0.probe() } + + fn receive_window(&self) -> Option { + self.0.receive_window() + } } impl poll::SendStream for Local { From 73a2984a522ca3080f530b6c53beec813ce3c329 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:38:36 -0700 Subject: [PATCH 07/16] style(net): rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/transport/boxed.rs | 16 ++++++++-------- rs/moq-net/src/transport/memory.rs | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 777788a7e6b..5db1c8ca6e0 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -364,8 +364,10 @@ impl ErasedSession for S { options: SendOptions<'_, Group>, budget: Budget, ) -> Poll, Error>> { - poll::Session::poll_open_request(self, waiter, unbox(options)) - .map(|res| res.map(|streams| streams.map(|bis| bi::(bis, budget))).map_err(Error::new)) + poll::Session::poll_open_request(self, waiter, unbox(options)).map(|res| { + res.map(|streams| streams.map(|bis| bi::(bis, budget))) + .map_err(Error::new) + }) } fn send_group(&self, priority: i64) -> Option { @@ -703,9 +705,8 @@ mod test { #[test] fn paused_streams_hold_at_most_half_the_connection_window() { let gate = kio::Producer::new(true); - let session = Session::new( - SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20)), - ); + let session = + Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20))); let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); assert!(streams[0].pause()); assert!(streams[0].pause(), "a paused stream reserves once"); @@ -726,9 +727,8 @@ mod test { let session = Session::new(SinkSession::accepted_bi(gate.consume())); assert!(!accept(&session).pause()); - let session = Session::new( - SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20)), - ); + let session = + Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20))); let mut stream = accept(&session); drop(session); assert!(!stream.pause()); diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index fb273b4ea8e..ec801e64062 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -407,7 +407,10 @@ impl poll::Session for Session { /// Streams share no connection window, and one holds at most its buffer. fn receive_window(&self) -> Option { - Some(super::ReceiveWindow::new(u64::MAX, self.conn().config.stream_buffer as u64)) + Some(super::ReceiveWindow::new( + u64::MAX, + self.conn().config.stream_buffer as u64, + )) } } From 2c968360338fb4d51082328d7994c18b96dfb65b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:52:09 -0700 Subject: [PATCH 08/16] fix(net): admit a frame only once its size fits the pace window Admitting a frame trims the live tail to fit its declared size, so a large frame could shed what the fastest reader still wants before the payload gate ran. Ingest now waits on the declared size first, under the same per-session credit cap. moq-quic reports the largest stream window ever in force, since credit granted outlives a shrink. Co-Authored-By: Claude Opus 5.5 --- quest/m1/unbounded-groups/fastest-reader.md | 8 +++- rs/moq-net/src/coding/reader.rs | 44 +++++++++++++----- rs/moq-net/src/ietf/subscriber.rs | 16 +++++-- rs/moq-net/src/lite/subscriber.rs | 21 +++++++-- rs/moq-net/src/model/group.rs | 3 +- rs/moq-net/src/model/pace.rs | 17 +++++++ rs/moq-net/src/recv.rs | 9 ++++ .../tests/integration/fastest_reader.rs | 46 ++++++++++++++++++- rs/moq-quic/src/connection/mod.rs | 4 +- rs/moq-quic/src/connection/streams/state.rs | 22 +++++++-- 10 files changed, 161 insertions(+), 29 deletions(-) diff --git a/quest/m1/unbounded-groups/fastest-reader.md b/quest/m1/unbounded-groups/fastest-reader.md index f0f04b0ee72..1779b5439e2 100644 --- a/quest/m1/unbounded-groups/fastest-reader.md +++ b/quest/m1/unbounded-groups/fastest-reader.md @@ -88,8 +88,9 @@ Landed in [moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388) (Rust, session ingest only): readers report the byte end of each frame they take (`Pace` in `rs/moq-net/src/model/pace.rs`), and the shared transport reader's payload loop (`coding::Reader::poll_read_frame`) waits on -`group::Producer::poll_ready` before taking more, so moq-lite and -moq-transport ingest both pause past the 16 MB window. A reader that took every +`group::Producer::poll_paced` before taking more, and admits a frame only once +its declared size fits the window (`coding::Reader::poll_admit`), so moq-lite +and moq-transport ingest both pause past the 16 MB window. A reader that took every committed frame counts as waiting on the writer, so a frame larger than the window never stalls it. Tests cover one hop on lite-06, lite-07-wip, and moq-transport-22: an idle reader holds the relay copy near the window and then @@ -105,6 +106,9 @@ What remains: - Receive-window reporting for the transports that report none yet (quinn, iroh, qmux, WebTransport in the browser, and the FFI adapters), whose ingest never pauses until they do. +- A sized readiness check for local writers: `group::Producer::ready()` + waits for room but not for the next frame's size, so a local frame larger + than the room left still sheds what the egress has yet to send. - JS parity. - A two-relay chain test with memory measured near the window. - Benchmarks of the hot path in `group.rs`: the cursor's report is one diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index b36891ddb8d..e1d3209bdbc 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -417,18 +417,10 @@ impl Reader { let result = loop { // A writer too far ahead of the group's fastest reader leaves the rest in the - // stream, so flow control pushes back upstream, if the session has room for the - // credit the paused stream holds; otherwise it reads on and sheds. - let paced = match frame.poll_ready(waiter) { - Poll::Ready(()) => { - self.stream.resume(); - false - } - Poll::Pending => self.stream.pause(), - }; - let lease = match paced { - false => self.poll_lease(waiter, frame.remaining().min(1)), - true => Poll::Pending, + // stream, so flow control pushes back upstream. + let lease = match self.poll_pace(frame.poll_ready(waiter)) { + Poll::Ready(()) => self.poll_lease(waiter, frame.remaining().min(1)), + Poll::Pending => Poll::Pending, }; let mut lease = match lease { Poll::Pending => { @@ -482,6 +474,34 @@ impl Reader { Poll::Ready(result) } + /// Poll until `group`'s writer may admit a frame of `size` bytes from this stream: until + /// that leaves no byte its fastest reader has yet to take for the live tail to shed. + /// + /// Pacing before the frame exists matters because admitting it trims the tail to fit + /// its declared size, so a large frame would otherwise shed what that reader still wants. + pub(crate) fn poll_admit(&mut self, waiter: &kio::Waiter, group: &crate::recv::Group, size: u64) -> Poll<()> + where + S: Pause, + { + self.poll_pace(group.poll_admit(waiter, size)) + } + + /// Wait out `paced`, the group's pace gate, if the session has room for the credit this + /// stream holds while paused (see [`Pause`]); otherwise read on, and the group sheds. + fn poll_pace(&mut self, paced: Poll<()>) -> Poll<()> + where + S: Pause, + { + match paced { + Poll::Ready(()) => { + self.stream.resume(); + Poll::Ready(()) + } + Poll::Pending if self.stream.pause() => Poll::Pending, + Poll::Pending => Poll::Ready(()), + } + } + /// Poll for exactly `size` bytes. pub fn poll_read_exact(&mut self, waiter: &kio::Waiter, size: usize) -> Poll> { self.want = 0; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 6327778d372..98057e77f0e 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -4984,6 +4984,8 @@ enum IngestPhase { Size { timestamp: Option }, /// Reading the status of an empty object. Status { timestamp: Option }, + /// Waiting to admit the object until the group's fastest reader leaves room for it. + Admit { header: frame::Header }, /// Streaming the object payload. Payload { frame: frame::Producer }, /// An explicit end-of-group or end-of-track status arrived. @@ -5312,6 +5314,7 @@ impl Subscriber { fields, inflated: super::object::inflated(decoded.inflated, producer.compressed())?, }; + kio::wait(|waiter| stream.poll_admit(waiter, producer, size)).await; let mut frame = producer.create_object(header)?; if let Err(err) = kio::wait(|waiter| stream.poll_read_frame(waiter, &mut frame)).await { let _ = frame.abort(err.clone()); @@ -5856,7 +5859,7 @@ struct FetchedObject { /// /// The `first` object on a stream has no prior to inherit from, so a field it leaves to /// the prior object is a protocol violation. -async fn decode_fetch_object( +async fn decode_fetch_object( stream: &mut Reader, first: bool, ) -> Result, Error> { @@ -6034,11 +6037,18 @@ impl GroupIngest { self.phase = IngestPhase::Status { timestamp: *timestamp }; continue; } + let timestamp = object_time(self.timescale, *timestamp)?; + let header = self.header(group, frame::Info { size, timestamp })?; + self.phase = IngestPhase::Admit { header }; + } + IngestPhase::Admit { header } => { + ready!(reader.poll_admit(waiter, group, header.info.size)); + let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Delta) else { + unreachable!() + }; // `create_object` is the allocation chokepoint: it rejects an oversized // `size` and allocates up front only within the origin's budget, so no // pre-check is needed. - let timestamp = object_time(self.timescale, *timestamp)?; - let header = self.header(group, frame::Info { size, timestamp })?; let frame = group.create_object(header)?; self.phase = IngestPhase::Payload { frame }; } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 7b3fc33ab26..5444c865e4a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1086,6 +1086,8 @@ enum IngestPhase { timestamp: Option, inflated: Option, }, + /// Waiting to admit the frame until the group's fastest reader leaves room for it. + Admit { header: frame::Header }, /// Streaming the frame payload. Payload { frame: frame::Producer }, } @@ -1267,16 +1269,25 @@ impl FrameIngest { } None => None, }; + self.phase = IngestPhase::Admit { + header: frame::Header { + info: frame::Info { size, timestamp }, + fields, + inflated, + }, + }; + } + IngestPhase::Admit { header } => { + ready!(reader.poll_admit(waiter, group, header.info.size)); + let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Timing) else { + unreachable!() + }; // `create_object` is the allocation chokepoint: it rejects an oversized // `size` and allocates up front only within the origin's budget, so no // pre-check is needed. No wire timestamp (pre-lite-05) means an untimed // frame. A compressed frame stores the slice as it arrives, its inflated // size checked against the same limits before anything inflates. - let frame = group.create_object(frame::Header { - info: frame::Info { size, timestamp }, - fields, - inflated, - })?; + let frame = group.create_object(header)?; self.phase = IngestPhase::Payload { frame }; } IngestPhase::Payload { frame } => { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 3b144491ce0..0dcab9931a9 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2224,7 +2224,8 @@ impl Producer { /// Ready once the group is closed, so the next write surfaces why. A reader that took /// every committed frame is waiting on the writer, so it never holds the writer back, /// and one whose next frame was shed has fallen off the group and is not waited on. - /// Writing without waiting never fails: the group sheds its front instead. + /// Writing without waiting never fails: the group sheds its front instead, as it also + /// does for a frame larger than the room left, which trims the tail to fit. pub fn poll_ready(&self, waiter: &kio::Waiter) -> Poll<()> { ready!(self.poll_paced(waiter, 0)); // A closed group reports `Closed`, which is ready too. diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 13114219e10..4c1657c157d 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -201,6 +201,23 @@ mod tests { assert_eq!(frame.poll_ready(&waiter), Poll::Pending); } + /// A frame is admitted only once its declared size fits the window, since admitting + /// it trims the tail to fit, which would otherwise shed what the reader still wants. + #[test] + fn a_large_frame_waits_to_be_admitted() { + let (flag, waiter) = flag(); + let group = written(15); + let mut reader = group.consume(); + assert_eq!(group.poll_paced(&waiter, 20 * MB as u64), Poll::Pending); + while reader.poll_read_frame(&kio::Waiter::noop()).is_ready() {} + assert!(flag.woken()); + assert_eq!( + group.poll_paced(&waiter, 20 * MB as u64), + Poll::Ready(()), + "the reader took everything, so the frame sheds nothing it wants" + ); + } + /// A group nobody reads keeps writing, and a parked writer moves on once its last /// reader leaves. #[test] diff --git a/rs/moq-net/src/recv.rs b/rs/moq-net/src/recv.rs index d005fd209c9..d72230da4d5 100644 --- a/rs/moq-net/src/recv.rs +++ b/rs/moq-net/src/recv.rs @@ -30,6 +30,15 @@ impl Group { .map(frame::Producer::cancel_on_drop) } + /// Poll until the writer may admit a frame of `size` bytes without shedding a byte its + /// fastest reader has yet to take; see [`group::Producer::poll_paced`]. + pub fn poll_admit(&self, waiter: &kio::Waiter, size: u64) -> std::task::Poll<()> { + self.writer + .as_ref() + .expect("group writer is available between frames") + .poll_paced(waiter, size) + } + /// Restore the writer returned by a completed frame. pub fn restore(&mut self, writer: group::Producer) { debug_assert!(self.writer.is_none()); diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index b1b8e72fb4f..3b525b8789f 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -89,7 +89,7 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { moq_clock::sleep(Duration::from_secs(1)).await; let held = reader.frame_count(); assert!( - (16..=18).contains(&held), + (15..=18).contains(&held), "{version}: the relay read {held} MB ahead of a reader that took nothing" ); @@ -108,6 +108,50 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { } } +/// A frame larger than the room left waits to be admitted, since admitting it trims the +/// tail to fit: an idle reader loses nothing to a large frame either. +#[moq_clock::test] +async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { + let payload = Bytes::from(vec![7u8; MB]); + for version in VERSIONS { + let mut relay = relay(version).await; + + let mut group = relay.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("group 0"); + for _ in 1..15 { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + // Let the relay take what fits, so the publisher's egress has sent every frame the + // large one sheds from the publisher's own tail. + moq_clock::sleep(Duration::from_secs(1)).await; + group + .write_frame(Timestamp::ZERO, Bytes::from(vec![7u8; 20 * MB])) + .unwrap(); + group.finish().unwrap(); + + moq_clock::sleep(Duration::from_secs(1)).await; + assert_eq!( + reader.frame_count(), + 15, + "{version}: the large frame waits for the reader" + ); + for index in 0..16 { + let frame = moq_clock::timeout(TIMEOUT, reader.read_frame()) + .await + .unwrap_or_else(|_| panic!("{version}: frame {index} never arrived")) + .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) + .expect("a frame"); + let size = if index < 15 { MB } else { 20 * MB }; + assert_eq!(frame.payload.len(), size, "{version}: frame {index}"); + } + } +} + /// A session whose paused streams would hold more than half its connection window reads /// on instead, and its idle reader keeps what the live tail holds. #[moq_clock::test] diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 7fa3be2e6da..1540a62b605 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1564,8 +1564,8 @@ impl Connection { self.streams.receive_window() } - /// The most receive credit one stream may hold: the per-stream window now in force, or the - /// larger one a stream may have started with; see [`Limits::stream_receive_window`] + /// The most receive credit one stream may hold: the largest per-stream window ever in force, + /// since credit already granted outlives a shrink; see [`Limits::stream_receive_window`] pub fn stream_receive_window(&self) -> u64 { self.streams.stream_receive_window() } diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 1f207b744bd..09e3e96ae34 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -174,6 +174,9 @@ pub(crate) struct StreamsState { /// new stream starts with whatever [`stream_receive_window`](Self::stream_receive_window) /// became since pub(super) initial_stream_receive_window: u64, + /// The largest stream receive window ever in force, which bounds the credit a stream may + /// still hold after the window shrinks + peak_stream_receive_window: u64, // Pertinent state from the TransportParameters supplied by the peer initial_max_stream_data_uni: VarInt, @@ -242,6 +245,7 @@ impl StreamsState { send_window, stream_receive_window: stream_receive_window.into(), initial_stream_receive_window: stream_receive_window.into(), + peak_stream_receive_window: stream_receive_window.into(), initial_max_stream_data_uni: 0u32.into(), initial_max_stream_data_bidi_local: 0u32.into(), initial_max_stream_data_bidi_remote: 0u32.into(), @@ -1068,16 +1072,17 @@ impl StreamsState { self.receive_window } - /// The most credit one receive stream may hold: the window now in force, or the larger one - /// a new stream started with. + /// The most credit one receive stream may hold: the largest window ever in force, since + /// credit already granted outlives a shrink. pub(crate) fn stream_receive_window(&self) -> u64 { - self.stream_receive_window.max(self.initial_stream_receive_window) + self.peak_stream_receive_window } /// Set the receive window of each stream, which applies as each stream's credit is next /// extended: credit already granted cannot be taken back. pub(crate) fn set_stream_receive_window(&mut self, window: u64) { self.stream_receive_window = window; + self.peak_stream_receive_window = self.peak_stream_receive_window.max(window); } /// Set the receive_window and returns whether the receive_window has been @@ -2815,6 +2820,17 @@ mod tests { assert!(should_transmit.should_transmit()); } + /// A stream may still hold credit granted under a larger window, so the bound reported to + /// the application never falls when the window shrinks. + #[test] + fn stream_receive_window_reports_its_peak() { + let mut server = make(Side::Server); + let initial = server.stream_receive_window(); + server.set_stream_receive_window(initial * 4); + server.set_stream_receive_window(initial / 2); + assert_eq!(server.stream_receive_window(), initial * 4); + } + #[test] fn shrink_receive_window() { let mut server = make(Side::Server); From a051f54b8bceb0abcfb77aa4da07bb502e186211 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:55:29 -0700 Subject: [PATCH 09/16] fix(quic): name the peak stream window getter for what it returns Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 2 +- rs/moq-quic/src/connection/streams/state.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 1540a62b605..106423e573e 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1567,7 +1567,7 @@ impl Connection { /// The most receive credit one stream may hold: the largest per-stream window ever in force, /// since credit already granted outlives a shrink; see [`Limits::stream_receive_window`] pub fn stream_receive_window(&self) -> u64 { - self.streams.stream_receive_window() + self.streams.peak_stream_receive_window() } /// Change every limit `limits` sets, leaving the others as they are diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 09e3e96ae34..bea26977243 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -1074,7 +1074,7 @@ impl StreamsState { /// The most credit one receive stream may hold: the largest window ever in force, since /// credit already granted outlives a shrink. - pub(crate) fn stream_receive_window(&self) -> u64 { + pub(crate) fn peak_stream_receive_window(&self) -> u64 { self.peak_stream_receive_window } @@ -2825,10 +2825,10 @@ mod tests { #[test] fn stream_receive_window_reports_its_peak() { let mut server = make(Side::Server); - let initial = server.stream_receive_window(); + let initial = server.peak_stream_receive_window(); server.set_stream_receive_window(initial * 4); server.set_stream_receive_window(initial / 2); - assert_eq!(server.stream_receive_window(), initial * 4); + assert_eq!(server.peak_stream_receive_window(), initial * 4); } #[test] From 775219ab156c3bd53236d7672528ab0a7de11db4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 01:27:52 -0700 Subject: [PATCH 10/16] fix(net): watch a paused ingest stream for resets A paced ingest stops reading its stream, so a publisher's reset went unseen until the group's reader moved. `poll::RecvStream::poll_reset` (and `poll::Chunks::poll_reset`) wait for a reset without taking bytes or granting credit; a paused reader polls it and fails the group. moq-quic answers it through moq-tokio and moq-uring, and the memory transport through its channel. The default never completes, which suits a transport that reports no receive window, since its streams never pause. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 81 ++++++++++++++++--- rs/moq-net/src/ietf/subscriber.rs | 4 +- rs/moq-net/src/lite/subscriber.rs | 2 +- rs/moq-net/src/transport.rs | 26 ++++++ rs/moq-net/src/transport/boxed.rs | 9 +++ rs/moq-net/src/transport/memory.rs | 17 ++++ .../tests/integration/fastest_reader.rs | 29 +++++++ rs/moq-net/tests/integration/support/mock.rs | 8 ++ rs/moq-tokio/src/quic/driver/recv.rs | 55 +++++++------ rs/moq-tokio/src/quic/web/recv.rs | 16 +++- rs/moq-tokio/src/transport/owned.rs | 3 + rs/moq-uring/src/quic/driver/recv.rs | 8 +- rs/moq-uring/src/quic/web.rs | 14 ++++ rs/moq-uring/src/transport.rs | 3 + rs/moq-uring/src/transport/local.rs | 4 + 15 files changed, 237 insertions(+), 42 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index e1d3209bdbc..8844f5ab0d0 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -418,8 +418,9 @@ impl Reader { let result = loop { // A writer too far ahead of the group's fastest reader leaves the rest in the // stream, so flow control pushes back upstream. - let lease = match self.poll_pace(frame.poll_ready(waiter)) { - Poll::Ready(()) => self.poll_lease(waiter, frame.remaining().min(1)), + let lease = match self.poll_pace(waiter, frame.poll_ready(waiter)) { + Poll::Ready(Ok(())) => self.poll_lease(waiter, frame.remaining().min(1)), + Poll::Ready(Err(err)) => break Err(err), Poll::Pending => Poll::Pending, }; let mut lease = match lease { @@ -479,26 +480,38 @@ impl Reader { /// /// Pacing before the frame exists matters because admitting it trims the tail to fit /// its declared size, so a large frame would otherwise shed what that reader still wants. - pub(crate) fn poll_admit(&mut self, waiter: &kio::Waiter, group: &crate::recv::Group, size: u64) -> Poll<()> + /// An error means the stream failed while paused. + pub(crate) fn poll_admit( + &mut self, + waiter: &kio::Waiter, + group: &crate::recv::Group, + size: u64, + ) -> Poll> where S: Pause, { - self.poll_pace(group.poll_admit(waiter, size)) + self.poll_pace(waiter, group.poll_admit(waiter, size)) } /// Wait out `paced`, the group's pace gate, if the session has room for the credit this /// stream holds while paused (see [`Pause`]); otherwise read on, and the group sheds. - fn poll_pace(&mut self, paced: Poll<()>) -> Poll<()> + /// + /// A paused stream is not read, so it is watched for a reset instead, which fails the + /// wait rather than leaving it to the group's reader to notice. + fn poll_pace(&mut self, waiter: &kio::Waiter, paced: Poll<()>) -> Poll> where S: Pause, { match paced { Poll::Ready(()) => { self.stream.resume(); - Poll::Ready(()) + Poll::Ready(Ok(())) + } + Poll::Pending if self.stream.pause() => { + let err = ready!(self.stream.poll_reset(waiter)); + Poll::Ready(Err(self.version.transport_error(err))) } - Poll::Pending if self.stream.pause() => Poll::Pending, - Poll::Pending => Poll::Ready(()), + Poll::Pending => Poll::Ready(Ok(())), } } @@ -736,11 +749,12 @@ mod tests { /// Hands back one queued chunk per poll, then blocks: a stream whose socket has /// delivered a burst and has nothing more for this turn. Finishes once `fin` is set and - /// the queue is empty. + /// the queue is empty, and reports a reset once `reset` is set. #[derive(Default)] struct Queue { chunks: VecDeque<&'static [u8]>, fin: bool, + reset: bool, } impl Chunks for Queue { @@ -764,13 +778,20 @@ mod tests { } fn stop(&mut self, _code: u32) {} + + fn poll_reset(&mut self, _: &kio::Waiter) -> Poll { + match self.reset { + true => Poll::Ready(Default::default()), + false => Poll::Pending, + } + } } fn queue(chunks: &[&'static [u8]], version: V) -> Reader, V> { Reader::new( Buffered::new(Queue { chunks: chunks.iter().copied().collect(), - fin: false, + ..Default::default() }), version, ) @@ -951,6 +972,44 @@ mod tests { )); } + /// A group 17 MB past an idle reader, so its writer is paced. + fn paced_group() -> (crate::group::Producer, crate::group::Consumer) { + let mut group = crate::group::Info { sequence: 0 }.produce(); + let idle = group.consume(); + let payload = Bytes::from(vec![0u8; 1 << 20]); + for _ in 0..17 { + group.write_frame(crate::Timestamp::ZERO, payload.clone()).unwrap(); + } + (group, idle) + } + + /// A paused stream is not read, so it is watched for a reset instead: one ends the wait + /// at once, whether a frame waits to be admitted or to fill, rather than once the + /// group's reader moves. + #[test] + fn a_reset_ends_a_paused_stream() { + let noop = &kio::Waiter::noop(); + + let (group, _idle) = paced_group(); + let group = crate::recv::Group::new(group); + let mut reader = queue(&[], crate::lite::Version::Lite05); + assert!(reader.poll_admit(noop, &group, 1).is_pending(), "paused to admit"); + reader.stream.get_mut().reset = true; + assert!(matches!(reader.poll_admit(noop, &group, 1), Poll::Ready(Err(_)))); + + let (group, _idle) = paced_group(); + let mut frame = group + .create_frame(crate::frame::Info { + size: 1, + timestamp: Some(crate::Timestamp::ZERO), + }) + .unwrap(); + let mut reader = queue(&[b"x"], crate::lite::Version::Lite05); + assert!(reader.poll_read_frame(noop, &mut frame).is_pending(), "paused to fill"); + reader.stream.get_mut().reset = true; + assert!(matches!(reader.poll_read_frame(noop, &mut frame), Poll::Ready(Err(_)))); + } + /// A consumer parked on the group cannot run until this task yields, so a burst of /// chunks owes exactly one wake: at the boundary, from whoever wrote them. Every /// other exit hands the wake to the caller's `finish` or `abort`. @@ -1023,7 +1082,7 @@ mod tests { Buffered::new(Burst { chunks: Queue { chunks: std::iter::repeat_n(CHUNK, chunks).collect(), - fin: false, + ..Default::default() }, payload, waiter, diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index b7d3181e7f2..3244e562da4 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -5360,7 +5360,7 @@ impl Subscriber { fields, inflated: super::object::inflated(decoded.inflated, producer.compressed())?, }; - kio::wait(|waiter| stream.poll_admit(waiter, producer, size)).await; + kio::wait(|waiter| stream.poll_admit(waiter, producer, size)).await?; let mut frame = producer.create_object(header)?; if let Err(err) = kio::wait(|waiter| stream.poll_read_frame(waiter, &mut frame)).await { let _ = frame.abort(err.clone()); @@ -6088,7 +6088,7 @@ impl GroupIngest { self.phase = IngestPhase::Admit { header }; } IngestPhase::Admit { header } => { - ready!(reader.poll_admit(waiter, group, header.info.size)); + ready!(reader.poll_admit(waiter, group, header.info.size))?; let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Delta) else { unreachable!() }; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 944b4c7a24e..ab556a1889d 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1278,7 +1278,7 @@ impl FrameIngest { }; } IngestPhase::Admit { header } => { - ready!(reader.poll_admit(waiter, group, header.info.size)); + ready!(reader.poll_admit(waiter, group, header.info.size))?; let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Timing) else { unreachable!() }; diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index ed227c0a864..7f43d67928e 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -888,6 +888,18 @@ pub mod poll { /// Call this method manually if you want to specify a code yourself. fn stop(&mut self, code: u32); + /// Poll until the stream fails without reading it: the peer resets it, or the + /// connection closes. Pending for good once the peer finished it. + /// + /// A session that stops reading a stream to pace it (see + /// [`Session::receive_window`]) watches this instead, so a reset still ends the group + /// it carries. It takes no bytes and grants no credit. The default never completes, + /// which suits a transport reporting no receive window, since none of its streams pause. + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + let _ = waiter; + Poll::Pending + } + /// Poll for the next chunk, up to `max` bytes, or `None` once the peer finished the /// stream. Its credit returns as it is read. /// @@ -965,6 +977,12 @@ pub mod poll { /// An implementation MUST also do this on Drop: [`Buffered`] forwards an explicit stop /// but leaves the drop to the wrapped stream. fn stop(&mut self, code: u32); + + /// Poll until the stream fails without reading it. See [`RecvStream::poll_reset`]. + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + let _ = waiter; + Poll::Pending + } } /// Leases over a transport that cannot keep bytes unconsumed, buffering what it reads. @@ -1035,6 +1053,14 @@ pub mod poll { self.len = 0; self.inner.stop(code); } + + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + // Every byte arrived, so nothing is left to reset. + match self.fin { + true => Poll::Pending, + false => self.inner.poll_reset(waiter), + } + } } /// A [`Lease`] of a [`Buffered`] stream. diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 5db1c8ca6e0..7335d9ce4ea 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -302,6 +302,7 @@ trait ErasedRecv: MaybeSend + MaybeSync { fn poll_read_chunk(&mut self, waiter: &kio::Waiter, max: usize) -> Poll, Error>>; fn set_priority(&mut self, priority: i64); fn stop(&mut self, code: u32); + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll; fn pause(&mut self) -> bool; fn resume(&mut self); } @@ -488,6 +489,10 @@ impl ErasedRecv for Budgeted { poll::RecvStream::stop(&mut self.stream, code) } + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + poll::RecvStream::poll_reset(&mut self.stream, waiter).map(Error::new) + } + fn pause(&mut self) -> bool { if self.held > 0 { return true; @@ -653,6 +658,10 @@ impl poll::RecvStream for RecvStream { fn stop(&mut self, code: u32) { self.0.stop(code) } + + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + self.0.poll_reset(waiter) + } } impl Buf for Lease<'_> { diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index ec801e64062..2b8add1d9f7 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -643,6 +643,23 @@ impl poll::Chunks for Chunks { drop(channel); self.release(); } + + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + if self.stopped { + return Poll::Ready(Error::Closed); + } + if let Poll::Ready(err) = poll_close(&self.close, waiter) { + return Poll::Ready(err); + } + let channel = ready!(self.channel.poll(waiter, |channel| match channel.end { + Some(End::Reset(_)) => Poll::Ready(()), + _ => Poll::Pending, + })); + let Some(End::Reset(code)) = channel.end else { + unreachable!("checked above") + }; + Poll::Ready(Error::Reset(code)) + } } impl Drop for Chunks { diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index 3b525b8789f..c2e8da80a6f 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -108,6 +108,35 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { } } +/// A paused stream is not read, so the relay watches it for a reset instead: a publisher +/// aborting the group ends the relay's copy while its reader still takes nothing. +#[moq_clock::test] +async fn a_reset_reaches_a_paused_relay() { + let payload = Bytes::from(vec![7u8; MB]); + for version in VERSIONS { + let mut relay = relay(version).await; + + let mut group = relay.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("group 0"); + for _ in 1..30 { + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + moq_clock::sleep(Duration::from_secs(1)).await; + assert!(reader.frame_count() < 30, "{version}: the relay never paused"); + + group.abort(moq_net::Error::Cancel).unwrap(); + let end = moq_clock::timeout(TIMEOUT, reader.finished()) + .await + .unwrap_or_else(|_| panic!("{version}: the reset never reached the paused relay")); + assert!(end.is_err(), "{version}: {end:?}"); + } +} + /// A frame larger than the room left waits to be admitted, since admitting it trims the /// tail to fit: an idle reader loses nothing to a large frame either. #[moq_clock::test] diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 4f2188cbb9e..dbfe36edb2e 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -413,6 +413,14 @@ impl poll::Chunks for MockChunks { fn stop(&mut self, code: u32) { self.stop_after_latency(code); } + + // Unlike data, a reset skips the link's latency: no test times one. + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + match &mut self.inner { + Some(inner) => poll::RecvStream::poll_reset(inner, waiter), + None => Poll::Ready(memory::Error::Closed), + } + } } impl Drop for MockChunks { diff --git a/rs/moq-tokio/src/quic/driver/recv.rs b/rs/moq-tokio/src/quic/driver/recv.rs index 6f9bdd405d1..fd5b219eeae 100644 --- a/rs/moq-tokio/src/quic/driver/recv.rs +++ b/rs/moq-tokio/src/quic/driver/recv.rs @@ -227,36 +227,39 @@ impl Stream { /// /// This operation is cancel-safe. pub async fn received_reset(&mut self) -> Result, Error> { - poll_fn(|cx| { - let mut conn = self.conn.state.lock(); + poll_fn(|cx| self.poll_received_reset(cx)).await + } - if let Some(code) = self.reset { - return Poll::Ready(Ok(Some(code))); - } + /// Poll form of [`Self::received_reset`]. It takes no bytes, so a lease in progress is + /// left as it was. + pub(crate) fn poll_received_reset(&mut self, cx: &mut Context<'_>) -> Poll, Error>> { + let mut conn = self.conn.state.lock(); - match conn.inner.recv_stream(self.id).received_reset() { - Err(_) => Poll::Ready(Ok(None)), - Ok(Some(error_code)) => { - // Stream state has just now been freed, so the connection may need to issue new - // stream ID flow control credit, and later reads fail with the cached code. - self.reset = Some(error_code); - conn.wake(); - Poll::Ready(Ok(Some(error_code))) - } - Ok(None) => { - if let Some(e) = &conn.error { - return Poll::Ready(Err(e.clone().into())); - } - // Resets always notify readers, since a reset is an immediate read error. We - // could introduce a dedicated channel to reduce the risk of spurious wakeups, - // but that increased complexity is probably not justified, as an application - // that is expecting a reset is not likely to receive large amounts of data. - conn.blocked_readers.insert(self.id, cx.waker().clone()); - Poll::Pending + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + + match conn.inner.recv_stream(self.id).received_reset() { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(error_code)) => { + // Stream state has just now been freed, so the connection may need to issue new + // stream ID flow control credit, and later reads fail with the cached code. + self.reset = Some(error_code); + conn.wake(); + Poll::Ready(Ok(Some(error_code))) + } + Ok(None) => { + if let Some(e) = &conn.error { + return Poll::Ready(Err(e.clone().into())); } + // Resets always notify readers, since a reset is an immediate read error. We + // could introduce a dedicated channel to reduce the risk of spurious wakeups, + // but that increased complexity is probably not justified, as an application + // that is expecting a reset is not likely to receive large amounts of data. + conn.blocked_readers.insert(self.id, cx.waker().clone()); + Poll::Pending } - }) - .await + } } /// Handle common logic related to reading out of a receive stream diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 541774131fb..af1188809c9 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -2,7 +2,7 @@ use std::{ io, pin::Pin, sync::Arc, - task::{Context, Poll}, + task::{Context, Poll, ready}, }; use bytes::{BufMut, Bytes}; @@ -45,6 +45,20 @@ impl RecvStream { } } + /// Poll until the peer resets the stream or the session closes, without reading it; pending + /// for good once every byte arrived. See [`recv::Stream::received_reset`]. + pub fn poll_reset(&mut self, cx: &mut Context<'_>) -> Poll { + let close = &self.close; + let Some(inner) = &mut self.inner else { + return Poll::Ready(ReadError::ClosedStream); + }; + match ready!(inner.poll_received_reset(cx)) { + Ok(None) => Poll::Pending, + Ok(Some(code)) => Poll::Ready(map_error(close, recv::Error::Reset(code))), + Err(err) => Poll::Ready(map_error(close, err)), + } + } + /// Rank this stream for eviction. See [`recv::Stream::set_priority`]. pub fn set_priority(&mut self, priority: i64) { if let Some(inner) = &mut self.inner { diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index b76dc375504..f5cd027d453 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -262,6 +262,9 @@ impl poll::RecvStream for crate::quic::web::RecvStream { // A stream already closed has nobody left to tell. crate::quic::web::RecvStream::stop(self, code); } + fn poll_reset(&mut self, waiter: &moq_net::kio::Waiter) -> Poll { + crate::quic::web::RecvStream::poll_reset(self, &mut waiter.context()).map(Error) + } } #[cfg(feature = "quic")] diff --git a/rs/moq-uring/src/quic/driver/recv.rs b/rs/moq-uring/src/quic/driver/recv.rs index 99056d71e0b..d4f3ccf3571 100644 --- a/rs/moq-uring/src/quic/driver/recv.rs +++ b/rs/moq-uring/src/quic/driver/recv.rs @@ -214,7 +214,13 @@ impl Stream { /// /// Cancel-safe. pub async fn received_reset(&mut self) -> Result, Error> { - poll_fn(|cx| self.parked(cx, Self::wait_received_reset)).await + poll_fn(|cx| self.poll_received_reset(cx)).await + } + + /// Poll form of [`Self::received_reset`]. It takes no bytes, so a lease in progress is + /// left as it was. + pub(crate) fn poll_received_reset(&mut self, cx: &Context<'_>) -> Poll, Error>> { + self.parked(cx, Self::wait_received_reset) } fn wait_received_reset(&mut self, waiter: &kio::Waiter) -> Poll, Error>> { diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index f6a2815cd2e..ec65caf49b9 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -1078,6 +1078,20 @@ impl RecvStream { inner.set_priority(priority); } } + + /// Poll until the peer resets the stream or the connection closes, without reading it; + /// pending for good once every byte arrived. See [`recv::Stream::received_reset`]. + pub fn poll_reset(&mut self, cx: &mut Context<'_>) -> Poll { + let web = self.web; + let Some(inner) = &mut self.inner else { + return Poll::Ready(Error::Closed); + }; + match ready!(inner.poll_received_reset(cx)) { + Ok(None) => Poll::Pending, + Ok(Some(code)) => Poll::Ready(map(web, Error::Reset(code.into_inner()))), + Err(err) => Poll::Ready(map(web, err.into())), + } + } } /// Unmap a stream error's code from the session's code space. diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 13e9bf5e977..7489caeef04 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -96,6 +96,9 @@ impl moq_net::transport::poll::RecvStream for quic::web::RecvStream { fn stop(&mut self, code: u32) { web_transport_trait::poll::RecvStream::stop(self, code) } + fn poll_reset(&mut self, waiter: &kio::Waiter) -> std::task::Poll { + quic::web::RecvStream::poll_reset(self, &mut waiter.context()).map(adapter::Error) + } } #[cfg(feature = "quic")] diff --git a/rs/moq-uring/src/transport/local.rs b/rs/moq-uring/src/transport/local.rs index ed476d36937..90890356063 100644 --- a/rs/moq-uring/src/transport/local.rs +++ b/rs/moq-uring/src/transport/local.rs @@ -177,6 +177,10 @@ impl poll::RecvStream for Local { self.0.stop(code) } + fn poll_reset(&mut self, waiter: &kio::Waiter) -> Poll { + self.0.poll_reset(waiter) + } + fn poll_read_chunk(&mut self, waiter: &kio::Waiter, max: usize) -> Poll, Self::Error>> { self.0.poll_read_chunk(waiter, max) } From b0e7943bdf9ba65815b626b23abe8062dc705230 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 01:27:55 -0700 Subject: [PATCH 11/16] perf(net): bench a paced writer; skip the locked swap when none is parked `group_paced` keeps a writer at the pacing window over 1, 8, and 64 readers, so the per-read progress report and the park and wake show up together. The fastest reader's wake check now loads `parked` before swapping it, so a read with no parked writer costs one locked RMW, not two. Co-Authored-By: Claude Opus 5.5 --- quest/m1/unbounded-groups/fastest-reader.md | 23 +++++--- rs/moq-net/benches/group.rs | 58 ++++++++++++++++++++- rs/moq-net/src/model/pace.rs | 4 +- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/quest/m1/unbounded-groups/fastest-reader.md b/quest/m1/unbounded-groups/fastest-reader.md index 1779b5439e2..80483f55060 100644 --- a/quest/m1/unbounded-groups/fastest-reader.md +++ b/quest/m1/unbounded-groups/fastest-reader.md @@ -92,10 +92,13 @@ payload loop (`coding::Reader::poll_read_frame`) waits on its declared size fits the window (`coding::Reader::poll_admit`), so moq-lite and moq-transport ingest both pause past the 16 MB window. A reader that took every committed frame counts as waiting on the writer, so a frame larger than the -window never stalls it. Tests cover one hop on lite-06, lite-07-wip, and -moq-transport-22: an idle reader holds the relay copy near the window and then -receives every frame, a slow reader ends `Evicted` while the fast one keeps -up, and a session past its credit cap reads on. The same PR landed the credit +window never stalls it. A paused stream is not read, so it is watched for a +reset instead (`poll::RecvStream::poll_reset`, which moq-quic answers through +moq-tokio and moq-uring without taking bytes). Tests cover one hop on lite-06, +lite-07-wip, and moq-transport-22: an idle reader holds the relay copy near the +window and then receives every frame, a slow reader ends `Evicted` while the +fast one keeps up, a session past its credit cap reads on, and a publisher's +reset ends a paused relay copy. The same PR landed the credit cap (each paused stream reserves its stream window from a per-session budget in `rs/moq-net/src/transport/boxed.rs`; moq-quic reports its windows through moq-tokio and moq-uring, and a transport that reports none never pauses) and @@ -105,15 +108,19 @@ What remains: - Receive-window reporting for the transports that report none yet (quinn, iroh, qmux, WebTransport in the browser, and the FFI adapters), whose ingest - never pauses until they do. + never pauses until they do. Each must also answer + `poll::RecvStream::poll_reset`, or a reset goes unseen while paused. - A sized readiness check for local writers: `group::Producer::ready()` waits for room but not for the next frame's size, so a local frame larger than the room left still sheds what the egress has yet to send. - JS parity. - A two-relay chain test with memory measured near the window. -- Benchmarks of the hot path in `group.rs`: the cursor's report is one - relaxed load per frame or batch, and a store only for the fastest reader. - They were not run on the busy host that landed the first slice. +- A quiet-host comparison against the base of `group_read_frames`, + `group_contended`, and `group_relay_frames`, plus `group_paced` (a writer + parked at the window, over 1, 8, and 64 readers), which the first slice + added. The cursor's report is one relaxed load per frame or batch, and a + `fetch_max` only for the fastest reader. On the busy host that landed it, + runs swung by up to 50% either way between repeats. - Fills that write a whole batch of groups at once evict their own earlier groups before the reader takes them (the follow-up from [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)). Pacing within diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index c833b2ef51a..d371631d8fa 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -22,6 +22,10 @@ //! holds (a small pool's debt, or the 32 MiB live tail) and the frames it sheds, so //! shedding the front stays a flat cost per frame. //! +//! `group_paced` keeps a writer one frame past the 16 MiB window ahead of its fastest +//! reader, as a paced ingest runs, swept over readers: each frame read reports progress, +//! and the fastest reader's read wakes the parked writer. +//! //! Run with `cargo bench -p moq-net`. #![expect( @@ -757,6 +761,57 @@ fn bench_shed(c: &mut Criterion) { g.finish(); } +/// Counts wakes, so a bench can check that reads let a parked writer on. +#[derive(Default)] +struct Wakes(AtomicU64); + +impl std::task::Wake for Wakes { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// A writer held at the pacing window: each round it polls [`group::Producer::poll_ready`], +/// parking whenever it is past the window, every reader takes one frame, and the writer +/// writes the next. A reader reports a prefetched batch at once, so the fastest one's batch +/// read wakes the parked writer. Throughput counts frames read, so the per-read progress +/// report and the park and wake show up together, and the readers trailing the fastest one +/// as a slope across readers. +fn bench_paced(c: &mut Criterion) { + // Payloads are shared clones, so a megabyte costs a refcount and the window is 16 frames. + let payload = Bytes::from(vec![0u8; 1 << 20]); + let mut g = c.benchmark_group("group_paced"); + for readers in [1, 8, 64] { + g.throughput(Throughput::Elements(readers as u64)); + g.bench_function(BenchmarkId::from_parameter(readers), |b| { + let mut ctx = fresh_group(); + let mut consumers: Vec<_> = (0..readers).map(|_| ctx.group.consume()).collect(); + // One frame past the window, so the writer is paced from the first round. + for _ in 0..17 { + ctx.group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + } + let wakes = Arc::new(Wakes::default()); + let writer = kio::Waiter::new(wakes.clone().into()); + let reader = kio::Waiter::noop(); + let mut parked = 0u64; + b.iter(|| { + parked += ctx.group.poll_ready(&writer).is_pending() as u64; + for consumer in &mut consumers { + let Poll::Ready(Ok(Some(frame))) = consumer.poll_read_frame(&reader) else { + panic!("a frame is ready"); + }; + black_box(frame); + } + ctx.group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + }); + // A writer still paced parks again on the next round, so wakes trail parks. + assert!(parked > 0, "the writer never parked"); + assert!(wakes.0.load(Ordering::Relaxed) > 0, "no read woke the writer"); + }); + } + g.finish(); +} + criterion_group!( benches, bench_write, @@ -767,6 +822,7 @@ criterion_group!( bench_relay_frames, bench_inflate, bench_contended, - bench_shed + bench_shed, + bench_paced ); criterion_main!(benches); diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 4c1657c157d..6165f2f0e52 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -52,7 +52,9 @@ impl Pace { /// Wake the writer, if parked, since what it waits on changed. fn wake(&self) { - if self.parked.swap(false, Ordering::SeqCst) { + // A load first, so a writer not parked costs a read rather than a locked swap; still + // SeqCst, to stay in the order `park` pairs with. + if self.parked.load(Ordering::SeqCst) && self.parked.swap(false, Ordering::SeqCst) { let writer = self.writer.lock().unwrap_or_else(PoisonError::into_inner).take(); if let Some(writer) = writer { writer.wake(); From 85beba91b42ee8b4e881fb5c948ee5e8710a63f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:05:54 -0700 Subject: [PATCH 12/16] fix(net): count frame metadata toward the pace window The live tail charges a frame's payload, metadata, and slot, but the pace scale and frame admission left the metadata out, so frames whose properties outweigh their payloads could fill the tail and shed bytes the fastest reader still wanted before ingest paused. Both now charge it, and admission takes the frame's header to do so. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 14 ++++++---- rs/moq-net/src/ietf/subscriber.rs | 4 +-- rs/moq-net/src/lite/subscriber.rs | 2 +- rs/moq-net/src/model/group.rs | 7 ++--- rs/moq-net/src/model/pace.rs | 44 +++++++++++++++++++++++++++++++ rs/moq-net/src/recv.rs | 10 ++++--- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 8844f5ab0d0..adde502b14e 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -475,7 +475,7 @@ impl Reader { Poll::Ready(result) } - /// Poll until `group`'s writer may admit a frame of `size` bytes from this stream: until + /// Poll until `group`'s writer may admit the frame `header` opens from this stream: until /// that leaves no byte its fastest reader has yet to take for the live tail to shed. /// /// Pacing before the frame exists matters because admitting it trims the tail to fit @@ -485,12 +485,12 @@ impl Reader { &mut self, waiter: &kio::Waiter, group: &crate::recv::Group, - size: u64, + header: &crate::frame::Header, ) -> Poll> where S: Pause, { - self.poll_pace(waiter, group.poll_admit(waiter, size)) + self.poll_pace(waiter, group.poll_admit(waiter, header)) } /// Wait out `paced`, the group's pace gate, if the session has room for the credit this @@ -992,10 +992,14 @@ mod tests { let (group, _idle) = paced_group(); let group = crate::recv::Group::new(group); + let header = crate::frame::Header::from(crate::frame::Info { + size: 1, + timestamp: Some(crate::Timestamp::ZERO), + }); let mut reader = queue(&[], crate::lite::Version::Lite05); - assert!(reader.poll_admit(noop, &group, 1).is_pending(), "paused to admit"); + assert!(reader.poll_admit(noop, &group, &header).is_pending(), "paused to admit"); reader.stream.get_mut().reset = true; - assert!(matches!(reader.poll_admit(noop, &group, 1), Poll::Ready(Err(_)))); + assert!(matches!(reader.poll_admit(noop, &group, &header), Poll::Ready(Err(_)))); let (group, _idle) = paced_group(); let mut frame = group diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 3244e562da4..4a5c938e75f 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -5360,7 +5360,7 @@ impl Subscriber { fields, inflated: super::object::inflated(decoded.inflated, producer.compressed())?, }; - kio::wait(|waiter| stream.poll_admit(waiter, producer, size)).await?; + kio::wait(|waiter| stream.poll_admit(waiter, producer, &header)).await?; let mut frame = producer.create_object(header)?; if let Err(err) = kio::wait(|waiter| stream.poll_read_frame(waiter, &mut frame)).await { let _ = frame.abort(err.clone()); @@ -6088,7 +6088,7 @@ impl GroupIngest { self.phase = IngestPhase::Admit { header }; } IngestPhase::Admit { header } => { - ready!(reader.poll_admit(waiter, group, header.info.size))?; + ready!(reader.poll_admit(waiter, group, header))?; let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Delta) else { unreachable!() }; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index ab556a1889d..a8d206663ea 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1278,7 +1278,7 @@ impl FrameIngest { }; } IngestPhase::Admit { header } => { - ready!(reader.poll_admit(waiter, group, header.info.size))?; + ready!(reader.poll_admit(waiter, group, header))?; let IngestPhase::Admit { header } = std::mem::replace(&mut self.phase, IngestPhase::Timing) else { unreachable!() }; diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 0dcab9931a9..48940a19dbc 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -968,13 +968,14 @@ impl GroupState { /// Append a completed frame, returning the slot growth it charged. /// - /// The frame's place on the pace scale costs its payload and its slot, as the live - /// tail does, so a paced writer of tiny frames is bounded too. + /// The frame's place on the pace scale costs its payload, metadata, and slot, as the + /// live tail does, so a paced writer of tiny frames is bounded too. fn push(&mut self, mut entry: Entry) -> u64 { let slots = self.reserve_frame(); + let metadata = entry.fields.as_ref().map_or(0, |fields| fields.heap()); entry.end = self .pace - .commit(entry.frame.payload.len() as u64 + size_of::() as u64); + .commit(entry.frame.payload.len() as u64 + metadata + size_of::() as u64); self.log.push(entry); slots } diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 6165f2f0e52..0616d5ed654 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -186,6 +186,50 @@ mod tests { assert_eq!(group.poll_paced(&waiter, 0), Poll::Ready(())); } + /// A one-byte frame whose properties hold `metadata` bytes. + fn fielded(id: u64, metadata: usize) -> frame::Header { + use crate::model::object::{Block, Fields, Properties}; + let block = Block { + bytes: Bytes::from(vec![0u8; metadata]), + immutable: false, + }; + frame::Header { + info: frame::Info { + size: 1, + timestamp: None, + }, + fields: Some(Arc::new(Fields { + id, + properties: Properties { + block: Some(block), + ..Default::default() + }, + })), + inflated: None, + } + } + + /// Metadata counts toward the window as it does toward the live tail, both committed + /// and to admit a frame, so frames whose properties outweigh their payloads are paced. + #[test] + fn metadata_counts_toward_the_window() { + let (_flag, waiter) = flag(); + let mut group = group::Info { sequence: 0 }.produce_untimed(); + let _reader = group.consume(); + for id in 0..15 { + let mut frame = group.create_object(fielded(id, MB)).unwrap(); + frame.write(Bytes::from_static(b"f")).unwrap(); + group = frame.finish().unwrap(); + } + let group = crate::recv::Group::new(group); + assert_eq!(group.poll_admit(&waiter, &fielded(15, 0)), Poll::Ready(())); + assert_eq!( + group.poll_admit(&waiter, &fielded(15, 2 * MB)), + Poll::Pending, + "its metadata would pass the window" + ); + } + /// Bytes measure the window, not frames, so an open frame counts as it fills. #[test] fn an_open_frame_counts_toward_the_window() { diff --git a/rs/moq-net/src/recv.rs b/rs/moq-net/src/recv.rs index d72230da4d5..8e87b288d58 100644 --- a/rs/moq-net/src/recv.rs +++ b/rs/moq-net/src/recv.rs @@ -30,13 +30,15 @@ impl Group { .map(frame::Producer::cancel_on_drop) } - /// Poll until the writer may admit a frame of `size` bytes without shedding a byte its - /// fastest reader has yet to take; see [`group::Producer::poll_paced`]. - pub fn poll_admit(&self, waiter: &kio::Waiter, size: u64) -> std::task::Poll<()> { + /// Poll until the writer may admit the frame `header` opens, its payload and metadata, + /// without shedding a byte its fastest reader has yet to take; see + /// [`group::Producer::poll_paced`]. + pub fn poll_admit(&self, waiter: &kio::Waiter, header: &frame::Header) -> std::task::Poll<()> { + let metadata = header.fields.as_ref().map_or(0, |fields| fields.heap()); self.writer .as_ref() .expect("group writer is available between frames") - .poll_paced(waiter, size) + .poll_paced(waiter, header.info.size.saturating_add(metadata)) } /// Restore the writer returned by a completed frame. From 3f5babd5b4419898256e34affbd06c74483028f7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:11:48 -0700 Subject: [PATCH 13/16] fix(net): admit an empty moq-transport object's properties An object with no payload and status 0 skipped the admission gate, so a stream of them with large property blocks could fill the live tail past the fastest reader. It now waits in `Admit` like any other object, and finishes as soon as it is admitted. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 4a5c938e75f..2822f310278 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -5030,7 +5030,8 @@ enum IngestPhase { Size { timestamp: Option }, /// Reading the status of an empty object. Status { timestamp: Option }, - /// Waiting to admit the object until the group's fastest reader leaves room for it. + /// Waiting to admit the object, or an empty one's properties, until the group's + /// fastest reader leaves room for it. Admit { header: frame::Header }, /// Streaming the object payload. Payload { frame: frame::Producer }, @@ -6095,17 +6096,20 @@ impl GroupIngest { // `create_object` is the allocation chokepoint: it rejects an oversized // `size` and allocates up front only within the origin's budget, so no // pre-check is needed. + let empty = header.info.size == 0; let frame = group.create_object(header)?; - self.phase = IngestPhase::Payload { frame }; + match empty { + // An empty object has no payload to read, only properties to admit. + true => group.restore(frame.finish()?), + false => self.phase = IngestPhase::Payload { frame }, + } } IngestPhase::Status { timestamp } => { let status = ready!(reader.poll_varint(waiter))?; if status == 0 { let timestamp = object_time(self.timescale, *timestamp)?; let header = self.header(group, frame::Info { size: 0, timestamp })?; - let frame = group.create_object(header)?; - group.restore(frame.finish()?); - self.phase = IngestPhase::Delta; + self.phase = IngestPhase::Admit { header }; continue; } // A status object has no frame to hold its time, so its block keeps it. From 27dfa2553f1f1ace31a82e4cefd1886246fe73d7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:17:46 -0700 Subject: [PATCH 14/16] test(net): gate the metadata pace test on async, which holds recv Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/pace.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 0616d5ed654..bb1ca8ff938 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -187,6 +187,7 @@ mod tests { } /// A one-byte frame whose properties hold `metadata` bytes. + #[cfg(feature = "async")] fn fielded(id: u64, metadata: usize) -> frame::Header { use crate::model::object::{Block, Fields, Properties}; let block = Block { @@ -212,6 +213,7 @@ mod tests { /// Metadata counts toward the window as it does toward the live tail, both committed /// and to admit a frame, so frames whose properties outweigh their payloads are paced. #[test] + #[cfg(feature = "async")] fn metadata_counts_toward_the_window() { let (_flag, waiter) = flag(); let mut group = group::Info { sequence: 0 }.produce_untimed(); From f6b01d3200822bf456fbee811c77825b0bcf0ef2 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 31 Oct 2026 23:22:59 -0700 Subject: [PATCH 15/16] fix(net): follow base wanted polling API rename --- rs/moq-net/src/model/group.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 70f6fdb547f..de50f68e970 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -2230,7 +2230,7 @@ impl Producer { pub fn poll_ready(&self, waiter: &kio::Waiter) -> Poll<()> { ready!(self.poll_paced(waiter, 0)); // A closed group reports `Closed`, which is ready too. - self.handle.state.poll_used(waiter).map(|_| ()) + self.handle.state.poll_wanted_changed(waiter, false).map(|_| ()) } /// Wait until a reader holds the group and the writer has room; see [`Self::poll_ready`]. @@ -2255,7 +2255,7 @@ impl Producer { return Poll::Ready(()); } // Either wakes the writer: the last reader leaving, or the group closing. - if self.handle.poll_unused(waiter).is_ready() || self.handle.state.poll_closed(waiter).is_ready() { + if self.handle.poll_unwanted(waiter).is_ready() || self.handle.state.poll_closed(waiter).is_ready() { return Poll::Ready(()); } match pace.paced(pace.park(waiter), partial) { From 646e6e1a1c772f448565b28f314521bb79c17eb8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:29:38 -0800 Subject: [PATCH 16/16] fix(net): follow the moq-clock move to kio::clock in fastest_reader tests Co-Authored-By: Claude Opus 5.5 --- .../tests/integration/fastest_reader.rs | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index c2e8da80a6f..d4112857e9b 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -43,11 +43,11 @@ async fn relay(version: &str) -> Relay { .unwrap(); broadcast.announce(Default::default()).unwrap(); - let remote = moq_clock::timeout(TIMEOUT, relay.consume().follow("live", None).unwrap().available()) + let remote = kio::clock::timeout(TIMEOUT, relay.consume().follow("live", None).unwrap().available()) .await .expect("resolve timeout") .expect("broadcast resolves"); - let subscriber = moq_clock::timeout( + let subscriber = kio::clock::timeout( TIMEOUT, remote .track("data") @@ -66,7 +66,7 @@ async fn relay(version: &str) -> Relay { } } -#[moq_clock::test] +#[kio::test] async fn an_idle_reader_paces_the_relay_and_loses_nothing() { let payload = Bytes::from(vec![7u8; MB]); for version in VERSIONS { @@ -74,7 +74,7 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { let mut group = relay.track.append_group().unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() @@ -86,7 +86,7 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { group.finish().unwrap(); // Simulated time moves only once every task is parked: the relay's ingest stopped. - moq_clock::sleep(Duration::from_secs(1)).await; + kio::clock::sleep(Duration::from_secs(1)).await; let held = reader.frame_count(); assert!( (15..=18).contains(&held), @@ -94,14 +94,14 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { ); for index in 0..30 { - let frame = moq_clock::timeout(TIMEOUT, reader.read_frame()) + let frame = kio::clock::timeout(TIMEOUT, reader.read_frame()) .await .unwrap_or_else(|_| panic!("{version}: frame {index} never arrived")) .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) .expect("a frame"); assert_eq!(frame.payload.len(), MB, "{version}: frame {index}"); } - let end = moq_clock::timeout(TIMEOUT, reader.read_frame()) + let end = kio::clock::timeout(TIMEOUT, reader.read_frame()) .await .expect("end timeout"); assert!(matches!(end, Ok(None)), "{version}: {end:?}"); @@ -110,7 +110,7 @@ async fn an_idle_reader_paces_the_relay_and_loses_nothing() { /// A paused stream is not read, so the relay watches it for a reset instead: a publisher /// aborting the group ends the relay's copy while its reader still takes nothing. -#[moq_clock::test] +#[kio::test] async fn a_reset_reaches_a_paused_relay() { let payload = Bytes::from(vec![7u8; MB]); for version in VERSIONS { @@ -118,7 +118,7 @@ async fn a_reset_reaches_a_paused_relay() { let mut group = relay.track.append_group().unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() @@ -126,11 +126,11 @@ async fn a_reset_reaches_a_paused_relay() { for _ in 1..30 { group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); } - moq_clock::sleep(Duration::from_secs(1)).await; + kio::clock::sleep(Duration::from_secs(1)).await; assert!(reader.frame_count() < 30, "{version}: the relay never paused"); group.abort(moq_net::Error::Cancel).unwrap(); - let end = moq_clock::timeout(TIMEOUT, reader.finished()) + let end = kio::clock::timeout(TIMEOUT, reader.finished()) .await .unwrap_or_else(|_| panic!("{version}: the reset never reached the paused relay")); assert!(end.is_err(), "{version}: {end:?}"); @@ -139,7 +139,7 @@ async fn a_reset_reaches_a_paused_relay() { /// A frame larger than the room left waits to be admitted, since admitting it trims the /// tail to fit: an idle reader loses nothing to a large frame either. -#[moq_clock::test] +#[kio::test] async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { let payload = Bytes::from(vec![7u8; MB]); for version in VERSIONS { @@ -147,7 +147,7 @@ async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { let mut group = relay.track.append_group().unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - let mut reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() @@ -157,20 +157,20 @@ async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { } // Let the relay take what fits, so the publisher's egress has sent every frame the // large one sheds from the publisher's own tail. - moq_clock::sleep(Duration::from_secs(1)).await; + kio::clock::sleep(Duration::from_secs(1)).await; group .write_frame(Timestamp::ZERO, Bytes::from(vec![7u8; 20 * MB])) .unwrap(); group.finish().unwrap(); - moq_clock::sleep(Duration::from_secs(1)).await; + kio::clock::sleep(Duration::from_secs(1)).await; assert_eq!( reader.frame_count(), 15, "{version}: the large frame waits for the reader" ); for index in 0..16 { - let frame = moq_clock::timeout(TIMEOUT, reader.read_frame()) + let frame = kio::clock::timeout(TIMEOUT, reader.read_frame()) .await .unwrap_or_else(|_| panic!("{version}: frame {index} never arrived")) .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) @@ -183,7 +183,7 @@ async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { /// A session whose paused streams would hold more than half its connection window reads /// on instead, and its idle reader keeps what the live tail holds. -#[moq_clock::test] +#[kio::test] async fn a_session_past_its_credit_cap_reads_on() { let payload = Bytes::from(vec![7u8; MB]); for version in VERSIONS { @@ -194,7 +194,7 @@ async fn a_session_past_its_credit_cap_reads_on() { let mut group = relay.track.append_group().unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - let reader = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + let reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() @@ -204,13 +204,13 @@ async fn a_session_past_its_credit_cap_reads_on() { } group.finish().unwrap(); - moq_clock::sleep(Duration::from_secs(1)).await; + kio::clock::sleep(Duration::from_secs(1)).await; let held = reader.frame_count(); assert_eq!(held, 30, "{version}: the relay paused past its credit cap"); } } -#[moq_clock::test] +#[kio::test] async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { let payload = Bytes::from(vec![7u8; MB]); for version in VERSIONS { @@ -218,7 +218,7 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { let mut group = relay.track.append_group().unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - let mut fast = moq_clock::timeout(TIMEOUT, relay.subscriber.recv_group()) + let mut fast = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() @@ -230,7 +230,7 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { if index > 0 { group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); } - let frame = moq_clock::timeout(TIMEOUT, fast.read_frame()) + let frame = kio::clock::timeout(TIMEOUT, fast.read_frame()) .await .unwrap_or_else(|_| panic!("{version}: the fast reader stalled at frame {index}")) .unwrap_or_else(|err| panic!("{version}: frame {index}: {err}")) @@ -239,7 +239,7 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { } group.finish().unwrap(); - let fell = moq_clock::timeout(TIMEOUT, slow.read_frame()) + let fell = kio::clock::timeout(TIMEOUT, slow.read_frame()) .await .expect("read timeout"); assert!(