From 06678a5f71e75b9dd8890a94a0cae4f0c3328114 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 09:26:49 -0700 Subject: [PATCH 1/9] quest(m1): plan keeping a resumed group's head across a route change Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 + quest/m1/resume-prefix.md | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 quest/m1/resume-prefix.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 0230ba6414c..3823c4f2ae5 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -60,6 +60,7 @@ lands before the cut. - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly, except cases that use a subgroup other than 0 - [Refresh](/quest/m1/refresh/README.md) - a join asks for a fresh group once the current group's deltas exceed a publisher-advertised byte ratio, at every hop; replaces `Demand::used`/`unused` and moq-json's `delta_ratio` +- [Resumed group head](/quest/m1/resume-prefix.md) - a group resumed on another route keeps the frames the replaced route received, so the replacement's cache holds it whole - [Compressed tracks](/quest/m1/compress/README.md) - a track flagged `compress` is DEFLATEd per group by moq-net, forwarded compressed, and inflated once on the relay for peers that cannot signal it; `catalog.json.z` and moq-flate go away - [Moxygen minimal interop](/quest/m1/smoke-moxygen.md) - waiting on Moxygen to send a registered filter type, then the smoke lane stops hiding its failure - [Claim answer epochs](/quest/m1/claim-answer-epochs.md) - a claim answers each output by publishing it at its exact path under its own epoch, so a restarted output never reaches a returning reader as the old one's cache diff --git a/quest/m1/resume-prefix.md b/quest/m1/resume-prefix.md new file mode 100644 index 00000000000..8ed57ca9b9f --- /dev/null +++ b/quest/m1/resume-prefix.md @@ -0,0 +1,55 @@ +# [M] Keep a resumed group's head across a route change + +## Goal + +When a front resumes a group on another route of the same epoch, the +replacement copy adopts frames 0..N-1 the replaced route already received and +fetches only N onward. The replacement route's cached group is then whole, so +a later reader or fetch of it from frame 0 is a cache hit, the same as on a +relay that never saw the route change. This holds for every group, compressed +or not, and for a replaced copy that failed as well as one that stalled. + +Boundaries: + +- A plain (non-front) consumer of an aborted group still pins nothing: the + abort releases its frames, and only readers that can resume keep the head + until they hand it over. +- Cache policy (expiry, eviction, too old) still frees the frames it drops; + only a copy that failed or was given up keeps its head for a successor. +- Not compression. [Compressed tracks](/quest/m1/compress/core.md) builds on + the adopted head to rebuild a group's inflater; this quest only names the + hook. + +## Plan + +Decided with the maintainer: frames 0..N-1 survive a route change for every +group; compression is not special. + +- A group handed out from a front's logical track holds a shared hand-off + for its copy. Aborting that copy (other than by cache policy), or dropping + it unfinished, moves the committed frames into the hand-off instead of + dropping them; a plain consumer still reads the abort, never the frames. + The hand-off is freed with the last reader holding it, so nothing outlives + the readers that could use it. +- On resume, the reader adopts the head into the replacement group before + reading from it: frames below where that group starts, from the stalled + copy's own cache or from the hand-off. The fetch still asks from the + reader's frame, so no wire, handler, or request shape changes. +- Adoption bills the replacement's cache, refuses an overflow by skipping + the adoption, and keeps IETF object fields and the first-object marker + aligned. Sparse groups already refuse ordinal recovery and stay that way. +- js/net has no route resume on main ([JS track handover](/quest/m1/js-group-handover.md) + ports it), so there is nothing to mirror yet; that quest mirrors this + contract. + +Tests: a failed and a stalled route both leave the replacement's cache whole, +so a later fetch from frame 0 is a hit; a plain consumer of the aborted group +still reads the abort with nothing pinned; a cache-policy abort keeps +nothing. Mocked time, no sleeps. + +Public API: none. Wire: none. + +## Related + +- [Compressed tracks in Rust moq-net](/quest/m1/compress/core.md) - rebuilds a resumed group's inflater from the adopted head +- [JS track handover](/quest/m1/js-group-handover.md) - mirrors route resume, including this, in js/net From 79396ca4f7573adad218b048e3c472cf025e9802 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 09:36:39 -0700 Subject: [PATCH 2/9] fix(net): a resumed group keeps the head the replaced route received When a front resumes a group on another route, the replacement copy now adopts frames 0..N-1 the replaced copy held and fetches only N onward, so the serving route caches the group whole, as a cold relay would. A group handed out from a logical track shares a hand-off with its readers. A release that is no cache policy moves the committed frames into it rather than dropping them, so a plain consumer still reads the abort and pins nothing, and the frames go with the last resuming reader. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/resume-prefix.md | 55 ------ rs/moq-net/src/model/group.rs | 294 ++++++++++++++++++++++++++++++++- rs/moq-net/src/model/resume.rs | 164 ++++++++++++++++-- rs/moq-net/src/model/track.rs | 20 +++ 5 files changed, 459 insertions(+), 75 deletions(-) delete mode 100644 quest/m1/resume-prefix.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 3823c4f2ae5..0230ba6414c 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -60,7 +60,6 @@ lands before the cut. - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly, except cases that use a subgroup other than 0 - [Refresh](/quest/m1/refresh/README.md) - a join asks for a fresh group once the current group's deltas exceed a publisher-advertised byte ratio, at every hop; replaces `Demand::used`/`unused` and moq-json's `delta_ratio` -- [Resumed group head](/quest/m1/resume-prefix.md) - a group resumed on another route keeps the frames the replaced route received, so the replacement's cache holds it whole - [Compressed tracks](/quest/m1/compress/README.md) - a track flagged `compress` is DEFLATEd per group by moq-net, forwarded compressed, and inflated once on the relay for peers that cannot signal it; `catalog.json.z` and moq-flate go away - [Moxygen minimal interop](/quest/m1/smoke-moxygen.md) - waiting on Moxygen to send a registered filter type, then the smoke lane stops hiding its failure - [Claim answer epochs](/quest/m1/claim-answer-epochs.md) - a claim answers each output by publishing it at its exact path under its own epoch, so a restarted output never reaches a returning reader as the old one's cache diff --git a/quest/m1/resume-prefix.md b/quest/m1/resume-prefix.md deleted file mode 100644 index 8ed57ca9b9f..00000000000 --- a/quest/m1/resume-prefix.md +++ /dev/null @@ -1,55 +0,0 @@ -# [M] Keep a resumed group's head across a route change - -## Goal - -When a front resumes a group on another route of the same epoch, the -replacement copy adopts frames 0..N-1 the replaced route already received and -fetches only N onward. The replacement route's cached group is then whole, so -a later reader or fetch of it from frame 0 is a cache hit, the same as on a -relay that never saw the route change. This holds for every group, compressed -or not, and for a replaced copy that failed as well as one that stalled. - -Boundaries: - -- A plain (non-front) consumer of an aborted group still pins nothing: the - abort releases its frames, and only readers that can resume keep the head - until they hand it over. -- Cache policy (expiry, eviction, too old) still frees the frames it drops; - only a copy that failed or was given up keeps its head for a successor. -- Not compression. [Compressed tracks](/quest/m1/compress/core.md) builds on - the adopted head to rebuild a group's inflater; this quest only names the - hook. - -## Plan - -Decided with the maintainer: frames 0..N-1 survive a route change for every -group; compression is not special. - -- A group handed out from a front's logical track holds a shared hand-off - for its copy. Aborting that copy (other than by cache policy), or dropping - it unfinished, moves the committed frames into the hand-off instead of - dropping them; a plain consumer still reads the abort, never the frames. - The hand-off is freed with the last reader holding it, so nothing outlives - the readers that could use it. -- On resume, the reader adopts the head into the replacement group before - reading from it: frames below where that group starts, from the stalled - copy's own cache or from the hand-off. The fetch still asks from the - reader's frame, so no wire, handler, or request shape changes. -- Adoption bills the replacement's cache, refuses an overflow by skipping - the adoption, and keeps IETF object fields and the first-object marker - aligned. Sparse groups already refuse ordinal recovery and stay that way. -- js/net has no route resume on main ([JS track handover](/quest/m1/js-group-handover.md) - ports it), so there is nothing to mirror yet; that quest mirrors this - contract. - -Tests: a failed and a stalled route both leave the replacement's cache whole, -so a later fetch from frame 0 is a hit; a plain consumer of the aborted group -still reads the abort with nothing pinned; a cache-policy abort keeps -nothing. Mocked time, no sleeps. - -Public API: none. Wire: none. - -## Related - -- [Compressed tracks in Rust moq-net](/quest/m1/compress/core.md) - rebuilds a resumed group's inflater from the adopted head -- [JS track handover](/quest/m1/js-group-handover.md) - mirrors route resume, including this, in js/net diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 96a887e9c29..13e9bda725e 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -20,8 +20,8 @@ use crate::{Cap, Timescale, stats, track}; use std::collections::VecDeque; use std::mem::MaybeUninit; use std::ops::{Bound, RangeBounds}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError, Weak}; use std::task::{Poll, ready}; use crate::{Error, IntoBytes, Result, Timestamp}; @@ -187,6 +187,43 @@ pub(super) struct GroupState { // `Alive::aborted`, so [`Producer::abort`] stays the only writer: anything else // setting this would leave track scans reading a group as live. pub(super) abort: Option, + + // The readers that could resume this group on another route. A release that is no + // cache policy moves the frames to them rather than dropping them. + handoff: Mutex>, + + // Whether an adopted head (see [`Producer::adopt_head`]) begins at the group's first + // object, for a wire header that arrives after it. + adopted: Option, +} + +/// The frames a copy of a group holds, from where it starts, which a successor copy of +/// the same group on another route adopts below where it starts. +pub(super) struct Head { + start: usize, + // Whether `start` is the group's first object. + first_object: bool, + frames: Vec, + // Relay metadata for `frames`, at the same index; frames past its end carry none. + objects: Vec>>, +} + +/// A copy's [`Head`], shared by every reader that could resume its group elsewhere. +/// +/// Read from the copy while it holds its frames. Once it fails, its release moves them +/// here instead, so a plain consumer of the failed copy still reads the abort and pins +/// nothing, and the frames go with the last reader holding this. +pub(super) struct Handoff { + state: kio::ConsumerWeak, + salvaged: Mutex>>, +} + +impl Handoff { + /// The copy's head, if it holds any frames, or held them when it failed. + pub(super) fn head(&self) -> Option> { + let salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner).clone(); + salvaged.or_else(|| self.state.read().head().map(Arc::new)) + } } /// What an IETF relay keeps beside a group's frames. @@ -493,8 +530,49 @@ impl GroupState { self.charge.add(written + slots) } + /// Whether the frames held start at the group's first object. + fn first_object(&self) -> bool { + self.wire().map_or(self.offset == 0, |wire| wire.first_object) + } + + /// The committed frames, for a successor copy to adopt. Sparse Object IDs refuse + /// ordinal recovery, so they lend nothing. + fn head(&self) -> Option { + if self.frames.is_empty() || self.sparse() { + return None; + } + Some(Head { + start: self.offset, + first_object: self.first_object(), + frames: self.frames.iter().cloned().collect(), + objects: self + .relay + .as_ref() + .map_or_else(Vec::new, |relay| relay.objects.iter().cloned().collect()), + }) + } + /// Drop the cached frames (and any in-flight tail) and release their pool charge. - fn release(&mut self) { + /// + /// With `salvage`, the committed frames move to the readers that could resume the + /// group elsewhere, if any are left, instead of being dropped. + fn release(&mut self, salvage: bool) { + let handoff = match salvage && !self.frames.is_empty() && !self.sparse() { + true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), + false => None, + }; + if let Some(handoff) = handoff { + let head = Head { + start: self.offset, + first_object: self.first_object(), + frames: std::mem::take(&mut self.frames).into(), + objects: self + .relay + .as_mut() + .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), + }; + *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(head)); + } self.frames = VecDeque::new(); self.partial = None; self.cache = 0; @@ -606,7 +684,7 @@ impl Drop for Alive { sequence = self.info.sequence, "group::Producer dropped without finish() or abort()" ); - state.release(); + state.release(true); } Err(state) => { if state.fin.is_some() || state.abort.is_some() { @@ -638,8 +716,15 @@ impl Producer { #[cfg(feature = "async")] // private: passthrough - pub(crate) fn set_wire(&self, wire: Wire) -> Result<()> { - modify(&self.state)?.relay().wire = Some(wire); + pub(crate) fn set_wire(&self, mut wire: Wire) -> Result<()> { + let mut state = modify(&self.state)?; + // A head adopted before the header arrived holds the objects below the stream's. + if let Some(first_object) = state.adopted { + wire.first_object = first_object; + wire.first_id = state.offset as u64; + wire.last = wire.last.or((state.next_index as u64).checked_sub(1)); + } + state.relay().wire = Some(wire); Ok(()) } @@ -784,6 +869,72 @@ impl Producer { Ok(()) } + /// Adopt the frames of `head` below where this copy starts, so it holds the group + /// from `head`'s start, as if this route had delivered them. + /// + /// A no-op unless `head` starts below this copy and reaches it, and the frames fit the + /// group's budget. Readers keep their positions, since frames keep their indices. A + /// group whose frames build on earlier ones (a compressed group) primes its decoder + /// from the adopted frames here. + pub(super) fn adopt_head(&self, head: &Head) { + let Ok(mut state) = self.state.write() else { + return; + }; + let offset = state.offset; + let reaches = head.start.saturating_add(head.frames.len()) >= offset; + if state.abort.is_some() || state.sparse() || head.start >= offset || !reaches { + return; + } + let count = offset - head.start; + let frames = &head.frames[..count]; + let fields = || head.objects.iter().take(count).flatten(); + let payload: u64 = frames.iter().map(|frame| frame.payload.len() as u64).sum(); + let metadata: u64 = fields().map(|fields| fields.heap()).sum(); + if state.would_overflow(count, payload + metadata) { + return; + } + + let before = state.frames.capacity().saturating_sub(FRAME_SLOTS); + state.frames.reserve(count); + let mut growth = ((state.frames.capacity().saturating_sub(FRAME_SLOTS) - before) * size_of::()) as u64; + for frame in frames.iter().rev() { + state.frames.push_front(frame.clone()); + } + // Keep the fields aligned with the frames: the adopted ones lead. + if fields().next().is_some() || state.relay.as_ref().is_some_and(|relay| !relay.objects.is_empty()) { + let relay = state.relay(); + let before = relay.objects.capacity(); + relay.objects.reserve(count); + let slots = ((relay.objects.capacity() - before) * size_of::>>()) as u64; + for index in (0..count).rev() { + relay.objects.push_front(head.objects.get(index).cloned().flatten()); + } + relay.bytes += slots + metadata; + growth += slots; + } + state.offset = head.start; + state.cache += payload; + let start = frames.iter().find_map(|frame| frame.timestamp); + state.timeline = match (state.timeline, start) { + (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, latest }, + (Timeline::Empty, Some(start)) => Timeline::Timed { + start, + latest: frames.iter().rev().find_map(|frame| frame.timestamp).unwrap_or(start), + }, + (Timeline::Empty, None) => Timeline::Untimed, + (timeline, _) => timeline, + }; + state.adopted = Some(head.first_object); + if let Some(wire) = state.wire_mut() { + wire.first_object = head.first_object; + wire.first_id = head.start as u64; + wire.last = wire.last.or(Some(offset as u64 - 1)); + } + let now = state.charge.add(payload + metadata + growth); + drop(state); + self.cache.settle(now); + } + /// A helper method to write a frame from a single byte buffer. /// /// If you want to write multiple chunks, use [Self::create_frame] to get a frame producer. @@ -1190,7 +1341,8 @@ impl Producer { guard.abort = Some(err); self.alive.cut.store(cut, Ordering::Release); self.alive.aborted.store(true, Ordering::Release); - guard.release(); + // Cache policy frees the frames; a copy that failed or was given up lends them on. + guard.release(cut); guard.close(); self.cache.aborted(self.sequence, cut); } @@ -1202,7 +1354,7 @@ impl Producer { state.abort = Some(err.clone()); self.alive.cut.store(true, Ordering::Release); self.alive.aborted.store(true, Ordering::Release); - state.release(); + state.release(false); state.close(); self.cache.aborted(self.sequence, true); err @@ -1760,7 +1912,6 @@ impl Consumer { } /// The index of the first frame the group holds (see [`Producer::start_at`]). - #[cfg(feature = "async")] // private: passthrough pub(crate) fn first_frame(&self) -> u64 { self.cursor.state.read().offset as u64 @@ -1813,6 +1964,22 @@ impl Consumer { self } + /// The hand-off of this copy's head, shared by every reader that could resume it + /// elsewhere. + pub(super) fn handoff(&self) -> Arc { + let state = self.cursor.state.read(); + let mut handoff = state.handoff.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(handoff) = handoff.upgrade() { + return handoff; + } + let shared = Arc::new(Handoff { + state: self.cursor.state.weak(), + salvaged: Mutex::default(), + }); + *handoff = Arc::downgrade(&shared); + shared + } + /// Carry this group across a front's route changes; see [`super::resume`]. pub(super) fn with_recover(mut self, recover: super::resume::Recover) -> Self { self.recover = Some(Box::new(recover)); @@ -2899,6 +3066,117 @@ mod test { assert_eq!(state.cache, 0); } + /// A failed copy moves its frames to the readers that could resume it, so they outlive + /// the release only as long as one of those readers does; cache policy keeps nothing. + #[test] + fn an_abort_hands_the_head_to_its_resumers() { + for (err, kept) in [(Error::Cancel, true), (Error::Evicted, false)] { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + producer.write_frame(None, Bytes::from_static(b"b")).unwrap(); + let handoff = producer.consume().handoff(); + assert_eq!(handoff.head().map(|head| head.frames.len()), Some(2), "read live"); + + producer.clone().abort(err.clone()).unwrap(); + assert!(producer.state.read().frames.is_empty(), "{err}: released"); + assert_eq!(pool.used(), 0, "{err}: uncharged"); + let head = handoff.head(); + assert_eq!(head.is_some(), kept, "{err}"); + if let Some(head) = head { + assert_eq!((head.start, head.first_object, head.frames.len()), (0, true, 2)); + let held = Arc::downgrade(&head); + drop((head, handoff)); + assert!(held.upgrade().is_none(), "freed with the last resumer"); + } + } + } + + /// An adopted head leads the fields and the wire header, whether the header was set + /// before the adoption or arrives after it. + #[test] + #[cfg(feature = "async")] + fn an_adopted_head_keeps_fields_and_the_wire_aligned() { + let fields = |id| { + Some(Arc::new(super::super::object::Fields { + id, + properties: Default::default(), + })) + }; + let head = Head { + start: 0, + first_object: true, + frames: vec![ + Frame { + timestamp: None, + payload: Bytes::from_static(b"a"), + }, + Frame { + timestamp: None, + payload: Bytes::from_static(b"b"), + }, + ], + objects: vec![None, fields(1)], + }; + let wire = Wire { + sparse: false, + first_object: false, + first_id: 2, + extensions: true, + has_end: true, + fetchable: true, + last: None, + }; + for wired_first in [true, false] { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut tail = Producer::new(Info::from(0u64), track::Info::default(), cache); + tail.start_at(2).unwrap(); + if wired_first { + tail.set_wire(wire.clone()).unwrap(); + } + let mut reader = tail.consume(); + reader.start_at(2); + let before = pool.used(); + tail.adopt_head(&head); + assert!(pool.used() >= before + 2, "the adopted frames are charged"); + if !wired_first { + tail.set_wire(wire.clone()).unwrap(); + } + let written = tail.state.read().wire().cloned().unwrap(); + assert_eq!( + (written.first_object, written.first_id, written.last), + (true, 0, Some(1)) + ); + tail.mark_gap(2, None).unwrap(); + assert!(tail.state.read().wire().unwrap().fetchable, "no gap below the stream"); + tail.write_frame(None, Bytes::from_static(b"c")).unwrap(); + tail.finish().unwrap(); + + let waiter = kio::Waiter::noop(); + let mut whole = tail.consume(); + let mut read = Vec::new(); + while let Poll::Ready(Ok(Some(object))) = whole.poll_read_object(&waiter) { + read.push((object.frame.payload, object.fields.map(|fields| fields.id))); + } + assert_eq!( + read, + [ + (Bytes::from_static(b"a"), None), + (Bytes::from_static(b"b"), Some(1)), + (Bytes::from_static(b"c"), None) + ] + ); + let next = reader.read_frame().now_or_never().unwrap().unwrap(); + assert_eq!( + next.map(|frame| frame.payload), + Some(Bytes::from_static(b"c")), + "a reader keeps its place" + ); + } + } + #[test] #[cfg(feature = "async")] fn drop_unfinished_clears_cached_frames() { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 953e19bb42d..d74cd31a3b7 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -589,6 +589,7 @@ impl Reader { let sequence = group.sequence; self.newest = self.newest.max(Some(sequence)); let copy = &self.copies[index]; + let handoff = group.handoff(); group.with_recover(Recover { sequence, reader: Some(self.this.clone()), @@ -596,6 +597,7 @@ impl Reader { route: self.route.clone(), generation: copy.generation, copy: copy.track.clone(), + handoff, lease: Some(copy.lease.clone()), logical: None, fetch: None, @@ -884,6 +886,9 @@ pub(super) struct Recover { /// The route generation of the copy being read, and that copy. generation: u64, copy: track::Consumer, + /// The head of the group as that copy holds it, which the copy continuing it adopts, + /// so the serving route caches the group whole. + handoff: Arc, /// Keeps a replaced copy subscribed while this group is read from it. lease: Option>, /// A fetch's logical track, held while the group is read. A fetch is no subscription, @@ -909,6 +914,7 @@ impl Clone for Recover { route: self.route.clone(), generation: self.generation, copy: self.copy.clone(), + handoff: self.handoff.clone(), lease: self.lease.clone(), logical: self.logical.clone(), // A clone reads in parallel, so it asks for itself. @@ -1014,6 +1020,7 @@ impl Recover { // failed delivered it already, so its subscription will not again. let same = generation == self.generation; if !same && let Poll::Ready(Some(mut group)) = serving.poll_group(self.sequence, waiter) { + lend_head(&self.handoff, self.sequence, serving, &group); group.start_at(index); if group.index() == index { return Poll::Ready(Ok(Replacement { @@ -1042,6 +1049,7 @@ impl Recover { Fetch::Refused(err) => err.clone(), Fetch::Pending(pending) => match ready!(kio::Task::poll(&mut **pending, waiter)) { Ok(mut group) => { + lend_head(&self.handoff, self.sequence, serving, &group); group.start_at(index); if group.index() == index { return Poll::Ready(Ok(Replacement { @@ -1091,6 +1099,7 @@ impl Recover { pub(super) fn adopt(&mut self, replacement: &Replacement) { self.generation = replacement.generation; self.copy = replacement.copy.clone(); + self.handoff = replacement.group.handoff(); // A later switch must keep this copy subscribed while the recovered group // still reads it, just like a group handed out from that copy directly. self.lease = self.reader.as_ref().and_then(Weak::upgrade).and_then(|reader| { @@ -1105,6 +1114,16 @@ impl Recover { } } +/// Lend the head `handoff` holds to `serving`'s copy of group `sequence`, `group`, when that +/// starts later: the fetch for the rest asks only from the reader's frame. +fn lend_head(handoff: &group::Handoff, sequence: u64, serving: &track::Consumer, group: &group::Consumer) { + if group.first_frame() > 0 + && let Some(head) = handoff.head() + { + serving.adopt_head(sequence, &head); + } +} + /// A candidate cursor, committed only when the requested read can use it. pub(super) struct Replacement { pub(super) group: group::Consumer, @@ -1158,17 +1177,21 @@ impl kio::Task for Fetching { } }; return match result { - Ok(group) => Poll::Ready(Ok(group.with_recover(Recover { - sequence: self.sequence, - reader: None, - max_delay: self.options.max_delay, - route: self.route.clone(), - generation, - copy: serving.clone(), - lease: None, - logical: Some(self.logical.clone()), - fetch: None, - }))), + Ok(group) => { + let handoff = group.handoff(); + Poll::Ready(Ok(group.with_recover(Recover { + sequence: self.sequence, + reader: None, + max_delay: self.options.max_delay, + route: self.route.clone(), + generation, + copy: serving.clone(), + handoff, + lease: None, + logical: Some(self.logical.clone()), + fetch: None, + }))) + } // The route failed: ask the next one, unless none will come. Err(err) if (route_failed(&err) || copy_failed(&serving)) && self.route.end().is_none() => { ready!(self.route.poll_changed(generation, waiter)); @@ -1365,6 +1388,125 @@ mod test { assert!(chunks.read_chunk().now_or_never().unwrap().unwrap().is_none()); } + /// Read a fetched group from frame 0, which must already be cached. + #[cfg(feature = "async")] + fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { + let mut group = track + .fetch_group(sequence, None) + .now_or_never() + .expect("the whole group is cached") + .unwrap(); + std::iter::from_fn(|| read(&mut group)).collect() + } + + /// The replacement adopts the head the failed route received, so its cache holds the + /// whole group, as on a relay that never saw the route change. + #[test] + #[cfg(feature = "async")] + fn a_failed_route_hands_its_head_to_the_replacement() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + open.write_frame(ts(1), b"b".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + let mut plain = open.consume(); + open.abort(Error::Dropped).unwrap(); + assert!( + matches!(plain.read_frame().now_or_never(), Some(Err(Error::Dropped))), + "a plain consumer reads the abort, not the head" + ); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!((request.sequence(), request.frame_start()), (0, 2)); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(2), b"c".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"c".to_vec())); + assert_eq!(read(&mut reading), None); + + let whole = [b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]; + assert_eq!(read_cached(&b.consume(), 0), whole); + assert_eq!(read_cached(&logical.consume(), 0), whole); + assert!(dynamic.requested_group().now_or_never().is_none(), "no second fetch"); + } + + /// A replaced route that only stalled lends the replacement its head the same way. + #[test] + #[cfg(feature = "async")] + fn a_stalled_route_lends_its_head_to_the_replacement() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!(request.frame_start(), 1); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + + assert_eq!(read_cached(&b.consume(), 0), [b"a".to_vec(), b"b".to_vec()]); + } + + /// Cache policy dropping a group frees its frames: nothing is kept for a successor. + #[test] + #[cfg(feature = "async")] + fn an_evicted_group_hands_nothing_to_the_replacement() { + let routes = Producer::new(); + let a = copy(); + routes.serve(a.consume()); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut fetched = logical(&routes) + .consume() + .fetch_group(0, None) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(read(&mut fetched), Some(b"a".to_vec())); + open.abort(Error::Evicted).unwrap(); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert!(fetched.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!(request.frame_start(), 1); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut fetched), Some(b"b".to_vec())); + assert_eq!(read(&mut fetched), None); + let _whole = b.consume().fetch_group(0, None); + assert!( + dynamic.requested_group().now_or_never().is_some(), + "the head was not kept" + ); + } + #[test] #[cfg(feature = "async")] fn a_group_both_routes_deliver_is_handed_out_once() { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 478270f7552..4d3bd569339 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3657,6 +3657,26 @@ impl Consumer { }) } + /// Lend `head` to the cached copies of group `sequence`; see + /// [`group::Producer::adopt_head`]. + pub(super) fn adopt_head(&self, sequence: u64, head: &group::Head) { + let copies: Vec = self.with_state(|state| { + let Some(state) = state else { + return Vec::new(); + }; + let state = state.read(); + state + .lookup + .get(&sequence) + .map_or_else(Vec::new, |slot| slot.copies().cloned().collect()) + }); + // With the track lock released (lock order is track then group): adopting settles + // the group's cache debt. + for group in copies { + group.adopt_head(head); + } + } + /// [`Self::poll_group`], except a newer group is no verdict: QUIC does not order /// streams, so this one may still land behind it. Pending until it is cached, or the /// feed starts past it or ends before it. From 307d50c8f319c30b3c6ba84fd31018e11ccfae01 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 10:42:15 -0700 Subject: [PATCH 3/9] fix(net): bill a held head and expire it with the pool window Nothing bounds how many group consumers a stalled downstream egress holds, so an unbilled hand-off let a peer that forces route changes pin failed groups' heads without limit. A held head is now billed to its track's pool, refused when the pool has no room for it, and dropped by the sweep once the pool's idle window passes, the same bounds as cached content. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/cache.rs | 66 ++++++++++++++++++++++ rs/moq-net/src/model/group.rs | 101 +++++++++++++++++++++++++++++----- 2 files changed, 152 insertions(+), 15 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index ceedefb22d0..90ec9c4392f 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -646,6 +646,10 @@ pub(super) struct Track { // Here rather than on the track state so a frame write advances it without the track // lock, and wakes only the readers following it. largest: kio::Shared, + + // Hand-offs holding a failed group's head for a successor route, which the sweep + // expires like idle groups. Only filled while the pool sweeps. + held: Mutex>>, } /// The greatest location a track has written or its source reported, followed by the track @@ -739,6 +743,7 @@ impl Track { wakes: Default::default(), aborts: OnceLock::new(), largest: Default::default(), + held: Mutex::default(), }); if let Some(key) = track.pool.register(&track) { let _ = track.sweep.set(key); @@ -801,6 +806,39 @@ impl Track { } } + /// Bill `bytes` of a failed group's head while `handoff` holds it for a successor route. + /// + /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed + /// like cached content and bounded the same way: `None` when it would take the pool past + /// its capacity, and the sweep drops it once the pool's idle window passes. + pub(super) fn hold(self: &Arc, handoff: &Arc, bytes: u64) -> Option { + if self + .pool + .capacity() + .is_some_and(|capacity| self.pool.used().saturating_add(bytes) > capacity) + { + return None; + } + if self.sweep.get().is_some() { + self.held + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(Arc::downgrade(handoff)); + } + self.pool.add(bytes); + Some(Bill { + track: self.clone(), + bytes, + at: self.pool.now(), + }) + } + + /// Drop the held heads past the pool's idle window, forgetting the hand-offs done with. + fn expire_held(&self) { + let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); + held.retain(|handoff| handoff.upgrade().is_some_and(|handoff| !handoff.expire())); + } + /// Take everything written since the last call, to be turned into eviction debt. pub(super) fn take_written(&self) -> u64 { self.written.swap(0, Ordering::Relaxed) @@ -823,6 +861,7 @@ impl Track { /// Settle from [`Pool::sweep`], dating activity and expiring every idle candidate. pub(super) fn sweep(&self) { + self.expire_held(); self.settle_inner(None, true); } @@ -911,6 +950,28 @@ impl Drop for Track { } } +/// The bytes a held head of a failed group bills to its track's pool, released on drop. +pub(super) struct Bill { + track: Arc, + bytes: u64, + // The coarse tick it was billed at. + at: u64, +} + +impl Bill { + /// Whether the pool's idle window passed since it was billed. + pub(super) fn expired(&self) -> bool { + let pool = &self.track.pool; + pool.now().saturating_sub(self.at) >= pool.expiry_ticks() + } +} + +impl Drop for Bill { + fn drop(&mut self) { + self.track.pool.sub(self.bytes); + } +} + /// The RAII byte accounting for one cached group, owned by the group's state. /// /// `add`/`sub` mirror the group's cached payload bytes into the pool with plain @@ -1077,6 +1138,11 @@ impl Charge { Some(target >> ACCESS_SHIFT) } + /// The track account this charge bills, unless it is detached. + pub(super) fn account(&self) -> Option<&Arc> { + self.track.as_ref() + } + /// Release everything this charge holds: bytes, overhead, and the access /// sample. Idempotent; used when the group aborts and clears its frames. pub(super) fn clear(&mut self) { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 13e9bda725e..a3dca5bf722 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -215,15 +215,43 @@ pub(super) struct Head { /// nothing, and the frames go with the last reader holding this. pub(super) struct Handoff { state: kio::ConsumerWeak, - salvaged: Mutex>>, + salvaged: Mutex>, +} + +/// A failed copy's head, billed to its pool while held; see [`cache::Track::hold`]. +struct Salvage { + head: Arc, + bill: cache::Bill, } impl Handoff { /// The copy's head, if it holds any frames, or held them when it failed. pub(super) fn head(&self) -> Option> { - let salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner); + let salvaged = salvaged.as_ref().map(|salvage| salvage.head.clone()); salvaged.or_else(|| self.state.read().head().map(Arc::new)) } + + /// Drop the held head once the pool's idle window passed, returning whether none is + /// held any more. + pub(super) fn expire(&self) -> bool { + let mut salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner); + if salvaged.as_ref().is_some_and(|salvage| salvage.bill.expired()) { + *salvaged = None; + } + salvaged.is_none() + } +} + +impl Head { + /// Roughly what holding it costs: payloads, fields, and their slots. + fn bytes(&self) -> u64 { + let payload: u64 = self.frames.iter().map(|frame| frame.payload.len() as u64).sum(); + let fields: u64 = self.objects.iter().flatten().map(|fields| fields.heap()).sum(); + let slots = self.frames.len() * size_of::() + + self.objects.len() * size_of::>>(); + payload + fields + slots as u64 + } } /// What an IETF relay keeps beside a group's frames. @@ -561,18 +589,15 @@ impl GroupState { true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), false => None, }; - if let Some(handoff) = handoff { - let head = Head { - start: self.offset, - first_object: self.first_object(), - frames: std::mem::take(&mut self.frames).into(), - objects: self - .relay - .as_mut() - .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), - }; - *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(head)); - } + let head = handoff.as_ref().map(|_| Head { + start: self.offset, + first_object: self.first_object(), + frames: std::mem::take(&mut self.frames).into(), + objects: self + .relay + .as_mut() + .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), + }); self.frames = VecDeque::new(); self.partial = None; self.cache = 0; @@ -583,6 +608,19 @@ impl GroupState { relay.bytes = size_of::() as u64 + relay.end_object.as_ref().map_or(0, |fields| fields.heap()); self.charge.add(relay.bytes); } + // Billed once the group's own charge is gone, so the head only takes its place. + if let (Some(handoff), Some(head)) = (handoff, head) + && let Some(bill) = self + .charge + .account() + .and_then(|track| track.hold(&handoff, head.bytes())) + { + let salvage = Salvage { + head: Arc::new(head), + bill, + }; + *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(salvage); + } } } @@ -3081,7 +3119,7 @@ mod test { producer.clone().abort(err.clone()).unwrap(); assert!(producer.state.read().frames.is_empty(), "{err}: released"); - assert_eq!(pool.used(), 0, "{err}: uncharged"); + assert_eq!(pool.used() > 0, kept, "{err}: billed only while held"); let head = handoff.head(); assert_eq!(head.is_some(), kept, "{err}"); if let Some(head) = head { @@ -3089,10 +3127,43 @@ mod test { let held = Arc::downgrade(&head); drop((head, handoff)); assert!(held.upgrade().is_none(), "freed with the last resumer"); + assert_eq!(pool.used(), 0, "unbilled with it"); } } } + /// A resuming reader that stops polling holds a failed copy's head no longer than an + /// idle group stays cached. + #[test] + fn a_held_head_expires_with_the_pool_window() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.clone().abort(Error::Cancel).unwrap(); + drop(producer); + assert!(handoff.head().is_some()); + + pool.step(std::time::Duration::from_secs(2)); + pool.sweep(); + assert!(handoff.head().is_none(), "the sweep drops a held head past the window"); + assert_eq!(pool.used(), 0); + } + + /// A failed copy's head is held only while the pool has room for it. + #[test] + fn a_full_pool_holds_no_head() { + let pool = cache::Pool::new(cache::Config::default().with_capacity(1)); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.clone().abort(Error::Cancel).unwrap(); + assert!(handoff.head().is_none()); + assert_eq!(pool.used(), 0); + } + /// An adopted head leads the fields and the wire header, whether the header was set /// before the adoption or arrives after it. #[test] From f0b20321bfd50b037d334fa800e488162c3c895f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:12:10 -0700 Subject: [PATCH 4/9] fix(net): lock a hand-off in release's order and reserve an open frame from adoption - Handoff::head takes the group's lock before the salvage, as release does, so it neither deadlocks against a concurrent release nor sees the head in neither place. - A held head registers for expiry only once stored, so a sweep in between cannot forget it. - Adoption counts the open frame's unwritten remainder against the group budget. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/cache.rs | 26 ++++++++------ rs/moq-net/src/model/group.rs | 64 ++++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index 90ec9c4392f..11f1b3ec982 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -806,12 +806,12 @@ impl Track { } } - /// Bill `bytes` of a failed group's head while `handoff` holds it for a successor route. + /// Bill `bytes` of a failed group's head while a hand-off holds it for a successor route. /// /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed /// like cached content and bounded the same way: `None` when it would take the pool past - /// its capacity, and the sweep drops it once the pool's idle window passes. - pub(super) fn hold(self: &Arc, handoff: &Arc, bytes: u64) -> Option { + /// its capacity, and [`Self::expire_held`] drops it once the pool's idle window passes. + pub(super) fn hold(self: &Arc, bytes: u64) -> Option { if self .pool .capacity() @@ -819,12 +819,6 @@ impl Track { { return None; } - if self.sweep.get().is_some() { - self.held - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(Arc::downgrade(handoff)); - } self.pool.add(bytes); Some(Bill { track: self.clone(), @@ -833,8 +827,18 @@ impl Track { }) } + /// Have the sweep drop `handoff`'s held head once the pool's idle window passes. + pub(super) fn expire_held(&self, handoff: &Arc) { + if self.sweep.get().is_some() { + self.held + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(Arc::downgrade(handoff)); + } + } + /// Drop the held heads past the pool's idle window, forgetting the hand-offs done with. - fn expire_held(&self) { + fn sweep_held(&self) { let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); held.retain(|handoff| handoff.upgrade().is_some_and(|handoff| !handoff.expire())); } @@ -861,7 +865,7 @@ impl Track { /// Settle from [`Pool::sweep`], dating activity and expiring every idle candidate. pub(super) fn sweep(&self) { - self.expire_held(); + self.sweep_held(); self.settle_inner(None, true); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index a3dca5bf722..6e64ec22a22 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -227,9 +227,13 @@ struct Salvage { impl Handoff { /// The copy's head, if it holds any frames, or held them when it failed. pub(super) fn head(&self) -> Option> { - let salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner); - let salvaged = salvaged.as_ref().map(|salvage| salvage.head.clone()); - salvaged.or_else(|| self.state.read().head().map(Arc::new)) + // The group's lock first, in `release`'s order: under it the frames are either + // still in the copy or already moved here, never in neither. + let state = self.state.read(); + let salvaged = (self.salvaged.lock().unwrap_or_else(PoisonError::into_inner)) + .as_ref() + .map(|salvage| salvage.head.clone()); + salvaged.or_else(|| state.head().map(Arc::new)) } /// Drop the held head once the pool's idle window passed, returning whether none is @@ -330,17 +334,21 @@ impl GroupState { !matches!(self.timeline, Timeline::Empty) || self.fin.is_some() || self.abort.is_some() } + /// The bytes the in-flight frame declared but has not written yet, which its open + /// already admitted. + fn unwritten(&self) -> u64 { + self.partial + .as_ref() + .map_or(0, |partial| partial.buf.size() as u64 - partial.charged) + } + /// Content still available to a reader of this group. /// /// Counts the in-flight frame at its declared size, like [`Self::content_range`]: a /// reader that skips it misses the whole frame, however much has arrived. fn content(&self) -> stats::Content { - let unwritten = self - .partial - .as_ref() - .map_or(0, |partial| partial.buf.size() as u64 - partial.charged); stats::Content { - bytes: self.cache + unwritten, + bytes: self.cache + self.unwritten(), frames: self.next_index.saturating_sub(self.offset) as u64, groups: 1, datagrams: 0, @@ -610,16 +618,16 @@ impl GroupState { } // Billed once the group's own charge is gone, so the head only takes its place. if let (Some(handoff), Some(head)) = (handoff, head) - && let Some(bill) = self - .charge - .account() - .and_then(|track| track.hold(&handoff, head.bytes())) + && let Some(track) = self.charge.account() + && let Some(bill) = track.hold(head.bytes()) { let salvage = Salvage { head: Arc::new(head), bill, }; *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(salvage); + // Only once it is held: a sweep finding the hand-off empty forgets it. + track.expire_held(&handoff); } } } @@ -928,7 +936,9 @@ impl Producer { let fields = || head.objects.iter().take(count).flatten(); let payload: u64 = frames.iter().map(|frame| frame.payload.len() as u64).sum(); let metadata: u64 = fields().map(|fields| fields.heap()).sum(); - if state.would_overflow(count, payload + metadata) { + // The in-flight frame's remainder was admitted at its open and is charged as it + // streams in, without another check. + if state.would_overflow(count, payload + metadata + state.unwritten()) { return; } @@ -3164,6 +3174,34 @@ mod test { assert_eq!(pool.used(), 0); } + /// The in-flight frame's unwritten remainder counts against the group budget, so an + /// adoption cannot take the group past it once the frame finishes streaming. + #[test] + fn an_open_frame_reserves_its_remainder_from_adoption() { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool, kio::Weak::new()); + let mut tail = Producer::new(Info::from(0u64), track::Info::default(), cache); + tail.start_at(1).unwrap(); + let adopter = tail.clone(); + let _frame = tail + .create_frame(frame::Info { + size: MAX_CACHE_BYTES - 1, + timestamp: None, + }) + .unwrap(); + let head = Head { + start: 0, + first_object: true, + frames: vec![Frame { + timestamp: None, + payload: Bytes::from_static(b"ab"), + }], + objects: Vec::new(), + }; + adopter.adopt_head(&head); + assert_eq!(adopter.state.read().offset, 1, "the head does not fit beside the open frame"); + } + /// An adopted head leads the fields and the wire header, whether the header was set /// before the adoption or arrives after it. #[test] From edf62b2ec82c576a1735ac6106d90057da378213 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:32:32 -0700 Subject: [PATCH 5/9] chore(net): format the open-frame adoption test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 997fb7c2d38..c055b4b341c 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3146,7 +3146,11 @@ mod test { objects: Vec::new(), }; adopter.adopt_head(&head); - assert_eq!(adopter.state.read().offset, 1, "the head does not fit beside the open frame"); + assert_eq!( + adopter.state.read().offset, + 1, + "the head does not fit beside the open frame" + ); } /// An adopted head leads the fields and the wire header, whether the header was set From e9c9c0a9f5c487ad0c8bc28f3c5c74eb11e10f6f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:40:14 -0700 Subject: [PATCH 6/9] fix(net): an oversized group lends no head, whichever write finds it A GroupTooLarge from set_end_object reached commit_abort through abort and was salvaged; abort_too_large now shares commit_abort, which frees it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index c055b4b341c..f5760a7c79a 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1386,26 +1386,22 @@ impl Producer { fn commit_abort(&self, mut guard: kio::Mut<'_, GroupState>, err: Error) { let cut = !matches!(err, Error::Old | Error::Timeout | Error::Evicted); + // Cache policy frees the frames, as does an oversized group whichever write found + // it; a copy that failed or was given up lends them on. + let salvage = cut && !matches!(err, Error::GroupTooLarge); guard.abort = Some(err); self.alive.cut.store(cut, Ordering::Release); self.alive.aborted.store(true, Ordering::Release); - // Cache policy frees the frames; a copy that failed or was given up lends them on. - guard.release(cut); + guard.release(salvage); guard.close(); self.cache.aborted(self.sequence, cut); } /// Abort a write that would grow the group past its budget, holding the lock already /// taken for that write so nothing else lands in between. - fn abort_too_large(&self, mut state: kio::Mut<'_, GroupState>) -> Error { - let err = Error::GroupTooLarge; - state.abort = Some(err.clone()); - self.alive.cut.store(true, Ordering::Release); - self.alive.aborted.store(true, Ordering::Release); - state.release(false); - state.close(); - self.cache.aborted(self.sequence, true); - err + fn abort_too_large(&self, state: kio::Mut<'_, GroupState>) -> Error { + self.commit_abort(state, Error::GroupTooLarge); + Error::GroupTooLarge } /// Whether the group has been aborted (including pool eviction). The track's @@ -3062,10 +3058,11 @@ mod test { } /// A failed copy moves its frames to the readers that could resume it, so they outlive - /// the release only as long as one of those readers does; cache policy keeps nothing. + /// the release only as long as one of those readers does; cache policy and an oversized + /// group keep nothing. #[test] fn an_abort_hands_the_head_to_its_resumers() { - for (err, kept) in [(Error::Cancel, true), (Error::Evicted, false)] { + for (err, kept) in [(Error::Cancel, true), (Error::Evicted, false), (Error::GroupTooLarge, false)] { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), kio::Weak::new()); let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); From f8a7e7c24315fea38ca3367619c486d45633eabd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:49:45 -0700 Subject: [PATCH 7/9] chore(net): format the abort hand-off test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index f5760a7c79a..02b3459d379 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3062,7 +3062,11 @@ mod test { /// group keep nothing. #[test] fn an_abort_hands_the_head_to_its_resumers() { - for (err, kept) in [(Error::Cancel, true), (Error::Evicted, false), (Error::GroupTooLarge, false)] { + for (err, kept) in [ + (Error::Cancel, true), + (Error::Evicted, false), + (Error::GroupTooLarge, false), + ] { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), kio::Weak::new()); let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); From 7ab175749027502d28632c7f6041a9747da35830 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 13:10:11 -0700 Subject: [PATCH 8/9] fix(net): size the held-head registry to the heads held Every failed group in an expiry-enabled pool registered its hand-off for the sweep, even when recovery adopted the head at once, and the sweep's retain kept the vector's peak allocation. A burst of failed groups could pin memory outside the pool's accounting. The registry now forgets dead hand-offs before it grows, and the sweep gives back what a burst of held heads grew it to. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/cache.rs | 24 ++++++++++++++++++++---- rs/moq-net/src/model/group.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index 11f1b3ec982..c0e642478a6 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -830,19 +830,35 @@ impl Track { /// Have the sweep drop `handoff`'s held head once the pool's idle window passes. pub(super) fn expire_held(&self, handoff: &Arc) { if self.sweep.get().is_some() { - self.held - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(Arc::downgrade(handoff)); + let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); + // Forget the hand-offs done with before growing, so a burst of failed groups whose + // heads recovery adopts at once grows the registry no further than the heads held. + if held.len() == held.capacity() { + Self::prune_held(&mut held); + } + held.push(Arc::downgrade(handoff)); } } /// Drop the held heads past the pool's idle window, forgetting the hand-offs done with. fn sweep_held(&self) { let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); + Self::prune_held(&mut held); + // Give back what a burst of held heads grew it to, which the pool no longer bills. + let len = held.len(); + held.shrink_to(len * 2); + } + + /// Expire the held heads past the window and forget the hand-offs holding none. + fn prune_held(held: &mut Vec>) { held.retain(|handoff| handoff.upgrade().is_some_and(|handoff| !handoff.expire())); } + #[cfg(test)] + pub(super) fn held_capacity(&self) -> usize { + self.held.lock().unwrap_or_else(PoisonError::into_inner).capacity() + } + /// Take everything written since the last call, to be turned into eviction debt. pub(super) fn take_written(&self) -> u64 { self.written.swap(0, Ordering::Relaxed) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 02b3459d379..76b48d04a99 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3109,6 +3109,36 @@ mod test { assert_eq!(pool.used(), 0); } + /// The registry the sweep expires held heads from grows with the heads held, not with + /// every group that failed, and gives back what a burst grew it to once they expire. + #[test] + fn the_held_registry_follows_the_heads_held() { + const BURST: usize = 64; + let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let fail = |sequence: u64| { + let mut producer = Producer::new(Info::from(sequence), track::Info::default(), cache.clone()); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.abort(Error::Cancel).unwrap(); + handoff + }; + + // Recovery adopts each head as soon as its group fails. + for sequence in 0..BURST as u64 { + drop(fail(sequence)); + } + assert!(cache.held_capacity() < BURST, "adopted hand-offs pinned the registry"); + + // Readers that stop polling hold every head until the window passes. + let held: Vec<_> = (0..BURST as u64).map(fail).collect(); + assert!(cache.held_capacity() >= BURST); + pool.step(std::time::Duration::from_secs(2)); + pool.sweep(); + assert!(held.iter().all(|handoff| handoff.head().is_none())); + assert!(cache.held_capacity() < BURST, "the sweep kept a burst's allocation"); + } + /// A failed copy's head is held only while the pool has room for it. #[test] fn a_full_pool_holds_no_head() { From 4e74809d10e176c5c7233b0c9643e034e7685935 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 14:12:56 -0700 Subject: [PATCH 9/9] fix(net): reclaim held heads under capacity pressure A held hand-off head sits outside every track's eviction order, and only an expiry window ever dropped it. In a capacity-only pool, a shrink below the held bills left a stalled resumer keeping them while ordinary groups were evicted. The held registry now lives on the pool, which drops the oldest held heads first whenever a track charges debt over capacity, and on each sweep. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/cache.rs | 95 ++++++++++++++++++++--------------- rs/moq-net/src/model/group.rs | 19 ++++--- rs/moq-net/src/model/track.rs | 34 +++++++++++++ 3 files changed, 100 insertions(+), 48 deletions(-) diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index c0e642478a6..640fedfa7bc 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -164,6 +164,10 @@ struct Inner { wall_micros: AtomicU64, // Tracks with reads parked on a wall-clock deadline, by their earliest one. wall: kio::Lock, + // Hand-offs holding a failed group's head for a successor route, oldest first. The + // heads sit outside every track's eviction order, so the pool reclaims them itself: + // first under capacity pressure, and once its idle window passes. + held: Mutex>>, } /// The pool-wide index of wall-clock deadlines that [`Pool::gc`] fires. @@ -218,6 +222,7 @@ impl Pool { wall_epoch: OnceLock::new(), wall_micros: AtomicU64::new(0), wall: kio::Lock::default(), + held: Mutex::default(), }), } } @@ -275,8 +280,56 @@ impl Pool { .map(|window| window / 2) } - /// Expire idle groups across the registered tracks. + /// Have the pool drop `handoff`'s held head under capacity pressure, oldest first, or + /// once its idle window passes. + pub(super) fn hold_head(&self, handoff: &Arc) { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + // Forget the hand-offs done with before growing, so a burst of failed groups whose + // heads recovery adopts at once grows the registry no further than the heads held. + if held.len() == held.capacity() { + self.prune_held(&mut held); + } + held.push(Arc::downgrade(handoff)); + } + + /// Drop held heads, oldest first, while the pool is over capacity. + pub(super) fn reclaim_held(&self) { + if self.used() > self.inner.capacity.load(Ordering::Relaxed) { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + self.prune_held(&mut held); + } + } + + /// Drop the held heads past the idle window, then the oldest while the pool is over + /// capacity, and forget the hand-offs holding none. + fn prune_held(&self, held: &mut Vec>) { + held.retain(|handoff| { + let Some(handoff) = handoff.upgrade() else { + return false; + }; + let over = self.used() > self.inner.capacity.load(Ordering::Relaxed); + !handoff.expire(over) + }); + } + + #[cfg(test)] + pub(super) fn held_capacity(&self) -> usize { + self.inner + .held + .lock() + .unwrap_or_else(PoisonError::into_inner) + .capacity() + } + + /// Expire idle groups and held heads across the registered tracks. pub(super) fn sweep(&self) { + { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + self.prune_held(&mut held); + // Give back what a burst of held heads grew it to, which the pool no longer bills. + let len = held.len(); + held.shrink_to(len * 2); + } // Upgrade outside each track's lock: dropping its last account unregisters it. let tracks: Vec<_> = self .inner @@ -646,10 +699,6 @@ pub(super) struct Track { // Here rather than on the track state so a frame write advances it without the track // lock, and wakes only the readers following it. largest: kio::Shared, - - // Hand-offs holding a failed group's head for a successor route, which the sweep - // expires like idle groups. Only filled while the pool sweeps. - held: Mutex>>, } /// The greatest location a track has written or its source reported, followed by the track @@ -743,7 +792,6 @@ impl Track { wakes: Default::default(), aborts: OnceLock::new(), largest: Default::default(), - held: Mutex::default(), }); if let Some(key) = track.pool.register(&track) { let _ = track.sweep.set(key); @@ -810,7 +858,7 @@ impl Track { /// /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed /// like cached content and bounded the same way: `None` when it would take the pool past - /// its capacity, and [`Self::expire_held`] drops it once the pool's idle window passes. + /// its capacity, and [`Pool::hold_head`] has the pool reclaim it like idle content. pub(super) fn hold(self: &Arc, bytes: u64) -> Option { if self .pool @@ -827,38 +875,6 @@ impl Track { }) } - /// Have the sweep drop `handoff`'s held head once the pool's idle window passes. - pub(super) fn expire_held(&self, handoff: &Arc) { - if self.sweep.get().is_some() { - let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); - // Forget the hand-offs done with before growing, so a burst of failed groups whose - // heads recovery adopts at once grows the registry no further than the heads held. - if held.len() == held.capacity() { - Self::prune_held(&mut held); - } - held.push(Arc::downgrade(handoff)); - } - } - - /// Drop the held heads past the pool's idle window, forgetting the hand-offs done with. - fn sweep_held(&self) { - let mut held = self.held.lock().unwrap_or_else(PoisonError::into_inner); - Self::prune_held(&mut held); - // Give back what a burst of held heads grew it to, which the pool no longer bills. - let len = held.len(); - held.shrink_to(len * 2); - } - - /// Expire the held heads past the window and forget the hand-offs holding none. - fn prune_held(held: &mut Vec>) { - held.retain(|handoff| handoff.upgrade().is_some_and(|handoff| !handoff.expire())); - } - - #[cfg(test)] - pub(super) fn held_capacity(&self) -> usize { - self.held.lock().unwrap_or_else(PoisonError::into_inner).capacity() - } - /// Take everything written since the last call, to be turned into eviction debt. pub(super) fn take_written(&self) -> u64 { self.written.swap(0, Ordering::Relaxed) @@ -881,7 +897,6 @@ impl Track { /// Settle from [`Pool::sweep`], dating activity and expiring every idle candidate. pub(super) fn sweep(&self) { - self.sweep_held(); self.settle_inner(None, true); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 76b48d04a99..0090be97737 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -236,11 +236,14 @@ impl Handoff { salvaged.or_else(|| state.head().map(Arc::new)) } - /// Drop the held head once the pool's idle window passed, returning whether none is - /// held any more. - pub(super) fn expire(&self) -> bool { + /// Drop the held head when `reclaim`, or once the pool's idle window passed, returning + /// whether none is held any more. + pub(super) fn expire(&self, reclaim: bool) -> bool { let mut salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner); - if salvaged.as_ref().is_some_and(|salvage| salvage.bill.expired()) { + if salvaged + .as_ref() + .is_some_and(|salvage| reclaim || salvage.bill.expired()) + { *salvaged = None; } salvaged.is_none() @@ -627,7 +630,7 @@ impl GroupState { }; *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(salvage); // Only once it is held: a sweep finding the hand-off empty forgets it. - track.expire_held(&handoff); + track.pool().hold_head(&handoff); } } } @@ -3128,15 +3131,15 @@ mod test { for sequence in 0..BURST as u64 { drop(fail(sequence)); } - assert!(cache.held_capacity() < BURST, "adopted hand-offs pinned the registry"); + assert!(pool.held_capacity() < BURST, "adopted hand-offs pinned the registry"); // Readers that stop polling hold every head until the window passes. let held: Vec<_> = (0..BURST as u64).map(fail).collect(); - assert!(cache.held_capacity() >= BURST); + assert!(pool.held_capacity() >= BURST); pool.step(std::time::Duration::from_secs(2)); pool.sweep(); assert!(held.iter().all(|handoff| handoff.head().is_none())); - assert!(cache.held_capacity() < BURST, "the sweep kept a burst's allocation"); + assert!(pool.held_capacity() < BURST, "the sweep kept a burst's allocation"); } /// A failed copy's head is held only while the pool has room for it. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 93e3f535daf..a7946100ea6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1631,6 +1631,8 @@ impl TrackState { pub(super) fn charge_debt(&mut self) { let written = self.cache.take_written(); let pool = self.cache.pool().clone(); + // Held heads go first: they are a failed copy's frames that no track can evict. + pool.reclaim_held(); match pool.accrue(written) { Some(mut accrued) => { if self.oldest_is_stale(&pool) { @@ -11978,6 +11980,38 @@ mod test { assert!(pool.used() <= 24 * unit, "usage hovers near capacity: {}", pool.used()); } + /// A capacity shrink reclaims a failed group's held head before any cached group, even + /// in a pool with no idle window and once the failed copy's track is gone. + #[test] + fn a_shrink_reclaims_held_heads_first() { + let pool = cache::Pool::new(cache::Config::default().with_capacity(1 << 40)); + let broadcast = Arc::new(broadcast::Info { + pool: pool.clone(), + ..Default::default() + }); + let failed = Producer::new(broadcast.clone(), "failed", Info::timed()); + let mut group = failed.append_group().unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 1_000])) + .unwrap(); + let handoff = group.consume().handoff(); + group.abort(crate::Error::Cancel).unwrap(); + drop(failed); + assert!(handoff.head().is_some(), "held for a successor route"); + + let mut producer = Producer::new(broadcast, "live", Info::timed()); + finished_group(&mut producer, 1_000); + // Room for the cached group, not the held head beside it. + pool.resize(pool.used() - 1); + finished_group(&mut producer, 1_000); + + assert!( + handoff.head().is_none(), + "the shrink left a stalled resumer's head held" + ); + assert!(producer.consume().peek_group(0).is_some(), "a cached group went first"); + } + /// One write pays at most twice what it produced, so a capacity shrink (or one /// track's burst) drains gradually instead of one writer dumping its whole /// backlog in a single call.