diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index ceedefb22d0..640fedfa7bc 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -164,6 +164,10 @@ struct Inner { wall_micros: AtomicU64, // Tracks with reads parked on a wall-clock deadline, by their earliest one. wall: kio::Lock, + // Hand-offs holding a failed group's head for a successor route, oldest first. The + // heads sit outside every track's eviction order, so the pool reclaims them itself: + // first under capacity pressure, and once its idle window passes. + held: Mutex>>, } /// The pool-wide index of wall-clock deadlines that [`Pool::gc`] fires. @@ -218,6 +222,7 @@ impl Pool { wall_epoch: OnceLock::new(), wall_micros: AtomicU64::new(0), wall: kio::Lock::default(), + held: Mutex::default(), }), } } @@ -275,8 +280,56 @@ impl Pool { .map(|window| window / 2) } - /// Expire idle groups across the registered tracks. + /// Have the pool drop `handoff`'s held head under capacity pressure, oldest first, or + /// once its idle window passes. + pub(super) fn hold_head(&self, handoff: &Arc) { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + // Forget the hand-offs done with before growing, so a burst of failed groups whose + // heads recovery adopts at once grows the registry no further than the heads held. + if held.len() == held.capacity() { + self.prune_held(&mut held); + } + held.push(Arc::downgrade(handoff)); + } + + /// Drop held heads, oldest first, while the pool is over capacity. + pub(super) fn reclaim_held(&self) { + if self.used() > self.inner.capacity.load(Ordering::Relaxed) { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + self.prune_held(&mut held); + } + } + + /// Drop the held heads past the idle window, then the oldest while the pool is over + /// capacity, and forget the hand-offs holding none. + fn prune_held(&self, held: &mut Vec>) { + held.retain(|handoff| { + let Some(handoff) = handoff.upgrade() else { + return false; + }; + let over = self.used() > self.inner.capacity.load(Ordering::Relaxed); + !handoff.expire(over) + }); + } + + #[cfg(test)] + pub(super) fn held_capacity(&self) -> usize { + self.inner + .held + .lock() + .unwrap_or_else(PoisonError::into_inner) + .capacity() + } + + /// Expire idle groups and held heads across the registered tracks. pub(super) fn sweep(&self) { + { + let mut held = self.inner.held.lock().unwrap_or_else(PoisonError::into_inner); + self.prune_held(&mut held); + // Give back what a burst of held heads grew it to, which the pool no longer bills. + let len = held.len(); + held.shrink_to(len * 2); + } // Upgrade outside each track's lock: dropping its last account unregisters it. let tracks: Vec<_> = self .inner @@ -801,6 +854,27 @@ impl Track { } } + /// Bill `bytes` of a failed group's head while a hand-off holds it for a successor route. + /// + /// Readers that stop polling can hold a hand-off indefinitely, so the head is billed + /// like cached content and bounded the same way: `None` when it would take the pool past + /// its capacity, and [`Pool::hold_head`] has the pool reclaim it like idle content. + pub(super) fn hold(self: &Arc, bytes: u64) -> Option { + if self + .pool + .capacity() + .is_some_and(|capacity| self.pool.used().saturating_add(bytes) > capacity) + { + return None; + } + self.pool.add(bytes); + Some(Bill { + track: self.clone(), + bytes, + at: self.pool.now(), + }) + } + /// Take everything written since the last call, to be turned into eviction debt. pub(super) fn take_written(&self) -> u64 { self.written.swap(0, Ordering::Relaxed) @@ -911,6 +985,28 @@ impl Drop for Track { } } +/// The bytes a held head of a failed group bills to its track's pool, released on drop. +pub(super) struct Bill { + track: Arc, + bytes: u64, + // The coarse tick it was billed at. + at: u64, +} + +impl Bill { + /// Whether the pool's idle window passed since it was billed. + pub(super) fn expired(&self) -> bool { + let pool = &self.track.pool; + pool.now().saturating_sub(self.at) >= pool.expiry_ticks() + } +} + +impl Drop for Bill { + fn drop(&mut self) { + self.track.pool.sub(self.bytes); + } +} + /// The RAII byte accounting for one cached group, owned by the group's state. /// /// `add`/`sub` mirror the group's cached payload bytes into the pool with plain @@ -1077,6 +1173,11 @@ impl Charge { Some(target >> ACCESS_SHIFT) } + /// The track account this charge bills, unless it is detached. + pub(super) fn account(&self) -> Option<&Arc> { + self.track.as_ref() + } + /// Release everything this charge holds: bytes, overhead, and the access /// sample. Idempotent; used when the group aborts and clears its frames. pub(super) fn clear(&mut self) { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 3d297530b43..0090be97737 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -20,8 +20,8 @@ use crate::{Cap, Timescale, stats, track}; use std::collections::VecDeque; use std::mem::MaybeUninit; use std::ops::{Bound, RangeBounds}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError, Weak}; use std::task::{Poll, ready}; use crate::{Error, IntoBytes, Result, Timestamp}; @@ -187,6 +187,78 @@ pub(super) struct GroupState { // `Alive::aborted`, so [`Producer::abort`] stays the only writer: anything else // setting this would leave track scans reading a group as live. pub(super) abort: Option, + + // The readers that could resume this group on another route. A release that is no + // cache policy moves the frames to them rather than dropping them. + handoff: Mutex>, + + // Whether an adopted head (see [`Producer::adopt_head`]) begins at the group's first + // object, for a wire header that arrives after it. + adopted: Option, +} + +/// The frames a copy of a group holds, from where it starts, which a successor copy of +/// the same group on another route adopts below where it starts. +pub(super) struct Head { + start: usize, + // Whether `start` is the group's first object. + first_object: bool, + frames: Vec, + // Relay metadata for `frames`, at the same index; frames past its end carry none. + objects: Vec>>, +} + +/// A copy's [`Head`], shared by every reader that could resume its group elsewhere. +/// +/// Read from the copy while it holds its frames. Once it fails, its release moves them +/// here instead, so a plain consumer of the failed copy still reads the abort and pins +/// nothing, and the frames go with the last reader holding this. +pub(super) struct Handoff { + state: kio::ConsumerWeak, + salvaged: Mutex>, +} + +/// A failed copy's head, billed to its pool while held; see [`cache::Track::hold`]. +struct Salvage { + head: Arc, + bill: cache::Bill, +} + +impl Handoff { + /// The copy's head, if it holds any frames, or held them when it failed. + pub(super) fn head(&self) -> Option> { + // The group's lock first, in `release`'s order: under it the frames are either + // still in the copy or already moved here, never in neither. + let state = self.state.read(); + let salvaged = (self.salvaged.lock().unwrap_or_else(PoisonError::into_inner)) + .as_ref() + .map(|salvage| salvage.head.clone()); + salvaged.or_else(|| state.head().map(Arc::new)) + } + + /// Drop the held head when `reclaim`, or once the pool's idle window passed, returning + /// whether none is held any more. + pub(super) fn expire(&self, reclaim: bool) -> bool { + let mut salvaged = self.salvaged.lock().unwrap_or_else(PoisonError::into_inner); + if salvaged + .as_ref() + .is_some_and(|salvage| reclaim || salvage.bill.expired()) + { + *salvaged = None; + } + salvaged.is_none() + } +} + +impl Head { + /// Roughly what holding it costs: payloads, fields, and their slots. + fn bytes(&self) -> u64 { + let payload: u64 = self.frames.iter().map(|frame| frame.payload.len() as u64).sum(); + let fields: u64 = self.objects.iter().flatten().map(|fields| fields.heap()).sum(); + let slots = self.frames.len() * size_of::() + + self.objects.len() * size_of::>>(); + payload + fields + slots as u64 + } } /// What an IETF relay keeps beside a group's frames. @@ -265,17 +337,21 @@ impl GroupState { !matches!(self.timeline, Timeline::Empty) || self.fin.is_some() || self.abort.is_some() } + /// The bytes the in-flight frame declared but has not written yet, which its open + /// already admitted. + fn unwritten(&self) -> u64 { + self.partial + .as_ref() + .map_or(0, |partial| partial.buf.size() as u64 - partial.charged) + } + /// Content still available to a reader of this group. /// /// Counts the in-flight frame at its declared size, like [`Self::content_range`]: a /// reader that skips it misses the whole frame, however much has arrived. fn content(&self) -> stats::Content { - let unwritten = self - .partial - .as_ref() - .map_or(0, |partial| partial.buf.size() as u64 - partial.charged); stats::Content { - bytes: self.cache + unwritten, + bytes: self.cache + self.unwritten(), frames: self.next_index.saturating_sub(self.offset) as u64, groups: 1, datagrams: 0, @@ -493,8 +569,46 @@ impl GroupState { self.charge.add(written + slots) } + /// Whether the frames held start at the group's first object. + fn first_object(&self) -> bool { + self.wire().map_or(self.offset == 0, |wire| wire.first_object) + } + + /// The committed frames, for a successor copy to adopt. Sparse Object IDs refuse + /// ordinal recovery, so they lend nothing. + fn head(&self) -> Option { + if self.frames.is_empty() || self.sparse() { + return None; + } + Some(Head { + start: self.offset, + first_object: self.first_object(), + frames: self.frames.iter().cloned().collect(), + objects: self + .relay + .as_ref() + .map_or_else(Vec::new, |relay| relay.objects.iter().cloned().collect()), + }) + } + /// Drop the cached frames (and any in-flight tail) and release their pool charge. - fn release(&mut self) { + /// + /// With `salvage`, the committed frames move to the readers that could resume the + /// group elsewhere, if any are left, instead of being dropped. + fn release(&mut self, salvage: bool) { + let handoff = match salvage && !self.frames.is_empty() && !self.sparse() { + true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), + false => None, + }; + let head = handoff.as_ref().map(|_| Head { + start: self.offset, + first_object: self.first_object(), + frames: std::mem::take(&mut self.frames).into(), + objects: self + .relay + .as_mut() + .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), + }); self.frames = VecDeque::new(); self.partial = None; self.cache = 0; @@ -505,6 +619,19 @@ impl GroupState { relay.bytes = size_of::() as u64 + relay.end_object.as_ref().map_or(0, |fields| fields.heap()); self.charge.add(relay.bytes); } + // Billed once the group's own charge is gone, so the head only takes its place. + if let (Some(handoff), Some(head)) = (handoff, head) + && let Some(track) = self.charge.account() + && let Some(bill) = track.hold(head.bytes()) + { + let salvage = Salvage { + head: Arc::new(head), + bill, + }; + *handoff.salvaged.lock().unwrap_or_else(PoisonError::into_inner) = Some(salvage); + // Only once it is held: a sweep finding the hand-off empty forgets it. + track.pool().hold_head(&handoff); + } } } @@ -606,7 +733,7 @@ impl Drop for Alive { sequence = self.info.sequence, "group::Producer dropped without finish() or abort()" ); - state.release(); + state.release(true); } Err(state) => { if state.fin.is_some() || state.abort.is_some() { @@ -638,8 +765,15 @@ impl Producer { #[cfg(feature = "async")] // private: passthrough - pub(crate) fn set_wire(&self, wire: Wire) -> Result<()> { - modify(&self.state)?.relay().wire = Some(wire); + pub(crate) fn set_wire(&self, mut wire: Wire) -> Result<()> { + let mut state = modify(&self.state)?; + // A head adopted before the header arrived holds the objects below the stream's. + if let Some(first_object) = state.adopted { + wire.first_object = first_object; + wire.first_id = state.offset as u64; + wire.last = wire.last.or((state.next_index as u64).checked_sub(1)); + } + state.relay().wire = Some(wire); Ok(()) } @@ -784,6 +918,74 @@ impl Producer { Ok(()) } + /// Adopt the frames of `head` below where this copy starts, so it holds the group + /// from `head`'s start, as if this route had delivered them. + /// + /// A no-op unless `head` starts below this copy and reaches it, and the frames fit the + /// group's budget. Readers keep their positions, since frames keep their indices. A + /// group whose frames build on earlier ones (a compressed group) primes its decoder + /// from the adopted frames here. + pub(super) fn adopt_head(&self, head: &Head) { + let Ok(mut state) = self.state.write() else { + return; + }; + let offset = state.offset; + let reaches = head.start.saturating_add(head.frames.len()) >= offset; + if state.abort.is_some() || state.sparse() || head.start >= offset || !reaches { + return; + } + let count = offset - head.start; + let frames = &head.frames[..count]; + let fields = || head.objects.iter().take(count).flatten(); + let payload: u64 = frames.iter().map(|frame| frame.payload.len() as u64).sum(); + let metadata: u64 = fields().map(|fields| fields.heap()).sum(); + // The in-flight frame's remainder was admitted at its open and is charged as it + // streams in, without another check. + if state.would_overflow(count, payload + metadata + state.unwritten()) { + return; + } + + let before = state.frames.capacity().saturating_sub(FRAME_SLOTS); + state.frames.reserve(count); + let mut growth = ((state.frames.capacity().saturating_sub(FRAME_SLOTS) - before) * size_of::()) as u64; + for frame in frames.iter().rev() { + state.frames.push_front(frame.clone()); + } + // Keep the fields aligned with the frames: the adopted ones lead. + if fields().next().is_some() || state.relay.as_ref().is_some_and(|relay| !relay.objects.is_empty()) { + let relay = state.relay(); + let before = relay.objects.capacity(); + relay.objects.reserve(count); + let slots = ((relay.objects.capacity() - before) * size_of::>>()) as u64; + for index in (0..count).rev() { + relay.objects.push_front(head.objects.get(index).cloned().flatten()); + } + relay.bytes += slots + metadata; + growth += slots; + } + state.offset = head.start; + state.cache += payload; + let start = frames.iter().find_map(|frame| frame.timestamp); + state.timeline = match (state.timeline, start) { + (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, latest }, + (Timeline::Empty, Some(start)) => Timeline::Timed { + start, + latest: frames.iter().rev().find_map(|frame| frame.timestamp).unwrap_or(start), + }, + (Timeline::Empty, None) => Timeline::Untimed, + (timeline, _) => timeline, + }; + state.adopted = Some(head.first_object); + if let Some(wire) = state.wire_mut() { + wire.first_object = head.first_object; + wire.first_id = head.start as u64; + wire.last = wire.last.or(Some(offset as u64 - 1)); + } + let now = state.charge.add(payload + metadata + growth); + drop(state); + self.cache.settle(now); + } + /// A helper method to write a frame from a single byte buffer. /// /// If you want to write multiple chunks, use [Self::create_frame] to get a frame producer. @@ -1187,25 +1389,22 @@ impl Producer { fn commit_abort(&self, mut guard: kio::Mut<'_, GroupState>, err: Error) { let cut = !matches!(err, Error::Old | Error::Timeout | Error::Evicted); + // Cache policy frees the frames, as does an oversized group whichever write found + // it; a copy that failed or was given up lends them on. + let salvage = cut && !matches!(err, Error::GroupTooLarge); guard.abort = Some(err); self.alive.cut.store(cut, Ordering::Release); self.alive.aborted.store(true, Ordering::Release); - guard.release(); + guard.release(salvage); guard.close(); self.cache.aborted(self.sequence, cut); } /// Abort a write that would grow the group past its budget, holding the lock already /// taken for that write so nothing else lands in between. - fn abort_too_large(&self, mut state: kio::Mut<'_, GroupState>) -> Error { - let err = Error::GroupTooLarge; - state.abort = Some(err.clone()); - self.alive.cut.store(true, Ordering::Release); - self.alive.aborted.store(true, Ordering::Release); - state.release(); - state.close(); - self.cache.aborted(self.sequence, true); - err + fn abort_too_large(&self, state: kio::Mut<'_, GroupState>) -> Error { + self.commit_abort(state, Error::GroupTooLarge); + Error::GroupTooLarge } /// Whether the group has been aborted (including pool eviction). The track's @@ -1753,7 +1952,6 @@ impl Consumer { } /// The index of the first frame the group holds (see [`Producer::start_at`]). - #[cfg(feature = "async")] // private: passthrough pub(crate) fn first_frame(&self) -> u64 { self.cursor.state.read().offset as u64 @@ -1806,6 +2004,22 @@ impl Consumer { self } + /// The hand-off of this copy's head, shared by every reader that could resume it + /// elsewhere. + pub(super) fn handoff(&self) -> Arc { + let state = self.cursor.state.read(); + let mut handoff = state.handoff.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(handoff) = handoff.upgrade() { + return handoff; + } + let shared = Arc::new(Handoff { + state: self.cursor.state.weak(), + salvaged: Mutex::default(), + }); + *handoff = Arc::downgrade(&shared); + shared + } + /// Carry this group across a front's route changes; see [`super::resume`]. pub(super) fn with_recover(mut self, recover: super::resume::Recover) -> Self { self.recover = Some(Box::new(recover)); @@ -2846,6 +3060,217 @@ mod test { assert_eq!(state.cache, 0); } + /// A failed copy moves its frames to the readers that could resume it, so they outlive + /// the release only as long as one of those readers does; cache policy and an oversized + /// group keep nothing. + #[test] + fn an_abort_hands_the_head_to_its_resumers() { + for (err, kept) in [ + (Error::Cancel, true), + (Error::Evicted, false), + (Error::GroupTooLarge, false), + ] { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + producer.write_frame(None, Bytes::from_static(b"b")).unwrap(); + let handoff = producer.consume().handoff(); + assert_eq!(handoff.head().map(|head| head.frames.len()), Some(2), "read live"); + + producer.clone().abort(err.clone()).unwrap(); + assert!(producer.state.read().frames.is_empty(), "{err}: released"); + assert_eq!(pool.used() > 0, kept, "{err}: billed only while held"); + let head = handoff.head(); + assert_eq!(head.is_some(), kept, "{err}"); + if let Some(head) = head { + assert_eq!((head.start, head.first_object, head.frames.len()), (0, true, 2)); + let held = Arc::downgrade(&head); + drop((head, handoff)); + assert!(held.upgrade().is_none(), "freed with the last resumer"); + assert_eq!(pool.used(), 0, "unbilled with it"); + } + } + } + + /// A resuming reader that stops polling holds a failed copy's head no longer than an + /// idle group stays cached. + #[test] + fn a_held_head_expires_with_the_pool_window() { + let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.clone().abort(Error::Cancel).unwrap(); + drop(producer); + assert!(handoff.head().is_some()); + + pool.step(std::time::Duration::from_secs(2)); + pool.sweep(); + assert!(handoff.head().is_none(), "the sweep drops a held head past the window"); + assert_eq!(pool.used(), 0); + } + + /// The registry the sweep expires held heads from grows with the heads held, not with + /// every group that failed, and gives back what a burst grew it to once they expire. + #[test] + fn the_held_registry_follows_the_heads_held() { + const BURST: usize = 64; + let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let fail = |sequence: u64| { + let mut producer = Producer::new(Info::from(sequence), track::Info::default(), cache.clone()); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.abort(Error::Cancel).unwrap(); + handoff + }; + + // Recovery adopts each head as soon as its group fails. + for sequence in 0..BURST as u64 { + drop(fail(sequence)); + } + assert!(pool.held_capacity() < BURST, "adopted hand-offs pinned the registry"); + + // Readers that stop polling hold every head until the window passes. + let held: Vec<_> = (0..BURST as u64).map(fail).collect(); + assert!(pool.held_capacity() >= BURST); + pool.step(std::time::Duration::from_secs(2)); + pool.sweep(); + assert!(held.iter().all(|handoff| handoff.head().is_none())); + assert!(pool.held_capacity() < BURST, "the sweep kept a burst's allocation"); + } + + /// A failed copy's head is held only while the pool has room for it. + #[test] + fn a_full_pool_holds_no_head() { + let pool = cache::Pool::new(cache::Config::default().with_capacity(1)); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); + producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let handoff = producer.consume().handoff(); + producer.clone().abort(Error::Cancel).unwrap(); + assert!(handoff.head().is_none()); + assert_eq!(pool.used(), 0); + } + + /// The in-flight frame's unwritten remainder counts against the group budget, so an + /// adoption cannot take the group past it once the frame finishes streaming. + #[test] + fn an_open_frame_reserves_its_remainder_from_adoption() { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool, kio::Weak::new()); + let mut tail = Producer::new(Info::from(0u64), track::Info::default(), cache); + tail.start_at(1).unwrap(); + let adopter = tail.clone(); + let _frame = tail + .create_frame(frame::Info { + size: MAX_CACHE_BYTES - 1, + timestamp: None, + }) + .unwrap(); + let head = Head { + start: 0, + first_object: true, + frames: vec![Frame { + timestamp: None, + payload: Bytes::from_static(b"ab"), + }], + objects: Vec::new(), + }; + adopter.adopt_head(&head); + assert_eq!( + adopter.state.read().offset, + 1, + "the head does not fit beside the open frame" + ); + } + + /// An adopted head leads the fields and the wire header, whether the header was set + /// before the adoption or arrives after it. + #[test] + #[cfg(feature = "async")] + fn an_adopted_head_keeps_fields_and_the_wire_aligned() { + let fields = |id| { + Some(Arc::new(super::super::object::Fields { + id, + properties: Default::default(), + })) + }; + let head = Head { + start: 0, + first_object: true, + frames: vec![ + Frame { + timestamp: None, + payload: Bytes::from_static(b"a"), + }, + Frame { + timestamp: None, + payload: Bytes::from_static(b"b"), + }, + ], + objects: vec![None, fields(1)], + }; + let wire = Wire { + sparse: false, + first_object: false, + first_id: 2, + extensions: true, + has_end: true, + fetchable: true, + last: None, + }; + for wired_first in [true, false] { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut tail = Producer::new(Info::from(0u64), track::Info::default(), cache); + tail.start_at(2).unwrap(); + if wired_first { + tail.set_wire(wire.clone()).unwrap(); + } + let mut reader = tail.consume(); + reader.start_at(2); + let before = pool.used(); + tail.adopt_head(&head); + assert!(pool.used() >= before + 2, "the adopted frames are charged"); + if !wired_first { + tail.set_wire(wire.clone()).unwrap(); + } + let written = tail.state.read().wire().cloned().unwrap(); + assert_eq!( + (written.first_object, written.first_id, written.last), + (true, 0, Some(1)) + ); + tail.mark_gap(2, None).unwrap(); + assert!(tail.state.read().wire().unwrap().fetchable, "no gap below the stream"); + tail.write_frame(None, Bytes::from_static(b"c")).unwrap(); + tail.finish().unwrap(); + + let waiter = kio::Waiter::noop(); + let mut whole = tail.consume(); + let mut read = Vec::new(); + while let Poll::Ready(Ok(Some(object))) = whole.poll_read_object(&waiter) { + read.push((object.frame.payload, object.fields.map(|fields| fields.id))); + } + assert_eq!( + read, + [ + (Bytes::from_static(b"a"), None), + (Bytes::from_static(b"b"), Some(1)), + (Bytes::from_static(b"c"), None) + ] + ); + let next = reader.read_frame().now_or_never().unwrap().unwrap(); + assert_eq!( + next.map(|frame| frame.payload), + Some(Bytes::from_static(b"c")), + "a reader keeps its place" + ); + } + } + #[test] #[cfg(feature = "async")] fn drop_unfinished_clears_cached_frames() { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f2b06f7d55e..2c54d0029ed 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -589,6 +589,7 @@ impl Reader { let sequence = group.sequence; self.newest = self.newest.max(Some(sequence)); let copy = &self.copies[index]; + let handoff = group.handoff(); group.with_recover(Recover { sequence, reader: Some(self.this.clone()), @@ -596,6 +597,7 @@ impl Reader { route: self.route.clone(), generation: copy.generation, copy: copy.track.clone(), + handoff, lease: Some(copy.lease.clone()), logical: None, fetch: None, @@ -900,6 +902,9 @@ pub(super) struct Recover { /// The route generation of the copy being read, and that copy. generation: u64, copy: track::Consumer, + /// The head of the group as that copy holds it, which the copy continuing it adopts, + /// so the serving route caches the group whole. + handoff: Arc, /// Keeps a replaced copy subscribed while this group is read from it. lease: Option>, /// A fetch's logical track, held while the group is read. A fetch is no subscription, @@ -925,6 +930,7 @@ impl Clone for Recover { route: self.route.clone(), generation: self.generation, copy: self.copy.clone(), + handoff: self.handoff.clone(), lease: self.lease.clone(), logical: self.logical.clone(), // A clone reads in parallel, so it asks for itself. @@ -1030,6 +1036,7 @@ impl Recover { // failed delivered it already, so its subscription will not again. let same = generation == self.generation; if !same && let Poll::Ready(Some(mut group)) = serving.poll_group(self.sequence, waiter) { + lend_head(&self.handoff, self.sequence, serving, &group); group.start_at(index); if group.index() == index { return Poll::Ready(Ok(Replacement { @@ -1058,6 +1065,7 @@ impl Recover { Fetch::Refused(err) => err.clone(), Fetch::Pending(pending) => match ready!(kio::Task::poll(&mut **pending, waiter)) { Ok(mut group) => { + lend_head(&self.handoff, self.sequence, serving, &group); group.start_at(index); if group.index() == index { return Poll::Ready(Ok(Replacement { @@ -1107,6 +1115,7 @@ impl Recover { pub(super) fn adopt(&mut self, replacement: &Replacement) { self.generation = replacement.generation; self.copy = replacement.copy.clone(); + self.handoff = replacement.group.handoff(); // A later switch must keep this copy subscribed while the recovered group // still reads it, just like a group handed out from that copy directly. self.lease = self.reader.as_ref().and_then(Weak::upgrade).and_then(|reader| { @@ -1121,6 +1130,16 @@ impl Recover { } } +/// Lend the head `handoff` holds to `serving`'s copy of group `sequence`, `group`, when that +/// starts later: the fetch for the rest asks only from the reader's frame. +fn lend_head(handoff: &group::Handoff, sequence: u64, serving: &track::Consumer, group: &group::Consumer) { + if group.first_frame() > 0 + && let Some(head) = handoff.head() + { + serving.adopt_head(sequence, &head); + } +} + /// A candidate cursor, committed only when the requested read can use it. pub(super) struct Replacement { pub(super) group: group::Consumer, @@ -1174,17 +1193,21 @@ impl kio::Task for Fetching { } }; return match result { - Ok(group) => Poll::Ready(Ok(group.with_recover(Recover { - sequence: self.sequence, - reader: None, - max_delay: self.options.max_delay, - route: self.route.clone(), - generation, - copy: serving.clone(), - lease: None, - logical: Some(self.logical.clone()), - fetch: None, - }))), + Ok(group) => { + let handoff = group.handoff(); + Poll::Ready(Ok(group.with_recover(Recover { + sequence: self.sequence, + reader: None, + max_delay: self.options.max_delay, + route: self.route.clone(), + generation, + copy: serving.clone(), + handoff, + lease: None, + logical: Some(self.logical.clone()), + fetch: None, + }))) + } // The route failed: ask the next one, unless none will come. Err(err) if (route_failed(&err) || copy_failed(&serving)) && self.route.end().is_none() => { ready!(self.route.poll_changed(generation, waiter)); @@ -1381,6 +1404,125 @@ mod test { assert!(chunks.read_chunk().now_or_never().unwrap().unwrap().is_none()); } + /// Read a fetched group from frame 0, which must already be cached. + #[cfg(feature = "async")] + fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { + let mut group = track + .fetch_group(sequence, None) + .now_or_never() + .expect("the whole group is cached") + .unwrap(); + std::iter::from_fn(|| read(&mut group)).collect() + } + + /// The replacement adopts the head the failed route received, so its cache holds the + /// whole group, as on a relay that never saw the route change. + #[test] + #[cfg(feature = "async")] + fn a_failed_route_hands_its_head_to_the_replacement() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + open.write_frame(ts(1), b"b".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + let mut plain = open.consume(); + open.abort(Error::Dropped).unwrap(); + assert!( + matches!(plain.read_frame().now_or_never(), Some(Err(Error::Dropped))), + "a plain consumer reads the abort, not the head" + ); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!((request.sequence(), request.frame_start()), (0, 2)); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(2), b"c".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"c".to_vec())); + assert_eq!(read(&mut reading), None); + + let whole = [b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]; + assert_eq!(read_cached(&b.consume(), 0), whole); + assert_eq!(read_cached(&logical.consume(), 0), whole); + assert!(dynamic.requested_group().now_or_never().is_none(), "no second fetch"); + } + + /// A replaced route that only stalled lends the replacement its head the same way. + #[test] + #[cfg(feature = "async")] + fn a_stalled_route_lends_its_head_to_the_replacement() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!(request.frame_start(), 1); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + + assert_eq!(read_cached(&b.consume(), 0), [b"a".to_vec(), b"b".to_vec()]); + } + + /// Cache policy dropping a group frees its frames: nothing is kept for a successor. + #[test] + #[cfg(feature = "async")] + fn an_evicted_group_hands_nothing_to_the_replacement() { + let routes = Producer::new(); + let a = copy(); + routes.serve(a.consume()); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut fetched = logical(&routes) + .consume() + .fetch_group(0, None) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(read(&mut fetched), Some(b"a".to_vec())); + open.abort(Error::Evicted).unwrap(); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + assert!(fetched.read_frame().now_or_never().is_none()); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + assert_eq!(request.frame_start(), 1); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut fetched), Some(b"b".to_vec())); + assert_eq!(read(&mut fetched), None); + let _whole = b.consume().fetch_group(0, None); + assert!( + dynamic.requested_group().now_or_never().is_some(), + "the head was not kept" + ); + } + #[test] #[cfg(feature = "async")] fn a_group_both_routes_deliver_is_handed_out_once() { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ffb8e42da81..a7946100ea6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1631,6 +1631,8 @@ impl TrackState { pub(super) fn charge_debt(&mut self) { let written = self.cache.take_written(); let pool = self.cache.pool().clone(); + // Held heads go first: they are a failed copy's frames that no track can evict. + pool.reclaim_held(); match pool.accrue(written) { Some(mut accrued) => { if self.oldest_is_stale(&pool) { @@ -3666,6 +3668,26 @@ impl Consumer { }) } + /// Lend `head` to the cached copies of group `sequence`; see + /// [`group::Producer::adopt_head`]. + pub(super) fn adopt_head(&self, sequence: u64, head: &group::Head) { + let copies: Vec = self.with_state(|state| { + let Some(state) = state else { + return Vec::new(); + }; + let state = state.read(); + state + .lookup + .get(&sequence) + .map_or_else(Vec::new, |slot| slot.copies().cloned().collect()) + }); + // With the track lock released (lock order is track then group): adopting settles + // the group's cache debt. + for group in copies { + group.adopt_head(head); + } + } + /// [`Self::poll_group`], except a newer group is no verdict: QUIC does not order /// streams, so this one may still land behind it. Pending until it is cached, or the /// feed starts past it or ends before it. @@ -11958,6 +11980,38 @@ mod test { assert!(pool.used() <= 24 * unit, "usage hovers near capacity: {}", pool.used()); } + /// A capacity shrink reclaims a failed group's held head before any cached group, even + /// in a pool with no idle window and once the failed copy's track is gone. + #[test] + fn a_shrink_reclaims_held_heads_first() { + let pool = cache::Pool::new(cache::Config::default().with_capacity(1 << 40)); + let broadcast = Arc::new(broadcast::Info { + pool: pool.clone(), + ..Default::default() + }); + let failed = Producer::new(broadcast.clone(), "failed", Info::timed()); + let mut group = failed.append_group().unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 1_000])) + .unwrap(); + let handoff = group.consume().handoff(); + group.abort(crate::Error::Cancel).unwrap(); + drop(failed); + assert!(handoff.head().is_some(), "held for a successor route"); + + let mut producer = Producer::new(broadcast, "live", Info::timed()); + finished_group(&mut producer, 1_000); + // Room for the cached group, not the held head beside it. + pool.resize(pool.used() - 1); + finished_group(&mut producer, 1_000); + + assert!( + handoff.head().is_none(), + "the shrink left a stalled resumer's head held" + ); + assert!(producer.consume().peek_group(0).is_some(), "a cached group went first"); + } + /// One write pays at most twice what it produced, so a capacity shrink (or one /// track's burst) drains gradually instead of one writer dumping its whole /// backlog in a single call.